Compare commits
65
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2d7da04502 | ||
|
|
b4099ddda4 | ||
|
|
04410a5022 | ||
|
|
682747e9f7 | ||
|
|
00f327da4b | ||
|
|
aa7bb6282d | ||
|
|
33d7ef8fa0 | ||
|
|
0ea6308dae | ||
|
|
e7351c16b0 | ||
|
|
00d96b7163 | ||
|
|
2b4bf4b7bd | ||
|
|
e7ebc92f83 | ||
|
|
181b0d637e | ||
|
|
b0733e53b2 | ||
|
|
618880d38d | ||
|
|
79b76c8789 | ||
|
|
5b12c3e085 | ||
|
|
623c31615d | ||
|
|
af719e387d | ||
|
|
dddb21bd70 | ||
|
|
e8d4e964cb | ||
|
|
eb24cb04dc | ||
|
|
e84057f402 | ||
|
|
15c3006d09 | ||
|
|
b3209c2012 | ||
|
|
6f5a424661 | ||
|
|
8d47d2670b | ||
|
|
b19de45a82 | ||
|
|
1e88893ef9 | ||
|
|
1fd2e09971 | ||
|
|
f85d831104 | ||
|
|
2b24db5a15 | ||
|
|
e1945d977c | ||
|
|
a10875d68e | ||
|
|
f35dcef4d0 | ||
|
|
f5ebae1134 | ||
|
|
6be4e8388b | ||
|
|
9e3f87ac52 | ||
|
|
ceb01f6eaa | ||
|
|
1b13563e19 | ||
|
|
848a1e9a5b | ||
|
|
a77453338c | ||
|
|
d12bc2ea0b | ||
|
|
a1df3742b1 | ||
|
|
d8818b776b | ||
|
|
40c36ccb1d | ||
|
|
80c151d084 | ||
|
|
c6a8759e7b | ||
|
|
f17b5212c7 | ||
|
|
85f70d4bc1 | ||
|
|
b284934e34 | ||
|
|
2d33321fab | ||
|
|
0975471986 | ||
|
|
214d6e21dd | ||
|
|
fca5c23345 | ||
|
|
f9b1500ba2 | ||
|
|
931bf3b27d | ||
|
|
6bb671be21 | ||
|
|
b8f2fa479c | ||
|
|
1639be9841 | ||
|
|
907392cce8 | ||
|
|
73a3e674c9 | ||
|
|
b5c105545c | ||
|
|
61c20b0995 | ||
|
|
b7a22146a0 |
Generated
+6
@@ -6712,15 +6712,21 @@ name = "yoi-e2e"
|
|||||||
version = "0.0.0"
|
version = "0.0.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"chrono",
|
"chrono",
|
||||||
|
"config-source",
|
||||||
"libc",
|
"libc",
|
||||||
|
"manifest",
|
||||||
|
"protocol",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"server-api",
|
"server-api",
|
||||||
|
"session-store",
|
||||||
"sha2 0.11.0",
|
"sha2 0.11.0",
|
||||||
"tempfile",
|
"tempfile",
|
||||||
|
"tungstenite 0.29.0",
|
||||||
"webauthn-authenticator-rs",
|
"webauthn-authenticator-rs",
|
||||||
"webauthn-rs-proto",
|
"webauthn-rs-proto",
|
||||||
|
"worker-runtime",
|
||||||
"yoi-workspace-server",
|
"yoi-workspace-server",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|||||||
@@ -72,6 +72,41 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn compiled_handoff_uses_host_owned_retry_identity() {
|
||||||
|
let flow = builtin_flow_source(CODER_REVIEW_FLOW_SLUG)
|
||||||
|
.unwrap()
|
||||||
|
.compile()
|
||||||
|
.unwrap();
|
||||||
|
let complete = flow
|
||||||
|
.state(&crate::StateId::new("complete").unwrap())
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
complete
|
||||||
|
.instructions
|
||||||
|
.contains("The Host owns transport retry identity")
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
complete
|
||||||
|
.instructions
|
||||||
|
.contains("do not supply operation_key")
|
||||||
|
);
|
||||||
|
assert!(!complete.instructions.contains("same key and full input"));
|
||||||
|
let fix = flow.state(&crate::StateId::new("fix").unwrap()).unwrap();
|
||||||
|
assert!(
|
||||||
|
fix.instructions
|
||||||
|
.contains("rerun validation affected by the fixes")
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
fix.instructions
|
||||||
|
.contains("Cite reusable evidence for unaffected scope")
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
fix.instructions
|
||||||
|
.contains("Source movement invalidates the prior verdict")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn every_builtin_flow_compiles_and_matches_catalog_identity() {
|
fn every_builtin_flow_compiles_and_matches_catalog_identity() {
|
||||||
assert!(!builtin_flow_sources().is_empty());
|
assert!(!builtin_flow_sources().is_empty());
|
||||||
|
|||||||
+184
-83
@@ -617,7 +617,8 @@ impl fmt::Display for McpRequestStage {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A running initialized stdio MCP client.
|
/// An owned stdio MCP process, initialized by `connect` or explicitly after
|
||||||
|
/// `spawn_uninitialized` when startup cleanup must retain it before any await.
|
||||||
pub struct McpStdioClient {
|
pub struct McpStdioClient {
|
||||||
server_name: String,
|
server_name: String,
|
||||||
limits: McpStdioLimits,
|
limits: McpStdioLimits,
|
||||||
@@ -636,6 +637,9 @@ pub struct McpStdioClient {
|
|||||||
|
|
||||||
impl McpStdioClient {
|
impl McpStdioClient {
|
||||||
/// Spawn, initialize, negotiate capabilities, and send notifications/initialized.
|
/// Spawn, initialize, negotiate capabilities, and send notifications/initialized.
|
||||||
|
/// Initialization errors attempt shutdown, but this convenience API cannot
|
||||||
|
/// return an unreaped client on failure/cancellation. Its error is not proof
|
||||||
|
/// of cleanup; use `spawn_uninitialized` with retained ownership when needed.
|
||||||
pub async fn connect(
|
pub async fn connect(
|
||||||
spec: McpStdioServerSpec,
|
spec: McpStdioServerSpec,
|
||||||
limits: McpStdioLimits,
|
limits: McpStdioLimits,
|
||||||
@@ -667,6 +671,21 @@ impl McpStdioClient {
|
|||||||
async fn spawn(
|
async fn spawn(
|
||||||
spec: McpStdioServerSpec,
|
spec: McpStdioServerSpec,
|
||||||
limits: McpStdioLimits,
|
limits: McpStdioLimits,
|
||||||
|
) -> Result<Self, McpClientError> {
|
||||||
|
Self::spawn_uninitialized(spec, limits)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Spawn without awaiting the handshake, so a cleanup owner can retain the
|
||||||
|
/// client before initialization can fail or be cancelled. Call
|
||||||
|
/// `initialize_with_timeout` before discovery; on error/cancellation, shut
|
||||||
|
/// down this same client rather than spawning a replacement.
|
||||||
|
///
|
||||||
|
/// Once the process spawns, this returns its owner even if an expected pipe
|
||||||
|
/// is missing. Initialization then fails, but cleanup can still reap the
|
||||||
|
/// actual child rather than relying on kill-on-drop for a post-spawn error.
|
||||||
|
pub fn spawn_uninitialized(
|
||||||
|
spec: McpStdioServerSpec,
|
||||||
|
limits: McpStdioLimits,
|
||||||
) -> Result<Self, McpClientError> {
|
) -> Result<Self, McpClientError> {
|
||||||
let redactor = spec.redactor();
|
let redactor = spec.redactor();
|
||||||
let mut command = Command::new(&spec.command);
|
let mut command = Command::new(&spec.command);
|
||||||
@@ -688,29 +707,23 @@ impl McpStdioClient {
|
|||||||
McpErrorKind::Io(redactor.redact(&err.to_string())),
|
McpErrorKind::Io(redactor.redact(&err.to_string())),
|
||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
let stdin = child.stdin.take().ok_or_else(|| {
|
let stdin = child.stdin.take();
|
||||||
McpClientError::new(
|
let stdout = child.stdout.take();
|
||||||
&spec.name,
|
let stderr = child.stderr.take();
|
||||||
McpPhase::Spawn,
|
let missing_pipe = if stdin.is_none() {
|
||||||
McpErrorKind::Protocol("child stdin was not piped".into()),
|
Some("stdin")
|
||||||
)
|
} else if stdout.is_none() {
|
||||||
})?;
|
Some("stdout")
|
||||||
let stdout = child.stdout.take().ok_or_else(|| {
|
} else if stderr.is_none() {
|
||||||
McpClientError::new(
|
Some("stderr")
|
||||||
&spec.name,
|
} else {
|
||||||
McpPhase::Spawn,
|
None
|
||||||
McpErrorKind::Protocol("child stdout was not piped".into()),
|
};
|
||||||
)
|
|
||||||
})?;
|
|
||||||
let stderr = child.stderr.take().ok_or_else(|| {
|
|
||||||
McpClientError::new(
|
|
||||||
&spec.name,
|
|
||||||
McpPhase::Spawn,
|
|
||||||
McpErrorKind::Protocol("child stderr was not piped".into()),
|
|
||||||
)
|
|
||||||
})?;
|
|
||||||
|
|
||||||
let stdin: SharedStdin = Arc::new(Mutex::new(Some(Box::new(stdin))));
|
let stdin: SharedStdin =
|
||||||
|
Arc::new(Mutex::new(stdin.map(|stdin| {
|
||||||
|
Box::new(stdin) as Box<dyn AsyncWrite + Send + Unpin>
|
||||||
|
})));
|
||||||
let diagnostics = Arc::new(Mutex::new(BoundedDiagnostics::new(
|
let diagnostics = Arc::new(Mutex::new(BoundedDiagnostics::new(
|
||||||
spec.name.clone(),
|
spec.name.clone(),
|
||||||
limits.max_diagnostic_lines,
|
limits.max_diagnostic_lines,
|
||||||
@@ -718,16 +731,32 @@ impl McpStdioClient {
|
|||||||
)));
|
)));
|
||||||
let list_changes = Arc::new(Mutex::new(BoundedListChanged::new(spec.name.clone())));
|
let list_changes = Arc::new(Mutex::new(BoundedListChanged::new(spec.name.clone())));
|
||||||
let (tx, rx) = mpsc::channel(16);
|
let (tx, rx) = mpsc::channel(16);
|
||||||
let reader_task = spawn_stdout_reader(
|
// Do not return an error that discards an already spawned child. Queue
|
||||||
spec.name.clone(),
|
// setup failure before reader events so initialization cannot succeed.
|
||||||
stdout,
|
if let Some(pipe) = missing_pipe {
|
||||||
stdin.clone(),
|
tx.try_send(ReaderEvent::Error(McpClientError::new(
|
||||||
tx,
|
&spec.name,
|
||||||
limits.clone(),
|
McpPhase::Spawn,
|
||||||
redactor.clone(),
|
McpErrorKind::Protocol(format!("child {pipe} was not piped")),
|
||||||
list_changes.clone(),
|
)))
|
||||||
);
|
.expect("new response channel has capacity");
|
||||||
let stderr_task = spawn_stderr_reader(stderr, diagnostics.clone(), limits.clone());
|
}
|
||||||
|
let reader_task = match stdout {
|
||||||
|
Some(stdout) => spawn_stdout_reader(
|
||||||
|
spec.name.clone(),
|
||||||
|
stdout,
|
||||||
|
stdin.clone(),
|
||||||
|
tx,
|
||||||
|
limits.clone(),
|
||||||
|
redactor.clone(),
|
||||||
|
list_changes.clone(),
|
||||||
|
),
|
||||||
|
None => tokio::spawn(async {}),
|
||||||
|
};
|
||||||
|
let stderr_task = match stderr {
|
||||||
|
Some(stderr) => spawn_stderr_reader(stderr, diagnostics.clone(), limits.clone()),
|
||||||
|
None => tokio::spawn(async {}),
|
||||||
|
};
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
server_name: spec.name,
|
server_name: spec.name,
|
||||||
@@ -746,6 +775,25 @@ impl McpStdioClient {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Perform the bounded handshake on an already retained client. Failure or
|
||||||
|
/// cancellation does not drop or shut down the client: its owner must do so.
|
||||||
|
/// Do not retry initialization on a partially initialized connection.
|
||||||
|
pub async fn initialize_with_timeout(&mut self) -> Result<(), McpClientError> {
|
||||||
|
let started = Instant::now();
|
||||||
|
match timeout(self.limits.startup_timeout, self.initialize()).await {
|
||||||
|
Ok(result) => result,
|
||||||
|
Err(_) => Err(McpClientError::new(
|
||||||
|
&self.server_name,
|
||||||
|
McpPhase::Initialize,
|
||||||
|
McpErrorKind::Timeout {
|
||||||
|
operation: "startup".to_string(),
|
||||||
|
elapsed: started.elapsed(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.with_diagnostics(self.snapshot_diagnostics().await)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn initialize(&mut self) -> Result<(), McpClientError> {
|
async fn initialize(&mut self) -> Result<(), McpClientError> {
|
||||||
let result: InitializeResult = self
|
let result: InitializeResult = self
|
||||||
.request(
|
.request(
|
||||||
@@ -1080,6 +1128,8 @@ impl McpStdioClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Close stdin and wait for process exit, falling back to terminate and kill.
|
/// Close stdin and wait for process exit, falling back to terminate and kill.
|
||||||
|
/// Errors and cancellation retain the child for another attempt on this
|
||||||
|
/// client. Only confirmed reaping releases the process handle.
|
||||||
pub async fn shutdown(&mut self) -> Result<ShutdownReport, McpClientError> {
|
pub async fn shutdown(&mut self) -> Result<ShutdownReport, McpClientError> {
|
||||||
self.shutdown_started = true;
|
self.shutdown_started = true;
|
||||||
let deadline = Instant::now() + self.limits.shutdown_timeout;
|
let deadline = Instant::now() + self.limits.shutdown_timeout;
|
||||||
@@ -1091,64 +1141,55 @@ impl McpStdioClient {
|
|||||||
.await
|
.await
|
||||||
.is_ok();
|
.is_ok();
|
||||||
|
|
||||||
let mut child = match self.child.take() {
|
// Keep the actual process handle in the client across every await and
|
||||||
Some(child) => child,
|
// error. Taking it before wait loses retry/reaping authority on cancel.
|
||||||
None => return Ok(ShutdownReport::already_finished()),
|
if self.child.is_none() {
|
||||||
};
|
return Ok(ShutdownReport::already_finished());
|
||||||
|
}
|
||||||
if !closed {
|
if !closed {
|
||||||
self.reader_task.abort();
|
self.reader_task.abort();
|
||||||
return self.terminate_then_kill(child).await;
|
return self.terminate_then_kill().await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let child = self.child.as_mut().expect("checked child");
|
||||||
if let Ok(Some(status)) = child.try_wait() {
|
if let Ok(Some(status)) = child.try_wait() {
|
||||||
self.reader_task.abort();
|
return Ok(self.finish_shutdown(status, false, false));
|
||||||
self.stderr_task.abort();
|
|
||||||
return Ok(ShutdownReport {
|
|
||||||
exit_status: Some(status),
|
|
||||||
terminated: false,
|
|
||||||
killed: false,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
match timeout_at(deadline, child.wait()).await {
|
match timeout_at(deadline, child.wait()).await {
|
||||||
Ok(Ok(status)) => {
|
Ok(Ok(status)) => Ok(self.finish_shutdown(status, false, false)),
|
||||||
self.reader_task.abort();
|
|
||||||
self.stderr_task.abort();
|
|
||||||
Ok(ShutdownReport {
|
|
||||||
exit_status: Some(status),
|
|
||||||
terminated: false,
|
|
||||||
killed: false,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
Ok(Err(err)) => Err(McpClientError::new(
|
Ok(Err(err)) => Err(McpClientError::new(
|
||||||
&self.server_name,
|
&self.server_name,
|
||||||
McpPhase::Shutdown,
|
McpPhase::Shutdown,
|
||||||
McpErrorKind::Io(self.redactor.redact(&err.to_string())),
|
McpErrorKind::Io(self.redactor.redact(&err.to_string())),
|
||||||
)
|
)
|
||||||
.with_diagnostics(self.snapshot_diagnostics().await)),
|
.with_diagnostics(self.snapshot_diagnostics().await)),
|
||||||
Err(_) => self.terminate_then_kill(child).await,
|
Err(_) => self.terminate_then_kill().await,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn terminate_then_kill(
|
fn finish_shutdown(
|
||||||
&mut self,
|
&mut self,
|
||||||
mut child: Child,
|
status: ExitStatus,
|
||||||
) -> Result<ShutdownReport, McpClientError> {
|
terminated: bool,
|
||||||
let mut terminated = false;
|
killed: bool,
|
||||||
let mut killed = false;
|
) -> ShutdownReport {
|
||||||
if send_terminate(&mut child).is_ok() {
|
// No await separates confirmed reaping from releasing the handle.
|
||||||
terminated = true;
|
drop(self.child.take());
|
||||||
|
self.reader_task.abort();
|
||||||
|
self.stderr_task.abort();
|
||||||
|
ShutdownReport {
|
||||||
|
exit_status: Some(status),
|
||||||
|
terminated,
|
||||||
|
killed,
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn terminate_then_kill(&mut self) -> Result<ShutdownReport, McpClientError> {
|
||||||
|
let child = self.child.as_mut().expect("shutdown retains child");
|
||||||
|
let terminated = send_terminate(child).is_ok();
|
||||||
match timeout(self.limits.kill_timeout, child.wait()).await {
|
match timeout(self.limits.kill_timeout, child.wait()).await {
|
||||||
Ok(Ok(status)) => {
|
Ok(Ok(status)) => Ok(self.finish_shutdown(status, terminated, false)),
|
||||||
self.reader_task.abort();
|
|
||||||
self.stderr_task.abort();
|
|
||||||
Ok(ShutdownReport {
|
|
||||||
exit_status: Some(status),
|
|
||||||
terminated,
|
|
||||||
killed,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
Ok(Err(err)) => Err(McpClientError::new(
|
Ok(Err(err)) => Err(McpClientError::new(
|
||||||
&self.server_name,
|
&self.server_name,
|
||||||
McpPhase::Shutdown,
|
McpPhase::Shutdown,
|
||||||
@@ -1163,7 +1204,6 @@ impl McpStdioClient {
|
|||||||
McpErrorKind::Io(self.redactor.redact(&err.to_string())),
|
McpErrorKind::Io(self.redactor.redact(&err.to_string())),
|
||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
killed = true;
|
|
||||||
let status = timeout(self.limits.kill_timeout, child.wait())
|
let status = timeout(self.limits.kill_timeout, child.wait())
|
||||||
.await
|
.await
|
||||||
.map_err(|_| {
|
.map_err(|_| {
|
||||||
@@ -1183,13 +1223,7 @@ impl McpStdioClient {
|
|||||||
McpErrorKind::Io(self.redactor.redact(&err.to_string())),
|
McpErrorKind::Io(self.redactor.redact(&err.to_string())),
|
||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
self.reader_task.abort();
|
Ok(self.finish_shutdown(status, terminated, true))
|
||||||
self.stderr_task.abort();
|
|
||||||
Ok(ShutdownReport {
|
|
||||||
exit_status: Some(status),
|
|
||||||
terminated,
|
|
||||||
killed,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1197,10 +1231,10 @@ impl McpStdioClient {
|
|||||||
|
|
||||||
impl Drop for McpStdioClient {
|
impl Drop for McpStdioClient {
|
||||||
fn drop(&mut self) {
|
fn drop(&mut self) {
|
||||||
if !self.shutdown_started {
|
// An attempted shutdown is not proof of exit. Errors and cancellation
|
||||||
if let Some(child) = &mut self.child {
|
// leave the child here so Drop must still make a best-effort kill.
|
||||||
let _ = child.start_kill();
|
if let Some(child) = &mut self.child {
|
||||||
}
|
let _ = child.start_kill();
|
||||||
}
|
}
|
||||||
self.reader_task.abort();
|
self.reader_task.abort();
|
||||||
self.stderr_task.abort();
|
self.stderr_task.abort();
|
||||||
@@ -1784,3 +1818,70 @@ fn send_terminate(child: &mut Child) -> Result<(), ()> {
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod deadline_tests;
|
mod deadline_tests;
|
||||||
|
|
||||||
|
#[cfg(all(test, unix))]
|
||||||
|
mod shutdown_retention_tests {
|
||||||
|
//! The real client and OS process are used; only the external server is a
|
||||||
|
//! local shell that acknowledges initialization and ignores TERM/EOF.
|
||||||
|
use super::*;
|
||||||
|
use std::future::{Future, poll_fn};
|
||||||
|
use std::task::Poll;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn cancelled_graceful_or_terminate_wait_retries_reaping_the_same_child() {
|
||||||
|
for cancel_in_fallback in [false, true] {
|
||||||
|
let script = r#"trap '' TERM
|
||||||
|
read init || exit 1
|
||||||
|
printf '%s\n' '{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":"2025-11-25","capabilities":{},"serverInfo":{"name":"retained","version":"1"}}}'
|
||||||
|
while :; do :; done
|
||||||
|
"#;
|
||||||
|
let mut client = McpStdioClient::connect(
|
||||||
|
McpStdioServerSpec::new("retained", "/bin/sh").args(["-c", script]),
|
||||||
|
McpStdioLimits {
|
||||||
|
shutdown_timeout: if cancel_in_fallback {
|
||||||
|
Duration::ZERO
|
||||||
|
} else {
|
||||||
|
Duration::from_secs(30)
|
||||||
|
},
|
||||||
|
kill_timeout: Duration::from_secs(30),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("initialize after installing TERM handler");
|
||||||
|
let pid = client.child.as_ref().unwrap().id();
|
||||||
|
let mut shutdown = Box::pin(client.shutdown());
|
||||||
|
poll_fn(|cx| {
|
||||||
|
assert!(shutdown.as_mut().poll(cx).is_pending());
|
||||||
|
Poll::Ready(())
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
drop(shutdown);
|
||||||
|
assert_eq!(client.child.as_ref().and_then(Child::id), pid);
|
||||||
|
assert!(client.shutdown_started);
|
||||||
|
|
||||||
|
client.limits.shutdown_timeout = Duration::from_millis(20);
|
||||||
|
client.limits.kill_timeout = Duration::from_millis(50);
|
||||||
|
let report = timeout(Duration::from_secs(2), client.shutdown())
|
||||||
|
.await
|
||||||
|
.expect("retry bounded")
|
||||||
|
.expect("retry reaps original child");
|
||||||
|
assert!(
|
||||||
|
report.exit_status.is_some(),
|
||||||
|
"not an already-finished receipt"
|
||||||
|
);
|
||||||
|
assert!(report.killed, "TERM was ignored");
|
||||||
|
assert!(client.child.is_none());
|
||||||
|
let mut status = 0;
|
||||||
|
assert_eq!(
|
||||||
|
unsafe { libc::waitpid(pid.unwrap() as libc::pid_t, &mut status, libc::WNOHANG) },
|
||||||
|
-1
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
std::io::Error::last_os_error().raw_os_error(),
|
||||||
|
Some(libc::ECHILD),
|
||||||
|
"shutdown, not a later observer, reaped the original process"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -210,12 +210,7 @@ impl MergeRequestThreadEvent {
|
|||||||
}
|
}
|
||||||
fn bound_bodies(&mut self) {
|
fn bound_bodies(&mut self) {
|
||||||
match self {
|
match self {
|
||||||
Self::Review(value) => {
|
Self::Review(_) => {}
|
||||||
truncate_body(&mut value.body);
|
|
||||||
for finding in &mut value.findings {
|
|
||||||
truncate_body(&mut finding.body);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Self::ReviewRevoked(value) => truncate_body(&mut value.reason),
|
Self::ReviewRevoked(value) => truncate_body(&mut value.reason),
|
||||||
Self::ReviewCancelled(value) => truncate_body(&mut value.reason),
|
Self::ReviewCancelled(value) => truncate_body(&mut value.reason),
|
||||||
Self::Comment(value) => truncate_body(&mut value.body),
|
Self::Comment(value) => truncate_body(&mut value.body),
|
||||||
@@ -939,10 +934,6 @@ impl MergeRequestStore {
|
|||||||
&self,
|
&self,
|
||||||
i: SubmitMergeRequestReview,
|
i: SubmitMergeRequestReview,
|
||||||
) -> Result<ReviewEvent, MergeRequestError> {
|
) -> Result<ReviewEvent, MergeRequestError> {
|
||||||
bounded_body("review body", &i.body)?;
|
|
||||||
for finding in &i.findings {
|
|
||||||
bounded_body("review finding", &finding.body)?;
|
|
||||||
}
|
|
||||||
let mut c = self.lock()?;
|
let mut c = self.lock()?;
|
||||||
let t = c.transaction()?;
|
let t = c.transaction()?;
|
||||||
let g: Option<(String, String, String, String, String, String)> = t
|
let g: Option<(String, String, String, String, String, String)> = t
|
||||||
@@ -1747,7 +1738,23 @@ fn ticket_event(
|
|||||||
) -> Result<(), MergeRequestError> {
|
) -> Result<(), MergeRequestError> {
|
||||||
let ticket = &mr.ticket_ids[0];
|
let ticket = &mr.ticket_ids[0];
|
||||||
let n:i64=t.query_row("SELECT COALESCE(MAX(event_index),-1)+1 FROM typed_ticket_events WHERE workspace_id=?1 AND ticket_id=?2",params![mr.workspace_id,ticket],|r|r.get(0))?;
|
let n:i64=t.query_row("SELECT COALESCE(MAX(event_index),-1)+1 FROM typed_ticket_events WHERE workspace_id=?1 AND ticket_id=?2",params![mr.workspace_id,ticket],|r|r.get(0))?;
|
||||||
t.execute("INSERT INTO typed_ticket_events(workspace_id,ticket_id,event_index,kind,author,at,from_state,to_state,heading,body)VALUES(?1,?2,?3,'comment',?4,?5,NULL,NULL,'Merge Request integrated',?6)",params![mr.workspace_id,ticket,n,format!("worker:{}:{}",e.merged_by.runtime_id,e.merged_by.worker_id),e.created_at.to_rfc3339(),format!("Merge Request `{}` integrated approved source ref `{}`; Ticket remains in progress until guarded Ticket completion.",mr.merge_request_id,e.approved_source_ref)])?;
|
t.execute(
|
||||||
|
"INSERT INTO typed_ticket_events(
|
||||||
|
workspace_id,ticket_id,event_index,kind,author,at,from_state,to_state,heading,body,event_ref
|
||||||
|
) VALUES(?1,?2,?3,'comment',?4,?5,NULL,NULL,'Merge Request integrated',?6,?7)",
|
||||||
|
params![
|
||||||
|
mr.workspace_id,
|
||||||
|
ticket,
|
||||||
|
n,
|
||||||
|
format!("worker:{}:{}", e.merged_by.runtime_id, e.merged_by.worker_id),
|
||||||
|
e.created_at.to_rfc3339(),
|
||||||
|
format!(
|
||||||
|
"Merge Request `{}` integrated approved source ref `{}`; Ticket remains in progress until guarded Ticket completion.",
|
||||||
|
mr.merge_request_id, e.approved_source_ref
|
||||||
|
),
|
||||||
|
format!("{ticket}:{n}"),
|
||||||
|
],
|
||||||
|
)?;
|
||||||
for (k, v) in [
|
for (k, v) in [
|
||||||
("implementation_assignment_id", a),
|
("implementation_assignment_id", a),
|
||||||
("merge_request_id", &mr.merge_request_id),
|
("merge_request_id", &mr.merge_request_id),
|
||||||
|
|||||||
@@ -53,8 +53,8 @@ fn fixture() -> (tempfile::TempDir, MergeRequestStore) {
|
|||||||
let d = tempfile::tempdir().unwrap();
|
let d = tempfile::tempdir().unwrap();
|
||||||
let p = d.path().join("db");
|
let p = d.path().join("db");
|
||||||
let c = Connection::open(&p).unwrap();
|
let c = Connection::open(&p).unwrap();
|
||||||
c.execute_batch("CREATE TABLE workspaces(workspace_id TEXT PRIMARY KEY);CREATE TABLE repositories(workspace_id TEXT,repository_id TEXT,PRIMARY KEY(workspace_id,repository_id));CREATE TABLE ticket_current_worker_assignments(workspace_id TEXT,ticket_id TEXT,assignment_id TEXT,runtime_id TEXT,worker_id TEXT,updated_at TEXT,PRIMARY KEY(workspace_id,ticket_id));CREATE TABLE typed_tickets(workspace_id TEXT,ticket_id TEXT,workflow_state TEXT,workflow_state_explicit INTEGER,updated_at TEXT,title TEXT NOT NULL,body TEXT NOT NULL,PRIMARY KEY(workspace_id,ticket_id));CREATE TABLE typed_ticket_targets(workspace_id TEXT,ticket_id TEXT,ordinal INTEGER,repository_key TEXT,ref_selector TEXT,access TEXT);CREATE TABLE typed_ticket_events(workspace_id TEXT,ticket_id TEXT,event_index INTEGER,kind TEXT,author TEXT,at TEXT,from_state TEXT,to_state TEXT,heading TEXT,body TEXT,PRIMARY KEY(workspace_id,ticket_id,event_index));CREATE TABLE typed_ticket_event_attributes(workspace_id TEXT,ticket_id TEXT,event_index INTEGER,key TEXT,value TEXT,PRIMARY KEY(workspace_id,ticket_id,event_index,key));INSERT INTO workspaces VALUES('W');INSERT INTO repositories VALUES('W','R');INSERT INTO repositories VALUES('W','R2');INSERT INTO ticket_current_worker_assignments VALUES('W','T','A','runtime','coder','t');INSERT INTO typed_tickets VALUES('W','T','inprogress',1,'t','Title','Requirements');CREATE VIEW ticket_active_worker_assignments AS SELECT current.* FROM ticket_current_worker_assignments current JOIN typed_tickets ticket ON ticket.workspace_id=current.workspace_id AND ticket.ticket_id=current.ticket_id WHERE ticket.workflow_state NOT IN ('done','closed');").unwrap();
|
c.execute_batch("CREATE TABLE workspaces(workspace_id TEXT PRIMARY KEY);CREATE TABLE repositories(workspace_id TEXT,repository_id TEXT,PRIMARY KEY(workspace_id,repository_id));CREATE TABLE ticket_current_worker_assignments(workspace_id TEXT,ticket_id TEXT,assignment_id TEXT,runtime_id TEXT,worker_id TEXT,updated_at TEXT,PRIMARY KEY(workspace_id,ticket_id));CREATE TABLE typed_tickets(workspace_id TEXT,ticket_id TEXT,workflow_state TEXT,workflow_state_explicit INTEGER,updated_at TEXT,title TEXT NOT NULL,body TEXT NOT NULL,PRIMARY KEY(workspace_id,ticket_id));CREATE TABLE typed_ticket_targets(workspace_id TEXT,ticket_id TEXT,ordinal INTEGER,repository_key TEXT,ref_selector TEXT,access TEXT);CREATE TABLE typed_ticket_events(workspace_id TEXT,ticket_id TEXT,event_index INTEGER,kind TEXT,author TEXT,at TEXT,from_state TEXT,to_state TEXT,heading TEXT,body TEXT,event_ref TEXT NOT NULL DEFAULT '',PRIMARY KEY(workspace_id,ticket_id,event_index),UNIQUE(workspace_id,ticket_id,event_ref));CREATE TABLE typed_ticket_event_attributes(workspace_id TEXT,ticket_id TEXT,event_index INTEGER,key TEXT,value TEXT,PRIMARY KEY(workspace_id,ticket_id,event_index,key));INSERT INTO workspaces VALUES('W');INSERT INTO repositories VALUES('W','R');INSERT INTO repositories VALUES('W','R2');INSERT INTO ticket_current_worker_assignments VALUES('W','T','A','runtime','coder','t');INSERT INTO typed_tickets VALUES('W','T','inprogress',1,'t','Title','Requirements');CREATE VIEW ticket_active_worker_assignments AS SELECT current.* FROM ticket_current_worker_assignments current JOIN typed_tickets ticket ON ticket.workspace_id=current.workspace_id AND ticket.ticket_id=current.ticket_id WHERE ticket.workflow_state NOT IN ('done','closed');").unwrap();
|
||||||
c.execute("INSERT INTO typed_ticket_events(workspace_id,ticket_id,event_index,kind,body) VALUES('W','T',-1,'comment','Requirements')",[]).unwrap();
|
c.execute("INSERT INTO typed_ticket_events(workspace_id,ticket_id,event_index,kind,body,event_ref) VALUES('W','T',-1,'comment','Requirements','T:-1')",[]).unwrap();
|
||||||
drop(c);
|
drop(c);
|
||||||
let a = Assignments(Arc::new(Mutex::new(CurrentAssignment {
|
let a = Assignments(Arc::new(Mutex::new(CurrentAssignment {
|
||||||
assignment_id: "A".into(),
|
assignment_id: "A".into(),
|
||||||
@@ -376,6 +376,49 @@ fn approve_for(
|
|||||||
fn approve(s: &MergeRequestStore, subject: &str, token: &str) -> ReviewEvent {
|
fn approve(s: &MergeRequestStore, subject: &str, token: &str) -> ReviewEvent {
|
||||||
approve_for(s, "MR", "R", subject, token)
|
approve_for(s, "MR", "R", subject, token)
|
||||||
}
|
}
|
||||||
|
#[test]
|
||||||
|
fn review_submission_preserves_long_body_and_findings_on_reload() {
|
||||||
|
for (body, finding_body) in [
|
||||||
|
("b".repeat(16 * 1024 + 1), "finding".into()),
|
||||||
|
("review".into(), "指摘".repeat(4096)),
|
||||||
|
] {
|
||||||
|
let (_dir, store) = fixture();
|
||||||
|
open(&store);
|
||||||
|
request(&store, "source", "token");
|
||||||
|
let findings = vec![ReviewFinding {
|
||||||
|
severity: FindingSeverity::Major,
|
||||||
|
code: None,
|
||||||
|
path: None,
|
||||||
|
line: None,
|
||||||
|
body: finding_body,
|
||||||
|
}];
|
||||||
|
let submitted = store
|
||||||
|
.submit_review(SubmitMergeRequestReview {
|
||||||
|
merge_request_id: "MR".into(),
|
||||||
|
ticket_id: "T".into(),
|
||||||
|
current_subject_ref: "source".into(),
|
||||||
|
capability_token: "token".into(),
|
||||||
|
decision: ReviewDecision::RequestChanges,
|
||||||
|
body: body.clone(),
|
||||||
|
findings: findings.clone(),
|
||||||
|
now: at(4),
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(submitted.body, body);
|
||||||
|
assert_eq!(submitted.findings, findings);
|
||||||
|
let reloaded = store.get_by_id("W", "MR").unwrap();
|
||||||
|
let saved = reloaded
|
||||||
|
.thread
|
||||||
|
.iter()
|
||||||
|
.find_map(|event| match event {
|
||||||
|
MergeRequestThreadEvent::Review(review) => Some(review),
|
||||||
|
_ => None,
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(saved, &submitted);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn review_submission_authorization_rejects_invalid_grants_before_side_effects() {
|
fn review_submission_authorization_rejects_invalid_grants_before_side_effects() {
|
||||||
let (_d, store) = fixture();
|
let (_d, store) = fixture();
|
||||||
@@ -457,6 +500,18 @@ fn integration_preserves_selectors_ticket_state_and_assignment_and_replays_its_m
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(replayed, merged);
|
assert_eq!(replayed, merged);
|
||||||
let connection = Connection::open(d.path().join("db")).unwrap();
|
let connection = Connection::open(d.path().join("db")).unwrap();
|
||||||
|
let ticket_refs = connection
|
||||||
|
.prepare("SELECT event_ref FROM typed_ticket_events WHERE event_index >= 0")
|
||||||
|
.unwrap()
|
||||||
|
.query_map([], |row| row.get::<_, String>(0))
|
||||||
|
.unwrap()
|
||||||
|
.collect::<Result<Vec<_>, _>>()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
ticket_refs,
|
||||||
|
vec!["T:0"],
|
||||||
|
"replay must not append another post"
|
||||||
|
);
|
||||||
let (state, active): (String, bool) = connection.query_row(
|
let (state, active): (String, bool) = connection.query_row(
|
||||||
"SELECT workflow_state, EXISTS(SELECT 1 FROM ticket_active_worker_assignments WHERE workspace_id='W' AND ticket_id='T') FROM typed_tickets WHERE workspace_id='W' AND ticket_id='T'",
|
"SELECT workflow_state, EXISTS(SELECT 1 FROM ticket_active_worker_assignments WHERE workspace_id='W' AND ticket_id='T') FROM typed_tickets WHERE workspace_id='W' AND ticket_id='T'",
|
||||||
[],
|
[],
|
||||||
@@ -477,6 +532,87 @@ fn integration_preserves_selectors_ticket_state_and_assignment_and_replays_its_m
|
|||||||
assert!(!json.contains(banned), "{banned} in {json}")
|
assert!(!json.contains(banned), "{banned} in {json}")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
#[test]
|
||||||
|
fn integration_post_failure_rolls_back_and_same_operation_recovers_with_legacy_empty_ref() {
|
||||||
|
let (dir, store) = fixture();
|
||||||
|
open(&store);
|
||||||
|
let review = approve(&store, "source", "approved");
|
||||||
|
request(&store, "source", "pending");
|
||||||
|
let completion = CompleteMergeRequest {
|
||||||
|
merge_request_id: "MR".into(),
|
||||||
|
ticket_id: "T".into(),
|
||||||
|
operation_id: "recover-integration".into(),
|
||||||
|
approval_event_id: review.event_id,
|
||||||
|
current_subject_ref: "source".into(),
|
||||||
|
target_ref_before: "before".into(),
|
||||||
|
target_ref_after: "after".into(),
|
||||||
|
strategy: MergeStrategy::FastForward,
|
||||||
|
resolution: ConflictResolution::None,
|
||||||
|
auth: auth(),
|
||||||
|
now: at(5),
|
||||||
|
};
|
||||||
|
let connection = Connection::open(dir.path().join("db")).unwrap();
|
||||||
|
// Retain a comment written before integration populated event_ref.
|
||||||
|
connection.execute_batch(
|
||||||
|
"INSERT INTO typed_ticket_events(workspace_id,ticket_id,event_index,kind,body,event_ref)
|
||||||
|
VALUES('W','T',0,'comment','Legacy integration','');
|
||||||
|
CREATE TRIGGER reject_integration_post BEFORE INSERT ON typed_ticket_events
|
||||||
|
WHEN NEW.heading='Merge Request integrated'
|
||||||
|
BEGIN SELECT RAISE(ABORT,'injected integration post failure'); END;",
|
||||||
|
).unwrap();
|
||||||
|
let before = store.get_by_id("W", "MR").unwrap();
|
||||||
|
let error = store.complete(completion.clone()).unwrap_err();
|
||||||
|
assert!(
|
||||||
|
error
|
||||||
|
.to_string()
|
||||||
|
.contains("injected integration post failure")
|
||||||
|
);
|
||||||
|
let failed = store.get_by_id("W", "MR").unwrap();
|
||||||
|
assert_eq!(failed.state, MergeRequestState::Open);
|
||||||
|
assert_eq!(failed.thread, before.thread);
|
||||||
|
assert!(store.authorize_review_submission("MR", "pending").is_ok());
|
||||||
|
assert_eq!(
|
||||||
|
connection
|
||||||
|
.query_row(
|
||||||
|
"SELECT COUNT(*) FROM typed_ticket_event_attributes",
|
||||||
|
[],
|
||||||
|
|row| row.get::<_, i64>(0),
|
||||||
|
)
|
||||||
|
.unwrap(),
|
||||||
|
0
|
||||||
|
);
|
||||||
|
|
||||||
|
connection
|
||||||
|
.execute_batch("DROP TRIGGER reject_integration_post")
|
||||||
|
.unwrap();
|
||||||
|
let merged = store.complete(completion.clone()).unwrap();
|
||||||
|
assert_eq!(store.complete(completion).unwrap(), merged);
|
||||||
|
assert_eq!(
|
||||||
|
store.get_by_id("W", "MR").unwrap().state,
|
||||||
|
MergeRequestState::Merged
|
||||||
|
);
|
||||||
|
assert!(store.authorize_review_submission("MR", "pending").is_err());
|
||||||
|
let posts = connection
|
||||||
|
.prepare("SELECT event_ref,body FROM typed_ticket_events WHERE event_index >= 0 ORDER BY event_index")
|
||||||
|
.unwrap()
|
||||||
|
.query_map([], |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)))
|
||||||
|
.unwrap()
|
||||||
|
.collect::<Result<Vec<_>, _>>()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
posts.len(),
|
||||||
|
2,
|
||||||
|
"retry and replay must append exactly one post"
|
||||||
|
);
|
||||||
|
assert_eq!(posts[0], (String::new(), "Legacy integration".into()));
|
||||||
|
assert_eq!(posts[1].0, "T:1");
|
||||||
|
assert!(posts[1].1.contains("`MR`"));
|
||||||
|
assert_eq!(connection.query_row(
|
||||||
|
"SELECT value FROM typed_ticket_event_attributes WHERE event_index=1 AND key='operation_id'",
|
||||||
|
[], |row| row.get::<_, String>(0),
|
||||||
|
).unwrap(), "recover-integration");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn source_move_cancels_submission_and_old_approval_is_reusable_when_source_returns() {
|
fn source_move_cancels_submission_and_old_approval_is_reusable_when_source_returns() {
|
||||||
let (_d, s) = fixture();
|
let (_d, s) = fixture();
|
||||||
@@ -955,7 +1091,7 @@ fn ticket_rescope_requires_fresh_review_before_merge_and_remains_recoverable() {
|
|||||||
let connection = Connection::open(dir.path().join("db")).unwrap();
|
let connection = Connection::open(dir.path().join("db")).unwrap();
|
||||||
connection
|
connection
|
||||||
.execute_batch(
|
.execute_batch(
|
||||||
"UPDATE typed_tickets SET title='Rescoped' WHERE workspace_id='W' AND ticket_id='T'; INSERT INTO typed_ticket_events VALUES('W','T',1,'item_edit','user','t2',NULL,NULL,NULL,NULL)",
|
"UPDATE typed_tickets SET title='Rescoped' WHERE workspace_id='W' AND ticket_id='T'; INSERT INTO typed_ticket_events VALUES('W','T',1,'item_edit','user','t2',NULL,NULL,NULL,NULL,'T:1')",
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
connection
|
connection
|
||||||
@@ -1082,7 +1218,7 @@ fn ticket_rescope_after_integration_accepts_fresh_requirement_attestation() {
|
|||||||
let connection = Connection::open(dir.path().join("db")).unwrap();
|
let connection = Connection::open(dir.path().join("db")).unwrap();
|
||||||
connection
|
connection
|
||||||
.execute_batch(
|
.execute_batch(
|
||||||
"UPDATE typed_tickets SET title='Rescoped' WHERE workspace_id='W' AND ticket_id='T'; INSERT INTO typed_ticket_events VALUES('W','T',1,'item_edit','user','t2',NULL,NULL,NULL,NULL)",
|
"UPDATE typed_tickets SET title='Rescoped' WHERE workspace_id='W' AND ticket_id='T'; INSERT INTO typed_ticket_events VALUES('W','T',1,'item_edit','user','t2',NULL,NULL,NULL,NULL,'T:1')",
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
connection
|
connection
|
||||||
@@ -1307,6 +1443,24 @@ fn partial_and_full_integration_retain_ticket_state_and_active_assignment() {
|
|||||||
assert_eq!(second_request.merged_result().unwrap(), &second_merge);
|
assert_eq!(second_request.merged_result().unwrap(), &second_merge);
|
||||||
assert_eq!(first_request.integration_approval().unwrap(), &first);
|
assert_eq!(first_request.integration_approval().unwrap(), &first);
|
||||||
assert_eq!(second_request.integration_approval().unwrap(), &second);
|
assert_eq!(second_request.integration_approval().unwrap(), &second);
|
||||||
|
let posts = connection
|
||||||
|
.prepare(
|
||||||
|
"SELECT event.event_ref, attribute.value
|
||||||
|
FROM typed_ticket_events event
|
||||||
|
JOIN typed_ticket_event_attributes attribute USING(workspace_id,ticket_id,event_index)
|
||||||
|
WHERE attribute.key='merge_request_id' ORDER BY event.event_index",
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
.query_map([], |row| {
|
||||||
|
Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
|
||||||
|
})
|
||||||
|
.unwrap()
|
||||||
|
.collect::<Result<Vec<_>, _>>()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
posts,
|
||||||
|
vec![("T:0".into(), "MR".into()), ("T:1".into(), "MR-2".into())]
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -1324,7 +1478,7 @@ fn historical_ticket_completion_payload_remains_readable_and_unchanged_on_migrat
|
|||||||
}"#;
|
}"#;
|
||||||
connection.execute_batch(
|
connection.execute_batch(
|
||||||
"UPDATE typed_tickets SET workflow_state='done' WHERE workspace_id='W' AND ticket_id='T';
|
"UPDATE typed_tickets SET workflow_state='done' WHERE workspace_id='W' AND ticket_id='T';
|
||||||
INSERT INTO typed_ticket_events VALUES('W','T',0,'state_changed','worker:runtime:coder','2026-07-26T12:00:07Z','inprogress','done','Ticket requirements completed','historical body');"
|
INSERT INTO typed_ticket_events VALUES('W','T',0,'state_changed','worker:runtime:coder','2026-07-26T12:00:07Z','inprogress','done','Ticket requirements completed','historical body','T:0');"
|
||||||
).unwrap();
|
).unwrap();
|
||||||
connection
|
connection
|
||||||
.execute(
|
.execute(
|
||||||
|
|||||||
@@ -86,6 +86,8 @@ pub enum WorkerRestoreState {
|
|||||||
Accepted,
|
Accepted,
|
||||||
Rejected,
|
Rejected,
|
||||||
RolledBack,
|
RolledBack,
|
||||||
|
/// A later Stop superseded this restore; neither rollback nor prior Accepted is asserted.
|
||||||
|
Superseded,
|
||||||
ReconciliationRequired,
|
ReconciliationRequired,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1578,13 +1580,24 @@ mod tests {
|
|||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert!(request.preparation.is_none());
|
assert!(request.preparation.is_none());
|
||||||
for state in [
|
for (state, wire) in [
|
||||||
"accepted",
|
(WorkerRestoreState::Accepted, "accepted"),
|
||||||
"rejected",
|
(WorkerRestoreState::Rejected, "rejected"),
|
||||||
"rolled_back",
|
(WorkerRestoreState::RolledBack, "rolled_back"),
|
||||||
"reconciliation_required",
|
(WorkerRestoreState::Superseded, "superseded"),
|
||||||
|
(
|
||||||
|
WorkerRestoreState::ReconciliationRequired,
|
||||||
|
"reconciliation_required",
|
||||||
|
),
|
||||||
] {
|
] {
|
||||||
assert!(serde_json::from_value::<WorkerRestoreState>(serde_json::json!(state)).is_ok());
|
assert_eq!(
|
||||||
|
serde_json::from_value::<WorkerRestoreState>(serde_json::json!(wire)).unwrap(),
|
||||||
|
state
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(state).unwrap(),
|
||||||
|
serde_json::json!(wire)
|
||||||
|
);
|
||||||
}
|
}
|
||||||
assert!(
|
assert!(
|
||||||
serde_json::from_value::<WorkerRestoreState>(serde_json::json!(
|
serde_json::from_value::<WorkerRestoreState>(serde_json::json!(
|
||||||
|
|||||||
+201
-10
@@ -229,6 +229,10 @@ impl_openapi_schema!(
|
|||||||
PasskeyLoginOptionsResponse,
|
PasskeyLoginOptionsResponse,
|
||||||
PasskeyLoginCompleteRequest,
|
PasskeyLoginCompleteRequest,
|
||||||
PasskeyAdditionOptionsRequest,
|
PasskeyAdditionOptionsRequest,
|
||||||
|
PasskeySummary,
|
||||||
|
PasskeyListResponse,
|
||||||
|
PasskeyAdditionResultRequest,
|
||||||
|
PasskeyAdditionResultResponse,
|
||||||
PasskeyRegistrationOptionsRequest,
|
PasskeyRegistrationOptionsRequest,
|
||||||
PasskeyRegistrationOptionsResponse,
|
PasskeyRegistrationOptionsResponse,
|
||||||
PasskeyRegistrationCompleteRequest,
|
PasskeyRegistrationCompleteRequest,
|
||||||
@@ -552,8 +556,9 @@ pub struct WorkspaceWorkerSessionHistoryResponse {
|
|||||||
/// comparisons do not replace admission checks under the Runtime operation lock; conflicts do
|
/// comparisons do not replace admission checks under the Runtime operation lock; conflicts do
|
||||||
/// not trigger automatic token replacement.
|
/// not trigger automatic token replacement.
|
||||||
/// [`WorkerRestoreState`] Accepted means Restore committed, not that the subsequent run succeeded.
|
/// [`WorkerRestoreState`] Accepted means Restore committed, not that the subsequent run succeeded.
|
||||||
/// Distinguish read-only preflight Rejected, confirmed RolledBack after side effects, and
|
/// Distinguish read-only preflight Rejected, confirmed RolledBack after side effects,
|
||||||
/// ReconciliationRequired needing settled-state checks. Response loss does not imply rollback.
|
/// Superseded by a later Stop, and ReconciliationRequired needing settled-state checks.
|
||||||
|
/// Response loss does not imply rollback.
|
||||||
#[api(reqwest, axum, openapi)]
|
#[api(reqwest, axum, openapi)]
|
||||||
pub trait ServerApi {
|
pub trait ServerApi {
|
||||||
#[get("/health", status = 200, error_status = 400)]
|
#[get("/health", status = 200, error_status = 400)]
|
||||||
@@ -652,6 +657,33 @@ pub trait ServerApi {
|
|||||||
#[body] request: PasskeyRegistrationCompleteRequest,
|
#[body] request: PasskeyRegistrationCompleteRequest,
|
||||||
) -> Result<server_api_responses::AuthPasskeyRegistrationComplete, RepositoryApiError>;
|
) -> Result<server_api_responses::AuthPasskeyRegistrationComplete, RepositoryApiError>;
|
||||||
|
|
||||||
|
#[get(
|
||||||
|
"/api/auth/passkeys",
|
||||||
|
status = 200,
|
||||||
|
error_status = 400,
|
||||||
|
additional_error_statuses = [403, 500],
|
||||||
|
bearer_auth = true,
|
||||||
|
browser_auth = true
|
||||||
|
)]
|
||||||
|
async fn auth_passkey_list(
|
||||||
|
&self,
|
||||||
|
#[extension] context: ServerRequestContext,
|
||||||
|
) -> Result<PasskeyListResponse, RepositoryApiError>;
|
||||||
|
|
||||||
|
#[post(
|
||||||
|
"/api/auth/passkeys/addition/result",
|
||||||
|
status = 200,
|
||||||
|
error_status = 400,
|
||||||
|
additional_error_statuses = [403, 500],
|
||||||
|
bearer_auth = true,
|
||||||
|
browser_auth = true
|
||||||
|
)]
|
||||||
|
async fn auth_passkey_addition_result(
|
||||||
|
&self,
|
||||||
|
#[extension] context: ServerRequestContext,
|
||||||
|
#[body] request: PasskeyAdditionResultRequest,
|
||||||
|
) -> Result<PasskeyAdditionResultResponse, RepositoryApiError>;
|
||||||
|
|
||||||
#[post(
|
#[post(
|
||||||
"/api/auth/passkeys/addition/options",
|
"/api/auth/passkeys/addition/options",
|
||||||
status = 200,
|
status = 200,
|
||||||
@@ -4119,6 +4151,8 @@ impl RequestActor {
|
|||||||
pub struct WhoamiResponse {
|
pub struct WhoamiResponse {
|
||||||
pub actor: Option<RequestActor>,
|
pub actor: Option<RequestActor>,
|
||||||
pub session: BrowserSessionState,
|
pub session: BrowserSessionState,
|
||||||
|
/// Non-secret comparison evidence, never an authentication credential.
|
||||||
|
pub authentication_reference: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Browser navigation evidence, not an authorization grant or a generic 401 classification.
|
/// Browser navigation evidence, not an authorization grant or a generic 401 classification.
|
||||||
@@ -4146,6 +4180,8 @@ pub struct AuthBootstrapUserRequest {
|
|||||||
#[serde(deny_unknown_fields)]
|
#[serde(deny_unknown_fields)]
|
||||||
pub struct AuthUserResponse {
|
pub struct AuthUserResponse {
|
||||||
pub user: AuthenticatedUser,
|
pub user: AuthenticatedUser,
|
||||||
|
/// Comparison evidence for the exact newly issued session, not a credential.
|
||||||
|
pub authentication_reference: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Starts new-account registration only; an existing handle is never interpreted as key addition.
|
/// Starts new-account registration only; an existing handle is never interpreted as key addition.
|
||||||
@@ -4162,16 +4198,64 @@ pub struct PasskeyRegistrationOptionsRequest {
|
|||||||
pub browser_origin: Option<String>,
|
pub browser_origin: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Adds a credential to the current authenticated actor; target identity is never public input.
|
/// Adds to the current actor. Expected identity/binding fields only fence stale UI;
|
||||||
|
/// they never select an owner or authorize an addition.
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
#[serde(deny_unknown_fields)]
|
#[serde(deny_unknown_fields)]
|
||||||
pub struct PasskeyAdditionOptionsRequest {
|
pub struct PasskeyAdditionOptionsRequest {
|
||||||
|
pub expected_user_id: String,
|
||||||
|
pub expected_account_id: String,
|
||||||
|
pub expected_authentication_reference: String,
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
#[cfg_attr(feature = "typescript", ts(optional = nullable))]
|
#[cfg_attr(feature = "typescript", ts(optional = nullable))]
|
||||||
pub browser_origin: Option<String>,
|
pub browser_origin: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Safe account metadata. Neither the WebAuthn credential ID nor key material is exposed.
|
||||||
|
/// Normal-login verification is observed verifier success, not inferred discoverability.
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct PasskeySummary {
|
||||||
|
pub key_reference: String,
|
||||||
|
pub created_at: String,
|
||||||
|
pub last_used_at: Option<String>,
|
||||||
|
pub normal_login_verified_at: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct PasskeyListResponse {
|
||||||
|
pub user: AuthenticatedUser,
|
||||||
|
pub keys: Vec<PasskeySummary>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct PasskeyAdditionResultRequest {
|
||||||
|
pub challenge_id: String,
|
||||||
|
pub expected_user_id: String,
|
||||||
|
pub expected_account_id: String,
|
||||||
|
/// Original options authentication reference; the current same-owner session may differ.
|
||||||
|
pub expected_authentication_reference: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Unknown is not proof that registration failed: completion may still be in flight.
|
||||||
|
/// Saved proves the credential commit, not session issuance or current session health.
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
|
#[serde(tag = "status", rename_all = "snake_case", deny_unknown_fields)]
|
||||||
|
pub enum PasskeyAdditionResultResponse {
|
||||||
|
Unknown {},
|
||||||
|
Saved {
|
||||||
|
user: AuthenticatedUser,
|
||||||
|
key: PasskeySummary,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize, JsonSchema)]
|
#[derive(Debug, Serialize, Deserialize, JsonSchema)]
|
||||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
#[serde(deny_unknown_fields)]
|
#[serde(deny_unknown_fields)]
|
||||||
@@ -6980,6 +7064,8 @@ pub struct WorkingDirectoryCleanupTarget {
|
|||||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
#[serde(deny_unknown_fields)]
|
#[serde(deny_unknown_fields)]
|
||||||
pub struct WorkingDirectoryRemovalRequest {
|
pub struct WorkingDirectoryRemovalRequest {
|
||||||
|
/// Stable for an exact resend; a new explicit request uses a new identity.
|
||||||
|
pub operation_id: String,
|
||||||
pub reason: String,
|
pub reason: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -7518,6 +7604,9 @@ pub struct CurrentWorkerWorkdirAttachmentResponse {
|
|||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
pub environment: Option<workdir::WorkdirSessionEnvironment>,
|
pub environment: Option<workdir::WorkdirSessionEnvironment>,
|
||||||
pub attached: bool,
|
pub attached: bool,
|
||||||
|
/// The Workdir was already attached to this Worker; alias is its existing name.
|
||||||
|
#[serde(default)]
|
||||||
|
pub already_attached: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
@@ -8889,6 +8978,9 @@ pub struct MergeRequestListQuery {
|
|||||||
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
#[serde(deny_unknown_fields)]
|
#[serde(deny_unknown_fields)]
|
||||||
pub struct MergeRequestThreadQuery {
|
pub struct MergeRequestThreadQuery {
|
||||||
|
/// Show only: summary omits history; detail returns an exact bounded thread page.
|
||||||
|
/// Omission retains the existing HTTP detail response.
|
||||||
|
pub view: Option<MergeRequestReadView>,
|
||||||
#[schemars(range(min = 0, max = 9_007_199_254_740_991_u64))]
|
#[schemars(range(min = 0, max = 9_007_199_254_740_991_u64))]
|
||||||
pub after: Option<u64>,
|
pub after: Option<u64>,
|
||||||
#[schemars(range(min = 0, max = 1000))]
|
#[schemars(range(min = 0, max = 1000))]
|
||||||
@@ -8916,6 +9008,73 @@ pub struct MergeRequestLinkedTicketResponse {
|
|||||||
pub key: Option<String>,
|
pub key: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum MergeRequestReadView {
|
||||||
|
Summary,
|
||||||
|
#[default]
|
||||||
|
Detail,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct MergeRequestEventReference {
|
||||||
|
pub event_id: String,
|
||||||
|
/// Exact event: read detail with after = sequence - 1 and limit = 1.
|
||||||
|
#[cfg_attr(feature = "typescript", ts(type = "number"))]
|
||||||
|
#[schemars(range(min = 0, max = 9_007_199_254_740_991_u64))]
|
||||||
|
pub sequence: u64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct MergeRequestReviewSummary {
|
||||||
|
pub event: MergeRequestEventReference,
|
||||||
|
pub request_event_id: String,
|
||||||
|
pub subject_ref: String,
|
||||||
|
pub ticket_content_digest: String,
|
||||||
|
pub decision: ReviewDecision,
|
||||||
|
/// Findings retain their order in the exact review event; no finding body is truncated here.
|
||||||
|
#[cfg_attr(feature = "typescript", ts(type = "number"))]
|
||||||
|
#[schemars(range(min = 0, max = 9_007_199_254_740_991_usize))]
|
||||||
|
pub findings_count: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Computed from complete persisted evidence, never from the returned history page.
|
||||||
|
/// This observation is not integration authority or current-requirements approval.
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct MergeRequestStatusSummary {
|
||||||
|
/// unknown, none, pending, approved, or request_changes for the observed source.
|
||||||
|
pub review_status: String,
|
||||||
|
pub review: Option<MergeRequestReviewSummary>,
|
||||||
|
pub latest_review_subject_ref: Option<String>,
|
||||||
|
pub latest_event: Option<MergeRequestEventReference>,
|
||||||
|
#[cfg_attr(feature = "typescript", ts(type = "number"))]
|
||||||
|
#[schemars(range(min = 0, max = 9_007_199_254_740_991_usize))]
|
||||||
|
pub thread_event_count: usize,
|
||||||
|
/// Existing readiness semantics; review bodies are omitted in this compact projection.
|
||||||
|
pub readiness: MergeRequestReadinessResponse,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct MergeRequestEventPage {
|
||||||
|
/// Summary deliberately omits events. Read detail from the beginning to obtain history.
|
||||||
|
pub omitted: bool,
|
||||||
|
pub has_more: bool,
|
||||||
|
#[cfg_attr(feature = "typescript", ts(type = "number | null"))]
|
||||||
|
#[schemars(range(min = 0, max = 9_007_199_254_740_991_u64))]
|
||||||
|
pub next_after: Option<u64>,
|
||||||
|
#[schemars(range(min = 1, max = 200))]
|
||||||
|
pub limit: usize,
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
#[serde(deny_unknown_fields)]
|
#[serde(deny_unknown_fields)]
|
||||||
@@ -8925,6 +9084,9 @@ pub struct MergeRequestDetailResponse {
|
|||||||
pub source: MergeRequestRefResponse,
|
pub source: MergeRequestRefResponse,
|
||||||
pub target: MergeRequestRefResponse,
|
pub target: MergeRequestRefResponse,
|
||||||
pub linked_tickets: Vec<MergeRequestLinkedTicketResponse>,
|
pub linked_tickets: Vec<MergeRequestLinkedTicketResponse>,
|
||||||
|
pub view: MergeRequestReadView,
|
||||||
|
pub summary: MergeRequestStatusSummary,
|
||||||
|
pub event_page: MergeRequestEventPage,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
@@ -10579,8 +10741,8 @@ pub enum WorkerOperationState {
|
|||||||
/// Cross-layer outcome of an explicit Worker restore operation.
|
/// Cross-layer outcome of an explicit Worker restore operation.
|
||||||
///
|
///
|
||||||
/// `Rejected` is reserved for read-only preflight failures. Once live restore
|
/// `Rejected` is reserved for read-only preflight failures. Once live restore
|
||||||
/// work starts, failure is either a confirmed `RolledBack` operation or a
|
/// work starts, distinguish confirmed `RolledBack`, unresolved
|
||||||
/// `ReconciliationRequired` result whose commit state must be reread/retried.
|
/// `ReconciliationRequired`, and a later Stop's `Superseded` outcome.
|
||||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
#[serde(rename_all = "snake_case")]
|
#[serde(rename_all = "snake_case")]
|
||||||
@@ -10588,6 +10750,8 @@ pub enum WorkerRestoreState {
|
|||||||
Accepted,
|
Accepted,
|
||||||
Rejected,
|
Rejected,
|
||||||
RolledBack,
|
RolledBack,
|
||||||
|
/// A later Stop superseded this restore; neither rollback nor prior Accepted is asserted.
|
||||||
|
Superseded,
|
||||||
ReconciliationRequired,
|
ReconciliationRequired,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -11651,6 +11815,10 @@ pub fn auth_api_typescript() -> String {
|
|||||||
AuthBootstrapUserRequest::decl(&config),
|
AuthBootstrapUserRequest::decl(&config),
|
||||||
AuthUserResponse::decl(&config),
|
AuthUserResponse::decl(&config),
|
||||||
PasskeyAdditionOptionsRequest::decl(&config),
|
PasskeyAdditionOptionsRequest::decl(&config),
|
||||||
|
PasskeySummary::decl(&config),
|
||||||
|
PasskeyListResponse::decl(&config),
|
||||||
|
PasskeyAdditionResultRequest::decl(&config),
|
||||||
|
PasskeyAdditionResultResponse::decl(&config),
|
||||||
PasskeyRegistrationOptionsRequest::decl(&config),
|
PasskeyRegistrationOptionsRequest::decl(&config),
|
||||||
PasskeyRegistrationOptionsResponse::decl(&config),
|
PasskeyRegistrationOptionsResponse::decl(&config),
|
||||||
PasskeyRegistrationCompleteRequest::decl(&config),
|
PasskeyRegistrationCompleteRequest::decl(&config),
|
||||||
@@ -12681,6 +12849,7 @@ mod tests {
|
|||||||
(WorkerRestoreState::Accepted, "accepted"),
|
(WorkerRestoreState::Accepted, "accepted"),
|
||||||
(WorkerRestoreState::Rejected, "rejected"),
|
(WorkerRestoreState::Rejected, "rejected"),
|
||||||
(WorkerRestoreState::RolledBack, "rolled_back"),
|
(WorkerRestoreState::RolledBack, "rolled_back"),
|
||||||
|
(WorkerRestoreState::Superseded, "superseded"),
|
||||||
(
|
(
|
||||||
WorkerRestoreState::ReconciliationRequired,
|
WorkerRestoreState::ReconciliationRequired,
|
||||||
"reconciliation_required",
|
"reconciliation_required",
|
||||||
@@ -13291,9 +13460,11 @@ mod tests {
|
|||||||
capabilities: workdir::WorkdirSessionCapabilities::EMPTY,
|
capabilities: workdir::WorkdirSessionCapabilities::EMPTY,
|
||||||
environment: None,
|
environment: None,
|
||||||
attached: false,
|
attached: false,
|
||||||
|
already_attached: false,
|
||||||
};
|
};
|
||||||
let value = serde_json::to_value(&response).unwrap();
|
let value = serde_json::to_value(&response).unwrap();
|
||||||
assert_eq!(value.as_object().unwrap().len(), 5);
|
assert_eq!(value.as_object().unwrap().len(), 6);
|
||||||
|
assert_eq!(value["already_attached"], false);
|
||||||
assert!(value.get("connection_id").is_none());
|
assert!(value.get("connection_id").is_none());
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
serde_json::from_value::<CurrentWorkerWorkdirAttachmentResponse>(value).unwrap(),
|
serde_json::from_value::<CurrentWorkerWorkdirAttachmentResponse>(value).unwrap(),
|
||||||
@@ -14079,7 +14250,8 @@ mod tests {
|
|||||||
"display_name": "Hare",
|
"display_name": "Hare",
|
||||||
"auth_method": "browser_session"
|
"auth_method": "browser_session"
|
||||||
},
|
},
|
||||||
"session": { "kind": "authenticated", "expires_at": "2026-10-20T00:00:00Z" }
|
"session": { "kind": "authenticated", "expires_at": "2026-10-20T00:00:00Z" },
|
||||||
|
"authentication_reference": "synthetic-session-reference"
|
||||||
});
|
});
|
||||||
let decoded = serde_json::from_value::<WhoamiResponse>(whoami.clone())
|
let decoded = serde_json::from_value::<WhoamiResponse>(whoami.clone())
|
||||||
.expect("server whoami fixture should match shared DTO");
|
.expect("server whoami fixture should match shared DTO");
|
||||||
@@ -14101,7 +14273,8 @@ mod tests {
|
|||||||
"account_id": "account-1",
|
"account_id": "account-1",
|
||||||
"handle": "hare",
|
"handle": "hare",
|
||||||
"display_name": "Hare"
|
"display_name": "Hare"
|
||||||
}
|
},
|
||||||
|
"authentication_reference": "synthetic-session-reference"
|
||||||
});
|
});
|
||||||
let decoded = serde_json::from_value::<AuthUserResponse>(auth_user.clone())
|
let decoded = serde_json::from_value::<AuthUserResponse>(auth_user.clone())
|
||||||
.expect("server auth-user fixture should match shared DTO");
|
.expect("server auth-user fixture should match shared DTO");
|
||||||
@@ -14274,7 +14447,7 @@ mod tests {
|
|||||||
serde_json::json!({"kind":"expired"}),
|
serde_json::json!({"kind":"expired"}),
|
||||||
serde_json::json!({"kind":"authenticated", "expires_at":null}),
|
serde_json::json!({"kind":"authenticated", "expires_at":null}),
|
||||||
] {
|
] {
|
||||||
let value = serde_json::json!({"actor":null, "session":session});
|
let value = serde_json::json!({"actor":null, "session":session, "authentication_reference": null});
|
||||||
let response: WhoamiResponse = serde_json::from_value(value.clone()).unwrap();
|
let response: WhoamiResponse = serde_json::from_value(value.clone()).unwrap();
|
||||||
assert_eq!(serde_json::to_value(response).unwrap(), value);
|
assert_eq!(serde_json::to_value(response).unwrap(), value);
|
||||||
}
|
}
|
||||||
@@ -14812,10 +14985,28 @@ mod openapi_artifact_tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
assert!(
|
assert!(
|
||||||
serde_json::from_value::<PasskeyAdditionOptionsRequest>(serde_json::json!({})).is_ok()
|
serde_json::from_value::<PasskeyAdditionOptionsRequest>(serde_json::json!({})).is_err()
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn passkey_addition_requires_expected_owner_and_binding() {
|
||||||
|
let expected = serde_json::json!({
|
||||||
|
"expected_user_id": "user-a", "expected_account_id": "account-a",
|
||||||
|
"expected_authentication_reference": "reference"
|
||||||
|
});
|
||||||
|
assert!(serde_json::from_value::<PasskeyAdditionOptionsRequest>(expected.clone()).is_ok());
|
||||||
|
for field in [
|
||||||
|
"expected_user_id",
|
||||||
|
"expected_account_id",
|
||||||
|
"expected_authentication_reference",
|
||||||
|
] {
|
||||||
|
let mut missing = expected.clone();
|
||||||
|
missing.as_object_mut().unwrap().remove(field);
|
||||||
|
assert!(serde_json::from_value::<PasskeyAdditionOptionsRequest>(missing).is_err());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn auth_workspace_catalog_and_typed_header_contract_is_strict() {
|
fn auth_workspace_catalog_and_typed_header_contract_is_strict() {
|
||||||
let document = canonical_openapi_document().expect("canonical OpenAPI contract must build");
|
let document = canonical_openapi_document().expect("canonical OpenAPI contract must build");
|
||||||
|
|||||||
@@ -53,8 +53,6 @@ pub struct PublicMaterialLimits {
|
|||||||
pub max_entries: usize,
|
pub max_entries: usize,
|
||||||
/// Allocation ceiling for one persisted JSONL record, not an entire run.
|
/// Allocation ceiling for one persisted JSONL record, not an entire run.
|
||||||
pub max_record_bytes: u64,
|
pub max_record_bytes: u64,
|
||||||
/// Aggregate fixed storage extent, bounding replay metadata as well as body decoding.
|
|
||||||
pub max_storage_bytes: u64,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Default for PublicMaterialLimits {
|
impl Default for PublicMaterialLimits {
|
||||||
@@ -63,7 +61,6 @@ impl Default for PublicMaterialLimits {
|
|||||||
max_segments: 256,
|
max_segments: 256,
|
||||||
max_entries: 1_000_000,
|
max_entries: 1_000_000,
|
||||||
max_record_bytes: 64 * 1024 * 1024,
|
max_record_bytes: 64 * 1024 * 1024,
|
||||||
max_storage_bytes: 256 * 1024 * 1024,
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -376,7 +373,6 @@ impl CommittedPublicReplay {
|
|||||||
Layout::Fs(root) => root.clone(),
|
Layout::Fs(root) => root.clone(),
|
||||||
};
|
};
|
||||||
let mut file_ends = HashMap::new();
|
let mut file_ends = HashMap::new();
|
||||||
let mut fixed_bytes = 0u64;
|
|
||||||
for entry in std::fs::read_dir(directory)? {
|
for entry in std::fs::read_dir(directory)? {
|
||||||
let entry = entry?;
|
let entry = entry?;
|
||||||
let path = entry.path();
|
let path = entry.path();
|
||||||
@@ -394,14 +390,8 @@ impl CommittedPublicReplay {
|
|||||||
return Err(PublicMaterialError::ResourceLimit);
|
return Err(PublicMaterialError::ResourceLimit);
|
||||||
}
|
}
|
||||||
let reader = RecordReader::open(&path, limits.max_record_bytes)?;
|
let reader = RecordReader::open(&path, limits.max_record_bytes)?;
|
||||||
// This also bounds retained descriptor/identity allocations during replay,
|
// Freeze the extent, but do not cap the lifetime storage total. Records
|
||||||
// not just the final capture. Oversized histories fail explicitly.
|
// are decoded individually; segment and entry limits bound replay metadata.
|
||||||
fixed_bytes = fixed_bytes
|
|
||||||
.checked_add(reader.end)
|
|
||||||
.ok_or(PublicMaterialError::ResourceLimit)?;
|
|
||||||
if fixed_bytes > limits.max_storage_bytes {
|
|
||||||
return Err(PublicMaterialError::ResourceLimit);
|
|
||||||
}
|
|
||||||
file_ends.insert(id, reader.end);
|
file_ends.insert(id, reader.end);
|
||||||
}
|
}
|
||||||
let mut chain = Vec::new();
|
let mut chain = Vec::new();
|
||||||
|
|||||||
@@ -1802,7 +1802,7 @@ fn recording_capture_fence_blocks_publication_and_value_survives_later_append_an
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn recording_capture_rejects_foreign_guard_and_storage_or_body_allocation_excess() {
|
fn recording_capture_rejects_foreign_guard_and_record_or_body_allocation_excess() {
|
||||||
use crate::capture::*;
|
use crate::capture::*;
|
||||||
let temp = tempfile::tempdir().unwrap();
|
let temp = tempfile::tempdir().unwrap();
|
||||||
let root = temp.path().join("session");
|
let root = temp.path().join("session");
|
||||||
@@ -1834,7 +1834,7 @@ fn recording_capture_rejects_foreign_guard_and_storage_or_body_allocation_excess
|
|||||||
));
|
));
|
||||||
let guard = retention.lock_live(session).unwrap();
|
let guard = retention.lock_live(session).unwrap();
|
||||||
let limits = PublicMaterialLimits {
|
let limits = PublicMaterialLimits {
|
||||||
max_storage_bytes: 1,
|
max_record_bytes: 1,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
};
|
};
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
@@ -2457,10 +2457,6 @@ fn recording_catalog_errors_do_not_invoke_callback_or_relax_resource_and_identit
|
|||||||
Err(PublicMaterialError::Inconsistent)
|
Err(PublicMaterialError::Inconsistent)
|
||||||
));
|
));
|
||||||
for limits in [
|
for limits in [
|
||||||
PublicMaterialLimits {
|
|
||||||
max_storage_bytes: 1,
|
|
||||||
..Default::default()
|
|
||||||
},
|
|
||||||
PublicMaterialLimits {
|
PublicMaterialLimits {
|
||||||
max_record_bytes: 1,
|
max_record_bytes: 1,
|
||||||
..Default::default()
|
..Default::default()
|
||||||
|
|||||||
@@ -194,6 +194,21 @@ impl WorkerSessionStore {
|
|||||||
Ok(store)
|
Ok(store)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Open a current retained Session without creating or migrating anything.
|
||||||
|
/// The live constraint is checked here and again by every later publication;
|
||||||
|
/// attaching the control store itself does not acquire publication authority.
|
||||||
|
pub fn open_existing_with_retention(
|
||||||
|
root: impl Into<PathBuf>,
|
||||||
|
retention: crate::retention::SessionRetentionStore,
|
||||||
|
) -> Result<Self, StoreError> {
|
||||||
|
let mut store = Self::open_read_only(root)?;
|
||||||
|
if let Some(session) = *store.session_id.lock().unwrap() {
|
||||||
|
retention.ensure_live(session)?;
|
||||||
|
}
|
||||||
|
store.retention = Some(retention);
|
||||||
|
Ok(store)
|
||||||
|
}
|
||||||
|
|
||||||
fn lock_publication(
|
fn lock_publication(
|
||||||
&self,
|
&self,
|
||||||
session: SessionId,
|
session: SessionId,
|
||||||
|
|||||||
@@ -194,6 +194,29 @@ impl SubjektivStore {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// End a body after the Host has confirmed its Runtime removal. This is not a
|
||||||
|
/// completed Journal handoff: source coverage, positions and saved receipts stay
|
||||||
|
/// unchanged. Revocation and closing generations are atomic, so no retained Job
|
||||||
|
/// can write after deletion even while Backend Job cleanup is being retried.
|
||||||
|
pub fn release_removed_journal_body(&self, host: &JournalHostContext) -> JournalResult<()> {
|
||||||
|
self.database.try_transaction(|tx| {
|
||||||
|
super::store::check_session_host(self, tx, host)?;
|
||||||
|
let attr = host.attribution.as_ref().expect("checked Session authority");
|
||||||
|
let binding: Option<String> = tx.query_row(
|
||||||
|
"SELECT binding_id FROM journal_writer_bindings WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3 ORDER BY rowid DESC LIMIT 1",
|
||||||
|
params![host.subject_id, attr.runtime_id, attr.worker_id], |r| r.get(0),
|
||||||
|
).optional()?;
|
||||||
|
let binding = binding.unwrap_or_else(|| crate::issued_id("journal-removed-body"));
|
||||||
|
tx.execute("INSERT OR IGNORE INTO journal_writer_bindings(binding_id,subject_id,runtime_id,worker_id,active) VALUES(?1,?2,?3,?4,0)",params![binding,host.subject_id,attr.runtime_id,attr.worker_id])?;
|
||||||
|
tx.execute("UPDATE journal_writer_bindings SET active=0 WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3",params![host.subject_id,attr.runtime_id,attr.worker_id])?;
|
||||||
|
// Bind legacy attributed Sessions to an inactive lifetime, never to the
|
||||||
|
// successor. Retained unfinished history is not an active writer.
|
||||||
|
tx.execute("INSERT OR IGNORE INTO journal_session_bindings(session_id,binding_id) SELECT session_id,?4 FROM subject_session_attributions WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3",params![host.subject_id,attr.runtime_id,attr.worker_id,binding])?;
|
||||||
|
tx.execute("UPDATE journal_generations SET active=0 WHERE journal_id IN (SELECT j.id FROM journals j JOIN subject_session_attributions a ON a.session_id=j.source_session_id WHERE a.subject_id=?1 AND a.runtime_id=?2 AND a.worker_id=?3)",params![host.subject_id,attr.runtime_id,attr.worker_id])?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
/// The Host must hold its lifecycle fence, confirm Idle, reconcile all committed original
|
/// The Host must hold its lifecycle fence, confirm Idle, reconcile all committed original
|
||||||
/// Sessions (including the save/notification gap), and exclude queued/running/Unknown Jobs.
|
/// Sessions (including the save/notification gap), and exclude queued/running/Unknown Jobs.
|
||||||
/// This transaction rejects retained writers, pending source bytes and active generations.
|
/// This transaction rejects retained writers, pending source bytes and active generations.
|
||||||
|
|||||||
@@ -651,6 +651,32 @@ impl SubjektivStore {
|
|||||||
Ok(())
|
Ok(())
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
/// Read the exact retained Job/attempt snapshot without granting writer
|
||||||
|
/// authority or creating a generation. Used for outcome recovery after removal.
|
||||||
|
pub fn retained_journal_generation(
|
||||||
|
&self,
|
||||||
|
host: &JournalHostContext,
|
||||||
|
id: &str,
|
||||||
|
binding: &JobAttemptBinding,
|
||||||
|
) -> JournalResult<Option<JournalGeneration>> {
|
||||||
|
self.database.try_with_connection(|tx| {
|
||||||
|
check_host(self, tx, host, false)?;
|
||||||
|
check_session_host(self, tx, host)?;
|
||||||
|
journal(tx, host, id, true)?;
|
||||||
|
let raw: Option<String> = tx.query_row(
|
||||||
|
"SELECT snapshot_json FROM journal_generations WHERE journal_id=?1 AND job_id=?2 AND attempt_id=?3",
|
||||||
|
params![id, binding.job_id, binding.attempt_id], |r| r.get(0),
|
||||||
|
).optional()?;
|
||||||
|
raw.map(|raw| {
|
||||||
|
let generation: JournalGeneration = serde_json::from_str(&raw)?;
|
||||||
|
if generation.journal_id != id || &generation.job != binding {
|
||||||
|
return Err(conflict("retained generation identity mismatch"));
|
||||||
|
}
|
||||||
|
Ok(generation)
|
||||||
|
}).transpose()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
pub fn read_journal_generation(
|
pub fn read_journal_generation(
|
||||||
&self,
|
&self,
|
||||||
host: &JournalHostContext,
|
host: &JournalHostContext,
|
||||||
|
|||||||
+319
-38
@@ -132,9 +132,7 @@ System events and case judgments cannot be edited. The Workspace backend supplie
|
|||||||
const LIST_DESCRIPTION: &str = "Query Tickets from the configured typed Ticket backend as a bounded \
|
const LIST_DESCRIPTION: &str = "Query Tickets from the configured typed Ticket backend as a bounded \
|
||||||
overview. Workspace-backed Workers replace this definition with the richer authoritative \
|
overview. Workspace-backed Workers replace this definition with the richer authoritative \
|
||||||
text/event/evidence/relation/Objective/time/attention query.";
|
text/event/evidence/relation/Objective/time/attention query.";
|
||||||
const SHOW_DESCRIPTION: &str = "Show one Ticket by id or exact query through the configured typed \
|
const SHOW_DESCRIPTION: &str = "Read one Ticket by id or exact query. Use view=summary for status/CAS without prose or history; detail (default) for canonical initial_post and bounded recent discussion; history with event_cursor for older thread pages; post with an observed event_ref for the exact full post. Never use truncated bodies for edit CAS or guess event identities. Workspace-backed Workers retain the same views with authoritative delegation and MR evidence.";
|
||||||
Ticket backend. Output includes the initial comment and recent events with stable event_ref, author/time and edit metadata, bounded Markdown body, resolution, and artifact \
|
|
||||||
metadata; Workspace-backed Workers replace this definition with the richer authoritative evidence projection.";
|
|
||||||
const COMMENT_DESCRIPTION: &str = "Append a typed Ticket comment event. `body` is Markdown.";
|
const COMMENT_DESCRIPTION: &str = "Append a typed Ticket comment event. `body` is Markdown.";
|
||||||
const PLAN_DESCRIPTION: &str = "Append a typed Ticket plan event. `body` is Markdown.";
|
const PLAN_DESCRIPTION: &str = "Append a typed Ticket plan event. `body` is Markdown.";
|
||||||
const DECISION_DESCRIPTION: &str = "Append a typed Ticket decision event. `body` is Markdown.";
|
const DECISION_DESCRIPTION: &str = "Append a typed Ticket decision event. `body` is Markdown.";
|
||||||
@@ -549,8 +547,50 @@ impl QueryTicketParams {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Model-facing read modes shared by generic Ticket and Workspace Tool/WIP reads.
|
||||||
|
#[derive(
|
||||||
|
Debug, Default, Clone, Copy, PartialEq, Eq, Deserialize, Serialize, schemars::JsonSchema,
|
||||||
|
)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum ShowTicketView {
|
||||||
|
Summary,
|
||||||
|
#[default]
|
||||||
|
Detail,
|
||||||
|
History,
|
||||||
|
Post,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ShowTicketView {
|
||||||
|
pub fn validate(
|
||||||
|
self,
|
||||||
|
event_cursor: Option<&str>,
|
||||||
|
event_ref: Option<&str>,
|
||||||
|
) -> Result<(), String> {
|
||||||
|
if (self == Self::Post) != event_ref.is_some()
|
||||||
|
|| event_ref.is_some_and(|value| value.trim().is_empty())
|
||||||
|
{
|
||||||
|
return Err(
|
||||||
|
"event_ref is required only for view=post; use an observed identity".into(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if event_cursor.is_some() && !matches!(self, Self::Detail | Self::History) {
|
||||||
|
return Err("event_cursor is only valid for detail/history".into());
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Deserialize, schemars::JsonSchema)]
|
#[derive(Debug, Deserialize, schemars::JsonSchema)]
|
||||||
struct ShowTicketParams {
|
struct ShowTicketParams {
|
||||||
|
/// summary omits prose/history; detail is the default; history returns only a thread page; post returns an exact full post up to 1 MiB (larger posts fail, never truncate).
|
||||||
|
#[serde(default)]
|
||||||
|
view: ShowTicketView,
|
||||||
|
/// Opaque continuation from event_page. Valid for detail/history only.
|
||||||
|
#[serde(default)]
|
||||||
|
event_cursor: Option<String>,
|
||||||
|
/// Exact observed post identity. Required only for view=post.
|
||||||
|
#[serde(default)]
|
||||||
|
event_ref: Option<String>,
|
||||||
/// Ticket reference. Prefer `T-*`; canonical internal ids remain accepted for compatibility. Exactly one of `id` or `query` must be provided.
|
/// Ticket reference. Prefer `T-*`; canonical internal ids remain accepted for compatibility. Exactly one of `id` or `query` must be provided.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
id: Option<String>,
|
id: Option<String>,
|
||||||
@@ -1096,6 +1136,10 @@ impl Tool for ShowTicketTool {
|
|||||||
_ctx: agen::tool::ToolExecutionContext,
|
_ctx: agen::tool::ToolExecutionContext,
|
||||||
) -> Result<ToolOutput, ToolError> {
|
) -> Result<ToolOutput, ToolError> {
|
||||||
let params: ShowTicketParams = parse_input("ShowTicket", input_json)?;
|
let params: ShowTicketParams = parse_input("ShowTicket", input_json)?;
|
||||||
|
params
|
||||||
|
.view
|
||||||
|
.validate(params.event_cursor.as_deref(), params.event_ref.as_deref())
|
||||||
|
.map_err(ToolError::InvalidArgument)?;
|
||||||
let query = id_or_query(params.id, params.query)?;
|
let query = id_or_query(params.id, params.query)?;
|
||||||
let event_limit = bounded(params.event_limit, DEFAULT_EVENT_LIMIT, MAX_EVENT_LIMIT);
|
let event_limit = bounded(params.event_limit, DEFAULT_EVENT_LIMIT, MAX_EVENT_LIMIT);
|
||||||
let artifact_limit = bounded(
|
let artifact_limit = bounded(
|
||||||
@@ -1117,10 +1161,45 @@ impl Tool for ShowTicketTool {
|
|||||||
ticket.meta.id,
|
ticket.meta.id,
|
||||||
ticket.meta.workflow_state.as_str()
|
ticket.meta.workflow_state.as_str()
|
||||||
);
|
);
|
||||||
Ok(json_output(
|
let value = match params.view {
|
||||||
summary,
|
ShowTicketView::Summary => ticket_status_json(&ticket),
|
||||||
ticket_json(&ticket, event_limit, artifact_limit, body_max_bytes),
|
ShowTicketView::Post => {
|
||||||
))
|
let event = ticket
|
||||||
|
.events
|
||||||
|
.iter()
|
||||||
|
.find(|event| Some(event.event_ref.as_str()) == params.event_ref.as_deref())
|
||||||
|
.ok_or_else(|| {
|
||||||
|
ToolError::InvalidArgument(
|
||||||
|
"Observed post identity not found in this Ticket".into(),
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
// Match the Workspace full-post route: never silently truncate CAS input.
|
||||||
|
if event.body.as_str().len() > 1024 * 1024 {
|
||||||
|
return Err(ToolError::ExecutionFailed(
|
||||||
|
"Exact post exceeds the 1 MiB full-read limit; do not edit from a truncated preview".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
json!({"ticket": ticket.meta.id, "post": ticket_event_json(event, usize::MAX)})
|
||||||
|
}
|
||||||
|
ShowTicketView::Detail | ShowTicketView::History => {
|
||||||
|
let (events, page) = ticket_event_page(
|
||||||
|
&ticket,
|
||||||
|
params.event_cursor.as_deref(),
|
||||||
|
event_limit,
|
||||||
|
body_max_bytes,
|
||||||
|
)?;
|
||||||
|
if params.view == ShowTicketView::History {
|
||||||
|
json!({"ticket": ticket.meta.id, "thread": events, "event_page": page})
|
||||||
|
} else {
|
||||||
|
let mut detail =
|
||||||
|
ticket_json(&ticket, event_limit, artifact_limit, body_max_bytes);
|
||||||
|
detail["events"] = events;
|
||||||
|
detail["event_page"] = page;
|
||||||
|
detail
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
Ok(json_output(summary, value))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1694,8 +1773,70 @@ fn ticket_relations_json(ticket: &Ticket) -> Value {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn ticket_event_json(event: &crate::TicketEvent, body_max_bytes: usize) -> Value {
|
fn ticket_status_json(ticket: &Ticket) -> Value {
|
||||||
json!({
|
json!({
|
||||||
|
"ticket": ticket.meta.resource_key.as_deref().unwrap_or(&ticket.meta.id),
|
||||||
|
"title": ticket.meta.title,
|
||||||
|
"state": ticket.meta.workflow_state,
|
||||||
|
"content_digest": crate::ticket_content_digest(ticket),
|
||||||
|
"closure_operation_key": ticket.closure_operation_key,
|
||||||
|
"current_closure": ticket.current_closure.as_ref().map(|j| json!({
|
||||||
|
"operation_key": j.operation_key, "reason": j.reason, "author": j.author, "at": j.at
|
||||||
|
})),
|
||||||
|
"targets": ticket.meta.targets,
|
||||||
|
"tags": ticket.tags,
|
||||||
|
"assignee": ticket.meta.assignee,
|
||||||
|
"readiness": ticket.meta.readiness,
|
||||||
|
"updated_at": ticket.meta.updated_at,
|
||||||
|
"latest_event_ref": ticket.events.last().map(|event| &event.event_ref),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Cursor is the persisted identity of the oldest returned event, never an offset.
|
||||||
|
// Appending newer events between pages therefore cannot shift the continuation.
|
||||||
|
fn ticket_event_page(
|
||||||
|
ticket: &Ticket,
|
||||||
|
cursor: Option<&str>,
|
||||||
|
limit: usize,
|
||||||
|
body_max_bytes: usize,
|
||||||
|
) -> Result<(Value, Value), ToolError> {
|
||||||
|
let events: Vec<_> = ticket
|
||||||
|
.events
|
||||||
|
.iter()
|
||||||
|
.filter(|event| !event.is_initial)
|
||||||
|
.collect();
|
||||||
|
let end = match cursor {
|
||||||
|
None => events.len(),
|
||||||
|
Some(cursor) => events
|
||||||
|
.iter()
|
||||||
|
.position(|event| event.event_ref == cursor)
|
||||||
|
.ok_or_else(|| {
|
||||||
|
ToolError::InvalidArgument(
|
||||||
|
"Unknown Ticket event_cursor; reuse a returned cursor".into(),
|
||||||
|
)
|
||||||
|
})?,
|
||||||
|
};
|
||||||
|
let start = end.saturating_sub(limit.max(1));
|
||||||
|
let items: Vec<_> = events[start..end]
|
||||||
|
.iter()
|
||||||
|
.map(|event| ticket_event_json(event, body_max_bytes))
|
||||||
|
.collect();
|
||||||
|
let next_cursor = if start > 0 {
|
||||||
|
Some(&events[start].event_ref)
|
||||||
|
} else {
|
||||||
|
None
|
||||||
|
};
|
||||||
|
Ok((
|
||||||
|
json!({"count": events.len(), "returned": items.len(), "truncated": start > 0, "items": items}),
|
||||||
|
json!({"next_cursor": next_cursor, "has_more": start > 0}),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn ticket_event_json(event: &crate::TicketEvent, body_max_bytes: usize) -> Value {
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
json!({
|
||||||
|
"body_digest": Sha256::digest(event.body.as_str().as_bytes()).iter().map(|byte| format!("{byte:02x}")).collect::<String>(),
|
||||||
|
"editable": event.kind == TicketEventKind::Comment,
|
||||||
"event_ref": event.event_ref,
|
"event_ref": event.event_ref,
|
||||||
"is_initial": event.is_initial,
|
"is_initial": event.is_initial,
|
||||||
"kind": event.kind.as_str(),
|
"kind": event.kind.as_str(),
|
||||||
@@ -1722,13 +1863,8 @@ fn ticket_json(
|
|||||||
artifact_limit: usize,
|
artifact_limit: usize,
|
||||||
body_max_bytes: usize,
|
body_max_bytes: usize,
|
||||||
) -> Value {
|
) -> Value {
|
||||||
let event_count = ticket.events.len();
|
let (events, event_page) = ticket_event_page(ticket, None, event_limit, body_max_bytes)
|
||||||
let events: Vec<_> = ticket
|
.expect("first thread page has no cursor to reject");
|
||||||
.events
|
|
||||||
.iter()
|
|
||||||
.skip(event_count.saturating_sub(event_limit))
|
|
||||||
.map(|event| ticket_event_json(event, body_max_bytes))
|
|
||||||
.collect();
|
|
||||||
let artifact_count = ticket.artifacts.len();
|
let artifact_count = ticket.artifacts.len();
|
||||||
let artifacts: Vec<_> = ticket
|
let artifacts: Vec<_> = ticket
|
||||||
.artifacts
|
.artifacts
|
||||||
@@ -1753,17 +1889,12 @@ fn ticket_json(
|
|||||||
},
|
},
|
||||||
"content_digest": crate::ticket_content_digest(ticket),
|
"content_digest": crate::ticket_content_digest(ticket),
|
||||||
"closure_operation_key": ticket.closure_operation_key,
|
"closure_operation_key": ticket.closure_operation_key,
|
||||||
"current_closure": ticket.current_closure.as_ref().map(|j| json!({"operation_key": j.operation_key, "reason": j.reason, "explanation": truncate_text(&j.explanation, body_max_bytes), "author": j.author, "references": j.references, "at": j.at})),
|
"current_closure": ticket.current_closure.as_ref().map(|j| json!({"operation_key": j.operation_key, "reason": j.reason, "author": j.author, "references": j.references, "at": j.at})),
|
||||||
"case_history": { "count": ticket.case_history.len(), "items": ticket.case_history.iter().rev().take(event_limit).map(|j| json!({"operation_key": j.operation_key, "previous_closure_operation_key": j.previous_closure_operation_key, "action": j.action, "explanation": truncate_text(&j.explanation, body_max_bytes), "author": j.author, "references": j.references, "at": j.at})).collect::<Vec<_>>() },
|
"case_history": { "count": ticket.case_history.len(), "items": ticket.case_history.iter().rev().take(event_limit).map(|j| json!({"operation_key": j.operation_key, "previous_closure_operation_key": j.previous_closure_operation_key, "action": j.action, "author": j.author, "references": j.references, "at": j.at})).collect::<Vec<_>>() },
|
||||||
"legacy_workflow_snapshot": ticket.legacy_workflow_snapshot,
|
"legacy_workflow_snapshot": ticket.legacy_workflow_snapshot,
|
||||||
"initial_post": ticket.events.iter().find(|event| event.is_initial).map(|event| ticket_event_json(event, body_max_bytes)),
|
"initial_post": ticket.events.iter().find(|event| event.is_initial).map(|event| ticket_event_json(event, body_max_bytes)),
|
||||||
"body": truncate_text(ticket.document.body.as_str(), body_max_bytes),
|
"events": events,
|
||||||
"events": {
|
"event_page": event_page,
|
||||||
"count": event_count,
|
|
||||||
"returned": events.len(),
|
|
||||||
"truncated": event_count > events.len(),
|
|
||||||
"items": events,
|
|
||||||
},
|
|
||||||
"artifacts": {
|
"artifacts": {
|
||||||
"count": artifact_count,
|
"count": artifact_count,
|
||||||
"returned": artifacts.len(),
|
"returned": artifacts.len(),
|
||||||
@@ -2063,13 +2194,17 @@ mod tests {
|
|||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let value: Value = serde_json::from_str(&output.content.unwrap()).unwrap();
|
let value: Value = serde_json::from_str(&output.content.unwrap()).unwrap();
|
||||||
assert_eq!(value["events"]["count"], 2);
|
assert_eq!(value["events"]["count"], 1);
|
||||||
let projected = value["events"]["items"]
|
let projected = if post.is_initial {
|
||||||
.as_array()
|
&value["initial_post"]
|
||||||
.unwrap()
|
} else {
|
||||||
.iter()
|
value["events"]["items"]
|
||||||
.find(|e| e["event_ref"] == post.event_ref)
|
.as_array()
|
||||||
.unwrap();
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.find(|e| e["event_ref"] == post.event_ref)
|
||||||
|
.unwrap()
|
||||||
|
};
|
||||||
assert_eq!(projected["author"], json!(post.author));
|
assert_eq!(projected["author"], json!(post.author));
|
||||||
assert_eq!(projected["at"], json!(post.at));
|
assert_eq!(projected["at"], json!(post.at));
|
||||||
assert_eq!(projected["is_initial"], post.is_initial);
|
assert_eq!(projected["is_initial"], post.is_initial);
|
||||||
@@ -2104,7 +2239,8 @@ mod tests {
|
|||||||
value["initial_post"]["event_ref"],
|
value["initial_post"]["event_ref"],
|
||||||
before.events[0].event_ref
|
before.events[0].event_ref
|
||||||
);
|
);
|
||||||
assert_eq!(value["initial_post"]["body"], value["body"]);
|
assert_eq!(value["initial_post"]["body"], "Edited Initial body");
|
||||||
|
assert!(value.get("body").is_none());
|
||||||
assert_eq!(value["meta"]["created_by"], "creator");
|
assert_eq!(value["meta"]["created_by"], "creator");
|
||||||
assert_eq!(value["initial_post"]["author"], "creator");
|
assert_eq!(value["initial_post"]["author"], "creator");
|
||||||
let initial_keys = value["initial_post"]
|
let initial_keys = value["initial_post"]
|
||||||
@@ -2518,7 +2654,10 @@ mod tests {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
let value: Value = serde_json::from_str(&output.content.unwrap()).unwrap();
|
let value: Value = serde_json::from_str(&output.content.unwrap()).unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
value["body"].as_str().unwrap().trim_start_matches('\n'),
|
value["initial_post"]["body"]
|
||||||
|
.as_str()
|
||||||
|
.unwrap()
|
||||||
|
.trim_start_matches('\n'),
|
||||||
"ONE\ntwo\nONE\n"
|
"ONE\ntwo\nONE\n"
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -2980,13 +3119,13 @@ mod tests {
|
|||||||
assert_eq!(value["closure_operation_key"], "tool-correct");
|
assert_eq!(value["closure_operation_key"], "tool-correct");
|
||||||
assert_eq!(value["case_history"]["count"], 2);
|
assert_eq!(value["case_history"]["count"], 2);
|
||||||
assert_eq!(value["case_history"]["items"].as_array().unwrap().len(), 1);
|
assert_eq!(value["case_history"]["items"].as_array().unwrap().len(), 1);
|
||||||
|
assert!(value["current_closure"].get("explanation").is_none());
|
||||||
assert!(
|
assert!(
|
||||||
value["current_closure"]["explanation"]
|
value["case_history"]["items"][0]
|
||||||
.as_str()
|
.get("explanation")
|
||||||
.unwrap()
|
.is_none()
|
||||||
.len()
|
|
||||||
< 256
|
|
||||||
);
|
);
|
||||||
|
assert_eq!(value["events"]["items"][0]["body_truncated"], true);
|
||||||
assert!(value["content_digest"].as_str().unwrap().len() == 64);
|
assert!(value["content_digest"].as_str().unwrap().len() == 64);
|
||||||
let mut reopen = correction;
|
let mut reopen = correction;
|
||||||
reopen["expected_closure_operation_key"] = json!("tool-correct");
|
reopen["expected_closure_operation_key"] = json!("tool-correct");
|
||||||
@@ -3015,6 +3154,148 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn show_views_deduplicate_bound_history_and_read_exact_unicode_posts() {
|
||||||
|
let temp = TempDir::new().unwrap();
|
||||||
|
let backend = backend(&temp);
|
||||||
|
let body = "long-requirement-".repeat(700);
|
||||||
|
let mut input = NewTicket::new("Read projections");
|
||||||
|
input.body = MarkdownText::new(body.clone());
|
||||||
|
let reference = backend.create(input).unwrap();
|
||||||
|
for index in 0..5 {
|
||||||
|
backend
|
||||||
|
.add_event(
|
||||||
|
reference.id.clone().into(),
|
||||||
|
NewTicketEvent::new(TicketEventKind::Comment, format!("comment-{index}")),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
let before = backend.show(reference.id.clone().into()).unwrap();
|
||||||
|
let detail = ticket_json(&before, 20, 20, DEFAULT_BODY_MAX_BYTES);
|
||||||
|
assert_eq!(detail.to_string().matches(&body).count(), 1);
|
||||||
|
assert!(detail.get("body").is_none());
|
||||||
|
assert!(
|
||||||
|
detail["events"]["items"]
|
||||||
|
.as_array()
|
||||||
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.all(|event| event["is_initial"] == false)
|
||||||
|
);
|
||||||
|
let mut larger = before.clone();
|
||||||
|
larger.document.body = MarkdownText::new("prose".repeat(100_000));
|
||||||
|
larger.events[0].body = larger.document.body.clone();
|
||||||
|
larger
|
||||||
|
.events
|
||||||
|
.extend(vec![before.events.last().unwrap().clone(); 1000]);
|
||||||
|
let small_summary = ticket_status_json(&before).to_string();
|
||||||
|
let large_summary = ticket_status_json(&larger).to_string();
|
||||||
|
assert_eq!(small_summary.len(), large_summary.len());
|
||||||
|
eprintln!(
|
||||||
|
"Ticket generic bytes: detail={}, summary={} (unchanged with 500KB body / 1000 extra events)",
|
||||||
|
detail.to_string().len(),
|
||||||
|
small_summary.len()
|
||||||
|
);
|
||||||
|
|
||||||
|
let show = tool_by_name(backend.clone(), "ShowTicket");
|
||||||
|
let mut cursor = Value::Null;
|
||||||
|
let mut seen = Vec::new();
|
||||||
|
loop {
|
||||||
|
let output = show.execute(&json!({"id": reference.id, "view": "history", "event_limit": 2, "event_cursor": cursor}).to_string(), Default::default()).await.unwrap();
|
||||||
|
let page: Value = serde_json::from_str(&output.content.unwrap()).unwrap();
|
||||||
|
assert_eq!(page.as_object().unwrap().len(), 3);
|
||||||
|
for event in page["thread"]["items"].as_array().unwrap() {
|
||||||
|
seen.push(event["event_ref"].as_str().unwrap().to_owned());
|
||||||
|
}
|
||||||
|
if cursor.is_null() {
|
||||||
|
backend
|
||||||
|
.add_event(
|
||||||
|
reference.id.clone().into(),
|
||||||
|
NewTicketEvent::new(TicketEventKind::Comment, "newer append"),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
cursor = page["event_page"]["next_cursor"].clone();
|
||||||
|
if cursor.is_null() {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
seen.sort();
|
||||||
|
let mut expected: Vec<_> = before
|
||||||
|
.events
|
||||||
|
.iter()
|
||||||
|
.filter(|event| !event.is_initial)
|
||||||
|
.map(|event| event.event_ref.clone())
|
||||||
|
.collect();
|
||||||
|
expected.sort();
|
||||||
|
assert_eq!(seen, expected);
|
||||||
|
assert!(
|
||||||
|
show.execute(
|
||||||
|
&json!({"id": reference.id, "view":"history", "event_cursor":"guessed"})
|
||||||
|
.to_string(),
|
||||||
|
Default::default()
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
|
||||||
|
let unicode = "全文🦀é\n".repeat(20_000);
|
||||||
|
let post = &before.events[0];
|
||||||
|
let edit = tool_by_name(backend.clone(), "TicketEditPost");
|
||||||
|
let edit_args = json!({"ticket":reference.id, "event_ref":post.event_ref, "expected_body":post.body.as_str(), "body":unicode});
|
||||||
|
edit.execute(&edit_args.to_string(), Default::default())
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let full = show.execute(&json!({"id":reference.id, "view":"post", "event_ref":post.event_ref, "body_max_bytes":1}).to_string(), Default::default()).await.unwrap();
|
||||||
|
let full: Value = serde_json::from_str(&full.content.unwrap()).unwrap();
|
||||||
|
assert_eq!(full["post"]["body"], unicode);
|
||||||
|
assert_eq!(full["post"]["body_truncated"], false);
|
||||||
|
assert_eq!(full["post"]["editable"], true);
|
||||||
|
assert!(
|
||||||
|
edit.execute(&edit_args.to_string(), Default::default())
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
for args in [
|
||||||
|
json!({"view":"post"}),
|
||||||
|
json!({"view":"summary", "event_ref":post.event_ref}),
|
||||||
|
json!({"view":"summary", "event_cursor":"x"}),
|
||||||
|
] {
|
||||||
|
let mut args = args;
|
||||||
|
args["id"] = json!(reference.id);
|
||||||
|
assert!(
|
||||||
|
show.execute(&args.to_string(), Default::default())
|
||||||
|
.await
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn exact_post_read_rejects_oversize_instead_of_truncating_edit_input() {
|
||||||
|
let temp = TempDir::new().unwrap();
|
||||||
|
let backend = backend(&temp);
|
||||||
|
let mut input = NewTicket::new("Full-post limit");
|
||||||
|
input.body = MarkdownText::new("x".repeat(1024 * 1024 + 1));
|
||||||
|
let reference = backend.create(input).unwrap();
|
||||||
|
let observed = backend.show(reference.id.clone().into()).unwrap();
|
||||||
|
let event_ref = &observed
|
||||||
|
.events
|
||||||
|
.iter()
|
||||||
|
.find(|event| event.is_initial)
|
||||||
|
.unwrap()
|
||||||
|
.event_ref;
|
||||||
|
let show = tool_by_name(backend.clone(), "ShowTicket");
|
||||||
|
let error = show
|
||||||
|
.execute(
|
||||||
|
&json!({"id":reference.id, "view":"post", "event_ref":event_ref}).to_string(),
|
||||||
|
Default::default(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(error.to_string().contains("1 MiB"));
|
||||||
|
assert_eq!(backend.show(reference.id.into()).unwrap(), observed);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn case_tool_rejects_legacy_states_and_unrecognized_input_fields() {
|
async fn case_tool_rejects_legacy_states_and_unrecognized_input_fields() {
|
||||||
let temp = TempDir::new().unwrap();
|
let temp = TempDir::new().unwrap();
|
||||||
|
|||||||
@@ -785,6 +785,12 @@ mod tests {
|
|||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
struct PendingRuntimeBackend {
|
struct PendingRuntimeBackend {
|
||||||
operations: std::sync::Mutex<Vec<worker_runtime::execution::WorkerLifecycleOperationId>>,
|
operations: std::sync::Mutex<Vec<worker_runtime::execution::WorkerLifecycleOperationId>>,
|
||||||
|
owners: std::sync::Mutex<
|
||||||
|
std::collections::HashMap<
|
||||||
|
worker_runtime::execution::WorkerLifecycleOperationId,
|
||||||
|
worker_runtime::identity::WorkerRef,
|
||||||
|
>,
|
||||||
|
>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl worker_runtime::execution::WorkerExecutionBackend for PendingRuntimeBackend {
|
impl worker_runtime::execution::WorkerExecutionBackend for PendingRuntimeBackend {
|
||||||
@@ -806,6 +812,21 @@ mod tests {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn restore_recovery_disposition(
|
||||||
|
&self,
|
||||||
|
operation_id: worker_runtime::execution::WorkerLifecycleOperationId,
|
||||||
|
worker_ref: &worker_runtime::identity::WorkerRef,
|
||||||
|
) -> Option<worker_runtime::execution::RestoreRecoveryDisposition> {
|
||||||
|
use worker_runtime::execution::RestoreRecoveryDisposition as D;
|
||||||
|
Some(
|
||||||
|
if self.owners.lock().unwrap().get(&operation_id) == Some(worker_ref) {
|
||||||
|
D::Continue
|
||||||
|
} else {
|
||||||
|
D::Absent
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
fn restore_worker(
|
fn restore_worker(
|
||||||
&self,
|
&self,
|
||||||
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||||
@@ -813,6 +834,10 @@ mod tests {
|
|||||||
use worker_runtime::execution::*;
|
use worker_runtime::execution::*;
|
||||||
let mut operations = self.operations.lock().unwrap();
|
let mut operations = self.operations.lock().unwrap();
|
||||||
operations.push(request.operation_id);
|
operations.push(request.operation_id);
|
||||||
|
self.owners
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.insert(request.operation_id, request.worker_ref.clone());
|
||||||
if operations.len() == 1 {
|
if operations.len() == 1 {
|
||||||
WorkerExecutionSpawnResult::ReconciliationRequired {
|
WorkerExecutionSpawnResult::ReconciliationRequired {
|
||||||
result: WorkerExecutionResult::errored(
|
result: WorkerExecutionResult::errored(
|
||||||
@@ -1119,39 +1144,45 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn rejected_resume_does_not_observe_or_open_retained_state() {
|
async fn settled_nonaccepted_resume_does_not_retry_observe_or_open_retained_state() {
|
||||||
let lifecycle = FakeBackendWorkerLifecycle {
|
for state in [
|
||||||
restore_response: restore_response(BackendWorkerRestoreState::Rejected),
|
BackendWorkerRestoreState::Rejected,
|
||||||
observation: session_observation(serde_json::json!({
|
BackendWorkerRestoreState::RolledBack,
|
||||||
"availability": "unavailable",
|
BackendWorkerRestoreState::Superseded,
|
||||||
"reason": "retention_missing",
|
] {
|
||||||
"message": "must not be opened"
|
let lifecycle = FakeBackendWorkerLifecycle {
|
||||||
})),
|
restore_response: restore_response(state),
|
||||||
calls: std::sync::Mutex::new(Vec::new()),
|
observation: session_observation(serde_json::json!({
|
||||||
};
|
"availability": "unavailable",
|
||||||
let target = client::BackendRuntimeTarget::new(
|
"reason": "retention_missing",
|
||||||
"http://127.0.0.1:3000",
|
"message": "must not be opened"
|
||||||
"workspace-a",
|
})),
|
||||||
"runtime-a",
|
calls: std::sync::Mutex::new(Vec::new()),
|
||||||
"worker-a",
|
};
|
||||||
);
|
let target = client::BackendRuntimeTarget::new(
|
||||||
|
"http://127.0.0.1:3000",
|
||||||
|
"workspace-a",
|
||||||
|
"runtime-a",
|
||||||
|
"worker-a",
|
||||||
|
);
|
||||||
|
|
||||||
let error = prepare_selected_worker(
|
let error = prepare_selected_worker(
|
||||||
&lifecycle,
|
&lifecycle,
|
||||||
target,
|
target,
|
||||||
BackendWorkerPickerIntent::Resume,
|
BackendWorkerPickerIntent::Resume,
|
||||||
Some(client::BackendWorkerRestoreRequest {
|
Some(client::BackendWorkerRestoreRequest {
|
||||||
expected_observation_token: "observed-worker-state".into(),
|
expected_observation_token: "observed-worker-state".into(),
|
||||||
request_id: "resume-operation".into(),
|
request_id: "resume-operation".into(),
|
||||||
}),
|
}),
|
||||||
|_| Ok(false),
|
|_| panic!("settled nonaccepted restore must not offer recovery retry"),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.unwrap_err()
|
.unwrap_err()
|
||||||
.to_string();
|
.to_string();
|
||||||
|
|
||||||
assert!(error.contains("Rejected"));
|
assert!(error.contains(&format!("{state:?}")));
|
||||||
assert_eq!(*lifecycle.calls.lock().unwrap(), ["restore"]);
|
assert_eq!(*lifecycle.calls.lock().unwrap(), ["restore"]);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -1176,6 +1207,7 @@ mod tests {
|
|||||||
for state in [
|
for state in [
|
||||||
BackendWorkerRestoreState::Rejected,
|
BackendWorkerRestoreState::Rejected,
|
||||||
BackendWorkerRestoreState::RolledBack,
|
BackendWorkerRestoreState::RolledBack,
|
||||||
|
BackendWorkerRestoreState::Superseded,
|
||||||
BackendWorkerRestoreState::ReconciliationRequired,
|
BackendWorkerRestoreState::ReconciliationRequired,
|
||||||
] {
|
] {
|
||||||
let mut response = restore_response(state);
|
let mut response = restore_response(state);
|
||||||
|
|||||||
@@ -19,15 +19,19 @@ required-features = ["ws-server", "fs-store"]
|
|||||||
[features]
|
[features]
|
||||||
default = ["ws-server", "fs-store"]
|
default = ["ws-server", "fs-store"]
|
||||||
fs-store = ["dep:fs4"]
|
fs-store = ["dep:fs4"]
|
||||||
|
# Offline fixture tooling only; never enabled by the product defaults.
|
||||||
|
test-support = ["fs-store", "ws-server"]
|
||||||
|
# Explicit process-test synchronization; independent of offline fixture tooling.
|
||||||
|
restore-checkpoints = ["fs-store"]
|
||||||
http-server = ["dep:axum", "dep:tower", "dep:reqwest"]
|
http-server = ["dep:axum", "dep:tower", "dep:reqwest"]
|
||||||
ws-server = ["http-server", "axum/ws", "dep:futures", "tokio/sync"]
|
ws-server = ["http-server", "axum/ws", "tokio/sync"]
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
agen.workspace = true
|
agen.workspace = true
|
||||||
async-trait.workspace = true
|
async-trait.workspace = true
|
||||||
base64.workspace = true
|
base64.workspace = true
|
||||||
axum = { workspace = true, optional = true }
|
axum = { workspace = true, optional = true }
|
||||||
futures = { workspace = true, optional = true }
|
futures.workspace = true
|
||||||
decodal.workspace = true
|
decodal.workspace = true
|
||||||
flow = { path = "../flow" }
|
flow = { path = "../flow" }
|
||||||
fs4 = { workspace = true, optional = true }
|
fs4 = { workspace = true, optional = true }
|
||||||
|
|||||||
@@ -8,3 +8,61 @@ Manages Worker creation, stopping, and Restore, execution connections within the
|
|||||||
- [`execution`] — execution backends and operation-specific results.
|
- [`execution`] — execution backends and operation-specific results.
|
||||||
- [`working_directory`] — repository materialization, session resources, and safe deletion.
|
- [`working_directory`] — repository materialization, session resources, and safe deletion.
|
||||||
- [`workspace_issuer`] — Workspace signature verification and replay prevention.
|
- [`workspace_issuer`] — Workspace signature verification and replay prevention.
|
||||||
|
|
||||||
|
## Offline stopped-Worker fixtures
|
||||||
|
|
||||||
|
The nondefault `test-support` feature exposes `test_support::StoppedWorkerTemplate`.
|
||||||
|
It is intentionally limited to current-schema, stopped, attachment-free artificial
|
||||||
|
conversations with one Segment, text history, and no pending execution/activation.
|
||||||
|
It does not add a product endpoint or change create/restore authorization.
|
||||||
|
|
||||||
|
1. With the source Runtime offline, call `StoppedWorkerTemplate::extract(worker_dir)`.
|
||||||
|
The source must have its existing Runtime owner lock and identity. Extraction is
|
||||||
|
read-only; the returned serde template excludes source auth, saved Manifest,
|
||||||
|
request/config references, run directories, and Runtime operation history.
|
||||||
|
2. Initialize **new** Runtime identity and Workspace trust in a fresh destination
|
||||||
|
using normal setup. Preserve the template's Runtime and Workspace IDs.
|
||||||
|
3. Call `template.install(&options, request, &resolved_manifest)` before starting
|
||||||
|
the Runtime. Supply a fresh typed `CreateWorkerRequest` (including config/profile
|
||||||
|
references and trusted memory settings) and `manifest::WorkerManifest` resolved
|
||||||
|
for that environment. Its Worker name must equal `template.worker_name()`.
|
||||||
|
4. Start normally and use the ordinary explicit restore operation. Installation
|
||||||
|
itself creates no execution and leaves the Worker stopped with a fresh guard.
|
||||||
|
|
||||||
|
The template preserves the original Session manifest/JSONL bytes, including stable
|
||||||
|
entry IDs, timestamps, and settled submission receipts. It rejects resumable queues
|
||||||
|
and Feature invocation history. A saved system prompt is required for restore;
|
||||||
|
checkpoint receipts must match the Worker's typed receipt schema. Preserved records
|
||||||
|
must use canonical writer field shapes: unknown/duplicate fields (including nested
|
||||||
|
fields), discarded optional fields, and opaque Feature history payloads are rejected,
|
||||||
|
not stripped or rewritten. Portability/secret-marker checks do not replace
|
||||||
|
review of the supplied artificial conversation or its independently verified hashes.
|
||||||
|
The installer refuses any destination content other than initialized `auth.toml`;
|
||||||
|
interrupted publication is fail-closed, not an overwrite/recovery facility.
|
||||||
|
|
||||||
|
## Owned-process Restore checkpoints
|
||||||
|
|
||||||
|
The independent, nondefault `restore-checkpoints` feature exists only for explicit
|
||||||
|
process tests in the shared `tests/e2e` harness. Offline `test-support` does not enable
|
||||||
|
it. Normal binaries reject `--test-restore-checkpoint`; no environment variable or
|
||||||
|
HTTP endpoint activates this facility. Do not deploy a checkpoint-enabled binary
|
||||||
|
as a replacement for the default product.
|
||||||
|
|
||||||
|
A test-enabled Runtime accepts `--test-restore-checkpoint <config.json>` with a
|
||||||
|
bounded regular JSON file selecting a loopback socket, Worker ID, one stage and a
|
||||||
|
diagnostic nonce. The nonce is only a handshake label, never execution identity,
|
||||||
|
request authority or routing state. The one-shot stages are real prepared candidate
|
||||||
|
with `NotCommitted`, and durable `Committed` before activation/terminal receipt.
|
||||||
|
Before acknowledging, the existing bounded canonical reader validates the exact
|
||||||
|
Worker/operation/decision, syncs its file/directories and requires no saved receipt.
|
||||||
|
No pending JSON is fabricated and no lifecycle work is skipped.
|
||||||
|
|
||||||
|
The Runtime sends bounded newline-delimited JSON evidence, then waits at most eight
|
||||||
|
seconds for byte `R`. EOF, invalid reply, unavailable peer or deadline releases the
|
||||||
|
wait into the original operation. The Runtime-state mutex is not held across that
|
||||||
|
wait; existing per-Worker ownership still covers the actual candidate and cleanup.
|
||||||
|
The harness verifies the ack, kills/reaps only its owned child and inspects the
|
||||||
|
canonical aggregate offline before restarting the same roots/identity without the
|
||||||
|
checkpoint flag or another fixture install. These tests are a separate cfg/binary
|
||||||
|
lane from featureless product regression; they do not certify arbitrary OS failure,
|
||||||
|
external power loss or a real model call.
|
||||||
|
|||||||
@@ -215,6 +215,38 @@ impl WorkerExecutionResult {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Private one-shot synchronization at real publication boundaries. Disconnect
|
||||||
|
// releases the waiter during unwinding; timeout bounds a forgotten test owner.
|
||||||
|
#[cfg(test)]
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub(crate) struct RestoreTestGate<T> {
|
||||||
|
pub(crate) entered: std::sync::mpsc::Sender<T>,
|
||||||
|
pub(crate) release: std::sync::mpsc::Receiver<()>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
impl<T> RestoreTestGate<T> {
|
||||||
|
pub(crate) fn wait(self, evidence: T) {
|
||||||
|
let _ = self.entered.send(evidence);
|
||||||
|
assert!(
|
||||||
|
!matches!(
|
||||||
|
self.release
|
||||||
|
.recv_timeout(std::time::Duration::from_secs(30)),
|
||||||
|
Err(std::sync::mpsc::RecvTimeoutError::Timeout)
|
||||||
|
),
|
||||||
|
"publication test gate timed out"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(all(test, feature = "ws-server"))]
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub(crate) struct RestoreFlushFault {
|
||||||
|
pub(crate) remaining_successes: usize,
|
||||||
|
pub(crate) gate: RestoreTestGate<protocol::Event>,
|
||||||
|
pub(crate) restored: std::sync::mpsc::Sender<Vec<protocol::Event>>,
|
||||||
|
}
|
||||||
|
|
||||||
/// Runtime hooks available to an execution backend for one Worker.
|
/// Runtime hooks available to an execution backend for one Worker.
|
||||||
#[cfg(feature = "ws-server")]
|
#[cfg(feature = "ws-server")]
|
||||||
type WorkerObservationPublisher = Arc<
|
type WorkerObservationPublisher = Arc<
|
||||||
@@ -242,10 +274,15 @@ enum WorkerExecutionPublicationState {
|
|||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct WorkerExecutionContext {
|
pub struct WorkerExecutionContext {
|
||||||
worker_ref: WorkerRef,
|
worker_ref: WorkerRef,
|
||||||
|
restore_progress: Option<Arc<dyn Fn(WorkerLifecycleOperationId) + Send + Sync>>,
|
||||||
|
restore_repair_authorizer:
|
||||||
|
Option<Arc<dyn Fn(WorkerLifecycleOperationId) -> Result<(), RuntimeError> + Send + Sync>>,
|
||||||
#[cfg(feature = "ws-server")]
|
#[cfg(feature = "ws-server")]
|
||||||
observation_publisher: WorkerObservationPublisher,
|
observation_publisher: WorkerObservationPublisher,
|
||||||
#[cfg(feature = "ws-server")]
|
#[cfg(feature = "ws-server")]
|
||||||
publication_state: Arc<Mutex<WorkerExecutionPublicationState>>,
|
publication_state: Arc<Mutex<WorkerExecutionPublicationState>>,
|
||||||
|
#[cfg(all(test, feature = "ws-server"))]
|
||||||
|
flush_fault: Arc<Mutex<Option<RestoreFlushFault>>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl WorkerExecutionContext {
|
impl WorkerExecutionContext {
|
||||||
@@ -256,7 +293,11 @@ impl WorkerExecutionContext {
|
|||||||
) -> Self {
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
worker_ref,
|
worker_ref,
|
||||||
|
restore_progress: None,
|
||||||
|
restore_repair_authorizer: None,
|
||||||
observation_publisher,
|
observation_publisher,
|
||||||
|
#[cfg(test)]
|
||||||
|
flush_fault: Arc::default(),
|
||||||
publication_state: Arc::new(Mutex::new(WorkerExecutionPublicationState::Active)),
|
publication_state: Arc::new(Mutex::new(WorkerExecutionPublicationState::Active)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -268,7 +309,11 @@ impl WorkerExecutionContext {
|
|||||||
) -> Self {
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
worker_ref,
|
worker_ref,
|
||||||
|
restore_progress: None,
|
||||||
|
restore_repair_authorizer: None,
|
||||||
observation_publisher,
|
observation_publisher,
|
||||||
|
#[cfg(test)]
|
||||||
|
flush_fault: Arc::default(),
|
||||||
publication_state: Arc::new(Mutex::new(WorkerExecutionPublicationState::Pending(
|
publication_state: Arc::new(Mutex::new(WorkerExecutionPublicationState::Pending(
|
||||||
VecDeque::new(),
|
VecDeque::new(),
|
||||||
))),
|
))),
|
||||||
@@ -277,12 +322,56 @@ impl WorkerExecutionContext {
|
|||||||
|
|
||||||
#[cfg(not(feature = "ws-server"))]
|
#[cfg(not(feature = "ws-server"))]
|
||||||
pub(crate) fn new(worker_ref: WorkerRef) -> Self {
|
pub(crate) fn new(worker_ref: WorkerRef) -> Self {
|
||||||
Self { worker_ref }
|
Self {
|
||||||
|
worker_ref,
|
||||||
|
restore_progress: None,
|
||||||
|
restore_repair_authorizer: None,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(not(feature = "ws-server"))]
|
#[cfg(not(feature = "ws-server"))]
|
||||||
pub(crate) fn candidate(worker_ref: WorkerRef) -> Self {
|
pub(crate) fn candidate(worker_ref: WorkerRef) -> Self {
|
||||||
Self { worker_ref }
|
Self {
|
||||||
|
worker_ref,
|
||||||
|
restore_progress: None,
|
||||||
|
restore_repair_authorizer: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn with_restore_repair_authorizer(
|
||||||
|
mut self,
|
||||||
|
authorizer: Arc<
|
||||||
|
dyn Fn(WorkerLifecycleOperationId) -> Result<(), RuntimeError> + Send + Sync,
|
||||||
|
>,
|
||||||
|
) -> Self {
|
||||||
|
self.restore_repair_authorizer = Some(authorizer);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn authorize_restore_source_repair(
|
||||||
|
&self,
|
||||||
|
operation: WorkerLifecycleOperationId,
|
||||||
|
) -> Result<(), RuntimeError> {
|
||||||
|
match &self.restore_repair_authorizer {
|
||||||
|
Some(authorizer) => authorizer(operation),
|
||||||
|
// Direct factory/component users own their source repair separately;
|
||||||
|
// only Runtime contexts carry a Runtime aggregate authority callback.
|
||||||
|
None => Ok(()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn with_restore_progress(
|
||||||
|
mut self,
|
||||||
|
progress: Arc<dyn Fn(WorkerLifecycleOperationId) + Send + Sync>,
|
||||||
|
) -> Self {
|
||||||
|
self.restore_progress = Some(progress);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn restore_progressed(&self, operation_id: WorkerLifecycleOperationId) {
|
||||||
|
if let Some(progress) = &self.restore_progress {
|
||||||
|
progress(operation_id);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn worker_ref(&self) -> &WorkerRef {
|
pub fn worker_ref(&self) -> &WorkerRef {
|
||||||
@@ -335,20 +424,65 @@ impl WorkerExecutionContext {
|
|||||||
let Some(event) = event else {
|
let Some(event) = event else {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
};
|
};
|
||||||
if let Err(error) = (self.observation_publisher)(self.worker_ref.clone(), event.clone())
|
#[cfg(test)]
|
||||||
{
|
let fault = {
|
||||||
|
let mut fault = self.flush_fault.lock().unwrap();
|
||||||
|
if fault.as_ref().is_some_and(|f| f.remaining_successes == 0) {
|
||||||
|
fault.take()
|
||||||
|
} else {
|
||||||
|
if let Some(fault) = fault.as_mut() {
|
||||||
|
fault.remaining_successes -= 1;
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
};
|
||||||
|
#[cfg(test)]
|
||||||
|
let mut restored = None;
|
||||||
|
#[cfg(test)]
|
||||||
|
let publication = if let Some(fault) = fault {
|
||||||
|
restored = Some(fault.restored);
|
||||||
|
fault.gate.wait(event.clone());
|
||||||
|
Err(RuntimeError::InvalidRequest(
|
||||||
|
"injected buffered publication failure".into(),
|
||||||
|
))
|
||||||
|
} else {
|
||||||
|
(self.observation_publisher)(self.worker_ref.clone(), event.clone())
|
||||||
|
};
|
||||||
|
#[cfg(not(test))]
|
||||||
|
let publication = (self.observation_publisher)(self.worker_ref.clone(), event.clone());
|
||||||
|
if let Err(error) = publication {
|
||||||
let mut state = self
|
let mut state = self
|
||||||
.publication_state
|
.publication_state
|
||||||
.lock()
|
.lock()
|
||||||
.map_err(|_| RuntimeError::StatePoisoned)?;
|
.map_err(|_| RuntimeError::StatePoisoned)?;
|
||||||
if let WorkerExecutionPublicationState::Activating(events) = &mut *state {
|
if let WorkerExecutionPublicationState::Activating(events) = &mut *state {
|
||||||
events.push_front(event);
|
events.push_front(event);
|
||||||
|
#[cfg(test)]
|
||||||
|
if let Some(restored) = restored {
|
||||||
|
let _ = restored.send(events.iter().cloned().collect());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return Err(error);
|
return Err(error);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(all(test, feature = "ws-server"))]
|
||||||
|
pub(crate) fn test_fail_flush(&self, fault: RestoreFlushFault) {
|
||||||
|
assert!(self.flush_fault.lock().unwrap().replace(fault).is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(all(test, feature = "ws-server"))]
|
||||||
|
pub(crate) fn test_buffered_events(&self) -> Vec<protocol::Event> {
|
||||||
|
match &*self.publication_state.lock().unwrap() {
|
||||||
|
WorkerExecutionPublicationState::Pending(events)
|
||||||
|
| WorkerExecutionPublicationState::Activating(events) => {
|
||||||
|
events.iter().cloned().collect()
|
||||||
|
}
|
||||||
|
_ => Vec::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(not(feature = "ws-server"))]
|
#[cfg(not(feature = "ws-server"))]
|
||||||
pub(crate) fn activate_candidate(&self) -> Result<(), RuntimeError> {
|
pub(crate) fn activate_candidate(&self) -> Result<(), RuntimeError> {
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -424,6 +558,7 @@ pub struct WorkerExecutionSpawnRequest {
|
|||||||
/// Request passed to a [`WorkerExecutionBackend`] when restoring a persisted Worker.
|
/// Request passed to a [`WorkerExecutionBackend`] when restoring a persisted Worker.
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug)]
|
||||||
pub struct WorkerExecutionRestoreRequest {
|
pub struct WorkerExecutionRestoreRequest {
|
||||||
|
pub source_repair_possible: bool,
|
||||||
pub operation_id: WorkerLifecycleOperationId,
|
pub operation_id: WorkerLifecycleOperationId,
|
||||||
pub worker_ref: WorkerRef,
|
pub worker_ref: WorkerRef,
|
||||||
pub request: crate::catalog::CreateWorkerRequest,
|
pub request: crate::catalog::CreateWorkerRequest,
|
||||||
@@ -565,6 +700,19 @@ impl fmt::Debug for WorkerProtocolTransport {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Derived resource disposition for an existing restore owner. Not durable
|
||||||
|
/// authority: Runtime must reread its decision before choosing any action.
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
|
pub enum RestoreRecoveryDisposition {
|
||||||
|
/// No live or pending owner for this exact operation. Not cleanup evidence.
|
||||||
|
Absent,
|
||||||
|
InFlight,
|
||||||
|
Continue,
|
||||||
|
Discard,
|
||||||
|
/// A retained cleanup-only owner; never reconstruct or activate it.
|
||||||
|
Reclaim,
|
||||||
|
}
|
||||||
|
|
||||||
/// The connection selects this backend; WorkerRef identifies an execution,
|
/// The connection selects this backend; WorkerRef identifies an execution,
|
||||||
/// never a capability or authorization token.
|
/// never a capability or authorization token.
|
||||||
pub trait WorkerExecutionBackend: Send + Sync + 'static {
|
pub trait WorkerExecutionBackend: Send + Sync + 'static {
|
||||||
@@ -633,6 +781,33 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Completed resource-owner evidence retained while Runtime settlement may
|
||||||
|
/// still need retry. This is not a new request or execution authorization.
|
||||||
|
fn completed_restore_failure(
|
||||||
|
&self,
|
||||||
|
_operation_id: WorkerLifecycleOperationId,
|
||||||
|
_worker_ref: &WorkerRef,
|
||||||
|
) -> Option<WorkerExecutionSpawnResult> {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Opt in to Runtime-owned autonomous restore continuation. Pending tasks
|
||||||
|
/// signal completion through the original execution context; callbacks must
|
||||||
|
/// not wait for Runtime locks. Independent of resource disposition support.
|
||||||
|
fn supports_restore_continuation(&self) -> bool {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Describe exact operation resource ownership for warm/cold recovery
|
||||||
|
/// dispatch. This does not opt in to callbacks or autonomous continuation.
|
||||||
|
fn restore_recovery_disposition(
|
||||||
|
&self,
|
||||||
|
_operation_id: WorkerLifecycleOperationId,
|
||||||
|
_worker_ref: &WorkerRef,
|
||||||
|
) -> Option<RestoreRecoveryDisposition> {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
fn reconcile_restore(
|
fn reconcile_restore(
|
||||||
&self,
|
&self,
|
||||||
request: WorkerExecutionRestoreRequest,
|
request: WorkerExecutionRestoreRequest,
|
||||||
@@ -640,6 +815,42 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static {
|
|||||||
self.restore_worker(request)
|
self.restore_worker(request)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Reclaim an uncommitted operation after restart without preparing,
|
||||||
|
/// repairing, or activating a replacement candidate. Runtime must establish
|
||||||
|
/// the durable uncommitted decision before calling this seam. Unsupported
|
||||||
|
/// cleanup is uncertainty, never a successful rollback.
|
||||||
|
fn reclaim_uncommitted_restore(
|
||||||
|
&self,
|
||||||
|
_request: WorkerExecutionRestoreRequest,
|
||||||
|
) -> WorkerExecutionSpawnResult {
|
||||||
|
WorkerExecutionSpawnResult::ReconciliationRequired {
|
||||||
|
result: WorkerExecutionResult::unsupported(
|
||||||
|
WorkerExecutionOperation::Restore,
|
||||||
|
"backend cannot reclaim an uncommitted restore after restart",
|
||||||
|
),
|
||||||
|
worker_state: None,
|
||||||
|
workdir_attachments: Vec::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Discard only the exact unpublished operation after Runtime has proved
|
||||||
|
/// its durable decision uncommitted. This is not Stop and must not affect a
|
||||||
|
/// connected execution. RolledBack requires confirmed cleanup and no repair.
|
||||||
|
fn discard_uncommitted_restore(
|
||||||
|
&self,
|
||||||
|
_operation_id: WorkerLifecycleOperationId,
|
||||||
|
_worker_ref: &WorkerRef,
|
||||||
|
) -> WorkerExecutionSpawnResult {
|
||||||
|
WorkerExecutionSpawnResult::ReconciliationRequired {
|
||||||
|
result: WorkerExecutionResult::unsupported(
|
||||||
|
WorkerExecutionOperation::Restore,
|
||||||
|
"backend cannot confirm unpublished restore cleanup",
|
||||||
|
),
|
||||||
|
worker_state: None,
|
||||||
|
workdir_attachments: Vec::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn activate_restored_worker(
|
fn activate_restored_worker(
|
||||||
&self,
|
&self,
|
||||||
_operation_id: WorkerLifecycleOperationId,
|
_operation_id: WorkerLifecycleOperationId,
|
||||||
@@ -901,6 +1112,28 @@ impl WorkerExecutionBackendRef {
|
|||||||
self.backend.preflight_restore(request)
|
self.backend.preflight_restore(request)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn completed_restore_failure(
|
||||||
|
&self,
|
||||||
|
operation_id: WorkerLifecycleOperationId,
|
||||||
|
worker_ref: &WorkerRef,
|
||||||
|
) -> Option<WorkerExecutionSpawnResult> {
|
||||||
|
self.backend
|
||||||
|
.completed_restore_failure(operation_id, worker_ref)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn supports_restore_continuation(&self) -> bool {
|
||||||
|
self.backend.supports_restore_continuation()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn restore_recovery_disposition(
|
||||||
|
&self,
|
||||||
|
operation_id: WorkerLifecycleOperationId,
|
||||||
|
worker_ref: &WorkerRef,
|
||||||
|
) -> Option<RestoreRecoveryDisposition> {
|
||||||
|
self.backend
|
||||||
|
.restore_recovery_disposition(operation_id, worker_ref)
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn reconcile_restore(
|
pub(crate) fn reconcile_restore(
|
||||||
&self,
|
&self,
|
||||||
request: WorkerExecutionRestoreRequest,
|
request: WorkerExecutionRestoreRequest,
|
||||||
@@ -908,6 +1141,22 @@ impl WorkerExecutionBackendRef {
|
|||||||
self.backend.reconcile_restore(request)
|
self.backend.reconcile_restore(request)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn reclaim_uncommitted_restore(
|
||||||
|
&self,
|
||||||
|
request: WorkerExecutionRestoreRequest,
|
||||||
|
) -> WorkerExecutionSpawnResult {
|
||||||
|
self.backend.reclaim_uncommitted_restore(request)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn discard_uncommitted_restore(
|
||||||
|
&self,
|
||||||
|
operation_id: WorkerLifecycleOperationId,
|
||||||
|
worker_ref: &WorkerRef,
|
||||||
|
) -> WorkerExecutionSpawnResult {
|
||||||
|
self.backend
|
||||||
|
.discard_uncommitted_restore(operation_id, worker_ref)
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn activate_restored_worker(
|
pub(crate) fn activate_restored_worker(
|
||||||
&self,
|
&self,
|
||||||
operation_id: WorkerLifecycleOperationId,
|
operation_id: WorkerLifecycleOperationId,
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -25,6 +25,7 @@ use crate::interaction::{WorkerInput, WorkerInteractionAck};
|
|||||||
use crate::management::{RuntimeSummary, WorkerDeleteResult};
|
use crate::management::{RuntimeSummary, WorkerDeleteResult};
|
||||||
#[cfg(feature = "ws-server")]
|
#[cfg(feature = "ws-server")]
|
||||||
use crate::observation::WorkerObservationCursor;
|
use crate::observation::WorkerObservationCursor;
|
||||||
|
#[cfg(feature = "fs-store")]
|
||||||
use crate::retention::{
|
use crate::retention::{
|
||||||
WorkerRetentionExecutionRequest, WorkerRetentionExecutionResult, WorkerRetentionInventory,
|
WorkerRetentionExecutionRequest, WorkerRetentionExecutionResult, WorkerRetentionInventory,
|
||||||
};
|
};
|
||||||
@@ -1568,6 +1569,7 @@ fn protocol_error_event(message: impl Into<String>) -> protocol::Event {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "fs-store")]
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
async fn worker_retention_inventory(
|
async fn worker_retention_inventory(
|
||||||
State(state): State<RuntimeHttpState>,
|
State(state): State<RuntimeHttpState>,
|
||||||
@@ -1589,6 +1591,7 @@ async fn worker_retention_inventory(
|
|||||||
.map_err(RuntimeHttpRestError::runtime)
|
.map_err(RuntimeHttpRestError::runtime)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "fs-store")]
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
async fn execute_worker_retention(
|
async fn execute_worker_retention(
|
||||||
State(state): State<RuntimeHttpState>,
|
State(state): State<RuntimeHttpState>,
|
||||||
@@ -2654,6 +2657,87 @@ mod tests {
|
|||||||
.unwrap()
|
.unwrap()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Generated management routes and real Workspace capability middleware stay
|
||||||
|
// connected even when this process has no filesystem retention provider.
|
||||||
|
#[cfg(not(feature = "fs-store"))]
|
||||||
|
#[tokio::test]
|
||||||
|
async fn memory_retention_routes_preserve_auth_and_return_typed_unavailable() {
|
||||||
|
let identity = RuntimeIdentityMaterial::generate("workspace-key").unwrap();
|
||||||
|
let trust = workspace_trust(&identity);
|
||||||
|
let runtime = Runtime::new_memory();
|
||||||
|
let app = runtime_http_router_with_workspace_auth(
|
||||||
|
runtime,
|
||||||
|
None,
|
||||||
|
workspace_test_auth(vec![trust.clone()]),
|
||||||
|
);
|
||||||
|
let worker_id = WorkerId::now_v7();
|
||||||
|
let body = runtime_api::WorkerRetentionExecutionRequest {
|
||||||
|
operation_id: "memory-retention".into(),
|
||||||
|
input_fingerprint: "input".into(),
|
||||||
|
archive_id: None,
|
||||||
|
workspace_id: "workspace-a".into(),
|
||||||
|
source_runtime_id: "runtime-test".into(),
|
||||||
|
worker_id,
|
||||||
|
source_created_at: "2026-01-01T00:00:00Z".into(),
|
||||||
|
removed_at: "2026-01-02T00:00:00Z".into(),
|
||||||
|
effective_profile: None,
|
||||||
|
retention_class: None,
|
||||||
|
policy_id: "policy".into(),
|
||||||
|
session_disposition: runtime_api::SessionDisposition::Purge,
|
||||||
|
diagnostics_disposition: runtime_api::DiagnosticsDisposition::Purge,
|
||||||
|
};
|
||||||
|
for (method, path, body) in [
|
||||||
|
(
|
||||||
|
"GET",
|
||||||
|
format!("/v1/workers/{worker_id}/retention/inventory"),
|
||||||
|
vec![],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"POST",
|
||||||
|
format!("/v1/workers/{worker_id}/retention/execute"),
|
||||||
|
serde_json::to_vec(&body).unwrap(),
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
let unsigned = app
|
||||||
|
.clone()
|
||||||
|
.oneshot(
|
||||||
|
Request::builder()
|
||||||
|
.method(method)
|
||||||
|
.uri(&path)
|
||||||
|
.header(header::CONTENT_TYPE, "application/json")
|
||||||
|
.body(Body::from(body.clone()))
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(unsigned.status(), StatusCode::UNAUTHORIZED);
|
||||||
|
let claims = signed_claims(&trust, method, &path, &body);
|
||||||
|
let response =
|
||||||
|
signed_request(app.clone(), &identity, &claims, body, "workspace-a").await;
|
||||||
|
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||||
|
let error: RuntimeHttpErrorResponse = read_json(response).await;
|
||||||
|
assert_eq!(error.error.code, "worker_retention_unavailable");
|
||||||
|
}
|
||||||
|
for (mut value, expected_status, expected_code) in [
|
||||||
|
(body.clone(), StatusCode::NOT_FOUND, "worker_not_found"),
|
||||||
|
(body.clone(), StatusCode::BAD_REQUEST, "worker_id_mismatch"),
|
||||||
|
] {
|
||||||
|
if expected_status == StatusCode::NOT_FOUND {
|
||||||
|
value.workspace_id = "workspace-b".into();
|
||||||
|
} else {
|
||||||
|
value.worker_id = WorkerId::now_v7();
|
||||||
|
}
|
||||||
|
let path = format!("/v1/workers/{worker_id}/retention/execute");
|
||||||
|
let bytes = serde_json::to_vec(&value).unwrap();
|
||||||
|
let claims = signed_claims(&trust, "POST", &path, &bytes);
|
||||||
|
let response =
|
||||||
|
signed_request(app.clone(), &identity, &claims, bytes, "workspace-a").await;
|
||||||
|
assert_eq!(response.status(), expected_status);
|
||||||
|
let error: RuntimeHttpErrorResponse = read_json(response).await;
|
||||||
|
assert_eq!(error.error.code, expected_code);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn ordinary_signed_ping_succeeds_without_handshake_and_rejects_replay() {
|
async fn ordinary_signed_ping_succeeds_without_handshake_and_rejects_replay() {
|
||||||
let identity = RuntimeIdentityMaterial::generate("workspace-key").unwrap();
|
let identity = RuntimeIdentityMaterial::generate("workspace-key").unwrap();
|
||||||
|
|||||||
@@ -719,6 +719,9 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
|
|||||||
server_api::WorkerRestoreState::RolledBack => {
|
server_api::WorkerRestoreState::RolledBack => {
|
||||||
runtime_api::WorkerRestoreState::RolledBack
|
runtime_api::WorkerRestoreState::RolledBack
|
||||||
}
|
}
|
||||||
|
server_api::WorkerRestoreState::Superseded => {
|
||||||
|
runtime_api::WorkerRestoreState::Superseded
|
||||||
|
}
|
||||||
server_api::WorkerRestoreState::ReconciliationRequired => {
|
server_api::WorkerRestoreState::ReconciliationRequired => {
|
||||||
runtime_api::WorkerRestoreState::ReconciliationRequired
|
runtime_api::WorkerRestoreState::ReconciliationRequired
|
||||||
}
|
}
|
||||||
@@ -769,6 +772,9 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
|
|||||||
server_api::WorkerRestoreState::RolledBack => {
|
server_api::WorkerRestoreState::RolledBack => {
|
||||||
runtime_api::WorkerRestoreState::RolledBack
|
runtime_api::WorkerRestoreState::RolledBack
|
||||||
}
|
}
|
||||||
|
server_api::WorkerRestoreState::Superseded => {
|
||||||
|
runtime_api::WorkerRestoreState::Superseded
|
||||||
|
}
|
||||||
server_api::WorkerRestoreState::ReconciliationRequired => {
|
server_api::WorkerRestoreState::ReconciliationRequired => {
|
||||||
runtime_api::WorkerRestoreState::ReconciliationRequired
|
runtime_api::WorkerRestoreState::ReconciliationRequired
|
||||||
}
|
}
|
||||||
@@ -881,13 +887,25 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
|
|||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
let worker_ref = worker_ref_for(&self.state.runtime, worker_id).map_err(api_error)?;
|
let worker_ref = worker_ref_for(&self.state.runtime, worker_id).map_err(api_error)?;
|
||||||
let inventory = self
|
#[cfg(feature = "fs-store")]
|
||||||
.state
|
{
|
||||||
.runtime
|
let inventory = self
|
||||||
.worker_retention_inventory(&scope.workspace_id, &worker_ref)
|
.state
|
||||||
.map_err(RuntimeHttpRestError::runtime)
|
.runtime
|
||||||
.map_err(api_error)?;
|
.worker_retention_inventory(&scope.workspace_id, &worker_ref)
|
||||||
response(inventory)
|
.map_err(RuntimeHttpRestError::runtime)
|
||||||
|
.map_err(api_error)?;
|
||||||
|
response(inventory)
|
||||||
|
}
|
||||||
|
#[cfg(not(feature = "fs-store"))]
|
||||||
|
{
|
||||||
|
let _ = (scope, worker_ref);
|
||||||
|
Err(runtime_api::RuntimeApiError::new(
|
||||||
|
503,
|
||||||
|
"worker_retention_unavailable",
|
||||||
|
"Worker retention requires filesystem storage",
|
||||||
|
))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn execute_retention(
|
async fn execute_retention(
|
||||||
@@ -905,27 +923,36 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
|
|||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
let worker_ref = worker_ref_for(&self.state.runtime, worker_id).map_err(api_error)?;
|
let worker_ref = worker_ref_for(&self.state.runtime, worker_id).map_err(api_error)?;
|
||||||
let request: WorkerRetentionExecutionRequest = request(value)?;
|
if value.workspace_id != scope.workspace_id {
|
||||||
if request.workspace_id != scope.workspace_id {
|
|
||||||
return Err(runtime_api::RuntimeApiError::new(
|
return Err(runtime_api::RuntimeApiError::new(
|
||||||
StatusCode::NOT_FOUND.as_u16(),
|
StatusCode::NOT_FOUND.as_u16(),
|
||||||
"worker_not_found",
|
"worker_not_found",
|
||||||
"worker does not belong to the authenticated Workspace",
|
"worker does not belong to the authenticated Workspace",
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
if request.worker_id != worker_ref.worker_id {
|
if value.worker_id != worker_ref.worker_id {
|
||||||
return Err(runtime_api::RuntimeApiError::new(
|
return Err(runtime_api::RuntimeApiError::new(
|
||||||
StatusCode::BAD_REQUEST.as_u16(),
|
StatusCode::BAD_REQUEST.as_u16(),
|
||||||
"worker_id_mismatch",
|
"worker_id_mismatch",
|
||||||
"Worker retention request worker_id does not match route worker_id",
|
"Worker retention request worker_id does not match route worker_id",
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
let result = self
|
#[cfg(feature = "fs-store")]
|
||||||
.state
|
{
|
||||||
.runtime
|
let request: WorkerRetentionExecutionRequest = request(value)?;
|
||||||
.execute_worker_retention(&request)
|
let result = self
|
||||||
.map_err(RuntimeHttpRestError::runtime)
|
.state
|
||||||
.map_err(api_error)?;
|
.runtime
|
||||||
response(result)
|
.execute_worker_retention(&request)
|
||||||
|
.map_err(RuntimeHttpRestError::runtime)
|
||||||
|
.map_err(api_error)?;
|
||||||
|
response(result)
|
||||||
|
}
|
||||||
|
#[cfg(not(feature = "fs-store"))]
|
||||||
|
Err(runtime_api::RuntimeApiError::new(
|
||||||
|
503,
|
||||||
|
"worker_retention_unavailable",
|
||||||
|
"Worker retention requires filesystem storage",
|
||||||
|
))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,6 +16,11 @@ pub mod management;
|
|||||||
pub mod observation;
|
pub mod observation;
|
||||||
pub mod profile_archive;
|
pub mod profile_archive;
|
||||||
pub mod resource;
|
pub mod resource;
|
||||||
|
#[cfg(feature = "restore-checkpoints")]
|
||||||
|
pub mod restore_checkpoints;
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "../../../tests/support/restore_environment.rs"]
|
||||||
|
mod restore_test_environment;
|
||||||
#[cfg(feature = "fs-store")]
|
#[cfg(feature = "fs-store")]
|
||||||
pub mod retention;
|
pub mod retention;
|
||||||
mod runtime;
|
mod runtime;
|
||||||
@@ -32,6 +37,8 @@ pub mod workspace_issuer;
|
|||||||
#[cfg(feature = "http-server")]
|
#[cfg(feature = "http-server")]
|
||||||
pub mod workspace_request;
|
pub mod workspace_request;
|
||||||
|
|
||||||
|
#[cfg(feature = "test-support")]
|
||||||
|
pub use fs_store::test_support;
|
||||||
#[cfg(feature = "fs-store")]
|
#[cfg(feature = "fs-store")]
|
||||||
pub use fs_store::{FsRuntimeStore, FsRuntimeStoreOptions};
|
pub use fs_store::{FsRuntimeStore, FsRuntimeStoreOptions};
|
||||||
pub use management::RuntimeOptions;
|
pub use management::RuntimeOptions;
|
||||||
|
|||||||
@@ -117,6 +117,12 @@ fn run() -> Result<(), ProcessError> {
|
|||||||
let listener = tokio::net::TcpListener::bind(config.http.bind_addr).await?;
|
let listener = tokio::net::TcpListener::bind(config.http.bind_addr).await?;
|
||||||
let local_addr = listener.local_addr()?;
|
let local_addr = listener.local_addr()?;
|
||||||
let worker_runtime = build_runtime(&config)?;
|
let worker_runtime = build_runtime(&config)?;
|
||||||
|
#[cfg(feature = "restore-checkpoints")]
|
||||||
|
if let Some(checkpoint) = &config.test_restore_checkpoint {
|
||||||
|
worker_runtime
|
||||||
|
.install_test_restore_checkpoint(checkpoint.clone())
|
||||||
|
.map_err(ProcessError::Runtime)?;
|
||||||
|
}
|
||||||
eprintln!(
|
eprintln!(
|
||||||
"yoi-runtime listening on {local_addr}; intended client is a trusted backend/proxy, not a browser"
|
"yoi-runtime listening on {local_addr}; intended client is a trusted backend/proxy, not a browser"
|
||||||
);
|
);
|
||||||
@@ -391,6 +397,21 @@ where
|
|||||||
let (flag, inline_value) = split_flag_value(arg)?;
|
let (flag, inline_value) = split_flag_value(arg)?;
|
||||||
match flag.as_str() {
|
match flag.as_str() {
|
||||||
"--help" | "-h" => return Ok(None),
|
"--help" | "-h" => return Ok(None),
|
||||||
|
#[cfg(feature = "restore-checkpoints")]
|
||||||
|
"--test-restore-checkpoint" => {
|
||||||
|
let path = take_value(&flag, inline_value, &mut args)?;
|
||||||
|
if config.test_restore_checkpoint.is_some() {
|
||||||
|
return Err(ProcessError::usage(
|
||||||
|
"--test-restore-checkpoint may only be specified once".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
config.test_restore_checkpoint = Some(
|
||||||
|
worker_runtime::restore_checkpoints::RestoreCheckpointConfig::from_path(
|
||||||
|
Path::new(&path),
|
||||||
|
)
|
||||||
|
.map_err(ProcessError::Runtime)?,
|
||||||
|
);
|
||||||
|
}
|
||||||
"--bind" => {
|
"--bind" => {
|
||||||
let value = take_value(&flag, inline_value, &mut args)?;
|
let value = take_value(&flag, inline_value, &mut args)?;
|
||||||
config.http.bind_addr = value.parse::<SocketAddr>().map_err(|error| {
|
config.http.bind_addr = value.parse::<SocketAddr>().map_err(|error| {
|
||||||
@@ -465,6 +486,12 @@ where
|
|||||||
}
|
}
|
||||||
|
|
||||||
apply_store_selection(&mut config);
|
apply_store_selection(&mut config);
|
||||||
|
#[cfg(feature = "restore-checkpoints")]
|
||||||
|
if config.test_restore_checkpoint.is_some() && config.no_store {
|
||||||
|
return Err(ProcessError::usage(
|
||||||
|
"--test-restore-checkpoint requires an fs Runtime store".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
Ok(Some(config))
|
Ok(Some(config))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -521,6 +548,8 @@ struct ProcessConfig {
|
|||||||
no_store: bool,
|
no_store: bool,
|
||||||
backend_resource_endpoint: Option<String>,
|
backend_resource_endpoint: Option<String>,
|
||||||
backend_resource_token: Option<String>,
|
backend_resource_token: Option<String>,
|
||||||
|
#[cfg(feature = "restore-checkpoints")]
|
||||||
|
test_restore_checkpoint: Option<worker_runtime::restore_checkpoints::RestoreCheckpointConfig>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
@@ -542,6 +571,8 @@ impl ProcessConfig {
|
|||||||
no_store: false,
|
no_store: false,
|
||||||
backend_resource_endpoint: None,
|
backend_resource_endpoint: None,
|
||||||
backend_resource_token: None,
|
backend_resource_token: None,
|
||||||
|
#[cfg(feature = "restore-checkpoints")]
|
||||||
|
test_restore_checkpoint: None,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1227,6 +1258,47 @@ Auth commands:
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn checkpoint_cli_is_absent_by_default() {
|
||||||
|
let config = parse_args([] as [&str; 0]).unwrap().unwrap();
|
||||||
|
#[cfg(feature = "restore-checkpoints")]
|
||||||
|
assert!(config.test_restore_checkpoint.is_none());
|
||||||
|
#[cfg(not(feature = "restore-checkpoints"))]
|
||||||
|
{
|
||||||
|
let _ = config;
|
||||||
|
let error = parse_args(["--test-restore-checkpoint", "never-opened.json"]).unwrap_err();
|
||||||
|
assert!(error.to_string().contains("unknown argument"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "restore-checkpoints")]
|
||||||
|
#[test]
|
||||||
|
fn checkpoint_cli_requires_valid_explicit_file_and_fs_store() {
|
||||||
|
use worker_runtime::restore_checkpoints::{
|
||||||
|
RestoreCheckpointConfig, RestoreCheckpointStage,
|
||||||
|
};
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = dir.path().join("checkpoint.json");
|
||||||
|
let checkpoint = RestoreCheckpointConfig {
|
||||||
|
endpoint: "127.0.0.1:1234".parse().unwrap(),
|
||||||
|
worker_id: protocol::WorkerId::now_v7(),
|
||||||
|
stage: RestoreCheckpointStage::CommittedPreterminal,
|
||||||
|
nonce: "cli-test".into(),
|
||||||
|
};
|
||||||
|
std::fs::write(&path, serde_json::to_vec(&checkpoint).unwrap()).unwrap();
|
||||||
|
let args = ["--test-restore-checkpoint", path.to_str().unwrap()];
|
||||||
|
let config = parse_args(args).unwrap().unwrap();
|
||||||
|
assert_eq!(config.test_restore_checkpoint, Some(checkpoint));
|
||||||
|
assert!(parse_args([args[0], args[1], "--no-store"]).is_err());
|
||||||
|
assert!(parse_args([args[0], args[1], args[0], args[1]]).is_err());
|
||||||
|
assert!(parse_args([args[0]]).is_err());
|
||||||
|
std::fs::write(&path, b"private-invalid-body").unwrap();
|
||||||
|
let error = parse_args(args).unwrap_err().to_string();
|
||||||
|
assert!(!error.contains("private-invalid-body"));
|
||||||
|
std::fs::remove_file(&path).unwrap();
|
||||||
|
assert!(parse_args(args).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn help_does_not_start_server() {
|
fn help_does_not_start_server() {
|
||||||
assert!(parse_args(["--help"]).unwrap().is_none());
|
assert!(parse_args(["--help"]).unwrap().is_none());
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
|
#[cfg(feature = "http-server")]
|
||||||
use crate::auth::BACKEND_RESOURCE_FETCH_PERMISSION;
|
use crate::auth::BACKEND_RESOURCE_FETCH_PERMISSION;
|
||||||
use crate::identity::WorkerId;
|
use crate::identity::WorkerId;
|
||||||
use crate::profile_archive::{ProfileSourceArchive, ProfileSourceArchiveRef, sha256_hex};
|
use crate::profile_archive::{ProfileSourceArchive, ProfileSourceArchiveRef, sha256_hex};
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
use crate::workspace_request::RuntimeWorkspaceRequestClient;
|
use crate::workspace_request::RuntimeWorkspaceRequestClient;
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|||||||
@@ -0,0 +1,144 @@
|
|||||||
|
//! Explicit, ephemeral synchronization for owned Runtime process tests only.
|
||||||
|
//!
|
||||||
|
//! The nonce authenticates no Runtime action and is never a routing identity.
|
||||||
|
//! Socket failure, EOF and the fixed deadline only release the gate; the lifecycle
|
||||||
|
//! owner continues its original operation and remains responsible for cleanup.
|
||||||
|
|
||||||
|
use std::fs::OpenOptions;
|
||||||
|
use std::io::{Read, Write};
|
||||||
|
use std::net::{SocketAddr, TcpStream};
|
||||||
|
use std::path::Path;
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use protocol::WorkerId;
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
use crate::error::RuntimeError;
|
||||||
|
use crate::execution::WorkerLifecycleOperationId;
|
||||||
|
|
||||||
|
pub const RESTORE_CHECKPOINT_TIMEOUT: Duration = Duration::from_secs(8);
|
||||||
|
pub const MAX_RESTORE_CHECKPOINT_CONFIG_BYTES: usize = 4096;
|
||||||
|
pub const MAX_RESTORE_CHECKPOINT_ACK_BYTES: usize = 4096;
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum RestoreCheckpointStage {
|
||||||
|
PreparedNotCommitted,
|
||||||
|
CommittedPreterminal,
|
||||||
|
ActivatedPreterminal,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum RestoreCheckpointCommit {
|
||||||
|
NotCommitted,
|
||||||
|
Committed,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct RestoreCheckpointConfig {
|
||||||
|
pub endpoint: SocketAddr,
|
||||||
|
pub worker_id: WorkerId,
|
||||||
|
pub stage: RestoreCheckpointStage,
|
||||||
|
pub nonce: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct RestoreCheckpointAck {
|
||||||
|
pub nonce: String,
|
||||||
|
pub stage: RestoreCheckpointStage,
|
||||||
|
pub runtime_id: String,
|
||||||
|
pub worker_id: WorkerId,
|
||||||
|
pub operation_id: WorkerLifecycleOperationId,
|
||||||
|
pub commit: RestoreCheckpointCommit,
|
||||||
|
pub receipt_absent: bool,
|
||||||
|
pub pid: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn invalid_config() -> RuntimeError {
|
||||||
|
// Do not include serde errors, paths or input contents in process diagnostics.
|
||||||
|
RuntimeError::InvalidRequest("invalid test restore checkpoint configuration".into())
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RestoreCheckpointConfig {
|
||||||
|
pub fn validate(&self) -> Result<(), RuntimeError> {
|
||||||
|
if !self.endpoint.ip().is_loopback()
|
||||||
|
|| self.endpoint.port() == 0
|
||||||
|
|| self.nonce.is_empty()
|
||||||
|
|| self.nonce.len() > 256
|
||||||
|
|| self.nonce.chars().any(char::is_control)
|
||||||
|
{
|
||||||
|
return Err(invalid_config());
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn from_path(path: &Path) -> Result<Self, RuntimeError> {
|
||||||
|
let mut options = OpenOptions::new();
|
||||||
|
options.read(true);
|
||||||
|
#[cfg(unix)]
|
||||||
|
{
|
||||||
|
use std::os::unix::fs::OpenOptionsExt;
|
||||||
|
// A FIFO must not stall startup before its regular-file check.
|
||||||
|
options.custom_flags(libc::O_NONBLOCK | libc::O_NOFOLLOW);
|
||||||
|
}
|
||||||
|
let file = options.open(path).map_err(|_| invalid_config())?;
|
||||||
|
let metadata = file.metadata().map_err(|_| invalid_config())?;
|
||||||
|
if !metadata.is_file() || metadata.len() > MAX_RESTORE_CHECKPOINT_CONFIG_BYTES as u64 {
|
||||||
|
return Err(invalid_config());
|
||||||
|
}
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
file.take(MAX_RESTORE_CHECKPOINT_CONFIG_BYTES as u64 + 1)
|
||||||
|
.read_to_end(&mut bytes)
|
||||||
|
.map_err(|_| invalid_config())?;
|
||||||
|
if bytes.len() > MAX_RESTORE_CHECKPOINT_CONFIG_BYTES {
|
||||||
|
return Err(invalid_config());
|
||||||
|
}
|
||||||
|
let config: Self = serde_json::from_slice(&bytes).map_err(|_| invalid_config())?;
|
||||||
|
config.validate()?;
|
||||||
|
Ok(config)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn wait_for_release(config: &RestoreCheckpointConfig, ack: &RestoreCheckpointAck) {
|
||||||
|
let _ = exchange(config, ack);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn remaining(deadline: Instant) -> std::io::Result<Duration> {
|
||||||
|
deadline
|
||||||
|
.checked_duration_since(Instant::now())
|
||||||
|
.filter(|remaining| !remaining.is_zero())
|
||||||
|
.ok_or_else(|| std::io::Error::from(std::io::ErrorKind::TimedOut))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn exchange(config: &RestoreCheckpointConfig, ack: &RestoreCheckpointAck) -> std::io::Result<()> {
|
||||||
|
let deadline = Instant::now() + RESTORE_CHECKPOINT_TIMEOUT;
|
||||||
|
let mut bytes = serde_json::to_vec(ack)?;
|
||||||
|
bytes.push(b'\n');
|
||||||
|
if bytes.len() > MAX_RESTORE_CHECKPOINT_ACK_BYTES {
|
||||||
|
return Err(std::io::Error::from(std::io::ErrorKind::InvalidData));
|
||||||
|
}
|
||||||
|
let mut stream = TcpStream::connect_timeout(&config.endpoint, remaining(deadline)?)?;
|
||||||
|
// Each syscall shares one budget; partial writes cannot reset the deadline.
|
||||||
|
let mut pending = bytes.as_slice();
|
||||||
|
while !pending.is_empty() {
|
||||||
|
stream.set_write_timeout(Some(remaining(deadline)?))?;
|
||||||
|
let written = stream.write(pending)?;
|
||||||
|
if written == 0 {
|
||||||
|
return Err(std::io::Error::from(std::io::ErrorKind::WriteZero));
|
||||||
|
}
|
||||||
|
pending = &pending[written..];
|
||||||
|
}
|
||||||
|
stream.set_read_timeout(Some(remaining(deadline)?))?;
|
||||||
|
let mut release = [0];
|
||||||
|
stream.read_exact(&mut release)?;
|
||||||
|
if release != [b'R'] {
|
||||||
|
return Err(std::io::Error::from(std::io::ErrorKind::InvalidData));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests;
|
||||||
@@ -0,0 +1,139 @@
|
|||||||
|
use super::*;
|
||||||
|
use std::io::{BufRead, BufReader};
|
||||||
|
use std::net::TcpListener;
|
||||||
|
use std::thread;
|
||||||
|
|
||||||
|
fn config(endpoint: SocketAddr) -> RestoreCheckpointConfig {
|
||||||
|
RestoreCheckpointConfig {
|
||||||
|
endpoint,
|
||||||
|
worker_id: WorkerId::now_v7(),
|
||||||
|
stage: RestoreCheckpointStage::PreparedNotCommitted,
|
||||||
|
nonce: "checkpoint-test-handshake".into(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn ack(config: &RestoreCheckpointConfig) -> RestoreCheckpointAck {
|
||||||
|
RestoreCheckpointAck {
|
||||||
|
nonce: config.nonce.clone(),
|
||||||
|
stage: config.stage,
|
||||||
|
runtime_id: "checkpoint-runtime".into(),
|
||||||
|
worker_id: config.worker_id,
|
||||||
|
operation_id: WorkerLifecycleOperationId::new(),
|
||||||
|
commit: RestoreCheckpointCommit::NotCommitted,
|
||||||
|
receipt_absent: true,
|
||||||
|
pid: std::process::id(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn config_rejects_nonliteral_nonloopback_bad_stage_unknown_fields_and_invalid_worker() {
|
||||||
|
let valid = serde_json::to_value(config("127.0.0.1:1234".parse().unwrap())).unwrap();
|
||||||
|
for (field, value) in [
|
||||||
|
("endpoint", "localhost:1234"),
|
||||||
|
("endpoint", "192.0.2.1:1234"),
|
||||||
|
("endpoint", "0.0.0.0:1234"),
|
||||||
|
("endpoint", "127.0.0.1:0"),
|
||||||
|
("endpoint", "[::ffff:127.0.0.1]:1234"),
|
||||||
|
("worker_id", "not-a-worker-id"),
|
||||||
|
("stage", "after_terminal"),
|
||||||
|
("nonce", ""),
|
||||||
|
("nonce", "secret\nbody"),
|
||||||
|
("timeout", "unbounded"),
|
||||||
|
] {
|
||||||
|
let mut invalid = valid.clone();
|
||||||
|
invalid[field] = value.into();
|
||||||
|
let parsed = serde_json::from_value::<RestoreCheckpointConfig>(invalid);
|
||||||
|
assert!(
|
||||||
|
parsed.is_err() || parsed.unwrap().validate().is_err(),
|
||||||
|
"{field}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for endpoint in ["127.0.0.1:1234", "[::1]:1234"] {
|
||||||
|
config(endpoint.parse().unwrap()).validate().unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn config_file_bounds_and_errors_do_not_disclose_contents() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = dir.path().join("config.json");
|
||||||
|
assert!(RestoreCheckpointConfig::from_path(&path).is_err());
|
||||||
|
std::fs::write(&path, vec![b'x'; MAX_RESTORE_CHECKPOINT_CONFIG_BYTES + 1]).unwrap();
|
||||||
|
assert!(RestoreCheckpointConfig::from_path(&path).is_err());
|
||||||
|
std::fs::write(
|
||||||
|
&path,
|
||||||
|
br#"{"nonce":"private-config-body","stage":"secret"}"#,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let error = RestoreCheckpointConfig::from_path(&path)
|
||||||
|
.unwrap_err()
|
||||||
|
.to_string();
|
||||||
|
assert!(!error.contains("private-config-body"));
|
||||||
|
assert!(!error.contains("secret"));
|
||||||
|
let mut valid = config("127.0.0.1:1234".parse().unwrap());
|
||||||
|
valid.nonce = "x".repeat(257);
|
||||||
|
assert!(valid.validate().is_err());
|
||||||
|
valid.nonce = "valid".into();
|
||||||
|
std::fs::write(&path, serde_json::to_vec(&valid).unwrap()).unwrap();
|
||||||
|
assert_eq!(RestoreCheckpointConfig::from_path(&path).unwrap(), valid);
|
||||||
|
assert!(RestoreCheckpointConfig::from_path(dir.path()).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn explicit_release_eof_and_invalid_release_close_the_owned_stream() {
|
||||||
|
for release in [Some(b'R'), None, Some(b'X')] {
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||||
|
let config = config(listener.local_addr().unwrap());
|
||||||
|
let expected = ack(&config);
|
||||||
|
let sent = expected.clone();
|
||||||
|
let task = thread::spawn(move || exchange(&config, &sent));
|
||||||
|
let (mut stream, _) = listener.accept().unwrap();
|
||||||
|
stream
|
||||||
|
.set_read_timeout(Some(Duration::from_secs(2)))
|
||||||
|
.unwrap();
|
||||||
|
let mut line = String::new();
|
||||||
|
BufReader::new(&mut stream).read_line(&mut line).unwrap();
|
||||||
|
assert!(line.len() <= MAX_RESTORE_CHECKPOINT_ACK_BYTES);
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::from_str::<RestoreCheckpointAck>(&line).unwrap(),
|
||||||
|
expected
|
||||||
|
);
|
||||||
|
if let Some(byte) = release {
|
||||||
|
stream.write_all(&[byte]).unwrap();
|
||||||
|
} else {
|
||||||
|
stream.shutdown(std::net::Shutdown::Write).unwrap();
|
||||||
|
}
|
||||||
|
assert_eq!(task.join().unwrap().is_ok(), release == Some(b'R'));
|
||||||
|
let mut byte = [0];
|
||||||
|
assert_eq!(stream.read(&mut byte).unwrap(), 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn absent_listener_resumes_without_waiting_or_retrying() {
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||||
|
let config = config(listener.local_addr().unwrap());
|
||||||
|
drop(listener);
|
||||||
|
let start = Instant::now();
|
||||||
|
wait_for_release(&config, &ack(&config));
|
||||||
|
assert!(start.elapsed() < Duration::from_secs(2));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn silent_peer_hits_fixed_deadline_and_stream_is_dropped() {
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||||
|
let config = config(listener.local_addr().unwrap());
|
||||||
|
let start = Instant::now();
|
||||||
|
let task = thread::spawn(move || exchange(&config, &ack(&config)));
|
||||||
|
let (mut stream, _) = listener.accept().unwrap();
|
||||||
|
stream
|
||||||
|
.set_read_timeout(Some(Duration::from_secs(12)))
|
||||||
|
.unwrap();
|
||||||
|
let mut reader = BufReader::new(&mut stream);
|
||||||
|
let mut line = String::new();
|
||||||
|
reader.read_line(&mut line).unwrap();
|
||||||
|
assert!(task.join().unwrap().is_err());
|
||||||
|
assert!(start.elapsed() >= RESTORE_CHECKPOINT_TIMEOUT - Duration::from_millis(100));
|
||||||
|
assert!(start.elapsed() < RESTORE_CHECKPOINT_TIMEOUT + Duration::from_secs(3));
|
||||||
|
assert_eq!(reader.read(&mut [0]).unwrap(), 0);
|
||||||
|
}
|
||||||
@@ -2069,6 +2069,103 @@ mod tests {
|
|||||||
compact
|
compact
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn archive_removal_preserves_history_larger_than_256_mib_and_replays_receipt() {
|
||||||
|
use session_store::{
|
||||||
|
LogEntry, LoggedContentPart, LoggedHistoryEntry, LoggedItem, LoggedRole,
|
||||||
|
LoggedSessionHistoryEntryId, LoggedSessionHistoryMetadata, LoggedSessionHistoryOrigin,
|
||||||
|
Store,
|
||||||
|
};
|
||||||
|
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let worker = WorkerId::now_v7();
|
||||||
|
let worker_dir = temp.path().join("workers").join(worker.to_string());
|
||||||
|
write_json(
|
||||||
|
&worker_dir.join("worker.json"),
|
||||||
|
&serde_json::json!({"workspace_id": "workspace-a"}),
|
||||||
|
);
|
||||||
|
let session = session_store::new_session_id();
|
||||||
|
let segment = session_store::new_segment_id();
|
||||||
|
let store = session_store::WorkerSessionStore::new(worker_dir.join("session")).unwrap();
|
||||||
|
store
|
||||||
|
.create_segment(
|
||||||
|
session,
|
||||||
|
segment,
|
||||||
|
&[LogEntry::AnnotatedSegmentStart {
|
||||||
|
ts: 1,
|
||||||
|
session_id: session,
|
||||||
|
system_prompt: None,
|
||||||
|
config: Default::default(),
|
||||||
|
history: Vec::new(),
|
||||||
|
forked_from: None,
|
||||||
|
compacted_from: None,
|
||||||
|
}],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
// Each record is small enough to decode independently. Only the lifetime
|
||||||
|
// total exceeds the former ceiling; no individual resource limit is relaxed.
|
||||||
|
let text = "x".repeat(1024 * 1024);
|
||||||
|
for index in 0..257 {
|
||||||
|
store
|
||||||
|
.append(
|
||||||
|
session,
|
||||||
|
segment,
|
||||||
|
&LogEntry::AnnotatedUserInput {
|
||||||
|
ts: index + 2,
|
||||||
|
segments: Vec::new(),
|
||||||
|
extensions: Vec::new(),
|
||||||
|
history: vec![LoggedHistoryEntry {
|
||||||
|
item: LoggedItem::Message {
|
||||||
|
role: LoggedRole::User,
|
||||||
|
content: vec![LoggedContentPart::Text { text: text.clone() }],
|
||||||
|
},
|
||||||
|
metadata: LoggedSessionHistoryMetadata {
|
||||||
|
input_metadata: None,
|
||||||
|
entry_id: LoggedSessionHistoryEntryId(format!("input-{index}")),
|
||||||
|
origin: LoggedSessionHistoryOrigin::LocalHumanInput,
|
||||||
|
derivation: None,
|
||||||
|
},
|
||||||
|
}],
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
let mut metadata = session_store::WorkerMetadata::new(
|
||||||
|
worker.to_string(),
|
||||||
|
Some(session_store::WorkerActiveSegmentRef::active_segment(
|
||||||
|
session, segment,
|
||||||
|
)),
|
||||||
|
);
|
||||||
|
metadata.workspace_id = Some("workspace-a".into());
|
||||||
|
write_json(&worker_dir.join("metadata.json"), &metadata);
|
||||||
|
let segment_path = format!("session/segments/{segment}.jsonl");
|
||||||
|
let source_bytes = fs::metadata(worker_dir.join(&segment_path)).unwrap().len();
|
||||||
|
assert!(source_bytes > 256 * 1024 * 1024);
|
||||||
|
|
||||||
|
let provider = FsWorkerRetentionProvider::new(temp.path());
|
||||||
|
let request = request(worker, SessionDisposition::Archive);
|
||||||
|
let result = provider.execute(&request).unwrap();
|
||||||
|
assert!(result.source_removed);
|
||||||
|
assert!(!worker_dir.exists());
|
||||||
|
let archive = provider.archive_dir("archive-a").unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
fs::metadata(archive.join(&segment_path)).unwrap().len(),
|
||||||
|
source_bytes
|
||||||
|
);
|
||||||
|
let reader = session_store::public_material::SessionPublicMaterialReader::open_worker(
|
||||||
|
&archive.join("session"),
|
||||||
|
segment,
|
||||||
|
Default::default(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(reader.session_id(), session);
|
||||||
|
assert_eq!(reader.descriptors().len(), 257);
|
||||||
|
let retry = provider.execute(&request).unwrap();
|
||||||
|
assert!(retry.source_removed);
|
||||||
|
assert_eq!(retry.archive, result.archive);
|
||||||
|
assert_eq!(retry.operation_id, result.operation_id);
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn session_migration_repairs_failed_archive_and_preserves_ingestion_receipt_on_retry() {
|
fn session_migration_repairs_failed_archive_and_preserves_ingestion_receipt_on_retry() {
|
||||||
let temp = tempfile::tempdir().unwrap();
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
|||||||
@@ -3,7 +3,7 @@
|
|||||||
//! only the empty segment envelope is built with current APIs.
|
//! only the empty segment envelope is built with current APIs.
|
||||||
use super::tests::{request, write_json};
|
use super::tests::{request, write_json};
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::fs_store::{FsRuntimeStore, FsRuntimeStoreOptions};
|
use crate::fs_store::{FsRuntimeStore, FsRuntimeStoreOptions, SCHEMA_VERSION};
|
||||||
use serde_json::{Value, json};
|
use serde_json::{Value, json};
|
||||||
use session_store::{LogEntry, SegmentId, SessionId, Store, WorkerSessionStore};
|
use session_store::{LogEntry, SegmentId, SessionId, Store, WorkerSessionStore};
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
@@ -226,7 +226,7 @@ fn completed_v4_archive(
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() {
|
fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() {
|
||||||
for runtime_schema in [11, 12] {
|
for runtime_schema in [11, 12, SCHEMA_VERSION] {
|
||||||
let temp = tempfile::tempdir().unwrap();
|
let temp = tempfile::tempdir().unwrap();
|
||||||
let root = temp.path().join("runtime");
|
let root = temp.path().join("runtime");
|
||||||
let (worker, session, segment) = runtime_fixture(&root, runtime_schema);
|
let (worker, session, segment) = runtime_fixture(&root, runtime_schema);
|
||||||
@@ -329,7 +329,7 @@ fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() {
|
|||||||
(storage.checksum_sha256, storage.bytes, storage.file_count),
|
(storage.checksum_sha256, storage.bytes, storage.file_count),
|
||||||
(checksum, bytes, count)
|
(checksum, bytes, count)
|
||||||
);
|
);
|
||||||
if runtime_schema == 12 {
|
if runtime_schema == SCHEMA_VERSION {
|
||||||
for name in [
|
for name in [
|
||||||
PathBuf::from("runtime.json"),
|
PathBuf::from("runtime.json"),
|
||||||
PathBuf::from(format!("workers/{worker}/worker.json")),
|
PathBuf::from(format!("workers/{worker}/worker.json")),
|
||||||
@@ -346,8 +346,8 @@ fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() {
|
|||||||
);
|
);
|
||||||
drop(FsRuntimeStore::open_or_create(root.clone(), "runtime-a").unwrap());
|
drop(FsRuntimeStore::open_or_create(root.clone(), "runtime-a").unwrap());
|
||||||
assert_eq!(tree_bytes(&root), once, "retry rewrote migrated evidence");
|
assert_eq!(tree_bytes(&root), once, "retry rewrote migrated evidence");
|
||||||
assert!(!temp.path().join(".runtime.schema-v12-staging").exists());
|
assert!(!temp.path().join(".runtime.schema-v13-staging").exists());
|
||||||
assert!(!temp.path().join(".runtime.pre-schema-v12-backup").exists());
|
assert!(!temp.path().join(".runtime.pre-schema-v13-backup").exists());
|
||||||
// session-storage, not the immutable receipt's old checksum, attests
|
// session-storage, not the immutable receipt's old checksum, attests
|
||||||
// current bytes. Tampering with those bytes must remain detectable.
|
// current bytes. Tampering with those bytes must remain detectable.
|
||||||
let path = archive_session
|
let path = archive_session
|
||||||
@@ -360,7 +360,7 @@ fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn corrupt_archive_does_not_block_startup_or_healthy_session_migration_and_repair_converges() {
|
fn corrupt_archive_does_not_block_startup_or_healthy_session_migration_and_repair_converges() {
|
||||||
for runtime_schema in [11, 12] {
|
for runtime_schema in [11, 12, SCHEMA_VERSION] {
|
||||||
let temp = tempfile::tempdir().unwrap();
|
let temp = tempfile::tempdir().unwrap();
|
||||||
let root = temp.path().join("runtime");
|
let root = temp.path().join("runtime");
|
||||||
let (worker, session, segment) = runtime_fixture(&root, runtime_schema);
|
let (worker, session, segment) = runtime_fixture(&root, runtime_schema);
|
||||||
@@ -440,8 +440,8 @@ fn unmigratable_archive_preserves_evidence_across_startup_and_reopen() {
|
|||||||
] {
|
] {
|
||||||
let temp = tempfile::tempdir().unwrap();
|
let temp = tempfile::tempdir().unwrap();
|
||||||
let root = temp.path().join("runtime");
|
let root = temp.path().join("runtime");
|
||||||
let (live_worker, live_session, live_segment) = runtime_fixture(&root, 12);
|
let (live_worker, live_session, live_segment) = runtime_fixture(&root, SCHEMA_VERSION);
|
||||||
let (worker, _, segment) = stopped_worker(&root, 12);
|
let (worker, _, segment) = stopped_worker(&root, SCHEMA_VERSION);
|
||||||
let provider = FsWorkerRetentionProvider::new(&root);
|
let provider = FsWorkerRetentionProvider::new(&root);
|
||||||
let path = provider
|
let path = provider
|
||||||
.worker_dir(worker)
|
.worker_dir(worker)
|
||||||
@@ -542,8 +542,8 @@ fn unreadable_archive_metadata_does_not_block_archive_only_runtime_startup() {
|
|||||||
] {
|
] {
|
||||||
let temp = tempfile::tempdir().unwrap();
|
let temp = tempfile::tempdir().unwrap();
|
||||||
let root = temp.path().join("runtime");
|
let root = temp.path().join("runtime");
|
||||||
let (worker, _, _) = runtime_fixture(&root, 12);
|
let (worker, _, _) = runtime_fixture(&root, SCHEMA_VERSION);
|
||||||
completed_v4_archive(&root, 12);
|
completed_v4_archive(&root, SCHEMA_VERSION);
|
||||||
let provider = FsWorkerRetentionProvider::new(&root);
|
let provider = FsWorkerRetentionProvider::new(&root);
|
||||||
fs::remove_dir_all(provider.worker_dir(worker)).unwrap();
|
fs::remove_dir_all(provider.worker_dir(worker)).unwrap();
|
||||||
let archive = provider.archive_dir("archive-a").unwrap();
|
let archive = provider.archive_dir("archive-a").unwrap();
|
||||||
@@ -575,8 +575,8 @@ fn unreadable_archive_metadata_does_not_block_archive_only_runtime_startup() {
|
|||||||
fn archive_isolation_does_not_bypass_corrupt_erasure_authority() {
|
fn archive_isolation_does_not_bypass_corrupt_erasure_authority() {
|
||||||
let temp = tempfile::tempdir().unwrap();
|
let temp = tempfile::tempdir().unwrap();
|
||||||
let root = temp.path().join("runtime");
|
let root = temp.path().join("runtime");
|
||||||
runtime_fixture(&root, 12);
|
runtime_fixture(&root, SCHEMA_VERSION);
|
||||||
let (session, _, _, _) = completed_v4_archive(&root, 12);
|
let (session, _, _, _) = completed_v4_archive(&root, SCHEMA_VERSION);
|
||||||
let ledger = root.join("retention/session-sources");
|
let ledger = root.join("retention/session-sources");
|
||||||
fs::create_dir_all(&ledger).unwrap();
|
fs::create_dir_all(&ledger).unwrap();
|
||||||
fs::write(ledger.join(format!("{session}.json")), b"{broken\n").unwrap();
|
fs::write(ledger.join(format!("{session}.json")), b"{broken\n").unwrap();
|
||||||
@@ -591,7 +591,7 @@ fn archive_isolation_does_not_bypass_corrupt_erasure_authority() {
|
|||||||
fn runtime_v5_probe_and_staging_skip_erased_originals() {
|
fn runtime_v5_probe_and_staging_skip_erased_originals() {
|
||||||
let temp = tempfile::tempdir().unwrap();
|
let temp = tempfile::tempdir().unwrap();
|
||||||
let root = temp.path().join("runtime");
|
let root = temp.path().join("runtime");
|
||||||
let (worker, live_session, live_segment) = runtime_fixture(&root, 12);
|
let (worker, live_session, live_segment) = runtime_fixture(&root, SCHEMA_VERSION);
|
||||||
let erased = root
|
let erased = root
|
||||||
.join("orphaned-workers")
|
.join("orphaned-workers")
|
||||||
.join(WorkerId::now_v7().to_string());
|
.join(WorkerId::now_v7().to_string());
|
||||||
@@ -643,7 +643,7 @@ fn runtime_v5_rejects_broken_or_future_source_without_rewriting_evidence() {
|
|||||||
for corruption in ["broken-json", "future-schema"] {
|
for corruption in ["broken-json", "future-schema"] {
|
||||||
let temp = tempfile::tempdir().unwrap();
|
let temp = tempfile::tempdir().unwrap();
|
||||||
let root = temp.path().join("runtime");
|
let root = temp.path().join("runtime");
|
||||||
let (worker, session, segment) = runtime_fixture(&root, 12);
|
let (worker, session, segment) = runtime_fixture(&root, SCHEMA_VERSION);
|
||||||
let session_root = root
|
let session_root = root
|
||||||
.join("workers")
|
.join("workers")
|
||||||
.join(worker.to_string())
|
.join(worker.to_string())
|
||||||
@@ -674,19 +674,19 @@ fn runtime_v5_rejects_broken_or_future_source_without_rewriting_evidence() {
|
|||||||
assert_eq!(tree_bytes(&root), before);
|
assert_eq!(tree_bytes(&root), before);
|
||||||
assert!(FsRuntimeStore::migrate(&options).is_err());
|
assert!(FsRuntimeStore::migrate(&options).is_err());
|
||||||
assert_eq!(tree_bytes(&root), before);
|
assert_eq!(tree_bytes(&root), before);
|
||||||
assert!(!temp.path().join(".runtime.schema-v12-staging").exists());
|
assert!(!temp.path().join(".runtime.schema-v13-staging").exists());
|
||||||
assert!(!temp.path().join(".runtime.pre-schema-v12-backup").exists());
|
assert!(!temp.path().join(".runtime.pre-schema-v13-backup").exists());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
#[test]
|
#[test]
|
||||||
fn current_runtime_session_migration_still_rejects_live_worker_sockets() {
|
fn current_runtime_session_migration_still_rejects_live_worker_sockets() {
|
||||||
// Relative socket names keep this fixture independent of the checkout/TMPDIR
|
// The same TempDir owns the relative bind/connect reference. No cwd change
|
||||||
// length. The TempDir still owns the same tree; no process cwd is changed.
|
// or alias is needed, and the real live-socket migration refusal stays tested.
|
||||||
let temp = tempfile::tempdir_in(".").unwrap();
|
let temp = tempfile::tempdir_in(".").unwrap();
|
||||||
let root = Path::new(temp.path().file_name().unwrap()).join("runtime");
|
let root = Path::new(temp.path().file_name().unwrap()).join("runtime");
|
||||||
let (worker, _, _) = runtime_fixture(&root, 12);
|
let (worker, _, _) = runtime_fixture(&root, SCHEMA_VERSION);
|
||||||
let before = tree_bytes(&root);
|
let before = tree_bytes(&root);
|
||||||
let runs = root.join("workers").join(worker.to_string()).join("runs");
|
let runs = root.join("workers").join(worker.to_string()).join("runs");
|
||||||
fs::create_dir_all(runs.join("1")).unwrap();
|
fs::create_dir_all(runs.join("1")).unwrap();
|
||||||
@@ -697,7 +697,7 @@ fn current_runtime_session_migration_still_rejects_live_worker_sockets() {
|
|||||||
drop(listener);
|
drop(listener);
|
||||||
fs::remove_dir_all(runs).unwrap();
|
fs::remove_dir_all(runs).unwrap();
|
||||||
assert_eq!(tree_bytes(&root), before);
|
assert_eq!(tree_bytes(&root), before);
|
||||||
assert!(!temp.path().join(".runtime.schema-v12-staging").exists());
|
assert!(!temp.path().join(".runtime.schema-v13-staging").exists());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -705,7 +705,7 @@ fn current_runtime_upgrades_v5_role_metadata_through_staging_without_reserializi
|
|||||||
{
|
{
|
||||||
let temp = tempfile::tempdir().unwrap();
|
let temp = tempfile::tempdir().unwrap();
|
||||||
let root = temp.path().join("runtime");
|
let root = temp.path().join("runtime");
|
||||||
let (worker, session, segment) = runtime_fixture(&root, 12);
|
let (worker, session, segment) = runtime_fixture(&root, SCHEMA_VERSION);
|
||||||
let aggregate = root.join("workers").join(worker.to_string());
|
let aggregate = root.join("workers").join(worker.to_string());
|
||||||
let path = aggregate
|
let path = aggregate
|
||||||
.join("session/segments")
|
.join("session/segments")
|
||||||
|
|||||||
+1945
-496
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,112 @@
|
|||||||
|
use super::*;
|
||||||
|
use crate::restore_checkpoints::{
|
||||||
|
RestoreCheckpointAck, RestoreCheckpointCommit, RestoreCheckpointConfig, RestoreCheckpointStage,
|
||||||
|
wait_for_release,
|
||||||
|
};
|
||||||
|
|
||||||
|
impl Runtime {
|
||||||
|
/// Install an explicitly selected, one-shot process-test gate after normal startup.
|
||||||
|
/// It is not persisted or consulted by admission, routing or recovery authority.
|
||||||
|
pub fn install_test_restore_checkpoint(
|
||||||
|
&self,
|
||||||
|
config: RestoreCheckpointConfig,
|
||||||
|
) -> Result<(), RuntimeError> {
|
||||||
|
config.validate()?;
|
||||||
|
let mut state = self.lock()?;
|
||||||
|
if !matches!(state.persistence, RuntimePersistence::Fs(_)) {
|
||||||
|
return Err(RuntimeError::InvalidRequest(
|
||||||
|
"test restore checkpoint requires an fs Runtime store".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
state.restore_checkpoint = Some(config);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
// The lifecycle caller holds the per-Worker operation lock. No Runtime-state
|
||||||
|
// mutex is retained while waiting; the real backend candidate remains owned by
|
||||||
|
// the same operation throughout this synchronous call.
|
||||||
|
pub(super) fn test_restore_checkpoint(
|
||||||
|
&self,
|
||||||
|
worker_ref: &WorkerRef,
|
||||||
|
operation_id: WorkerLifecycleOperationId,
|
||||||
|
stage: RestoreCheckpointStage,
|
||||||
|
) {
|
||||||
|
let config = {
|
||||||
|
let Ok(mut state) = self.lock() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let slot = &mut state.restore_checkpoint;
|
||||||
|
if !slot.as_ref().is_some_and(|config| {
|
||||||
|
config.worker_id == worker_ref.worker_id && config.stage == stage
|
||||||
|
}) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// Consume even if canonical verification or the handshake fails.
|
||||||
|
slot.take().expect("matched checkpoint")
|
||||||
|
};
|
||||||
|
let Ok(ack) = self.restore_checkpoint_ack(&config, worker_ref, operation_id) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
wait_for_release(&config, &ack);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn restore_checkpoint_ack(
|
||||||
|
&self,
|
||||||
|
config: &RestoreCheckpointConfig,
|
||||||
|
worker_ref: &WorkerRef,
|
||||||
|
operation_id: WorkerLifecycleOperationId,
|
||||||
|
) -> Result<RestoreCheckpointAck, RuntimeError> {
|
||||||
|
let invalid = || {
|
||||||
|
RuntimeError::InvalidRequest("test restore checkpoint authority did not match".into())
|
||||||
|
};
|
||||||
|
let state = self.lock()?;
|
||||||
|
let RuntimePersistence::Fs(store) = &state.persistence else {
|
||||||
|
return Err(invalid());
|
||||||
|
};
|
||||||
|
// The existing helper validates the exact bounded canonical aggregate and
|
||||||
|
// syncs that same file plus its owning directories. Never manufacture a
|
||||||
|
// journal marker or infer commit from an in-memory projection.
|
||||||
|
let record = store.read_durable_worker_record(worker_ref)?;
|
||||||
|
let PersistedWorkerExecutionState::ReconciliationRequired(
|
||||||
|
PersistedWorkerLifecycleOperation::Restore(pending),
|
||||||
|
) = &record.execution_state
|
||||||
|
else {
|
||||||
|
return Err(invalid());
|
||||||
|
};
|
||||||
|
let expected = match config.stage {
|
||||||
|
RestoreCheckpointStage::PreparedNotCommitted => RestoreCommitFact::NotCommitted,
|
||||||
|
RestoreCheckpointStage::CommittedPreterminal
|
||||||
|
| RestoreCheckpointStage::ActivatedPreterminal => RestoreCommitFact::Committed,
|
||||||
|
};
|
||||||
|
if config.worker_id != worker_ref.worker_id
|
||||||
|
|| pending.operation_id != operation_id
|
||||||
|
|| pending.commit != expected
|
||||||
|
{
|
||||||
|
return Err(invalid());
|
||||||
|
}
|
||||||
|
let guard = &record.restore_guard;
|
||||||
|
let id = guard.active_request_id.as_ref().ok_or_else(invalid)?;
|
||||||
|
let owner = guard.owners.get(id).ok_or_else(invalid)?;
|
||||||
|
if owner.operation_id != operation_id
|
||||||
|
|| owner.request.request_id != *id
|
||||||
|
|| owner.receipt.is_some()
|
||||||
|
{
|
||||||
|
return Err(invalid());
|
||||||
|
}
|
||||||
|
let runtime_id = state.runtime_identity.clone().ok_or_else(invalid)?;
|
||||||
|
Ok(RestoreCheckpointAck {
|
||||||
|
nonce: config.nonce.clone(),
|
||||||
|
stage: config.stage,
|
||||||
|
runtime_id,
|
||||||
|
worker_id: record.worker_id,
|
||||||
|
operation_id,
|
||||||
|
commit: match expected {
|
||||||
|
RestoreCommitFact::NotCommitted => RestoreCheckpointCommit::NotCommitted,
|
||||||
|
RestoreCommitFact::Committed => RestoreCheckpointCommit::Committed,
|
||||||
|
RestoreCommitFact::LegacyUnknown => unreachable!(),
|
||||||
|
},
|
||||||
|
receipt_absent: true,
|
||||||
|
pid: std::process::id(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,304 @@
|
|||||||
|
use super::*;
|
||||||
|
use crate::restore_checkpoints::{
|
||||||
|
RestoreCheckpointAck, RestoreCheckpointCommit, RestoreCheckpointConfig, RestoreCheckpointStage,
|
||||||
|
};
|
||||||
|
use std::io::{BufRead, BufReader, Write};
|
||||||
|
use std::net::TcpListener;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn checkpoint_stages_ack_canonical_decision_and_eof_resumes_same_operation_once() {
|
||||||
|
// Component scope: durable Runtime with a substituted execution backend.
|
||||||
|
// Real prepared Controllers and process death are owned by the E2E suite.
|
||||||
|
for (stage, release) in [
|
||||||
|
(RestoreCheckpointStage::PreparedNotCommitted, "release"),
|
||||||
|
(RestoreCheckpointStage::CommittedPreterminal, "eof"),
|
||||||
|
(RestoreCheckpointStage::CommittedPreterminal, "timeout"),
|
||||||
|
(RestoreCheckpointStage::ActivatedPreterminal, "release"),
|
||||||
|
] {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let backend = Arc::new(TestExecutionBackend::default());
|
||||||
|
let runtime = Runtime::with_fs_store_and_execution_backend(
|
||||||
|
FsRuntimeStoreOptions::new(dir.path().join("runtime"))
|
||||||
|
.with_runtime_id("checkpoint-runtime"),
|
||||||
|
backend.clone(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
runtime.bind_runtime_identity("checkpoint-runtime").unwrap();
|
||||||
|
runtime.store_config_bundle(test_bundle()).unwrap();
|
||||||
|
let worker = runtime
|
||||||
|
.create_worker(task_request("checkpoint original operation"))
|
||||||
|
.unwrap();
|
||||||
|
runtime.stop_worker(&worker.worker_ref, None).unwrap();
|
||||||
|
let request = runtime.test_restore_request(&worker.worker_ref);
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||||
|
runtime
|
||||||
|
.install_test_restore_checkpoint(RestoreCheckpointConfig {
|
||||||
|
endpoint: listener.local_addr().unwrap(),
|
||||||
|
worker_id: worker.worker_id,
|
||||||
|
stage,
|
||||||
|
nonce: "runtime-stage-test".into(),
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
let cloned = runtime.clone();
|
||||||
|
let worker_ref = worker.worker_ref.clone();
|
||||||
|
let sent_request = request.clone();
|
||||||
|
let (sent, received) = std::sync::mpsc::channel();
|
||||||
|
let task = std::thread::spawn(move || {
|
||||||
|
sent.send(cloned.restore_worker_operation(&worker_ref, sent_request))
|
||||||
|
.unwrap();
|
||||||
|
});
|
||||||
|
listener.set_nonblocking(true).unwrap();
|
||||||
|
let stream = tokio::runtime::Builder::new_current_thread()
|
||||||
|
.enable_all()
|
||||||
|
.build()
|
||||||
|
.unwrap()
|
||||||
|
.block_on(async {
|
||||||
|
let listener = tokio::net::TcpListener::from_std(listener).unwrap();
|
||||||
|
let (stream, _) = tokio::time::timeout(Duration::from_secs(5), listener.accept())
|
||||||
|
.await
|
||||||
|
.expect("checkpoint did not connect")
|
||||||
|
.unwrap();
|
||||||
|
stream.into_std().unwrap()
|
||||||
|
});
|
||||||
|
stream.set_nonblocking(false).unwrap();
|
||||||
|
stream
|
||||||
|
.set_read_timeout(Some(Duration::from_secs(3)))
|
||||||
|
.unwrap();
|
||||||
|
let mut reader = BufReader::new(stream);
|
||||||
|
let mut line = String::new();
|
||||||
|
reader.read_line(&mut line).unwrap();
|
||||||
|
let ack: RestoreCheckpointAck = serde_json::from_str(&line).unwrap();
|
||||||
|
assert_eq!(ack.stage, stage);
|
||||||
|
assert_eq!(ack.nonce, "runtime-stage-test");
|
||||||
|
assert_eq!(ack.runtime_id, "checkpoint-runtime");
|
||||||
|
assert_eq!(ack.worker_id, worker.worker_id);
|
||||||
|
assert_eq!(ack.pid, std::process::id());
|
||||||
|
assert!(ack.receipt_absent);
|
||||||
|
assert_eq!(
|
||||||
|
ack.commit,
|
||||||
|
match stage {
|
||||||
|
RestoreCheckpointStage::PreparedNotCommitted =>
|
||||||
|
RestoreCheckpointCommit::NotCommitted,
|
||||||
|
RestoreCheckpointStage::CommittedPreterminal
|
||||||
|
| RestoreCheckpointStage::ActivatedPreterminal => RestoreCheckpointCommit::Committed,
|
||||||
|
}
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
backend.restore_operation_ids.lock().unwrap().as_slice(),
|
||||||
|
&[ack.operation_id]
|
||||||
|
);
|
||||||
|
if stage == RestoreCheckpointStage::ActivatedPreterminal {
|
||||||
|
assert_eq!(backend.activation_operation_ids.lock().unwrap().as_slice(), &[ack.operation_id]);
|
||||||
|
} else {
|
||||||
|
assert!(backend.activation_operation_ids.lock().unwrap().is_empty());
|
||||||
|
}
|
||||||
|
assert!(received.try_recv().is_err());
|
||||||
|
let canonical = runtime_store(&runtime)
|
||||||
|
.read_durable_worker_record(&worker.worker_ref)
|
||||||
|
.unwrap();
|
||||||
|
let id = canonical.restore_guard.active_request_id.as_ref().unwrap();
|
||||||
|
assert_eq!(id, &request.request_id);
|
||||||
|
assert!(canonical.restore_guard.owners[id].receipt.is_none());
|
||||||
|
// Release, EOF and timeout all resume the original lifecycle owner.
|
||||||
|
let start = std::time::Instant::now();
|
||||||
|
if release == "release" {
|
||||||
|
reader.get_mut().write_all(b"R").unwrap();
|
||||||
|
} else if release == "eof" {
|
||||||
|
reader
|
||||||
|
.get_mut()
|
||||||
|
.shutdown(std::net::Shutdown::Write)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
let result = received
|
||||||
|
.recv_timeout(Duration::from_secs(12))
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert!(start.elapsed() < Duration::from_secs(11));
|
||||||
|
task.join().unwrap();
|
||||||
|
assert_eq!(result.state, WorkerRestoreState::Accepted);
|
||||||
|
assert_eq!(
|
||||||
|
backend.activation_operation_ids.lock().unwrap().as_slice(),
|
||||||
|
&[ack.operation_id]
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.restore_worker_operation(&worker.worker_ref, request)
|
||||||
|
.unwrap(),
|
||||||
|
result
|
||||||
|
);
|
||||||
|
assert_eq!(*backend.restore_count.lock().unwrap(), 1);
|
||||||
|
assert!(runtime.lock().unwrap().restore_checkpoint.is_none());
|
||||||
|
runtime.stop_worker(&worker.worker_ref, None).unwrap();
|
||||||
|
let next = runtime.test_restore_request(&worker.worker_ref);
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.restore_worker_operation(&worker.worker_ref, next)
|
||||||
|
.unwrap()
|
||||||
|
.state,
|
||||||
|
WorkerRestoreState::Accepted
|
||||||
|
);
|
||||||
|
assert_eq!(*backend.restore_count.lock().unwrap(), 2);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn checkpoint_disabled_mismatched_or_unreachable_does_not_change_restore_result() {
|
||||||
|
for mode in ["disabled", "mismatch", "unreachable"] {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let backend = Arc::new(TestExecutionBackend::default());
|
||||||
|
let runtime = Runtime::with_fs_store_and_execution_backend(
|
||||||
|
FsRuntimeStoreOptions::new(dir.path().join("runtime"))
|
||||||
|
.with_runtime_id("checkpoint-runtime"),
|
||||||
|
backend.clone(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
runtime.bind_runtime_identity("checkpoint-runtime").unwrap();
|
||||||
|
runtime.store_config_bundle(test_bundle()).unwrap();
|
||||||
|
let worker = runtime
|
||||||
|
.create_worker(task_request("checkpoint absent"))
|
||||||
|
.unwrap();
|
||||||
|
runtime.stop_worker(&worker.worker_ref, None).unwrap();
|
||||||
|
let request = runtime.test_restore_request(&worker.worker_ref);
|
||||||
|
if mode != "disabled" {
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||||
|
let endpoint = listener.local_addr().unwrap();
|
||||||
|
drop(listener);
|
||||||
|
runtime
|
||||||
|
.install_test_restore_checkpoint(RestoreCheckpointConfig {
|
||||||
|
endpoint,
|
||||||
|
worker_id: if mode == "mismatch" {
|
||||||
|
WorkerId::now_v7()
|
||||||
|
} else {
|
||||||
|
worker.worker_id
|
||||||
|
},
|
||||||
|
stage: RestoreCheckpointStage::PreparedNotCommitted,
|
||||||
|
nonce: "absent-test".into(),
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
let result = runtime
|
||||||
|
.restore_worker_operation(&worker.worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(result.state, WorkerRestoreState::Accepted, "{mode}");
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.restore_worker_operation(&worker.worker_ref, request)
|
||||||
|
.unwrap(),
|
||||||
|
result
|
||||||
|
);
|
||||||
|
assert_eq!(*backend.restore_count.lock().unwrap(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
runtime.lock().unwrap().restore_checkpoint.is_some(),
|
||||||
|
mode == "mismatch"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn checkpoint_verification_requires_canonical_worker_operation_commit_and_absent_receipt() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let root = dir.path().join("runtime");
|
||||||
|
let backend = Arc::new(TestExecutionBackend::default());
|
||||||
|
let runtime = Runtime::with_fs_store_and_execution_backend(
|
||||||
|
FsRuntimeStoreOptions::new(&root).with_runtime_id("checkpoint-runtime"),
|
||||||
|
backend.clone(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
runtime.bind_runtime_identity("checkpoint-runtime").unwrap();
|
||||||
|
runtime.store_config_bundle(test_bundle()).unwrap();
|
||||||
|
let worker = runtime
|
||||||
|
.create_worker(task_request("checkpoint validation"))
|
||||||
|
.unwrap();
|
||||||
|
runtime.stop_worker(&worker.worker_ref, None).unwrap();
|
||||||
|
let request = runtime.test_restore_request(&worker.worker_ref);
|
||||||
|
*backend.restore_result.lock().unwrap() = Some(WorkerExecutionSpawnResult::Errored(
|
||||||
|
WorkerExecutionResult::errored(WorkerExecutionOperation::Restore, "not yet connected"),
|
||||||
|
));
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.restore_worker_operation(&worker.worker_ref, request.clone())
|
||||||
|
.unwrap()
|
||||||
|
.state,
|
||||||
|
WorkerRestoreState::ReconciliationRequired
|
||||||
|
);
|
||||||
|
let operation = backend.restore_operation_ids.lock().unwrap()[0];
|
||||||
|
let mut config = RestoreCheckpointConfig {
|
||||||
|
endpoint: "127.0.0.1:1234".parse().unwrap(),
|
||||||
|
worker_id: worker.worker_id,
|
||||||
|
stage: RestoreCheckpointStage::PreparedNotCommitted,
|
||||||
|
nonce: "verification-test".into(),
|
||||||
|
};
|
||||||
|
let path = root
|
||||||
|
.join("workers")
|
||||||
|
.join(worker.worker_id.to_string())
|
||||||
|
.join("worker.json");
|
||||||
|
let before = std::fs::read(&path).unwrap();
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.restore_checkpoint_ack(&config, &worker.worker_ref, operation)
|
||||||
|
.is_ok()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.restore_checkpoint_ack(
|
||||||
|
&config,
|
||||||
|
&worker.worker_ref,
|
||||||
|
WorkerLifecycleOperationId::new()
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
config.worker_id = WorkerId::now_v7();
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.restore_checkpoint_ack(&config, &worker.worker_ref, operation)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
config.worker_id = worker.worker_id;
|
||||||
|
config.stage = RestoreCheckpointStage::CommittedPreterminal;
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.restore_checkpoint_ack(&config, &worker.worker_ref, operation)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
config.stage = RestoreCheckpointStage::ActivatedPreterminal;
|
||||||
|
assert!(runtime.restore_checkpoint_ack(&config, &worker.worker_ref, operation).is_err());
|
||||||
|
config.stage = RestoreCheckpointStage::PreparedNotCommitted;
|
||||||
|
runtime_store(&runtime).fail_next_worker_reconcile_sync();
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.restore_checkpoint_ack(&config, &worker.worker_ref, operation)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read(&path).unwrap(),
|
||||||
|
before,
|
||||||
|
"verification must not write authority"
|
||||||
|
);
|
||||||
|
// In-memory equality must not hide an unreadable canonical aggregate.
|
||||||
|
std::fs::write(&path, b"unreadable canonical aggregate").unwrap();
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.restore_checkpoint_ack(&config, &worker.worker_ref, operation)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read(&path).unwrap(),
|
||||||
|
b"unreadable canonical aggregate"
|
||||||
|
);
|
||||||
|
std::fs::write(&path, before).unwrap();
|
||||||
|
backend.restore_result.lock().unwrap().take();
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.restore_worker_operation(&worker.worker_ref, request)
|
||||||
|
.unwrap()
|
||||||
|
.state,
|
||||||
|
WorkerRestoreState::Accepted
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.restore_checkpoint_ack(&config, &worker.worker_ref, operation)
|
||||||
|
.is_err(),
|
||||||
|
"terminal operation cannot ack a preterminal checkpoint"
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,522 @@
|
|||||||
|
//! Runtime-owned continuation of an admitted restore, independent of its caller.
|
||||||
|
//!
|
||||||
|
//! Completion callbacks only signal a condition variable. They never acquire a
|
||||||
|
//! Runtime/lifecycle lock and are never joined by this driver while holding one.
|
||||||
|
//! Each step revalidates the canonical owner under the existing Worker lock;
|
||||||
|
//! waiting/backoff and driver teardown happen outside it. A repeated obstruction
|
||||||
|
//! parks this owner with its reason, rather than spinning or inventing rollback.
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
use crate::execution::RestoreRecoveryDisposition;
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::sync::{Condvar, Weak};
|
||||||
|
use std::thread::JoinHandle;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
type Key = (WorkerRef, WorkerLifecycleOperationId);
|
||||||
|
|
||||||
|
#[derive(Debug, Default)]
|
||||||
|
pub(super) struct RestoreContinuations {
|
||||||
|
owners: Mutex<HashMap<Key, Arc<Continuation>>>,
|
||||||
|
#[cfg(test)]
|
||||||
|
paused: std::sync::atomic::AtomicBool,
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(super) terminal_receipt_gate:
|
||||||
|
Mutex<Option<crate::execution::RestoreTestGate<WorkerLifecycleOperationId>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Default)]
|
||||||
|
struct Schedule {
|
||||||
|
signaled: bool,
|
||||||
|
cancelled: bool,
|
||||||
|
finished: bool,
|
||||||
|
blocked_reason: Option<String>,
|
||||||
|
parked: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Default)]
|
||||||
|
struct Continuation {
|
||||||
|
schedule: Mutex<Schedule>,
|
||||||
|
wake: Condvar,
|
||||||
|
task: Mutex<Option<JoinHandle<()>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RestoreContinuations {
|
||||||
|
pub(super) fn signal(&self, worker_ref: &WorkerRef, operation: WorkerLifecycleOperationId) {
|
||||||
|
if let Some(owner) = self
|
||||||
|
.owners
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||||
|
.get(&(worker_ref.clone(), operation))
|
||||||
|
{
|
||||||
|
let mut schedule = owner
|
||||||
|
.schedule
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||||
|
schedule.signaled = true;
|
||||||
|
owner.wake.notify_all();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for RestoreContinuations {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
let owners = self
|
||||||
|
.owners
|
||||||
|
.get_mut()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||||
|
for owner in owners.values() {
|
||||||
|
let mut schedule = owner
|
||||||
|
.schedule
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||||
|
schedule.cancelled = true;
|
||||||
|
owner.wake.notify_all();
|
||||||
|
}
|
||||||
|
for owner in owners.values() {
|
||||||
|
if let Some(task) = owner
|
||||||
|
.task
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||||
|
.take()
|
||||||
|
{
|
||||||
|
if task.thread().id() != std::thread::current().id() {
|
||||||
|
let _ = task.join();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) struct ContinuationPause(Arc<RestoreContinuations>);
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
impl Drop for ContinuationPause {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
self.0
|
||||||
|
.paused
|
||||||
|
.store(false, std::sync::atomic::Ordering::Release);
|
||||||
|
for owner in self.0.owners.lock().unwrap().values() {
|
||||||
|
owner.schedule.lock().unwrap().signaled = true;
|
||||||
|
owner.wake.notify_all();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
enum Step {
|
||||||
|
Finished,
|
||||||
|
Waiting,
|
||||||
|
Contended,
|
||||||
|
Retry(String),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Runtime {
|
||||||
|
pub(super) fn restore_repair_authorizer(
|
||||||
|
&self,
|
||||||
|
worker_ref: WorkerRef,
|
||||||
|
) -> Arc<dyn Fn(WorkerLifecycleOperationId) -> Result<(), RuntimeError> + Send + Sync> {
|
||||||
|
let inner = Arc::downgrade(&self.inner);
|
||||||
|
Arc::new(move |operation| {
|
||||||
|
let inner = inner.upgrade().ok_or(RuntimeError::RuntimeStopped)?;
|
||||||
|
// The factory caller owns the lifecycle lock. Never reacquire it
|
||||||
|
// here: it may be awaiting this exact factory task.
|
||||||
|
let mut state = inner.lock().map_err(|_| RuntimeError::StatePoisoned)?;
|
||||||
|
state.ensure_running()?;
|
||||||
|
state.refresh_uncertain_worker_write(&worker_ref)?;
|
||||||
|
let worker = state.worker(&worker_ref)?;
|
||||||
|
state.ensure_subject_not_reserved(&worker_ref, false)?;
|
||||||
|
if worker.pending_stop.is_some()
|
||||||
|
|| worker.pending_restore.is_none_or(|pending| {
|
||||||
|
pending.operation_id != operation
|
||||||
|
|| pending.commit == RestoreCommitFact::LegacyUnknown
|
||||||
|
})
|
||||||
|
{
|
||||||
|
return Err(RuntimeError::RestoreObservationConflict {
|
||||||
|
worker_id: worker_ref.worker_id,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if let Some(owner) = worker
|
||||||
|
.restore_guard
|
||||||
|
.active_request_id
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|id| worker.restore_guard.owners.get(id))
|
||||||
|
{
|
||||||
|
owner.validate_workdir_preparation_authority(worker)?;
|
||||||
|
}
|
||||||
|
if worker
|
||||||
|
.pending_restore
|
||||||
|
.is_some_and(|pending| pending.source_repair_possible)
|
||||||
|
{
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let mut candidate = worker.clone();
|
||||||
|
candidate
|
||||||
|
.pending_restore
|
||||||
|
.as_mut()
|
||||||
|
.expect("validated restore owner")
|
||||||
|
.source_repair_possible = true;
|
||||||
|
// Grant before touching another canonical store. Unknown durability
|
||||||
|
// denies repair until authoritative reconciliation; no old-memory write.
|
||||||
|
state.persist_restore_worker_record(&candidate)?;
|
||||||
|
state.workers.insert(worker_ref.worker_id, candidate);
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) fn test_wait_restore_continuation_parked(
|
||||||
|
&self,
|
||||||
|
worker_ref: &WorkerRef,
|
||||||
|
operation: WorkerLifecycleOperationId,
|
||||||
|
timeout: Duration,
|
||||||
|
) -> String {
|
||||||
|
let owner = self
|
||||||
|
.restore_continuations
|
||||||
|
.owners
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.get(&(worker_ref.clone(), operation))
|
||||||
|
.cloned()
|
||||||
|
.unwrap();
|
||||||
|
let (schedule, waited) = owner
|
||||||
|
.wake
|
||||||
|
.wait_timeout_while(owner.schedule.lock().unwrap(), timeout, |s| {
|
||||||
|
!s.parked && !s.finished
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
!waited.timed_out() && schedule.parked,
|
||||||
|
"continuation did not park: {:?}",
|
||||||
|
schedule.blocked_reason
|
||||||
|
);
|
||||||
|
schedule.blocked_reason.clone().expect("parked reason")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Gate autonomous steps when a component test observes a single fault
|
||||||
|
/// boundary. Dedicated continuation tests run without this gate.
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) fn test_pause_restore_continuation(&self) -> ContinuationPause {
|
||||||
|
assert!(
|
||||||
|
!self
|
||||||
|
.restore_continuations
|
||||||
|
.paused
|
||||||
|
.swap(true, std::sync::atomic::Ordering::AcqRel)
|
||||||
|
);
|
||||||
|
ContinuationPause(self.restore_continuations.clone())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) fn test_wait_restore_continuation(
|
||||||
|
&self,
|
||||||
|
worker_ref: &WorkerRef,
|
||||||
|
operation: WorkerLifecycleOperationId,
|
||||||
|
timeout: Duration,
|
||||||
|
) {
|
||||||
|
let owner = self
|
||||||
|
.restore_continuations
|
||||||
|
.owners
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.get(&(worker_ref.clone(), operation))
|
||||||
|
.cloned()
|
||||||
|
.expect("continuation owner");
|
||||||
|
let schedule = owner.schedule.lock().unwrap();
|
||||||
|
let (schedule, waited) = owner
|
||||||
|
.wake
|
||||||
|
.wait_timeout_while(schedule, timeout, |s| !s.finished && !s.cancelled)
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
!waited.timed_out() && schedule.finished,
|
||||||
|
"restore continuation did not settle: {:?}",
|
||||||
|
schedule.blocked_reason
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn ensure_restore_continuation(
|
||||||
|
&self,
|
||||||
|
worker_ref: &WorkerRef,
|
||||||
|
) -> Result<(), RuntimeError> {
|
||||||
|
let operation = {
|
||||||
|
let state = self.lock()?;
|
||||||
|
let worker = state.worker(worker_ref)?;
|
||||||
|
worker
|
||||||
|
.pending_restore
|
||||||
|
.map(|pending| pending.operation_id)
|
||||||
|
.or_else(|| {
|
||||||
|
worker
|
||||||
|
.restore_guard
|
||||||
|
.active_request_id
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|id| worker.restore_guard.owners.get(id))
|
||||||
|
.filter(|owner| owner.receipt.is_none())
|
||||||
|
.map(|owner| owner.operation_id)
|
||||||
|
})
|
||||||
|
};
|
||||||
|
if let Some(operation) = operation {
|
||||||
|
self.start_restore_continuation(worker_ref.clone(), operation)?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn start_restore_continuation(
|
||||||
|
&self,
|
||||||
|
worker_ref: WorkerRef,
|
||||||
|
operation: WorkerLifecycleOperationId,
|
||||||
|
) -> Result<(), RuntimeError> {
|
||||||
|
{
|
||||||
|
let state = self.lock()?;
|
||||||
|
let Some(backend) = &state.execution_backend else {
|
||||||
|
return Ok(());
|
||||||
|
};
|
||||||
|
if !backend.supports_restore_continuation()
|
||||||
|
|| backend
|
||||||
|
.restore_recovery_disposition(operation, &worker_ref)
|
||||||
|
.is_none()
|
||||||
|
{
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let key = (worker_ref.clone(), operation);
|
||||||
|
let mut owners = self
|
||||||
|
.restore_continuations
|
||||||
|
.owners
|
||||||
|
.lock()
|
||||||
|
.map_err(|_| RuntimeError::StatePoisoned)?;
|
||||||
|
if owners.contains_key(&key) {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let owner = Arc::new(Continuation::default());
|
||||||
|
let retained = owner.clone();
|
||||||
|
let state = Arc::downgrade(&self.inner);
|
||||||
|
let operations = Arc::downgrade(&self.worker_operations);
|
||||||
|
let continuations = Arc::downgrade(&self.restore_continuations);
|
||||||
|
let task = std::thread::Builder::new()
|
||||||
|
.name("restore-continuation".into())
|
||||||
|
.spawn(move || {
|
||||||
|
run(
|
||||||
|
state,
|
||||||
|
operations,
|
||||||
|
continuations,
|
||||||
|
retained,
|
||||||
|
worker_ref,
|
||||||
|
operation,
|
||||||
|
);
|
||||||
|
})
|
||||||
|
.map_err(|error| {
|
||||||
|
RuntimeError::InvalidRequest(format!(
|
||||||
|
"Restore continuation could not start: {error}"
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
*owner.task.lock().map_err(|_| RuntimeError::StatePoisoned)? = Some(task);
|
||||||
|
owners.insert(key, owner);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn continue_restore_operation(
|
||||||
|
&self,
|
||||||
|
worker_ref: &WorkerRef,
|
||||||
|
operation: WorkerLifecycleOperationId,
|
||||||
|
) -> Result<Step, RuntimeError> {
|
||||||
|
#[cfg(test)]
|
||||||
|
if self
|
||||||
|
.restore_continuations
|
||||||
|
.paused
|
||||||
|
.load(std::sync::atomic::Ordering::Acquire)
|
||||||
|
{
|
||||||
|
return Ok(Step::Contended);
|
||||||
|
}
|
||||||
|
let lease = self.worker_operation_lock(worker_ref.worker_id)?;
|
||||||
|
// Another caller may be waiting for a backend task. Do not queue a join
|
||||||
|
// of that caller/driver while retaining the lock it needs to complete.
|
||||||
|
let _guard = match lease.try_lock() {
|
||||||
|
Ok(guard) => guard,
|
||||||
|
Err(std::sync::TryLockError::WouldBlock) => return Ok(Step::Contended),
|
||||||
|
Err(_) => return Err(RuntimeError::StatePoisoned),
|
||||||
|
};
|
||||||
|
let stop_is_owned = {
|
||||||
|
let mut state = self.lock()?;
|
||||||
|
state.ensure_running()?;
|
||||||
|
state.refresh_uncertain_worker_write(worker_ref)?;
|
||||||
|
let worker = state.worker(worker_ref)?;
|
||||||
|
worker.pending_stop.is_some()
|
||||||
|
&& worker
|
||||||
|
.pending_restore
|
||||||
|
.is_some_and(|pending| pending.operation_id == operation)
|
||||||
|
};
|
||||||
|
if stop_is_owned {
|
||||||
|
// Stop supersedes publication, not ownership of late resources.
|
||||||
|
// Retain this same continuation until its cleanup is confirmed.
|
||||||
|
self.reconcile_pending_stop_under_lock(worker_ref)?;
|
||||||
|
return Ok(Step::Finished);
|
||||||
|
}
|
||||||
|
let (backend, pending) = {
|
||||||
|
let mut state = self.lock()?;
|
||||||
|
state.ensure_running()?;
|
||||||
|
state.refresh_uncertain_worker_write(worker_ref)?;
|
||||||
|
let worker = state.worker(worker_ref)?;
|
||||||
|
if worker.pending_stop.is_some() {
|
||||||
|
return Ok(Step::Finished);
|
||||||
|
}
|
||||||
|
let owner = worker
|
||||||
|
.restore_guard
|
||||||
|
.active_request_id
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|id| worker.restore_guard.owners.get(id));
|
||||||
|
if owner.is_some_and(|owner| owner.operation_id != operation || owner.receipt.is_some())
|
||||||
|
{
|
||||||
|
return Ok(Step::Finished);
|
||||||
|
}
|
||||||
|
let Some(pending) = worker.pending_restore else {
|
||||||
|
return Ok(Step::Finished);
|
||||||
|
};
|
||||||
|
if pending.operation_id != operation {
|
||||||
|
return Ok(Step::Finished);
|
||||||
|
}
|
||||||
|
if pending.commit == RestoreCommitFact::LegacyUnknown {
|
||||||
|
return Ok(Step::Retry(
|
||||||
|
"Saved Restore has no authoritative commit evidence".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
state.ensure_subject_not_reserved(worker_ref, false)?;
|
||||||
|
if let Some(owner) = owner {
|
||||||
|
owner.validate_workdir_preparation_authority(worker)?;
|
||||||
|
}
|
||||||
|
let Some(backend) = state.execution_backend.clone() else {
|
||||||
|
return Ok(Step::Finished);
|
||||||
|
};
|
||||||
|
(backend, pending)
|
||||||
|
};
|
||||||
|
match backend.restore_recovery_disposition(operation, worker_ref) {
|
||||||
|
None => return Ok(Step::Finished),
|
||||||
|
Some(RestoreRecoveryDisposition::InFlight) => return Ok(Step::Waiting),
|
||||||
|
Some(RestoreRecoveryDisposition::Discard)
|
||||||
|
if pending.commit == RestoreCommitFact::NotCommitted =>
|
||||||
|
{
|
||||||
|
match backend.discard_uncommitted_restore(operation, worker_ref) {
|
||||||
|
WorkerExecutionSpawnResult::RolledBack(result) => {
|
||||||
|
let receipt = RuntimeWorkerRestoreResult::failed(
|
||||||
|
WorkerRestoreState::RolledBack,
|
||||||
|
"worker_restore_rolled_back",
|
||||||
|
result.message_or_default(),
|
||||||
|
);
|
||||||
|
#[cfg(feature = "fs-store")]
|
||||||
|
self.lock()?
|
||||||
|
.record_restore_failure(worker_ref, &result, &receipt)?;
|
||||||
|
#[cfg(not(feature = "fs-store"))]
|
||||||
|
self.settle_restore_rollback(worker_ref, &receipt)?;
|
||||||
|
return Ok(Step::Finished);
|
||||||
|
}
|
||||||
|
result => {
|
||||||
|
return Ok(Step::Retry(format!(
|
||||||
|
"Restore cleanup remains unresolved: {result:?}"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Some(_) => {}
|
||||||
|
}
|
||||||
|
let result = self.restore_admitted_worker_under_lock(worker_ref, pending.mode)?;
|
||||||
|
if backend.restore_recovery_disposition(operation, worker_ref)
|
||||||
|
== Some(RestoreRecoveryDisposition::InFlight)
|
||||||
|
{
|
||||||
|
return Ok(Step::Waiting);
|
||||||
|
}
|
||||||
|
if result.state == WorkerRestoreState::ReconciliationRequired {
|
||||||
|
Ok(Step::Retry(result.message.unwrap_or_else(|| {
|
||||||
|
"Restore reconciliation remains pending".into()
|
||||||
|
})))
|
||||||
|
} else {
|
||||||
|
Ok(Step::Finished)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn run(
|
||||||
|
inner: Weak<Mutex<RuntimeState>>,
|
||||||
|
operations: Weak<Mutex<BTreeMap<WorkerId, Arc<Mutex<()>>>>>,
|
||||||
|
continuations: Weak<RestoreContinuations>,
|
||||||
|
owner: Arc<Continuation>,
|
||||||
|
worker_ref: WorkerRef,
|
||||||
|
operation: WorkerLifecycleOperationId,
|
||||||
|
) {
|
||||||
|
let mut repeated = 0u32;
|
||||||
|
let mut last_reason = None;
|
||||||
|
loop {
|
||||||
|
let (Some(inner), Some(worker_operations), Some(restore_continuations)) = (
|
||||||
|
inner.upgrade(),
|
||||||
|
operations.upgrade(),
|
||||||
|
continuations.upgrade(),
|
||||||
|
) else {
|
||||||
|
break;
|
||||||
|
};
|
||||||
|
owner
|
||||||
|
.schedule
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||||
|
.parked = false;
|
||||||
|
let runtime = Runtime {
|
||||||
|
inner,
|
||||||
|
worker_operations,
|
||||||
|
restore_continuations,
|
||||||
|
};
|
||||||
|
let step = runtime.continue_restore_operation(&worker_ref, operation);
|
||||||
|
// No lifecycle lock or strong Runtime/registry reference survives waiting.
|
||||||
|
drop(runtime);
|
||||||
|
let mut schedule = owner
|
||||||
|
.schedule
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||||
|
if schedule.cancelled {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
let contended = matches!(&step, Ok(Step::Contended));
|
||||||
|
let reason = match step {
|
||||||
|
Ok(Step::Finished) => {
|
||||||
|
schedule.finished = true;
|
||||||
|
owner.wake.notify_all();
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
Ok(Step::Waiting) | Ok(Step::Contended) => None,
|
||||||
|
Ok(Step::Retry(reason)) => Some(reason),
|
||||||
|
Err(error) => Some(sanitize_worker_create_failure_message(&error.to_string())),
|
||||||
|
};
|
||||||
|
if let Some(reason) = &reason {
|
||||||
|
repeated = repeated.saturating_add(1);
|
||||||
|
last_reason = Some(reason.clone());
|
||||||
|
schedule.blocked_reason = Some(reason.clone());
|
||||||
|
}
|
||||||
|
// Completed failed attempts can have signaled while this step joined
|
||||||
|
// them. Do not let our own unchanged failure reset the retry budget.
|
||||||
|
if reason.is_some() {
|
||||||
|
schedule.signaled = false;
|
||||||
|
}
|
||||||
|
if (!contended && reason.is_none()) || repeated >= 4 {
|
||||||
|
if repeated >= 4 {
|
||||||
|
schedule.parked = true;
|
||||||
|
owner.wake.notify_all();
|
||||||
|
tracing::warn!(worker_id = %worker_ref.worker_id, operation_id = %operation, reason = ?last_reason,
|
||||||
|
"Restore continuation parked after bounded attempts; operation remains unresolved");
|
||||||
|
}
|
||||||
|
schedule = owner
|
||||||
|
.wake
|
||||||
|
.wait_while(schedule, |s| !s.signaled && !s.cancelled)
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||||
|
} else {
|
||||||
|
let delay = Duration::from_millis(100 * (1u64 << repeated.min(4)));
|
||||||
|
schedule = owner
|
||||||
|
.wake
|
||||||
|
.wait_timeout_while(schedule, delay, |s| !s.signaled && !s.cancelled)
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||||
|
.0;
|
||||||
|
}
|
||||||
|
if schedule.cancelled {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if schedule.signaled {
|
||||||
|
schedule.signaled = false;
|
||||||
|
// A completion wakes this same owner, not a new request. Its next
|
||||||
|
// observed result, not the signal itself, determines progress.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,872 @@
|
|||||||
|
//! Offline support for the approved, attachment-free stopped Worker fixture.
|
||||||
|
//!
|
||||||
|
//! Enabled only by `test-support`. This is not an import API: extraction accepts a
|
||||||
|
//! current-schema, settled, single-Segment aggregate; installation accepts only a
|
||||||
|
//! fresh Runtime directory containing its newly initialized `auth.toml`. No
|
||||||
|
//! Runtime or execution backend is constructed. Original Session bytes are never
|
||||||
|
//! rewritten. The caller must separately review/attest the artificial conversation;
|
||||||
|
//! structural checks and secret-marker checks cannot prove arbitrary prose safe.
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
use crate::catalog::ProfileSourceArchiveSource;
|
||||||
|
use crate::workspace_issuer::{
|
||||||
|
WorkspaceIssuerTrustRecord, WorkspaceIssuerTrustState, validate_workspace_issuer_trust_records,
|
||||||
|
};
|
||||||
|
use session_store::{
|
||||||
|
SegmentId, SessionId, WorkerActiveSegmentRef, WorkerMetadata, WorkerSessionStore,
|
||||||
|
};
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "test_support_tests.rs"]
|
||||||
|
mod tests;
|
||||||
|
|
||||||
|
const TEMPLATE_VERSION: u32 = 1;
|
||||||
|
const MAX_SESSION_BYTES: usize = 8 * 1024 * 1024;
|
||||||
|
|
||||||
|
/// Portable fixture, intentionally without Debug (Session material is private).
|
||||||
|
/// Fields are private and every deserialized template is validated on install.
|
||||||
|
/// Serialize this value to retain the fixture; do not serialize its source aggregate.
|
||||||
|
#[derive(Clone, Serialize, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct StoppedWorkerTemplate {
|
||||||
|
template_version: u32,
|
||||||
|
runtime_schema_version: u32,
|
||||||
|
runtime_id: String,
|
||||||
|
workspace_id: String,
|
||||||
|
worker_id: WorkerId,
|
||||||
|
created_at_ms: Option<u64>,
|
||||||
|
session_id: SessionId,
|
||||||
|
segment_id: SegmentId,
|
||||||
|
session_manifest: Vec<u8>,
|
||||||
|
segment_jsonl: Vec<u8>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
struct SessionManifest {
|
||||||
|
schema_version: u32,
|
||||||
|
session_id: SessionId,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read-only projection: never retain source key material in a template.
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
struct AuthIdentity {
|
||||||
|
identity_id: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
struct SourceAuth {
|
||||||
|
identity: AuthIdentity,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
struct InitializedAuth {
|
||||||
|
identity: crate::auth::RuntimeIdentityMaterial,
|
||||||
|
workspace_issuers: Vec<WorkspaceIssuerTrustRecord>,
|
||||||
|
#[serde(default)]
|
||||||
|
archived_legacy_issuers: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
struct GuardState {
|
||||||
|
active_request_id: Option<String>,
|
||||||
|
owners: BTreeMap<String, GuardOwner>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
struct GuardOwner {
|
||||||
|
receipt: Option<serde::de::IgnoredAny>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reject(message: &'static str) -> RuntimeError {
|
||||||
|
RuntimeError::InvalidRequest(format!("stopped Worker fixture: {message}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn io<T>(result: std::io::Result<T>) -> Result<T, RuntimeError> {
|
||||||
|
result.map_err(|_| reject("filesystem operation failed"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn decode<T: serde::de::DeserializeOwned>(bytes: &[u8]) -> Result<T, RuntimeError> {
|
||||||
|
serde_json::from_slice(bytes).map_err(|_| reject("invalid typed document"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn no_symlink_components(path: &Path) -> Result<(), RuntimeError> {
|
||||||
|
let mut current = PathBuf::new();
|
||||||
|
for part in path.components() {
|
||||||
|
if matches!(part, std::path::Component::ParentDir) {
|
||||||
|
return Err(reject("parent traversal is unsupported"));
|
||||||
|
}
|
||||||
|
current.push(part);
|
||||||
|
let metadata = io(fs::symlink_metadata(¤t))?;
|
||||||
|
if metadata.file_type().is_symlink() {
|
||||||
|
return Err(reject("symlinks are unsupported"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn check_tree(path: &Path, budget: &mut usize) -> Result<(), RuntimeError> {
|
||||||
|
if *budget == 0 {
|
||||||
|
return Err(reject("file count exceeds fixture limit"));
|
||||||
|
}
|
||||||
|
*budget -= 1;
|
||||||
|
let metadata = io(fs::symlink_metadata(path))?;
|
||||||
|
if metadata.is_dir() {
|
||||||
|
for entry in io(fs::read_dir(path))? {
|
||||||
|
check_tree(&io(entry)?.path(), budget)?;
|
||||||
|
}
|
||||||
|
} else if !metadata.is_file() {
|
||||||
|
return Err(reject("symlinks and special files are unsupported"));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn regular_file(path: &Path) -> Result<(), RuntimeError> {
|
||||||
|
no_symlink_components(path)?;
|
||||||
|
if !io(fs::symlink_metadata(path))?.is_file() {
|
||||||
|
return Err(reject("expected a regular file"));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn bytes(path: &Path) -> Result<Vec<u8>, RuntimeError> {
|
||||||
|
regular_file(path)?;
|
||||||
|
let mut options = OpenOptions::new();
|
||||||
|
options.read(true);
|
||||||
|
#[cfg(target_os = "linux")]
|
||||||
|
{
|
||||||
|
use std::os::unix::fs::OpenOptionsExt;
|
||||||
|
options.custom_flags(libc::O_NOATIME | libc::O_NOFOLLOW);
|
||||||
|
}
|
||||||
|
let file = io(options.open(path))?;
|
||||||
|
let mut result = Vec::new();
|
||||||
|
io(file
|
||||||
|
.take(MAX_SESSION_BYTES as u64 + 1)
|
||||||
|
.read_to_end(&mut result))?;
|
||||||
|
if result.len() > MAX_SESSION_BYTES {
|
||||||
|
return Err(reject("file exceeds fixture limit"));
|
||||||
|
}
|
||||||
|
Ok(result)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn text(path: &Path) -> Result<String, RuntimeError> {
|
||||||
|
String::from_utf8(bytes(path)?).map_err(|_| reject("invalid UTF-8 document"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn portable_id(id: &str) -> bool {
|
||||||
|
!id.is_empty()
|
||||||
|
&& id.len() <= 256
|
||||||
|
&& id
|
||||||
|
.bytes()
|
||||||
|
.all(|c| c.is_ascii_alphanumeric() || b"-_".contains(&c))
|
||||||
|
}
|
||||||
|
|
||||||
|
impl StoppedWorkerTemplate {
|
||||||
|
pub fn runtime_id(&self) -> &str {
|
||||||
|
&self.runtime_id
|
||||||
|
}
|
||||||
|
pub fn workspace_id(&self) -> &str {
|
||||||
|
&self.workspace_id
|
||||||
|
}
|
||||||
|
pub fn worker_id(&self) -> WorkerId {
|
||||||
|
self.worker_id
|
||||||
|
}
|
||||||
|
pub fn worker_name(&self) -> String {
|
||||||
|
runtime_worker_name(self.worker_id)
|
||||||
|
}
|
||||||
|
pub fn session_id(&self) -> SessionId {
|
||||||
|
self.session_id
|
||||||
|
}
|
||||||
|
pub fn segment_id(&self) -> SegmentId {
|
||||||
|
self.segment_id
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Extract without migration, rewriting metadata, or copying credentials,
|
||||||
|
/// resolved manifests, execution-operation receipts, or run directories.
|
||||||
|
/// The source Runtime must be offline; its existing owner lock is held for
|
||||||
|
/// the entire read. The path must be `<runtime>/workers/<WorkerId>`.
|
||||||
|
pub fn extract(worker_dir: &Path) -> Result<Self, RuntimeError> {
|
||||||
|
no_symlink_components(worker_dir)?;
|
||||||
|
let workers = worker_dir
|
||||||
|
.parent()
|
||||||
|
.ok_or_else(|| reject("invalid aggregate path"))?;
|
||||||
|
if workers.file_name().and_then(|s| s.to_str()) != Some(WORKERS_DIR) {
|
||||||
|
return Err(reject("invalid aggregate path"));
|
||||||
|
}
|
||||||
|
let root = workers
|
||||||
|
.parent()
|
||||||
|
.ok_or_else(|| reject("invalid Runtime path"))?;
|
||||||
|
// Require the existing lock, rather than creating a file in the source.
|
||||||
|
let lock_path = root.with_file_name(format!(
|
||||||
|
".{}.runtime-owner.lock",
|
||||||
|
root.file_name()
|
||||||
|
.and_then(|s| s.to_str())
|
||||||
|
.ok_or_else(|| reject("invalid Runtime path"))?
|
||||||
|
));
|
||||||
|
regular_file(&lock_path)?;
|
||||||
|
let lock = io(OpenOptions::new().read(true).open(lock_path))?;
|
||||||
|
if !io(FileExt::try_lock_shared(&lock))? {
|
||||||
|
return Err(reject("source Runtime is not offline"));
|
||||||
|
}
|
||||||
|
let _lock = RuntimeStoreOwnerLock { file: lock };
|
||||||
|
check_tree(worker_dir, &mut 256)?;
|
||||||
|
let snapshot: RuntimeSnapshot = decode(&bytes(&root.join(RUNTIME_FILE))?)?;
|
||||||
|
snapshot
|
||||||
|
.validate(&root.join(RUNTIME_FILE))
|
||||||
|
.map_err(|_| reject("unsupported Runtime schema"))?;
|
||||||
|
if !snapshot.subject_create_recovery_fences.is_empty()
|
||||||
|
|| !snapshot.subject_releases.is_empty()
|
||||||
|
{
|
||||||
|
return Err(reject("source has lifecycle operations"));
|
||||||
|
}
|
||||||
|
let record: WorkerAggregateRecord = decode(&bytes(&worker_dir.join(WORKER_FILE))?)?;
|
||||||
|
let record = record
|
||||||
|
.validate(&worker_dir.join(WORKER_FILE))
|
||||||
|
.map_err(|_| reject("invalid Worker schema or identity"))?;
|
||||||
|
let guard: GuardState = decode(
|
||||||
|
&serde_json::to_vec(&record.restore_guard)
|
||||||
|
.map_err(|_| reject("invalid restore guard"))?,
|
||||||
|
)?;
|
||||||
|
if record.status != WorkerStatus::Stopped
|
||||||
|
|| guard.active_request_id.is_some()
|
||||||
|
|| guard.owners.values().any(|owner| owner.receipt.is_none())
|
||||||
|
{
|
||||||
|
return Err(reject("Worker is not settled and stopped"));
|
||||||
|
}
|
||||||
|
let PersistedWorkerExecutionState::Available(execution) = &record.execution_state else {
|
||||||
|
return Err(reject("pending or unavailable execution metadata"));
|
||||||
|
};
|
||||||
|
// A stopped aggregate may retain the empty historical binding marker.
|
||||||
|
// It is deliberately not copied to the installed Worker.
|
||||||
|
if execution.restore_intent != WorkerRestoreIntent::Explicit {
|
||||||
|
return Err(reject("Worker requires explicit restore"));
|
||||||
|
}
|
||||||
|
if !record.workdir_attachments.is_empty() || !record.logical_workdir_attachments.is_empty()
|
||||||
|
{
|
||||||
|
return Err(reject("attachments are unsupported"));
|
||||||
|
}
|
||||||
|
check_request(&execution.request, false)?;
|
||||||
|
let workspace_id = record
|
||||||
|
.workspace_id
|
||||||
|
.ok_or_else(|| reject("missing Workspace identity"))?;
|
||||||
|
let metadata: WorkerMetadata = decode(&bytes(&worker_dir.join(WORKER_METADATA_FILE))?)?;
|
||||||
|
crate::worker_backend::validate_fixture_restore_metadata(&metadata, &execution.request)
|
||||||
|
.map_err(|_| reject("invalid source restore Manifest or settings"))?;
|
||||||
|
if metadata.worker_name != runtime_worker_name(record.worker_id)
|
||||||
|
|| metadata.workspace_id.as_deref() != Some(&workspace_id)
|
||||||
|
|| !metadata.spawned_children.is_empty()
|
||||||
|
|| !metadata.reclaimed_children.is_empty()
|
||||||
|
|| !metadata.peers.is_empty()
|
||||||
|
|| metadata.subjektiv_session_attribution.is_some()
|
||||||
|
{
|
||||||
|
return Err(reject("unsupported or inconsistent Worker metadata"));
|
||||||
|
}
|
||||||
|
let active = metadata
|
||||||
|
.active
|
||||||
|
.ok_or_else(|| reject("missing active Session"))?;
|
||||||
|
let segment_id = active
|
||||||
|
.segment_id
|
||||||
|
.ok_or_else(|| reject("unmaterialized Session"))?;
|
||||||
|
let source_auth: SourceAuth = toml::from_str(&text(&root.join("auth.toml"))?)
|
||||||
|
.map_err(|_| reject("missing Runtime identity"))?;
|
||||||
|
let session = worker_dir.join("session");
|
||||||
|
let fixture = Self {
|
||||||
|
template_version: TEMPLATE_VERSION,
|
||||||
|
runtime_schema_version: SCHEMA_VERSION,
|
||||||
|
runtime_id: source_auth.identity.identity_id,
|
||||||
|
workspace_id,
|
||||||
|
worker_id: record.worker_id,
|
||||||
|
created_at_ms: record.created_at_ms,
|
||||||
|
session_id: active.session_id,
|
||||||
|
segment_id,
|
||||||
|
session_manifest: bytes(&session.join("session.json"))?,
|
||||||
|
segment_jsonl: bytes(&session.join("segments").join(format!("{segment_id}.jsonl")))?,
|
||||||
|
};
|
||||||
|
fixture.validate_shape()?;
|
||||||
|
fixture.validate_session(&session)?;
|
||||||
|
Ok(fixture)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Install after normal identity/trust initialization, before Runtime start.
|
||||||
|
/// `request` and `manifest` supply NEW backend/config/profile references and
|
||||||
|
/// freshly resolved policy, not a launch. The saved source Manifest is never
|
||||||
|
/// reused. `manifest.worker.name` must equal `worker_name()`.
|
||||||
|
/// Only `auth.toml` may already exist. The owner lock fences live Runtimes;
|
||||||
|
/// staged files are synced and published without overwriting any Worker.
|
||||||
|
/// An interrupted publication fails closed as an incomplete Runtime layout.
|
||||||
|
pub fn install(
|
||||||
|
&self,
|
||||||
|
options: &FsRuntimeStoreOptions,
|
||||||
|
request: CreateWorkerRequest,
|
||||||
|
manifest: &manifest::WorkerManifest,
|
||||||
|
) -> Result<(), RuntimeError> {
|
||||||
|
self.validate_shape()?;
|
||||||
|
check_request(&request, true)?;
|
||||||
|
if manifest.worker.name != self.worker_name() {
|
||||||
|
return Err(reject("resolved Manifest Worker name mismatch"));
|
||||||
|
}
|
||||||
|
let mut metadata = WorkerMetadata::new(
|
||||||
|
self.worker_name(),
|
||||||
|
Some(WorkerActiveSegmentRef {
|
||||||
|
session_id: self.session_id,
|
||||||
|
segment_id: Some(self.segment_id),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.with_workspace_id(&self.workspace_id);
|
||||||
|
metadata.resolved_manifest_snapshot = Some(
|
||||||
|
manifest::write_persisted_worker_manifest_snapshot(manifest)
|
||||||
|
.map_err(|_| reject("invalid resolved Manifest"))?,
|
||||||
|
);
|
||||||
|
crate::worker_backend::validate_fixture_restore_metadata(&metadata, &request)
|
||||||
|
.map_err(|_| reject("resolved Manifest does not match fresh request settings"))?;
|
||||||
|
if options.runtime_id != self.runtime_id
|
||||||
|
|| request.worker_id != self.worker_id
|
||||||
|
|| request
|
||||||
|
.workspace_api
|
||||||
|
.as_ref()
|
||||||
|
.map(|api| api.workspace_id.as_str())
|
||||||
|
!= Some(&self.workspace_id)
|
||||||
|
{
|
||||||
|
return Err(reject("installation identity mismatch"));
|
||||||
|
}
|
||||||
|
no_symlink_components(&options.root)?;
|
||||||
|
check_tree(&options.root, &mut 256)?;
|
||||||
|
// Validate the lock location before the ordinary owner-lock opener.
|
||||||
|
let lock_path = options.root.with_file_name(format!(
|
||||||
|
".{}.runtime-owner.lock",
|
||||||
|
options
|
||||||
|
.root
|
||||||
|
.file_name()
|
||||||
|
.and_then(|s| s.to_str())
|
||||||
|
.ok_or_else(|| reject("invalid Runtime path"))?
|
||||||
|
));
|
||||||
|
if fs::symlink_metadata(&lock_path).is_ok() {
|
||||||
|
regular_file(&lock_path)?;
|
||||||
|
}
|
||||||
|
let lock = acquire_runtime_store_owner_lock(&options.root)?;
|
||||||
|
validate_fresh_runtime_store_layout(&options.root)
|
||||||
|
.map_err(|_| reject("destination is not fresh; refusing overwrite"))?;
|
||||||
|
let auth: InitializedAuth = toml::from_str(&text(&options.root.join("auth.toml"))?)
|
||||||
|
.map_err(|_| reject("initialize Runtime identity and trust before install"))?;
|
||||||
|
if auth.identity.identity_id != self.runtime_id
|
||||||
|
|| !auth.archived_legacy_issuers.is_empty()
|
||||||
|
|| auth.identity.signing_key().is_err()
|
||||||
|
{
|
||||||
|
return Err(reject("invalid destination Runtime identity"));
|
||||||
|
}
|
||||||
|
validate_workspace_issuer_trust_records(&auth.workspace_issuers)
|
||||||
|
.map_err(|_| reject("invalid destination Workspace trust"))?;
|
||||||
|
let api = request
|
||||||
|
.workspace_api
|
||||||
|
.as_ref()
|
||||||
|
.expect("validated Workspace binding");
|
||||||
|
if !auth.workspace_issuers.iter().any(|trust| {
|
||||||
|
trust.workspace_id == self.workspace_id
|
||||||
|
&& trust.backend_url == api.base_url
|
||||||
|
&& trust.state == WorkspaceIssuerTrustState::Active
|
||||||
|
}) {
|
||||||
|
return Err(reject("request does not match initialized Workspace trust"));
|
||||||
|
}
|
||||||
|
let stage = options
|
||||||
|
.root
|
||||||
|
.join(format!(".fixture-{}", uuid::Uuid::now_v7()));
|
||||||
|
io(fs::create_dir(&stage))?;
|
||||||
|
let store = FsRuntimeStore {
|
||||||
|
root: stage.clone(),
|
||||||
|
_owner_lock: Some(lock),
|
||||||
|
worker_write_fault: Arc::new(AtomicU8::new(AtomicWriteFault::None as u8)),
|
||||||
|
#[cfg(test)]
|
||||||
|
worker_reconcile_sync_fault: Default::default(),
|
||||||
|
#[cfg(test)]
|
||||||
|
worker_reconcile_read_fault: Default::default(),
|
||||||
|
#[cfg(test)]
|
||||||
|
worker_reconcile_counts: Default::default(),
|
||||||
|
};
|
||||||
|
let result = (|| {
|
||||||
|
let worker = PersistedWorkerRecord {
|
||||||
|
restore_guard: crate::runtime::RestoreGuard::default(),
|
||||||
|
worker_ref: WorkerRef::new(self.worker_id),
|
||||||
|
worker_id: self.worker_id,
|
||||||
|
profile: request.profile.clone(),
|
||||||
|
display_name: request.display_name.clone(),
|
||||||
|
profile_source: request.profile_source.reference(),
|
||||||
|
config_bundle: request.config_bundle.clone(),
|
||||||
|
created_at_ms: self.created_at_ms,
|
||||||
|
status: WorkerStatus::Stopped,
|
||||||
|
last_finished_submission_request_id: None,
|
||||||
|
workspace_id: Some(self.workspace_id.clone()),
|
||||||
|
workdir_attachments: Vec::new(),
|
||||||
|
logical_workdir_attachments: Vec::new(),
|
||||||
|
execution_state: PersistedWorkerExecutionState::Available(
|
||||||
|
PersistedWorkerExecution {
|
||||||
|
request,
|
||||||
|
binding: None,
|
||||||
|
restore_intent: WorkerRestoreIntent::Explicit,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
};
|
||||||
|
WorkerAggregateRecord::from_persisted(&worker)
|
||||||
|
.validate(&store.worker_dir(&self.worker_id).join(WORKER_FILE))
|
||||||
|
.map_err(|_| reject("invalid installation request"))?;
|
||||||
|
store.write_worker_record(&worker)?;
|
||||||
|
let worker_dir = store.worker_dir(&self.worker_id);
|
||||||
|
atomic_write_json(
|
||||||
|
&worker_dir.join(WORKER_METADATA_FILE),
|
||||||
|
&metadata,
|
||||||
|
"write fixture metadata",
|
||||||
|
)?;
|
||||||
|
let session = worker_dir.join("session");
|
||||||
|
io(fs::create_dir_all(session.join("segments")))?;
|
||||||
|
write_new(&session.join("session.json"), &self.session_manifest)?;
|
||||||
|
write_new(
|
||||||
|
&session
|
||||||
|
.join("segments")
|
||||||
|
.join(format!("{}.jsonl", self.segment_id)),
|
||||||
|
&self.segment_jsonl,
|
||||||
|
)?;
|
||||||
|
self.validate_session(&session)?;
|
||||||
|
store.write_runtime_snapshot(&PersistedRuntimeState {
|
||||||
|
subject_create_recovery_fences: BTreeMap::new(),
|
||||||
|
subject_releases: Vec::new(),
|
||||||
|
// Match normal store initialization: this is catalog availability,
|
||||||
|
// not a process or Worker execution start.
|
||||||
|
display_name: options.display_name.clone(),
|
||||||
|
status: RuntimeStatus::Running,
|
||||||
|
next_diagnostic_id: 1,
|
||||||
|
workers: BTreeMap::new(),
|
||||||
|
workspace_owners: BTreeMap::new(),
|
||||||
|
diagnostics: Vec::new(),
|
||||||
|
})?;
|
||||||
|
let loaded = store.load_runtime_state()?;
|
||||||
|
if loaded.workers.len() != 1 || !loaded.diagnostics.is_empty() {
|
||||||
|
return Err(reject("staged aggregate failed ordinary store roundtrip"));
|
||||||
|
}
|
||||||
|
sync_runtime_tree(&stage)?;
|
||||||
|
// No destination existed when checked under the owner lock. Hard-link
|
||||||
|
// the snapshot last with create-new semantics; never replace a file.
|
||||||
|
// Reserve the destination catalog with create-new semantics. Even
|
||||||
|
// an empty preexisting workers directory must never be replaced.
|
||||||
|
let destination_workers = options.root.join(WORKERS_DIR);
|
||||||
|
io(fs::create_dir(&destination_workers))?;
|
||||||
|
io(fs::rename(
|
||||||
|
store.worker_dir(&self.worker_id),
|
||||||
|
destination_workers.join(self.worker_id.to_string()),
|
||||||
|
))?;
|
||||||
|
sync_directory(&destination_workers, "publish stopped Worker aggregate")?;
|
||||||
|
io(fs::hard_link(
|
||||||
|
stage.join(RUNTIME_FILE),
|
||||||
|
options.root.join(RUNTIME_FILE),
|
||||||
|
))?;
|
||||||
|
sync_directory(&options.root, "publish stopped fixture")?;
|
||||||
|
Ok(())
|
||||||
|
})();
|
||||||
|
// Only remove our private staging directory, never existing destination data.
|
||||||
|
let cleanup = fs::remove_dir_all(&stage);
|
||||||
|
result?;
|
||||||
|
io(cleanup)?;
|
||||||
|
sync_directory(&options.root, "finish stopped fixture")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_shape(&self) -> Result<(), RuntimeError> {
|
||||||
|
// Version 1 templates carry only settled Session material, never a
|
||||||
|
// lifecycle operation. Schema 12 -> 13 changes pending decision evidence,
|
||||||
|
// not that payload. Accept the fixed schema-12 artifact read-only and
|
||||||
|
// install through the current aggregate owner; never rewrite its bytes.
|
||||||
|
if self.template_version != TEMPLATE_VERSION
|
||||||
|
|| !matches!(
|
||||||
|
self.runtime_schema_version,
|
||||||
|
RESTORE_ADMISSION_SCHEMA_VERSION | SCHEMA_VERSION
|
||||||
|
)
|
||||||
|
{
|
||||||
|
return Err(reject("unsupported template schema"));
|
||||||
|
}
|
||||||
|
if !portable_id(&self.runtime_id)
|
||||||
|
|| !portable_id(&self.workspace_id)
|
||||||
|
|| self.session_id.get_version_num() != 7
|
||||||
|
|| self.segment_id.get_version_num() != 7
|
||||||
|
|| self.session_manifest.len() > MAX_SESSION_BYTES
|
||||||
|
|| self.segment_jsonl.len() > MAX_SESSION_BYTES
|
||||||
|
{
|
||||||
|
return Err(reject("invalid portable identity or size"));
|
||||||
|
}
|
||||||
|
let manifest: SessionManifest = decode(&self.session_manifest)?;
|
||||||
|
if manifest.session_id != self.session_id
|
||||||
|
|| manifest.schema_version == 0
|
||||||
|
|| !self.segment_jsonl.ends_with(b"\n")
|
||||||
|
{
|
||||||
|
return Err(reject("Session identity or committed boundary mismatch"));
|
||||||
|
}
|
||||||
|
// Read-only JSON traversal, never mutation/reconstruction of Session bytes.
|
||||||
|
// Reject known secret/host markers even in otherwise valid typed records.
|
||||||
|
for line in self
|
||||||
|
.segment_jsonl
|
||||||
|
.split(|b| *b == b'\n')
|
||||||
|
.filter(|line| !line.is_empty())
|
||||||
|
{
|
||||||
|
let _: PortableJson = decode(line)?;
|
||||||
|
validate_preserved_record(line)?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_history_entry(
|
||||||
|
&self,
|
||||||
|
entry: &session_store::LoggedHistoryEntry,
|
||||||
|
ids: &mut std::collections::BTreeSet<String>,
|
||||||
|
) -> Result<(), RuntimeError> {
|
||||||
|
if !portable_id(&entry.metadata.entry_id.0)
|
||||||
|
|| !ids.insert(entry.metadata.entry_id.0.clone())
|
||||||
|
{
|
||||||
|
return Err(reject("duplicate or invalid entry identity"));
|
||||||
|
}
|
||||||
|
if entry
|
||||||
|
.metadata
|
||||||
|
.input_metadata
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|metadata| {
|
||||||
|
metadata.feature_payload().is_some() || metadata.public_item().is_some()
|
||||||
|
})
|
||||||
|
{
|
||||||
|
// These are opaque owner-specific payloads, not part of the narrow
|
||||||
|
// artificial conversation contract. Never preserve unchecked JSON.
|
||||||
|
return Err(reject("Feature history payloads are unsupported"));
|
||||||
|
}
|
||||||
|
if !matches!(
|
||||||
|
entry.item,
|
||||||
|
session_store::LoggedItem::Message { .. }
|
||||||
|
| session_store::LoggedItem::AssistantResponseBoundary { .. }
|
||||||
|
) {
|
||||||
|
return Err(reject("only text conversation records are supported"));
|
||||||
|
}
|
||||||
|
if let session_store::LoggedSessionHistoryOrigin::ModelOutput { worker } =
|
||||||
|
&entry.metadata.origin
|
||||||
|
{
|
||||||
|
// Session producer IDs identify the original execution, not the
|
||||||
|
// Workspace catalog Worker. Preserve them; do not rebind/rewrite them.
|
||||||
|
if !portable_id(&worker.worker_id)
|
||||||
|
|| worker
|
||||||
|
.workspace_id
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|id| id != &self.workspace_id)
|
||||||
|
|| worker
|
||||||
|
.runtime_id
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|id| id != &self.runtime_id)
|
||||||
|
{
|
||||||
|
return Err(reject("Session origin identity mismatch"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_session(&self, root: &Path) -> Result<(), RuntimeError> {
|
||||||
|
for entry in io(fs::read_dir(root))? {
|
||||||
|
let entry = io(entry)?;
|
||||||
|
match entry.file_name().to_str() {
|
||||||
|
Some("session.json" | "segments") => {}
|
||||||
|
Some("artifacts") => check_empty_artifacts(&entry.path())?,
|
||||||
|
_ => return Err(reject("unsupported Session files")),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let entries = io(fs::read_dir(root.join("segments")))?.collect::<Result<Vec<_>, _>>();
|
||||||
|
let entries = io(entries)?;
|
||||||
|
if entries.len() != 1
|
||||||
|
|| entries[0].file_name() != format!("{}.jsonl", self.segment_id).as_str()
|
||||||
|
{
|
||||||
|
return Err(reject("only the fixed single-Segment fixture is supported"));
|
||||||
|
}
|
||||||
|
let store = WorkerSessionStore::open_read_only(root)
|
||||||
|
.map_err(|_| reject("unsupported Session schema"))?;
|
||||||
|
let logs = store
|
||||||
|
.read_all_read_only_bounded(self.session_id, self.segment_id, MAX_SESSION_BYTES as u64)
|
||||||
|
.map_err(|_| reject("invalid canonical Session records"))?
|
||||||
|
.0;
|
||||||
|
if !matches!(logs.first(), Some(session_store::LogEntry::AnnotatedSegmentStart {
|
||||||
|
session_id, system_prompt: Some(_), forked_from: None, compacted_from: None, ..
|
||||||
|
}) if *session_id == self.session_id)
|
||||||
|
{
|
||||||
|
return Err(reject(
|
||||||
|
"missing system prompt, invalid Segment seed, or unsupported lineage",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let mut ids = std::collections::BTreeSet::new();
|
||||||
|
for (index, log) in logs.iter().enumerate() {
|
||||||
|
use session_store::LogEntry;
|
||||||
|
match log {
|
||||||
|
LogEntry::AnnotatedSegmentStart { history, .. }
|
||||||
|
if index == 0 && history.is_empty() => {}
|
||||||
|
LogEntry::AnnotatedUserInput {
|
||||||
|
history,
|
||||||
|
extensions,
|
||||||
|
..
|
||||||
|
} => {
|
||||||
|
for extension in extensions {
|
||||||
|
validate_settled_activation(extension)?;
|
||||||
|
}
|
||||||
|
for entry in history {
|
||||||
|
self.validate_history_entry(entry, &mut ids)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
LogEntry::AnnotatedAssistantItem { entry, .. } => {
|
||||||
|
self.validate_history_entry(entry, &mut ids)?
|
||||||
|
}
|
||||||
|
LogEntry::RunCompleted {
|
||||||
|
entry_id: Some(id), ..
|
||||||
|
} => {
|
||||||
|
if !portable_id(&id.0) || !ids.insert(id.0.clone()) {
|
||||||
|
return Err(reject("duplicate or invalid entry identity"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
LogEntry::Invoke { .. } | LogEntry::LlmUsage { .. } | LogEntry::TurnEnd { .. } => {}
|
||||||
|
_ => {
|
||||||
|
return Err(reject(
|
||||||
|
"unsupported conversation record or pending operation",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let state = session_store::collect_state(&logs);
|
||||||
|
if state.last_run_interrupted || state.active_run_turn_count.is_some() {
|
||||||
|
return Err(reject("Session has pending execution"));
|
||||||
|
}
|
||||||
|
for segments in &state.user_segments {
|
||||||
|
if segments
|
||||||
|
.iter()
|
||||||
|
.any(|segment| !matches!(segment, protocol::Segment::Text { .. }))
|
||||||
|
{
|
||||||
|
return Err(reject("Session attachments are unsupported"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compare *fields*, not re-encoded bytes or scalar values (e.g. f32 JSON
|
||||||
|
// normalization). Serde's permissive/untagged/flattened compatibility readers
|
||||||
|
// can drop unknown data without surfacing it to an ignored-field callback. A
|
||||||
|
// retained field must survive the owner's typed projection at every depth.
|
||||||
|
// This intentionally accepts only canonical writer shapes: even an explicit
|
||||||
|
// empty optional field omitted by that writer is rejected, not repaired.
|
||||||
|
// The projection is validation-only; installation always writes original bytes.
|
||||||
|
fn validate_preserved_record(bytes: &[u8]) -> Result<(), RuntimeError> {
|
||||||
|
let original: serde_json::Value = decode(bytes)?;
|
||||||
|
let record: session_store::LogEntry = decode(bytes)?;
|
||||||
|
let projection =
|
||||||
|
serde_json::to_value(&record).map_err(|_| reject("invalid typed Session record"))?;
|
||||||
|
require_preserved_fields(&original, &projection)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn require_preserved_fields(
|
||||||
|
original: &serde_json::Value,
|
||||||
|
projection: &serde_json::Value,
|
||||||
|
) -> Result<(), RuntimeError> {
|
||||||
|
use serde_json::Value;
|
||||||
|
match (original, projection) {
|
||||||
|
(Value::Object(original), Value::Object(projection)) => {
|
||||||
|
for (key, value) in original {
|
||||||
|
let projected = projection
|
||||||
|
.get(key)
|
||||||
|
.ok_or_else(|| reject("unknown or noncanonical Session field"))?;
|
||||||
|
require_preserved_fields(value, projected)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
(Value::Array(original), Value::Array(projection))
|
||||||
|
if original.len() == projection.len() =>
|
||||||
|
{
|
||||||
|
for (value, projected) in original.iter().zip(projection) {
|
||||||
|
require_preserved_fields(value, projected)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
(Value::Null, Value::Null)
|
||||||
|
| (Value::Bool(_), Value::Bool(_))
|
||||||
|
| (Value::Number(_), Value::Number(_))
|
||||||
|
| (Value::String(_), Value::String(_)) => {}
|
||||||
|
_ => return Err(reject("noncanonical Session field shape")),
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Exact receipt wire fields owned by worker::PendingActivationState. Keep these
|
||||||
|
// fixture-only mirrors strict; do not change the production compatibility reader.
|
||||||
|
#[derive(Serialize, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
struct SubmissionReceipt {
|
||||||
|
submission_request_id: String,
|
||||||
|
source_namespace: String,
|
||||||
|
submission_id: String,
|
||||||
|
payload_digest: String,
|
||||||
|
disposition: protocol::SubmissionDisposition,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
struct NotificationReceipt {
|
||||||
|
notification_request_id: String,
|
||||||
|
source_namespace: String,
|
||||||
|
payload_digest: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
// The approved conversation has committed submission checkpoints in its original
|
||||||
|
// Session records. Keep their receipt bytes, but reject any resumable queue or
|
||||||
|
// Feature invocation history. This projection confers no execution authority.
|
||||||
|
fn validate_settled_activation(
|
||||||
|
extension: &session_store::SessionExtension,
|
||||||
|
) -> Result<(), RuntimeError> {
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
struct Checkpoint {
|
||||||
|
#[serde(rename = "next_activation_sequence")]
|
||||||
|
_next_activation_sequence: u64,
|
||||||
|
activating: Option<serde::de::IgnoredAny>,
|
||||||
|
activating_notification: Vec<serde::de::IgnoredAny>,
|
||||||
|
pending: Vec<serde::de::IgnoredAny>,
|
||||||
|
pending_notifications: Vec<serde::de::IgnoredAny>,
|
||||||
|
#[serde(rename = "receipts")]
|
||||||
|
_receipts: Vec<SubmissionReceipt>,
|
||||||
|
#[serde(rename = "notification_receipts")]
|
||||||
|
_notification_receipts: Vec<NotificationReceipt>,
|
||||||
|
invocation_receipts: Vec<serde::de::IgnoredAny>,
|
||||||
|
invocation_recovery_blocked: bool,
|
||||||
|
}
|
||||||
|
if extension.domain != "worker.pending_activations.v1" {
|
||||||
|
return Err(reject("unsupported Session extension"));
|
||||||
|
}
|
||||||
|
let checkpoint: Checkpoint = serde_json::from_value(extension.payload.clone())
|
||||||
|
.map_err(|_| reject("invalid Session activation checkpoint"))?;
|
||||||
|
if checkpoint.activating.is_some()
|
||||||
|
|| !checkpoint.activating_notification.is_empty()
|
||||||
|
|| !checkpoint.pending.is_empty()
|
||||||
|
|| !checkpoint.pending_notifications.is_empty()
|
||||||
|
|| !checkpoint.invocation_receipts.is_empty()
|
||||||
|
|| checkpoint.invocation_recovery_blocked
|
||||||
|
{
|
||||||
|
return Err(reject(
|
||||||
|
"Session has pending activation or invocation history",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn check_request(request: &CreateWorkerRequest, install: bool) -> Result<(), RuntimeError> {
|
||||||
|
if !request.workdir_attachment_requests.is_empty()
|
||||||
|
|| !request.workdir_attachments.is_empty()
|
||||||
|
|| request.backend_job.is_some()
|
||||||
|
|| request.subjektiv_attached
|
||||||
|
|| request.worker_observation_enabled
|
||||||
|
|| !request.worker_observation_grants.is_empty()
|
||||||
|
|| (install && request.initial_input.is_some())
|
||||||
|
{
|
||||||
|
return Err(reject(
|
||||||
|
"unsupported capabilities, attachments, or initial input",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if install
|
||||||
|
&& (request.create_fingerprint.trim().is_empty()
|
||||||
|
|| request.config_bundle.is_none()
|
||||||
|
|| !matches!(
|
||||||
|
request.profile_source,
|
||||||
|
ProfileSourceArchiveSource::WorkspaceConfig { .. }
|
||||||
|
))
|
||||||
|
{
|
||||||
|
return Err(reject(
|
||||||
|
"fresh config/profile references and create fingerprint are required",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn check_empty_artifacts(root: &Path) -> Result<(), RuntimeError> {
|
||||||
|
for entry in io(fs::read_dir(root))? {
|
||||||
|
let entry = io(entry)?;
|
||||||
|
let metadata = io(entry.metadata())?;
|
||||||
|
if metadata.is_dir() {
|
||||||
|
check_empty_artifacts(&entry.path())?;
|
||||||
|
} else if entry.file_name() != ".aggregate.lock" || metadata.len() != 0 {
|
||||||
|
return Err(reject("Session artifacts are unsupported"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write_new(path: &Path, bytes: &[u8]) -> Result<(), RuntimeError> {
|
||||||
|
let mut file = io(OpenOptions::new().write(true).create_new(true).open(path))?;
|
||||||
|
io(file.write_all(bytes))?;
|
||||||
|
io(file.sync_all())
|
||||||
|
}
|
||||||
|
|
||||||
|
// A streaming validation visitor, not a generic document available to callers.
|
||||||
|
struct PortableJson;
|
||||||
|
impl<'de> Deserialize<'de> for PortableJson {
|
||||||
|
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||||
|
struct Visitor;
|
||||||
|
impl<'de> serde::de::Visitor<'de> for Visitor {
|
||||||
|
type Value = PortableJson;
|
||||||
|
fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
f.write_str("portable fixture JSON")
|
||||||
|
}
|
||||||
|
fn visit_bool<E: serde::de::Error>(self, _: bool) -> Result<PortableJson, E> {
|
||||||
|
Ok(PortableJson)
|
||||||
|
}
|
||||||
|
fn visit_i64<E: serde::de::Error>(self, _: i64) -> Result<PortableJson, E> {
|
||||||
|
Ok(PortableJson)
|
||||||
|
}
|
||||||
|
fn visit_u64<E: serde::de::Error>(self, _: u64) -> Result<PortableJson, E> {
|
||||||
|
Ok(PortableJson)
|
||||||
|
}
|
||||||
|
fn visit_f64<E: serde::de::Error>(self, _: f64) -> Result<PortableJson, E> {
|
||||||
|
Ok(PortableJson)
|
||||||
|
}
|
||||||
|
fn visit_unit<E: serde::de::Error>(self) -> Result<PortableJson, E> {
|
||||||
|
Ok(PortableJson)
|
||||||
|
}
|
||||||
|
fn visit_str<E: serde::de::Error>(self, value: &str) -> Result<PortableJson, E> {
|
||||||
|
let lower = value.to_ascii_lowercase();
|
||||||
|
if [
|
||||||
|
"://",
|
||||||
|
"private key",
|
||||||
|
"bearer ",
|
||||||
|
"sk-",
|
||||||
|
"api_key",
|
||||||
|
"access_token",
|
||||||
|
"refresh_token",
|
||||||
|
"private_key",
|
||||||
|
"authorization",
|
||||||
|
]
|
||||||
|
.iter()
|
||||||
|
.any(|marker| lower.contains(marker))
|
||||||
|
|| value
|
||||||
|
.split_whitespace()
|
||||||
|
.any(|word| word.starts_with('/') || word.contains(":\\"))
|
||||||
|
{
|
||||||
|
return Err(E::custom("nonportable or sensitive fixture material"));
|
||||||
|
}
|
||||||
|
Ok(PortableJson)
|
||||||
|
}
|
||||||
|
fn visit_seq<A: serde::de::SeqAccess<'de>>(
|
||||||
|
self,
|
||||||
|
mut seq: A,
|
||||||
|
) -> Result<PortableJson, A::Error> {
|
||||||
|
while seq.next_element::<PortableJson>()?.is_some() {}
|
||||||
|
Ok(PortableJson)
|
||||||
|
}
|
||||||
|
fn visit_map<A: serde::de::MapAccess<'de>>(
|
||||||
|
self,
|
||||||
|
mut map: A,
|
||||||
|
) -> Result<PortableJson, A::Error> {
|
||||||
|
let mut keys = std::collections::BTreeSet::new();
|
||||||
|
while let Some(key) = map.next_key::<String>()? {
|
||||||
|
// A duplicate key can hide discarded data even inside an
|
||||||
|
// otherwise opaque extension payload. Never retain it.
|
||||||
|
if !keys.insert(key.clone()) {
|
||||||
|
return Err(serde::de::Error::custom("duplicate fixture JSON field"));
|
||||||
|
}
|
||||||
|
serde::de::Visitor::visit_str::<A::Error>(Visitor, &key)?;
|
||||||
|
map.next_value::<PortableJson>()?;
|
||||||
|
}
|
||||||
|
Ok(PortableJson)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
deserializer.deserialize_any(Visitor)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,919 @@
|
|||||||
|
use super::*;
|
||||||
|
use crate::auth::RuntimeIdentityMaterial;
|
||||||
|
use crate::catalog::{ProfileSelector, WorkspaceApiRef};
|
||||||
|
use crate::profile_archive::ProfileSourceGraphSummary;
|
||||||
|
use session_store::{LogEntry, Store};
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
|
||||||
|
const RUNTIME: &str = "fixture-runtime";
|
||||||
|
const WORKSPACE: &str = "fixture-workspace";
|
||||||
|
const NEW_URL: &str = "http://127.0.0.1:34567";
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
#[serde(untagged)]
|
||||||
|
enum ReceiptFixture<T> {
|
||||||
|
Valid(T),
|
||||||
|
Malformed(u8),
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default, Serialize)]
|
||||||
|
struct CheckpointFixture {
|
||||||
|
next_activation_sequence: u64,
|
||||||
|
activating: Option<u8>,
|
||||||
|
activating_notification: Vec<u8>,
|
||||||
|
pending: Vec<u8>,
|
||||||
|
pending_notifications: Vec<u8>,
|
||||||
|
receipts: Vec<ReceiptFixture<SubmissionReceipt>>,
|
||||||
|
notification_receipts: Vec<ReceiptFixture<NotificationReceipt>>,
|
||||||
|
invocation_receipts: Vec<u8>,
|
||||||
|
invocation_recovery_blocked: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn settled_checkpoint() -> CheckpointFixture {
|
||||||
|
CheckpointFixture {
|
||||||
|
next_activation_sequence: 2,
|
||||||
|
receipts: vec![ReceiptFixture::Valid(SubmissionReceipt {
|
||||||
|
submission_request_id: "completed-submission-request".into(),
|
||||||
|
source_namespace: "fixture-source".into(),
|
||||||
|
submission_id: "completed-submission".into(),
|
||||||
|
payload_digest: "fixture-payload-digest".into(),
|
||||||
|
disposition: protocol::SubmissionDisposition::Started,
|
||||||
|
})],
|
||||||
|
notification_receipts: vec![ReceiptFixture::Valid(NotificationReceipt {
|
||||||
|
notification_request_id: "completed-notification-request".into(),
|
||||||
|
source_namespace: "fixture-source".into(),
|
||||||
|
payload_digest: "fixture-notification-digest".into(),
|
||||||
|
})],
|
||||||
|
..Default::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn checkpoint_extension(checkpoint: &CheckpointFixture) -> session_store::SessionExtension {
|
||||||
|
session_store::SessionExtension::new(
|
||||||
|
"worker.pending_activations.v1",
|
||||||
|
serde_json::to_value(checkpoint).unwrap(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn worker_id() -> WorkerId {
|
||||||
|
"01a1657d-b762-7791-87ae-a49f28cadbce".parse().unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn resolved() -> manifest::WorkerManifest {
|
||||||
|
let mut config = manifest::WorkerManifestConfig::resolution_defaults();
|
||||||
|
config.worker.name = Some(runtime_worker_name(worker_id()));
|
||||||
|
manifest::WorkerManifest::try_from(config).unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn request() -> CreateWorkerRequest {
|
||||||
|
CreateWorkerRequest {
|
||||||
|
worker_id: worker_id(),
|
||||||
|
create_fingerprint: "new-create-fingerprint".into(),
|
||||||
|
profile: ProfileSelector::Builtin("default".into()),
|
||||||
|
display_name: Some("fixture".into()),
|
||||||
|
profile_source: ProfileSourceArchiveSource::WorkspaceConfig {
|
||||||
|
archive: ProfileSourceArchiveRef {
|
||||||
|
id: "new-profile".into(),
|
||||||
|
digest: "new-profile-digest".into(),
|
||||||
|
size_bytes: 1,
|
||||||
|
source_graph: ProfileSourceGraphSummary {
|
||||||
|
source_count: 1,
|
||||||
|
total_source_bytes: 1,
|
||||||
|
entrypoints: BTreeMap::from([("builtin:default".into(), "default.dcl".into())]),
|
||||||
|
import_count: 0,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
config_bundle: Some(ConfigBundleRef {
|
||||||
|
id: "new-config".into(),
|
||||||
|
digest: "new-config-digest".into(),
|
||||||
|
}),
|
||||||
|
workspace_api: Some(WorkspaceApiRef {
|
||||||
|
workspace_id: WORKSPACE.into(),
|
||||||
|
base_url: NEW_URL.into(),
|
||||||
|
}),
|
||||||
|
initial_input: None,
|
||||||
|
workdir_attachment_requests: Vec::new(),
|
||||||
|
workdir_attachments: Vec::new(),
|
||||||
|
worker_observation_enabled: false,
|
||||||
|
worker_observation_grants: Vec::new(),
|
||||||
|
memory_settings: Some(manifest::WorkspaceMemorySettingsSnapshot {
|
||||||
|
workspace_id: WORKSPACE.into(),
|
||||||
|
language: "en".into(),
|
||||||
|
}),
|
||||||
|
subjektiv_attached: false,
|
||||||
|
backend_job: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn initialize(root: &Path, runtime_id: &str, workspace_id: &str, url: &str) {
|
||||||
|
#[derive(Serialize)]
|
||||||
|
struct Auth {
|
||||||
|
identity: RuntimeIdentityMaterial,
|
||||||
|
workspace_issuers: Vec<WorkspaceIssuerTrustRecord>,
|
||||||
|
}
|
||||||
|
let issuer = RuntimeIdentityMaterial::generate("fixture-issuer").unwrap();
|
||||||
|
let digest = Sha256::digest(crate::auth::decode_public_key(&issuer.public_key).unwrap());
|
||||||
|
let fingerprint = format!(
|
||||||
|
"sha256:{}",
|
||||||
|
digest
|
||||||
|
.iter()
|
||||||
|
.map(|byte| format!("{byte:02x}"))
|
||||||
|
.collect::<String>()
|
||||||
|
);
|
||||||
|
let trust = WorkspaceIssuerTrustRecord::from_bundle(
|
||||||
|
server_api::WorkspacePublicIdentityBundle {
|
||||||
|
workspace_id: workspace_id.into(),
|
||||||
|
backend_url: url.into(),
|
||||||
|
key_id: "fixture-issuer".into(),
|
||||||
|
algorithm: "ed25519".into(),
|
||||||
|
public_key: issuer.public_key,
|
||||||
|
public_key_fingerprint: fingerprint,
|
||||||
|
},
|
||||||
|
"fixture-trust".into(),
|
||||||
|
100,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
fs::create_dir_all(root).unwrap();
|
||||||
|
fs::write(
|
||||||
|
root.join("auth.toml"),
|
||||||
|
toml::to_string(&Auth {
|
||||||
|
identity: RuntimeIdentityMaterial::generate(runtime_id).unwrap(),
|
||||||
|
workspace_issuers: vec![trust],
|
||||||
|
})
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn record(request: CreateWorkerRequest) -> PersistedWorkerRecord {
|
||||||
|
PersistedWorkerRecord {
|
||||||
|
restore_guard: crate::runtime::RestoreGuard::default(),
|
||||||
|
worker_ref: WorkerRef::new(request.worker_id),
|
||||||
|
worker_id: request.worker_id,
|
||||||
|
profile: request.profile.clone(),
|
||||||
|
display_name: request.display_name.clone(),
|
||||||
|
profile_source: request.profile_source.reference(),
|
||||||
|
config_bundle: request.config_bundle.clone(),
|
||||||
|
created_at_ms: Some(123456),
|
||||||
|
status: WorkerStatus::Stopped,
|
||||||
|
last_finished_submission_request_id: Some("old-submission-operation".into()),
|
||||||
|
workspace_id: Some(WORKSPACE.into()),
|
||||||
|
workdir_attachments: vec![],
|
||||||
|
logical_workdir_attachments: vec![],
|
||||||
|
execution_state: PersistedWorkerExecutionState::Available(PersistedWorkerExecution {
|
||||||
|
request,
|
||||||
|
binding: Some(PersistedWorkerExecutionBinding {}),
|
||||||
|
restore_intent: WorkerRestoreIntent::Explicit,
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn source() -> (tempfile::TempDir, PathBuf, StoppedWorkerTemplate) {
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let root = temp.path().join("runtime");
|
||||||
|
initialize(&root, RUNTIME, WORKSPACE, "http://127.0.0.1:23456");
|
||||||
|
drop(
|
||||||
|
crate::Runtime::with_fs_store(FsRuntimeStoreOptions::new(&root).with_runtime_id(RUNTIME))
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
let opened = FsRuntimeStore::open_or_create(root.clone(), RUNTIME).unwrap();
|
||||||
|
let mut old = request();
|
||||||
|
old.workspace_api.as_mut().unwrap().base_url = "http://127.0.0.1:23456".into();
|
||||||
|
old.config_bundle.as_mut().unwrap().id = "old-config".into();
|
||||||
|
opened.store.write_worker_record(&record(old)).unwrap();
|
||||||
|
let worker = opened.store.worker_dir(&worker_id());
|
||||||
|
let session_id = "01a1657d-b800-7777-8888-111111111111".parse().unwrap();
|
||||||
|
let segment_id = "01a1657d-b818-77c1-bd73-176172a4bd82".parse().unwrap();
|
||||||
|
let mut metadata = WorkerMetadata::new(
|
||||||
|
runtime_worker_name(worker_id()),
|
||||||
|
Some(WorkerActiveSegmentRef {
|
||||||
|
session_id,
|
||||||
|
segment_id: Some(segment_id),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.with_workspace_id(WORKSPACE)
|
||||||
|
.with_workspace_root(PathBuf::from("/old/host/path"));
|
||||||
|
metadata.resolved_manifest_snapshot =
|
||||||
|
Some(manifest::write_persisted_worker_manifest_snapshot(&resolved()).unwrap());
|
||||||
|
atomic_write_json(
|
||||||
|
&worker.join(WORKER_METADATA_FILE),
|
||||||
|
&metadata,
|
||||||
|
"test metadata",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let sessions = WorkerSessionStore::new(worker.join("session")).unwrap();
|
||||||
|
let history = session_store::LoggedHistoryEntry {
|
||||||
|
item: session_store::LoggedItem::Message {
|
||||||
|
role: session_store::LoggedRole::Assistant,
|
||||||
|
content: vec![session_store::LoggedContentPart::Text {
|
||||||
|
text: "artificial fixture reply".into(),
|
||||||
|
}],
|
||||||
|
},
|
||||||
|
metadata: session_store::LoggedSessionHistoryMetadata {
|
||||||
|
entry_id: session_store::LoggedSessionHistoryEntryId("fixed-entry-id".into()),
|
||||||
|
origin: session_store::LoggedSessionHistoryOrigin::ModelOutput {
|
||||||
|
worker: session_store::LoggedWorkerSubject {
|
||||||
|
workspace_id: Some(WORKSPACE.into()),
|
||||||
|
runtime_id: Some(RUNTIME.into()),
|
||||||
|
worker_id: worker_id().to_string(),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
derivation: None,
|
||||||
|
input_metadata: None,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
sessions
|
||||||
|
.create_segment(
|
||||||
|
session_id,
|
||||||
|
segment_id,
|
||||||
|
&[
|
||||||
|
LogEntry::AnnotatedSegmentStart {
|
||||||
|
ts: 123,
|
||||||
|
session_id,
|
||||||
|
system_prompt: Some("Harmless artificial fixture instructions.".into()),
|
||||||
|
config: Default::default(),
|
||||||
|
history: vec![],
|
||||||
|
forked_from: None,
|
||||||
|
compacted_from: None,
|
||||||
|
},
|
||||||
|
LogEntry::Invoke {
|
||||||
|
ts: 124,
|
||||||
|
trigger: protocol::InvokeKind::UserSend,
|
||||||
|
},
|
||||||
|
LogEntry::AnnotatedUserInput {
|
||||||
|
ts: 124,
|
||||||
|
segments: vec![protocol::Segment::text(
|
||||||
|
"Harmless artificial fixture input.",
|
||||||
|
)],
|
||||||
|
extensions: vec![checkpoint_extension(&settled_checkpoint())],
|
||||||
|
history: vec![session_store::LoggedHistoryEntry {
|
||||||
|
item: session_store::LoggedItem::Message {
|
||||||
|
role: session_store::LoggedRole::User,
|
||||||
|
content: vec![session_store::LoggedContentPart::Text {
|
||||||
|
text: "Harmless artificial fixture input.".into(),
|
||||||
|
}],
|
||||||
|
},
|
||||||
|
metadata: session_store::LoggedSessionHistoryMetadata {
|
||||||
|
entry_id: session_store::LoggedSessionHistoryEntryId(
|
||||||
|
"fixed-user-id".into(),
|
||||||
|
),
|
||||||
|
origin: session_store::LoggedSessionHistoryOrigin::BackendInstruction {
|
||||||
|
operation_id: None,
|
||||||
|
},
|
||||||
|
derivation: None,
|
||||||
|
input_metadata: None,
|
||||||
|
},
|
||||||
|
}],
|
||||||
|
},
|
||||||
|
LogEntry::AnnotatedAssistantItem {
|
||||||
|
ts: 125,
|
||||||
|
entry: history,
|
||||||
|
},
|
||||||
|
LogEntry::LlmUsage {
|
||||||
|
ts: 125,
|
||||||
|
history_len: 2,
|
||||||
|
input_total_tokens: 1,
|
||||||
|
cache_read_tokens: 0,
|
||||||
|
cache_write_tokens: 0,
|
||||||
|
output_tokens: 1,
|
||||||
|
},
|
||||||
|
LogEntry::TurnEnd {
|
||||||
|
ts: 126,
|
||||||
|
turn_count: 1,
|
||||||
|
},
|
||||||
|
LogEntry::RunCompleted {
|
||||||
|
entry_id: Some(session_store::LoggedSessionHistoryEntryId(
|
||||||
|
"fixed-terminal-id".into(),
|
||||||
|
)),
|
||||||
|
ts: 126,
|
||||||
|
interrupted: false,
|
||||||
|
result: agen::EngineResult::Finished,
|
||||||
|
active_run_turn_count: None,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
fs::create_dir_all(worker.join("runs/old-run")).unwrap();
|
||||||
|
fs::write(worker.join("runs/old-run/private.log"), "not exported").unwrap();
|
||||||
|
drop(opened);
|
||||||
|
let template = StoppedWorkerTemplate::extract(&worker).unwrap();
|
||||||
|
(temp, worker, template)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn destination(template: &StoppedWorkerTemplate) -> (tempfile::TempDir, FsRuntimeStoreOptions) {
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let root = temp.path().join("runtime");
|
||||||
|
initialize(
|
||||||
|
&root,
|
||||||
|
template.runtime_id(),
|
||||||
|
template.workspace_id(),
|
||||||
|
NEW_URL,
|
||||||
|
);
|
||||||
|
(
|
||||||
|
temp,
|
||||||
|
FsRuntimeStoreOptions::new(root).with_runtime_id(template.runtime_id()),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_fresh(options: &FsRuntimeStoreOptions) {
|
||||||
|
assert_eq!(fs::read_dir(&options.root).unwrap().count(), 1);
|
||||||
|
assert!(options.root.join("auth.toml").is_file());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn extraction_install_roundtrip_preserves_bytes_and_is_inactive() {
|
||||||
|
let (_source, worker, template) = source();
|
||||||
|
let original_worker = fs::read(worker.join(WORKER_FILE)).unwrap();
|
||||||
|
let original_metadata = fs::read(worker.join(WORKER_METADATA_FILE)).unwrap();
|
||||||
|
let exported = serde_json::to_vec(&template).unwrap();
|
||||||
|
let exported_text = String::from_utf8(exported.clone()).unwrap();
|
||||||
|
for forbidden in [
|
||||||
|
"old-config",
|
||||||
|
"old-submission-operation",
|
||||||
|
"restore_guard",
|
||||||
|
"private_key",
|
||||||
|
"http://",
|
||||||
|
"/old/host",
|
||||||
|
"private.log",
|
||||||
|
"resolved_manifest_snapshot",
|
||||||
|
] {
|
||||||
|
assert!(!exported_text.contains(forbidden));
|
||||||
|
}
|
||||||
|
let template: StoppedWorkerTemplate = serde_json::from_slice(&exported).unwrap();
|
||||||
|
let (_target, options) = destination(&template);
|
||||||
|
let auth_before = fs::read(options.root.join("auth.toml")).unwrap();
|
||||||
|
template.install(&options, request(), &resolved()).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
fs::read(options.root.join("auth.toml")).unwrap(),
|
||||||
|
auth_before
|
||||||
|
);
|
||||||
|
assert_eq!(fs::read(worker.join(WORKER_FILE)).unwrap(), original_worker);
|
||||||
|
assert_eq!(
|
||||||
|
fs::read(worker.join(WORKER_METADATA_FILE)).unwrap(),
|
||||||
|
original_metadata
|
||||||
|
);
|
||||||
|
let installed = options
|
||||||
|
.root
|
||||||
|
.join(WORKERS_DIR)
|
||||||
|
.join(template.worker_id().to_string());
|
||||||
|
assert_eq!(
|
||||||
|
fs::read(installed.join("session/session.json")).unwrap(),
|
||||||
|
template.session_manifest
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
fs::read(installed.join(format!("session/segments/{}.jsonl", template.segment_id())))
|
||||||
|
.unwrap(),
|
||||||
|
template.segment_jsonl
|
||||||
|
);
|
||||||
|
assert!(!installed.join("runs").exists());
|
||||||
|
let metadata: WorkerMetadata =
|
||||||
|
decode(&fs::read(installed.join(WORKER_METADATA_FILE)).unwrap()).unwrap();
|
||||||
|
assert_eq!(metadata.active.unwrap().session_id, template.session_id());
|
||||||
|
assert!(metadata.workspace_root.is_none() && metadata.resolved_manifest_snapshot.is_some());
|
||||||
|
|
||||||
|
let opened = FsRuntimeStore::open_or_create(options.root.clone(), RUNTIME).unwrap();
|
||||||
|
let state = opened.state.unwrap();
|
||||||
|
assert!(state.diagnostics.is_empty());
|
||||||
|
let saved = state.workers.get(&worker_id()).unwrap();
|
||||||
|
assert_eq!(saved.status, WorkerStatus::Stopped);
|
||||||
|
assert!(saved.last_finished_submission_request_id.is_none());
|
||||||
|
let PersistedWorkerExecutionState::Available(execution) = &saved.execution_state else {
|
||||||
|
panic!("not available")
|
||||||
|
};
|
||||||
|
assert_eq!(execution.restore_intent, WorkerRestoreIntent::Explicit);
|
||||||
|
assert!(execution.binding.is_none());
|
||||||
|
assert_eq!(execution.request, request());
|
||||||
|
// Fresh restore guard, no operation owner/receipt history.
|
||||||
|
assert_ne!(
|
||||||
|
serde_json::to_vec(&saved.restore_guard).unwrap(),
|
||||||
|
serde_json::to_vec(
|
||||||
|
&decode::<WorkerAggregateRecord>(&original_worker)
|
||||||
|
.unwrap()
|
||||||
|
.restore_guard
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
);
|
||||||
|
let guard: GuardState = decode(&serde_json::to_vec(&saved.restore_guard).unwrap()).unwrap();
|
||||||
|
assert!(guard.active_request_id.is_none() && guard.owners.is_empty());
|
||||||
|
drop(opened.store);
|
||||||
|
|
||||||
|
struct NoExecution;
|
||||||
|
impl crate::execution::WorkerExecutionBackend for NoExecution {
|
||||||
|
fn backend_id(&self) -> &str {
|
||||||
|
"fixture-no-execution"
|
||||||
|
}
|
||||||
|
fn spawn_worker(
|
||||||
|
&self,
|
||||||
|
_: crate::execution::WorkerExecutionSpawnRequest,
|
||||||
|
) -> crate::execution::WorkerExecutionSpawnResult {
|
||||||
|
panic!("fixture executed")
|
||||||
|
}
|
||||||
|
fn dispatch_input(
|
||||||
|
&self,
|
||||||
|
_: &WorkerRef,
|
||||||
|
_: crate::interaction::WorkerInput,
|
||||||
|
) -> crate::execution::WorkerExecutionResult {
|
||||||
|
panic!("fixture submitted")
|
||||||
|
}
|
||||||
|
fn restore_worker(
|
||||||
|
&self,
|
||||||
|
_: crate::execution::WorkerExecutionRestoreRequest,
|
||||||
|
) -> crate::execution::WorkerExecutionSpawnResult {
|
||||||
|
panic!("fixture auto-restored")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let runtime =
|
||||||
|
crate::Runtime::with_fs_store_and_execution_backend(options.clone(), Arc::new(NoExecution))
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(runtime.list_workers().unwrap().len(), 1);
|
||||||
|
assert_eq!(runtime.summary().unwrap().active_worker_count, 0);
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.worker_detail(&WorkerRef::new(worker_id()))
|
||||||
|
.unwrap()
|
||||||
|
.status,
|
||||||
|
WorkerStatus::Stopped
|
||||||
|
);
|
||||||
|
drop(runtime);
|
||||||
|
let runtime = crate::Runtime::with_fs_store(options).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
runtime.list_workers().unwrap()[0].status,
|
||||||
|
WorkerStatus::Stopped
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn install_rejects_schema_identity_input_and_overwrite() {
|
||||||
|
let (_source, _, template) = source();
|
||||||
|
for change in [
|
||||||
|
"template",
|
||||||
|
"runtime-schema",
|
||||||
|
"session-schema",
|
||||||
|
"session-id",
|
||||||
|
"segment-id",
|
||||||
|
"runtime-id",
|
||||||
|
"workspace-id",
|
||||||
|
"worker-id",
|
||||||
|
"origin-id",
|
||||||
|
"duplicate-id",
|
||||||
|
"truncated",
|
||||||
|
] {
|
||||||
|
let (_target, options) = destination(&template);
|
||||||
|
let mut invalid = template.clone();
|
||||||
|
match change {
|
||||||
|
"template" => invalid.template_version += 1,
|
||||||
|
"runtime-schema" => invalid.runtime_schema_version += 1,
|
||||||
|
"session-schema" => {
|
||||||
|
invalid.session_manifest = format!(
|
||||||
|
"{{\"schema_version\":999,\"session_id\":\"{}\"}}",
|
||||||
|
invalid.session_id
|
||||||
|
)
|
||||||
|
.into_bytes()
|
||||||
|
}
|
||||||
|
"session-id" => invalid.session_id = uuid::Uuid::now_v7(),
|
||||||
|
"segment-id" => invalid.segment_id = uuid::Uuid::nil(), // no external attestation: valid new Segment ID is otherwise portable
|
||||||
|
"runtime-id" => invalid.runtime_id = "other-runtime".into(),
|
||||||
|
"workspace-id" => invalid.workspace_id = "other-workspace".into(),
|
||||||
|
"worker-id" => invalid.worker_id = WorkerId::now_v7(),
|
||||||
|
"origin-id" => {
|
||||||
|
invalid.segment_jsonl = String::from_utf8(invalid.segment_jsonl)
|
||||||
|
.unwrap()
|
||||||
|
.replace(RUNTIME, "other-runtime")
|
||||||
|
.into_bytes()
|
||||||
|
}
|
||||||
|
"duplicate-id" => {
|
||||||
|
let last = invalid
|
||||||
|
.segment_jsonl
|
||||||
|
.split_inclusive(|b| *b == b'\n')
|
||||||
|
.last()
|
||||||
|
.unwrap()
|
||||||
|
.to_vec();
|
||||||
|
invalid.segment_jsonl.extend(last);
|
||||||
|
}
|
||||||
|
"truncated" => {
|
||||||
|
invalid.segment_jsonl.pop();
|
||||||
|
}
|
||||||
|
_ => unreachable!(),
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
invalid.install(&options, request(), &resolved()).is_err(),
|
||||||
|
"accepted {change}"
|
||||||
|
);
|
||||||
|
assert_fresh(&options);
|
||||||
|
}
|
||||||
|
let (_target, options) = destination(&template);
|
||||||
|
let mut launch = request();
|
||||||
|
launch.initial_input = Some(crate::interaction::WorkerInput::user("do not execute"));
|
||||||
|
assert!(template.install(&options, launch, &resolved()).is_err());
|
||||||
|
assert_fresh(&options);
|
||||||
|
template.install(&options, request(), &resolved()).unwrap();
|
||||||
|
let worker_path = options
|
||||||
|
.root
|
||||||
|
.join(WORKERS_DIR)
|
||||||
|
.join(worker_id().to_string())
|
||||||
|
.join(WORKER_FILE);
|
||||||
|
let before = fs::read(&worker_path).unwrap();
|
||||||
|
assert!(template.install(&options, request(), &resolved()).is_err());
|
||||||
|
assert_eq!(fs::read(worker_path).unwrap(), before);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rejects_uninitialized_untrusted_live_and_partial_destination() {
|
||||||
|
let (_source, _, template) = source();
|
||||||
|
let (_target, options) = destination(&template);
|
||||||
|
let held = acquire_runtime_store_owner_lock(&options.root).unwrap();
|
||||||
|
assert!(template.install(&options, request(), &resolved()).is_err());
|
||||||
|
assert_fresh(&options);
|
||||||
|
drop(held);
|
||||||
|
initialize(&options.root, "wrong-runtime", WORKSPACE, NEW_URL);
|
||||||
|
assert!(template.install(&options, request(), &resolved()).is_err());
|
||||||
|
initialize(&options.root, RUNTIME, WORKSPACE, "http://127.0.0.1:23456");
|
||||||
|
assert!(template.install(&options, request(), &resolved()).is_err());
|
||||||
|
fs::remove_file(options.root.join("auth.toml")).unwrap();
|
||||||
|
assert!(template.install(&options, request(), &resolved()).is_err());
|
||||||
|
initialize(&options.root, RUNTIME, WORKSPACE, NEW_URL);
|
||||||
|
fs::create_dir(options.root.join(WORKERS_DIR)).unwrap();
|
||||||
|
assert!(template.install(&options, request(), &resolved()).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn extraction_rejects_nonstopped_pending_schema_and_metadata_mismatch() {
|
||||||
|
for change in [
|
||||||
|
"running",
|
||||||
|
"pending",
|
||||||
|
"guard-pending",
|
||||||
|
"attachment",
|
||||||
|
"schema",
|
||||||
|
"worker-id",
|
||||||
|
"metadata-workspace",
|
||||||
|
"metadata-session",
|
||||||
|
"artifact",
|
||||||
|
"secret",
|
||||||
|
"url",
|
||||||
|
"path",
|
||||||
|
] {
|
||||||
|
let (_source, worker, _) = source();
|
||||||
|
let mut aggregate: WorkerAggregateRecord =
|
||||||
|
decode(&fs::read(worker.join(WORKER_FILE)).unwrap()).unwrap();
|
||||||
|
match change {
|
||||||
|
"running" => aggregate.status = WorkerStatus::Running,
|
||||||
|
"pending" => {
|
||||||
|
aggregate.execution_state = WorkerExecutionStateRecord::ReconciliationRequired(
|
||||||
|
PersistedWorkerLifecycleOperation::Restore(PersistedWorkerRestoreOperation {
|
||||||
|
operation_id: WorkerLifecycleOperationId::new(),
|
||||||
|
mode: PersistedWorkerRestoreMode::Explicit,
|
||||||
|
commit: crate::runtime::RestoreCommitFact::NotCommitted,
|
||||||
|
source_repair_possible: false,
|
||||||
|
request: request(),
|
||||||
|
binding: None,
|
||||||
|
last_settled_status: WorkerStatus::Stopped,
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
"guard-pending" => aggregate.restore_guard = serde_json::from_str(r#"{"observation_token":"old-token","active_request_id":"pending-restore","owners":{}}"#).unwrap(),
|
||||||
|
"attachment" => aggregate.logical_workdir_attachments.push(LogicalWorkdirAttachment {
|
||||||
|
alias: workdir::WorkdirAttachmentAlias::new("source").unwrap(),
|
||||||
|
working_directory_id: "old-workdir".into(),
|
||||||
|
capabilities: workdir::WorkdirSessionCapabilities::READ_ONLY,
|
||||||
|
}),
|
||||||
|
"schema" => aggregate.schema_version += 1,
|
||||||
|
"worker-id" => aggregate.worker_id = WorkerId::now_v7(),
|
||||||
|
"metadata-workspace" | "metadata-session" => {
|
||||||
|
let mut metadata: WorkerMetadata =
|
||||||
|
decode(&fs::read(worker.join(WORKER_METADATA_FILE)).unwrap()).unwrap();
|
||||||
|
if change == "metadata-workspace" {
|
||||||
|
metadata.workspace_id = Some("wrong-workspace".into());
|
||||||
|
} else {
|
||||||
|
metadata.active.as_mut().unwrap().session_id = uuid::Uuid::now_v7();
|
||||||
|
}
|
||||||
|
atomic_write_json(
|
||||||
|
&worker.join(WORKER_METADATA_FILE),
|
||||||
|
&metadata,
|
||||||
|
"test mismatch",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
"artifact" => {
|
||||||
|
fs::create_dir_all(worker.join("session/artifacts/paste")).unwrap();
|
||||||
|
fs::write(
|
||||||
|
worker.join("session/artifacts/paste/payload"),
|
||||||
|
"unsupported",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
"secret" | "url" | "path" => {
|
||||||
|
let segment = fs::read_dir(worker.join("session/segments"))
|
||||||
|
.unwrap()
|
||||||
|
.next()
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
.path();
|
||||||
|
let replacement = match change {
|
||||||
|
"secret" => "Bearer secret",
|
||||||
|
"url" => "https://old-host.invalid",
|
||||||
|
_ => "/old/host/path",
|
||||||
|
};
|
||||||
|
let text = fs::read_to_string(&segment)
|
||||||
|
.unwrap()
|
||||||
|
.replace("artificial fixture reply", replacement);
|
||||||
|
fs::write(segment, text).unwrap();
|
||||||
|
}
|
||||||
|
_ => unreachable!(),
|
||||||
|
}
|
||||||
|
atomic_write_json(&worker.join(WORKER_FILE), &aggregate, "test aggregate").unwrap();
|
||||||
|
assert!(
|
||||||
|
StoppedWorkerTemplate::extract(&worker).is_err(),
|
||||||
|
"accepted {change}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let (_source, worker, _) = source();
|
||||||
|
let held =
|
||||||
|
acquire_runtime_store_owner_lock(worker.parent().unwrap().parent().unwrap()).unwrap();
|
||||||
|
assert!(StoppedWorkerTemplate::extract(&worker).is_err());
|
||||||
|
drop(held);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn schema_v12_settled_template_installs_current_aggregate_without_rewriting_session() {
|
||||||
|
let (_source, _, mut template) = source();
|
||||||
|
template.runtime_schema_version = RESTORE_ADMISSION_SCHEMA_VERSION;
|
||||||
|
let bytes = serde_json::to_vec(&template).unwrap();
|
||||||
|
let (_target, options) = destination(&template);
|
||||||
|
template.install(&options, request(), &resolved()).unwrap();
|
||||||
|
assert_eq!(serde_json::to_vec(&template).unwrap(), bytes);
|
||||||
|
let worker = options.root.join(WORKERS_DIR).join(worker_id().to_string());
|
||||||
|
assert_eq!(
|
||||||
|
fs::read(worker.join("session/session.json")).unwrap(),
|
||||||
|
template.session_manifest
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
fs::read(worker.join(format!("session/segments/{}.jsonl", template.segment_id))).unwrap(),
|
||||||
|
template.segment_jsonl
|
||||||
|
);
|
||||||
|
let aggregate: WorkerAggregateRecord =
|
||||||
|
read_bounded_json(&worker.join(WORKER_FILE), "read installed fixture").unwrap();
|
||||||
|
let saved = aggregate.validate(&worker.join(WORKER_FILE)).unwrap();
|
||||||
|
assert_eq!(saved.status, WorkerStatus::Stopped);
|
||||||
|
assert!(matches!(
|
||||||
|
saved.execution_state,
|
||||||
|
PersistedWorkerExecutionState::Available(_)
|
||||||
|
));
|
||||||
|
// Unsupported template schemas do not enter the versioned reader.
|
||||||
|
template.runtime_schema_version = RESTORE_ADMISSION_SCHEMA_VERSION - 1;
|
||||||
|
assert!(template.validate_shape().is_err());
|
||||||
|
template.runtime_schema_version = SCHEMA_VERSION + 1;
|
||||||
|
assert!(template.validate_shape().is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn fresh_manifest_and_memory_settings_are_required() {
|
||||||
|
let (_source, _, template) = source();
|
||||||
|
let (_target, options) = destination(&template);
|
||||||
|
let mut manifest = resolved();
|
||||||
|
manifest.worker.name = "wrong-worker".into();
|
||||||
|
assert!(template.install(&options, request(), &manifest).is_err());
|
||||||
|
let mut binding = request();
|
||||||
|
binding.memory_settings = None;
|
||||||
|
assert!(template.install(&options, binding, &resolved()).is_err());
|
||||||
|
let mut binding = request();
|
||||||
|
binding.memory_settings.as_mut().unwrap().workspace_id = "other-workspace".into();
|
||||||
|
assert!(template.install(&options, binding, &resolved()).is_err());
|
||||||
|
assert_fresh(&options);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn settled_checkpoints_preserve_receipts_but_refuse_pending_work() {
|
||||||
|
let mut checkpoint = settled_checkpoint();
|
||||||
|
assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_ok());
|
||||||
|
checkpoint.pending.push(1);
|
||||||
|
assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_err());
|
||||||
|
checkpoint.pending.clear();
|
||||||
|
checkpoint.activating = Some(1);
|
||||||
|
assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_err());
|
||||||
|
checkpoint.activating = None;
|
||||||
|
checkpoint.invocation_receipts.push(1);
|
||||||
|
assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_err());
|
||||||
|
checkpoint.invocation_receipts.clear();
|
||||||
|
checkpoint.invocation_recovery_blocked = true;
|
||||||
|
assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
// Deliberately corrupt only test-owned synthetic source files, never the lab.
|
||||||
|
fn assert_rejected_session_bytes(worker: &Path, template: &StoppedWorkerTemplate, bytes: Vec<u8>) {
|
||||||
|
let path = worker.join(format!("session/segments/{}.jsonl", template.segment_id()));
|
||||||
|
fs::write(path, &bytes).unwrap();
|
||||||
|
let extraction_error = StoppedWorkerTemplate::extract(worker)
|
||||||
|
.err()
|
||||||
|
.expect("invalid source accepted");
|
||||||
|
assert!(
|
||||||
|
!extraction_error
|
||||||
|
.to_string()
|
||||||
|
.contains("synthetic-discarded-material")
|
||||||
|
);
|
||||||
|
let (_target, options) = destination(template);
|
||||||
|
let mut invalid = template.clone();
|
||||||
|
invalid.segment_jsonl = bytes;
|
||||||
|
let installation_error = invalid
|
||||||
|
.install(&options, request(), &resolved())
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(
|
||||||
|
!installation_error
|
||||||
|
.to_string()
|
||||||
|
.contains("synthetic-discarded-material")
|
||||||
|
);
|
||||||
|
assert_fresh(&options);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn synthetic_logs(template: &StoppedWorkerTemplate) -> Vec<LogEntry> {
|
||||||
|
template
|
||||||
|
.segment_jsonl
|
||||||
|
.split(|byte| *byte == b'\n')
|
||||||
|
.filter(|line| !line.is_empty())
|
||||||
|
.map(|line| serde_json::from_slice(line).unwrap())
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn encode_synthetic_logs(logs: &[LogEntry]) -> Vec<u8> {
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
for log in logs {
|
||||||
|
serde_json::to_writer(&mut bytes, log).unwrap();
|
||||||
|
bytes.push(b'\n');
|
||||||
|
}
|
||||||
|
bytes
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn missing_system_prompt_matches_worker_restore_rejection() {
|
||||||
|
let (_source, worker_dir, template) = source();
|
||||||
|
let mut logs = synthetic_logs(&template);
|
||||||
|
assert!(session_store::collect_state(&logs).system_prompt.is_some());
|
||||||
|
let LogEntry::AnnotatedSegmentStart { system_prompt, .. } = &mut logs[0] else {
|
||||||
|
panic!("missing seed")
|
||||||
|
};
|
||||||
|
*system_prompt = None;
|
||||||
|
assert_rejected_session_bytes(&worker_dir, &template, encode_synthetic_logs(&logs));
|
||||||
|
// The owning Worker constructor rejects this before model-client creation,
|
||||||
|
// allocation, or execution. Do not start a controller or submit any input.
|
||||||
|
let store = session_store::CombinedStore::new(
|
||||||
|
WorkerSessionStore::open_read_only(worker_dir.join("session")).unwrap(),
|
||||||
|
session_store::WorkerAggregateStore::open_read_only(&worker_dir, template.worker_name())
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
let restored = worker::Worker::restore_from_manifest(
|
||||||
|
template.session_id(),
|
||||||
|
template.segment_id(),
|
||||||
|
resolved(),
|
||||||
|
store,
|
||||||
|
worker::PromptCatalogSource::builtins_only(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(matches!(
|
||||||
|
restored,
|
||||||
|
Err(worker::WorkerError::SegmentSystemPromptMissing { .. })
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn malformed_submission_and_notification_receipts_fail_extraction_and_install() {
|
||||||
|
let (_source, worker, template) = source();
|
||||||
|
for notification in [false, true] {
|
||||||
|
let mut checkpoint = settled_checkpoint();
|
||||||
|
if notification {
|
||||||
|
checkpoint.notification_receipts =
|
||||||
|
vec![ReceiptFixture::Malformed(1), ReceiptFixture::Malformed(2)];
|
||||||
|
} else {
|
||||||
|
checkpoint.receipts = vec![ReceiptFixture::Malformed(1), ReceiptFixture::Malformed(2)];
|
||||||
|
}
|
||||||
|
let mut logs = synthetic_logs(&template);
|
||||||
|
for log in &mut logs {
|
||||||
|
if let LogEntry::AnnotatedUserInput { extensions, .. } = log {
|
||||||
|
*extensions = vec![checkpoint_extension(&checkpoint)];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert_rejected_session_bytes(&worker, &template, encode_synthetic_logs(&logs));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unknown_fields_at_every_retained_record_depth_are_rejected_without_secret_diagnostics() {
|
||||||
|
let (_source, worker, template) = source();
|
||||||
|
let original = String::from_utf8(template.segment_jsonl.clone()).unwrap();
|
||||||
|
// These values are harmless synthetic markers, not lab/private material.
|
||||||
|
// The compatibility decoder normally discards many of these fields.
|
||||||
|
let locations = [
|
||||||
|
r#""kind":"annotated_segment_start""#,
|
||||||
|
r#""kind":"invoke""#,
|
||||||
|
r#""kind":"annotated_user_input""#,
|
||||||
|
r#""kind":"annotated_assistant_item""#,
|
||||||
|
r#""kind":"llm_usage""#,
|
||||||
|
r#""kind":"turn_end""#,
|
||||||
|
r#""kind":"run_completed""#,
|
||||||
|
r#""max_tokens":null"#,
|
||||||
|
r#""kind":"message","role":"assistant""#,
|
||||||
|
r#""kind":"text","text":"artificial fixture reply""#,
|
||||||
|
r#""kind":"text","content":"Harmless artificial fixture input.""#,
|
||||||
|
r#""entry_id":"fixed-entry-id""#,
|
||||||
|
r#""kind":"model_output""#,
|
||||||
|
r#""worker_id":"01a1657d-b762-7791-87ae-a49f28cadbce""#,
|
||||||
|
r#""domain":"worker.pending_activations.v1""#,
|
||||||
|
r#""next_activation_sequence":2"#,
|
||||||
|
r#""submission_request_id":"completed-submission-request""#,
|
||||||
|
r#""notification_request_id":"completed-notification-request""#,
|
||||||
|
];
|
||||||
|
for location in locations {
|
||||||
|
assert!(original.contains(location));
|
||||||
|
for extra in [
|
||||||
|
r#", "reasoning":{"text":"synthetic-discarded-material"}"#,
|
||||||
|
r#", "encrypted_content":"synthetic-discarded-material""#,
|
||||||
|
] {
|
||||||
|
let altered = original.replacen(location, &format!("{location}{extra}"), 1);
|
||||||
|
assert_rejected_session_bytes(&worker, &template, altered.into_bytes());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let duplicate = original.replacen(
|
||||||
|
r#""role":"assistant""#,
|
||||||
|
r#""role":"assistant","role":"assistant""#,
|
||||||
|
1,
|
||||||
|
);
|
||||||
|
assert_rejected_session_bytes(&worker, &template, duplicate.into_bytes());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn opaque_feature_payloads_cannot_bypass_strict_session_validation() {
|
||||||
|
let (_source, worker, template) = source();
|
||||||
|
let mut logs = synthetic_logs(&template);
|
||||||
|
for log in &mut logs {
|
||||||
|
if let LogEntry::AnnotatedUserInput { history, .. } = log {
|
||||||
|
if let session_store::LoggedItem::Message { role, .. } = &mut history[0].item {
|
||||||
|
*role = session_store::LoggedRole::System;
|
||||||
|
}
|
||||||
|
history[0].metadata.input_metadata = Some(
|
||||||
|
session_store::HistoryInputMetadata::new(
|
||||||
|
session_store::HistoryInputUsePolicy {
|
||||||
|
recording_material: session_store::InputEvidenceUse::Exclude,
|
||||||
|
memory_evidence: session_store::InputEvidenceUse::Exclude,
|
||||||
|
observation: session_store::InputObservationUse::Hidden,
|
||||||
|
compaction: session_store::InputCompactionUse::Retain,
|
||||||
|
},
|
||||||
|
Some(
|
||||||
|
session_store::FeatureInputPayload::encode(
|
||||||
|
"fixture.test",
|
||||||
|
1,
|
||||||
|
&BTreeMap::from([("reasoning", "harmless synthetic payload")]),
|
||||||
|
)
|
||||||
|
.unwrap(),
|
||||||
|
),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert_rejected_session_bytes(&worker, &template, encode_synthetic_logs(&logs));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
#[test]
|
||||||
|
fn refuses_source_and_destination_symlinks() {
|
||||||
|
let (_source, worker, template) = source();
|
||||||
|
let (_target, options) = destination(&template);
|
||||||
|
std::os::unix::fs::symlink(worker.join(WORKER_FILE), options.root.join("linked")).unwrap();
|
||||||
|
assert!(template.install(&options, request(), &resolved()).is_err());
|
||||||
|
fs::remove_file(options.root.join("linked")).unwrap();
|
||||||
|
std::os::unix::fs::symlink(worker.join(WORKER_FILE), worker.join("runs/link")).unwrap();
|
||||||
|
assert!(StoppedWorkerTemplate::extract(&worker).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Opt-in only: source path supplied explicitly; assertions never display content.
|
||||||
|
#[test]
|
||||||
|
#[ignore = "requires the approved artificial lab aggregate via YOI_STOPPED_FIXTURE_SOURCE"]
|
||||||
|
fn approved_lab_extract_install() {
|
||||||
|
let path = PathBuf::from(
|
||||||
|
std::env::var_os("YOI_STOPPED_FIXTURE_SOURCE").expect("explicit source required"),
|
||||||
|
);
|
||||||
|
let template = StoppedWorkerTemplate::extract(&path).unwrap();
|
||||||
|
let (_target, options) = destination(&template);
|
||||||
|
let mut bind = request();
|
||||||
|
bind.worker_id = template.worker_id();
|
||||||
|
bind.workspace_api.as_mut().unwrap().workspace_id = template.workspace_id().into();
|
||||||
|
bind.memory_settings.as_mut().unwrap().workspace_id = template.workspace_id().into();
|
||||||
|
let mut manifest = resolved();
|
||||||
|
manifest.worker.name = template.worker_name();
|
||||||
|
template.install(&options, bind, &manifest).unwrap();
|
||||||
|
let installed = options
|
||||||
|
.root
|
||||||
|
.join(WORKERS_DIR)
|
||||||
|
.join(template.worker_id().to_string());
|
||||||
|
for relative in [
|
||||||
|
"session/session.json".to_string(),
|
||||||
|
format!("session/segments/{}.jsonl", template.segment_id()),
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
fs::read(path.join(&relative)).unwrap() == fs::read(installed.join(relative)).unwrap()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let runtime = crate::Runtime::with_fs_store(options).unwrap();
|
||||||
|
assert_eq!(runtime.list_workers().unwrap().len(), 1);
|
||||||
|
assert_eq!(runtime.summary().unwrap().active_worker_count, 0);
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -193,6 +193,21 @@ mod tests {
|
|||||||
use super::*;
|
use super::*;
|
||||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
|
|
||||||
|
fn short_tempdir() -> tempfile::TempDir {
|
||||||
|
let target = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||||
|
.parent()
|
||||||
|
.unwrap()
|
||||||
|
.parent()
|
||||||
|
.unwrap()
|
||||||
|
.join("target");
|
||||||
|
std::fs::create_dir_all(&target).unwrap();
|
||||||
|
tempfile::Builder::new()
|
||||||
|
.prefix("d")
|
||||||
|
.rand_bytes(3)
|
||||||
|
.tempdir_in(target)
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
fn read_request() -> ReadRequest {
|
fn read_request() -> ReadRequest {
|
||||||
ReadRequest {
|
ReadRequest {
|
||||||
path: WorkdirPath::new("README.md").unwrap(),
|
path: WorkdirPath::new("README.md").unwrap(),
|
||||||
@@ -225,9 +240,60 @@ mod tests {
|
|||||||
assert_eq!(attempts.load(Ordering::SeqCst), 0);
|
assert_eq!(attempts.load(Ordering::SeqCst), 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn router_cleanup_of_unopened_session_never_connects_or_follows_reused_alias() {
|
||||||
|
let attempts = Arc::new(AtomicUsize::new(0));
|
||||||
|
let observed = attempts.clone();
|
||||||
|
let session = DeferredWorkdirSession::handle(
|
||||||
|
"old",
|
||||||
|
WorkdirSessionCapabilities::READ_ONLY,
|
||||||
|
move || {
|
||||||
|
observed.fetch_add(1, Ordering::SeqCst);
|
||||||
|
Err("offline".into())
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let router = WorkdirSessionRouter::new();
|
||||||
|
let alias = WorkdirAttachmentAlias::new("w").unwrap();
|
||||||
|
let routed = router.attach(alias.clone(), session.clone()).unwrap();
|
||||||
|
assert!(routed.subscribe_command_events().is_some());
|
||||||
|
assert!(routed.command_snapshot().is_empty());
|
||||||
|
router.close_all().await.unwrap();
|
||||||
|
router.close_all().await.unwrap();
|
||||||
|
assert!(router.aliases().is_empty());
|
||||||
|
assert!(matches!(
|
||||||
|
session.read(read_request()).await,
|
||||||
|
Err(WorkdirError::SessionClosed)
|
||||||
|
));
|
||||||
|
assert_eq!(attempts.load(Ordering::SeqCst), 0);
|
||||||
|
|
||||||
|
let new_attempts = Arc::new(AtomicUsize::new(0));
|
||||||
|
let new_observed = new_attempts.clone();
|
||||||
|
let replacement = DeferredWorkdirSession::handle(
|
||||||
|
"new",
|
||||||
|
WorkdirSessionCapabilities::READ_ONLY,
|
||||||
|
move || {
|
||||||
|
new_observed.fetch_add(1, Ordering::SeqCst);
|
||||||
|
Err("offline".into())
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let new_routed = router.attach(alias, replacement).unwrap();
|
||||||
|
// A retained old connection cannot open the replacement provider.
|
||||||
|
assert!(routed.read(read_request()).await.is_err());
|
||||||
|
assert_eq!(new_attempts.load(Ordering::SeqCst), 0);
|
||||||
|
assert!(matches!(
|
||||||
|
new_routed.read(read_request()).await,
|
||||||
|
Err(ref error) if matches!(error.classification_source(), WorkdirError::Unavailable(_))
|
||||||
|
));
|
||||||
|
assert_eq!(new_attempts.load(Ordering::SeqCst), 1);
|
||||||
|
// Even after a failed open, cleanup must not try connecting again.
|
||||||
|
router.close_all().await.unwrap();
|
||||||
|
assert_eq!(new_attempts.load(Ordering::SeqCst), 1);
|
||||||
|
assert_eq!(attempts.load(Ordering::SeqCst), 0);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn failed_open_retries_on_use_and_keeps_read_only_capabilities() {
|
async fn failed_open_retries_on_use_and_keeps_read_only_capabilities() {
|
||||||
let temp = tempfile::tempdir().unwrap();
|
let temp = short_tempdir();
|
||||||
std::fs::write(temp.path().join("README.md"), "restored").unwrap();
|
std::fs::write(temp.path().join("README.md"), "restored").unwrap();
|
||||||
let root = temp.path().to_path_buf();
|
let root = temp.path().to_path_buf();
|
||||||
let attempts = Arc::new(AtomicUsize::new(0));
|
let attempts = Arc::new(AtomicUsize::new(0));
|
||||||
|
|||||||
@@ -0,0 +1,185 @@
|
|||||||
|
//! Explicit, operation-owned repair of saved Session material. This is not
|
||||||
|
//! read-only candidate preparation: the canonical Session/attribution is the
|
||||||
|
//! receipt for replay after a lost response, and effects are reported as such.
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
impl ProfileRuntimeWorkerFactory {
|
||||||
|
pub(super) async fn recover_saved_restore_source(
|
||||||
|
&self,
|
||||||
|
request: &WorkerExecutionRestoreRequest,
|
||||||
|
) -> Result<bool, RuntimeRestoreFailure> {
|
||||||
|
let cause = |cause| RuntimeRestoreFailure::started(RuntimeRestoreStage::Recovery, cause);
|
||||||
|
let name = Self::runtime_worker_name_for_ref(&request.worker_ref);
|
||||||
|
let root = self
|
||||||
|
.worker_aggregate_dir(&request.worker_ref)
|
||||||
|
.map_err(|_| cause("restore recovery aggregate unavailable"))?;
|
||||||
|
let metadata_store = WorkerAggregateStore::open_read_only(&root, &name)
|
||||||
|
.map_err(|_| cause("restore recovery metadata open failed"))?;
|
||||||
|
let metadata = metadata_store
|
||||||
|
.read_by_name(&name)
|
||||||
|
.map_err(|_| cause("restore recovery metadata read failed"))?
|
||||||
|
.ok_or_else(|| cause("restore recovery metadata missing"))?;
|
||||||
|
let active = metadata
|
||||||
|
.active
|
||||||
|
.as_ref()
|
||||||
|
.ok_or_else(|| cause("restore recovery Session missing"))?;
|
||||||
|
let pending = active.segment_id.is_none();
|
||||||
|
let incomplete = request.request.subjektiv_attached
|
||||||
|
&& !matches!(
|
||||||
|
metadata.subjektiv_session_attribution,
|
||||||
|
Some(session_store::SubjektivSessionAttributionState::Confirmed { .. })
|
||||||
|
);
|
||||||
|
if !pending && !incomplete {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
request
|
||||||
|
.context
|
||||||
|
.authorize_restore_source_repair(request.operation_id)
|
||||||
|
.map_err(|_| cause("canonical source repair authority is not durably confirmed"))?;
|
||||||
|
// The accepted Runtime operation already owns this Session. Its stable
|
||||||
|
// Session identity and sole committed seed, not a new restore request,
|
||||||
|
// are used to reconcile an interrupted first-head publication.
|
||||||
|
let session_store = WorkerSessionStore::new_with_retention(
|
||||||
|
root.join("session"),
|
||||||
|
self.session_retention()
|
||||||
|
.map_err(|_| cause("restore recovery retention unavailable"))?,
|
||||||
|
)
|
||||||
|
.map_err(|_| cause("restore recovery Session open failed"))?;
|
||||||
|
let (manifest, mut loader) = Self::manifest_for_restore(&metadata, &request.request)
|
||||||
|
.map_err(|_| cause("restore recovery saved manifest invalid"))?;
|
||||||
|
let workspace = RuntimeWorkspaceBackendRef::from_worker_request(
|
||||||
|
&request.request,
|
||||||
|
self.runtime_id.as_deref(),
|
||||||
|
)
|
||||||
|
.map_err(cause)?;
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
|
let request_client = request
|
||||||
|
.request
|
||||||
|
.workspace_api
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|api| self.workspace_request_clients.get(&api.workspace_id));
|
||||||
|
let context = workspace.worker_context(
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
|
&request.worker_ref,
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
|
request.workspace_scope.as_ref(),
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
|
self.worker_mutation_identity.as_ref(),
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
|
request_client,
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
|
self.embedded_worker_mutation_dispatcher.as_ref(),
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
|
Some(self.prompt_projection_cache.clone()),
|
||||||
|
);
|
||||||
|
if pending && context.workspace_id().is_some() {
|
||||||
|
let bundle = request
|
||||||
|
.config_bundle
|
||||||
|
.as_ref()
|
||||||
|
.ok_or_else(|| cause("pending recovery launch material missing"))?;
|
||||||
|
let projection = self
|
||||||
|
.observe_bundle_prompt_projection(
|
||||||
|
bundle,
|
||||||
|
request
|
||||||
|
.request
|
||||||
|
.workspace_api
|
||||||
|
.as_ref()
|
||||||
|
.map(|api| api.workspace_id.as_str()),
|
||||||
|
)
|
||||||
|
.map_err(|_| cause("pending recovery Prompt projection invalid"))?
|
||||||
|
.ok_or_else(|| cause("pending recovery Prompt projection missing"))?;
|
||||||
|
loader = loader.with_effective_catalog(projection.projection.catalog.clone());
|
||||||
|
}
|
||||||
|
self.failed_restore_resources
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||||
|
.insert(
|
||||||
|
request.worker_ref.clone(),
|
||||||
|
FailedRestoreResources {
|
||||||
|
operation_id: request.operation_id,
|
||||||
|
sessions: Vec::new(),
|
||||||
|
candidate_workdir_cleanup: None,
|
||||||
|
allocation: None,
|
||||||
|
feature_tasks: None,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let store = CombinedStore::new(session_store, metadata_store);
|
||||||
|
let worker_result = Worker::restore_pending_from_worker_metadata_with_context(
|
||||||
|
&name,
|
||||||
|
manifest,
|
||||||
|
store,
|
||||||
|
loader,
|
||||||
|
context,
|
||||||
|
WorkerFilesystemAuthority::None,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let mut worker = worker_result.map_err(|error| {
|
||||||
|
self.retain_construction_failure(request, &error);
|
||||||
|
cause(restore_worker_cause(&error))
|
||||||
|
})?;
|
||||||
|
let tasks = worker.retain_startup_feature_cleanup();
|
||||||
|
let sessions = worker.workdir_sessions();
|
||||||
|
let allocation = worker.restore_allocation_cleanup_handle();
|
||||||
|
self.failed_restore_resources
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||||
|
.insert(
|
||||||
|
request.worker_ref.clone(),
|
||||||
|
FailedRestoreResources {
|
||||||
|
operation_id: request.operation_id,
|
||||||
|
sessions: vec![sessions.clone()],
|
||||||
|
candidate_workdir_cleanup: None,
|
||||||
|
allocation: allocation.clone(),
|
||||||
|
feature_tasks: Some(tasks.clone()),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let result = async {
|
||||||
|
if let Some(binding) = request.request.backend_job.as_ref() {
|
||||||
|
worker
|
||||||
|
.bind_backend_job(binding.clone())
|
||||||
|
.map_err(|_| cause("recovery Backend Job binding failed"))?;
|
||||||
|
}
|
||||||
|
worker
|
||||||
|
.finalize_subjektiv_session_attribution(
|
||||||
|
request.request.subjektiv_attached,
|
||||||
|
SubjektivSessionAttributionLifecycle::RestoredSession,
|
||||||
|
)
|
||||||
|
.map_err(|error| cause(restore_worker_cause(&error)))?;
|
||||||
|
if pending {
|
||||||
|
worker::WorkerController::recover_pending_head(&mut worker, &root)
|
||||||
|
.await
|
||||||
|
.map_err(|_| cause("pending Session head recovery failed"))?;
|
||||||
|
}
|
||||||
|
Ok::<_, RuntimeRestoreFailure>(())
|
||||||
|
}
|
||||||
|
.await;
|
||||||
|
// Never release allocation while the Worker guard is attached. Retained
|
||||||
|
// handles in the same operation remain available on every cleanup error.
|
||||||
|
drop(worker);
|
||||||
|
let cleanup = async {
|
||||||
|
tasks
|
||||||
|
.shutdown()
|
||||||
|
.await
|
||||||
|
.map_err(|_| cause("recovery Feature settlement failed"))?;
|
||||||
|
sessions
|
||||||
|
.close_all()
|
||||||
|
.await
|
||||||
|
.map_err(|_| cause("recovery Workdir close failed"))?;
|
||||||
|
if let Some(allocation) = allocation {
|
||||||
|
allocation
|
||||||
|
.release()
|
||||||
|
.map_err(|_| cause("recovery allocation release failed"))?;
|
||||||
|
}
|
||||||
|
Ok::<_, RuntimeRestoreFailure>(())
|
||||||
|
}
|
||||||
|
.await;
|
||||||
|
result?;
|
||||||
|
cleanup?;
|
||||||
|
self.failed_restore_resources
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||||
|
.remove(&request.worker_ref);
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,927 @@
|
|||||||
|
// Included by the production Profile publication suite. Each fault is at a
|
||||||
|
// distinct partial-publication boundary; no factory result is substituted.
|
||||||
|
|
||||||
|
fn publication_gate<T>() -> (
|
||||||
|
mpsc::Sender<()>,
|
||||||
|
mpsc::Receiver<T>,
|
||||||
|
crate::execution::RestoreTestGate<T>,
|
||||||
|
) {
|
||||||
|
let (release, receiver) = mpsc::channel();
|
||||||
|
let (entered, evidence) = mpsc::channel();
|
||||||
|
(
|
||||||
|
release,
|
||||||
|
evidence,
|
||||||
|
crate::execution::RestoreTestGate {
|
||||||
|
entered,
|
||||||
|
release: receiver,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_single_publication(fixture: &Fixture, expected: usize) {
|
||||||
|
let faults = &fixture.backend.factory.restore_faults;
|
||||||
|
assert_eq!(
|
||||||
|
faults.executions.load(Ordering::SeqCst),
|
||||||
|
expected,
|
||||||
|
"construction reran"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
faults.effects_calls.load(Ordering::SeqCst),
|
||||||
|
expected,
|
||||||
|
"restore effects reran"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
faults.publication_attempts.load(Ordering::SeqCst),
|
||||||
|
expected,
|
||||||
|
"Controller start reran"
|
||||||
|
);
|
||||||
|
fixture.assert_quiet();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_real_attach(
|
||||||
|
fixture: &Fixture,
|
||||||
|
handle: &WorkerHandle,
|
||||||
|
) -> crate::execution::WorkerProtocolTransport {
|
||||||
|
let worker_ref = &fixture.request.worker_ref;
|
||||||
|
assert!(matches!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.worker_session(crate::execution::WorkerSessionObservationRequest {
|
||||||
|
worker_ref: worker_ref.clone(),
|
||||||
|
}),
|
||||||
|
runtime_api::WorkerSessionAvailability::LiveProtocol
|
||||||
|
));
|
||||||
|
let transport = fixture
|
||||||
|
.backend
|
||||||
|
.clone()
|
||||||
|
.attach_worker_protocol(worker_ref)
|
||||||
|
.unwrap();
|
||||||
|
transport.validate().unwrap();
|
||||||
|
let paired = subscribe_worker_protocol_session(handle);
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(&transport.snapshot).unwrap(),
|
||||||
|
serde_json::to_value(paired.snapshot_event).unwrap()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
matches!(&transport.snapshot, Event::Snapshot { session, greeting, state, in_flight, .. }
|
||||||
|
if session == &session_store::public_snapshot::project_current_session_snapshot(&fixture.entries())
|
||||||
|
&& !greeting.tools.is_empty() && state.state == protocol::WorkerState::Idle && in_flight.is_empty())
|
||||||
|
);
|
||||||
|
transport
|
||||||
|
}
|
||||||
|
|
||||||
|
fn observe_publication_markers(
|
||||||
|
fixture: &Fixture,
|
||||||
|
observations: tokio::sync::broadcast::Receiver<crate::observation::WorkerObservationEvent>,
|
||||||
|
markers: &'static [&'static str],
|
||||||
|
) -> tokio::sync::broadcast::Receiver<crate::observation::WorkerObservationEvent> {
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.run_on_adapter_runtime(async move {
|
||||||
|
let mut observations = observations;
|
||||||
|
let mut counts = vec![0; markers.len()];
|
||||||
|
let mut snapshots = 0;
|
||||||
|
tokio::time::timeout(WAIT, async {
|
||||||
|
loop {
|
||||||
|
let observed = observations.recv().await.unwrap();
|
||||||
|
if matches!(observed.payload, Event::Snapshot { .. }) {
|
||||||
|
snapshots += 1;
|
||||||
|
}
|
||||||
|
for (index, marker) in markers.iter().enumerate() {
|
||||||
|
if is_marker(&observed.payload, marker) {
|
||||||
|
counts[index] += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if counts.last() == Some(&1) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(|_| format!("lost original subscription markers: {counts:?}"))?;
|
||||||
|
assert_eq!(
|
||||||
|
counts,
|
||||||
|
vec![1; markers.len()],
|
||||||
|
"lost or duplicated prefix on original receiver"
|
||||||
|
);
|
||||||
|
assert_eq!(snapshots, 1, "initial full snapshot was lost or replayed");
|
||||||
|
while let Ok(event) = observations.try_recv() {
|
||||||
|
assert!(
|
||||||
|
!markers
|
||||||
|
.iter()
|
||||||
|
.any(|marker| is_marker(&event.payload, marker)),
|
||||||
|
"duplicate after tail fence"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Ok(observations)
|
||||||
|
})
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn partial_registration_recovers_same_controller(hub: bool) {
|
||||||
|
let mut fixture = Fixture::no_model();
|
||||||
|
Arc::get_mut(&mut fixture.backend)
|
||||||
|
.unwrap()
|
||||||
|
.spawn_restore_timeout = Duration::from_millis(250);
|
||||||
|
let runtime = stopped_runtime(&fixture);
|
||||||
|
let pause = runtime.test_pause_restore_continuation();
|
||||||
|
let worker_ref = fixture.request.worker_ref.clone();
|
||||||
|
let scope = fixture.request.workspace_scope.as_ref().unwrap();
|
||||||
|
let request = runtime.test_restore_request(&worker_ref);
|
||||||
|
let observations = runtime.subscribe_worker_observation().unwrap();
|
||||||
|
let (started_release, started_entered) = activation_gate(&fixture, true);
|
||||||
|
let first = runtime
|
||||||
|
.restore_worker_operation_scoped(scope, &worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
first.state,
|
||||||
|
server_api::WorkerRestoreState::ReconciliationRequired
|
||||||
|
);
|
||||||
|
let started = started_entered.recv_timeout(WAIT).unwrap();
|
||||||
|
let pending = assert_activation_owned(&fixture, &runtime, &request.request_id, &started);
|
||||||
|
let old = started.handle.as_ref().unwrap().clone();
|
||||||
|
let canonical = fixture.canonical_files();
|
||||||
|
alert_marker(&old, "registration-original-paired-prefix");
|
||||||
|
let (release, entered, fault) = publication_gate();
|
||||||
|
if hub {
|
||||||
|
*fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.observation_hub
|
||||||
|
.registration_gate
|
||||||
|
.lock()
|
||||||
|
.unwrap() = Some(RestoreHubRegistrationGate {
|
||||||
|
gate: fault,
|
||||||
|
fail: true,
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
*fixture.backend.bridge_fault.lock().unwrap() = Some(fault);
|
||||||
|
}
|
||||||
|
started_release.release();
|
||||||
|
drop(pause);
|
||||||
|
let boundary_handle = entered.recv_timeout(WAIT).unwrap();
|
||||||
|
assert!(boundary_handle.same_controller(&old));
|
||||||
|
let durable = durable_operation(&runtime, &worker_ref).unwrap();
|
||||||
|
assert_eq!(durable.operation_id, started.operation_id);
|
||||||
|
assert_eq!(durable.commit, crate::runtime::RestoreCommitFact::Committed);
|
||||||
|
assert!(restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"].is_null());
|
||||||
|
if hub {
|
||||||
|
let (_, shared, _) = fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.observation_hub
|
||||||
|
.get(&worker_ref)
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
Arc::ptr_eq(&shared, &old.shared_state),
|
||||||
|
"real hub insert preceded fault"
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
assert!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.observation_hub
|
||||||
|
.get(&worker_ref)
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let (operation, context, uncertain) = runtime.test_pending_restore_context(&worker_ref);
|
||||||
|
assert_eq!(operation, started.operation_id);
|
||||||
|
assert!(!uncertain);
|
||||||
|
assert!(
|
||||||
|
context
|
||||||
|
.test_buffered_events()
|
||||||
|
.iter()
|
||||||
|
.any(|event| matches!(event, Event::Snapshot { .. }))
|
||||||
|
);
|
||||||
|
alert_marker(&old, "registration-during-fault");
|
||||||
|
let pause = runtime.test_pause_restore_continuation();
|
||||||
|
release.send(()).unwrap();
|
||||||
|
// This lock also observes the end of the actual registration attempt, not
|
||||||
|
// merely the test's entered notification. Its retained scope owns cleanup.
|
||||||
|
let execution = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
|
||||||
|
assert!(old.same_controller(&execution.handle));
|
||||||
|
assert_eq!(execution.restore_operation_id, Some(operation));
|
||||||
|
assert_eq!(
|
||||||
|
execution.tasks.tasks.lock().unwrap().len(),
|
||||||
|
if hub { 2 } else { 1 }
|
||||||
|
);
|
||||||
|
let bridge = execution.tasks.protocol_bridge.lock().unwrap();
|
||||||
|
if hub {
|
||||||
|
assert!(bridge.is_none());
|
||||||
|
} else {
|
||||||
|
let bridge = bridge.as_ref().unwrap();
|
||||||
|
assert!(
|
||||||
|
!bridge.snapshot_pending,
|
||||||
|
"snapshot already registered, do not replay it"
|
||||||
|
);
|
||||||
|
let streams = bridge.streams.lock().unwrap();
|
||||||
|
assert!(
|
||||||
|
streams.is_some(),
|
||||||
|
"original paired streams survived failed task admission"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(&streams.as_ref().unwrap().snapshot_event).unwrap(),
|
||||||
|
serde_json::to_value(&started.snapshot).unwrap()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
drop(bridge);
|
||||||
|
assert!(
|
||||||
|
!old.protocol_is_closed(),
|
||||||
|
"publication failure must not clean a committed Controller"
|
||||||
|
);
|
||||||
|
assert_single_publication(&fixture, 1);
|
||||||
|
drop(pause);
|
||||||
|
runtime.test_wait_restore_continuation(&worker_ref, operation, WAIT);
|
||||||
|
assert!(durable_operation(&runtime, &worker_ref).is_none());
|
||||||
|
let owner = restore_owner(&runtime, &worker_ref, &request.request_id);
|
||||||
|
assert_eq!(
|
||||||
|
owner["operation_id"],
|
||||||
|
serde_json::to_value(operation).unwrap()
|
||||||
|
);
|
||||||
|
assert_eq!(owner["receipt"]["state"], "accepted");
|
||||||
|
assert!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.pending_restore(&worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert!(!pending.task_in_flight.load(Ordering::Acquire));
|
||||||
|
assert!(pending.activated.lock().unwrap().is_none());
|
||||||
|
assert!(pending.result.lock().unwrap().is_none());
|
||||||
|
assert!(pending.task.try_lock().unwrap().is_none());
|
||||||
|
assert_eq!(fixture.backend.workers.lock().unwrap().len(), 1);
|
||||||
|
assert!(old.same_controller(&fixture.backend.workers.lock().unwrap()[&worker_ref].handle));
|
||||||
|
let transport = assert_real_attach(&fixture, &old);
|
||||||
|
assert_eq!(fixture.canonical_files(), canonical);
|
||||||
|
assert_single_publication(&fixture, 1);
|
||||||
|
alert_marker(&old, "registration-tail-fence");
|
||||||
|
let observations = observe_publication_markers(
|
||||||
|
&fixture,
|
||||||
|
observations,
|
||||||
|
&[
|
||||||
|
"registration-original-paired-prefix",
|
||||||
|
"registration-during-fault",
|
||||||
|
"registration-tail-fence",
|
||||||
|
],
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.restore_worker_operation_scoped(scope, &worker_ref, request.clone())
|
||||||
|
.unwrap()
|
||||||
|
.state,
|
||||||
|
server_api::WorkerRestoreState::Accepted
|
||||||
|
);
|
||||||
|
assert_single_publication(&fixture, 1);
|
||||||
|
// Retain the original transport/handle and scope through Stop/replacement.
|
||||||
|
runtime
|
||||||
|
.stop_worker_scoped(scope, &worker_ref, None)
|
||||||
|
.unwrap();
|
||||||
|
assert!(old.protocol_is_closed());
|
||||||
|
assert!(transport.validate().is_err());
|
||||||
|
let replacement_request = runtime.test_restore_request(&worker_ref);
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.restore_worker_operation_scoped(scope, &worker_ref, replacement_request.clone())
|
||||||
|
.unwrap()
|
||||||
|
.state,
|
||||||
|
server_api::WorkerRestoreState::Accepted
|
||||||
|
);
|
||||||
|
let replacement = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
|
||||||
|
assert!(!old.same_controller(&replacement.handle));
|
||||||
|
assert_ne!(replacement.restore_operation_id, Some(operation));
|
||||||
|
assert!(
|
||||||
|
transport
|
||||||
|
.dispatch(Method::Pause {
|
||||||
|
command: protocol::WorkerCommandEnvelope::new(801)
|
||||||
|
})
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
transport
|
||||||
|
.dispatch(Method::Shutdown {
|
||||||
|
command: protocol::WorkerCommandEnvelope::new(802)
|
||||||
|
})
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
for method in [
|
||||||
|
Method::Pause {
|
||||||
|
command: protocol::WorkerCommandEnvelope::new(803),
|
||||||
|
},
|
||||||
|
Method::Shutdown {
|
||||||
|
command: protocol::WorkerCommandEnvelope::new(804),
|
||||||
|
},
|
||||||
|
] {
|
||||||
|
let old_command = old.clone();
|
||||||
|
assert!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.run_on_adapter_runtime(async move {
|
||||||
|
old_command
|
||||||
|
.send(method)
|
||||||
|
.await
|
||||||
|
.map_err(|error| error.to_string())
|
||||||
|
})
|
||||||
|
.is_err(),
|
||||||
|
"closed original Controller command must not route to replacement"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.deactivate_stopped_controller(&worker_ref, &old)
|
||||||
|
.unwrap();
|
||||||
|
let (_, shared, _) = fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.observation_hub
|
||||||
|
.get(&worker_ref)
|
||||||
|
.unwrap();
|
||||||
|
assert!(Arc::ptr_eq(&shared, &replacement.handle.shared_state));
|
||||||
|
alert_marker(&old, "registration-stopped-event-must-not-leak");
|
||||||
|
alert_marker(&replacement.handle, "registration-replacement-fence");
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.run_on_adapter_runtime(async move {
|
||||||
|
let mut observations = observations;
|
||||||
|
tokio::time::timeout(WAIT, async {
|
||||||
|
loop {
|
||||||
|
let event = observations.recv().await.unwrap();
|
||||||
|
assert!(!is_marker(
|
||||||
|
&event.payload,
|
||||||
|
"registration-stopped-event-must-not-leak"
|
||||||
|
));
|
||||||
|
if is_marker(&event.payload, "registration-replacement-fence") {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(|_| "replacement observation fence missing".to_string())?;
|
||||||
|
drop(transport);
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_single_publication(&fixture, 2);
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.restore_worker_operation_scoped(scope, &worker_ref, request.clone())
|
||||||
|
.unwrap()
|
||||||
|
.state,
|
||||||
|
server_api::WorkerRestoreState::Accepted
|
||||||
|
);
|
||||||
|
assert_eq!(restore_owner_count(&runtime, &worker_ref), 2);
|
||||||
|
let current = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
|
||||||
|
assert!(current.handle.same_controller(&replacement.handle));
|
||||||
|
assert_eq!(
|
||||||
|
current.restore_operation_id,
|
||||||
|
replacement.restore_operation_id
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
replacement.handle.shared_state.snapshot().state,
|
||||||
|
protocol::WorkerState::Idle
|
||||||
|
);
|
||||||
|
drop(current);
|
||||||
|
drop(execution);
|
||||||
|
drop(replacement);
|
||||||
|
drop(pending);
|
||||||
|
runtime
|
||||||
|
.stop_worker_scoped(scope, &worker_ref, None)
|
||||||
|
.unwrap();
|
||||||
|
fixture.assert_released();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn restore_owner_count(runtime: &EmbeddedRuntime, worker_ref: &WorkerRef) -> usize {
|
||||||
|
let saved = runtime
|
||||||
|
.test_runtime_store()
|
||||||
|
.read_worker_record(worker_ref)
|
||||||
|
.unwrap();
|
||||||
|
serde_json::to_value(saved.restore_guard).unwrap()["owners"]
|
||||||
|
.as_object()
|
||||||
|
.unwrap()
|
||||||
|
.len()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn bridge_partial_snapshot_registration_retains_original_streams_and_autonomously_recovers() {
|
||||||
|
partial_registration_recovers_same_controller(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn hub_partial_insert_registration_retains_original_controller_and_autonomously_recovers() {
|
||||||
|
partial_registration_recovers_same_controller(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn context_partial_flush_retains_only_undelivered_queue_and_recovers_without_repost() {
|
||||||
|
let mut fixture = Fixture::no_model();
|
||||||
|
Arc::get_mut(&mut fixture.backend)
|
||||||
|
.unwrap()
|
||||||
|
.spawn_restore_timeout = Duration::from_millis(250);
|
||||||
|
let runtime = stopped_runtime(&fixture);
|
||||||
|
let pause = runtime.test_pause_restore_continuation();
|
||||||
|
let worker_ref = fixture.request.worker_ref.clone();
|
||||||
|
let scope = fixture.request.workspace_scope.as_ref().unwrap();
|
||||||
|
let request = runtime.test_restore_request(&worker_ref);
|
||||||
|
let observations = runtime.subscribe_worker_observation().unwrap();
|
||||||
|
let (started_release, started_entered) = activation_gate(&fixture, true);
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.restore_worker_operation_scoped(scope, &worker_ref, request.clone())
|
||||||
|
.unwrap()
|
||||||
|
.state,
|
||||||
|
server_api::WorkerRestoreState::ReconciliationRequired
|
||||||
|
);
|
||||||
|
let started = started_entered.recv_timeout(WAIT).unwrap();
|
||||||
|
let pending = assert_activation_owned(&fixture, &runtime, &request.request_id, &started);
|
||||||
|
let old = started.handle.unwrap();
|
||||||
|
let (operation, context, _) = runtime.test_pending_restore_context(&worker_ref);
|
||||||
|
// Stop after the actual bridge and hub registration, without failing that
|
||||||
|
// boundary, so the flush begins with its real original full snapshot.
|
||||||
|
let (hub_release, hub_entered, hub_gate) = publication_gate();
|
||||||
|
*fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.observation_hub
|
||||||
|
.registration_gate
|
||||||
|
.lock()
|
||||||
|
.unwrap() = Some(RestoreHubRegistrationGate {
|
||||||
|
gate: hub_gate,
|
||||||
|
fail: false,
|
||||||
|
});
|
||||||
|
started_release.release();
|
||||||
|
drop(pause);
|
||||||
|
assert!(
|
||||||
|
hub_entered
|
||||||
|
.recv_timeout(WAIT)
|
||||||
|
.unwrap()
|
||||||
|
.same_controller(&old)
|
||||||
|
);
|
||||||
|
let buffered_prefix = context.test_buffered_events();
|
||||||
|
assert!(matches!(
|
||||||
|
buffered_prefix.first(),
|
||||||
|
Some(Event::Snapshot { .. })
|
||||||
|
));
|
||||||
|
let prefix_length = buffered_prefix.len();
|
||||||
|
let marker = |text: &str| Event::Error {
|
||||||
|
code: protocol::ErrorCode::Internal,
|
||||||
|
message: text.into(),
|
||||||
|
};
|
||||||
|
context
|
||||||
|
.publish_protocol_event(marker("flush-delivered"))
|
||||||
|
.unwrap();
|
||||||
|
context
|
||||||
|
.publish_protocol_event(marker("flush-failed"))
|
||||||
|
.unwrap();
|
||||||
|
context
|
||||||
|
.publish_protocol_event(marker("flush-pending"))
|
||||||
|
.unwrap();
|
||||||
|
let (release, entered, gate) = publication_gate();
|
||||||
|
let (restored, restored_queue) = mpsc::channel();
|
||||||
|
context.test_fail_flush(crate::execution::RestoreFlushFault {
|
||||||
|
remaining_successes: prefix_length + 1,
|
||||||
|
gate,
|
||||||
|
restored,
|
||||||
|
});
|
||||||
|
drop(hub_release);
|
||||||
|
assert!(
|
||||||
|
matches!(entered.recv_timeout(WAIT).unwrap(), Event::Error { message, .. } if message == "flush-failed")
|
||||||
|
);
|
||||||
|
let partial_execution = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
|
||||||
|
assert!(old.same_controller(&partial_execution.handle));
|
||||||
|
assert_eq!(partial_execution.restore_operation_id, Some(operation));
|
||||||
|
assert_eq!(partial_execution.tasks.tasks.lock().unwrap().len(), 2);
|
||||||
|
let (_, shared, _) = fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.observation_hub
|
||||||
|
.get(&worker_ref)
|
||||||
|
.unwrap();
|
||||||
|
assert!(Arc::ptr_eq(&shared, &old.shared_state));
|
||||||
|
let pause = runtime.test_pause_restore_continuation();
|
||||||
|
context
|
||||||
|
.publish_protocol_event(marker("flush-arrived-during-error"))
|
||||||
|
.unwrap();
|
||||||
|
release.send(()).unwrap();
|
||||||
|
let restored = restored_queue.recv_timeout(WAIT).unwrap();
|
||||||
|
let undelivered: Vec<_> = restored
|
||||||
|
.iter()
|
||||||
|
.filter_map(|event| match event {
|
||||||
|
Event::Error { message, .. } if message.starts_with("flush-") => Some(message.as_str()),
|
||||||
|
_ => None,
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
assert_eq!(
|
||||||
|
undelivered,
|
||||||
|
[
|
||||||
|
"flush-failed",
|
||||||
|
"flush-pending",
|
||||||
|
"flush-arrived-during-error"
|
||||||
|
]
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!restored
|
||||||
|
.iter()
|
||||||
|
.any(|event| matches!(event, Event::Snapshot { .. })),
|
||||||
|
"delivered snapshot must not rejoin the failed suffix"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(context.test_buffered_events()).unwrap(),
|
||||||
|
serde_json::to_value(&restored).unwrap()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
durable_operation(&runtime, &worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.operation_id,
|
||||||
|
operation
|
||||||
|
);
|
||||||
|
assert!(restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"].is_null());
|
||||||
|
// The next terminal gate observes successful retry only after the same
|
||||||
|
// queued owner has repaired its failed callback and remaining suffix.
|
||||||
|
let (terminal_release, terminal_entered, gate) = publication_gate();
|
||||||
|
runtime.test_terminal_receipt_gate(gate);
|
||||||
|
drop(pause);
|
||||||
|
assert_eq!(terminal_entered.recv_timeout(WAIT).unwrap(), operation);
|
||||||
|
assert!(context.test_buffered_events().is_empty());
|
||||||
|
assert!(restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"].is_null());
|
||||||
|
assert_eq!(
|
||||||
|
durable_operation(&runtime, &worker_ref).unwrap().commit,
|
||||||
|
crate::runtime::RestoreCommitFact::Committed
|
||||||
|
);
|
||||||
|
let execution = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
|
||||||
|
assert!(old.same_controller(&execution.handle));
|
||||||
|
assert_eq!(execution.restore_operation_id, Some(operation));
|
||||||
|
drop(terminal_release); // RAII disconnect, not a second request.
|
||||||
|
runtime.test_wait_restore_continuation(&worker_ref, operation, WAIT);
|
||||||
|
assert_eq!(
|
||||||
|
restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"]["state"],
|
||||||
|
"accepted"
|
||||||
|
);
|
||||||
|
assert_real_attach(&fixture, &old);
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.run_on_adapter_runtime(async move {
|
||||||
|
let mut observations = observations;
|
||||||
|
let mut messages = Vec::new();
|
||||||
|
let mut snapshots = 0;
|
||||||
|
while let Ok(event) = observations.try_recv() {
|
||||||
|
match event.payload {
|
||||||
|
Event::Error { message, .. } if message.starts_with("flush-") => {
|
||||||
|
messages.push(message)
|
||||||
|
}
|
||||||
|
Event::Snapshot { .. } => snapshots += 1,
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
messages,
|
||||||
|
[
|
||||||
|
"flush-delivered",
|
||||||
|
"flush-failed",
|
||||||
|
"flush-pending",
|
||||||
|
"flush-arrived-during-error"
|
||||||
|
]
|
||||||
|
);
|
||||||
|
assert_eq!(snapshots, 1);
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_single_publication(&fixture, 1);
|
||||||
|
drop(partial_execution);
|
||||||
|
drop(execution);
|
||||||
|
drop(pending);
|
||||||
|
runtime
|
||||||
|
.stop_worker_scoped(scope, &worker_ref, None)
|
||||||
|
.unwrap();
|
||||||
|
fixture.assert_released();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug)]
|
||||||
|
enum TerminalFault {
|
||||||
|
BeforeRename,
|
||||||
|
AfterRename,
|
||||||
|
SyncUnknown,
|
||||||
|
ReadUnknown,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn published_terminal_receipt_fault(fault: TerminalFault) {
|
||||||
|
let mut fixture = Fixture::no_model();
|
||||||
|
// Reuse the existing ordinary non-network model boundary, not the product
|
||||||
|
// Codex profile. All fault/convergence assertions precede the sole input.
|
||||||
|
let script = MockClient::sequential(vec![MockResponse::Complete(simple_text_events())]);
|
||||||
|
let injected = script.clone();
|
||||||
|
let factory = fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.as_ref()
|
||||||
|
.clone()
|
||||||
|
.with_spawn_model_client_factory(move || Box::new(injected.clone()));
|
||||||
|
fixture.backend = Arc::new(WorkerRuntimeExecutionBackend::new(factory).unwrap());
|
||||||
|
let runtime = Arc::new(stopped_runtime(&fixture));
|
||||||
|
let pause = runtime.test_pause_restore_continuation();
|
||||||
|
let worker_ref = fixture.request.worker_ref.clone();
|
||||||
|
let scope = fixture.request.workspace_scope.clone().unwrap();
|
||||||
|
let request = runtime.test_restore_request(&worker_ref);
|
||||||
|
let observations = runtime.subscribe_worker_observation().unwrap();
|
||||||
|
let mut catalog = runtime
|
||||||
|
.subscribe_event_selector_scoped(
|
||||||
|
&scope,
|
||||||
|
protocol::subscription::EventSubscriptionSelector::RuntimeWorkers,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let (release, entered, gate) = publication_gate();
|
||||||
|
runtime.test_terminal_receipt_gate(gate);
|
||||||
|
let restoring = {
|
||||||
|
let runtime = runtime.clone();
|
||||||
|
let worker_ref = worker_ref.clone();
|
||||||
|
let scope = scope.clone();
|
||||||
|
let request = request.clone();
|
||||||
|
std::thread::spawn(move || {
|
||||||
|
runtime.restore_worker_operation_scoped(&scope, &worker_ref, request)
|
||||||
|
})
|
||||||
|
};
|
||||||
|
let operation = entered.recv_timeout(WAIT).unwrap();
|
||||||
|
// Restore legitimately repairs derived metadata before publication. Receipt
|
||||||
|
// failures must not rerun those effects; compare from this actual boundary.
|
||||||
|
let canonical = fixture.canonical_files();
|
||||||
|
let execution = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
|
||||||
|
let old = execution.handle.clone();
|
||||||
|
assert_eq!(execution.restore_operation_id, Some(operation));
|
||||||
|
assert!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.pending_restore(&worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert_eq!(execution.tasks.tasks.lock().unwrap().len(), 2);
|
||||||
|
assert!(execution.tasks.protocol_bridge.lock().unwrap().is_none());
|
||||||
|
assert_eq!(
|
||||||
|
durable_operation(&runtime, &worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.operation_id,
|
||||||
|
operation
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
durable_operation(&runtime, &worker_ref).unwrap().commit,
|
||||||
|
crate::runtime::RestoreCommitFact::Committed
|
||||||
|
);
|
||||||
|
let owner = restore_owner(&runtime, &worker_ref, &request.request_id);
|
||||||
|
assert!(owner["receipt"].is_null());
|
||||||
|
let (same_operation, context, uncertain) = runtime.test_pending_restore_context(&worker_ref);
|
||||||
|
assert_eq!(same_operation, operation);
|
||||||
|
assert!(!uncertain);
|
||||||
|
assert!(context.test_buffered_events().is_empty());
|
||||||
|
let transport = assert_real_attach(&fixture, &old);
|
||||||
|
assert_single_publication(&fixture, 1);
|
||||||
|
alert_marker(&old, "terminal-before-receipt");
|
||||||
|
let store = runtime.test_runtime_store();
|
||||||
|
let reconcile_before = store.test_worker_reconcile_counts();
|
||||||
|
store.fail_next_worker_write(if matches!(fault, TerminalFault::BeforeRename) {
|
||||||
|
AtomicWriteFault::BeforeRename
|
||||||
|
} else {
|
||||||
|
AtomicWriteFault::AfterRename
|
||||||
|
});
|
||||||
|
match fault {
|
||||||
|
TerminalFault::SyncUnknown => store.fail_next_worker_reconcile_sync(),
|
||||||
|
TerminalFault::ReadUnknown => store.fail_next_worker_reconcile_read(),
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
release.send(()).unwrap();
|
||||||
|
let first = restoring.join().unwrap().unwrap();
|
||||||
|
let reconcile_after = store.test_worker_reconcile_counts();
|
||||||
|
let reconciliation = (
|
||||||
|
reconcile_after.0 - reconcile_before.0,
|
||||||
|
reconcile_after.1 - reconcile_before.1,
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
reconciliation,
|
||||||
|
match fault {
|
||||||
|
TerminalFault::BeforeRename => (0, 0),
|
||||||
|
TerminalFault::AfterRename => (1, 4), // canonical file plus Worker/workers/root directories
|
||||||
|
TerminalFault::SyncUnknown => (1, 1),
|
||||||
|
TerminalFault::ReadUnknown => (1, 0),
|
||||||
|
},
|
||||||
|
"{fault:?}: actual canonical read/sync attempts"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
first.state,
|
||||||
|
if matches!(fault, TerminalFault::AfterRename) {
|
||||||
|
server_api::WorkerRestoreState::Accepted
|
||||||
|
} else {
|
||||||
|
server_api::WorkerRestoreState::ReconciliationRequired
|
||||||
|
},
|
||||||
|
"{fault:?}: {first:?}"
|
||||||
|
);
|
||||||
|
assert_eq!(fixture.canonical_files(), canonical);
|
||||||
|
assert!(!old.protocol_is_closed());
|
||||||
|
transport.validate().unwrap();
|
||||||
|
assert_single_publication(&fixture, 1);
|
||||||
|
let disk_owner = restore_owner(&runtime, &worker_ref, &request.request_id);
|
||||||
|
assert_eq!(disk_owner["operation_id"], owner["operation_id"]);
|
||||||
|
match fault {
|
||||||
|
TerminalFault::BeforeRename => {
|
||||||
|
assert!(disk_owner["receipt"].is_null());
|
||||||
|
assert!(!runtime.test_pending_restore_context(&worker_ref).2);
|
||||||
|
}
|
||||||
|
TerminalFault::AfterRename => {
|
||||||
|
assert_eq!(disk_owner["receipt"]["state"], "accepted");
|
||||||
|
assert!(durable_operation(&runtime, &worker_ref).is_none());
|
||||||
|
}
|
||||||
|
TerminalFault::SyncUnknown | TerminalFault::ReadUnknown => {
|
||||||
|
assert_eq!(
|
||||||
|
disk_owner["receipt"]["state"], "accepted",
|
||||||
|
"visible rename alone is not durable proof"
|
||||||
|
);
|
||||||
|
let (same_operation, _, fenced) = runtime.test_pending_restore_context(&worker_ref);
|
||||||
|
assert_eq!(same_operation, operation);
|
||||||
|
assert!(
|
||||||
|
fenced,
|
||||||
|
"unknown canonical outcome must retain the memory owner/write fence"
|
||||||
|
);
|
||||||
|
// Keep durability unproved for one competing Stop as well: it must
|
||||||
|
// fail before any Shutdown, never manufacture a Stopped judgment.
|
||||||
|
match fault {
|
||||||
|
TerminalFault::SyncUnknown => store.fail_next_worker_reconcile_sync(),
|
||||||
|
TerminalFault::ReadUnknown => store.fail_next_worker_reconcile_read(),
|
||||||
|
_ => unreachable!(),
|
||||||
|
}
|
||||||
|
assert!(matches!(
|
||||||
|
runtime.stop_worker_scoped(&scope, &worker_ref, None),
|
||||||
|
Err(crate::error::RuntimeError::StoreCommitOutcomeUnknown { .. })
|
||||||
|
));
|
||||||
|
assert!(!old.protocol_is_closed());
|
||||||
|
assert!(!execution.shutdown_requested.load(Ordering::Acquire));
|
||||||
|
assert!(runtime.test_pending_restore_context(&worker_ref).2);
|
||||||
|
assert_single_publication(&fixture, 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Recovery is autonomous after the failed write / confirmed unknown fence.
|
||||||
|
// No POST, fresh request, or input is needed to make it converge.
|
||||||
|
drop(pause);
|
||||||
|
if !matches!(fault, TerminalFault::AfterRename) {
|
||||||
|
runtime.test_wait_restore_continuation(&worker_ref, operation, WAIT);
|
||||||
|
}
|
||||||
|
assert!(durable_operation(&runtime, &worker_ref).is_none());
|
||||||
|
assert_eq!(
|
||||||
|
restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"]["state"],
|
||||||
|
"accepted"
|
||||||
|
);
|
||||||
|
let current = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
|
||||||
|
assert!(old.same_controller(¤t.handle));
|
||||||
|
assert_eq!(current.restore_operation_id, Some(operation));
|
||||||
|
assert_eq!(fixture.backend.workers.lock().unwrap().len(), 1);
|
||||||
|
assert_real_attach(&fixture, &old);
|
||||||
|
assert_single_publication(&fixture, 1);
|
||||||
|
// A canonical terminal reread may bypass the final redundant upsert. The
|
||||||
|
// original subscriber must still have the current state and observation token.
|
||||||
|
let fresh_catalog = runtime
|
||||||
|
.subscribe_event_selector_scoped(
|
||||||
|
&scope,
|
||||||
|
protocol::subscription::EventSubscriptionSelector::RuntimeWorkers,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let protocol::subscription::SubscriptionSnapshot::Workers { workers } =
|
||||||
|
fresh_catalog.snapshot()
|
||||||
|
else {
|
||||||
|
panic!("Runtime catalog must return Workers");
|
||||||
|
};
|
||||||
|
assert_eq!(workers.len(), 1);
|
||||||
|
let mut last_applied = None;
|
||||||
|
use futures::FutureExt;
|
||||||
|
while let Some(update) = catalog.recv().now_or_never() {
|
||||||
|
match update.unwrap().payload {
|
||||||
|
protocol::subscription::SubscriptionEventPayload::WorkerUpserted { worker } => {
|
||||||
|
last_applied = Some(worker);
|
||||||
|
}
|
||||||
|
payload => panic!("unexpected catalog update: {payload:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
serde_json::to_value(last_applied.expect("restore must publish its catalog state"))
|
||||||
|
.unwrap()
|
||||||
|
== serde_json::to_value(&workers[0]).unwrap(),
|
||||||
|
"original catalog subscriber diverged after {fault:?}"
|
||||||
|
);
|
||||||
|
drop(fresh_catalog);
|
||||||
|
drop(catalog);
|
||||||
|
alert_marker(&old, "terminal-after-recovery");
|
||||||
|
let _observations = observe_publication_markers(
|
||||||
|
&fixture,
|
||||||
|
observations,
|
||||||
|
&["terminal-before-receipt", "terminal-after-recovery"],
|
||||||
|
);
|
||||||
|
let accepted = runtime
|
||||||
|
.restore_worker_operation_scoped(&scope, &worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(accepted.state, server_api::WorkerRestoreState::Accepted);
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.restore_worker_operation_scoped(&scope, &worker_ref, request.clone())
|
||||||
|
.unwrap(),
|
||||||
|
accepted
|
||||||
|
);
|
||||||
|
assert_eq!(restore_owner_count(&runtime, &worker_ref), 1);
|
||||||
|
let mut stale = request;
|
||||||
|
stale.request_id = "terminal-fresh-stale-intent".into();
|
||||||
|
assert!(matches!(
|
||||||
|
runtime.restore_worker_operation_scoped(&scope, &worker_ref, stale),
|
||||||
|
Err(crate::error::RuntimeError::RestoreObservationConflict { .. })
|
||||||
|
));
|
||||||
|
assert_single_publication(&fixture, 1);
|
||||||
|
assert_eq!(
|
||||||
|
script.call_count.load(Ordering::SeqCst),
|
||||||
|
0,
|
||||||
|
"receipt recovery cannot trigger input or a model call"
|
||||||
|
);
|
||||||
|
// This transport was paired before the failed receipt write. Exercise its
|
||||||
|
// normal log-entry receiver after autonomous settlement, not a new attach.
|
||||||
|
transport
|
||||||
|
.dispatch(Method::submit_text(
|
||||||
|
"terminal-ordinary-input",
|
||||||
|
"terminal-ordinary-input",
|
||||||
|
))
|
||||||
|
.unwrap();
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.run_on_adapter_runtime(async move {
|
||||||
|
let mut transport = transport;
|
||||||
|
let (mut admitted, mut committed, mut text, mut ended) = (false, false, false, false);
|
||||||
|
tokio::time::timeout(WAIT, async {
|
||||||
|
while !(admitted && committed && text && ended) {
|
||||||
|
match transport.events.recv().await.unwrap() {
|
||||||
|
Event::SubmissionAccepted {
|
||||||
|
submission_request_id,
|
||||||
|
..
|
||||||
|
} if submission_request_id == "terminal-ordinary-input" => admitted = true,
|
||||||
|
Event::UserMessage { segments, .. } => {
|
||||||
|
assert_eq!(
|
||||||
|
segments,
|
||||||
|
vec![protocol::Segment::text("terminal-ordinary-input")]
|
||||||
|
);
|
||||||
|
assert!(!committed, "paired log receiver duplicated input");
|
||||||
|
committed = true;
|
||||||
|
}
|
||||||
|
Event::TextDone { text: output } => {
|
||||||
|
assert_eq!(output, "hello from worker");
|
||||||
|
text = true;
|
||||||
|
}
|
||||||
|
Event::RunEnd { .. } => ended = true,
|
||||||
|
Event::SubmissionRejected { .. } | Event::Error { .. } => {
|
||||||
|
panic!("ordinary input failed after receipt recovery")
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(|_| {
|
||||||
|
"ordinary input failed to settle on original paired transport".to_string()
|
||||||
|
})?;
|
||||||
|
drop(transport);
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(script.call_count.load(Ordering::SeqCst), 1);
|
||||||
|
assert_eq!(fixture.entries().iter().filter(|entry| matches!(entry,
|
||||||
|
LogEntry::AnnotatedUserInput { segments, .. } if segments == &vec![protocol::Segment::text("terminal-ordinary-input")]
|
||||||
|
)).count(), 1);
|
||||||
|
assert_single_publication(&fixture, 1);
|
||||||
|
drop(execution);
|
||||||
|
drop(current);
|
||||||
|
runtime
|
||||||
|
.stop_worker_scoped(&scope, &worker_ref, None)
|
||||||
|
.unwrap();
|
||||||
|
assert!(old.protocol_is_closed());
|
||||||
|
fixture.assert_released();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn published_controller_terminal_receipt_before_rename_autonomously_settles_without_reconstruction()
|
||||||
|
{
|
||||||
|
published_terminal_receipt_fault(TerminalFault::BeforeRename);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn published_controller_terminal_receipt_after_rename_is_canonically_reread_and_synced() {
|
||||||
|
published_terminal_receipt_fault(TerminalFault::AfterRename);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn published_controller_terminal_receipt_sync_unknown_retains_owner_and_write_fence() {
|
||||||
|
published_terminal_receipt_fault(TerminalFault::SyncUnknown);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn published_controller_terminal_receipt_read_unknown_retains_owner_and_write_fence() {
|
||||||
|
published_terminal_receipt_fault(TerminalFault::ReadUnknown);
|
||||||
|
}
|
||||||
@@ -1,23 +1,32 @@
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
#[cfg(feature = "http-server")]
|
||||||
|
use std::time::Duration;
|
||||||
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
use crate::workspace_request::{
|
use crate::workspace_request::{
|
||||||
RuntimeWorkspaceRequest, RuntimeWorkspaceRequestClient, RuntimeWorkspaceRequestError,
|
RuntimeWorkspaceRequest, RuntimeWorkspaceRequestClient, RuntimeWorkspaceRequestError,
|
||||||
};
|
};
|
||||||
|
use worker::WorkspaceResponse;
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
use worker::{
|
use worker::{
|
||||||
WorkspaceBinaryRequest, WorkspaceBinaryResponse, WorkspaceClient, WorkspaceClientError,
|
WorkspaceBinaryRequest, WorkspaceBinaryResponse, WorkspaceClient, WorkspaceClientError,
|
||||||
WorkspacePromptCatalogResolution, WorkspacePromptProjection, WorkspaceRequest,
|
WorkspacePromptCatalogResolution, WorkspacePromptProjection, WorkspaceRequest,
|
||||||
WorkspaceRequestMethod, WorkspaceResponse, WorkspaceServerOperation,
|
WorkspaceRequestMethod, WorkspaceServerOperation,
|
||||||
};
|
};
|
||||||
|
|
||||||
use crate::auth::{
|
use crate::auth::{
|
||||||
RuntimeAuthError, RuntimeIdentityMaterial, RuntimeWorkerMutationSourceSigner,
|
RuntimeAuthError, RuntimeIdentityMaterial, RuntimeWorkerMutationSourceSigner,
|
||||||
WORKER_REMOVE_PERMISSION, WORKSPACE_REQUEST_PERMISSION, WORKSPACE_WORKER_DISCOVERY_PERMISSION,
|
WORKER_REMOVE_PERMISSION, WorkerMutationActorKind, WorkerMutationOperation,
|
||||||
WorkerMutationActorKind, WorkerMutationOperation, WorkerMutationSourceClaims, new_token_id,
|
WorkerMutationSourceClaims, new_token_id,
|
||||||
};
|
};
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
|
use crate::auth::{WORKSPACE_REQUEST_PERMISSION, WORKSPACE_WORKER_DISCOVERY_PERMISSION};
|
||||||
use crate::runtime::RuntimeWorkspaceScope;
|
use crate::runtime::RuntimeWorkspaceScope;
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
use crate::worker_backend::WorkspacePromptProjectionCache;
|
use crate::worker_backend::WorkspacePromptProjectionCache;
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
fn ticket_assignment_role_segment(role: server_api::TicketAssignmentRole) -> &'static str {
|
fn ticket_assignment_role_segment(role: server_api::TicketAssignmentRole) -> &'static str {
|
||||||
match role {
|
match role {
|
||||||
server_api::TicketAssignmentRole::Orchestrator => "orchestrator",
|
server_api::TicketAssignmentRole::Orchestrator => "orchestrator",
|
||||||
@@ -144,6 +153,7 @@ pub trait EmbeddedWorkerMutationDispatcher: Send + Sync {
|
|||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
enum RuntimeWorkerMutationTransport {
|
enum RuntimeWorkerMutationTransport {
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
Remote {
|
Remote {
|
||||||
request_client: RuntimeWorkspaceRequestClient,
|
request_client: RuntimeWorkspaceRequestClient,
|
||||||
},
|
},
|
||||||
@@ -161,6 +171,7 @@ pub struct RuntimeWorkerMutationForwarder {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl RuntimeWorkerMutationForwarder {
|
impl RuntimeWorkerMutationForwarder {
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
pub fn remote(
|
pub fn remote(
|
||||||
identity: &RuntimeIdentityMaterial,
|
identity: &RuntimeIdentityMaterial,
|
||||||
scope: RuntimeWorkspaceScope,
|
scope: RuntimeWorkspaceScope,
|
||||||
@@ -206,6 +217,7 @@ impl RuntimeWorkerMutationForwarder {
|
|||||||
target_worker_id,
|
target_worker_id,
|
||||||
)?;
|
)?;
|
||||||
match (&self.transport, proof) {
|
match (&self.transport, proof) {
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
(
|
(
|
||||||
RuntimeWorkerMutationTransport::Remote { request_client },
|
RuntimeWorkerMutationTransport::Remote { request_client },
|
||||||
RuntimeOwnedWorkerMutationProof::Remote(token),
|
RuntimeOwnedWorkerMutationProof::Remote(token),
|
||||||
@@ -231,6 +243,7 @@ impl RuntimeWorkerMutationForwarder {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
struct RemoteWorkerRemoveHttpRequest {
|
struct RemoteWorkerRemoveHttpRequest {
|
||||||
request_client: RuntimeWorkspaceRequestClient,
|
request_client: RuntimeWorkspaceRequestClient,
|
||||||
source_worker_id: String,
|
source_worker_id: String,
|
||||||
@@ -240,6 +253,7 @@ struct RemoteWorkerRemoveHttpRequest {
|
|||||||
reason: String,
|
reason: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
fn execute_remote_worker_remove_http(
|
fn execute_remote_worker_remove_http(
|
||||||
request: RemoteWorkerRemoveHttpRequest,
|
request: RemoteWorkerRemoveHttpRequest,
|
||||||
) -> Result<WorkspaceResponse, RuntimeWorkerMutationForwardError> {
|
) -> Result<WorkspaceResponse, RuntimeWorkerMutationForwardError> {
|
||||||
@@ -263,6 +277,7 @@ fn execute_remote_worker_remove_http(
|
|||||||
execute_remote_worker_remove_http_blocking(request)
|
execute_remote_worker_remove_http_blocking(request)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
fn execute_remote_worker_remove_http_blocking(
|
fn execute_remote_worker_remove_http_blocking(
|
||||||
request: RemoteWorkerRemoveHttpRequest,
|
request: RemoteWorkerRemoveHttpRequest,
|
||||||
) -> Result<WorkspaceResponse, RuntimeWorkerMutationForwardError> {
|
) -> Result<WorkspaceResponse, RuntimeWorkerMutationForwardError> {
|
||||||
@@ -315,6 +330,7 @@ fn execute_remote_worker_remove_http_blocking(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct RuntimeOwnedWorkspaceClient {
|
pub struct RuntimeOwnedWorkspaceClient {
|
||||||
workspace_id: String,
|
workspace_id: String,
|
||||||
@@ -325,6 +341,7 @@ pub struct RuntimeOwnedWorkspaceClient {
|
|||||||
prompt_projection_cache: Option<Arc<WorkspacePromptProjectionCache>>,
|
prompt_projection_cache: Option<Arc<WorkspacePromptProjectionCache>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
impl RuntimeOwnedWorkspaceClient {
|
impl RuntimeOwnedWorkspaceClient {
|
||||||
pub fn new(
|
pub fn new(
|
||||||
workspace_id: impl Into<String>,
|
workspace_id: impl Into<String>,
|
||||||
@@ -495,6 +512,7 @@ impl RuntimeOwnedWorkspaceClient {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
impl std::fmt::Debug for RuntimeOwnedWorkspaceClient {
|
impl std::fmt::Debug for RuntimeOwnedWorkspaceClient {
|
||||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
formatter
|
formatter
|
||||||
@@ -510,6 +528,7 @@ impl std::fmt::Debug for RuntimeOwnedWorkspaceClient {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
fn generated_workspace_response<T: serde::Serialize>(
|
fn generated_workspace_response<T: serde::Serialize>(
|
||||||
response: Result<T, server_api::client_support::ClientError<server_api::RepositoryApiError>>,
|
response: Result<T, server_api::client_support::ClientError<server_api::RepositoryApiError>>,
|
||||||
) -> Result<WorkspaceResponse, WorkspaceClientError> {
|
) -> Result<WorkspaceResponse, WorkspaceClientError> {
|
||||||
@@ -530,6 +549,7 @@ fn generated_workspace_response<T: serde::Serialize>(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
impl WorkspaceClient for RuntimeOwnedWorkspaceClient {
|
impl WorkspaceClient for RuntimeOwnedWorkspaceClient {
|
||||||
fn workspace_id(&self) -> Option<&str> {
|
fn workspace_id(&self) -> Option<&str> {
|
||||||
Some(self.request_client.workspace_id())
|
Some(self.request_client.workspace_id())
|
||||||
@@ -929,12 +949,17 @@ fn unix_now_seconds() -> u64 {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
use crate::auth::{
|
use crate::auth::{
|
||||||
RUNTIME_REQUEST_SOURCE_PROOF_HEADER, WorkerMutationSourceExpectation,
|
RUNTIME_REQUEST_SOURCE_PROOF_HEADER, decode_runtime_request_source_claims,
|
||||||
decode_runtime_request_source_claims, decode_worker_mutation_source_claims,
|
request_body_digest,
|
||||||
request_body_digest, verify_worker_mutation_source_proof,
|
};
|
||||||
|
use crate::auth::{
|
||||||
|
WorkerMutationSourceExpectation, decode_worker_mutation_source_claims,
|
||||||
|
verify_worker_mutation_source_proof,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
fn binary_loopback(response: Vec<u8>) -> (String, std::thread::JoinHandle<(String, Vec<u8>)>) {
|
fn binary_loopback(response: Vec<u8>) -> (String, std::thread::JoinHandle<(String, Vec<u8>)>) {
|
||||||
use std::io::{Read, Write};
|
use std::io::{Read, Write};
|
||||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||||
@@ -972,6 +997,7 @@ mod tests {
|
|||||||
(base_url, server)
|
(base_url, server)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
#[test]
|
#[test]
|
||||||
fn generated_ticket_assignment_transport_signs_exact_authoring_and_bounded_read_subjects() {
|
fn generated_ticket_assignment_transport_signs_exact_authoring_and_bounded_read_subjects() {
|
||||||
use crate::auth::{RuntimeRequestSourceExpectation, verify_runtime_request_source};
|
use crate::auth::{RuntimeRequestSourceExpectation, verify_runtime_request_source};
|
||||||
@@ -1104,6 +1130,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
#[test]
|
#[test]
|
||||||
fn binary_roundtrip_signs_exact_query_method_worker_and_raw_body() {
|
fn binary_roundtrip_signs_exact_query_method_worker_and_raw_body() {
|
||||||
use crate::auth::{RuntimeRequestSourceExpectation, verify_runtime_request_source};
|
use crate::auth::{RuntimeRequestSourceExpectation, verify_runtime_request_source};
|
||||||
@@ -1194,6 +1221,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
#[test]
|
#[test]
|
||||||
fn binary_responses_enforce_declared_and_streamed_limits_with_fixed_errors() {
|
fn binary_responses_enforce_declared_and_streamed_limits_with_fixed_errors() {
|
||||||
for response in [
|
for response in [
|
||||||
@@ -1213,6 +1241,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
#[test]
|
#[test]
|
||||||
fn binary_redirect_returns_status_without_forwarding_authentication() {
|
fn binary_redirect_returns_status_without_forwarding_authentication() {
|
||||||
let (base_url, server) = binary_loopback(b"HTTP/1.1 307 Temporary Redirect\r\nLocation: http://127.0.0.1:1/private\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_vec());
|
let (base_url, server) = binary_loopback(b"HTTP/1.1 307 Temporary Redirect\r\nLocation: http://127.0.0.1:1/private\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_vec());
|
||||||
@@ -1231,6 +1260,7 @@ mod tests {
|
|||||||
assert!(result.body.is_empty());
|
assert!(result.body.is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
#[test]
|
#[test]
|
||||||
fn binary_invalid_or_normalized_targets_fail_with_fixed_errors_before_http() {
|
fn binary_invalid_or_normalized_targets_fail_with_fixed_errors_before_http() {
|
||||||
let client = RuntimeOwnedWorkspaceClient::new(
|
let client = RuntimeOwnedWorkspaceClient::new(
|
||||||
@@ -1277,6 +1307,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
#[test]
|
#[test]
|
||||||
fn current_prompt_projection_uses_the_shared_runtime_cache_without_http() {
|
fn current_prompt_projection_uses_the_shared_runtime_cache_without_http() {
|
||||||
let cache = Arc::new(WorkspacePromptProjectionCache::default());
|
let cache = Arc::new(WorkspacePromptProjectionCache::default());
|
||||||
@@ -1312,6 +1343,7 @@ mod tests {
|
|||||||
assert!(Arc::ptr_eq(&projection.catalog, &second.catalog));
|
assert!(Arc::ptr_eq(&projection.catalog, &second.catalog));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
#[test]
|
#[test]
|
||||||
fn prompt_projection_cache_miss_fetches_content_identity() {
|
fn prompt_projection_cache_miss_fetches_content_identity() {
|
||||||
use std::io::{Read, Write};
|
use std::io::{Read, Write};
|
||||||
@@ -1356,6 +1388,7 @@ mod tests {
|
|||||||
assert_eq!(fetched.projection.source_digest, "source-3");
|
assert_eq!(fetched.projection.source_digest, "source-3");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
#[test]
|
#[test]
|
||||||
fn concurrent_prompt_projection_miss_fetches_once_and_shares_catalog() {
|
fn concurrent_prompt_projection_miss_fetches_once_and_shares_catalog() {
|
||||||
use std::io::{Read, Write};
|
use std::io::{Read, Write};
|
||||||
@@ -1427,6 +1460,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
#[test]
|
#[test]
|
||||||
fn current_prompt_projection_rejects_cross_workspace_response() {
|
fn current_prompt_projection_rejects_cross_workspace_response() {
|
||||||
use std::io::{Read, Write};
|
use std::io::{Read, Write};
|
||||||
@@ -1473,6 +1507,7 @@ mod tests {
|
|||||||
assert!(error.to_string().contains("scope mismatch"));
|
assert!(error.to_string().contains("scope mismatch"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
#[test]
|
#[test]
|
||||||
fn ordinary_workspace_forwarding_stamps_runtime_identity_and_signs_path_and_query() {
|
fn ordinary_workspace_forwarding_stamps_runtime_identity_and_signs_path_and_query() {
|
||||||
use std::io::{Read, Write};
|
use std::io::{Read, Write};
|
||||||
@@ -1531,6 +1566,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
#[test]
|
#[test]
|
||||||
fn workspace_worker_discovery_signs_dedicated_permission_and_encoded_query() {
|
fn workspace_worker_discovery_signs_dedicated_permission_and_encoded_query() {
|
||||||
use std::io::{Read, Write};
|
use std::io::{Read, Write};
|
||||||
@@ -1652,6 +1688,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
async fn remote_forwarder_is_safe_in_async_runtime_and_stamps_signed_proof() {
|
async fn remote_forwarder_is_safe_in_async_runtime_and_stamps_signed_proof() {
|
||||||
use std::io::{Read, Write};
|
use std::io::{Read, Write};
|
||||||
@@ -1815,6 +1852,7 @@ mod tests {
|
|||||||
assert_eq!(reason, "retire obsolete Worker");
|
assert_eq!(reason, "retire obsolete Worker");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
#[test]
|
#[test]
|
||||||
fn runtime_owned_workspace_client_returns_successful_response_without_request_timeout() {
|
fn runtime_owned_workspace_client_returns_successful_response_without_request_timeout() {
|
||||||
// Effective deadline clearing is exercised with a short inherited deadline in
|
// Effective deadline clearing is exercised with a short inherited deadline in
|
||||||
@@ -1838,6 +1876,7 @@ mod tests {
|
|||||||
server.join().unwrap();
|
server.join().unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
#[test]
|
#[test]
|
||||||
fn runtime_owned_workspace_client_honors_per_request_timeouts() {
|
fn runtime_owned_workspace_client_honors_per_request_timeouts() {
|
||||||
let (base_url, server) = delayed_workspace_response(Duration::from_millis(75));
|
let (base_url, server) = delayed_workspace_response(Duration::from_millis(75));
|
||||||
@@ -1857,6 +1896,7 @@ mod tests {
|
|||||||
server.join().unwrap();
|
server.join().unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
#[test]
|
#[test]
|
||||||
fn runtime_owned_workspace_client_reports_request_timeouts() {
|
fn runtime_owned_workspace_client_reports_request_timeouts() {
|
||||||
let (base_url, server) = delayed_workspace_response(Duration::from_millis(75));
|
let (base_url, server) = delayed_workspace_response(Duration::from_millis(75));
|
||||||
@@ -1878,6 +1918,7 @@ mod tests {
|
|||||||
server.join().unwrap();
|
server.join().unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
fn delayed_workspace_response(delay: Duration) -> (String, std::thread::JoinHandle<()>) {
|
fn delayed_workspace_response(delay: Duration) -> (String, std::thread::JoinHandle<()>) {
|
||||||
use std::io::{Read, Write};
|
use std::io::{Read, Write};
|
||||||
use std::net::TcpListener;
|
use std::net::TcpListener;
|
||||||
|
|||||||
@@ -3676,6 +3676,7 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn bound_ssh_workdir_uses_attachment_scoped_agent_and_cleans_it_up() {
|
fn bound_ssh_workdir_uses_attachment_scoped_agent_and_cleans_it_up() {
|
||||||
|
crate::restore_test_environment::require_restore_test_environment();
|
||||||
let repo = create_clean_repo();
|
let repo = create_clean_repo();
|
||||||
// ssh-agent and the broker receive relative socket paths in this owned
|
// ssh-agent and the broker receive relative socket paths in this owned
|
||||||
// fixture tree, so the test does not depend on checkout/TMPDIR length.
|
// fixture tree, so the test does not depend on checkout/TMPDIR length.
|
||||||
|
|||||||
@@ -7,7 +7,8 @@
|
|||||||
//! identity only and report CleanupPending/unknown, not current cleanliness or removal permission.
|
//! identity only and report CleanupPending/unknown, not current cleanliness or removal permission.
|
||||||
//!
|
//!
|
||||||
//! The cleanup walker never reads ordinary file bodies or computes content digests. It records
|
//! The cleanup walker never reads ordinary file bodies or computes content digests. It records
|
||||||
//! device/inode, type, regular-file executable bits, and size/nanosecond mtime for files and symlinks.
|
//! device/inode, type, regular-file executable bits, and size/nanosecond mtime for non-directory
|
||||||
|
//! entries, including FIFOs and Unix sockets. IPC endpoints are never opened for I/O or connected.
|
||||||
//! Directories retain identity/type only because child unlink changes their timestamps. Symlink
|
//! Directories retain identity/type only because child unlink changes their timestamps. Symlink
|
||||||
//! targets are neither read nor followed. This is metadata protection, not byte equality:
|
//! targets are neither read nor followed. This is metadata protection, not byte equality:
|
||||||
//! same-size writes restoring mtime are not guaranteed to be detected. Normal Git inspection may
|
//! same-size writes restoring mtime are not guaranteed to be detected. Normal Git inspection may
|
||||||
@@ -735,7 +736,7 @@ mod linux {
|
|||||||
use std::ffi::{CString, OsStr};
|
use std::ffi::{CString, OsStr};
|
||||||
use std::os::fd::{AsRawFd, FromRawFd};
|
use std::os::fd::{AsRawFd, FromRawFd};
|
||||||
use std::os::unix::ffi::OsStrExt;
|
use std::os::unix::ffi::OsStrExt;
|
||||||
use std::os::unix::fs::MetadataExt;
|
use std::os::unix::fs::{FileTypeExt, MetadataExt};
|
||||||
|
|
||||||
#[repr(C)]
|
#[repr(C)]
|
||||||
struct OpenHow {
|
struct OpenHow {
|
||||||
@@ -916,7 +917,16 @@ mod linux {
|
|||||||
io_observation::record(io_observation::Event::EntryInspection);
|
io_observation::record(io_observation::Event::EntryInspection);
|
||||||
let file = open(dir, name, libc::O_PATH)?;
|
let file = open(dir, name, libc::O_PATH)?;
|
||||||
let metadata = file.metadata()?;
|
let metadata = file.metadata()?;
|
||||||
if !metadata.is_file() && !metadata.is_dir() && !metadata.file_type().is_symlink() {
|
let kind = metadata.file_type();
|
||||||
|
// O_PATH inspects IPC entries without opening a FIFO stream or connecting
|
||||||
|
// to a socket. They are unlinked, never traversed, under the same identity
|
||||||
|
// and mount checks as regular entries; their type alone is not occupancy.
|
||||||
|
if !kind.is_file()
|
||||||
|
&& !kind.is_dir()
|
||||||
|
&& !kind.is_symlink()
|
||||||
|
&& !kind.is_fifo()
|
||||||
|
&& !kind.is_socket()
|
||||||
|
{
|
||||||
return Err(io::Error::from_raw_os_error(libc::EBUSY));
|
return Err(io::Error::from_raw_os_error(libc::EBUSY));
|
||||||
}
|
}
|
||||||
Ok(Entry {
|
Ok(Entry {
|
||||||
@@ -1163,6 +1173,91 @@ pub(super) mod tests {
|
|||||||
(runtime, materializer, binding)
|
(runtime, materializer, binding)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn create_fifo(path: &Path) {
|
||||||
|
use std::os::unix::ffi::OsStrExt;
|
||||||
|
let name = std::ffi::CString::new(path.as_os_str().as_bytes()).unwrap();
|
||||||
|
// SAFETY: NUL-terminated path inside the isolated test directory.
|
||||||
|
assert_eq!(unsafe { libc::mkfifo(name.as_ptr(), 0o600) }, 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn cleanup_unlinks_fifo_and_socket_without_opening_or_connecting() {
|
||||||
|
use std::os::unix::net::UnixListener;
|
||||||
|
for retry in [false, true] {
|
||||||
|
let (runtime, materializer, binding) = fixture();
|
||||||
|
fs::write(binding.root.join(".git/info/exclude"), "target/\n").unwrap();
|
||||||
|
let output = binding.root.join("target");
|
||||||
|
fs::create_dir(&output).unwrap();
|
||||||
|
let fifo = output.join("ready.fifo");
|
||||||
|
let socket = output.join("service.sock");
|
||||||
|
create_fifo(&fifo);
|
||||||
|
let listener = UnixListener::bind(&socket).unwrap();
|
||||||
|
listener.set_nonblocking(true).unwrap();
|
||||||
|
let mut watch = BodyWatch::new(&[fifo, socket]);
|
||||||
|
let id = &binding.working_directory.id;
|
||||||
|
if retry {
|
||||||
|
let (result, audit) =
|
||||||
|
io_observation::observe(Some(1), || materializer.cleanup_working_directory(id));
|
||||||
|
assert_eq!(
|
||||||
|
result.unwrap_err().code,
|
||||||
|
"working_directory_cleanup_permission_denied"
|
||||||
|
);
|
||||||
|
assert_eq!(audit.unlinks, 1);
|
||||||
|
assert_eq!(audit.entry_readable_opens, 0);
|
||||||
|
}
|
||||||
|
drop(materializer);
|
||||||
|
let materializer = RuntimeGitMaterializer::new(runtime.path());
|
||||||
|
let (result, audit) =
|
||||||
|
io_observation::observe(None, || materializer.cleanup_working_directory(id));
|
||||||
|
assert_eq!(
|
||||||
|
result.unwrap().summary.status,
|
||||||
|
WorkingDirectoryStatusKind::NotFound
|
||||||
|
);
|
||||||
|
assert!(!binding.working_directory_root.exists());
|
||||||
|
assert_eq!(audit.entry_readable_opens, 0);
|
||||||
|
watch.assert_no_body_access();
|
||||||
|
assert_eq!(
|
||||||
|
listener.accept().unwrap_err().kind(),
|
||||||
|
io::ErrorKind::WouldBlock
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn cleanup_preserves_replaced_fifo_and_socket_before_unlink() {
|
||||||
|
use std::os::unix::net::UnixListener;
|
||||||
|
for socket in [false, true] {
|
||||||
|
let (runtime, materializer, binding) = fixture();
|
||||||
|
fs::write(binding.root.join(".git/info/exclude"), "target/\n").unwrap();
|
||||||
|
let output = binding.root.join("target");
|
||||||
|
fs::create_dir(&output).unwrap();
|
||||||
|
let path = output.join("endpoint");
|
||||||
|
let _original = if socket {
|
||||||
|
Some(UnixListener::bind(&path).unwrap())
|
||||||
|
} else {
|
||||||
|
create_fifo(&path);
|
||||||
|
None
|
||||||
|
};
|
||||||
|
let error = remove_with_eraser(
|
||||||
|
&materializer,
|
||||||
|
&binding.working_directory.id,
|
||||||
|
|tree, relative, entries| {
|
||||||
|
fs::rename(&path, runtime.path().join("original-endpoint")).unwrap();
|
||||||
|
let _replacement = if socket {
|
||||||
|
Some(UnixListener::bind(&path).unwrap())
|
||||||
|
} else {
|
||||||
|
create_fifo(&path);
|
||||||
|
None
|
||||||
|
};
|
||||||
|
erase(tree, relative, entries)
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap_err();
|
||||||
|
assert_eq!(error.code, "working_directory_cleanup_changes_present");
|
||||||
|
assert!(path.symlink_metadata().is_ok());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn fail_once(materializer: &RuntimeGitMaterializer, id: &str) {
|
pub(crate) fn fail_once(materializer: &RuntimeGitMaterializer, id: &str) {
|
||||||
let error = remove_with_eraser(materializer, id, |_, _, _| {
|
let error = remove_with_eraser(materializer, id, |_, _, _| {
|
||||||
Err(EraseError::Io(io::Error::from_raw_os_error(libc::EACCES)))
|
Err(EraseError::Io(io::Error::from_raw_os_error(libc::EACCES)))
|
||||||
|
|||||||
@@ -28,7 +28,19 @@ pub(super) struct Lease {
|
|||||||
id: String,
|
id: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn busy() -> WorkingDirectoryDiagnostic {
|
fn busy(id: &str, action: &str, blocking: &Entry) -> WorkingDirectoryDiagnostic {
|
||||||
|
let (busy_reason, active_use_count) = match blocking {
|
||||||
|
Entry::Use(count) => ("active_use", *count),
|
||||||
|
Entry::Create => ("creation_in_progress", 0),
|
||||||
|
Entry::Cleanup => ("cleanup_in_progress", 0),
|
||||||
|
};
|
||||||
|
tracing::warn!(target: "yoi::workdir_cleanup",
|
||||||
|
working_directory_id = id,
|
||||||
|
action,
|
||||||
|
busy_source = "materializer_occupancy",
|
||||||
|
busy_reason,
|
||||||
|
active_use_count,
|
||||||
|
"Workdir operation blocked by busy resource");
|
||||||
WorkingDirectoryDiagnostic::new(
|
WorkingDirectoryDiagnostic::new(
|
||||||
"working_directory_cleanup_resource_busy",
|
"working_directory_cleanup_resource_busy",
|
||||||
"A filesystem resource is busy or changed during removal. Release the resource and retry.",
|
"A filesystem resource is busy or changed during removal. Release the resource and retry.",
|
||||||
@@ -42,7 +54,9 @@ impl Occupancy {
|
|||||||
.lock()
|
.lock()
|
||||||
.unwrap_or_else(|error| error.into_inner());
|
.unwrap_or_else(|error| error.into_inner());
|
||||||
match entries.get_mut(id) {
|
match entries.get_mut(id) {
|
||||||
Some(Entry::Cleanup | Entry::Create) => return Err(busy()),
|
Some(blocking @ (Entry::Cleanup | Entry::Create)) => {
|
||||||
|
return Err(busy(id, "acquire_use", blocking));
|
||||||
|
}
|
||||||
Some(Entry::Use(count)) => *count += 1,
|
Some(Entry::Use(count)) => *count += 1,
|
||||||
None => {
|
None => {
|
||||||
entries.insert(id.to_string(), Entry::Use(1));
|
entries.insert(id.to_string(), Entry::Use(1));
|
||||||
@@ -101,8 +115,8 @@ impl Occupancy {
|
|||||||
.entries
|
.entries
|
||||||
.lock()
|
.lock()
|
||||||
.unwrap_or_else(|error| error.into_inner());
|
.unwrap_or_else(|error| error.into_inner());
|
||||||
if entries.contains_key(id) {
|
if let Some(blocking) = entries.get(id) {
|
||||||
return Err(busy());
|
return Err(busy(id, "acquire_creation", blocking));
|
||||||
}
|
}
|
||||||
entries.insert(id.to_string(), Entry::Create);
|
entries.insert(id.to_string(), Entry::Create);
|
||||||
Ok(Lease {
|
Ok(Lease {
|
||||||
@@ -116,8 +130,8 @@ impl Occupancy {
|
|||||||
.entries
|
.entries
|
||||||
.lock()
|
.lock()
|
||||||
.unwrap_or_else(|error| error.into_inner());
|
.unwrap_or_else(|error| error.into_inner());
|
||||||
if entries.contains_key(id) {
|
if let Some(blocking) = entries.get(id) {
|
||||||
return Err(busy());
|
return Err(busy(id, "acquire_cleanup", blocking));
|
||||||
}
|
}
|
||||||
entries.insert(id.to_string(), Entry::Cleanup);
|
entries.insert(id.to_string(), Entry::Cleanup);
|
||||||
Ok(Lease {
|
Ok(Lease {
|
||||||
@@ -131,6 +145,71 @@ impl Occupancy {
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn busy_admission_logs_blocker_and_use_count_without_changing_error() {
|
||||||
|
use std::io::{Read, Seek, SeekFrom};
|
||||||
|
|
||||||
|
let mut output = tempfile::tempfile().unwrap();
|
||||||
|
let writer = output.try_clone().unwrap();
|
||||||
|
let subscriber = tracing_subscriber::fmt()
|
||||||
|
.with_ansi(false)
|
||||||
|
.without_time()
|
||||||
|
.with_writer(move || writer.try_clone().unwrap())
|
||||||
|
.finish();
|
||||||
|
tracing::subscriber::with_default(subscriber, || {
|
||||||
|
tracing::callsite::rebuild_interest_cache();
|
||||||
|
let occupancy = Occupancy::default();
|
||||||
|
for (id, leases) in [
|
||||||
|
(
|
||||||
|
"workdir-used",
|
||||||
|
vec![
|
||||||
|
occupancy.acquire_use("workdir-used").unwrap(),
|
||||||
|
occupancy.acquire_use("workdir-used").unwrap(),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"workdir-creating",
|
||||||
|
vec![occupancy.acquire_creation("workdir-creating").unwrap()],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"workdir-removing",
|
||||||
|
vec![occupancy.acquire_cleanup("workdir-removing").unwrap()],
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
let error = occupancy.acquire_cleanup(id).unwrap_err();
|
||||||
|
assert_eq!(error.code, "working_directory_cleanup_resource_busy");
|
||||||
|
assert_eq!(
|
||||||
|
error.message,
|
||||||
|
"A filesystem resource is busy or changed during removal. Release the resource and retry."
|
||||||
|
);
|
||||||
|
drop(leases);
|
||||||
|
drop(occupancy.acquire_cleanup(id).unwrap());
|
||||||
|
}
|
||||||
|
});
|
||||||
|
output.seek(SeekFrom::Start(0)).unwrap();
|
||||||
|
let mut log = String::new();
|
||||||
|
output.read_to_string(&mut log).unwrap();
|
||||||
|
let events: Vec<_> = log.lines().collect();
|
||||||
|
assert_eq!(events.len(), 3, "{log}");
|
||||||
|
for (event, (id, reason, count)) in events.iter().zip([
|
||||||
|
("workdir-used", "active_use", 2),
|
||||||
|
("workdir-creating", "creation_in_progress", 0),
|
||||||
|
("workdir-removing", "cleanup_in_progress", 0),
|
||||||
|
]) {
|
||||||
|
for field in [
|
||||||
|
"WARN".to_string(),
|
||||||
|
"yoi::workdir_cleanup".to_string(),
|
||||||
|
format!("working_directory_id=\"{id}\""),
|
||||||
|
"action=\"acquire_cleanup\"".to_string(),
|
||||||
|
"busy_source=\"materializer_occupancy\"".to_string(),
|
||||||
|
format!("busy_reason=\"{reason}\""),
|
||||||
|
format!("active_use_count={count}"),
|
||||||
|
] {
|
||||||
|
assert!(event.contains(&field), "missing {field}: {event}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn use_and_cleanup_leases_release_during_unwind() {
|
fn use_and_cleanup_leases_release_during_unwind() {
|
||||||
let occupancy = Occupancy::default();
|
let occupancy = Occupancy::default();
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ use workdir::WorkdirSessionHandle;
|
|||||||
|
|
||||||
use crate::PromptCatalogSource;
|
use crate::PromptCatalogSource;
|
||||||
use crate::controller::{ShutdownReceiver, WorkerController, WorkerHandle};
|
use crate::controller::{ShutdownReceiver, WorkerController, WorkerHandle};
|
||||||
|
pub use crate::worker::{PreparedRestoredWorker, RestoreCandidateWorkdirCleanup};
|
||||||
use crate::worker::{Worker, WorkerError, WorkerFilesystemAuthority, WorkerWorkspaceContext};
|
use crate::worker::{Worker, WorkerError, WorkerFilesystemAuthority, WorkerWorkspaceContext};
|
||||||
use manifest::WorkerManifest;
|
use manifest::WorkerManifest;
|
||||||
|
|
||||||
@@ -161,6 +162,38 @@ where
|
|||||||
Ok(PreparedWorker::new(worker, self.layout))
|
Ok(PreparedWorker::new(worker, self.layout))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Reconstruct a saved materialized Worker without allocation, attribution
|
||||||
|
/// repair, Feature installation, or controller exposure. The caller owns the
|
||||||
|
/// later controller layout and must activate before starting it.
|
||||||
|
pub async fn prepare_restored_candidate(
|
||||||
|
self,
|
||||||
|
worker_name: &str,
|
||||||
|
) -> Result<PreparedRestoredWorker<St>, WorkerBootstrapError> {
|
||||||
|
let mut candidate =
|
||||||
|
Worker::prepare_restored_from_worker_metadata_with_context_and_model_client(
|
||||||
|
worker_name,
|
||||||
|
self.manifest,
|
||||||
|
self.store,
|
||||||
|
self.prompt_catalog,
|
||||||
|
self.workspace_context,
|
||||||
|
self.filesystem_authority,
|
||||||
|
self.model_client,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(WorkerBootstrapError::Worker)?;
|
||||||
|
if let Some(workdir_session) = self.workdir_session {
|
||||||
|
candidate
|
||||||
|
.worker_mut()
|
||||||
|
.map_err(WorkerBootstrapError::Worker)?
|
||||||
|
.bind_single_workdir_session(Some(workdir_session));
|
||||||
|
}
|
||||||
|
candidate
|
||||||
|
.worker_mut()
|
||||||
|
.map_err(WorkerBootstrapError::Worker)?
|
||||||
|
.bind_subjektiv_host(self.subjektiv_host);
|
||||||
|
Ok(candidate)
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn prepare_restored(
|
pub async fn prepare_restored(
|
||||||
self,
|
self,
|
||||||
worker_name: &str,
|
worker_name: &str,
|
||||||
@@ -243,6 +276,7 @@ where
|
|||||||
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||||
{
|
{
|
||||||
let cleanup_sessions = worker.workdir_sessions();
|
let cleanup_sessions = worker.workdir_sessions();
|
||||||
|
let factory_owned_cleanup = worker.startup_feature_cleanup_handle().is_some();
|
||||||
let controller = match layout {
|
let controller = match layout {
|
||||||
WorkerBootstrapLayout::Direct { runtime_base } => {
|
WorkerBootstrapLayout::Direct { runtime_base } => {
|
||||||
WorkerController::spawn_owned(worker, &runtime_base).await
|
WorkerController::spawn_owned(worker, &runtime_base).await
|
||||||
@@ -259,7 +293,10 @@ where
|
|||||||
controller_task,
|
controller_task,
|
||||||
}),
|
}),
|
||||||
Err(source) => {
|
Err(source) => {
|
||||||
let cleanup_failed = cleanup_sessions.close_all().await.is_err();
|
// Factory-owned task scopes must settle before sessions close.
|
||||||
|
// Cleanup remains unconfirmed here; preserve the startup source.
|
||||||
|
let cleanup_failed =
|
||||||
|
factory_owned_cleanup || cleanup_sessions.close_all().await.is_err();
|
||||||
Err(WorkerBootstrapError::Controller {
|
Err(WorkerBootstrapError::Controller {
|
||||||
source,
|
source,
|
||||||
cleanup_failed,
|
cleanup_failed,
|
||||||
@@ -267,3 +304,229 @@ where
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Host bindings for one unpublished restored Controller. These do not expose
|
||||||
|
/// Worker execution or permit arbitrary mutation during preparation.
|
||||||
|
pub struct RestoredControllerBindings {
|
||||||
|
pub backend_job: Option<crate::BackendJobExecutionBinding>,
|
||||||
|
pub workdir_sessions: std::sync::Arc<workdir::WorkdirSessionRouter>,
|
||||||
|
pub observation_provider:
|
||||||
|
Option<std::sync::Arc<dyn crate::feature::builtin::WorkerObservationProvider>>,
|
||||||
|
pub subjektiv_attached: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The same real receiver, snapshot sources and paired subscription retained
|
||||||
|
/// privately across the Runtime's durable restore decision.
|
||||||
|
pub struct UnpublishedRestoredController<St: Store> {
|
||||||
|
initialized: Option<crate::controller::InitializedWorkerController<Box<dyn LlmClient>, St>>,
|
||||||
|
activation: Option<crate::worker::RestoredWorkerActivation>,
|
||||||
|
streams: Option<crate::ipc::protocol_session::WorkerProtocolSessionStreams>,
|
||||||
|
subjektiv_attached: bool,
|
||||||
|
effects_applied: bool,
|
||||||
|
activation_failed: bool,
|
||||||
|
closing: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct ActivatedRestoredController {
|
||||||
|
pub controller: BootstrappedWorker,
|
||||||
|
pub streams: crate::ipc::protocol_session::WorkerProtocolSessionStreams,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<St> PreparedRestoredWorker<St>
|
||||||
|
where
|
||||||
|
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||||
|
{
|
||||||
|
pub fn workspace_client_handle(
|
||||||
|
&mut self,
|
||||||
|
) -> Result<std::sync::Arc<dyn crate::WorkspaceClient>, WorkerError> {
|
||||||
|
Ok(self.worker_mut()?.workspace_client_handle())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn retain_startup_feature_cleanup(
|
||||||
|
&mut self,
|
||||||
|
) -> Result<crate::WorkerStartupFeatureCleanup, WorkerError> {
|
||||||
|
Ok(self.worker_mut()?.retain_startup_feature_cleanup())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Initialize the real Controller without allocation, canonical restore
|
||||||
|
/// writes, attribution changes, registry reclamation or input consumption.
|
||||||
|
/// MCP discovery and Feature tasks belong to the retained startup cleanup.
|
||||||
|
pub async fn prepare_runtime_controller(
|
||||||
|
mut self,
|
||||||
|
run_dir: &std::path::Path,
|
||||||
|
bindings: RestoredControllerBindings,
|
||||||
|
) -> Result<UnpublishedRestoredController<St>, WorkerBootstrapError> {
|
||||||
|
let worker = self.worker_mut().map_err(WorkerBootstrapError::Worker)?;
|
||||||
|
if let Some(binding) = bindings.backend_job {
|
||||||
|
worker.bind_backend_job(binding).map_err(|error| {
|
||||||
|
WorkerBootstrapError::Worker(WorkerError::FeatureLifecycle(error.into()))
|
||||||
|
})?;
|
||||||
|
}
|
||||||
|
worker
|
||||||
|
.workdir_sessions()
|
||||||
|
.close_all()
|
||||||
|
.await
|
||||||
|
.map_err(|error| {
|
||||||
|
WorkerBootstrapError::Worker(WorkerError::FeatureLifecycle(error.to_string()))
|
||||||
|
})?;
|
||||||
|
worker.bind_workdir_sessions(bindings.workdir_sessions);
|
||||||
|
worker.bind_worker_observation_provider(bindings.observation_provider);
|
||||||
|
let flow_enabled = worker.manifest().feature.flow.enabled;
|
||||||
|
if flow_enabled {
|
||||||
|
let report = worker
|
||||||
|
.install_runtime_flow_transition_feature()
|
||||||
|
.map_err(|error| {
|
||||||
|
WorkerBootstrapError::Worker(WorkerError::FeatureLifecycle(error))
|
||||||
|
})?;
|
||||||
|
if report.reports.iter().any(|report| !report.installed) {
|
||||||
|
return Err(WorkerBootstrapError::Worker(WorkerError::FeatureLifecycle(
|
||||||
|
"Flow transition installation refused".into(),
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let (worker, activation) = self
|
||||||
|
.into_controller_parts()
|
||||||
|
.map_err(WorkerBootstrapError::Worker)?;
|
||||||
|
let initialized = WorkerController::initialize_restored(worker, run_dir)
|
||||||
|
.await
|
||||||
|
.map_err(|source| WorkerBootstrapError::Controller {
|
||||||
|
source,
|
||||||
|
cleanup_failed: true,
|
||||||
|
})?;
|
||||||
|
if flow_enabled {
|
||||||
|
initialized.handle.shared_state.enable_flow_transition();
|
||||||
|
}
|
||||||
|
let streams = {
|
||||||
|
let _guard = initialized.handle.protocol_snapshot_publish_guard();
|
||||||
|
crate::ipc::protocol_session::subscribe_worker_protocol_session(&initialized.handle)
|
||||||
|
};
|
||||||
|
Ok(UnpublishedRestoredController {
|
||||||
|
initialized: Some(initialized),
|
||||||
|
activation: Some(activation),
|
||||||
|
streams: Some(streams),
|
||||||
|
subjektiv_attached: bindings.subjektiv_attached,
|
||||||
|
effects_applied: false,
|
||||||
|
activation_failed: false,
|
||||||
|
closing: false,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<St> UnpublishedRestoredController<St>
|
||||||
|
where
|
||||||
|
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||||
|
{
|
||||||
|
pub fn snapshot(&self) -> &protocol::Event {
|
||||||
|
&self
|
||||||
|
.streams
|
||||||
|
.as_ref()
|
||||||
|
.expect("unpublished restore streams")
|
||||||
|
.snapshot_event
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Only publication failures after confirmed effects can retry on this
|
||||||
|
/// exact owner. Partial activation must be reconciled, never blindly rerun.
|
||||||
|
pub fn can_retry_publication(&self) -> bool {
|
||||||
|
self.initialized.is_some()
|
||||||
|
&& self.effects_applied
|
||||||
|
&& !self.activation_failed
|
||||||
|
&& !self.closing
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Apply canonical restore effects after the host's durable decision, while
|
||||||
|
/// retaining the same private Controller for subsequent publication.
|
||||||
|
pub async fn apply_effects(&mut self) -> Result<(), WorkerBootstrapError> {
|
||||||
|
let initialized = self
|
||||||
|
.initialized
|
||||||
|
.as_mut()
|
||||||
|
.ok_or_else(|| WorkerBootstrapError::Worker(WorkerError::RestoreCandidateClosing))?;
|
||||||
|
if self.activation_failed || self.closing {
|
||||||
|
return Err(WorkerBootstrapError::Worker(
|
||||||
|
WorkerError::RestoreCandidateClosing,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
// Preparation's read-only authorization is not a publication grant.
|
||||||
|
// Revalidate current Host authority after commit, before effects/input,
|
||||||
|
// including a publication retry on this same private Controller.
|
||||||
|
initialized
|
||||||
|
.worker
|
||||||
|
.validate_subjektiv_body_publication()
|
||||||
|
.map_err(WorkerBootstrapError::Worker)?;
|
||||||
|
if !self.effects_applied {
|
||||||
|
self.activation_failed = true;
|
||||||
|
self.activation
|
||||||
|
.as_ref()
|
||||||
|
.expect("restore activation")
|
||||||
|
.activate(&mut initialized.worker)
|
||||||
|
.await
|
||||||
|
.map_err(WorkerBootstrapError::Worker)?;
|
||||||
|
initialized
|
||||||
|
.worker
|
||||||
|
.finalize_subjektiv_session_attribution(
|
||||||
|
self.subjektiv_attached,
|
||||||
|
crate::SubjektivSessionAttributionLifecycle::RestoredSession,
|
||||||
|
)
|
||||||
|
.map_err(WorkerBootstrapError::Worker)?;
|
||||||
|
self.effects_applied = true;
|
||||||
|
self.activation_failed = false;
|
||||||
|
self.activation = None;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn allocation_cleanup(
|
||||||
|
&self,
|
||||||
|
) -> Option<crate::runtime::worker_allocation::ScopeAllocationCleanup> {
|
||||||
|
self.initialized
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|initialized| initialized.worker.restore_allocation_cleanup_handle())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn activate(&mut self) -> Result<ActivatedRestoredController, WorkerBootstrapError> {
|
||||||
|
self.apply_effects().await?;
|
||||||
|
let initialized = self
|
||||||
|
.initialized
|
||||||
|
.as_mut()
|
||||||
|
.expect("same initialized Controller");
|
||||||
|
let publication = initialized.prepare_publication().await.map_err(|source| {
|
||||||
|
WorkerBootstrapError::Controller {
|
||||||
|
source,
|
||||||
|
cleanup_failed: true,
|
||||||
|
}
|
||||||
|
})?;
|
||||||
|
let initialized = self
|
||||||
|
.initialized
|
||||||
|
.take()
|
||||||
|
.expect("same initialized Controller");
|
||||||
|
let (handle, shutdown, controller_task) = initialized.start(publication);
|
||||||
|
Ok(ActivatedRestoredController {
|
||||||
|
controller: BootstrappedWorker {
|
||||||
|
handle,
|
||||||
|
shutdown,
|
||||||
|
controller_task,
|
||||||
|
},
|
||||||
|
streams: self.streams.take().expect("same paired subscription"),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn discard(&mut self) -> Result<(), WorkerError> {
|
||||||
|
self.closing = true;
|
||||||
|
if let Some(initialized) = self.initialized.as_mut() {
|
||||||
|
initialized
|
||||||
|
.worker
|
||||||
|
.retain_startup_feature_cleanup()
|
||||||
|
.shutdown()
|
||||||
|
.await
|
||||||
|
.map_err(|error| WorkerError::FeatureLifecycle(error.to_string()))?;
|
||||||
|
initialized
|
||||||
|
.worker
|
||||||
|
.workdir_sessions()
|
||||||
|
.close_all()
|
||||||
|
.await
|
||||||
|
.map_err(|error| WorkerError::FeatureLifecycle(error.to_string()))?;
|
||||||
|
}
|
||||||
|
self.streams = None;
|
||||||
|
self.initialized = None;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+388
-45
@@ -21,7 +21,7 @@ use crate::shutdown_after_idle::{
|
|||||||
ShutdownAfterIdleRequest, TicketIntakeReadyShutdownHook, is_ticket_intake_role,
|
ShutdownAfterIdleRequest, TicketIntakeReadyShutdownHook, is_ticket_intake_role,
|
||||||
take_shutdown_request_after_status,
|
take_shutdown_request_after_status,
|
||||||
};
|
};
|
||||||
use crate::spawn::registry::SpawnedWorkerRegistry;
|
use crate::spawn::registry::{PreparedSpawnedWorkerRegistryLoad, SpawnedWorkerRegistry};
|
||||||
use crate::spawn::tool::sub_worker_spawn_tool;
|
use crate::spawn::tool::sub_worker_spawn_tool;
|
||||||
use crate::worker::{
|
use crate::worker::{
|
||||||
PendingSubmissionHandle, SystemItemCommitter, Worker, WorkerError, WorkerRunResult,
|
PendingSubmissionHandle, SystemItemCommitter, Worker, WorkerError, WorkerRunResult,
|
||||||
@@ -654,7 +654,56 @@ pub type ShutdownReceiver = oneshot::Receiver<()>;
|
|||||||
|
|
||||||
pub struct WorkerController;
|
pub struct WorkerController;
|
||||||
|
|
||||||
|
// Own the real command receiver and all snapshot sources before starting any
|
||||||
|
// protocol listener or controller loop. Fresh startup may materialize the head;
|
||||||
|
// Runtime restore requires an already materialized source and defers canonical
|
||||||
|
// restore effects until its durable decision. Tool discovery can own external
|
||||||
|
// resources, so private initialization is not synonymous with side-effect-free.
|
||||||
|
pub(crate) struct InitializedWorkerController<C: LlmClient, St: Store> {
|
||||||
|
pub(crate) worker: Worker<C, St>,
|
||||||
|
pub(crate) handle: WorkerHandle,
|
||||||
|
method_rx: mpsc::Receiver<Method>,
|
||||||
|
shutdown_tx: oneshot::Sender<()>,
|
||||||
|
shutdown_rx: ShutdownReceiver,
|
||||||
|
registry_reclaim: PreparedSpawnedWorkerRegistryLoad<St>,
|
||||||
|
shutdown_after_idle: ShutdownAfterIdleRequest,
|
||||||
|
command_events: Option<PreparedWorkdirCommandEvents>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) struct PreparedControllerPublication {
|
||||||
|
command_observer: Option<tokio::task::JoinHandle<()>>,
|
||||||
|
}
|
||||||
|
|
||||||
impl WorkerController {
|
impl WorkerController {
|
||||||
|
/// Explicit recovery of an unmaterialized Session head. This installs the
|
||||||
|
/// saved Feature composition needed to produce the seed, but creates no
|
||||||
|
/// Controller, input channel, socket, observer, or queue consumer. The caller
|
||||||
|
/// owns the same operation's Feature/session/allocation cleanup on every exit.
|
||||||
|
pub async fn recover_pending_head<C, St>(
|
||||||
|
worker: &mut Worker<C, St>,
|
||||||
|
runtime_base: &Path,
|
||||||
|
) -> std::io::Result<()>
|
||||||
|
where
|
||||||
|
C: LlmClient + Clone + 'static,
|
||||||
|
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||||
|
{
|
||||||
|
if worker.needs_initial_session_head_materialization() {
|
||||||
|
worker
|
||||||
|
.validate_subjektiv_body_publication()
|
||||||
|
.map_err(std::io::Error::other)?;
|
||||||
|
let registry = SpawnedWorkerRegistry::new_internal(
|
||||||
|
worker.manifest().worker.name.clone(),
|
||||||
|
worker.scope().clone(),
|
||||||
|
);
|
||||||
|
register_worker_tools(worker, runtime_base.to_path_buf(), registry, None, None).await?;
|
||||||
|
worker
|
||||||
|
.materialize_durable_session_head()
|
||||||
|
.await
|
||||||
|
.map_err(std::io::Error::other)?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn spawn<C, St>(
|
pub async fn spawn<C, St>(
|
||||||
worker: Worker<C, St>,
|
worker: Worker<C, St>,
|
||||||
runtime_base: &Path,
|
runtime_base: &Path,
|
||||||
@@ -734,8 +783,12 @@ impl WorkerController {
|
|||||||
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||||
{
|
{
|
||||||
let sessions = worker.workdir_sessions();
|
let sessions = worker.workdir_sessions();
|
||||||
|
let factory_owned_cleanup = worker.startup_feature_cleanup_handle().is_some();
|
||||||
let result = Self::spawn_initialized(worker, runtime_base, runtime_run).await;
|
let result = Self::spawn_initialized(worker, runtime_base, runtime_run).await;
|
||||||
|
// An opted-in factory must quiesce its captured Feature tasks before
|
||||||
|
// closing their Workdir sessions; this wrapper cannot confirm that order.
|
||||||
if result.is_err()
|
if result.is_err()
|
||||||
|
&& !factory_owned_cleanup
|
||||||
&& let Err(error) = sessions.close_all().await
|
&& let Err(error) = sessions.close_all().await
|
||||||
{
|
{
|
||||||
tracing::warn!(%error, "Workdir sessions close after controller startup failure failed");
|
tracing::warn!(%error, "Workdir sessions close after controller startup failure failed");
|
||||||
@@ -744,7 +797,7 @@ impl WorkerController {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn spawn_initialized<C, St>(
|
async fn spawn_initialized<C, St>(
|
||||||
mut worker: Worker<C, St>,
|
worker: Worker<C, St>,
|
||||||
runtime_base: &Path,
|
runtime_base: &Path,
|
||||||
runtime_run: Option<&Path>,
|
runtime_run: Option<&Path>,
|
||||||
) -> Result<(WorkerHandle, ShutdownReceiver, tokio::task::JoinHandle<()>), std::io::Error>
|
) -> Result<(WorkerHandle, ShutdownReceiver, tokio::task::JoinHandle<()>), std::io::Error>
|
||||||
@@ -752,6 +805,52 @@ impl WorkerController {
|
|||||||
C: LlmClient + Clone + 'static,
|
C: LlmClient + Clone + 'static,
|
||||||
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||||
{
|
{
|
||||||
|
Self::initialize(worker, runtime_base, runtime_run)
|
||||||
|
.await?
|
||||||
|
.publish()
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn initialize<C, St>(
|
||||||
|
worker: Worker<C, St>,
|
||||||
|
runtime_base: &Path,
|
||||||
|
runtime_run: Option<&Path>,
|
||||||
|
) -> Result<InitializedWorkerController<C, St>, std::io::Error>
|
||||||
|
where
|
||||||
|
C: LlmClient + Clone + 'static,
|
||||||
|
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||||
|
{
|
||||||
|
Self::initialize_inner(worker, runtime_base, runtime_run, true).await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn initialize_restored<St>(
|
||||||
|
worker: Worker<Box<dyn LlmClient>, St>,
|
||||||
|
run_dir: &Path,
|
||||||
|
) -> std::io::Result<InitializedWorkerController<Box<dyn LlmClient>, St>>
|
||||||
|
where
|
||||||
|
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||||
|
{
|
||||||
|
let parent = run_dir
|
||||||
|
.parent()
|
||||||
|
.ok_or_else(|| std::io::Error::other("run path has no parent"))?;
|
||||||
|
Self::initialize_inner(worker, parent, Some(run_dir), false).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn initialize_inner<C, St>(
|
||||||
|
mut worker: Worker<C, St>,
|
||||||
|
runtime_base: &Path,
|
||||||
|
runtime_run: Option<&Path>,
|
||||||
|
canonical_startup: bool,
|
||||||
|
) -> Result<InitializedWorkerController<C, St>, std::io::Error>
|
||||||
|
where
|
||||||
|
C: LlmClient + Clone + 'static,
|
||||||
|
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||||
|
{
|
||||||
|
if !canonical_startup && worker.needs_initial_session_head_materialization() {
|
||||||
|
return Err(std::io::Error::other(
|
||||||
|
"restore source requires operation-owned head recovery",
|
||||||
|
));
|
||||||
|
}
|
||||||
worker
|
worker
|
||||||
.validate_subjektiv_body_publication()
|
.validate_subjektiv_body_publication()
|
||||||
.map_err(std::io::Error::other)?;
|
.map_err(std::io::Error::other)?;
|
||||||
@@ -773,21 +872,13 @@ impl WorkerController {
|
|||||||
});
|
});
|
||||||
|
|
||||||
let spawner_name = worker.manifest().worker.name.clone();
|
let spawner_name = worker.manifest().worker.name.clone();
|
||||||
let loaded_registry = SpawnedWorkerRegistry::load_from_worker_state_with_reclaim(
|
let registry_reclaim = SpawnedWorkerRegistry::prepare_from_worker_state_with_reclaim(
|
||||||
runtime_dir.clone(),
|
runtime_dir.clone(),
|
||||||
worker.store().clone(),
|
worker.store().clone(),
|
||||||
spawner_name.clone(),
|
spawner_name.clone(),
|
||||||
Some(worker.scope().clone()),
|
Some(worker.scope().clone()),
|
||||||
)
|
)?;
|
||||||
.await?;
|
let spawned_registry = registry_reclaim.registry.clone();
|
||||||
let reclaimed_unreachable = loaded_registry.reclaimed_unreachable;
|
|
||||||
let spawned_registry = loaded_registry.registry;
|
|
||||||
if reclaimed_unreachable {
|
|
||||||
worker.push_notify(
|
|
||||||
"Restored Worker state contained unreachable delegated child Workers; their delegated write scopes were reclaimed before resume."
|
|
||||||
.to_string(),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Hand the alerter to the Worker so internal operations (compaction,
|
// Hand the alerter to the Worker so internal operations (compaction,
|
||||||
// AGENTS.md ingestion during the first turn) can emit user-facing
|
// AGENTS.md ingestion during the first turn) can emit user-facing
|
||||||
@@ -878,7 +969,6 @@ impl WorkerController {
|
|||||||
if let Some(fs_for_view) = fs_for_view.as_ref() {
|
if let Some(fs_for_view) = fs_for_view.as_ref() {
|
||||||
shared_state.set_fs_view(crate::fs_view::WorkerFsView::new(fs_for_view.clone()));
|
shared_state.set_fs_view(crate::fs_view::WorkerFsView::new(fs_for_view.clone()));
|
||||||
}
|
}
|
||||||
|
|
||||||
let artifact_store: Arc<dyn Store> = Arc::new(worker.store().clone());
|
let artifact_store: Arc<dyn Store> = Arc::new(worker.store().clone());
|
||||||
let session_id = worker.session_id();
|
let session_id = worker.session_id();
|
||||||
let pending_activations = worker.pending_activation_state();
|
let pending_activations = worker.pending_activation_state();
|
||||||
@@ -903,10 +993,71 @@ impl WorkerController {
|
|||||||
pending_activations,
|
pending_activations,
|
||||||
};
|
};
|
||||||
|
|
||||||
// Start the observer only after fallible head/recovery initialization.
|
let command_events = fs_for_view
|
||||||
let command_observer = fs_for_view
|
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.and_then(|session| wire_workdir_command_events(session, &in_flight));
|
.map(|session| PreparedWorkdirCommandEvents::new(session, &handle.in_flight));
|
||||||
|
Ok(InitializedWorkerController {
|
||||||
|
worker,
|
||||||
|
handle,
|
||||||
|
method_rx,
|
||||||
|
shutdown_tx,
|
||||||
|
shutdown_rx,
|
||||||
|
registry_reclaim,
|
||||||
|
shutdown_after_idle,
|
||||||
|
command_events,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<C, St> InitializedWorkerController<C, St>
|
||||||
|
where
|
||||||
|
C: LlmClient + Clone + 'static,
|
||||||
|
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||||
|
{
|
||||||
|
async fn publish(
|
||||||
|
mut self,
|
||||||
|
) -> Result<(WorkerHandle, ShutdownReceiver, tokio::task::JoinHandle<()>), std::io::Error> {
|
||||||
|
let publication = self.prepare_publication().await?;
|
||||||
|
Ok(self.start(publication))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn prepare_publication(
|
||||||
|
&mut self,
|
||||||
|
) -> std::io::Result<PreparedControllerPublication> {
|
||||||
|
let loaded_registry = self.registry_reclaim.apply().await?;
|
||||||
|
debug_assert!(Arc::ptr_eq(
|
||||||
|
&loaded_registry.registry,
|
||||||
|
&self.handle.spawned_registry
|
||||||
|
));
|
||||||
|
if loaded_registry.reclaimed_unreachable {
|
||||||
|
self.worker.push_notify("Restored Worker state contained unreachable delegated child Workers; their delegated write scopes were reclaimed before resume.".to_string());
|
||||||
|
}
|
||||||
|
let command_observer = self
|
||||||
|
.command_events
|
||||||
|
.take()
|
||||||
|
.and_then(PreparedWorkdirCommandEvents::start);
|
||||||
|
Ok(PreparedControllerPublication { command_observer })
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn start(
|
||||||
|
self,
|
||||||
|
publication: PreparedControllerPublication,
|
||||||
|
) -> (WorkerHandle, ShutdownReceiver, tokio::task::JoinHandle<()>) {
|
||||||
|
let Self {
|
||||||
|
mut worker,
|
||||||
|
handle,
|
||||||
|
method_rx,
|
||||||
|
shutdown_tx,
|
||||||
|
shutdown_rx,
|
||||||
|
shutdown_after_idle,
|
||||||
|
..
|
||||||
|
} = self;
|
||||||
|
let PreparedControllerPublication { command_observer } = publication;
|
||||||
|
let runtime_dir = handle.runtime_dir.clone();
|
||||||
|
let shared_state = handle.shared_state.clone();
|
||||||
|
let working_event_tx = handle.working_event_tx.clone();
|
||||||
|
let spawned_registry = handle.spawned_registry.clone();
|
||||||
|
|
||||||
// === 5. controller_loop ===
|
// === 5. controller_loop ===
|
||||||
// Clone cancel sender and notification buffer before moving worker
|
// Clone cancel sender and notification buffer before moving worker
|
||||||
// into the controller task so the in-flight turn can be reached
|
// into the controller task so the in-flight turn can be reached
|
||||||
@@ -915,6 +1066,7 @@ impl WorkerController {
|
|||||||
let pause_tx = worker.engine_mut().pause_sender();
|
let pause_tx = worker.engine_mut().pause_sender();
|
||||||
let notify_buffer = worker.notify_buffer_handle();
|
let notify_buffer = worker.notify_buffer_handle();
|
||||||
|
|
||||||
|
worker.finish_startup_feature_cleanup_capture();
|
||||||
let controller_task = tokio::spawn(controller_loop(
|
let controller_task = tokio::spawn(controller_loop(
|
||||||
worker,
|
worker,
|
||||||
method_rx,
|
method_rx,
|
||||||
@@ -930,7 +1082,7 @@ impl WorkerController {
|
|||||||
command_observer,
|
command_observer,
|
||||||
));
|
));
|
||||||
|
|
||||||
Ok((handle, shutdown_rx, controller_task))
|
(handle, shutdown_rx, controller_task)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -938,30 +1090,51 @@ pub(crate) fn wire_workdir_command_events(
|
|||||||
session: &Arc<dyn WorkdirSession>,
|
session: &Arc<dyn WorkdirSession>,
|
||||||
in_flight: &InFlightEvents,
|
in_flight: &InFlightEvents,
|
||||||
) -> Option<tokio::task::JoinHandle<()>> {
|
) -> Option<tokio::task::JoinHandle<()>> {
|
||||||
in_flight.replace_command_snapshot(protocol_command_snapshots(session.as_ref()));
|
PreparedWorkdirCommandEvents::new(session, in_flight).start()
|
||||||
let Some(mut events) = session.subscribe_command_events() else {
|
}
|
||||||
return None;
|
|
||||||
};
|
struct PreparedWorkdirCommandEvents {
|
||||||
// Keep only a weak reference in the observer task. Holding the session
|
session: std::sync::Weak<dyn WorkdirSession>,
|
||||||
// strongly here would keep its broadcast sender alive forever and prevent
|
in_flight: InFlightEvents,
|
||||||
// the receiver from observing closure during Worker teardown.
|
events: Option<broadcast::Receiver<WorkdirCommandEvent>>,
|
||||||
let session = Arc::downgrade(session);
|
}
|
||||||
let in_flight = in_flight.clone();
|
|
||||||
Some(tokio::spawn(async move {
|
impl PreparedWorkdirCommandEvents {
|
||||||
loop {
|
fn new(session: &Arc<dyn WorkdirSession>, in_flight: &InFlightEvents) -> Self {
|
||||||
match events.recv().await {
|
// Subscribe before reading command state; hold the receiver privately
|
||||||
Ok(event) => in_flight.publish_command_event(protocol_command_event(event)),
|
// across the durable decision instead of creating a publication gap.
|
||||||
Err(broadcast::error::RecvError::Lagged(_)) => {
|
let events = session.subscribe_command_events();
|
||||||
let Some(session) = session.upgrade() else {
|
in_flight.replace_command_snapshot(protocol_command_snapshots(session.as_ref()));
|
||||||
break;
|
Self {
|
||||||
};
|
session: Arc::downgrade(session),
|
||||||
in_flight
|
in_flight: in_flight.clone(),
|
||||||
.replace_command_snapshot(protocol_command_snapshots(session.as_ref()));
|
events,
|
||||||
}
|
|
||||||
Err(broadcast::error::RecvError::Closed) => break,
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}))
|
}
|
||||||
|
|
||||||
|
fn start(self) -> Option<tokio::task::JoinHandle<()>> {
|
||||||
|
let Self {
|
||||||
|
session,
|
||||||
|
in_flight,
|
||||||
|
events,
|
||||||
|
} = self;
|
||||||
|
let mut events = events?;
|
||||||
|
Some(tokio::spawn(async move {
|
||||||
|
loop {
|
||||||
|
match events.recv().await {
|
||||||
|
Ok(event) => in_flight.publish_command_event(protocol_command_event(event)),
|
||||||
|
Err(broadcast::error::RecvError::Lagged(_)) => {
|
||||||
|
let Some(session) = session.upgrade() else {
|
||||||
|
break;
|
||||||
|
};
|
||||||
|
in_flight
|
||||||
|
.replace_command_snapshot(protocol_command_snapshots(session.as_ref()));
|
||||||
|
}
|
||||||
|
Err(broadcast::error::RecvError::Closed) => break,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn protocol_command_snapshots(session: &dyn WorkdirSession) -> Vec<ProtocolCommandSnapshot> {
|
fn protocol_command_snapshots(session: &dyn WorkdirSession) -> Vec<ProtocolCommandSnapshot> {
|
||||||
@@ -1687,9 +1860,17 @@ where
|
|||||||
.add_module(crate::feature::builtin::orchestration::orchestration_feature());
|
.add_module(crate::feature::builtin::orchestration::orchestration_feature());
|
||||||
}
|
}
|
||||||
if let Some(workspace_root) = local_workspace_root.as_ref() {
|
if let Some(workspace_root) = local_workspace_root.as_ref() {
|
||||||
if let Some(module) =
|
let module = if let Some(cleanup) = worker.startup_feature_cleanup_handle() {
|
||||||
|
crate::feature::mcp::discover_stdio_tool_feature_with_cleanup(
|
||||||
|
&mcp_config,
|
||||||
|
workspace_root,
|
||||||
|
&cleanup.mcp_discovery(),
|
||||||
|
)
|
||||||
|
.await?
|
||||||
|
} else {
|
||||||
crate::feature::mcp::discover_stdio_tool_feature(&mcp_config, workspace_root).await
|
crate::feature::mcp::discover_stdio_tool_feature(&mcp_config, workspace_root).await
|
||||||
{
|
};
|
||||||
|
if let Some(module) = module {
|
||||||
feature_registry = feature_registry.with_module(module);
|
feature_registry = feature_registry.with_module(module);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -3750,6 +3931,159 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Component connection: real Worker/Store, initialized endpoint, paired
|
||||||
|
// protocol subscription and Controller loop. The model boundary is an
|
||||||
|
// empty non-network script; this test must not request any generation.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn initialized_controller_publishes_the_same_endpoint_and_subscription() {
|
||||||
|
let root = TempDir::new().unwrap();
|
||||||
|
let manifest = manifest::WorkerManifest::from_toml(
|
||||||
|
"[worker]\nname = 'p'\n[model]\nscheme = 'anthropic'\nmodel_id = 'test'\n[engine]\n[scope]\n",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let client = ScriptedClient::new(vec![]);
|
||||||
|
let requests = client.requests.clone();
|
||||||
|
let engine =
|
||||||
|
Engine::<_, agen::state::Mutable, crate::SessionHistoryMetadata>::new_annotated(
|
||||||
|
client,
|
||||||
|
);
|
||||||
|
let store = session_store::CombinedStore::new(
|
||||||
|
session_store::FsStore::new(root.path().join("sessions")).unwrap(),
|
||||||
|
session_store::FsWorkerStore::new(root.path().join("workers")).unwrap(),
|
||||||
|
);
|
||||||
|
let mut worker = Worker::new(
|
||||||
|
manifest,
|
||||||
|
engine,
|
||||||
|
store,
|
||||||
|
crate::worker::WorkerWorkspaceContext::local_filesystem(None),
|
||||||
|
crate::worker::WorkerFilesystemAuthority::None,
|
||||||
|
manifest::Scope::empty(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
worker.set_system_prompt_template(
|
||||||
|
crate::SystemPromptTemplate::parse(
|
||||||
|
"default",
|
||||||
|
crate::PromptCatalogSource::builtins_only(),
|
||||||
|
)
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
let prepared = WorkerController::initialize(worker, root.path(), None)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let endpoint = prepared.handle.clone();
|
||||||
|
let mut streams =
|
||||||
|
crate::ipc::protocol_session::subscribe_worker_protocol_session(&endpoint);
|
||||||
|
let entries = endpoint.committed_entries();
|
||||||
|
assert!(matches!(
|
||||||
|
&streams.snapshot_event,
|
||||||
|
Event::Snapshot { greeting, state, session, .. }
|
||||||
|
if !greeting.tools.is_empty()
|
||||||
|
&& state.state == WorkerState::Idle
|
||||||
|
&& session == &session_store::public_snapshot::project_current_session_snapshot(&entries)
|
||||||
|
));
|
||||||
|
assert!(!endpoint.protocol_is_closed());
|
||||||
|
assert!(!endpoint.runtime_dir.path().join("sock").exists());
|
||||||
|
assert!(!endpoint.runtime_dir.path().join("status.json").exists());
|
||||||
|
assert!(!endpoint.runtime_dir.path().join("manifest.toml").exists());
|
||||||
|
assert!(matches!(
|
||||||
|
streams.events.try_recv(),
|
||||||
|
Err(broadcast::error::TryRecvError::Empty)
|
||||||
|
));
|
||||||
|
assert!(requests.lock().unwrap().is_empty());
|
||||||
|
|
||||||
|
let (handle, shutdown, task) = prepared.publish().await.unwrap();
|
||||||
|
assert!(handle.same_controller(&endpoint));
|
||||||
|
assert!(!handle.runtime_dir.path().join("sock").exists());
|
||||||
|
assert!(!handle.runtime_dir.path().join("status.json").exists());
|
||||||
|
assert!(!handle.runtime_dir.path().join("manifest.toml").exists());
|
||||||
|
handle.send(Method::ListPendingSubmissions).await.unwrap();
|
||||||
|
let pending = tokio::time::timeout(Duration::from_secs(5), async {
|
||||||
|
loop {
|
||||||
|
if let Event::PendingSubmissionsChanged { pending } =
|
||||||
|
streams.events.recv().await.unwrap()
|
||||||
|
{
|
||||||
|
break pending;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(pending, protocol::PendingSubmissionsSnapshot::default());
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(handle.committed_entries()).unwrap(),
|
||||||
|
serde_json::to_value(entries).unwrap(),
|
||||||
|
);
|
||||||
|
handle
|
||||||
|
.send(Method::Shutdown {
|
||||||
|
command: WorkerCommandEnvelope { command_id: 1 },
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
tokio::time::timeout(Duration::from_secs(5), shutdown)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
tokio::time::timeout(Duration::from_secs(5), task)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert!(endpoint.protocol_is_closed());
|
||||||
|
assert!(!endpoint.runtime_dir.path().join("sock").exists());
|
||||||
|
assert!(requests.lock().unwrap().is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn tool_initialization_failure_does_not_reclaim_saved_children() {
|
||||||
|
let root = TempDir::new().unwrap();
|
||||||
|
let mut manifest = manifest::WorkerManifest::from_toml(
|
||||||
|
"[worker]\nname = 'p'\n[model]\nscheme = 'anthropic'\nmodel_id = 'test'\n[engine]\n[scope]\n",
|
||||||
|
).unwrap();
|
||||||
|
manifest.feature.ticket.enabled = true;
|
||||||
|
let client = ScriptedClient::new(vec![]);
|
||||||
|
let requests = client.requests.clone();
|
||||||
|
let store = session_store::CombinedStore::new(
|
||||||
|
session_store::FsStore::new(root.path().join("sessions")).unwrap(),
|
||||||
|
session_store::FsWorkerStore::new(root.path().join("workers")).unwrap(),
|
||||||
|
);
|
||||||
|
// New feature activation is refused by the real unavailable host;
|
||||||
|
// legacy child metadata must not be consumed before that refusal.
|
||||||
|
let worker = Worker::new(
|
||||||
|
manifest,
|
||||||
|
Engine::<_, agen::state::Mutable, crate::SessionHistoryMetadata>::new_annotated(
|
||||||
|
client,
|
||||||
|
),
|
||||||
|
store.clone(),
|
||||||
|
crate::worker::WorkerWorkspaceContext::no_workspace(),
|
||||||
|
crate::worker::WorkerFilesystemAuthority::None,
|
||||||
|
manifest::Scope::empty(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let mut metadata = session_store::WorkerMetadata::new(
|
||||||
|
"p",
|
||||||
|
Some(session_store::WorkerActiveSegmentRef {
|
||||||
|
session_id: worker.session_id(),
|
||||||
|
segment_id: Some(worker.segment_id()),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
metadata
|
||||||
|
.spawned_children
|
||||||
|
.push(session_store::WorkerSpawnedChild {
|
||||||
|
worker_name: "legacy".into(),
|
||||||
|
socket_path: root.path().join("legacy.sock"),
|
||||||
|
callback_address: root.path().join("p/sock"),
|
||||||
|
scope_delegated: Vec::new(),
|
||||||
|
});
|
||||||
|
store.write(&metadata).unwrap();
|
||||||
|
let before = store.read_by_name("p").unwrap();
|
||||||
|
let result = WorkerController::initialize(worker, root.path(), None).await;
|
||||||
|
assert!(result.is_err());
|
||||||
|
assert_eq!(store.read_by_name("p").unwrap(), before);
|
||||||
|
assert!(!root.path().join("p/spawned_workers.json").exists());
|
||||||
|
assert!(requests.lock().unwrap().is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
struct DriveWorkspaceHost {
|
struct DriveWorkspaceHost {
|
||||||
available: bool,
|
available: bool,
|
||||||
@@ -4077,10 +4411,19 @@ mod tests {
|
|||||||
assert!(feature_install < materialize_gate);
|
assert!(feature_install < materialize_gate);
|
||||||
assert!(materialize_gate < materialize);
|
assert!(materialize_gate < materialize);
|
||||||
assert!(materialize < exposure);
|
assert!(materialize < exposure);
|
||||||
let observer = startup.find("let command_observer = fs_for_view").unwrap();
|
let observer_preparation = startup
|
||||||
|
.find("PreparedWorkdirCommandEvents::new(session")
|
||||||
|
.unwrap();
|
||||||
let recover = startup.find(".recover_unfinished_compaction()").unwrap();
|
let recover = startup.find(".recover_unfinished_compaction()").unwrap();
|
||||||
|
let reclaim = startup
|
||||||
|
.find("self.registry_reclaim.apply().await?")
|
||||||
|
.unwrap();
|
||||||
|
let observer_start = startup
|
||||||
|
.find(".and_then(PreparedWorkdirCommandEvents::start)")
|
||||||
|
.unwrap();
|
||||||
let loop_start = startup.find("tokio::spawn(controller_loop(").unwrap();
|
let loop_start = startup.find("tokio::spawn(controller_loop(").unwrap();
|
||||||
assert!(recover < observer && observer < loop_start);
|
assert!(recover < observer_preparation && observer_preparation < reclaim);
|
||||||
|
assert!(reclaim < observer_start && observer_start < loop_start);
|
||||||
assert!(
|
assert!(
|
||||||
startup[materialize_gate..materialize]
|
startup[materialize_gate..materialize]
|
||||||
.contains("needs_initial_session_head_materialization")
|
.contains("needs_initial_session_head_materialization")
|
||||||
|
|||||||
@@ -578,7 +578,8 @@ where
|
|||||||
"flow:{}:{}",
|
"flow:{}:{}",
|
||||||
attempt.instance_id, attempt.attempt_id
|
attempt.instance_id, attempt.attempt_id
|
||||||
)),
|
)),
|
||||||
max_turns: Some(12),
|
// Evidence exploration is not bounded by an arbitrary model-turn count.
|
||||||
|
max_turns: None,
|
||||||
compaction_budget: None,
|
compaction_budget: None,
|
||||||
engine_configurator: None,
|
engine_configurator: None,
|
||||||
features,
|
features,
|
||||||
@@ -882,35 +883,45 @@ mod tests {
|
|||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
struct FinishVerifierClient {
|
struct FinishVerifierClient {
|
||||||
calls: Arc<AtomicUsize>,
|
calls: Arc<AtomicUsize>,
|
||||||
|
exploration_turns: usize,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[async_trait]
|
#[async_trait]
|
||||||
impl LlmClient for FinishVerifierClient {
|
impl LlmClient for FinishVerifierClient {
|
||||||
async fn stream(&self, _request: Request) -> Result<ResponseStream, ClientError> {
|
async fn stream(&self, _request: Request) -> Result<ResponseStream, ClientError> {
|
||||||
let call = self.calls.fetch_add(1, Ordering::SeqCst);
|
let call = self.calls.fetch_add(1, Ordering::SeqCst);
|
||||||
if call == 0 {
|
if call <= self.exploration_turns {
|
||||||
let input = serde_json::json!({
|
let input = if call < self.exploration_turns {
|
||||||
"results": [
|
"{}".to_string()
|
||||||
{
|
} else {
|
||||||
"transition_id": "done",
|
serde_json::json!({
|
||||||
"verdict": "met",
|
"results": [
|
||||||
"rationale": "the captured implementation evidence is complete"
|
{
|
||||||
},
|
"transition_id": "done",
|
||||||
{
|
"verdict": "met",
|
||||||
"transition_id": "cancel",
|
"rationale": "the captured implementation evidence is complete"
|
||||||
"verdict": "not_met",
|
},
|
||||||
"rationale": "no exceptional cancellation condition exists"
|
{
|
||||||
}
|
"transition_id": "cancel",
|
||||||
]
|
"verdict": "not_met",
|
||||||
})
|
"rationale": "no exceptional cancellation condition exists"
|
||||||
.to_string();
|
}
|
||||||
|
]
|
||||||
|
})
|
||||||
|
.to_string()
|
||||||
|
};
|
||||||
Ok(Box::pin(stream::iter(vec![
|
Ok(Box::pin(stream::iter(vec![
|
||||||
Ok(LlmEvent::BlockStart(BlockStart {
|
Ok(LlmEvent::BlockStart(BlockStart {
|
||||||
index: 0,
|
index: 0,
|
||||||
block_type: BlockType::ToolUse,
|
block_type: BlockType::ToolUse,
|
||||||
metadata: BlockMetadata::ToolUse {
|
metadata: BlockMetadata::ToolUse {
|
||||||
id: "finish-flow".to_string(),
|
id: format!("flow-evidence-{call}"),
|
||||||
name: "FinishFlowVerification".to_string(),
|
name: if call < self.exploration_turns {
|
||||||
|
"ShowOverview"
|
||||||
|
} else {
|
||||||
|
"FinishFlowVerification"
|
||||||
|
}
|
||||||
|
.to_string(),
|
||||||
},
|
},
|
||||||
})),
|
})),
|
||||||
Ok(LlmEvent::BlockDelta(BlockDelta {
|
Ok(LlmEvent::BlockDelta(BlockDelta {
|
||||||
@@ -978,6 +989,15 @@ permission = "read"
|
|||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn worker_backed_verifier_requires_structured_finish_result() {
|
async fn worker_backed_verifier_requires_structured_finish_result() {
|
||||||
|
assert_worker_backed_verifier_completes(0).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn worker_backed_verifier_can_finish_after_more_than_twelve_turns() {
|
||||||
|
assert_worker_backed_verifier_completes(16).await;
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn assert_worker_backed_verifier_completes(exploration_turns: usize) {
|
||||||
let directory = tempfile::tempdir().unwrap();
|
let directory = tempfile::tempdir().unwrap();
|
||||||
let store = session_store::FsStore::new(directory.path()).unwrap();
|
let store = session_store::FsStore::new(directory.path()).unwrap();
|
||||||
let session_id: SessionId = Uuid::now_v7();
|
let session_id: SessionId = Uuid::now_v7();
|
||||||
@@ -998,11 +1018,16 @@ permission = "read"
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
let calls = Arc::new(AtomicUsize::new(0));
|
||||||
|
let mut manifest = verifier_manifest();
|
||||||
|
// A parent's own turn limit must not cap the private verifier either.
|
||||||
|
manifest.engine.max_turns = std::num::NonZeroU32::new(2);
|
||||||
let verifier = WorkerBackedFlowVerifier::new(
|
let verifier = WorkerBackedFlowVerifier::new(
|
||||||
FinishVerifierClient {
|
FinishVerifierClient {
|
||||||
calls: Arc::new(AtomicUsize::new(0)),
|
calls: calls.clone(),
|
||||||
|
exploration_turns,
|
||||||
},
|
},
|
||||||
verifier_manifest(),
|
manifest,
|
||||||
StoreFlowParentCapture::new(store, session_id, segment_id),
|
StoreFlowParentCapture::new(store, session_id, segment_id),
|
||||||
Vec::new(),
|
Vec::new(),
|
||||||
);
|
);
|
||||||
@@ -1013,6 +1038,7 @@ permission = "read"
|
|||||||
assert_eq!(results.len(), 2);
|
assert_eq!(results.len(), 2);
|
||||||
assert_eq!(results[0].verdict, ConditionVerdict::Met);
|
assert_eq!(results[0].verdict, ConditionVerdict::Met);
|
||||||
assert_eq!(results[1].verdict, ConditionVerdict::NotMet);
|
assert_eq!(results[1].verdict, ConditionVerdict::NotMet);
|
||||||
|
assert!(calls.load(Ordering::SeqCst) > exploration_turns);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -702,31 +702,46 @@ impl WorkspaceHttpWorkdirBackend {
|
|||||||
ATTACH_TOOL,
|
ATTACH_TOOL,
|
||||||
"working_directory_id",
|
"working_directory_id",
|
||||||
)?;
|
)?;
|
||||||
let alias_key = workdir::WorkdirAttachmentAlias::new(alias)
|
|
||||||
.map_err(|error| ToolError::InvalidArgument(error.to_string()))?;
|
|
||||||
if self.session_router.session(&alias_key).is_some() {
|
|
||||||
return Err(ToolError::InvalidArgument(format!(
|
|
||||||
"attachment alias `{alias}` already exists"
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
let response = self.attach_response(alias, working_directory_id)?;
|
let response = self.attach_response(alias, working_directory_id)?;
|
||||||
if let Err(error) = self.session_router.attach(
|
if !response.attachment.attached
|
||||||
alias_key.clone(),
|
|| response.attachment.working_directory_id != working_directory_id
|
||||||
|
{
|
||||||
|
return Err(ToolError::ExecutionFailed(
|
||||||
|
"Workdir attachment response does not match the target".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
// Backend ownership decides the name. A requested alias cannot rename
|
||||||
|
// an existing attachment or create a second route to the same Workdir.
|
||||||
|
let attached_alias = &response.attachment.alias;
|
||||||
|
let alias_key = workdir::WorkdirAttachmentAlias::new(attached_alias.clone())
|
||||||
|
.map_err(|error| ToolError::ExecutionFailed(error.to_string()))?;
|
||||||
|
if let Some(session) = self.session_router.session(&alias_key) {
|
||||||
|
if session.workdir().id().as_str() != working_directory_id {
|
||||||
|
return Err(ToolError::ExecutionFailed("Local Workdir attachment differs from Backend; refresh attachments before retrying".into()));
|
||||||
|
}
|
||||||
|
} else if let Err(error) = self.session_router.attach(
|
||||||
|
alias_key,
|
||||||
WorkspaceAttachedWorkdirSession::handle_for_workdir_with_environment(
|
WorkspaceAttachedWorkdirSession::handle_for_workdir_with_environment(
|
||||||
self.client.clone(),
|
self.client.clone(),
|
||||||
alias,
|
attached_alias,
|
||||||
working_directory_id,
|
working_directory_id,
|
||||||
response.attachment.capabilities,
|
response.attachment.capabilities,
|
||||||
response.environment.clone(),
|
response.environment.clone(),
|
||||||
),
|
),
|
||||||
) {
|
) {
|
||||||
let _ = self.detach(alias);
|
if !response.attachment.already_attached {
|
||||||
|
let _ = self.detach(attached_alias);
|
||||||
|
}
|
||||||
return Err(ToolError::ExecutionFailed(format!(
|
return Err(ToolError::ExecutionFailed(format!(
|
||||||
"attach Workdir session route: {error}"
|
"attach Workdir session route: {error}"
|
||||||
)));
|
)));
|
||||||
}
|
}
|
||||||
workdir_output(
|
workdir_output(
|
||||||
format!("Attached Workdir {working_directory_id} as {alias}"),
|
if response.attachment.already_attached {
|
||||||
|
format!("Workdir {working_directory_id} is already attached as {attached_alias}")
|
||||||
|
} else {
|
||||||
|
format!("Attached Workdir {working_directory_id} as {attached_alias}")
|
||||||
|
},
|
||||||
&response,
|
&response,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -737,15 +752,31 @@ impl WorkspaceHttpWorkdirBackend {
|
|||||||
working_directory_id: &str,
|
working_directory_id: &str,
|
||||||
) -> Result<WorkdirSessionAttachmentResponse, ToolError> {
|
) -> Result<WorkdirSessionAttachmentResponse, ToolError> {
|
||||||
let workspace_id = encode_path_segment(self.workspace_id()?);
|
let workspace_id = encode_path_segment(self.workspace_id()?);
|
||||||
self.execute_json::<WorkdirSessionAttachmentResponse>(WorkspaceRequest::json(
|
let response = self
|
||||||
WorkspaceRequestMethod::Post,
|
.client
|
||||||
format!("/api/w/{workspace_id}/workers/self/workdir-attachments"),
|
.execute(WorkspaceRequest::json(
|
||||||
serde_json::to_string(&WorkdirAttachRequest {
|
WorkspaceRequestMethod::Post,
|
||||||
alias: alias.to_string(),
|
format!("/api/w/{workspace_id}/workers/self/workdir-attachments"),
|
||||||
working_directory_id: working_directory_id.to_string(),
|
serde_json::to_string(&WorkdirAttachRequest {
|
||||||
})
|
alias: alias.to_string(),
|
||||||
.map_err(decode_error)?,
|
working_directory_id: working_directory_id.to_string(),
|
||||||
))
|
})
|
||||||
|
.map_err(decode_error)?,
|
||||||
|
))
|
||||||
|
.map_err(|error| {
|
||||||
|
ToolError::ExecutionFailed(format!("Workspace Workdir request failed: {error}"))
|
||||||
|
})?;
|
||||||
|
if response.status == 409
|
||||||
|
&& let Ok(error) =
|
||||||
|
serde_json::from_str::<server_api::RepositoryApiError>(&response.body)
|
||||||
|
&& error.error == "workdir_occupied"
|
||||||
|
{
|
||||||
|
return Err(ToolError::StructuredConflict {
|
||||||
|
code: "workdir_occupied".into(),
|
||||||
|
message: "Workdir is occupied by another Worker".into(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
decode_response(response)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn detach(&self, alias: &str) -> Result<ToolOutput, ToolError> {
|
fn detach(&self, alias: &str) -> Result<ToolOutput, ToolError> {
|
||||||
@@ -778,7 +809,11 @@ impl WorkspaceHttpWorkdirBackend {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn delete(&self, input: WorkdirDeleteInput) -> Result<ToolOutput, ToolError> {
|
fn delete(
|
||||||
|
&self,
|
||||||
|
input: WorkdirDeleteInput,
|
||||||
|
operation_id: String,
|
||||||
|
) -> Result<ToolOutput, ToolError> {
|
||||||
let workdir_id = validate_identity(
|
let workdir_id = validate_identity(
|
||||||
&input.working_directory_id,
|
&input.working_directory_id,
|
||||||
DELETE_TOOL,
|
DELETE_TOOL,
|
||||||
@@ -790,6 +825,7 @@ impl WorkspaceHttpWorkdirBackend {
|
|||||||
WorkspaceRequestMethod::Delete,
|
WorkspaceRequestMethod::Delete,
|
||||||
format!("/api/w/{workspace_id}/working-directories/{workdir_path}"),
|
format!("/api/w/{workspace_id}/working-directories/{workdir_path}"),
|
||||||
serde_json::to_string(&WorkdirRemovalRequest {
|
serde_json::to_string(&WorkdirRemovalRequest {
|
||||||
|
operation_id,
|
||||||
reason: validate_delete_reason(&input.reason)?.to_string(),
|
reason: validate_delete_reason(&input.reason)?.to_string(),
|
||||||
})
|
})
|
||||||
.map_err(decode_error)?,
|
.map_err(decode_error)?,
|
||||||
@@ -800,7 +836,7 @@ impl WorkspaceHttpWorkdirBackend {
|
|||||||
response.disposition,
|
response.disposition,
|
||||||
removal_guidance(response.failure_category.as_deref()),
|
removal_guidance(response.failure_category.as_deref()),
|
||||||
if response.retryable {
|
if response.retryable {
|
||||||
" After resolving the cause, repeat this WorkdirDelete with the same id and reason; current safety conditions will be rechecked."
|
" After resolving the cause, a new explicit WorkdirDelete request will recheck current safety conditions. This failed request will not be automatically re-executed."
|
||||||
} else {
|
} else {
|
||||||
""
|
""
|
||||||
}
|
}
|
||||||
@@ -954,9 +990,10 @@ impl Tool for WorkspaceHttpWorkdirTool {
|
|||||||
self.backend.detach_managed(&input.alias, None).await
|
self.backend.detach_managed(&input.alias, None).await
|
||||||
}
|
}
|
||||||
|
|
||||||
WorkdirOperation::Delete => self
|
WorkdirOperation::Delete => self.backend.delete(
|
||||||
.backend
|
parse_input::<WorkdirDeleteInput>(input_json)?,
|
||||||
.delete(parse_input::<WorkdirDeleteInput>(input_json)?),
|
ctx.call_id.to_string(),
|
||||||
|
),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1018,8 +1055,14 @@ fn removal_guidance(category: Option<&str>) -> &'static str {
|
|||||||
Some("runtime_unavailable" | "provider_unavailable") => {
|
Some("runtime_unavailable" | "provider_unavailable") => {
|
||||||
"Restore the Runtime connection; the registry remains until deletion is confirmed."
|
"Restore the Runtime connection; the registry remains until deletion is confirmed."
|
||||||
}
|
}
|
||||||
Some("provider_observation_unknown" | "provider_cleanup_outcome_unknown") => {
|
Some("provider_cleanup_outcome_unknown") => {
|
||||||
"Deletion is not confirmed. Re-observe with normal removal after restoring provider access; the registry is retained."
|
"Deletion outcome is unknown. Restore provider access and reconcile the existing request; do not submit a fresh deletion intent. Recovery only observes the target and never repeats deletion."
|
||||||
|
}
|
||||||
|
Some("provider_observation_unknown") => {
|
||||||
|
"The target could not be observed before deletion. Restore provider access before a new request."
|
||||||
|
}
|
||||||
|
Some("request_interrupted") => {
|
||||||
|
"The request was interrupted before execution. A new explicit request is required."
|
||||||
}
|
}
|
||||||
Some("unsupported_target") => {
|
Some("unsupported_target") => {
|
||||||
"Ask the owner for the provider's supported cleanup operation."
|
"Ask the owner for the provider's supported cleanup operation."
|
||||||
@@ -1219,6 +1262,8 @@ struct WorkdirAttachmentResponse {
|
|||||||
working_directory_id: String,
|
working_directory_id: String,
|
||||||
capabilities: WorkdirSessionCapabilities,
|
capabilities: WorkdirSessionCapabilities,
|
||||||
attached: bool,
|
attached: bool,
|
||||||
|
#[serde(default)]
|
||||||
|
already_attached: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
|
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
@@ -1337,8 +1382,44 @@ mod tests {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn delete_call_identity_distinguishes_new_requests_from_exact_resends() {
|
||||||
|
let client = Arc::new(RecordingWorkspaceClient::new(
|
||||||
|
(0..3)
|
||||||
|
.map(|_| {
|
||||||
|
response(json!({
|
||||||
|
"working_directory_id": "wd-request",
|
||||||
|
"disposition": "attention_required",
|
||||||
|
"retryable": true,
|
||||||
|
"failure_category": "resource_busy"
|
||||||
|
}))
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
));
|
||||||
|
let tool = WorkspaceHttpWorkdirTool {
|
||||||
|
backend: WorkspaceHttpWorkdirBackend::new(client.clone()),
|
||||||
|
operation: WorkdirOperation::Delete,
|
||||||
|
};
|
||||||
|
let input = r#"{"working_directory_id":"wd-request","reason":"remove unused Workdir"}"#;
|
||||||
|
for id in ["delete-1", "delete-1", "delete-2"] {
|
||||||
|
tool.execute(input, ToolExecutionContext::new(id, "batch", 0))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
let bodies: Vec<serde_json::Value> = client
|
||||||
|
.requests()
|
||||||
|
.iter()
|
||||||
|
.map(|r| serde_json::from_str(r.body.as_deref().unwrap()).unwrap())
|
||||||
|
.collect();
|
||||||
|
assert_eq!(bodies[0], bodies[1]);
|
||||||
|
assert_eq!(bodies[0]["operation_id"], "delete-1");
|
||||||
|
assert_eq!(bodies[2]["operation_id"], "delete-2");
|
||||||
|
assert_eq!(bodies[0]["reason"], bodies[2]["reason"]);
|
||||||
|
assert!(delete_schema()["properties"].get("operation_id").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn ordinary_workdir_delete_reports_safe_cause_and_exact_retry_guidance() {
|
fn ordinary_workdir_delete_reports_safe_cause_and_new_request_guidance() {
|
||||||
let client = Arc::new(RecordingWorkspaceClient::new(vec![response(json!({
|
let client = Arc::new(RecordingWorkspaceClient::new(vec![response(json!({
|
||||||
"working_directory_id": "wd-retry",
|
"working_directory_id": "wd-retry",
|
||||||
"disposition": "attention_required",
|
"disposition": "attention_required",
|
||||||
@@ -1347,10 +1428,13 @@ mod tests {
|
|||||||
}))]));
|
}))]));
|
||||||
let backend = WorkspaceHttpWorkdirBackend::new(client.clone());
|
let backend = WorkspaceHttpWorkdirBackend::new(client.clone());
|
||||||
let output = backend
|
let output = backend
|
||||||
.delete(WorkdirDeleteInput {
|
.delete(
|
||||||
working_directory_id: "wd-retry".into(),
|
WorkdirDeleteInput {
|
||||||
reason: "remove unused Workdir".into(),
|
working_directory_id: "wd-retry".into(),
|
||||||
})
|
reason: "remove unused Workdir".into(),
|
||||||
|
},
|
||||||
|
"delete-call-1".to_string(),
|
||||||
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert!(output.summary.contains("Confirm its owner"));
|
assert!(output.summary.contains("Confirm its owner"));
|
||||||
assert!(
|
assert!(
|
||||||
@@ -1358,7 +1442,11 @@ mod tests {
|
|||||||
.summary
|
.summary
|
||||||
.contains("Do not recursively delete mounted data")
|
.contains("Do not recursively delete mounted data")
|
||||||
);
|
);
|
||||||
assert!(output.summary.contains("same id and reason"));
|
assert!(
|
||||||
|
output
|
||||||
|
.summary
|
||||||
|
.contains("new explicit WorkdirDelete request")
|
||||||
|
);
|
||||||
assert!(output.summary.len() <= 600);
|
assert!(output.summary.len() <= 600);
|
||||||
let content: serde_json::Value =
|
let content: serde_json::Value =
|
||||||
serde_json::from_str(output.content.as_deref().unwrap()).unwrap();
|
serde_json::from_str(output.content.as_deref().unwrap()).unwrap();
|
||||||
@@ -1596,10 +1684,13 @@ mod tests {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
backend.detach("checkout").unwrap();
|
backend.detach("checkout").unwrap();
|
||||||
backend
|
backend
|
||||||
.delete(WorkdirDeleteInput {
|
.delete(
|
||||||
working_directory_id: "wd-created".to_string(),
|
WorkdirDeleteInput {
|
||||||
reason: "remove stale Workdir".to_string(),
|
working_directory_id: "wd-created".to_string(),
|
||||||
})
|
reason: "remove stale Workdir".to_string(),
|
||||||
|
},
|
||||||
|
"delete-call-1".to_string(),
|
||||||
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let requests = client.requests();
|
let requests = client.requests();
|
||||||
@@ -1637,7 +1728,102 @@ mod tests {
|
|||||||
assert_eq!(requests[4].method, WorkspaceRequestMethod::Delete);
|
assert_eq!(requests[4].method, WorkspaceRequestMethod::Delete);
|
||||||
let body: serde_json::Value =
|
let body: serde_json::Value =
|
||||||
serde_json::from_str(requests[4].body.as_deref().unwrap()).unwrap();
|
serde_json::from_str(requests[4].body.as_deref().unwrap()).unwrap();
|
||||||
assert_eq!(body, json!({"reason": "remove stale Workdir"}));
|
assert_eq!(
|
||||||
|
body,
|
||||||
|
json!({"reason": "remove stale Workdir", "operation_id": "delete-call-1"})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn attach_reports_other_worker_occupancy_without_exposing_backend_details() {
|
||||||
|
let client = Arc::new(RecordingWorkspaceClient::new(vec![WorkspaceResponse {
|
||||||
|
status: 409,
|
||||||
|
body: serde_json::to_string(&server_api::RepositoryApiError::new(
|
||||||
|
409,
|
||||||
|
"workdir_occupied",
|
||||||
|
"private backend detail",
|
||||||
|
Vec::new(),
|
||||||
|
))
|
||||||
|
.unwrap(),
|
||||||
|
}]));
|
||||||
|
let backend = WorkspaceHttpWorkdirBackend::new(client.clone());
|
||||||
|
let error = backend
|
||||||
|
.attach(WorkdirAttachInput {
|
||||||
|
alias: "work".into(),
|
||||||
|
working_directory_id: "occupied".into(),
|
||||||
|
})
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(
|
||||||
|
matches!(error, ToolError::StructuredConflict { code, message } if code == "workdir_occupied" && message == "Workdir is occupied by another Worker")
|
||||||
|
);
|
||||||
|
assert_eq!(client.requests().len(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn attach_reuses_backend_alias_without_duplicate_routes_or_detach() {
|
||||||
|
let client = Arc::new(RecordingWorkspaceClient::new((0..3).map(|_| response(json!({
|
||||||
|
"workspace_id": "workspace/test", "alias": "existing", "working_directory_id": "wd-existing",
|
||||||
|
"capabilities": {"bits": 63}, "attached": true, "already_attached": true,
|
||||||
|
"environment": workdir::WorkdirSessionEnvironment::root(WorkdirSessionCapabilities::ALL)
|
||||||
|
}))).collect()));
|
||||||
|
let backend = WorkspaceHttpWorkdirBackend::new(client.clone());
|
||||||
|
let occupied_alias = workdir::WorkdirAttachmentAlias::new("other").unwrap();
|
||||||
|
backend
|
||||||
|
.session_router
|
||||||
|
.attach(
|
||||||
|
occupied_alias.clone(),
|
||||||
|
WorkspaceAttachedWorkdirSession::handle_for_workdir(
|
||||||
|
client.clone(),
|
||||||
|
"other",
|
||||||
|
"wd-other",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
for alias in ["new-name", "existing", "other"] {
|
||||||
|
let output = backend
|
||||||
|
.attach(WorkdirAttachInput {
|
||||||
|
alias: alias.into(),
|
||||||
|
working_directory_id: "wd-existing".into(),
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
output.summary,
|
||||||
|
"Workdir wd-existing is already attached as existing"
|
||||||
|
);
|
||||||
|
let body: serde_json::Value =
|
||||||
|
serde_json::from_str(output.content.as_deref().unwrap()).unwrap();
|
||||||
|
assert_eq!(body["alias"], "existing");
|
||||||
|
assert_eq!(body["already_attached"], true);
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
backend
|
||||||
|
.session_router
|
||||||
|
.session(&workdir::WorkdirAttachmentAlias::new("new-name").unwrap())
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
backend
|
||||||
|
.session_router
|
||||||
|
.session(&occupied_alias)
|
||||||
|
.unwrap()
|
||||||
|
.workdir()
|
||||||
|
.id()
|
||||||
|
.as_str(),
|
||||||
|
"wd-other"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
backend
|
||||||
|
.session_router
|
||||||
|
.session(&workdir::WorkdirAttachmentAlias::new("existing").unwrap())
|
||||||
|
.is_some()
|
||||||
|
);
|
||||||
|
assert_eq!(client.requests().len(), 3);
|
||||||
|
assert!(
|
||||||
|
client
|
||||||
|
.requests()
|
||||||
|
.iter()
|
||||||
|
.all(|request| request.method == WorkspaceRequestMethod::Post)
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|||||||
@@ -91,13 +91,22 @@ struct MergeRequestInput {
|
|||||||
#[derive(Debug, Deserialize, JsonSchema)]
|
#[derive(Debug, Deserialize, JsonSchema)]
|
||||||
struct ShowMergeRequestInput {
|
struct ShowMergeRequestInput {
|
||||||
merge_request_id: String,
|
merge_request_id: String,
|
||||||
|
/// summary (default) omits review bodies/history; detail returns exact paged events.
|
||||||
|
/// For compatibility, supplying after/limit without view selects detail.
|
||||||
|
view: Option<MergeRequestReadView>,
|
||||||
/// Return thread events strictly after this sequence.
|
/// Return thread events strictly after this sequence.
|
||||||
#[schemars(range(min = 0, max = 9_007_199_254_740_991_u64))]
|
#[schemars(range(min = 0, max = 9_007_199_254_740_991_u64))]
|
||||||
after: Option<u64>,
|
after: Option<u64>,
|
||||||
/// Bounded thread page size. The Backend also enforces its authoritative cap.
|
/// Exact detail/history page size (default 20, maximum 200). Continue with event_page.next_after.
|
||||||
#[schemars(range(min = 1, max = 200))]
|
#[schemars(range(min = 1, max = 200))]
|
||||||
limit: Option<usize>,
|
limit: Option<usize>,
|
||||||
}
|
}
|
||||||
|
#[derive(Clone, Copy, Debug, Deserialize, JsonSchema)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
enum MergeRequestReadView {
|
||||||
|
Summary,
|
||||||
|
Detail,
|
||||||
|
}
|
||||||
#[derive(Debug, Deserialize, JsonSchema)]
|
#[derive(Debug, Deserialize, JsonSchema)]
|
||||||
struct OpenMergeRequestInput {
|
struct OpenMergeRequestInput {
|
||||||
ticket: String,
|
ticket: String,
|
||||||
@@ -203,7 +212,27 @@ impl Tool for MergeRequestTool {
|
|||||||
"limit must be between 1 and 200".into(),
|
"limit must be between 1 and 200".into(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
return self.show_merge_request(ws, &value.merge_request_id, value.after, value.limit);
|
let view = value.view.unwrap_or_else(|| {
|
||||||
|
if value.after.is_some() || value.limit.is_some() {
|
||||||
|
MergeRequestReadView::Detail
|
||||||
|
} else {
|
||||||
|
MergeRequestReadView::Summary
|
||||||
|
}
|
||||||
|
});
|
||||||
|
if matches!(view, MergeRequestReadView::Summary)
|
||||||
|
&& (value.after.is_some() || value.limit.is_some())
|
||||||
|
{
|
||||||
|
return Err(ToolError::InvalidArgument(
|
||||||
|
"after/limit require view=detail".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
return self.show_merge_request(
|
||||||
|
ws,
|
||||||
|
&value.merge_request_id,
|
||||||
|
view,
|
||||||
|
value.after,
|
||||||
|
value.limit,
|
||||||
|
);
|
||||||
}
|
}
|
||||||
let (method, path, body) = match self.kind {
|
let (method, path, body) = match self.kind {
|
||||||
Kind::Readiness => {
|
Kind::Readiness => {
|
||||||
@@ -290,12 +319,19 @@ impl MergeRequestTool {
|
|||||||
&self,
|
&self,
|
||||||
workspace_id: &str,
|
workspace_id: &str,
|
||||||
merge_request_id: &str,
|
merge_request_id: &str,
|
||||||
|
view: MergeRequestReadView,
|
||||||
after: Option<u64>,
|
after: Option<u64>,
|
||||||
limit: Option<usize>,
|
limit: Option<usize>,
|
||||||
) -> Result<ToolOutput, ToolError> {
|
) -> Result<ToolOutput, ToolError> {
|
||||||
let merge_request_id = encode_path_segment(merge_request_id);
|
let merge_request_id = encode_path_segment(merge_request_id);
|
||||||
let mut path = format!("/api/w/{workspace_id}/merge-requests/{merge_request_id}");
|
let mut path = format!("/api/w/{workspace_id}/merge-requests/{merge_request_id}");
|
||||||
let mut query = Vec::new();
|
let mut query = vec![format!(
|
||||||
|
"view={}",
|
||||||
|
match view {
|
||||||
|
MergeRequestReadView::Summary => "summary",
|
||||||
|
MergeRequestReadView::Detail => "detail",
|
||||||
|
}
|
||||||
|
)];
|
||||||
if let Some(after) = after {
|
if let Some(after) = after {
|
||||||
query.push(format!("after={after}"));
|
query.push(format!("after={after}"));
|
||||||
}
|
}
|
||||||
@@ -1099,7 +1135,7 @@ fn merge_request_operation_not_found() -> WipOperationError {
|
|||||||
pub fn description(n: &str) -> Option<&'static str> {
|
pub fn description(n: &str) -> Option<&'static str> {
|
||||||
match n {
|
match n {
|
||||||
"ShowMergeRequest" => Some(
|
"ShowMergeRequest" => Some(
|
||||||
"Read the selector-based Merge Request, append-only thread, source-review freshness, and target-integration evidence before review, fix, or handoff decisions.",
|
"Read an authoritative compact MR summary by default: source/target observations, full-evidence review status/readiness and stable event references, without review bodies/history. Use view=detail with bounded after/limit for exact reviews/findings and history; continue with event_page.next_after. Status is never inferred from a history page. Summary is not approval or integration authority.",
|
||||||
),
|
),
|
||||||
"CheckMergeRequestReadiness" => Some(
|
"CheckMergeRequestReadiness" => Some(
|
||||||
"Resolve current provider refs and derive readiness from exact-source review evidence; source movement requires fresh review while target-only movement preserves unchanged-source approval.",
|
"Resolve current provider refs and derive readiness from exact-source review evidence; source movement requires fresh review while target-only movement preserves unchanged-source approval.",
|
||||||
@@ -1222,7 +1258,10 @@ mod tests {
|
|||||||
let requests = client.requests.lock().expect("request lock");
|
let requests = client.requests.lock().expect("request lock");
|
||||||
assert_eq!(requests.len(), 1);
|
assert_eq!(requests.len(), 1);
|
||||||
assert_eq!(requests[0].method, WorkspaceRequestMethod::Get);
|
assert_eq!(requests[0].method, WorkspaceRequestMethod::Get);
|
||||||
assert_eq!(requests[0].path, "/api/w/ws/merge-requests/MR%2F1");
|
assert_eq!(
|
||||||
|
requests[0].path,
|
||||||
|
"/api/w/ws/merge-requests/MR%2F1?view=summary"
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
@@ -1243,7 +1282,7 @@ mod tests {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
client.requests.lock().expect("request lock")[0].path,
|
client.requests.lock().expect("request lock")[0].path,
|
||||||
"/api/w/ws/merge-requests/MR-1?after=42&limit=200"
|
"/api/w/ws/merge-requests/MR-1?view=detail&after=42&limit=200"
|
||||||
);
|
);
|
||||||
|
|
||||||
let rejected = tool
|
let rejected = tool
|
||||||
@@ -1257,6 +1296,64 @@ mod tests {
|
|||||||
assert_eq!(client.requests.lock().expect("request lock").len(), 1);
|
assert_eq!(client.requests.lock().expect("request lock").len(), 1);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn native_read_forwards_explicit_projection_and_rejects_ambiguous_summary_paging() {
|
||||||
|
let client = Arc::new(RecordingWorkspaceClient::new(vec![response(json!({
|
||||||
|
"merge_request_id": "MR-1", "view": "detail", "thread": [],
|
||||||
|
"summary": {"review_status": "approved"},
|
||||||
|
"event_page": {"omitted": false, "has_more": true, "next_after": 9, "limit": 1}
|
||||||
|
}))]));
|
||||||
|
let tool = native_merge_request_tools(
|
||||||
|
client.clone(),
|
||||||
|
MergeRequestFeatureConfig {
|
||||||
|
show: true,
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
.remove(0);
|
||||||
|
let input = native_merge_request_input(
|
||||||
|
&BTreeMap::from([
|
||||||
|
("view".into(), WipValue::String("detail".into())),
|
||||||
|
("after".into(), json_to_wip(&json!(8)).unwrap()),
|
||||||
|
("limit".into(), json_to_wip(&json!(1)).unwrap()),
|
||||||
|
]),
|
||||||
|
Some("MR-1"),
|
||||||
|
Some("merge_request_id"),
|
||||||
|
)
|
||||||
|
.unwrap_or_else(|_| panic!("native read arguments should bind"));
|
||||||
|
let output = execute_native_merge_request_tool(
|
||||||
|
&tool,
|
||||||
|
&None,
|
||||||
|
input,
|
||||||
|
Some("MR-1"),
|
||||||
|
WipCallContext {
|
||||||
|
execution: ToolExecutionContext::direct(),
|
||||||
|
security_context: "reader".into(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap_or_else(|_| panic!("native detail read should succeed"));
|
||||||
|
let output = merge_request_tool_output_json(output)
|
||||||
|
.unwrap_or_else(|_| panic!("valid JSON response"));
|
||||||
|
assert_eq!(output["summary"]["review_status"], "approved");
|
||||||
|
assert_eq!(output["event_page"]["next_after"], 9);
|
||||||
|
assert_eq!(
|
||||||
|
client.requests.lock().unwrap()[0].path,
|
||||||
|
"/api/w/ws/merge-requests/MR-1?view=detail&after=8&limit=1"
|
||||||
|
);
|
||||||
|
let rejected = tool
|
||||||
|
.tool
|
||||||
|
.execute(
|
||||||
|
r#"{"merge_request_id":"MR-1","view":"summary","limit":1}"#,
|
||||||
|
ToolExecutionContext::default(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(rejected.to_string().contains("require view=detail"));
|
||||||
|
assert_eq!(client.requests.lock().unwrap().len(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn show_requires_explicit_merge_request_identity() {
|
async fn show_requires_explicit_merge_request_identity() {
|
||||||
let client = Arc::new(RecordingWorkspaceClient::new(vec![]));
|
let client = Arc::new(RecordingWorkspaceClient::new(vec![]));
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ struct ModelTicketQueryItem {
|
|||||||
title: String,
|
title: String,
|
||||||
state: ticket::TicketState,
|
state: ticket::TicketState,
|
||||||
closure_operation_key: Option<String>,
|
closure_operation_key: Option<String>,
|
||||||
current_closure: Option<ticket::TicketClosureJudgment>,
|
current_closure: Option<Value>,
|
||||||
legacy_handoff: Option<ticket::TicketLegacyWorkflowSnapshot>,
|
legacy_handoff: Option<ticket::TicketLegacyWorkflowSnapshot>,
|
||||||
assignment_tracking: Vec<ModelAssignmentTracking>,
|
assignment_tracking: Vec<ModelAssignmentTracking>,
|
||||||
tags: Vec<ticket::WorkspaceTag>,
|
tags: Vec<ticket::WorkspaceTag>,
|
||||||
@@ -34,7 +34,6 @@ struct ModelTicketQueryItem {
|
|||||||
unresolved_review_count: usize,
|
unresolved_review_count: usize,
|
||||||
evidence: Option<ModelTicketEvidence>,
|
evidence: Option<ModelTicketEvidence>,
|
||||||
merge_requests: Vec<ModelMergeRequest>,
|
merge_requests: Vec<ModelMergeRequest>,
|
||||||
merge_request: Option<ModelMergeRequest>,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, serde::Deserialize, Serialize)]
|
#[derive(Debug, serde::Deserialize, Serialize)]
|
||||||
@@ -50,14 +49,12 @@ pub(super) struct ModelTicketDetail {
|
|||||||
ticket: String,
|
ticket: String,
|
||||||
content_digest: String,
|
content_digest: String,
|
||||||
title: String,
|
title: String,
|
||||||
body: String,
|
|
||||||
body_truncated: bool,
|
|
||||||
initial_post: ModelTicketEvent,
|
initial_post: ModelTicketEvent,
|
||||||
targets: Vec<ModelTicketTarget>,
|
targets: Vec<ModelTicketTarget>,
|
||||||
created_by: Option<String>,
|
created_by: Option<String>,
|
||||||
state: ticket::TicketState,
|
state: ticket::TicketState,
|
||||||
closure_operation_key: Option<String>,
|
closure_operation_key: Option<String>,
|
||||||
current_closure: Option<ticket::TicketClosureJudgment>,
|
current_closure: Option<Value>,
|
||||||
legacy_handoff: Option<ticket::TicketLegacyWorkflowSnapshot>,
|
legacy_handoff: Option<ticket::TicketLegacyWorkflowSnapshot>,
|
||||||
assignment_tracking: Vec<ModelAssignmentTracking>,
|
assignment_tracking: Vec<ModelAssignmentTracking>,
|
||||||
tags: Vec<ticket::WorkspaceTag>,
|
tags: Vec<ticket::WorkspaceTag>,
|
||||||
@@ -65,7 +62,7 @@ pub(super) struct ModelTicketDetail {
|
|||||||
priority: Option<String>,
|
priority: Option<String>,
|
||||||
created_at: Option<String>,
|
created_at: Option<String>,
|
||||||
updated_at: Option<String>,
|
updated_at: Option<String>,
|
||||||
closure_history: Vec<ticket::TicketCaseHistoryEntry>,
|
closure_history: Vec<Value>,
|
||||||
thread: Vec<ModelTicketEvent>,
|
thread: Vec<ModelTicketEvent>,
|
||||||
relations: ModelTicketRelations,
|
relations: ModelTicketRelations,
|
||||||
linked_objectives: Vec<ModelObjectiveSummary>,
|
linked_objectives: Vec<ModelObjectiveSummary>,
|
||||||
@@ -73,7 +70,6 @@ pub(super) struct ModelTicketDetail {
|
|||||||
current_worker: Option<ModelWorkerSummary>,
|
current_worker: Option<ModelWorkerSummary>,
|
||||||
implementation_reports: Vec<ModelEvidenceEvent>,
|
implementation_reports: Vec<ModelEvidenceEvent>,
|
||||||
merge_requests: Vec<ModelMergeRequest>,
|
merge_requests: Vec<ModelMergeRequest>,
|
||||||
merge_request: Option<ModelMergeRequest>,
|
|
||||||
evidence: Option<ModelTicketEvidence>,
|
evidence: Option<ModelTicketEvidence>,
|
||||||
actions: Option<ModelTicketActions>,
|
actions: Option<ModelTicketActions>,
|
||||||
event_page: Option<ModelEventPage>,
|
event_page: Option<ModelEventPage>,
|
||||||
@@ -280,6 +276,21 @@ fn typed_field<T: serde::de::DeserializeOwned>(
|
|||||||
serde_json::from_value(value.clone()).map_err(|error| format!("Invalid Ticket {key}: {error}"))
|
serde_json::from_value(value.clone()).map_err(|error| format!("Invalid Ticket {key}: {error}"))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn closure_metadata<T: Serialize>(value: T) -> Result<Value, String> {
|
||||||
|
let mut value = serde_json::to_value(value).map_err(|error| error.to_string())?;
|
||||||
|
if let Some(object) = value.as_object_mut() {
|
||||||
|
// The full explanation is available in thread history/post, not repeated
|
||||||
|
// in current judgment and closure-index metadata as well.
|
||||||
|
object.remove("explanation");
|
||||||
|
}
|
||||||
|
Ok(value)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn project_current_closure(root: &Map<String, Value>) -> Result<Option<Value>, String> {
|
||||||
|
let closure: Option<ticket::TicketClosureJudgment> = typed_field(root, "current_closure")?;
|
||||||
|
closure.map(closure_metadata).transpose()
|
||||||
|
}
|
||||||
|
|
||||||
pub(super) fn project_ticket_query(value: Value) -> Result<ModelTicketQueryResponse, String> {
|
pub(super) fn project_ticket_query(value: Value) -> Result<ModelTicketQueryResponse, String> {
|
||||||
let root = object(&value, "Ticket query response")?;
|
let root = object(&value, "Ticket query response")?;
|
||||||
let page = object_field(root, "page")?;
|
let page = object_field(root, "page")?;
|
||||||
@@ -301,7 +312,7 @@ fn project_ticket_query_item(value: &Value) -> Result<ModelTicketQueryItem, Stri
|
|||||||
title: string_field(item, "title")?,
|
title: string_field(item, "title")?,
|
||||||
state: typed_field(item, "state")?,
|
state: typed_field(item, "state")?,
|
||||||
closure_operation_key: typed_field(item, "closure_operation_key")?,
|
closure_operation_key: typed_field(item, "closure_operation_key")?,
|
||||||
current_closure: typed_field(item, "current_closure")?,
|
current_closure: project_current_closure(item)?,
|
||||||
legacy_handoff: typed_field(item, "legacy_handoff")?,
|
legacy_handoff: typed_field(item, "legacy_handoff")?,
|
||||||
assignment_tracking: project_assignment_tracking(item)?,
|
assignment_tracking: project_assignment_tracking(item)?,
|
||||||
tags: project_tags(item)?,
|
tags: project_tags(item)?,
|
||||||
@@ -330,14 +341,84 @@ fn project_ticket_query_item(value: &Value) -> Result<ModelTicketQueryItem, Stri
|
|||||||
.iter()
|
.iter()
|
||||||
.map(project_merge_request)
|
.map(project_merge_request)
|
||||||
.collect::<Result<Vec<_>, _>>()?,
|
.collect::<Result<Vec<_>, _>>()?,
|
||||||
merge_request: item
|
|
||||||
.get("merge_request")
|
|
||||||
.filter(|value| !value.is_null())
|
|
||||||
.map(project_merge_request)
|
|
||||||
.transpose()?,
|
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Status is deliberately projected from authority fields, not from serialized detail:
|
||||||
|
/// prose/history size and history parsing have no effect on this response.
|
||||||
|
pub(super) fn project_ticket_status(value: Value) -> Result<Value, String> {
|
||||||
|
use serde_json::json;
|
||||||
|
let root = object(&value, "Ticket summary response")?;
|
||||||
|
let state: ticket::TicketState = typed_field(root, "state")?;
|
||||||
|
let closure_key: Option<String> = typed_field(root, "closure_operation_key")?;
|
||||||
|
let closure: Option<ticket::TicketClosureJudgment> = typed_field(root, "current_closure")?;
|
||||||
|
let targets: Vec<ModelTicketTarget> = typed_field(root, "targets")?;
|
||||||
|
let tracking = project_assignment_tracking(root)?.into_iter().map(|row| json!({
|
||||||
|
"assignment_id": row.assignment_id, "role": row.role, "principal": row.principal,
|
||||||
|
"assigned_by": row.assigned_by, "assigned_at": row.assigned_at, "current": row.current,
|
||||||
|
"status": row.tracking.map(|tracking| tracking.status),
|
||||||
|
"last_event_id": row.last_event_id, "updated_at": row.updated_at,
|
||||||
|
})).collect::<Vec<_>>();
|
||||||
|
let merges = array_field(root, "merge_requests")?
|
||||||
|
.iter()
|
||||||
|
.map(|value| {
|
||||||
|
let mut merge = serde_json::to_value(project_merge_request(value)?)
|
||||||
|
.map_err(|error| error.to_string())?;
|
||||||
|
merge
|
||||||
|
.as_object_mut()
|
||||||
|
.expect("typed merge object")
|
||||||
|
.remove("review_excerpt");
|
||||||
|
Ok(merge)
|
||||||
|
})
|
||||||
|
.collect::<Result<Vec<Value>, String>>()?;
|
||||||
|
let latest_event_ref = array_field(root, "events")?
|
||||||
|
.last()
|
||||||
|
.map(|event| string_field(object(event, "Ticket event")?, "event_ref"))
|
||||||
|
.transpose()?;
|
||||||
|
Ok(json!({
|
||||||
|
"ticket": resource_ref(root, "resource_key", "T-")?,
|
||||||
|
"title": string_field(root, "title")?,
|
||||||
|
"content_digest": string_field(root, "content_digest")?,
|
||||||
|
"state": state, "closure_operation_key": closure_key,
|
||||||
|
"current_closure": closure.map(|j| json!({"operation_key": j.operation_key, "reason": j.reason, "author": j.author, "at": j.at})),
|
||||||
|
"targets": targets, "tags": project_tags(root)?,
|
||||||
|
"readiness": optional_string(root, "readiness")?,
|
||||||
|
"priority": optional_string(root, "priority")?,
|
||||||
|
"updated_at": optional_string(root, "updated_at")?,
|
||||||
|
"assignment_tracking": tracking,
|
||||||
|
"current_worker": root.get("current_worker").filter(|value| !value.is_null()).map(project_worker).transpose()?,
|
||||||
|
"merge_requests": merges,
|
||||||
|
"evidence": root.get("evidence").map(project_evidence).transpose()?,
|
||||||
|
"actions": root.get("action_eligibility").filter(|value| !value.is_null()).map(project_actions).transpose()?,
|
||||||
|
"latest_event_ref": latest_event_ref,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn project_ticket_history(value: Value) -> Result<Value, String> {
|
||||||
|
let root = object(&value, "Ticket history response")?;
|
||||||
|
let thread = array_field(root, "events")?
|
||||||
|
.iter()
|
||||||
|
.filter(|event| event.get("is_initial") != Some(&Value::Bool(true)))
|
||||||
|
.map(project_ticket_event)
|
||||||
|
.collect::<Result<Vec<_>, _>>()?;
|
||||||
|
Ok(serde_json::json!({
|
||||||
|
"ticket": resource_ref(root, "resource_key", "T-")?,
|
||||||
|
"thread": thread,
|
||||||
|
"event_page": project_event_page(root.get("event_page").ok_or("missing event_page")?)?,
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn project_ticket_post(value: Value) -> Result<Value, String> {
|
||||||
|
let post = project_ticket_event(&value)?;
|
||||||
|
if post.body_truncated || post.body.is_none() {
|
||||||
|
return Err(
|
||||||
|
"Full-post read returned an absent or truncated body; do not use it for edit CAS"
|
||||||
|
.into(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
serde_json::to_value(post).map_err(|error| error.to_string())
|
||||||
|
}
|
||||||
|
|
||||||
pub(super) fn project_ticket_detail(value: Value) -> Result<ModelTicketDetail, String> {
|
pub(super) fn project_ticket_detail(value: Value) -> Result<ModelTicketDetail, String> {
|
||||||
let root = object(&value, "Ticket detail response")?;
|
let root = object(&value, "Ticket detail response")?;
|
||||||
let current_worker = root
|
let current_worker = root
|
||||||
@@ -354,8 +435,6 @@ pub(super) fn project_ticket_detail(value: Value) -> Result<ModelTicketDetail, S
|
|||||||
ticket: resource_ref(root, "resource_key", "T-")?,
|
ticket: resource_ref(root, "resource_key", "T-")?,
|
||||||
content_digest: string_field(root, "content_digest")?,
|
content_digest: string_field(root, "content_digest")?,
|
||||||
title: string_field(root, "title")?,
|
title: string_field(root, "title")?,
|
||||||
body: string_field(root, "body")?,
|
|
||||||
body_truncated: bool_field(root, "body_truncated")?,
|
|
||||||
initial_post: project_ticket_event(
|
initial_post: project_ticket_event(
|
||||||
root.get("initial_post")
|
root.get("initial_post")
|
||||||
.ok_or("Ticket detail response is missing initial_post")?,
|
.ok_or("Ticket detail response is missing initial_post")?,
|
||||||
@@ -364,9 +443,15 @@ pub(super) fn project_ticket_detail(value: Value) -> Result<ModelTicketDetail, S
|
|||||||
created_by: optional_string(root, "created_by")?,
|
created_by: optional_string(root, "created_by")?,
|
||||||
state: typed_field(root, "state")?,
|
state: typed_field(root, "state")?,
|
||||||
closure_operation_key: typed_field(root, "closure_operation_key")?,
|
closure_operation_key: typed_field(root, "closure_operation_key")?,
|
||||||
current_closure: typed_field(root, "current_closure")?,
|
current_closure: project_current_closure(root)?,
|
||||||
legacy_handoff: typed_field(root, "legacy_handoff")?,
|
legacy_handoff: typed_field(root, "legacy_handoff")?,
|
||||||
closure_history: typed_field(root, "closure_history")?,
|
closure_history: typed_field::<Vec<ticket::TicketCaseHistoryEntry>>(
|
||||||
|
root,
|
||||||
|
"closure_history",
|
||||||
|
)?
|
||||||
|
.into_iter()
|
||||||
|
.map(closure_metadata)
|
||||||
|
.collect::<Result<Vec<_>, _>>()?,
|
||||||
assignment_tracking: project_assignment_tracking(root)?,
|
assignment_tracking: project_assignment_tracking(root)?,
|
||||||
tags: project_tags(root)?,
|
tags: project_tags(root)?,
|
||||||
readiness: optional_string(root, "readiness")?,
|
readiness: optional_string(root, "readiness")?,
|
||||||
@@ -375,6 +460,7 @@ pub(super) fn project_ticket_detail(value: Value) -> Result<ModelTicketDetail, S
|
|||||||
updated_at: optional_string(root, "updated_at")?,
|
updated_at: optional_string(root, "updated_at")?,
|
||||||
thread: array_field(root, "events")?
|
thread: array_field(root, "events")?
|
||||||
.iter()
|
.iter()
|
||||||
|
.filter(|event| event.get("is_initial") != Some(&Value::Bool(true)))
|
||||||
.map(project_ticket_event)
|
.map(project_ticket_event)
|
||||||
.collect::<Result<Vec<_>, _>>()?,
|
.collect::<Result<Vec<_>, _>>()?,
|
||||||
relations: project_relations(root.get("relations"))?,
|
relations: project_relations(root.get("relations"))?,
|
||||||
@@ -392,11 +478,6 @@ pub(super) fn project_ticket_detail(value: Value) -> Result<ModelTicketDetail, S
|
|||||||
.iter()
|
.iter()
|
||||||
.map(project_merge_request)
|
.map(project_merge_request)
|
||||||
.collect::<Result<Vec<_>, _>>()?,
|
.collect::<Result<Vec<_>, _>>()?,
|
||||||
merge_request: root
|
|
||||||
.get("merge_request")
|
|
||||||
.filter(|value| !value.is_null())
|
|
||||||
.map(project_merge_request)
|
|
||||||
.transpose()?,
|
|
||||||
evidence: root.get("evidence").map(project_evidence).transpose()?,
|
evidence: root.get("evidence").map(project_evidence).transpose()?,
|
||||||
actions: root
|
actions: root
|
||||||
.get("action_eligibility")
|
.get("action_eligibility")
|
||||||
@@ -919,7 +1000,18 @@ mod tests {
|
|||||||
assert_eq!(projected["state"], "open");
|
assert_eq!(projected["state"], "open");
|
||||||
assert_eq!(projected["closure_operation_key"], "close-1");
|
assert_eq!(projected["closure_operation_key"], "close-1");
|
||||||
assert_eq!(projected["current_closure"], Value::Null);
|
assert_eq!(projected["current_closure"], Value::Null);
|
||||||
assert_eq!(projected["closure_history"], history);
|
assert_eq!(
|
||||||
|
projected["closure_history"],
|
||||||
|
json!(
|
||||||
|
history
|
||||||
|
.as_array()
|
||||||
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.map(closure_metadata)
|
||||||
|
.collect::<Result<Vec<_>, _>>()
|
||||||
|
.unwrap()
|
||||||
|
)
|
||||||
|
);
|
||||||
assert_eq!(projected["legacy_handoff"], legacy);
|
assert_eq!(projected["legacy_handoff"], legacy);
|
||||||
assert_eq!(projected["tags"], response["tags"]);
|
assert_eq!(projected["tags"], response["tags"]);
|
||||||
assert_eq!(projected["content_digest"], "content-digest");
|
assert_eq!(projected["content_digest"], "content-digest");
|
||||||
@@ -1191,7 +1283,8 @@ mod tests {
|
|||||||
.expect("source evidence must project");
|
.expect("source evidence must project");
|
||||||
let projected = serde_json::to_value(projected).expect("serialize Ticket detail");
|
let projected = serde_json::to_value(projected).expect("serialize Ticket detail");
|
||||||
|
|
||||||
for merge in [&projected["merge_requests"][0], &projected["merge_request"]] {
|
assert!(projected.get("merge_request").is_none());
|
||||||
|
for merge in [&projected["merge_requests"][0]] {
|
||||||
assert_eq!(merge["subject_ref"], json!(subject_ref));
|
assert_eq!(merge["subject_ref"], json!(subject_ref));
|
||||||
assert_eq!(merge["source_ref_observation"], observation);
|
assert_eq!(merge["source_ref_observation"], observation);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
@@ -1378,9 +1471,14 @@ mod tests {
|
|||||||
"current_closure",
|
"current_closure",
|
||||||
"legacy_handoff",
|
"legacy_handoff",
|
||||||
] {
|
] {
|
||||||
assert_eq!(projected_detail[field], detail[field], "detail {field}");
|
let expected = if field == "current_closure" {
|
||||||
|
closure_metadata(&detail[field]).unwrap()
|
||||||
|
} else {
|
||||||
|
detail[field].clone()
|
||||||
|
};
|
||||||
|
assert_eq!(projected_detail[field], expected, "detail {field}");
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
projected_query["tickets"][0][field], detail[field],
|
projected_query["tickets"][0][field], expected,
|
||||||
"query {field}"
|
"query {field}"
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -1392,10 +1490,22 @@ mod tests {
|
|||||||
assert!(actual.get("workspace_id").is_none());
|
assert!(actual.get("workspace_id").is_none());
|
||||||
assert!(actual.get("ticket_id").is_none());
|
assert!(actual.get("ticket_id").is_none());
|
||||||
}
|
}
|
||||||
assert_eq!(projected_detail["closure_history"], history);
|
assert_eq!(
|
||||||
|
projected_detail["closure_history"],
|
||||||
|
json!(
|
||||||
|
history
|
||||||
|
.as_array()
|
||||||
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.map(closure_metadata)
|
||||||
|
.collect::<Result<Vec<_>, _>>()
|
||||||
|
.unwrap()
|
||||||
|
)
|
||||||
|
);
|
||||||
assert_eq!(projected_detail["content_digest"], "content-1");
|
assert_eq!(projected_detail["content_digest"], "content-1");
|
||||||
assert_eq!(projected_detail["created_by"], "workspace-user:creator");
|
assert_eq!(projected_detail["created_by"], "workspace-user:creator");
|
||||||
assert_eq!(projected_detail["body_truncated"], false);
|
assert_eq!(projected_detail["initial_post"]["body_truncated"], false);
|
||||||
|
assert!(projected_detail.get("body").is_none());
|
||||||
for initial in [true, false] {
|
for initial in [true, false] {
|
||||||
let input = post_fixture(initial, "The same post contract");
|
let input = post_fixture(initial, "The same post contract");
|
||||||
let output = serde_json::to_value(project_ticket_event(&input).unwrap()).unwrap();
|
let output = serde_json::to_value(project_ticket_event(&input).unwrap()).unwrap();
|
||||||
@@ -1488,6 +1598,93 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn ticket_views_keep_prose_once_and_summary_independent_of_prose_and_history() {
|
||||||
|
let body = "unique-long-requirement-".repeat(600);
|
||||||
|
let post = post_fixture(true, &body);
|
||||||
|
let closure = json!({"operation_key":"close", "reason":"completed", "explanation":"closure prose", "references":[], "author":"user", "at":"today"});
|
||||||
|
let tracking = json!({"workspace_id":"ws", "ticket_id":"internal", "assignment_id":"assignment", "role":"orchestrator", "principal":{"kind":"workspace_agent","agent_key":"workspace-orchestrator"}, "assigned_by":"user", "assigned_at":"today", "release":null, "current":true, "last_event_id":"handling-event", "updated_at":"today", "migration_diagnostic":null, "tracking":{"status":"waiting", "waiting_reason":"prose", "progress":null, "next_action":"prose", "result":null}});
|
||||||
|
let merge = json!({"merge_request_id":"MR-1", "repository_key":"main", "state":"open", "selector_from":"topic", "selector_to":"main", "review_status":"request_changes", "current_subject_ref":"sha1", "source_ref_observation":{"status":"observed"}, "integration_evidence_error":null, "review_excerpt":"review prose"});
|
||||||
|
let mut input = json!({
|
||||||
|
"resource_key":"T-848", "title":"Views", "body":body, "body_truncated":false, "initial_post":post,
|
||||||
|
"content_digest":"content", "state":"closed", "closure_operation_key":"close", "current_closure":closure,
|
||||||
|
"closure_history":[], "legacy_handoff":null, "targets":[], "tags":[], "assignment_tracking":[tracking],
|
||||||
|
"events":[post, post_fixture(false, "discussion")], "assignments":[], "linked_objectives":[],
|
||||||
|
"implementation_reports":[], "merge_requests":[merge.clone()], "merge_request":merge,
|
||||||
|
"event_page":{"next_cursor":"stable-cursor", "has_more":true}
|
||||||
|
});
|
||||||
|
let detail = serde_json::to_value(project_ticket_detail(input.clone()).unwrap()).unwrap();
|
||||||
|
assert_eq!(detail.to_string().matches(&body).count(), 1);
|
||||||
|
assert!(detail.get("body").is_none());
|
||||||
|
assert!(detail.get("merge_request").is_none());
|
||||||
|
assert_eq!(detail["thread"].as_array().unwrap().len(), 1);
|
||||||
|
assert_eq!(detail["initial_post"]["event_ref"], "post-initial");
|
||||||
|
assert_eq!(detail["initial_post"]["body_digest"], "a".repeat(64));
|
||||||
|
assert!(detail["current_closure"].get("explanation").is_none());
|
||||||
|
let history = project_ticket_history(input.clone()).unwrap();
|
||||||
|
assert_eq!(history.as_object().unwrap().len(), 3);
|
||||||
|
assert_eq!(history["event_page"]["next_cursor"], "stable-cursor");
|
||||||
|
let summary = project_ticket_status(input.clone()).unwrap();
|
||||||
|
// Reconstruct the removed model-facing aliases on this same fixture.
|
||||||
|
let mut duplicated = detail.clone();
|
||||||
|
duplicated["body"] = detail["initial_post"]["body"].clone();
|
||||||
|
duplicated["body_truncated"] = detail["initial_post"]["body_truncated"].clone();
|
||||||
|
duplicated["merge_request"] = detail["merge_requests"][0].clone();
|
||||||
|
duplicated["thread"]
|
||||||
|
.as_array_mut()
|
||||||
|
.unwrap()
|
||||||
|
.insert(0, detail["initial_post"].clone());
|
||||||
|
assert_eq!(duplicated.to_string().matches(&body).count(), 3);
|
||||||
|
assert!(duplicated.to_string().len() > detail.to_string().len());
|
||||||
|
let mut changed = input.clone();
|
||||||
|
changed["content_digest"] = json!("new-requirements-digest");
|
||||||
|
changed["merge_requests"][0]["current_subject_ref"] = json!("sha2");
|
||||||
|
changed["merge_requests"][0]["review_status"] = json!("pending");
|
||||||
|
let changed = project_ticket_status(changed).unwrap();
|
||||||
|
assert_ne!(summary["content_digest"], changed["content_digest"]);
|
||||||
|
assert_ne!(
|
||||||
|
summary["merge_requests"][0]["subject_ref"],
|
||||||
|
changed["merge_requests"][0]["subject_ref"]
|
||||||
|
);
|
||||||
|
assert_ne!(
|
||||||
|
summary["merge_requests"][0]["review_status"],
|
||||||
|
changed["merge_requests"][0]["review_status"]
|
||||||
|
);
|
||||||
|
assert_eq!(summary["content_digest"], "content");
|
||||||
|
assert_eq!(summary["closure_operation_key"], "close");
|
||||||
|
assert_eq!(summary["assignment_tracking"][0]["status"], "waiting");
|
||||||
|
assert_eq!(
|
||||||
|
summary["assignment_tracking"][0]["last_event_id"],
|
||||||
|
"handling-event"
|
||||||
|
);
|
||||||
|
assert_eq!(summary["merge_requests"][0]["subject_ref"], "sha1");
|
||||||
|
assert_eq!(
|
||||||
|
summary["merge_requests"][0]["review_status"],
|
||||||
|
"request_changes"
|
||||||
|
);
|
||||||
|
let huge = "全文🦀".repeat(100_000);
|
||||||
|
input["body"] = json!(huge);
|
||||||
|
input["initial_post"]["body"] = json!(huge);
|
||||||
|
input["current_closure"]["explanation"] = json!(huge);
|
||||||
|
input["assignment_tracking"][0]["tracking"]["waiting_reason"] = json!(huge);
|
||||||
|
input["merge_requests"][0]["review_excerpt"] = json!(huge);
|
||||||
|
input["closure_history"] = json!(vec![json!({"explanation":huge}); 20]);
|
||||||
|
input["implementation_reports"] = json!(vec![json!({"excerpt":huge}); 20]);
|
||||||
|
input["events"] = json!(vec![post_fixture(false, "history prose"); 1000]);
|
||||||
|
let larger = project_ticket_status(input).unwrap();
|
||||||
|
assert_eq!(summary, larger);
|
||||||
|
eprintln!(
|
||||||
|
"Ticket workspace bytes: duplicated fixture={}, detail={}, summary={} (identical with large prose and 1000 events)",
|
||||||
|
duplicated.to_string().len(),
|
||||||
|
detail.to_string().len(),
|
||||||
|
summary.to_string().len()
|
||||||
|
);
|
||||||
|
let mut full = post_fixture(false, &huge);
|
||||||
|
assert_eq!(project_ticket_post(full.clone()).unwrap()["body"], huge);
|
||||||
|
full["body_truncated"] = json!(true);
|
||||||
|
assert!(project_ticket_post(full).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn ticket_tag_history_projection_preserves_observed_names_and_identities() {
|
fn ticket_tag_history_projection_preserves_observed_names_and_identities() {
|
||||||
let tag = serde_json::json!({"workspace_id":"ws","id":"tag-1","name":"Old name","last_operation_key":"create","deleted":false});
|
let tag = serde_json::json!({"workspace_id":"ws","id":"tag-1","name":"Old name","last_operation_key":"create","deleted":false});
|
||||||
|
|||||||
@@ -80,7 +80,11 @@ use crate::wip::{
|
|||||||
use crate::worker::{WorkspaceClient, WorkspaceRequest, WorkspaceRequestMethod};
|
use crate::worker::{WorkspaceClient, WorkspaceRequest, WorkspaceRequestMethod};
|
||||||
use agen::tool::{Tool, ToolError, ToolExecutionContext, ToolMeta, ToolOutput};
|
use agen::tool::{Tool, ToolError, ToolExecutionContext, ToolMeta, ToolOutput};
|
||||||
|
|
||||||
use super::resource_projection::{project_ticket_detail, project_ticket_query};
|
use super::resource_projection::{
|
||||||
|
project_ticket_detail, project_ticket_history, project_ticket_post, project_ticket_query,
|
||||||
|
project_ticket_status,
|
||||||
|
};
|
||||||
|
use ticket::tool::ShowTicketView;
|
||||||
|
|
||||||
#[derive(Clone, Copy)]
|
#[derive(Clone, Copy)]
|
||||||
enum WorkspaceTicketReadKind {
|
enum WorkspaceTicketReadKind {
|
||||||
@@ -102,7 +106,7 @@ impl WorkspaceTicketReadKind {
|
|||||||
"Query authoritative Workspace Tickets with bounded typed filters, stable snippets, evidence summaries, and cursor metadata."
|
"Query authoritative Workspace Tickets with bounded typed filters, stable snippets, evidence summaries, and cursor metadata."
|
||||||
}
|
}
|
||||||
Self::Show => {
|
Self::Show => {
|
||||||
"Show one authoritative Workspace Ticket with its ordered implementation targets (repository_key, ref_selector, access), content digest, paged thread, links, historical implementation reports, and current Merge Request readiness evidence. Read targets here before choosing a work repository; a null selector remains unspecified, not an inferred default."
|
"Read one authoritative Workspace Ticket. Use view=summary for current state/CAS, targets, delegation identities/status and MR evidence without prose/history; detail (default) for canonical initial_post and bounded discussion; history with event_cursor for only thread/page metadata; post with an observed event_ref for the exact full post. Never use truncated text for edit CAS or guess event IDs. Read targets before choosing a repository; a null selector remains unspecified. Summary alone is not approval or completion authority."
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -227,9 +231,14 @@ struct WorkspaceQueryTicketInput {
|
|||||||
|
|
||||||
#[derive(Debug, Deserialize, Serialize, JsonSchema)]
|
#[derive(Debug, Deserialize, Serialize, JsonSchema)]
|
||||||
struct WorkspaceShowTicketInput {
|
struct WorkspaceShowTicketInput {
|
||||||
|
/// summary omits prose/history; detail is default; history returns only a thread page; post returns the exact full post up to 1 MiB (larger posts fail, never truncate).
|
||||||
|
#[serde(default)]
|
||||||
|
view: ShowTicketView,
|
||||||
|
/// Exact observed event_ref, required only for view=post. Never a display ordinal.
|
||||||
|
event_ref: Option<String>,
|
||||||
/// Ticket reference. Prefer `T-*`; canonical internal ids remain accepted for compatibility.
|
/// Ticket reference. Prefer `T-*`; canonical internal ids remain accepted for compatibility.
|
||||||
id: String,
|
id: String,
|
||||||
/// Most-recent thread entries to return, bounded by the Backend to 1..=50.
|
/// Most-recent thread entries to return, bounded by the Backend to 1..=100.
|
||||||
event_limit: Option<usize>,
|
event_limit: Option<usize>,
|
||||||
/// Opaque event cursor returned by a prior ShowTicket page.
|
/// Opaque event cursor returned by a prior ShowTicket page.
|
||||||
event_cursor: Option<String>,
|
event_cursor: Option<String>,
|
||||||
@@ -251,6 +260,8 @@ impl Tool for WorkspaceTicketReadTool {
|
|||||||
let workspace_id = self.client.workspace_id().ok_or_else(|| {
|
let workspace_id = self.client.workspace_id().ok_or_else(|| {
|
||||||
ToolError::InvalidArgument("Workspace Ticket reads require workspace identity".into())
|
ToolError::InvalidArgument("Workspace Ticket reads require workspace identity".into())
|
||||||
})?;
|
})?;
|
||||||
|
let mut view = ShowTicketView::Detail;
|
||||||
|
let mut method = WorkspaceRequestMethod::Post;
|
||||||
let (path, body) = match self.kind {
|
let (path, body) = match self.kind {
|
||||||
WorkspaceTicketReadKind::Query => {
|
WorkspaceTicketReadKind::Query => {
|
||||||
let input: WorkspaceQueryTicketInput = serde_json::from_str(&input)
|
let input: WorkspaceQueryTicketInput = serde_json::from_str(&input)
|
||||||
@@ -276,22 +287,41 @@ impl Tool for WorkspaceTicketReadTool {
|
|||||||
"ShowTicket.id must not be empty".into(),
|
"ShowTicket.id must not be empty".into(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
let path = format!("/api/w/{workspace_id}/tickets/{}/show", input.id.trim());
|
view = input.view;
|
||||||
let body = json!({
|
view.validate(input.event_cursor.as_deref(), input.event_ref.as_deref())
|
||||||
"event_limit": input.event_limit,
|
.map_err(ToolError::InvalidArgument)?;
|
||||||
"event_cursor": input.event_cursor,
|
let id =
|
||||||
});
|
WorkspaceHttpTicketBackend::ticket_path(&TicketIdOrSlug::from(input.id.trim()));
|
||||||
(path, body)
|
if let Some(event_ref) = input.event_ref {
|
||||||
|
method = WorkspaceRequestMethod::Get;
|
||||||
|
let event_ref =
|
||||||
|
WorkspaceHttpTicketBackend::ticket_path(&TicketIdOrSlug::from(event_ref));
|
||||||
|
(
|
||||||
|
format!("/api/w/{workspace_id}/tickets/{id}/posts/{event_ref}"),
|
||||||
|
Value::Null,
|
||||||
|
)
|
||||||
|
} else {
|
||||||
|
let path = format!("/api/w/{workspace_id}/tickets/{id}/show");
|
||||||
|
let body = json!({
|
||||||
|
"event_limit": if view == ShowTicketView::Summary { Some(1) } else { input.event_limit },
|
||||||
|
"event_cursor": input.event_cursor,
|
||||||
|
});
|
||||||
|
(path, body)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let response = self
|
let response = self
|
||||||
.client
|
.client
|
||||||
.execute(WorkspaceRequest::json(
|
.execute(if method == WorkspaceRequestMethod::Get {
|
||||||
WorkspaceRequestMethod::Post,
|
WorkspaceRequest::get(path)
|
||||||
path,
|
} else {
|
||||||
serde_json::to_string(&body)
|
WorkspaceRequest::json(
|
||||||
.map_err(|error| ToolError::Internal(error.to_string()))?,
|
method,
|
||||||
))
|
path,
|
||||||
|
serde_json::to_string(&body)
|
||||||
|
.map_err(|error| ToolError::Internal(error.to_string()))?,
|
||||||
|
)
|
||||||
|
})
|
||||||
.map_err(|error| ToolError::ExecutionFailed(error.to_string()))?;
|
.map_err(|error| ToolError::ExecutionFailed(error.to_string()))?;
|
||||||
if !response.is_success() {
|
if !response.is_success() {
|
||||||
return Err(ToolError::ExecutionFailed(ticket_api_error_message(
|
return Err(ToolError::ExecutionFailed(ticket_api_error_message(
|
||||||
@@ -308,9 +338,20 @@ impl Tool for WorkspaceTicketReadTool {
|
|||||||
WorkspaceTicketReadKind::Query => serde_json::to_string(
|
WorkspaceTicketReadKind::Query => serde_json::to_string(
|
||||||
&project_ticket_query(response_value).map_err(ToolError::ExecutionFailed)?,
|
&project_ticket_query(response_value).map_err(ToolError::ExecutionFailed)?,
|
||||||
),
|
),
|
||||||
WorkspaceTicketReadKind::Show => serde_json::to_string(
|
WorkspaceTicketReadKind::Show => match view {
|
||||||
&project_ticket_detail(response_value).map_err(ToolError::ExecutionFailed)?,
|
ShowTicketView::Detail => serde_json::to_string(
|
||||||
),
|
&project_ticket_detail(response_value).map_err(ToolError::ExecutionFailed)?,
|
||||||
|
),
|
||||||
|
ShowTicketView::Summary => serde_json::to_string(
|
||||||
|
&project_ticket_status(response_value).map_err(ToolError::ExecutionFailed)?,
|
||||||
|
),
|
||||||
|
ShowTicketView::History => serde_json::to_string(
|
||||||
|
&project_ticket_history(response_value).map_err(ToolError::ExecutionFailed)?,
|
||||||
|
),
|
||||||
|
ShowTicketView::Post => serde_json::to_string(
|
||||||
|
&json!({"post": project_ticket_post(response_value).map_err(ToolError::ExecutionFailed)?}),
|
||||||
|
),
|
||||||
|
},
|
||||||
}
|
}
|
||||||
.map_err(|error| ToolError::Internal(error.to_string()))?;
|
.map_err(|error| ToolError::Internal(error.to_string()))?;
|
||||||
Ok(ToolOutput {
|
Ok(ToolOutput {
|
||||||
@@ -2894,6 +2935,127 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn show_views_share_tool_and_route_bound_wip_wiring() {
|
||||||
|
for view in ["summary", "detail", "history", "post"] {
|
||||||
|
let mut detail = target_detail_fixture(
|
||||||
|
json!([{"repository_key":"main", "ref_selector":null, "access":"read_write"}]),
|
||||||
|
);
|
||||||
|
let unicode = "全文🦀é\n".repeat(20_000);
|
||||||
|
detail["event_page"] = json!({"next_cursor":"opaque-before", "has_more":true});
|
||||||
|
let mut post = detail["initial_post"].clone();
|
||||||
|
post["event_ref"] = json!("persisted:id/with unicode界");
|
||||||
|
post["body"] = json!(unicode);
|
||||||
|
let response = if view == "post" { post.clone() } else { detail };
|
||||||
|
let client = Arc::new(DecisionClient {
|
||||||
|
requests: Mutex::new(Vec::new()),
|
||||||
|
response: crate::worker::WorkspaceResponse {
|
||||||
|
status: 200,
|
||||||
|
body: response.to_string(),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
let (meta, tool) =
|
||||||
|
workspace_ticket_read_definition(client.clone(), WorkspaceTicketReadKind::Show)();
|
||||||
|
assert!(meta.input_schema["properties"]["view"].is_object());
|
||||||
|
assert!(meta.input_schema["properties"]["event_ref"].is_object());
|
||||||
|
let mut input = json!({"id":"T-739", "view":view});
|
||||||
|
if view == "post" {
|
||||||
|
input["event_ref"] = post["event_ref"].clone();
|
||||||
|
}
|
||||||
|
if view == "history" {
|
||||||
|
input["event_cursor"] = json!("opaque-before");
|
||||||
|
}
|
||||||
|
let direct = ticket_tool_output_json(
|
||||||
|
tool.execute(&input.to_string(), ToolExecutionContext::direct())
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap_or_else(|_| panic!("Ticket {view} returned invalid JSON"));
|
||||||
|
let tools =
|
||||||
|
native_ticket_tools(client.clone(), TicketFeatureAccess::read_only()).unwrap();
|
||||||
|
let descriptor = ticket_descriptor(&tools, "Ticket", "bound").unwrap();
|
||||||
|
let read = descriptor
|
||||||
|
.operations
|
||||||
|
.iter()
|
||||||
|
.find(|op| op.name == "read")
|
||||||
|
.unwrap();
|
||||||
|
assert!(read.parameters.iter().any(|param| param.name == "view"));
|
||||||
|
assert!(
|
||||||
|
read.parameters
|
||||||
|
.iter()
|
||||||
|
.any(|param| param.name == "event_ref")
|
||||||
|
);
|
||||||
|
assert!(!read.parameters.iter().any(|param| param.name == "id"));
|
||||||
|
let resolver = TicketItemResolver {
|
||||||
|
tools: operation_map(tools),
|
||||||
|
permissions: None,
|
||||||
|
collection_route: "/tickets".into(),
|
||||||
|
observations: Arc::new(Mutex::new(TicketObservationState::default())),
|
||||||
|
};
|
||||||
|
input.as_object_mut().unwrap().remove("id");
|
||||||
|
let arguments = input
|
||||||
|
.as_object()
|
||||||
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.map(|(key, value)| (key.clone(), json_to_wip(value).unwrap()))
|
||||||
|
.collect();
|
||||||
|
let bound = resolver.resolve("T-739").unwrap();
|
||||||
|
let output = bound
|
||||||
|
.handler
|
||||||
|
.call(
|
||||||
|
"read",
|
||||||
|
&arguments,
|
||||||
|
WipCallContext {
|
||||||
|
execution: ToolExecutionContext::direct(),
|
||||||
|
security_context: "reader".into(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap_or_else(|_| panic!("WIP {view} failed"));
|
||||||
|
assert_eq!(wip_to_json(&output.value).unwrap(), direct);
|
||||||
|
match view {
|
||||||
|
"post" => {
|
||||||
|
assert_eq!(direct["post"]["body"], unicode);
|
||||||
|
assert_eq!(direct["post"]["body_truncated"], false);
|
||||||
|
}
|
||||||
|
"history" => {
|
||||||
|
assert_eq!(direct.as_object().unwrap().len(), 3);
|
||||||
|
}
|
||||||
|
"summary" => {
|
||||||
|
assert!(direct.get("initial_post").is_none());
|
||||||
|
assert_eq!(direct["content_digest"], "observed-content");
|
||||||
|
}
|
||||||
|
_ => {
|
||||||
|
assert_eq!(direct["initial_post"]["body"], "Requirements");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let requests = client.requests.lock().unwrap();
|
||||||
|
assert_eq!(requests.len(), 2);
|
||||||
|
assert_eq!(requests[0].method, requests[1].method);
|
||||||
|
assert_eq!(requests[0].path, requests[1].path);
|
||||||
|
assert_eq!(requests[0].body, requests[1].body);
|
||||||
|
if view == "post" {
|
||||||
|
assert_eq!(requests[0].method, WorkspaceRequestMethod::Get);
|
||||||
|
assert_eq!(
|
||||||
|
requests[0].path,
|
||||||
|
"/api/w/workspace/tickets/T-739/posts/persisted%3Aid%2Fwith%20unicode%E7%95%8C"
|
||||||
|
);
|
||||||
|
assert!(requests[0].body.is_none());
|
||||||
|
} else {
|
||||||
|
let body: Value =
|
||||||
|
serde_json::from_str(requests[0].body.as_deref().unwrap()).unwrap();
|
||||||
|
assert!(body.get("view").is_none());
|
||||||
|
assert!(body.get("event_ref").is_none());
|
||||||
|
if view == "summary" {
|
||||||
|
assert_eq!(body["event_limit"], 1);
|
||||||
|
}
|
||||||
|
if view == "history" {
|
||||||
|
assert_eq!(body["event_cursor"], "opaque-before");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn show_rejects_missing_or_malformed_targets_instead_of_reporting_unassigned() {
|
async fn show_rejects_missing_or_malformed_targets_instead_of_reporting_unassigned() {
|
||||||
for malformed in [
|
for malformed in [
|
||||||
|
|||||||
@@ -8,6 +8,7 @@
|
|||||||
//! the regular registry decides Tool-name collisions.
|
//! the regular registry decides Tool-name collisions.
|
||||||
|
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
|
use std::io;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
@@ -25,6 +26,7 @@ use mcp::stdio::{
|
|||||||
};
|
};
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
use serde_json::{Map, Value};
|
use serde_json::{Map, Value};
|
||||||
|
use tokio::sync::Mutex;
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
FeatureDescriptor, FeatureDiagnostic, FeatureInstallContext, FeatureInstallError,
|
FeatureDescriptor, FeatureDiagnostic, FeatureInstallContext, FeatureInstallError,
|
||||||
@@ -52,8 +54,88 @@ const MAX_LIST_ITEMS: usize = 128;
|
|||||||
const MAX_RESOURCE_CONTENTS: usize = 16;
|
const MAX_RESOURCE_CONTENTS: usize = 16;
|
||||||
const MAX_PROMPT_MESSAGES: usize = 32;
|
const MAX_PROMPT_MESSAGES: usize = 32;
|
||||||
|
|
||||||
|
/// Owns discovery clients until process reaping is confirmed. Retain a clone in
|
||||||
|
/// startup cleanup *before* awaiting discovery. Stop/cancel discovery before
|
||||||
|
/// calling `shutdown`; the shared lock serializes discovery and cleanup, not
|
||||||
|
/// cancellation of an in-flight discovery task.
|
||||||
|
///
|
||||||
|
/// Error or cancellation leaves the actual clients here for another shutdown
|
||||||
|
/// attempt. Dropping the final owner only provides best-effort process killing.
|
||||||
|
#[derive(Clone, Default)]
|
||||||
|
pub struct McpDiscoveryCleanup {
|
||||||
|
clients: Arc<Mutex<Vec<McpStdioClient>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl McpDiscoveryCleanup {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self::default()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Try every retained client once, removing only confirmed shutdowns. An
|
||||||
|
/// error is not a cleanup receipt; retry this same owner, not discovery.
|
||||||
|
pub async fn shutdown(&self) -> io::Result<()> {
|
||||||
|
let mut clients = self.clients.lock().await;
|
||||||
|
let mut first_error = None;
|
||||||
|
let mut index = 0;
|
||||||
|
while index < clients.len() {
|
||||||
|
match clients[index].shutdown().await {
|
||||||
|
Ok(_) => {
|
||||||
|
clients.remove(index);
|
||||||
|
}
|
||||||
|
Err(err) => {
|
||||||
|
first_error.get_or_insert_with(|| discovery_shutdown_error(&err));
|
||||||
|
index += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
match first_error {
|
||||||
|
Some(err) => Err(err),
|
||||||
|
None => Ok(()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn discovery_shutdown_error(err: &McpClientError) -> io::Error {
|
||||||
|
io::Error::other(bounded_diagnostic(format!(
|
||||||
|
"MCP discovery shutdown unconfirmed; client retained for cleanup retry: {err}"
|
||||||
|
)))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Restore preparation must fail if discovery cannot confirm process shutdown.
|
||||||
|
/// The caller retains `cleanup` before this future is polled, including during
|
||||||
|
/// initialization. Ordinary discovery failures remain isolated contributions
|
||||||
|
/// only after their client has been reaped.
|
||||||
|
pub async fn discover_stdio_tool_feature_with_cleanup(
|
||||||
|
config: &McpConfig,
|
||||||
|
workspace_root: &Path,
|
||||||
|
cleanup: &McpDiscoveryCleanup,
|
||||||
|
) -> Result<Option<McpStdioToolFeature>, io::Error> {
|
||||||
|
if config.stdio_servers.is_empty() {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
let mut feature = McpStdioToolFeature::new();
|
||||||
|
for server in &config.stdio_servers {
|
||||||
|
match resolve_stdio_server(server, workspace_root, None) {
|
||||||
|
Ok(spec) => {
|
||||||
|
feature.add_contribution(
|
||||||
|
discover_server_tools_with_cleanup(spec, cleanup, McpStdioLimits::default())
|
||||||
|
.await?,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Err(err) => {
|
||||||
|
feature.add_diagnostic(FeatureDiagnostic::error(bounded_diagnostic(format!(
|
||||||
|
"failed to resolve MCP stdio server `{}`: {err}",
|
||||||
|
server.name
|
||||||
|
))))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(Some(feature))
|
||||||
|
}
|
||||||
|
|
||||||
/// Discover enabled MCP stdio server tools and return a single feature module
|
/// Discover enabled MCP stdio server tools and return a single feature module
|
||||||
/// containing startup contributions for normal ToolRegistry installation.
|
/// containing startup contributions for normal ToolRegistry installation.
|
||||||
|
/// Callers needing retryable startup cleanup must use the cleanup-aware API.
|
||||||
pub async fn discover_stdio_tool_feature(
|
pub async fn discover_stdio_tool_feature(
|
||||||
config: &McpConfig,
|
config: &McpConfig,
|
||||||
workspace_root: &Path,
|
workspace_root: &Path,
|
||||||
@@ -81,21 +163,66 @@ pub async fn discover_stdio_tool_feature(
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn discover_server_tools(spec: McpStdioServerSpec) -> ProtocolProviderContribution {
|
async fn discover_server_tools(spec: McpStdioServerSpec) -> ProtocolProviderContribution {
|
||||||
|
let declaration = provider_declaration(&spec.name, None);
|
||||||
|
let cleanup = McpDiscoveryCleanup::new();
|
||||||
|
match discover_server_tools_with_cleanup(spec, &cleanup, McpStdioLimits::default()).await {
|
||||||
|
Ok(contribution) => contribution,
|
||||||
|
Err(err) => {
|
||||||
|
ProtocolProviderContribution::failed(declaration, bounded_diagnostic(err.to_string()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn discover_server_tools_with_cleanup(
|
||||||
|
spec: McpStdioServerSpec,
|
||||||
|
cleanup: &McpDiscoveryCleanup,
|
||||||
|
limits: McpStdioLimits,
|
||||||
|
) -> io::Result<ProtocolProviderContribution> {
|
||||||
|
let declaration = provider_declaration(&spec.name, None);
|
||||||
|
let mut clients = cleanup.clients.lock().await;
|
||||||
|
if !clients.is_empty() {
|
||||||
|
return Err(io::Error::other(
|
||||||
|
"MCP discovery cleanup is pending; shut down the retained owner before retrying discovery",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let client = match McpStdioClient::spawn_uninitialized(spec.clone(), limits) {
|
||||||
|
Ok(client) => client,
|
||||||
|
Err(err) => {
|
||||||
|
// The synchronous spawn API only errors before a process exists;
|
||||||
|
// pipe/handshake failures retain a client and use the shutdown path.
|
||||||
|
return Ok(ProtocolProviderContribution::failed(
|
||||||
|
declaration,
|
||||||
|
bounded_diagnostic(err.to_string()),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
// Spawn is synchronous: the owner holds the client before the first await
|
||||||
|
// that can leave a running process on initialization failure/cancellation.
|
||||||
|
clients.push(client);
|
||||||
|
let client = clients.last_mut().expect("retained discovery client");
|
||||||
|
let contribution = match client.initialize_with_timeout().await {
|
||||||
|
Ok(()) => discover_connected_server_tools(spec, client).await,
|
||||||
|
Err(err) => {
|
||||||
|
ProtocolProviderContribution::failed(declaration, bounded_diagnostic(err.to_string()))
|
||||||
|
}
|
||||||
|
};
|
||||||
|
client
|
||||||
|
.shutdown()
|
||||||
|
.await
|
||||||
|
.map_err(|err| discovery_shutdown_error(&err))?;
|
||||||
|
drop(clients.pop());
|
||||||
|
Ok(contribution)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn discover_connected_server_tools(
|
||||||
|
spec: McpStdioServerSpec,
|
||||||
|
client: &mut McpStdioClient,
|
||||||
|
) -> ProtocolProviderContribution {
|
||||||
let declaration = provider_declaration(&spec.name, None);
|
let declaration = provider_declaration(&spec.name, None);
|
||||||
let mut contribution = ProtocolProviderContribution::ready(declaration.clone());
|
let mut contribution = ProtocolProviderContribution::ready(declaration.clone());
|
||||||
let server_namespace = sanitize_segment(&spec.name);
|
let server_namespace = sanitize_segment(&spec.name);
|
||||||
let execution_spec = spec.clone();
|
let execution_spec = spec.clone();
|
||||||
|
|
||||||
let mut client = match McpStdioClient::connect(spec, McpStdioLimits::default()).await {
|
|
||||||
Ok(client) => client,
|
|
||||||
Err(err) => {
|
|
||||||
return ProtocolProviderContribution::failed(
|
|
||||||
declaration,
|
|
||||||
bounded_diagnostic(err.to_string()),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
let (server_version, has_tools, has_resources, has_prompts) = client
|
let (server_version, has_tools, has_resources, has_prompts) = client
|
||||||
.initialize_result()
|
.initialize_result()
|
||||||
.map(|result| {
|
.map(|result| {
|
||||||
@@ -133,18 +260,10 @@ async fn discover_server_tools(spec: McpStdioServerSpec) -> ProtocolProviderCont
|
|||||||
{
|
{
|
||||||
Ok(list) => list,
|
Ok(list) => list,
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
let mut failed = ProtocolProviderContribution::failed(
|
return ProtocolProviderContribution::failed(
|
||||||
declaration,
|
declaration,
|
||||||
bounded_diagnostic(err.to_string()),
|
bounded_diagnostic(err.to_string()),
|
||||||
);
|
);
|
||||||
if let Err(shutdown_err) = client.shutdown().await {
|
|
||||||
failed = failed.with_diagnostic(FeatureDiagnostic::warning(
|
|
||||||
bounded_diagnostic(format!(
|
|
||||||
"MCP server shutdown after discovery failure failed: {shutdown_err}"
|
|
||||||
)),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
return failed;
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let list_changes = client.snapshot_list_changes().await;
|
let list_changes = client.snapshot_list_changes().await;
|
||||||
@@ -166,20 +285,12 @@ async fn discover_server_tools(spec: McpStdioServerSpec) -> ProtocolProviderCont
|
|||||||
{
|
{
|
||||||
Ok(list) => list,
|
Ok(list) => list,
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
let mut failed = ProtocolProviderContribution::failed(
|
return ProtocolProviderContribution::failed(
|
||||||
declaration,
|
declaration,
|
||||||
bounded_diagnostic(format!(
|
bounded_diagnostic(format!(
|
||||||
"MCP server sent notifications/tools/list_changed during tool discovery and refresh failed: {err}"
|
"MCP server sent notifications/tools/list_changed during tool discovery and refresh failed: {err}"
|
||||||
)),
|
)),
|
||||||
);
|
);
|
||||||
if let Err(shutdown_err) = client.shutdown().await {
|
|
||||||
failed = failed.with_diagnostic(FeatureDiagnostic::warning(
|
|
||||||
bounded_diagnostic(format!(
|
|
||||||
"MCP server shutdown after discovery refresh failure failed: {shutdown_err}"
|
|
||||||
)),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
return failed;
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let refresh_changes = client.snapshot_list_changes().await;
|
let refresh_changes = client.snapshot_list_changes().await;
|
||||||
@@ -197,13 +308,6 @@ async fn discover_server_tools(spec: McpStdioServerSpec) -> ProtocolProviderCont
|
|||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
};
|
};
|
||||||
let shutdown_result = client.shutdown().await;
|
|
||||||
if let Err(err) = shutdown_result {
|
|
||||||
contribution =
|
|
||||||
contribution.with_diagnostic(FeatureDiagnostic::warning(bounded_diagnostic(format!(
|
|
||||||
"MCP server shutdown after capability discovery failed: {err}"
|
|
||||||
))));
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(list) = list {
|
if let Some(list) = list {
|
||||||
contribution = normalize_listed_tools(
|
contribution = normalize_listed_tools(
|
||||||
@@ -1819,6 +1923,140 @@ read exit_notification || true
|
|||||||
McpStdioServerSpec::new("shell-mock", "/bin/sh").args(["-c".to_string(), script])
|
McpStdioServerSpec::new("shell-mock", "/bin/sh").args(["-c".to_string(), script])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
fn discovery_config(spec: McpStdioServerSpec) -> McpConfig {
|
||||||
|
McpConfig {
|
||||||
|
stdio_servers: vec![manifest::McpStdioServerConfig {
|
||||||
|
name: spec.name,
|
||||||
|
command: spec.command,
|
||||||
|
args: spec.args,
|
||||||
|
cwd: None,
|
||||||
|
env: Default::default(),
|
||||||
|
}],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
#[tokio::test]
|
||||||
|
async fn cancelled_discovery_retains_initializing_client_in_cleanup_clone() {
|
||||||
|
use std::future::{Future, poll_fn};
|
||||||
|
use std::task::Poll;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
let config = discovery_config(
|
||||||
|
McpStdioServerSpec::new("initializing", "/bin/sh")
|
||||||
|
.args(["-c", "while read line; do :; done"]),
|
||||||
|
);
|
||||||
|
let cleanup = McpDiscoveryCleanup::new();
|
||||||
|
let retained = cleanup.clone();
|
||||||
|
let mut discovery = Box::pin(discover_stdio_tool_feature_with_cleanup(
|
||||||
|
&config,
|
||||||
|
Path::new("."),
|
||||||
|
&cleanup,
|
||||||
|
));
|
||||||
|
poll_fn(|cx| {
|
||||||
|
assert!(discovery.as_mut().poll(cx).is_pending());
|
||||||
|
Poll::Ready(())
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
drop(discovery);
|
||||||
|
drop(cleanup);
|
||||||
|
assert_eq!(retained.clients.lock().await.len(), 1);
|
||||||
|
let retry_error =
|
||||||
|
discover_stdio_tool_feature_with_cleanup(&config, Path::new("."), &retained)
|
||||||
|
.await
|
||||||
|
.err()
|
||||||
|
.expect("discovery must not replace an unreaped client");
|
||||||
|
assert!(retry_error.to_string().contains("cleanup is pending"));
|
||||||
|
assert_eq!(retained.clients.lock().await.len(), 1);
|
||||||
|
tokio::time::timeout(Duration::from_secs(3), retained.shutdown())
|
||||||
|
.await
|
||||||
|
.expect("cleanup bounded")
|
||||||
|
.expect("initializing process reaped");
|
||||||
|
assert!(retained.clients.lock().await.is_empty());
|
||||||
|
retained
|
||||||
|
.shutdown()
|
||||||
|
.await
|
||||||
|
.expect("confirmed cleanup repeatable");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
#[tokio::test]
|
||||||
|
async fn cancelled_owner_shutdown_keeps_client_for_retry() {
|
||||||
|
use std::future::{Future, poll_fn};
|
||||||
|
use std::task::Poll;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
let cleanup = McpDiscoveryCleanup::new();
|
||||||
|
cleanup.clients.lock().await.push(
|
||||||
|
McpStdioClient::spawn_uninitialized(
|
||||||
|
McpStdioServerSpec::new("shutdown-pending", "/bin/sh")
|
||||||
|
.args(["-c", "while :; do :; done"]),
|
||||||
|
McpStdioLimits {
|
||||||
|
shutdown_timeout: Duration::from_millis(50),
|
||||||
|
kill_timeout: Duration::from_millis(100),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.expect("local process spawned"),
|
||||||
|
);
|
||||||
|
let mut shutdown = Box::pin(cleanup.shutdown());
|
||||||
|
poll_fn(|cx| {
|
||||||
|
assert!(shutdown.as_mut().poll(cx).is_pending());
|
||||||
|
Poll::Ready(())
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
drop(shutdown);
|
||||||
|
assert_eq!(cleanup.clients.lock().await.len(), 1);
|
||||||
|
tokio::time::timeout(Duration::from_secs(2), cleanup.shutdown())
|
||||||
|
.await
|
||||||
|
.expect("retry bounded")
|
||||||
|
.expect("retained client shutdown");
|
||||||
|
assert!(cleanup.clients.lock().await.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
#[tokio::test]
|
||||||
|
async fn cleanup_aware_discovery_reaps_success_and_initialize_failure() {
|
||||||
|
let cleanup = McpDiscoveryCleanup::new();
|
||||||
|
let config = discovery_config(shell_capability_server(r#"{"resources":{}}"#));
|
||||||
|
let feature = discover_stdio_tool_feature_with_cleanup(&config, Path::new("."), &cleanup)
|
||||||
|
.await
|
||||||
|
.expect("discovery cleanup confirmed")
|
||||||
|
.expect("configured feature");
|
||||||
|
assert!(
|
||||||
|
feature.contributions[0]
|
||||||
|
.tools
|
||||||
|
.iter()
|
||||||
|
.any(|tool| { tool.name() == "Mcp_shell_mock_resources_read" })
|
||||||
|
);
|
||||||
|
assert!(cleanup.clients.lock().await.is_empty());
|
||||||
|
|
||||||
|
let spec = McpStdioServerSpec::new("init-rejected", "/bin/sh").args([
|
||||||
|
"-c",
|
||||||
|
r#"read init || exit 1
|
||||||
|
printf '%s\n' '{"jsonrpc":"2.0","id":1,"error":{"code":-32000,"message":"initialization rejected"}}'
|
||||||
|
while read line; do :; done
|
||||||
|
"#,
|
||||||
|
]);
|
||||||
|
let config = discovery_config(spec);
|
||||||
|
let feature = discover_stdio_tool_feature_with_cleanup(&config, Path::new("."), &cleanup)
|
||||||
|
.await
|
||||||
|
.expect("failure isolated only after cleanup")
|
||||||
|
.expect("failed contribution preserved");
|
||||||
|
assert!(feature.contributions[0].tools.is_empty());
|
||||||
|
assert!(
|
||||||
|
feature.contributions[0]
|
||||||
|
.diagnostics
|
||||||
|
.iter()
|
||||||
|
.any(|diagnostic| {
|
||||||
|
diagnostic.severity == FeatureDiagnosticSeverity::Error
|
||||||
|
&& diagnostic.message.contains("initialization rejected")
|
||||||
|
})
|
||||||
|
);
|
||||||
|
assert!(cleanup.clients.lock().await.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn mcp_resource_prompt_operation_tools_are_namespaced_and_untrusted() {
|
fn mcp_resource_prompt_operation_tools_are_namespaced_and_untrusted() {
|
||||||
let declaration = provider_declaration("demo", Some("1.0.0"));
|
let declaration = provider_declaration("demo", Some("1.0.0"));
|
||||||
|
|||||||
@@ -19,6 +19,9 @@ pub mod model_client;
|
|||||||
mod paste_artifact_tool;
|
mod paste_artifact_tool;
|
||||||
pub mod profile_manifest;
|
pub mod profile_manifest;
|
||||||
pub mod prompt;
|
pub mod prompt;
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "../../../tests/support/restore_environment.rs"]
|
||||||
|
mod restore_test_environment;
|
||||||
pub mod runtime;
|
pub mod runtime;
|
||||||
pub mod segment_log_sink;
|
pub mod segment_log_sink;
|
||||||
mod session_capture;
|
mod session_capture;
|
||||||
@@ -69,8 +72,9 @@ pub use session_history::{
|
|||||||
};
|
};
|
||||||
pub use shared_state::WorkerSharedState;
|
pub use shared_state::WorkerSharedState;
|
||||||
pub use worker::{
|
pub use worker::{
|
||||||
LocalWorkingDirectory, SubjektivSessionAttributionLifecycle, Worker, WorkerError,
|
LocalWorkingDirectory, ReviewerChildWorkspaceClient, ReviewerContext,
|
||||||
WorkerFilesystemAuthority, WorkerRunResult, WorkerWorkspaceContext, WorkspaceBinaryRequest,
|
SubjektivSessionAttributionLifecycle, Worker, WorkerError, WorkerFilesystemAuthority,
|
||||||
|
WorkerRunResult, WorkerStartupFeatureCleanup, WorkerWorkspaceContext, WorkspaceBinaryRequest,
|
||||||
WorkspaceBinaryResponse, WorkspaceClient, WorkspaceClientError, WorkspaceId, WorkspaceIdError,
|
WorkspaceBinaryResponse, WorkspaceClient, WorkspaceClientError, WorkspaceId, WorkspaceIdError,
|
||||||
WorkspacePromptCatalogResolution, WorkspaceRequest, WorkspaceRequestMethod, WorkspaceResponse,
|
WorkspacePromptCatalogResolution, WorkspaceRequest, WorkspaceRequestMethod, WorkspaceResponse,
|
||||||
WorkspaceServerOperation, WorkspaceWorkerDiscoveryRequest, apply_worker_manifest,
|
WorkspaceServerOperation, WorkspaceWorkerDiscoveryRequest, apply_worker_manifest,
|
||||||
|
|||||||
@@ -823,6 +823,20 @@ mod tests {
|
|||||||
assert!(prompt.contains("SubmitBackendJobResult"));
|
assert!(prompt.contains("SubmitBackendJobResult"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn flow_verifier_reuses_inspected_evidence_without_claiming_live_read_authority() {
|
||||||
|
let prompt = PromptCatalog::builtins_only()
|
||||||
|
.unwrap()
|
||||||
|
.flow_verifier_system()
|
||||||
|
.unwrap();
|
||||||
|
assert!(prompt.contains("stable entry references"));
|
||||||
|
assert!(prompt.contains("unchanged tree under equivalent conditions"));
|
||||||
|
assert!(prompt.contains("never transfers an old approval or authority"));
|
||||||
|
assert!(prompt.contains("cannot perform live MR or other-session reads"));
|
||||||
|
assert!(prompt.contains("return `indeterminate`"));
|
||||||
|
assert!(prompt.contains("FinishFlowVerification"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn merge_request_prompts_keep_recording_and_reviewer_contracts() {
|
fn merge_request_prompts_keep_recording_and_reviewer_contracts() {
|
||||||
let catalog = PromptCatalog::builtins_only().unwrap();
|
let catalog = PromptCatalog::builtins_only().unwrap();
|
||||||
@@ -834,7 +848,10 @@ mod tests {
|
|||||||
assert!(common.contains("target-only movement preserves approval"));
|
assert!(common.contains("target-only movement preserves approval"));
|
||||||
assert!(reviewer.contains("complete linked MR/result snapshot"));
|
assert!(reviewer.contains("complete linked MR/result snapshot"));
|
||||||
assert!(reviewer.contains("Submit `ReviewMergeRequest` once"));
|
assert!(reviewer.contains("Submit `ReviewMergeRequest` once"));
|
||||||
assert!(reviewer.contains("without modifying implementation files"));
|
assert!(reviewer.contains("Bash"));
|
||||||
|
assert!(reviewer.contains("Git"));
|
||||||
|
assert!(!reviewer.contains("without modifying"));
|
||||||
|
assert!(!reviewer.contains("read-only"));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -21,8 +21,9 @@ pub use error::ScopeLockError;
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) use lifecycle::install_top_level_at_for_test;
|
pub(crate) use lifecycle::install_top_level_at_for_test;
|
||||||
pub use lifecycle::{
|
pub use lifecycle::{
|
||||||
ScopeAllocationGuard, SegmentActivationCommitError, SegmentActivationGuard, SegmentLockInfo,
|
ScopeAllocationCleanup, ScopeAllocationGuard, SegmentActivationCommitError,
|
||||||
adopt_allocation, install_top_level, install_top_level_with_deny, lookup_segment,
|
SegmentActivationGuard, SegmentLockInfo, adopt_allocation, install_top_level,
|
||||||
|
install_top_level_with_deny, lookup_segment,
|
||||||
};
|
};
|
||||||
pub use mutate::{
|
pub use mutate::{
|
||||||
delegate_scope, reclaim_delegated_scope, reclaim_stale, reclaim_stale_with, register_worker,
|
delegate_scope, reclaim_delegated_scope, reclaim_stale, reclaim_stale_with, register_worker,
|
||||||
|
|||||||
@@ -11,6 +11,14 @@ use session_store::SegmentId;
|
|||||||
pub enum ScopeLockError {
|
pub enum ScopeLockError {
|
||||||
#[error("I/O error on workers.json: {0}")]
|
#[error("I/O error on workers.json: {0}")]
|
||||||
Io(#[from] io::Error),
|
Io(#[from] io::Error),
|
||||||
|
#[error("worker `{worker_name}` allocation still has a live guard")]
|
||||||
|
AllocationGuardAttached { worker_name: String },
|
||||||
|
#[error("allocation acquisition was published but its save failed: {source}")]
|
||||||
|
AcquisitionSaveFailed {
|
||||||
|
#[source]
|
||||||
|
source: io::Error,
|
||||||
|
cleanup: super::lifecycle::ScopeAllocationCleanup,
|
||||||
|
},
|
||||||
#[error("worker `{0}` is already allocated")]
|
#[error("worker `{0}` is already allocated")]
|
||||||
DuplicateWorkerName(String),
|
DuplicateWorkerName(String),
|
||||||
#[error("requested scope `{}` conflicts with worker allocation `{competitor}` rule `{}`", .rule.target.display(), .competitor_rule.target.display())]
|
#[error("requested scope `{}` conflicts with worker allocation `{competitor}` rule `{}`", .rule.target.display(), .competitor_rule.target.display())]
|
||||||
@@ -46,3 +54,15 @@ pub enum ScopeLockError {
|
|||||||
actual: Option<SegmentId>,
|
actual: Option<SegmentId>,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl ScopeLockError {
|
||||||
|
/// Retain the release authority for an owned acquisition whose table rename
|
||||||
|
/// succeeded before save returned an error. Refusals and unpublished saves
|
||||||
|
/// do not confer ownership of any preexisting allocation.
|
||||||
|
pub fn allocation_cleanup_handle(&self) -> Option<super::lifecycle::ScopeAllocationCleanup> {
|
||||||
|
match self {
|
||||||
|
Self::AcquisitionSaveFailed { cleanup, .. } => Some(cleanup.clone()),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
//! up on drop.
|
//! up on drop.
|
||||||
|
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
||||||
use manifest::ScopeRule;
|
use manifest::ScopeRule;
|
||||||
use session_store::SegmentId;
|
use session_store::SegmentId;
|
||||||
@@ -11,13 +12,82 @@ use super::error::ScopeLockError;
|
|||||||
use super::mutate::release_worker;
|
use super::mutate::release_worker;
|
||||||
use super::table::{LockFileGuard, default_allocation_path};
|
use super::table::{LockFileGuard, default_allocation_path};
|
||||||
|
|
||||||
/// Owned allocation: on drop, opens the lock file and releases this
|
/// Owned allocation shared with any retained cleanup handles. Only the final
|
||||||
/// Worker's entry. The guard keeps only the name + lock-file path; it
|
/// owner's drop attempts best-effort release; no `flock` is held for its lifetime.
|
||||||
/// does not hold the `flock` for the Worker's lifetime.
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub struct ScopeAllocationGuard {
|
pub struct ScopeAllocationGuard {
|
||||||
|
owner: Arc<ScopeAllocationOwner>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Retains an allocation's release authority even after its Worker is consumed
|
||||||
|
/// or dropped. A successful controller transfer should drop this handle without
|
||||||
|
/// explicitly releasing the live Worker's allocation.
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct ScopeAllocationCleanup {
|
||||||
|
owner: Arc<ScopeAllocationOwner>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
struct ScopeAllocationOwner {
|
||||||
worker_name: String,
|
worker_name: String,
|
||||||
lock_path: PathBuf,
|
lock_path: PathBuf,
|
||||||
|
state: Mutex<ScopeAllocationOwnerState>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
struct ScopeAllocationOwnerState {
|
||||||
|
guard_attached: bool,
|
||||||
|
released: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ScopeAllocationOwner {
|
||||||
|
fn release(&self) -> Result<(), ScopeLockError> {
|
||||||
|
let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner());
|
||||||
|
if state.guard_attached {
|
||||||
|
return Err(ScopeLockError::AllocationGuardAttached {
|
||||||
|
worker_name: self.worker_name.clone(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if state.released {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let mut guard = LockFileGuard::open(&self.lock_path)?;
|
||||||
|
if guard.data().find(&self.worker_name).is_some() {
|
||||||
|
release_worker(&mut guard, &self.worker_name)?;
|
||||||
|
} else {
|
||||||
|
// A previous save may have removed the entry but failed after rename.
|
||||||
|
// Complete the durable save before reporting release as confirmed.
|
||||||
|
guard.save()?;
|
||||||
|
}
|
||||||
|
state.released = true;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for ScopeAllocationOwner {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
let _ = self.release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for ScopeAllocationGuard {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
self.owner
|
||||||
|
.state
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|error| error.into_inner())
|
||||||
|
.guard_attached = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ScopeAllocationCleanup {
|
||||||
|
/// Release only after the factory has returned an error and no live
|
||||||
|
/// Controller can still use this allocation. An attached guard rejects
|
||||||
|
/// release. Success is idempotent across all clones; failure retains the
|
||||||
|
/// authority for an explicit retry.
|
||||||
|
pub fn release(&self) -> Result<(), ScopeLockError> {
|
||||||
|
self.owner.release()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Exclusive authority to activate a replacement Segment for one allocated Worker.
|
/// Exclusive authority to activate a replacement Segment for one allocated Worker.
|
||||||
@@ -56,12 +126,58 @@ impl SegmentActivationCommitError {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl ScopeAllocationGuard {
|
impl ScopeAllocationGuard {
|
||||||
|
fn new(worker_name: String, lock_path: PathBuf) -> Self {
|
||||||
|
Self {
|
||||||
|
owner: Arc::new(ScopeAllocationOwner {
|
||||||
|
worker_name,
|
||||||
|
lock_path,
|
||||||
|
state: Mutex::new(ScopeAllocationOwnerState {
|
||||||
|
guard_attached: true,
|
||||||
|
released: false,
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn commit_acquisition(self, table: &mut LockFileGuard) -> Result<Self, ScopeLockError> {
|
||||||
|
let mut published = false;
|
||||||
|
match table.save_with_publication(&mut published) {
|
||||||
|
Ok(()) => Ok(self),
|
||||||
|
Err(source) => {
|
||||||
|
let cleanup = if published {
|
||||||
|
Some(self.cleanup_handle())
|
||||||
|
} else {
|
||||||
|
// A staged entry is not ours until rename. Disarm before
|
||||||
|
// detaching so Drop cannot remove a preexisting allocation
|
||||||
|
// (or reopen the table lock still held by this caller).
|
||||||
|
self.owner
|
||||||
|
.state
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|error| error.into_inner())
|
||||||
|
.released = true;
|
||||||
|
None
|
||||||
|
};
|
||||||
|
drop(self);
|
||||||
|
match cleanup {
|
||||||
|
Some(cleanup) => Err(ScopeLockError::AcquisitionSaveFailed { source, cleanup }),
|
||||||
|
None => Err(ScopeLockError::Io(source)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub fn worker_name(&self) -> &str {
|
pub fn worker_name(&self) -> &str {
|
||||||
&self.worker_name
|
&self.owner.worker_name
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn lock_path(&self) -> &Path {
|
pub fn lock_path(&self) -> &Path {
|
||||||
&self.lock_path
|
&self.owner.lock_path
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn cleanup_handle(&self) -> ScopeAllocationCleanup {
|
||||||
|
ScopeAllocationCleanup {
|
||||||
|
owner: Arc::clone(&self.owner),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Lock the machine-wide table and validate a Segment replacement before
|
/// Lock the machine-wide table and validate a Segment replacement before
|
||||||
@@ -71,24 +187,25 @@ impl ScopeAllocationGuard {
|
|||||||
expected: SegmentId,
|
expected: SegmentId,
|
||||||
replacement: SegmentId,
|
replacement: SegmentId,
|
||||||
) -> Result<SegmentActivationGuard, ScopeLockError> {
|
) -> Result<SegmentActivationGuard, ScopeLockError> {
|
||||||
let guard = LockFileGuard::open(&self.lock_path)?;
|
let guard = LockFileGuard::open(self.lock_path())?;
|
||||||
let actual = guard
|
let actual = guard
|
||||||
.data()
|
.data()
|
||||||
.allocations
|
.allocations
|
||||||
.iter()
|
.iter()
|
||||||
.find(|allocation| allocation.worker_name == self.worker_name)
|
.find(|allocation| allocation.worker_name == self.worker_name())
|
||||||
.and_then(|allocation| allocation.segment_id);
|
.and_then(|allocation| allocation.segment_id);
|
||||||
|
|
||||||
if actual != Some(expected) {
|
if actual != Some(expected) {
|
||||||
return Err(ScopeLockError::SegmentChanged {
|
return Err(ScopeLockError::SegmentChanged {
|
||||||
worker_name: self.worker_name.clone(),
|
worker_name: self.worker_name().to_owned(),
|
||||||
expected,
|
expected,
|
||||||
actual,
|
actual,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(existing) = guard.data().allocations.iter().find(|allocation| {
|
if let Some(existing) = guard.data().allocations.iter().find(|allocation| {
|
||||||
allocation.segment_id == Some(replacement) && allocation.worker_name != self.worker_name
|
allocation.segment_id == Some(replacement)
|
||||||
|
&& allocation.worker_name != self.worker_name()
|
||||||
}) {
|
}) {
|
||||||
return Err(ScopeLockError::SegmentConflict {
|
return Err(ScopeLockError::SegmentConflict {
|
||||||
segment_id: replacement,
|
segment_id: replacement,
|
||||||
@@ -99,7 +216,7 @@ impl ScopeAllocationGuard {
|
|||||||
|
|
||||||
Ok(SegmentActivationGuard {
|
Ok(SegmentActivationGuard {
|
||||||
guard,
|
guard,
|
||||||
worker_name: self.worker_name.clone(),
|
worker_name: self.worker_name().to_owned(),
|
||||||
expected,
|
expected,
|
||||||
replacement,
|
replacement,
|
||||||
})
|
})
|
||||||
@@ -145,14 +262,6 @@ impl SegmentActivationGuard {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Drop for ScopeAllocationGuard {
|
|
||||||
fn drop(&mut self) {
|
|
||||||
if let Ok(mut guard) = LockFileGuard::open(&self.lock_path) {
|
|
||||||
let _ = release_worker(&mut guard, &self.worker_name);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Open the default lock file, register a top-level Worker, and return a
|
/// Open the default lock file, register a top-level Worker, and return a
|
||||||
/// guard that will release the allocation on drop.
|
/// guard that will release the allocation on drop.
|
||||||
pub fn install_top_level(
|
pub fn install_top_level(
|
||||||
@@ -185,7 +294,7 @@ pub fn install_top_level_with_deny(
|
|||||||
) -> Result<ScopeAllocationGuard, ScopeLockError> {
|
) -> Result<ScopeAllocationGuard, ScopeLockError> {
|
||||||
let lock_path = default_allocation_path()?;
|
let lock_path = default_allocation_path()?;
|
||||||
let mut guard = LockFileGuard::open(&lock_path)?;
|
let mut guard = LockFileGuard::open(&lock_path)?;
|
||||||
super::mutate::register_worker_with_deny(
|
super::mutate::prepare_worker_with_deny(
|
||||||
&mut guard,
|
&mut guard,
|
||||||
worker_name.clone(),
|
worker_name.clone(),
|
||||||
pid,
|
pid,
|
||||||
@@ -194,10 +303,7 @@ pub fn install_top_level_with_deny(
|
|||||||
scope_deny,
|
scope_deny,
|
||||||
segment_id,
|
segment_id,
|
||||||
)?;
|
)?;
|
||||||
Ok(ScopeAllocationGuard {
|
ScopeAllocationGuard::new(worker_name, lock_path).commit_acquisition(&mut guard)
|
||||||
worker_name,
|
|
||||||
lock_path,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
@@ -210,7 +316,7 @@ pub(crate) fn install_top_level_at_for_test(
|
|||||||
segment_id: SegmentId,
|
segment_id: SegmentId,
|
||||||
) -> Result<ScopeAllocationGuard, ScopeLockError> {
|
) -> Result<ScopeAllocationGuard, ScopeLockError> {
|
||||||
let mut guard = LockFileGuard::open(&lock_path)?;
|
let mut guard = LockFileGuard::open(&lock_path)?;
|
||||||
super::mutate::register_worker_with_deny(
|
super::mutate::prepare_worker_with_deny(
|
||||||
&mut guard,
|
&mut guard,
|
||||||
worker_name.clone(),
|
worker_name.clone(),
|
||||||
pid,
|
pid,
|
||||||
@@ -219,10 +325,7 @@ pub(crate) fn install_top_level_at_for_test(
|
|||||||
Vec::new(),
|
Vec::new(),
|
||||||
segment_id,
|
segment_id,
|
||||||
)?;
|
)?;
|
||||||
Ok(ScopeAllocationGuard {
|
ScopeAllocationGuard::new(worker_name, lock_path).commit_acquisition(&mut guard)
|
||||||
worker_name,
|
|
||||||
lock_path,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Take ownership of an existing allocation that was pre-registered by
|
/// Take ownership of an existing allocation that was pre-registered by
|
||||||
@@ -240,6 +343,15 @@ pub fn adopt_allocation(
|
|||||||
segment_id: SegmentId,
|
segment_id: SegmentId,
|
||||||
) -> Result<ScopeAllocationGuard, ScopeLockError> {
|
) -> Result<ScopeAllocationGuard, ScopeLockError> {
|
||||||
let lock_path = default_allocation_path()?;
|
let lock_path = default_allocation_path()?;
|
||||||
|
adopt_allocation_at(lock_path, worker_name, new_pid, segment_id)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn adopt_allocation_at(
|
||||||
|
lock_path: PathBuf,
|
||||||
|
worker_name: String,
|
||||||
|
new_pid: u32,
|
||||||
|
segment_id: SegmentId,
|
||||||
|
) -> Result<ScopeAllocationGuard, ScopeLockError> {
|
||||||
let mut guard = LockFileGuard::open(&lock_path)?;
|
let mut guard = LockFileGuard::open(&lock_path)?;
|
||||||
let alloc = guard
|
let alloc = guard
|
||||||
.data_mut()
|
.data_mut()
|
||||||
@@ -247,11 +359,7 @@ pub fn adopt_allocation(
|
|||||||
.ok_or_else(|| ScopeLockError::UnknownWorker(worker_name.clone()))?;
|
.ok_or_else(|| ScopeLockError::UnknownWorker(worker_name.clone()))?;
|
||||||
alloc.pid = new_pid;
|
alloc.pid = new_pid;
|
||||||
alloc.segment_id = Some(segment_id);
|
alloc.segment_id = Some(segment_id);
|
||||||
guard.save()?;
|
ScopeAllocationGuard::new(worker_name, lock_path).commit_acquisition(&mut guard)
|
||||||
Ok(ScopeAllocationGuard {
|
|
||||||
worker_name,
|
|
||||||
lock_path,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Information about a Worker that currently holds an allocation for a
|
/// Information about a Worker that currently holds an allocation for a
|
||||||
@@ -329,6 +437,388 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn retained_cleanup_handle_defers_release_until_final_owner_drop() {
|
||||||
|
let dir = TempDir::new().unwrap();
|
||||||
|
let lock_path = dir.path().join("workers.json");
|
||||||
|
let guard = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"retained".into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("retained"),
|
||||||
|
Vec::new(),
|
||||||
|
sid(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let cleanup = guard.cleanup_handle();
|
||||||
|
let retained = cleanup.clone();
|
||||||
|
|
||||||
|
drop(guard);
|
||||||
|
drop(cleanup);
|
||||||
|
{
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(table.data().find("retained").is_some());
|
||||||
|
}
|
||||||
|
drop(retained);
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(table.data().find("retained").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn dropping_cleanup_handle_after_transfer_preserves_live_guard() {
|
||||||
|
let dir = TempDir::new().unwrap();
|
||||||
|
let lock_path = dir.path().join("workers.json");
|
||||||
|
let segment = sid();
|
||||||
|
let guard = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"transferred".into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("transferred"),
|
||||||
|
Vec::new(),
|
||||||
|
segment,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
drop(guard.cleanup_handle());
|
||||||
|
// A successful controller transfer leaves the guard usable, including
|
||||||
|
// the existing activation protocol, without retaining a cleanup handle.
|
||||||
|
let replacement = sid();
|
||||||
|
guard
|
||||||
|
.begin_segment_activation(segment, replacement)
|
||||||
|
.unwrap()
|
||||||
|
.commit()
|
||||||
|
.unwrap();
|
||||||
|
{
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
table.data().find("transferred").unwrap().segment_id,
|
||||||
|
Some(replacement)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
drop(guard);
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(table.data().find("transferred").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn cleanup_rejects_live_guard_then_releases_only_detached_allocation() {
|
||||||
|
let dir = TempDir::new().unwrap();
|
||||||
|
let lock_path = dir.path().join("workers.json");
|
||||||
|
let guard = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"live".into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("live"),
|
||||||
|
Vec::new(),
|
||||||
|
sid(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let _competitor = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"competitor".into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("competitor"),
|
||||||
|
Vec::new(),
|
||||||
|
sid(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let cleanup = guard.cleanup_handle();
|
||||||
|
assert!(matches!(
|
||||||
|
cleanup.release(),
|
||||||
|
Err(ScopeLockError::AllocationGuardAttached { worker_name }) if worker_name == "live"
|
||||||
|
));
|
||||||
|
drop(cleanup.clone());
|
||||||
|
{
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(table.data().find("live").is_some());
|
||||||
|
assert!(table.data().find("competitor").is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
drop(guard);
|
||||||
|
cleanup.release().unwrap();
|
||||||
|
cleanup.clone().release().unwrap();
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(table.data().find("live").is_none());
|
||||||
|
assert!(table.data().find("competitor").is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn published_install_failure_retains_cleanup_across_error_drop_and_retry() {
|
||||||
|
use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test};
|
||||||
|
|
||||||
|
let dir = TempDir::new().unwrap();
|
||||||
|
let lock_path = dir.path().join("workers.json");
|
||||||
|
let _competitor = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"competitor".into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("competitor"),
|
||||||
|
Vec::new(),
|
||||||
|
sid(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let segment = sid();
|
||||||
|
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::AfterRename);
|
||||||
|
let error = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"failed".into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("failed"),
|
||||||
|
Vec::new(),
|
||||||
|
segment,
|
||||||
|
)
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(matches!(
|
||||||
|
&error,
|
||||||
|
ScopeLockError::AcquisitionSaveFailed { .. }
|
||||||
|
));
|
||||||
|
let cleanup = error.allocation_cleanup_handle().unwrap();
|
||||||
|
let retry = error.allocation_cleanup_handle().unwrap();
|
||||||
|
drop(error);
|
||||||
|
{
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
table.data().find("failed").unwrap().segment_id,
|
||||||
|
Some(segment)
|
||||||
|
);
|
||||||
|
assert!(table.data().find("competitor").is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::BeforeRename);
|
||||||
|
assert!(matches!(cleanup.release(), Err(ScopeLockError::Io(_))));
|
||||||
|
drop(cleanup);
|
||||||
|
{
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(table.data().find("failed").is_some());
|
||||||
|
}
|
||||||
|
retry.release().unwrap();
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(table.data().find("failed").is_none());
|
||||||
|
assert!(table.data().find("competitor").is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn refused_install_has_no_cleanup_authority_over_existing_allocation() {
|
||||||
|
let dir = TempDir::new().unwrap();
|
||||||
|
let lock_path = dir.path().join("workers.json");
|
||||||
|
let segment = sid();
|
||||||
|
let _competitor = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"competitor".into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("competitor"),
|
||||||
|
vec![write_rule("/shared", true)],
|
||||||
|
segment,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let cases = [
|
||||||
|
("competitor", sid(), Vec::new()),
|
||||||
|
("other", segment, Vec::new()),
|
||||||
|
("other", sid(), vec![write_rule("/shared", true)]),
|
||||||
|
];
|
||||||
|
for (name, requested_segment, scope) in cases {
|
||||||
|
let error = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
name.into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("refused"),
|
||||||
|
scope,
|
||||||
|
requested_segment,
|
||||||
|
)
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(matches!(
|
||||||
|
&error,
|
||||||
|
ScopeLockError::DuplicateWorkerName(_)
|
||||||
|
| ScopeLockError::SegmentConflict { .. }
|
||||||
|
| ScopeLockError::WriteConflict { .. }
|
||||||
|
));
|
||||||
|
assert!(error.allocation_cleanup_handle().is_none());
|
||||||
|
drop(error);
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert_eq!(table.data().allocations.len(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
table.data().find("competitor").unwrap().socket,
|
||||||
|
sock("competitor")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
table.data().find("competitor").unwrap().segment_id,
|
||||||
|
Some(segment)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unpublished_install_failure_has_no_cleanup_authority() {
|
||||||
|
use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test};
|
||||||
|
|
||||||
|
let dir = TempDir::new().unwrap();
|
||||||
|
let lock_path = dir.path().join("workers.json");
|
||||||
|
let _competitor = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"competitor".into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("competitor"),
|
||||||
|
Vec::new(),
|
||||||
|
sid(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::BeforeRename);
|
||||||
|
let error = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"failed".into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("failed"),
|
||||||
|
Vec::new(),
|
||||||
|
sid(),
|
||||||
|
)
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(matches!(&error, ScopeLockError::Io(_)));
|
||||||
|
assert!(error.allocation_cleanup_handle().is_none());
|
||||||
|
drop(error);
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(table.data().find("failed").is_none());
|
||||||
|
assert!(table.data().find("competitor").is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn published_adoption_failure_retains_cleanup_for_retry() {
|
||||||
|
use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test};
|
||||||
|
|
||||||
|
let dir = TempDir::new().unwrap();
|
||||||
|
let lock_path = dir.path().join("workers.json");
|
||||||
|
{
|
||||||
|
let mut table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
delegate_placeholder(&mut table, "child", std::process::id());
|
||||||
|
delegate_placeholder(&mut table, "competitor", std::process::id());
|
||||||
|
}
|
||||||
|
let segment = sid();
|
||||||
|
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::AfterRename);
|
||||||
|
let error = adopt_allocation_at(
|
||||||
|
lock_path.clone(),
|
||||||
|
"child".into(),
|
||||||
|
std::process::id(),
|
||||||
|
segment,
|
||||||
|
)
|
||||||
|
.unwrap_err();
|
||||||
|
let cleanup = error.allocation_cleanup_handle().unwrap();
|
||||||
|
drop(error);
|
||||||
|
{
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
table.data().find("child").unwrap().segment_id,
|
||||||
|
Some(segment)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::AfterRename);
|
||||||
|
assert!(matches!(cleanup.release(), Err(ScopeLockError::Io(_))));
|
||||||
|
{
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(table.data().find("child").is_none());
|
||||||
|
assert!(table.data().find("competitor").is_some());
|
||||||
|
}
|
||||||
|
cleanup.release().unwrap();
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(table.data().find("child").is_none());
|
||||||
|
assert!(table.data().find("competitor").is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unpublished_adoption_failure_preserves_placeholder_without_cleanup_authority() {
|
||||||
|
use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test};
|
||||||
|
|
||||||
|
let dir = TempDir::new().unwrap();
|
||||||
|
let lock_path = dir.path().join("workers.json");
|
||||||
|
let placeholder_pid = std::process::id();
|
||||||
|
{
|
||||||
|
let mut table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
delegate_placeholder(&mut table, "child", placeholder_pid);
|
||||||
|
}
|
||||||
|
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::BeforeRename);
|
||||||
|
let error = adopt_allocation_at(
|
||||||
|
lock_path.clone(),
|
||||||
|
"child".into(),
|
||||||
|
placeholder_pid.wrapping_add(1),
|
||||||
|
sid(),
|
||||||
|
)
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(matches!(&error, ScopeLockError::Io(_)));
|
||||||
|
assert!(error.allocation_cleanup_handle().is_none());
|
||||||
|
drop(error);
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
let allocation = table.data().find("child").unwrap();
|
||||||
|
assert_eq!(allocation.pid, placeholder_pid);
|
||||||
|
assert_eq!(allocation.segment_id, None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn failed_cleanup_release_retains_authority_for_retry() {
|
||||||
|
use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test};
|
||||||
|
|
||||||
|
for point in [SaveFaultPoint::BeforeRename, SaveFaultPoint::AfterRename] {
|
||||||
|
let dir = TempDir::new().unwrap();
|
||||||
|
let lock_path = dir.path().join("workers.json");
|
||||||
|
let guard = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"failed".into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("failed"),
|
||||||
|
vec![read_rule("/shared", true)],
|
||||||
|
sid(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let _other = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"other".into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("other"),
|
||||||
|
vec![read_rule("/shared", true)],
|
||||||
|
sid(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let cleanup = guard.cleanup_handle();
|
||||||
|
let retry = cleanup.clone();
|
||||||
|
drop(guard);
|
||||||
|
fail_next_save_at_for_test(&lock_path, point);
|
||||||
|
|
||||||
|
assert!(matches!(cleanup.release(), Err(ScopeLockError::Io(_))));
|
||||||
|
drop(cleanup);
|
||||||
|
{
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
table.data().find("failed").is_some(),
|
||||||
|
point == SaveFaultPoint::BeforeRename
|
||||||
|
);
|
||||||
|
assert!(table.data().find("other").is_some());
|
||||||
|
}
|
||||||
|
retry.release().unwrap();
|
||||||
|
{
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(table.data().find("failed").is_none());
|
||||||
|
assert!(table.data().find("other").is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
// Success disarms every clone and final Drop, even if the name is
|
||||||
|
// subsequently allocated to a different Worker.
|
||||||
|
let _replacement = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"failed".into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("replacement"),
|
||||||
|
vec![read_rule("/shared", true)],
|
||||||
|
sid(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
retry.clone().release().unwrap();
|
||||||
|
drop(retry);
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
table.data().find("failed").unwrap().socket,
|
||||||
|
sock("replacement")
|
||||||
|
);
|
||||||
|
assert!(table.data().find("other").is_some());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn adopt_allocation_rewrites_pid_and_releases_on_drop() {
|
fn adopt_allocation_rewrites_pid_and_releases_on_drop() {
|
||||||
let dir = TempDir::new().unwrap();
|
let dir = TempDir::new().unwrap();
|
||||||
@@ -358,6 +848,7 @@ mod tests {
|
|||||||
let dir = TempDir::new().unwrap();
|
let dir = TempDir::new().unwrap();
|
||||||
let _sandbox = RuntimeDirSandbox::new(dir.path());
|
let _sandbox = RuntimeDirSandbox::new(dir.path());
|
||||||
let err = adopt_allocation("ghost".into(), 42, sid()).unwrap_err();
|
let err = adopt_allocation("ghost".into(), 42, sid()).unwrap_err();
|
||||||
|
assert!(err.allocation_cleanup_handle().is_none());
|
||||||
assert!(matches!(err, ScopeLockError::UnknownWorker(ref n) if n == "ghost"));
|
assert!(matches!(err, ScopeLockError::UnknownWorker(ref n) if n == "ghost"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -57,6 +57,30 @@ pub fn register_worker_with_deny(
|
|||||||
scope_allow: Vec<ScopeRule>,
|
scope_allow: Vec<ScopeRule>,
|
||||||
scope_deny: Vec<ScopeRule>,
|
scope_deny: Vec<ScopeRule>,
|
||||||
segment_id: SegmentId,
|
segment_id: SegmentId,
|
||||||
|
) -> Result<(), ScopeLockError> {
|
||||||
|
prepare_worker_with_deny(
|
||||||
|
guard,
|
||||||
|
worker_name,
|
||||||
|
pid,
|
||||||
|
socket,
|
||||||
|
scope_allow,
|
||||||
|
scope_deny,
|
||||||
|
segment_id,
|
||||||
|
)?;
|
||||||
|
guard.save()?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Admit and stage a registration without publishing it. Owned installation
|
||||||
|
/// constructs its cleanup authority after admission and before committing.
|
||||||
|
pub(super) fn prepare_worker_with_deny(
|
||||||
|
guard: &mut LockFileGuard,
|
||||||
|
worker_name: String,
|
||||||
|
pid: u32,
|
||||||
|
socket: PathBuf,
|
||||||
|
scope_allow: Vec<ScopeRule>,
|
||||||
|
scope_deny: Vec<ScopeRule>,
|
||||||
|
segment_id: SegmentId,
|
||||||
) -> Result<(), ScopeLockError> {
|
) -> Result<(), ScopeLockError> {
|
||||||
reclaim_stale(guard);
|
reclaim_stale(guard);
|
||||||
if guard.data().find(&worker_name).is_some() {
|
if guard.data().find(&worker_name).is_some() {
|
||||||
@@ -101,7 +125,6 @@ pub fn register_worker_with_deny(
|
|||||||
delegated_from: None,
|
delegated_from: None,
|
||||||
segment_id: Some(segment_id),
|
segment_id: Some(segment_id),
|
||||||
});
|
});
|
||||||
guard.save()?;
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -246,6 +246,14 @@ impl LockFileGuard {
|
|||||||
/// Persist with atomic replacement while the separate allocation lock stays
|
/// Persist with atomic replacement while the separate allocation lock stays
|
||||||
/// held. A crash exposes either the old complete table or the new one.
|
/// held. A crash exposes either the old complete table or the new one.
|
||||||
pub fn save(&mut self) -> io::Result<()> {
|
pub fn save(&mut self) -> io::Result<()> {
|
||||||
|
let mut published = false;
|
||||||
|
self.save_with_publication(&mut published)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Report this save's rename while the caller still holds the table lock.
|
||||||
|
/// In-memory mutation alone is not evidence that an acquisition took place.
|
||||||
|
pub(super) fn save_with_publication(&mut self, published: &mut bool) -> io::Result<()> {
|
||||||
|
*published = false;
|
||||||
let json = serde_json::to_vec_pretty(&self.data).map_err(io::Error::other)?;
|
let json = serde_json::to_vec_pretty(&self.data).map_err(io::Error::other)?;
|
||||||
let mut temp_os = self.data_path.as_os_str().to_owned();
|
let mut temp_os = self.data_path.as_os_str().to_owned();
|
||||||
temp_os.push(".tmp");
|
temp_os.push(".tmp");
|
||||||
@@ -265,6 +273,7 @@ impl LockFileGuard {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
fs::rename(&temp_path, &self.data_path)?;
|
fs::rename(&temp_path, &self.data_path)?;
|
||||||
|
*published = true;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
if take_save_fault(&self.data_path, SaveFaultPoint::AfterRename) {
|
if take_save_fault(&self.data_path, SaveFaultPoint::AfterRename) {
|
||||||
return Err(io::Error::other(
|
return Err(io::Error::other(
|
||||||
|
|||||||
@@ -340,6 +340,84 @@ pub struct SpawnedWorkerRegistryLoad {
|
|||||||
pub reclaimed_unreachable: bool,
|
pub reclaimed_unreachable: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Read-only restore preparation using the registry that will be published by the Host.
|
||||||
|
/// The Host must retain its lifecycle exclusion from preparation through application;
|
||||||
|
/// the metadata reread detects source drift, but is not an atomic store transaction.
|
||||||
|
/// Dropping an unapplied plan does not reclaim legacy child state.
|
||||||
|
pub(crate) struct PreparedSpawnedWorkerRegistryLoad<St> {
|
||||||
|
pub(crate) registry: Arc<SpawnedWorkerRegistry>,
|
||||||
|
/// Projected outcome, not evidence that reclamation has already happened.
|
||||||
|
pub(crate) reclaimed_unreachable: bool,
|
||||||
|
runtime_dir: Arc<RuntimeDir>,
|
||||||
|
store: St,
|
||||||
|
worker_name: String,
|
||||||
|
parent_scope: Option<SharedScope>,
|
||||||
|
persisted_children: Vec<WorkerSpawnedChild>,
|
||||||
|
valid_records: Vec<SpawnedWorkerRecord>,
|
||||||
|
legacy_projection_exists: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<St> PreparedSpawnedWorkerRegistryLoad<St>
|
||||||
|
where
|
||||||
|
St: WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||||
|
{
|
||||||
|
/// Apply only after all other fallible tool initialization, before socket publication.
|
||||||
|
/// Effects retain their original order: projection, canonical reclaim history, then
|
||||||
|
/// allocation/scope. An error may follow committed effects; no rollback is attempted.
|
||||||
|
pub(crate) async fn apply(&mut self) -> io::Result<SpawnedWorkerRegistryLoad> {
|
||||||
|
let metadata = self
|
||||||
|
.store
|
||||||
|
.read_by_name(&self.worker_name)
|
||||||
|
.map_err(store_error_to_io)?;
|
||||||
|
let reclaimed: Vec<_> = self
|
||||||
|
.persisted_children
|
||||||
|
.iter()
|
||||||
|
.map(reclaimed_child_from_metadata)
|
||||||
|
.collect();
|
||||||
|
// A prior apply may have committed the canonical reclaim and then failed
|
||||||
|
// in allocation/scope cleanup. Under the host's same lifecycle lock,
|
||||||
|
// confirm its exact retained facts rather than appending them again.
|
||||||
|
let already_reclaimed = !reclaimed.is_empty()
|
||||||
|
&& metadata.as_ref().is_some_and(|metadata| {
|
||||||
|
metadata.spawned_children.is_empty()
|
||||||
|
&& metadata.reclaimed_children.ends_with(&reclaimed)
|
||||||
|
});
|
||||||
|
if !already_reclaimed
|
||||||
|
&& metadata
|
||||||
|
.as_ref()
|
||||||
|
.map(|metadata| metadata.spawned_children.as_slice())
|
||||||
|
.unwrap_or_default()
|
||||||
|
!= self.persisted_children.as_slice()
|
||||||
|
{
|
||||||
|
return Err(io::Error::new(
|
||||||
|
io::ErrorKind::InvalidData,
|
||||||
|
"spawned-child metadata changed after registry preparation",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Runtime projection is migration input only; the normal Internal registry is never
|
||||||
|
// materialized into spawned_workers.json. Only existence was captured, not its contents.
|
||||||
|
if !self.persisted_children.is_empty() || self.legacy_projection_exists {
|
||||||
|
self.runtime_dir.write_spawned_workers(&[]).await?;
|
||||||
|
}
|
||||||
|
if !reclaimed.is_empty() && !already_reclaimed {
|
||||||
|
self.store
|
||||||
|
.reclaim_spawned_children(&self.worker_name, reclaimed)
|
||||||
|
.map_err(store_error_to_io)?;
|
||||||
|
}
|
||||||
|
for record in &self.valid_records {
|
||||||
|
reclaim_record(&self.worker_name, self.parent_scope.as_ref(), record)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
let reclaimed_unreachable = self.reclaimed_unreachable;
|
||||||
|
self.reclaimed_unreachable = false;
|
||||||
|
Ok(SpawnedWorkerRegistryLoad {
|
||||||
|
registry: self.registry.clone(),
|
||||||
|
reclaimed_unreachable,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl SpawnedWorkerRegistry {
|
impl SpawnedWorkerRegistry {
|
||||||
pub(crate) fn bash_output(&self) -> tools::bash_output::BashOutputView {
|
pub(crate) fn bash_output(&self) -> tools::bash_output::BashOutputView {
|
||||||
self.bash_output.lock().unwrap().clone()
|
self.bash_output.lock().unwrap().clone()
|
||||||
@@ -426,6 +504,23 @@ impl SpawnedWorkerRegistry {
|
|||||||
worker_name: String,
|
worker_name: String,
|
||||||
parent_scope: Option<SharedScope>,
|
parent_scope: Option<SharedScope>,
|
||||||
) -> io::Result<SpawnedWorkerRegistryLoad>
|
) -> io::Result<SpawnedWorkerRegistryLoad>
|
||||||
|
where
|
||||||
|
St: WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||||
|
{
|
||||||
|
Self::prepare_from_worker_state_with_reclaim(runtime_dir, store, worker_name, parent_scope)?
|
||||||
|
.apply()
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Capture legacy child facts without starting children or changing metadata,
|
||||||
|
/// projection, allocation, or scope. Tool initialization must clone this registry,
|
||||||
|
/// not construct a replacement, and defer reclamation to the owned plan's `apply`.
|
||||||
|
pub(crate) fn prepare_from_worker_state_with_reclaim<St>(
|
||||||
|
runtime_dir: Arc<RuntimeDir>,
|
||||||
|
store: St,
|
||||||
|
worker_name: String,
|
||||||
|
parent_scope: Option<SharedScope>,
|
||||||
|
) -> io::Result<PreparedSpawnedWorkerRegistryLoad<St>>
|
||||||
where
|
where
|
||||||
St: WorkerMetadataStore + Clone + Send + Sync + 'static,
|
St: WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||||
{
|
{
|
||||||
@@ -442,54 +537,33 @@ impl SpawnedWorkerRegistry {
|
|||||||
Ok(record) => {
|
Ok(record) => {
|
||||||
warn!(
|
warn!(
|
||||||
worker = %record.worker_name,
|
worker = %record.worker_name,
|
||||||
"reclaiming legacy persisted process SubWorker during Internal session restore"
|
"captured legacy persisted process SubWorker for reclamation during Internal session restore"
|
||||||
);
|
);
|
||||||
valid_records.push(record);
|
valid_records.push(record);
|
||||||
}
|
}
|
||||||
Err(error) => warn!(
|
Err(error) => warn!(
|
||||||
error = %error,
|
error = %error,
|
||||||
worker = %child.worker_name,
|
worker = %child.worker_name,
|
||||||
"clearing corrupt legacy persisted process SubWorker record"
|
"captured corrupt legacy persisted process SubWorker record for clearing"
|
||||||
),
|
),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Runtime projection is migration input only; the normal Internal registry is never
|
|
||||||
// materialized into spawned_workers.json.
|
|
||||||
let legacy_projection_exists = runtime_dir.path().join("spawned_workers.json").exists();
|
let legacy_projection_exists = runtime_dir.path().join("spawned_workers.json").exists();
|
||||||
if !persisted_children.is_empty() || legacy_projection_exists {
|
let registry = match &parent_scope {
|
||||||
runtime_dir.write_spawned_workers(&[]).await?;
|
Some(scope) => Self::new_internal(worker_name.clone(), scope.clone()),
|
||||||
}
|
None => Self::new(runtime_dir.clone()),
|
||||||
if !persisted_children.is_empty() {
|
};
|
||||||
let reclaimed = persisted_children
|
Ok(PreparedSpawnedWorkerRegistryLoad {
|
||||||
.iter()
|
registry,
|
||||||
.map(reclaimed_child_from_metadata)
|
|
||||||
.collect();
|
|
||||||
store
|
|
||||||
.reclaim_spawned_children(&worker_name, reclaimed)
|
|
||||||
.map_err(store_error_to_io)?;
|
|
||||||
}
|
|
||||||
for record in &valid_records {
|
|
||||||
reclaim_record(&worker_name, parent_scope.as_ref(), record)?;
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(SpawnedWorkerRegistryLoad {
|
|
||||||
registry: Arc::new(Self {
|
|
||||||
internal_records: std::sync::Mutex::new(Vec::new()),
|
|
||||||
service_records: std::sync::Mutex::new(Vec::new()),
|
|
||||||
#[cfg(test)]
|
|
||||||
fail_service_stops: std::sync::Mutex::new(HashMap::new()),
|
|
||||||
internal_names: std::sync::Mutex::new(HashSet::new()),
|
|
||||||
internal_shutting_down: AtomicBool::new(false),
|
|
||||||
pending_internal_spawns: AtomicUsize::new(0),
|
|
||||||
pending_internal_notify: Notify::new(),
|
|
||||||
internal_spawn_cleanup_failed: AtomicBool::new(false),
|
|
||||||
parent_scope,
|
|
||||||
bash_output: Mutex::new(tools::bash_output::BashOutputView::new()),
|
|
||||||
parent_protocol: Mutex::new(None),
|
|
||||||
protocol_publish_lock: Arc::new(Mutex::new(())),
|
|
||||||
}),
|
|
||||||
reclaimed_unreachable: !persisted_children.is_empty(),
|
reclaimed_unreachable: !persisted_children.is_empty(),
|
||||||
|
runtime_dir,
|
||||||
|
store,
|
||||||
|
worker_name,
|
||||||
|
parent_scope,
|
||||||
|
persisted_children,
|
||||||
|
valid_records,
|
||||||
|
legacy_projection_exists,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1140,6 +1214,10 @@ impl Drop for SpawnedWorkerRegistry {
|
|||||||
.filter(|record| !record.scope_reclaimed.load(Ordering::Acquire))
|
.filter(|record| !record.scope_reclaimed.load(Ordering::Acquire))
|
||||||
.flat_map(delegated_write_rules)
|
.flat_map(delegated_write_rules)
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
|
// Empty prepared registries own no delegations, including when abandoned or rejected.
|
||||||
|
if write_rules.is_empty() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
let _ = parent_scope.update(|current| current.with_removed_deny_rules(write_rules));
|
let _ = parent_scope.update(|current| current.with_removed_deny_rules(write_rules));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1165,19 +1243,18 @@ fn reclaim_record(
|
|||||||
parent_scope: Option<&SharedScope>,
|
parent_scope: Option<&SharedScope>,
|
||||||
record: &SpawnedWorkerRecord,
|
record: &SpawnedWorkerRecord,
|
||||||
) -> io::Result<()> {
|
) -> io::Result<()> {
|
||||||
if let Ok(path) = worker_allocation::default_allocation_path() {
|
let path = worker_allocation::default_allocation_path().map_err(io::Error::other)?;
|
||||||
if let Ok(mut guard) = worker_allocation::LockFileGuard::open(&path) {
|
let mut guard = worker_allocation::LockFileGuard::open(&path).map_err(io::Error::other)?;
|
||||||
match worker_allocation::reclaim_delegated_scope(
|
match worker_allocation::reclaim_delegated_scope(
|
||||||
&mut guard,
|
&mut guard,
|
||||||
parent_name,
|
parent_name,
|
||||||
&record.worker_name,
|
&record.worker_name,
|
||||||
&record.scope_delegated,
|
&record.scope_delegated,
|
||||||
) {
|
) {
|
||||||
Ok(()) | Err(worker_allocation::ScopeLockError::UnknownWorker(_)) => {}
|
Ok(()) | Err(worker_allocation::ScopeLockError::UnknownWorker(_)) => {}
|
||||||
Err(error) => return Err(io::Error::other(error)),
|
Err(error) => return Err(io::Error::other(error)),
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
drop(guard);
|
||||||
if let Some(parent_scope) = parent_scope {
|
if let Some(parent_scope) = parent_scope {
|
||||||
let write_rules = record
|
let write_rules = record
|
||||||
.scope_delegated
|
.scope_delegated
|
||||||
@@ -1240,10 +1317,18 @@ fn store_error_to_io(error: WorkerStoreError) -> io::Error {
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
|
//! Legacy restore cases connect the registry to canonical aggregate storage,
|
||||||
|
//! runtime projections, and temporary SharedScope rules. Successful apply uses
|
||||||
|
//! a corrupt legacy record to exercise durable clearing without consulting the
|
||||||
|
//! process-global legacy allocation file; valid-record scope reclaim is also
|
||||||
|
//! covered by the existing external restore test.
|
||||||
|
|
||||||
|
use std::path::PathBuf;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use manifest::{Scope, ScopeConfig};
|
use manifest::{Scope, ScopeConfig};
|
||||||
use session_store::LogEntry;
|
use session_store::{LogEntry, WorkerAggregateStore, WorkerMetadata, WorkerSpawnedScopeRule};
|
||||||
|
use tempfile::TempDir;
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::internal_worker::{
|
use crate::internal_worker::{
|
||||||
@@ -1251,6 +1336,249 @@ mod tests {
|
|||||||
test_internal_worker_session_with_greeting,
|
test_internal_worker_session_with_greeting,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
struct LegacyRestoreFixture {
|
||||||
|
runtime_dir: Arc<RuntimeDir>,
|
||||||
|
store: WorkerAggregateStore,
|
||||||
|
parent_scope: SharedScope,
|
||||||
|
scope_root: PathBuf,
|
||||||
|
metadata_path: PathBuf,
|
||||||
|
// Keep all storage and scope paths alive until the other fixture fields are dropped.
|
||||||
|
_root: TempDir,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl LegacyRestoreFixture {
|
||||||
|
async fn new(permission: &str) -> Self {
|
||||||
|
let root = TempDir::new().unwrap();
|
||||||
|
let scope_root = root.path().join("scope");
|
||||||
|
std::fs::create_dir(&scope_root).unwrap();
|
||||||
|
let aggregate = root.path().join("aggregate");
|
||||||
|
let store = WorkerAggregateStore::new(&aggregate, "parent").unwrap();
|
||||||
|
let runtime_dir = Arc::new(
|
||||||
|
RuntimeDir::create(&root.path().join("runtime"), "parent")
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
let mut metadata =
|
||||||
|
WorkerMetadata::new("parent", None).with_workspace_id("restore-test-workspace");
|
||||||
|
metadata.spawned_children.push(WorkerSpawnedChild {
|
||||||
|
worker_name: "legacy-child".into(),
|
||||||
|
socket_path: root.path().join("legacy.sock"),
|
||||||
|
callback_address: runtime_dir.path().join("sock"),
|
||||||
|
scope_delegated: vec![WorkerSpawnedScopeRule {
|
||||||
|
target: scope_root.clone(),
|
||||||
|
permission: permission.into(),
|
||||||
|
recursive: true,
|
||||||
|
symlink_policy: Default::default(),
|
||||||
|
}],
|
||||||
|
});
|
||||||
|
store.write(&metadata).unwrap();
|
||||||
|
let write_rule = ScopeRule {
|
||||||
|
target: scope_root.clone(),
|
||||||
|
permission: Permission::Write,
|
||||||
|
recursive: true,
|
||||||
|
symlink_policy: Default::default(),
|
||||||
|
};
|
||||||
|
let parent_scope = SharedScope::new(
|
||||||
|
Scope::from_config(&ScopeConfig {
|
||||||
|
allow: vec![write_rule.clone()],
|
||||||
|
deny: vec![write_rule],
|
||||||
|
})
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
Self {
|
||||||
|
runtime_dir,
|
||||||
|
store,
|
||||||
|
parent_scope,
|
||||||
|
scope_root,
|
||||||
|
metadata_path: aggregate.join("metadata.json"),
|
||||||
|
_root: root,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn prepare(&self) -> PreparedSpawnedWorkerRegistryLoad<WorkerAggregateStore> {
|
||||||
|
SpawnedWorkerRegistry::prepare_from_worker_state_with_reclaim(
|
||||||
|
self.runtime_dir.clone(),
|
||||||
|
self.store.clone(),
|
||||||
|
"parent".into(),
|
||||||
|
Some(self.parent_scope.clone()),
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn projection_path(&self) -> PathBuf {
|
||||||
|
self.runtime_dir.path().join("spawned_workers.json")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn registry_preparation_and_abandonment_leave_legacy_state_unchanged() {
|
||||||
|
for projection_exists in [false, true] {
|
||||||
|
let fixture = LegacyRestoreFixture::new("write").await;
|
||||||
|
if projection_exists {
|
||||||
|
// Projection contents are deliberately not decoded during preparation.
|
||||||
|
std::fs::write(fixture.projection_path(), b"obsolete projection").unwrap();
|
||||||
|
}
|
||||||
|
let metadata_before = std::fs::read(&fixture.metadata_path).unwrap();
|
||||||
|
let projection_before = std::fs::read(fixture.projection_path()).ok();
|
||||||
|
let scope_before = fixture.parent_scope.snapshot();
|
||||||
|
assert!(!scope_before.is_writable(&fixture.scope_root));
|
||||||
|
|
||||||
|
let prepared = fixture.prepare();
|
||||||
|
|
||||||
|
assert!(prepared.reclaimed_unreachable);
|
||||||
|
assert!(prepared.registry.list_internal().is_empty());
|
||||||
|
assert!(prepared.registry.internal_worker_snapshots().is_empty());
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read(&fixture.metadata_path).unwrap(),
|
||||||
|
metadata_before
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read(fixture.projection_path()).ok(),
|
||||||
|
projection_before
|
||||||
|
);
|
||||||
|
assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot()));
|
||||||
|
assert!(!fixture.runtime_dir.path().join("sock").exists());
|
||||||
|
drop(prepared);
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read(&fixture.metadata_path).unwrap(),
|
||||||
|
metadata_before
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read(fixture.projection_path()).ok(),
|
||||||
|
projection_before
|
||||||
|
);
|
||||||
|
assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn prepared_apply_clears_canonical_children_and_projection_on_same_registry() {
|
||||||
|
let fixture = LegacyRestoreFixture::new("unsupported-legacy-permission").await;
|
||||||
|
std::fs::write(fixture.projection_path(), b"obsolete projection").unwrap();
|
||||||
|
let scope_before = fixture.parent_scope.snapshot();
|
||||||
|
let mut prepared = fixture.prepare();
|
||||||
|
let shared_registry = prepared.registry.clone();
|
||||||
|
// Unrelated metadata may advance while tools are initialized.
|
||||||
|
fixture.store.add_peer("parent", "new-peer").unwrap();
|
||||||
|
let mut expected = fixture.store.read_by_name("parent").unwrap().unwrap();
|
||||||
|
expected.reclaimed_children.extend(
|
||||||
|
expected
|
||||||
|
.spawned_children
|
||||||
|
.iter()
|
||||||
|
.map(reclaimed_child_from_metadata),
|
||||||
|
);
|
||||||
|
expected.spawned_children.clear();
|
||||||
|
|
||||||
|
let loaded = prepared.apply().await.unwrap();
|
||||||
|
|
||||||
|
assert!(loaded.reclaimed_unreachable);
|
||||||
|
assert!(Arc::ptr_eq(&shared_registry, &loaded.registry));
|
||||||
|
assert!(shared_registry.list_internal().is_empty());
|
||||||
|
assert_eq!(
|
||||||
|
fixture.store.read_by_name("parent").unwrap(),
|
||||||
|
Some(expected)
|
||||||
|
);
|
||||||
|
let projection: serde_json::Value =
|
||||||
|
serde_json::from_slice(&std::fs::read(fixture.projection_path()).unwrap()).unwrap();
|
||||||
|
assert_eq!(projection, serde_json::json!([]));
|
||||||
|
// Corrupt records are audited and cleared, not interpreted as scope grants.
|
||||||
|
assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot()));
|
||||||
|
assert!(
|
||||||
|
!fixture
|
||||||
|
.parent_scope
|
||||||
|
.snapshot()
|
||||||
|
.is_writable(&fixture.scope_root)
|
||||||
|
);
|
||||||
|
assert!(!fixture.runtime_dir.path().join("sock").exists());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn prepared_apply_rejects_child_source_drift_before_any_effects() {
|
||||||
|
for drift in ["scope", "socket", "removal", "metadata-deletion"] {
|
||||||
|
let fixture = LegacyRestoreFixture::new("write").await;
|
||||||
|
std::fs::write(fixture.projection_path(), b"obsolete projection").unwrap();
|
||||||
|
let scope_before = fixture.parent_scope.snapshot();
|
||||||
|
let mut prepared = fixture.prepare();
|
||||||
|
if drift == "metadata-deletion" {
|
||||||
|
fixture.store.delete_by_name("parent").unwrap();
|
||||||
|
} else {
|
||||||
|
fixture
|
||||||
|
.store
|
||||||
|
.update_by_name("parent", |metadata| match drift {
|
||||||
|
"scope" => {
|
||||||
|
metadata.spawned_children[0].scope_delegated[0].recursive = false
|
||||||
|
}
|
||||||
|
"socket" => {
|
||||||
|
metadata.spawned_children[0].socket_path =
|
||||||
|
fixture.runtime_dir.path().join("replacement.sock");
|
||||||
|
}
|
||||||
|
"removal" => metadata.spawned_children.clear(),
|
||||||
|
_ => unreachable!(),
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
let metadata_before = std::fs::read(&fixture.metadata_path).ok();
|
||||||
|
let projection_before = std::fs::read(fixture.projection_path()).unwrap();
|
||||||
|
|
||||||
|
let error = prepared
|
||||||
|
.apply()
|
||||||
|
.await
|
||||||
|
.err()
|
||||||
|
.expect("source drift must fail");
|
||||||
|
|
||||||
|
assert_eq!(error.kind(), io::ErrorKind::InvalidData);
|
||||||
|
assert_eq!(std::fs::read(&fixture.metadata_path).ok(), metadata_before);
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read(fixture.projection_path()).unwrap(),
|
||||||
|
projection_before
|
||||||
|
);
|
||||||
|
assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot()));
|
||||||
|
assert!(!fixture.runtime_dir.path().join("sock").exists());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn prepared_apply_clears_projection_only_without_reporting_child_reclaim() {
|
||||||
|
let fixture = LegacyRestoreFixture::new("write").await;
|
||||||
|
fixture
|
||||||
|
.store
|
||||||
|
.set_spawned_children("parent", Vec::new())
|
||||||
|
.unwrap();
|
||||||
|
std::fs::write(fixture.projection_path(), b"obsolete projection").unwrap();
|
||||||
|
let metadata_before = std::fs::read(&fixture.metadata_path).unwrap();
|
||||||
|
let mut prepared = fixture.prepare();
|
||||||
|
assert!(!prepared.reclaimed_unreachable);
|
||||||
|
|
||||||
|
let loaded = prepared.apply().await.unwrap();
|
||||||
|
|
||||||
|
assert!(!loaded.reclaimed_unreachable);
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read(&fixture.metadata_path).unwrap(),
|
||||||
|
metadata_before
|
||||||
|
);
|
||||||
|
let projection: serde_json::Value =
|
||||||
|
serde_json::from_slice(&std::fs::read(fixture.projection_path()).unwrap()).unwrap();
|
||||||
|
assert_eq!(projection, serde_json::json!([]));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn prepared_apply_projection_failure_keeps_canonical_children_and_scope() {
|
||||||
|
let fixture = LegacyRestoreFixture::new("write").await;
|
||||||
|
std::fs::create_dir(fixture.projection_path()).unwrap();
|
||||||
|
let metadata_before = std::fs::read(&fixture.metadata_path).unwrap();
|
||||||
|
let scope_before = fixture.parent_scope.snapshot();
|
||||||
|
let mut prepared = fixture.prepare();
|
||||||
|
|
||||||
|
assert!(prepared.apply().await.is_err());
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read(&fixture.metadata_path).unwrap(),
|
||||||
|
metadata_before
|
||||||
|
);
|
||||||
|
assert!(fixture.projection_path().is_dir());
|
||||||
|
assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot()));
|
||||||
|
}
|
||||||
|
|
||||||
fn registry() -> Arc<SpawnedWorkerRegistry> {
|
fn registry() -> Arc<SpawnedWorkerRegistry> {
|
||||||
let scope = Scope::from_config(&ScopeConfig {
|
let scope = Scope::from_config(&ScopeConfig {
|
||||||
allow: vec![ScopeRule {
|
allow: vec![ScopeRule {
|
||||||
|
|||||||
@@ -70,7 +70,8 @@ struct SubWorkerSpawnInput {
|
|||||||
/// Explicitly grant command execution through the parent-owned Workdir tool broker.
|
/// Explicitly grant command execution through the parent-owned Workdir tool broker.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
command: bool,
|
command: bool,
|
||||||
/// Binds an actual read-only builtin Reviewer child to the current Merge Request candidate.
|
/// Binds an actual builtin Reviewer child to the current Merge Request candidate.
|
||||||
|
/// Bash may be enabled for Git and other commands.
|
||||||
/// Review capability material is generated by the trusted spawn layer.
|
/// Review capability material is generated by the trusted spawn layer.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
review: Option<ReviewerHandoffInput>,
|
review: Option<ReviewerHandoffInput>,
|
||||||
@@ -1176,6 +1177,17 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn reviewer_handoff_schema_allows_bash_for_git_commands() {
|
||||||
|
let schema = serde_json::to_value(schemars::schema_for!(SubWorkerSpawnInput)).unwrap();
|
||||||
|
let description = schema["properties"]["review"]["description"]
|
||||||
|
.as_str()
|
||||||
|
.expect("review handoff description");
|
||||||
|
assert!(description.contains("Bash"));
|
||||||
|
assert!(description.contains("Git"));
|
||||||
|
assert!(!description.contains("read-only"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn reviewer_handoff_requires_explicit_builtin_profile_and_writable_scope() {
|
fn reviewer_handoff_requires_explicit_builtin_profile_and_writable_scope() {
|
||||||
let valid: SubWorkerSpawnInput = serde_json::from_value(serde_json::json!({
|
let valid: SubWorkerSpawnInput = serde_json::from_value(serde_json::json!({
|
||||||
|
|||||||
+1932
-60
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,180 @@
|
|||||||
|
// Construction coverage only, not model-behavior measurement or a restored prompt update.
|
||||||
|
use super::*;
|
||||||
|
use crate::feature::builtin::ticket::{TicketFeature, TicketFeatureAccess};
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
struct GuidanceHost;
|
||||||
|
impl WorkspaceClient for GuidanceHost {
|
||||||
|
fn workspace_id(&self) -> Option<&str> {
|
||||||
|
Some("workspace-test")
|
||||||
|
}
|
||||||
|
fn kind(&self) -> &str {
|
||||||
|
"guidance-fixture"
|
||||||
|
}
|
||||||
|
fn is_available(&self) -> bool {
|
||||||
|
true
|
||||||
|
}
|
||||||
|
fn authorize_feature_activation(&self, _feature: &str) -> Result<(), WorkspaceClientError> {
|
||||||
|
// This fixture tests construction, not Backend authorization policy.
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
fn execute(
|
||||||
|
&self,
|
||||||
|
request: WorkspaceRequest,
|
||||||
|
) -> Result<WorkspaceResponse, WorkspaceClientError> {
|
||||||
|
if request.path.ends_with("/authorization/check") {
|
||||||
|
return Ok(WorkspaceResponse {
|
||||||
|
status: 200,
|
||||||
|
body: "{}".into(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
assert_eq!(request.path, "/api/w/workspace-test/flows/resolve");
|
||||||
|
let definition = flow::builtin_flow_source("coder-review")
|
||||||
|
.unwrap()
|
||||||
|
.compile()
|
||||||
|
.unwrap();
|
||||||
|
let source = flow::ResolvedFlowSource {
|
||||||
|
selector: "builtin:coder-review".parse().unwrap(),
|
||||||
|
workspace_id: "workspace-test".into(),
|
||||||
|
flow_id: "guidance-fixture".into(),
|
||||||
|
content_digest: definition.content_digest.clone(),
|
||||||
|
definition,
|
||||||
|
};
|
||||||
|
Ok(WorkspaceResponse {
|
||||||
|
status: 200,
|
||||||
|
body: serde_json::to_string(&source).unwrap(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn fresh_builtin_profiles_materialize_validation_guidance() {
|
||||||
|
for (profile, guidance) in [
|
||||||
|
("coder", "own diagnosis, test selection, and fix iterations"),
|
||||||
|
("orchestrator", "Same-named files on separate branches"),
|
||||||
|
("reviewer", "Responsibility for the complete snapshot"),
|
||||||
|
] {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let manifest = manifest::ProfileResolver::new()
|
||||||
|
.with_workspace_base(dir.path())
|
||||||
|
.resolve(
|
||||||
|
&manifest::ProfileSelector::parse_cli(&format!("builtin:{profile}")),
|
||||||
|
manifest::ProfileResolveOptions::with_worker_name("prompt-wiring"),
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
.manifest;
|
||||||
|
let instruction = manifest.engine.instruction.clone();
|
||||||
|
let config = manifest.feature.merge_request;
|
||||||
|
let ticket = &manifest.feature.ticket;
|
||||||
|
let access = TicketFeatureAccess {
|
||||||
|
authoring: ticket.authoring,
|
||||||
|
thread: ticket.thread,
|
||||||
|
intake: ticket.intake,
|
||||||
|
workflow: ticket.workflow,
|
||||||
|
};
|
||||||
|
let orchestration = manifest.feature.sub_worker.enabled || manifest.feature.worker.enabled;
|
||||||
|
assert!(config.any() && ticket.enabled);
|
||||||
|
let cwd = dir.path().join("workspace");
|
||||||
|
std::fs::create_dir_all(&cwd).unwrap();
|
||||||
|
let mut worker = Worker::new(
|
||||||
|
manifest,
|
||||||
|
Engine::<_, Mutable, SessionHistoryMetadata>::new_annotated(NoopClient),
|
||||||
|
session_store::FsStore::new(dir.path().join("sessions")).unwrap(),
|
||||||
|
WorkerWorkspaceContext::with_client(
|
||||||
|
Some(WorkspaceId::new("workspace-test").unwrap()),
|
||||||
|
Arc::new(GuidanceHost),
|
||||||
|
),
|
||||||
|
WorkerFilesystemAuthority::local(cwd.clone(), cwd.clone()),
|
||||||
|
Scope::writable(&cwd).unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let feature = crate::feature::builtin::merge_request::MergeRequestFeature::new(
|
||||||
|
worker.workspace_client_handle(),
|
||||||
|
config,
|
||||||
|
);
|
||||||
|
let report = worker.install_features(
|
||||||
|
FeatureRegistryBuilder::new()
|
||||||
|
.with_module(feature)
|
||||||
|
.with_module(TicketFeature::new(worker.workspace_client_handle(), access)),
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!report.has_errors(),
|
||||||
|
"{profile}: {}",
|
||||||
|
report.error_message()
|
||||||
|
);
|
||||||
|
if orchestration {
|
||||||
|
worker.register_worker_orchestration_instruction();
|
||||||
|
}
|
||||||
|
worker.set_system_prompt_template(
|
||||||
|
SystemPromptTemplate::parse(
|
||||||
|
&instruction,
|
||||||
|
crate::prompt::source::PromptCatalogSource::builtins_only(),
|
||||||
|
)
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
worker.ensure_system_prompt_materialized().await.unwrap();
|
||||||
|
let prompt = worker.engine().get_system_prompt().unwrap();
|
||||||
|
assert!(prompt.contains(guidance), "{profile}");
|
||||||
|
assert_eq!(
|
||||||
|
prompt.matches("Record validation evidence once").count(),
|
||||||
|
1,
|
||||||
|
"{profile}"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
prompt.matches("Record a Ticket dependency when").count(),
|
||||||
|
1,
|
||||||
|
"{profile}"
|
||||||
|
);
|
||||||
|
assert!(prompt.contains("rerun affected or uncertain checks"));
|
||||||
|
assert!(prompt.contains("Changed source or requirements need fresh attestation"));
|
||||||
|
if orchestration {
|
||||||
|
assert!(prompt.contains("SubWorkerSpawn"));
|
||||||
|
assert!(prompt.contains("delegated authority"));
|
||||||
|
}
|
||||||
|
if profile == "coder" {
|
||||||
|
worker.ensure_segment_head().await.unwrap();
|
||||||
|
let report = worker.install_runtime_flow_transition_feature().unwrap();
|
||||||
|
assert!(!report.has_errors(), "{}", report.error_message());
|
||||||
|
let (segments, state, projection) = worker
|
||||||
|
.prepare_flow_input(vec![Segment::Flow {
|
||||||
|
selector: "builtin:coder-review".into(),
|
||||||
|
}])
|
||||||
|
.unwrap();
|
||||||
|
let state = state.expect("new Flow");
|
||||||
|
let builtin = flow::builtin_flow_source("coder-review")
|
||||||
|
.unwrap()
|
||||||
|
.compile()
|
||||||
|
.unwrap();
|
||||||
|
let entry = worker.projected_input_history(
|
||||||
|
&segments,
|
||||||
|
projection.as_ref(),
|
||||||
|
&[SessionHistoryEntryId::new()],
|
||||||
|
&WorkerHistoryProvenance::HumanInput {
|
||||||
|
account_id: "fixture".into(),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
assert_eq!(entry.len(), 1);
|
||||||
|
assert!(matches!(
|
||||||
|
entry[0].annotation.origin,
|
||||||
|
WorkerHistoryProvenance::FlowInstruction { .. }
|
||||||
|
));
|
||||||
|
assert_eq!(
|
||||||
|
entry[0].item.as_text().as_deref(),
|
||||||
|
Some(
|
||||||
|
builtin
|
||||||
|
.state(&builtin.initial)
|
||||||
|
.unwrap()
|
||||||
|
.instructions
|
||||||
|
.as_str()
|
||||||
|
)
|
||||||
|
);
|
||||||
|
// The resolved immutable Flow, rather than a hard-coded test recipe, was installed.
|
||||||
|
assert!(
|
||||||
|
serde_json::to_string(&state)
|
||||||
|
.unwrap()
|
||||||
|
.contains(&builtin.content_digest)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,3 +1,4 @@
|
|||||||
|
use std::path::{Path, PathBuf};
|
||||||
use std::pin::Pin;
|
use std::pin::Pin;
|
||||||
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
|
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
@@ -150,6 +151,19 @@ impl LlmClient for MockClient {
|
|||||||
// Helpers
|
// Helpers
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
#[path = "../../../tests/support/restore_environment.rs"]
|
||||||
|
mod restore_test_environment;
|
||||||
|
use restore_test_environment::require_restore_test_environment;
|
||||||
|
|
||||||
|
fn owned_relative_runtime_base() -> (tempfile::TempDir, PathBuf) {
|
||||||
|
require_restore_test_environment();
|
||||||
|
let temp = tempfile::tempdir_in(".").unwrap();
|
||||||
|
// The relative reference and the TempDir name the same owned tree, without
|
||||||
|
// putting the checkout prefix into sockaddr_un or changing process cwd.
|
||||||
|
let runtime_base = Path::new(temp.path().file_name().unwrap()).join("runtime");
|
||||||
|
(temp, runtime_base)
|
||||||
|
}
|
||||||
|
|
||||||
fn simple_text_events() -> Vec<LlmEvent> {
|
fn simple_text_events() -> Vec<LlmEvent> {
|
||||||
vec![
|
vec![
|
||||||
LlmEvent::text_block_start(0),
|
LlmEvent::text_block_start(0),
|
||||||
@@ -295,6 +309,7 @@ async fn make_worker_with_manifest_and_workspace_context(
|
|||||||
manifest: WorkerManifest,
|
manifest: WorkerManifest,
|
||||||
workspace_context: WorkerWorkspaceContext,
|
workspace_context: WorkerWorkspaceContext,
|
||||||
) -> (Worker<MockClient, TestStore>, std::path::PathBuf) {
|
) -> (Worker<MockClient, TestStore>, std::path::PathBuf) {
|
||||||
|
require_restore_test_environment();
|
||||||
let store_tmp = tempfile::tempdir().unwrap();
|
let store_tmp = tempfile::tempdir().unwrap();
|
||||||
let store = CombinedStore::new(
|
let store = CombinedStore::new(
|
||||||
FsStore::new(store_tmp.path()).unwrap(),
|
FsStore::new(store_tmp.path()).unwrap(),
|
||||||
@@ -320,18 +335,21 @@ async fn make_worker_with_manifest_and_workspace_context(
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn spawn_controller(worker: Worker<MockClient, TestStore>) -> WorkerHandle {
|
async fn spawn_controller(worker: Worker<MockClient, TestStore>) -> WorkerHandle {
|
||||||
let tmp = tempfile::tempdir().unwrap();
|
let (tmp, runtime_base) = owned_relative_runtime_base();
|
||||||
let runtime_base = tmp.path().to_owned();
|
let (handle, shutdown_rx) = WorkerController::spawn(worker, &runtime_base)
|
||||||
std::mem::forget(tmp);
|
|
||||||
let (handle, _shutdown_rx) = WorkerController::spawn(worker, &runtime_base)
|
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
// Retain ownership until controller shutdown (or test-runtime teardown).
|
||||||
|
tokio::spawn(async move {
|
||||||
|
let _ = shutdown_rx.await;
|
||||||
|
drop(tmp);
|
||||||
|
});
|
||||||
handle
|
handle
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn canonical_worker_handle_uploads_and_submits_attachment() {
|
async fn canonical_worker_handle_uploads_and_submits_attachment() {
|
||||||
let root = tempfile::tempdir().unwrap();
|
let (root, runtime_base) = owned_relative_runtime_base();
|
||||||
let manifest = WorkerManifest::from_toml(MANIFEST_TOML).unwrap();
|
let manifest = WorkerManifest::from_toml(MANIFEST_TOML).unwrap();
|
||||||
let session = session_store::WorkerSessionStore::new(root.path().join("session")).unwrap();
|
let session = session_store::WorkerSessionStore::new(root.path().join("session")).unwrap();
|
||||||
let store = CombinedStore::new(
|
let store = CombinedStore::new(
|
||||||
@@ -353,7 +371,7 @@ async fn canonical_worker_handle_uploads_and_submits_attachment() {
|
|||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let (handle, shutdown_rx) = WorkerController::spawn(worker, &root.path().join("runtime"))
|
let (handle, shutdown_rx) = WorkerController::spawn(worker, &runtime_base)
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let mut rx = handle.subscribe();
|
let mut rx = handle.subscribe();
|
||||||
@@ -762,6 +780,227 @@ async fn controller_startup_failure_closes_bound_workdir_session() {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
mod registry_startup_failure {
|
||||||
|
//! Connect real controller preparation, registry source fences and session cleanup.
|
||||||
|
//! The Workdir telemetry boundary substitutes a concurrent metadata change at subscription;
|
||||||
|
//! the real registry must reject drift before starting its prepared observer.
|
||||||
|
use super::*;
|
||||||
|
use session_store::WorkerMetadataStore;
|
||||||
|
use tokio::sync::broadcast;
|
||||||
|
use workdir::WorkdirSession;
|
||||||
|
|
||||||
|
struct ObservedSession {
|
||||||
|
local: LocalWorkdirSession,
|
||||||
|
metadata_store: TestStore,
|
||||||
|
worker_name: String,
|
||||||
|
events: broadcast::Sender<workdir::CommandEvent>,
|
||||||
|
subscriptions: AtomicUsize,
|
||||||
|
receivers_at_close: Mutex<Vec<usize>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Debug for ObservedSession {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
f.debug_struct("ObservedSession")
|
||||||
|
.field("worker_name", &self.worker_name)
|
||||||
|
.finish_non_exhaustive()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl WorkdirSession for ObservedSession {
|
||||||
|
fn workdir(&self) -> &Workdir {
|
||||||
|
self.local.workdir()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn capabilities(&self) -> WorkdirSessionCapabilities {
|
||||||
|
self.local.capabilities()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn stat(
|
||||||
|
&self,
|
||||||
|
request: workdir::StatRequest,
|
||||||
|
) -> Result<workdir::StatResult, WorkdirError> {
|
||||||
|
self.local.stat(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn read(
|
||||||
|
&self,
|
||||||
|
request: workdir::ReadRequest,
|
||||||
|
) -> Result<workdir::ReadResult, WorkdirError> {
|
||||||
|
self.local.read(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn write(
|
||||||
|
&self,
|
||||||
|
request: workdir::WriteRequest,
|
||||||
|
) -> Result<workdir::WriteResult, WorkdirError> {
|
||||||
|
self.local.write(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn edit(
|
||||||
|
&self,
|
||||||
|
request: workdir::EditRequest,
|
||||||
|
) -> Result<workdir::EditResult, WorkdirError> {
|
||||||
|
self.local.edit(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn list(
|
||||||
|
&self,
|
||||||
|
request: workdir::ListRequest,
|
||||||
|
) -> Result<workdir::ListResult, WorkdirError> {
|
||||||
|
self.local.list(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn glob(
|
||||||
|
&self,
|
||||||
|
request: workdir::GlobRequest,
|
||||||
|
) -> Result<workdir::GlobResult, WorkdirError> {
|
||||||
|
self.local.glob(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn grep(
|
||||||
|
&self,
|
||||||
|
request: workdir::GrepRequest,
|
||||||
|
) -> Result<workdir::GrepResult, WorkdirError> {
|
||||||
|
self.local.grep(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn start_command(
|
||||||
|
&self,
|
||||||
|
request: CommandRequest,
|
||||||
|
) -> Result<workdir::CommandHandle, WorkdirError> {
|
||||||
|
self.local.start_command(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn command_status(
|
||||||
|
&self,
|
||||||
|
handle: workdir::CommandHandle,
|
||||||
|
) -> Result<workdir::CommandStatus, WorkdirError> {
|
||||||
|
self.local.command_status(handle).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn command_output(
|
||||||
|
&self,
|
||||||
|
request: CommandOutputRequest,
|
||||||
|
) -> Result<workdir::CommandOutput, WorkdirError> {
|
||||||
|
self.local.command_output(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn cancel_command(&self, handle: workdir::CommandHandle) -> Result<(), WorkdirError> {
|
||||||
|
self.local.cancel_command(handle).await
|
||||||
|
}
|
||||||
|
|
||||||
|
fn subscribe_command_events(&self) -> Option<broadcast::Receiver<workdir::CommandEvent>> {
|
||||||
|
// Interleave a canonical change after read-only registry preparation.
|
||||||
|
let mut metadata = self
|
||||||
|
.metadata_store
|
||||||
|
.read_by_name(&self.worker_name)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap_or_else(|| session_store::WorkerMetadata::new(&self.worker_name, None));
|
||||||
|
metadata
|
||||||
|
.spawned_children
|
||||||
|
.push(session_store::WorkerSpawnedChild {
|
||||||
|
worker_name: "legacy-drift".into(),
|
||||||
|
socket_path: PathBuf::new(),
|
||||||
|
callback_address: PathBuf::new(),
|
||||||
|
scope_delegated: vec![],
|
||||||
|
});
|
||||||
|
self.metadata_store.write(&metadata).unwrap();
|
||||||
|
self.subscriptions.fetch_add(1, Ordering::SeqCst);
|
||||||
|
Some(self.events.subscribe())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn close(&self) -> Result<(), WorkdirError> {
|
||||||
|
// Record before any await: aborting without joining must not pass by getting a
|
||||||
|
// scheduler turn during the provider's asynchronous close. The sender stays alive
|
||||||
|
// even after close, so channel closure cannot accidentally dispose of the observer.
|
||||||
|
self.receivers_at_close
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.push(self.events.receiver_count());
|
||||||
|
self.local.close().await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "current_thread")]
|
||||||
|
async fn registry_drift_drops_prepared_observer_before_closing_session() {
|
||||||
|
let root = tempfile::tempdir().unwrap();
|
||||||
|
let run_dir = root.path().join("runs/1");
|
||||||
|
let client = MockClient::new(simple_text_events());
|
||||||
|
let manifest = WorkerManifest::from_toml(MANIFEST_TOML).unwrap();
|
||||||
|
let store = CombinedStore::new(
|
||||||
|
FsStore::new(root.path().join("sessions")).unwrap(),
|
||||||
|
FsWorkerStore::new(root.path().join("workers")).unwrap(),
|
||||||
|
);
|
||||||
|
let engine =
|
||||||
|
Engine::<_, agen::state::Mutable, worker::SessionHistoryMetadata>::new_annotated(
|
||||||
|
client.clone(),
|
||||||
|
);
|
||||||
|
let mut worker = Worker::new(
|
||||||
|
manifest,
|
||||||
|
engine,
|
||||||
|
store.clone(),
|
||||||
|
WorkerWorkspaceContext::local_filesystem(None),
|
||||||
|
WorkerFilesystemAuthority::local(root.path().into(), root.path().into()),
|
||||||
|
manifest::Scope::writable(root.path()).unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let (events, unused_receiver) = broadcast::channel(8);
|
||||||
|
drop(unused_receiver);
|
||||||
|
let session = Arc::new(ObservedSession {
|
||||||
|
local: LocalWorkdirSession::materialized_bound(
|
||||||
|
Workdir::new("registry-startup-failure"),
|
||||||
|
root.path().into(),
|
||||||
|
root.path().into(),
|
||||||
|
worker.scope().clone(),
|
||||||
|
WorkdirSessionCapabilities::ALL,
|
||||||
|
),
|
||||||
|
metadata_store: store,
|
||||||
|
worker_name: worker.manifest().worker.name.clone(),
|
||||||
|
events,
|
||||||
|
subscriptions: AtomicUsize::new(0),
|
||||||
|
receivers_at_close: Mutex::new(Vec::new()),
|
||||||
|
});
|
||||||
|
worker.bind_single_workdir_session(Some(session.clone()));
|
||||||
|
let sessions = worker.workdir_sessions();
|
||||||
|
|
||||||
|
let result = tokio::time::timeout(
|
||||||
|
std::time::Duration::from_secs(5),
|
||||||
|
WorkerController::spawn_runtime_managed_run(worker, &run_dir),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("registry rejection and cleanup must settle");
|
||||||
|
let error = match result {
|
||||||
|
Err(error) => error,
|
||||||
|
Ok(_) => panic!("registry source drift must reject controller startup"),
|
||||||
|
};
|
||||||
|
assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
|
||||||
|
assert_eq!(session.subscriptions.load(Ordering::SeqCst), 1);
|
||||||
|
assert_eq!(*session.receivers_at_close.lock().unwrap(), vec![0]);
|
||||||
|
assert_eq!(session.events.receiver_count(), 0);
|
||||||
|
assert!(sessions.aliases().is_empty());
|
||||||
|
// No controller retains the Worker/Engine after the failed startup.
|
||||||
|
assert_eq!(Arc::strong_count(&client.call_count), 1);
|
||||||
|
assert_eq!(client.call_count.load(Ordering::SeqCst), 0);
|
||||||
|
assert!(!run_dir.join("worker.sock").exists());
|
||||||
|
assert!(!run_dir.join("status.json").exists());
|
||||||
|
assert!(!run_dir.join("manifest.toml").exists());
|
||||||
|
assert!(matches!(
|
||||||
|
session
|
||||||
|
.start_command(CommandRequest {
|
||||||
|
command: "printf unreachable".to_owned(),
|
||||||
|
timeout_secs: 5,
|
||||||
|
transport_byte_budget: workdir::MAX_COMMAND_TRANSPORT_BYTES,
|
||||||
|
cwd: workdir::WorkdirPath::root(),
|
||||||
|
tool_call_id: None,
|
||||||
|
})
|
||||||
|
.await,
|
||||||
|
Err(ref error) if matches!(error.classification_source(), WorkdirError::Unavailable(_))
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn wait_for_status(handle: &WorkerHandle, status: WorkerStatus) {
|
async fn wait_for_status(handle: &WorkerHandle, status: WorkerStatus) {
|
||||||
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
|
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
|
||||||
loop {
|
loop {
|
||||||
@@ -1561,11 +1800,12 @@ permission = "write"
|
|||||||
"#;
|
"#;
|
||||||
let client = MockClient::new(simple_text_events());
|
let client = MockClient::new(simple_text_events());
|
||||||
let worker = make_worker_with_pwd_and_manifest(client, manifest).await.0;
|
let worker = make_worker_with_pwd_and_manifest(client, manifest).await.0;
|
||||||
let tmp = tempfile::tempdir().unwrap();
|
let (_tmp, runtime_base) = owned_relative_runtime_base();
|
||||||
let result = WorkerController::spawn(worker, tmp.path()).await;
|
let result = WorkerController::spawn(worker, &runtime_base).await;
|
||||||
assert!(
|
assert!(
|
||||||
result.is_ok(),
|
result.is_ok(),
|
||||||
"feature exposure must not imply delegation authority"
|
"feature exposure must not imply delegation authority: {:?}",
|
||||||
|
result.as_ref().err()
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -10,6 +10,10 @@ autobins = false
|
|||||||
name = "yoi-server"
|
name = "yoi-server"
|
||||||
path = "src/main.rs"
|
path = "src/main.rs"
|
||||||
|
|
||||||
|
[features]
|
||||||
|
# Offline fixture preparation only; build product binaries without this feature.
|
||||||
|
test-support = []
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
api-macros = { workspace = true, features = ["axum"] }
|
api-macros = { workspace = true, features = ["axum"] }
|
||||||
async-trait.workspace = true
|
async-trait.workspace = true
|
||||||
|
|||||||
@@ -477,7 +477,8 @@ fn tracking_store_real_schema91_to92_retains_queue_cases_principals_audit_receip
|
|||||||
(100, "Provisional users and scoped passkey activation".into()),
|
(100, "Provisional users and scoped passkey activation".into()),
|
||||||
(101, "Durable explicit Subject body release".into()),
|
(101, "Durable explicit Subject body release".into()),
|
||||||
(102, "Durable failed Subject creation recovery".into()),
|
(102, "Durable failed Subject creation recovery".into()),
|
||||||
(103, "Job input retirement independent of immutable receipts".into())
|
(103, "Passkey metadata and atomic addition results".into()),
|
||||||
|
(104, "Converge Passkey results and Job input retention".into())
|
||||||
]
|
]
|
||||||
);
|
);
|
||||||
for (table, rows) in retained.iter().zip(original.iter()) {
|
for (table, rows) in retained.iter().zip(original.iter()) {
|
||||||
|
|||||||
@@ -228,13 +228,21 @@ fn resolve_whoami_sync(
|
|||||||
use server_api::{BrowserSessionState, WhoamiResponse};
|
use server_api::{BrowserSessionState, WhoamiResponse};
|
||||||
let anonymous = || WhoamiResponse {
|
let anonymous = || WhoamiResponse {
|
||||||
actor: None,
|
actor: None,
|
||||||
|
authentication_reference: None,
|
||||||
session: BrowserSessionState::Anonymous {},
|
session: BrowserSessionState::Anonymous {},
|
||||||
};
|
};
|
||||||
// Preserve the existing API-token precedence and fallback rules, including expired tokens.
|
// Preserve the existing API-token precedence and fallback rules, including expired tokens.
|
||||||
if let Some((actor, expires_at)) =
|
if let Some((actor, expires_at)) =
|
||||||
resolve_credentials_sync(store, bearer, session_token.clone())?
|
resolve_credentials_sync(store, bearer.clone(), session_token.clone())?
|
||||||
{
|
{
|
||||||
|
let credential = match actor.auth_method {
|
||||||
|
ActorAuthMethod::ApiToken => bearer.as_deref(),
|
||||||
|
ActorAuthMethod::BrowserSession => session_token.as_deref(),
|
||||||
|
}
|
||||||
|
.ok_or_else(|| auth_error("auth_required", "authentication binding missing"))?;
|
||||||
|
let authentication_reference = Some(authentication_reference(&token_hash(credential)));
|
||||||
return Ok(WhoamiResponse {
|
return Ok(WhoamiResponse {
|
||||||
|
authentication_reference,
|
||||||
actor: Some(actor),
|
actor: Some(actor),
|
||||||
session: BrowserSessionState::Authenticated { expires_at },
|
session: BrowserSessionState::Authenticated { expires_at },
|
||||||
});
|
});
|
||||||
@@ -255,6 +263,7 @@ fn resolve_whoami_sync(
|
|||||||
if within_session_expiry_grace(&session.expires_at, Utc::now()) {
|
if within_session_expiry_grace(&session.expires_at, Utc::now()) {
|
||||||
return Ok(WhoamiResponse {
|
return Ok(WhoamiResponse {
|
||||||
actor: None,
|
actor: None,
|
||||||
|
authentication_reference: None,
|
||||||
session: BrowserSessionState::Expired {},
|
session: BrowserSessionState::Expired {},
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -262,6 +271,13 @@ fn resolve_whoami_sync(
|
|||||||
Ok(anonymous())
|
Ok(anonymous())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Domain-separated public comparison value; never accepted as a credential.
|
||||||
|
pub(crate) fn authentication_reference(credential_hash: &str) -> String {
|
||||||
|
token_hash(&format!(
|
||||||
|
"passkey-authentication-reference:{credential_hash}"
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
fn within_session_expiry_grace(expires_at: &str, now: DateTime<Utc>) -> bool {
|
fn within_session_expiry_grace(expires_at: &str, now: DateTime<Utc>) -> bool {
|
||||||
DateTime::parse_from_rfc3339(expires_at).is_ok_and(|expires_at| {
|
DateTime::parse_from_rfc3339(expires_at).is_ok_and(|expires_at| {
|
||||||
expires_at <= now && now < expires_at + Duration::seconds(SESSION_EXPIRY_GRACE_SECONDS)
|
expires_at <= now && now < expires_at + Duration::seconds(SESSION_EXPIRY_GRACE_SECONDS)
|
||||||
|
|||||||
@@ -4194,6 +4194,7 @@ pub struct RemoteWorkerRuntime {
|
|||||||
host_id: String,
|
host_id: String,
|
||||||
resource_broker: BackendResourceBroker,
|
resource_broker: BackendResourceBroker,
|
||||||
http: BlockingHttpClient,
|
http: BlockingHttpClient,
|
||||||
|
workdir_cleanup_http: BlockingHttpClient,
|
||||||
async_http: AsyncHttpClient,
|
async_http: AsyncHttpClient,
|
||||||
request_timeout: Duration,
|
request_timeout: Duration,
|
||||||
}
|
}
|
||||||
@@ -4346,15 +4347,20 @@ impl RemoteWorkerRuntime {
|
|||||||
};
|
};
|
||||||
let timeout = config.timeout;
|
let timeout = config.timeout;
|
||||||
let blocking_resolution = pinned_endpoint.clone();
|
let blocking_resolution = pinned_endpoint.clone();
|
||||||
let http = run_blocking_http(move || {
|
let (http, workdir_cleanup_http) = run_blocking_http(move || {
|
||||||
let mut builder = BlockingHttpClient::builder()
|
let build_client = |timeout: Option<Duration>| {
|
||||||
.timeout(timeout)
|
let mut builder = BlockingHttpClient::builder()
|
||||||
.redirect(reqwest::redirect::Policy::none())
|
.timeout(timeout)
|
||||||
.no_proxy();
|
.redirect(reqwest::redirect::Policy::none())
|
||||||
if let Some((host, addresses)) = &blocking_resolution {
|
.no_proxy();
|
||||||
builder = builder.resolve_to_addrs(host, addresses);
|
if let Some((host, addresses)) = &blocking_resolution {
|
||||||
}
|
builder = builder.resolve_to_addrs(host, addresses);
|
||||||
builder.build()
|
}
|
||||||
|
builder.build()
|
||||||
|
};
|
||||||
|
// Cleanup duration depends on the retained tree. A response deadline
|
||||||
|
// does not cancel Runtime deletion, so it must not bound this wait.
|
||||||
|
Ok::<_, reqwest::Error>((build_client(Some(timeout))?, build_client(None)?))
|
||||||
})
|
})
|
||||||
.map_err(|err| RuntimeRegistryError::RuntimeOperationFailed {
|
.map_err(|err| RuntimeRegistryError::RuntimeOperationFailed {
|
||||||
runtime_id: config.runtime_id.clone(),
|
runtime_id: config.runtime_id.clone(),
|
||||||
@@ -4393,6 +4399,7 @@ impl RemoteWorkerRuntime {
|
|||||||
cached_status: config.cached_status,
|
cached_status: config.cached_status,
|
||||||
resource_broker: BackendResourceBroker::default(),
|
resource_broker: BackendResourceBroker::default(),
|
||||||
http,
|
http,
|
||||||
|
workdir_cleanup_http,
|
||||||
async_http,
|
async_http,
|
||||||
request_timeout: timeout,
|
request_timeout: timeout,
|
||||||
})
|
})
|
||||||
@@ -5113,6 +5120,9 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
|
|||||||
runtime_api::WorkerRestoreState::RolledBack => {
|
runtime_api::WorkerRestoreState::RolledBack => {
|
||||||
server_api::WorkerRestoreState::RolledBack
|
server_api::WorkerRestoreState::RolledBack
|
||||||
}
|
}
|
||||||
|
runtime_api::WorkerRestoreState::Superseded => {
|
||||||
|
server_api::WorkerRestoreState::Superseded
|
||||||
|
}
|
||||||
runtime_api::WorkerRestoreState::ReconciliationRequired => {
|
runtime_api::WorkerRestoreState::ReconciliationRequired => {
|
||||||
server_api::WorkerRestoreState::ReconciliationRequired
|
server_api::WorkerRestoreState::ReconciliationRequired
|
||||||
}
|
}
|
||||||
@@ -5322,9 +5332,13 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
|
|||||||
&self,
|
&self,
|
||||||
working_directory_id: &str,
|
working_directory_id: &str,
|
||||||
) -> RuntimeWorkingDirectoryResult {
|
) -> RuntimeWorkingDirectoryResult {
|
||||||
match self.delete_json::<RuntimeHttpWorkingDirectoryResponse>(&format!(
|
let path = format!("/v1/working-directories/{working_directory_id}");
|
||||||
"/v1/working-directories/{working_directory_id}"
|
match self.send_json::<RuntimeHttpWorkingDirectoryResponse>(
|
||||||
)) {
|
&path,
|
||||||
|
"DELETE",
|
||||||
|
&[],
|
||||||
|
self.workdir_cleanup_http.delete(self.endpoint(&path)),
|
||||||
|
) {
|
||||||
Ok(response) => RuntimeWorkingDirectoryResult {
|
Ok(response) => RuntimeWorkingDirectoryResult {
|
||||||
state: InternalWorkerOperationState::Accepted,
|
state: InternalWorkerOperationState::Accepted,
|
||||||
working_directory: Some(response.working_directory),
|
working_directory: Some(response.working_directory),
|
||||||
@@ -6825,6 +6839,7 @@ pub fn placeholder_spawn_response(host_id: impl Into<String>) -> WorkerSpawnResu
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
mod restore_auth_tests;
|
mod restore_auth_tests;
|
||||||
|
mod workdir_cleanup_tests;
|
||||||
mod workspace_auth_tests;
|
mod workspace_auth_tests;
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|||||||
@@ -0,0 +1,95 @@
|
|||||||
|
//! Exercises the production RemoteWorkerRuntime HTTP client against a gated loopback
|
||||||
|
//! provider. Only the provider is replaced; no live Runtime or Workdir is used.
|
||||||
|
//! Runs with the ordinary workspace-server crate tests.
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
use axum::{Json, Router, routing::delete};
|
||||||
|
use tokio::{sync::Notify, task::JoinSet, time::timeout};
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn workdir_cleanup_waits_past_ordinary_request_timeout_for_provider_confirmation() {
|
||||||
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
|
let runtime = RemoteWorkerRuntime::new(
|
||||||
|
RemoteRuntimeConfig::new(
|
||||||
|
"runtime-test",
|
||||||
|
"Test Runtime",
|
||||||
|
format!("http://{}", listener.local_addr().unwrap()),
|
||||||
|
Some("secret-token".into()),
|
||||||
|
)
|
||||||
|
.with_timeout(Duration::from_millis(100)),
|
||||||
|
"workspace-test".into(),
|
||||||
|
"http://127.0.0.1:1".into(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let entered = Arc::new(Notify::new());
|
||||||
|
let release = Arc::new(Notify::new());
|
||||||
|
let provider_entered = entered.clone();
|
||||||
|
let provider_release = release.clone();
|
||||||
|
let app = Router::new().route(
|
||||||
|
"/v1/working-directories/workdir-test",
|
||||||
|
delete(move |headers: axum::http::HeaderMap| async move {
|
||||||
|
assert_eq!(headers[AUTHORIZATION], "Bearer secret-token");
|
||||||
|
assert_eq!(headers["x-yoi-workspace-id"], "workspace-test");
|
||||||
|
provider_entered.notify_one();
|
||||||
|
// The fixture's deadline only bounds test failure/teardown. Completion
|
||||||
|
// is controlled by the test after an ordinary request has timed out.
|
||||||
|
let _ = timeout(Duration::from_secs(5), provider_release.notified()).await;
|
||||||
|
Json(json!({
|
||||||
|
"working_directory": { "summary": {
|
||||||
|
"working_directory_id": "workdir-test",
|
||||||
|
"repository_id": "main",
|
||||||
|
"materializer_kind": "runtime_git_clone",
|
||||||
|
"status": "not_found"
|
||||||
|
}}
|
||||||
|
}))
|
||||||
|
})
|
||||||
|
.get(|| async {
|
||||||
|
let _ = timeout(Duration::from_secs(5), std::future::pending::<()>()).await;
|
||||||
|
Json(json!({}))
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
let mut server = JoinSet::new();
|
||||||
|
server.spawn(async move { axum::serve(listener, app).await.unwrap() });
|
||||||
|
let cleanup_runtime = runtime.clone();
|
||||||
|
let cleanup = tokio::task::spawn_blocking(move || {
|
||||||
|
cleanup_runtime.cleanup_working_directory("workdir-test")
|
||||||
|
});
|
||||||
|
timeout(Duration::from_secs(5), entered.notified())
|
||||||
|
.await
|
||||||
|
.expect("provider received cleanup");
|
||||||
|
|
||||||
|
let ordinary = timeout(
|
||||||
|
Duration::from_secs(5),
|
||||||
|
tokio::task::spawn_blocking(move || runtime.working_directory("workdir-test")),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("ordinary request remains bounded")
|
||||||
|
.unwrap();
|
||||||
|
let cleanup_still_waiting = !cleanup.is_finished();
|
||||||
|
release.notify_one();
|
||||||
|
let removed = timeout(Duration::from_secs(5), cleanup)
|
||||||
|
.await
|
||||||
|
.expect("cleanup receives provider confirmation")
|
||||||
|
.unwrap();
|
||||||
|
server.shutdown().await;
|
||||||
|
|
||||||
|
assert_eq!(ordinary.state, InternalWorkerOperationState::Rejected);
|
||||||
|
assert!(
|
||||||
|
ordinary
|
||||||
|
.diagnostics
|
||||||
|
.iter()
|
||||||
|
.any(|d| d.code == "remote_runtime_timeout")
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
cleanup_still_waiting,
|
||||||
|
"cleanup must not inherit the ordinary timeout"
|
||||||
|
);
|
||||||
|
assert_eq!(removed.state, InternalWorkerOperationState::Accepted);
|
||||||
|
assert!(removed.diagnostics.is_empty());
|
||||||
|
let summary = removed.working_directory.unwrap().summary;
|
||||||
|
assert_eq!(summary.working_directory_id, "workdir-test");
|
||||||
|
assert_eq!(
|
||||||
|
summary.status,
|
||||||
|
workdir::workspace::WorkingDirectoryStatusKind::NotFound
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -381,10 +381,13 @@ async fn implementation_result_v97_to_v98_preserves_legacy_bytes_receipts_and_no
|
|||||||
DROP TABLE ticket_implementation_results;
|
DROP TABLE ticket_implementation_results;
|
||||||
DROP TABLE ticket_implementation_delegations;
|
DROP TABLE ticket_implementation_delegations;
|
||||||
DROP TABLE merge_request_source_observations;
|
DROP TABLE merge_request_source_observations;
|
||||||
|
DROP TRIGGER passkey_metadata_insert;
|
||||||
|
DROP TABLE passkey_addition_receipts;
|
||||||
|
DROP TABLE passkey_metadata;
|
||||||
DROP TRIGGER backend_job_input_retirement;
|
DROP TRIGGER backend_job_input_retirement;
|
||||||
DROP TRIGGER backend_job_identity_immutable;
|
DROP TRIGGER backend_job_identity_immutable;
|
||||||
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
|
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
|
||||||
UPDATE __yoi_schema_migrations SET version=102 WHERE version=103;
|
UPDATE __yoi_schema_migrations SET version=102 WHERE version=104;
|
||||||
DROP TABLE subject_create_recoveries;
|
DROP TABLE subject_create_recoveries;
|
||||||
UPDATE __yoi_schema_migrations SET version=101 WHERE version=102;
|
UPDATE __yoi_schema_migrations SET version=101 WHERE version=102;
|
||||||
DROP TABLE subject_worker_releases;
|
DROP TABLE subject_worker_releases;
|
||||||
@@ -521,10 +524,13 @@ async fn mr_notification_upgrade_recovers_events_not_comments_and_retains_unknow
|
|||||||
// Reconstruct the prior composition before its new business capture existed.
|
// Reconstruct the prior composition before its new business capture existed.
|
||||||
conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture;
|
conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture;
|
||||||
DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations;
|
DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations;
|
||||||
|
DROP TRIGGER passkey_metadata_insert;
|
||||||
|
DROP TABLE passkey_addition_receipts;
|
||||||
|
DROP TABLE passkey_metadata;
|
||||||
DROP TRIGGER backend_job_input_retirement;
|
DROP TRIGGER backend_job_input_retirement;
|
||||||
DROP TRIGGER backend_job_identity_immutable;
|
DROP TRIGGER backend_job_identity_immutable;
|
||||||
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
|
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
|
||||||
UPDATE __yoi_schema_migrations SET version=102 WHERE version=103;
|
UPDATE __yoi_schema_migrations SET version=102 WHERE version=104;
|
||||||
DROP TABLE subject_create_recoveries;
|
DROP TABLE subject_create_recoveries;
|
||||||
UPDATE __yoi_schema_migrations SET version=101 WHERE version=102;
|
UPDATE __yoi_schema_migrations SET version=101 WHERE version=102;
|
||||||
DROP TABLE subject_worker_releases;
|
DROP TABLE subject_worker_releases;
|
||||||
@@ -687,10 +693,13 @@ async fn mr_event_actors_preserve_self_suppression_in_capture_and_backfill() {
|
|||||||
if historical {
|
if historical {
|
||||||
conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture;
|
conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture;
|
||||||
DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations;
|
DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations;
|
||||||
DROP TRIGGER backend_job_input_retirement;
|
DROP TRIGGER passkey_metadata_insert;
|
||||||
|
DROP TABLE passkey_addition_receipts;
|
||||||
|
DROP TABLE passkey_metadata;
|
||||||
|
DROP TRIGGER backend_job_input_retirement;
|
||||||
DROP TRIGGER backend_job_identity_immutable;
|
DROP TRIGGER backend_job_identity_immutable;
|
||||||
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
|
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
|
||||||
UPDATE __yoi_schema_migrations SET version=102 WHERE version=103;
|
UPDATE __yoi_schema_migrations SET version=102 WHERE version=104;
|
||||||
DROP TABLE subject_create_recoveries;
|
DROP TABLE subject_create_recoveries;
|
||||||
UPDATE __yoi_schema_migrations SET version=101 WHERE version=102;
|
UPDATE __yoi_schema_migrations SET version=101 WHERE version=102;
|
||||||
DROP TABLE subject_worker_releases;
|
DROP TABLE subject_worker_releases;
|
||||||
|
|||||||
@@ -19,12 +19,17 @@ pub mod repositories;
|
|||||||
pub mod repository_access;
|
pub mod repository_access;
|
||||||
pub mod repository_source;
|
pub mod repository_source;
|
||||||
pub mod resource_broker;
|
pub mod resource_broker;
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "../../../tests/support/restore_environment.rs"]
|
||||||
|
mod restore_test_environment;
|
||||||
pub mod retention;
|
pub mod retention;
|
||||||
pub mod runtime_settings;
|
pub mod runtime_settings;
|
||||||
pub mod runtime_subscription;
|
pub mod runtime_subscription;
|
||||||
pub mod server;
|
pub mod server;
|
||||||
pub mod skills;
|
pub mod skills;
|
||||||
pub mod store;
|
pub mod store;
|
||||||
|
#[cfg(feature = "test-support")]
|
||||||
|
pub use store::fixture_support as test_support;
|
||||||
pub mod subjektiv;
|
pub mod subjektiv;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod ticket_browse_performance_tests;
|
mod ticket_browse_performance_tests;
|
||||||
@@ -129,6 +134,12 @@ pub enum Error {
|
|||||||
TicketAssignmentConflict(String),
|
TicketAssignmentConflict(String),
|
||||||
#[error("Workdir attachment conflict: {0}")]
|
#[error("Workdir attachment conflict: {0}")]
|
||||||
WorkdirAttachmentConflict(String),
|
WorkdirAttachmentConflict(String),
|
||||||
|
#[error("Workdir {workdir_id} is already attached to Worker {runtime_id}:{worker_id}")]
|
||||||
|
WorkdirOccupied {
|
||||||
|
workdir_id: String,
|
||||||
|
runtime_id: String,
|
||||||
|
worker_id: String,
|
||||||
|
},
|
||||||
#[error("Workspace permission denied: {0}")]
|
#[error("Workspace permission denied: {0}")]
|
||||||
WorkspacePermissionDenied(String),
|
WorkspacePermissionDenied(String),
|
||||||
#[error("Workspace config update conflict: {0}")]
|
#[error("Workspace config update conflict: {0}")]
|
||||||
|
|||||||
@@ -0,0 +1,18 @@
|
|||||||
|
CREATE TABLE passkey_metadata (
|
||||||
|
credential_id TEXT PRIMARY KEY REFERENCES passkey_credentials(credential_id) ON DELETE CASCADE,
|
||||||
|
key_reference TEXT NOT NULL UNIQUE,
|
||||||
|
normal_login_verified_at TEXT
|
||||||
|
);
|
||||||
|
INSERT INTO passkey_metadata(credential_id, key_reference)
|
||||||
|
SELECT credential_id, lower(hex(randomblob(16))) FROM passkey_credentials;
|
||||||
|
CREATE TRIGGER passkey_metadata_insert AFTER INSERT ON passkey_credentials
|
||||||
|
BEGIN
|
||||||
|
INSERT INTO passkey_metadata(credential_id, key_reference)
|
||||||
|
VALUES (NEW.credential_id, lower(hex(randomblob(16))));
|
||||||
|
END;
|
||||||
|
CREATE TABLE passkey_addition_receipts (
|
||||||
|
challenge_id TEXT PRIMARY KEY,
|
||||||
|
credential_id TEXT NOT NULL UNIQUE REFERENCES passkey_credentials(credential_id) ON DELETE CASCADE,
|
||||||
|
user_id TEXT NOT NULL REFERENCES users(user_id) ON DELETE CASCADE,
|
||||||
|
authentication_reference TEXT NOT NULL
|
||||||
|
);
|
||||||
@@ -325,7 +325,7 @@ impl SqliteWorkspaceStore {
|
|||||||
let now = Utc::now().to_rfc3339();
|
let now = Utc::now().to_rfc3339();
|
||||||
self.with_conn_mut(|conn| {
|
self.with_conn_mut(|conn| {
|
||||||
let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||||
let policy=load_removal_policy(&tx,&req.workspace_id,&req.worker,erased_journal)?.ok_or_else(|| StoreError::InvalidInput(format!("policy-missing:{}",req.workspace_id)))?;
|
let policy=load_removal_policy(&tx,&req.workspace_id,&req.worker,erased_journal,true)?.ok_or_else(|| StoreError::InvalidInput(format!("policy-missing:{}",req.workspace_id)))?;
|
||||||
let worker=match load_worker(&tx,&req.workspace_id,&req.worker)? {
|
let worker=match load_worker(&tx,&req.workspace_id,&req.worker)? {
|
||||||
Some(v)=>v,
|
Some(v)=>v,
|
||||||
None=>{
|
None=>{
|
||||||
@@ -371,7 +371,7 @@ impl SqliteWorkspaceStore {
|
|||||||
return Err(StoreError::InvalidInput(format!("stale:{plan_id}:plan state {} is not executable", state_s(plan.state))));
|
return Err(StoreError::InvalidInput(format!("stale:{plan_id}:plan state {} is not executable", state_s(plan.state))));
|
||||||
}
|
}
|
||||||
if !plan.blockers.is_empty(){return Err(StoreError::InvalidInput(format!("blocked:{}",serde_json::to_string(&plan.blockers).unwrap())));}
|
if !plan.blockers.is_empty(){return Err(StoreError::InvalidInput(format!("blocked:{}",serde_json::to_string(&plan.blockers).unwrap())));}
|
||||||
let policy=load_removal_policy(&tx,workspace_id,&plan.worker,plan.policy_id==ERASED_JOURNAL_POLICY_ID)?.ok_or_else(||StoreError::InvalidInput(format!("policy-missing:{workspace_id}")))?;
|
let policy=load_removal_policy(&tx,workspace_id,&plan.worker,plan.policy_id==ERASED_JOURNAL_POLICY_ID,plan.policy_id=="backend-subject-removal-archive-v1")?.ok_or_else(||StoreError::InvalidInput(format!("policy-missing:{workspace_id}")))?;
|
||||||
if policy.policy_id!=plan.policy_id||policy.policy_digest!=plan.policy_digest{
|
if policy.policy_id!=plan.policy_id||policy.policy_digest!=plan.policy_digest{
|
||||||
mark_stale(&tx,&plan,"policy content changed")?;tx.commit()?;
|
mark_stale(&tx,&plan,"policy content changed")?;tx.commit()?;
|
||||||
return Err(stale_error(&plan,"policy content changed"));
|
return Err(stale_error(&plan,"policy content changed"));
|
||||||
@@ -1020,6 +1020,7 @@ fn load_removal_policy(
|
|||||||
w: &str,
|
w: &str,
|
||||||
worker: &RuntimeWorkerRef,
|
worker: &RuntimeWorkerRef,
|
||||||
erased_journal: bool,
|
erased_journal: bool,
|
||||||
|
preserve_subject_history: bool,
|
||||||
) -> crate::Result<Option<WorkerRetentionPolicy>> {
|
) -> crate::Result<Option<WorkerRetentionPolicy>> {
|
||||||
let Some(mut policy) = load_policy(c, w)? else {
|
let Some(mut policy) = load_policy(c, w)? else {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
@@ -1044,6 +1045,25 @@ fn load_removal_policy(
|
|||||||
policy.archive_retention = update.archive_retention;
|
policy.archive_retention = update.archive_retention;
|
||||||
policy.diagnostics_disposition = update.diagnostics_disposition;
|
policy.diagnostics_disposition = update.diagnostics_disposition;
|
||||||
policy.diagnostics_retention_seconds = None;
|
policy.diagnostics_retention_seconds = None;
|
||||||
|
} else if preserve_subject_history && policy.session_disposition == SessionDisposition::Purge && c.query_row(
|
||||||
|
"SELECT EXISTS(SELECT 1 FROM worker_create_reservations WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND substr(singleton_key,1,10)='subjektiv:')",
|
||||||
|
params![w, worker.runtime_id, worker.worker_id],
|
||||||
|
|r| r.get::<_, bool>(0),
|
||||||
|
)? {
|
||||||
|
// Removing a Subject body releases occupancy, not its original history.
|
||||||
|
// A configured Purge must not silently erase an unfinished Journal. Explicit
|
||||||
|
// Session erasure uses the separate, proof-bound branch above.
|
||||||
|
let update = WorkerRetentionPolicyUpdate {
|
||||||
|
policy_id: "backend-subject-removal-archive-v1".into(),
|
||||||
|
session_disposition: SessionDisposition::Archive,
|
||||||
|
metadata_disposition: policy.metadata_disposition,
|
||||||
|
archive_retention: policy.archive_retention,
|
||||||
|
diagnostics_disposition: policy.diagnostics_disposition,
|
||||||
|
diagnostics_retention_seconds: policy.diagnostics_retention_seconds,
|
||||||
|
};
|
||||||
|
policy.policy_id = update.policy_id.clone();
|
||||||
|
policy.policy_digest = worker_retention_policy_digest(&update);
|
||||||
|
policy.session_disposition = update.session_disposition;
|
||||||
}
|
}
|
||||||
Ok(Some(policy))
|
Ok(Some(policy))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,240 @@
|
|||||||
|
// Included in store::tests. Exercise the production migration entrypoints on
|
||||||
|
// isolated SQLite files; both published branches are built from retained DDL.
|
||||||
|
mod schema_convergence {
|
||||||
|
use super::*;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
fn variant_fixture(job_variant: bool, baseline_only: bool) -> (tempfile::TempDir, PathBuf) {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = dir.path().join("schema-103.sqlite");
|
||||||
|
prepare_retained_schema(&path, 102);
|
||||||
|
let conn = Connection::open(&path).unwrap();
|
||||||
|
configure_sqlite(&conn).unwrap();
|
||||||
|
if job_variant {
|
||||||
|
(JOB_INPUT_RETENTION_SCHEMA_103.apply)(&conn).unwrap();
|
||||||
|
} else {
|
||||||
|
migrate_passkey_results_v102_to_v103(&conn).unwrap();
|
||||||
|
}
|
||||||
|
if baseline_only {
|
||||||
|
conn.execute_batch("DELETE FROM __yoi_schema_migrations;
|
||||||
|
INSERT INTO __yoi_schema_migrations(version,name) VALUES(103,'workspace schema baseline');").unwrap();
|
||||||
|
}
|
||||||
|
conn.execute_batch(r#"
|
||||||
|
INSERT INTO accounts(account_id,kind,handle,display_name,created_at,updated_at)
|
||||||
|
VALUES('owner','user','owner','Owner','1','2');
|
||||||
|
INSERT INTO users(user_id,account_id,handle,display_name,created_at,updated_at)
|
||||||
|
VALUES('user','owner','owner','Owner','1','2');
|
||||||
|
INSERT INTO workspaces(workspace_id,owner_account_id,display_name,state,created_at,updated_at)
|
||||||
|
VALUES('workspace','owner','Workspace','active','1','2');
|
||||||
|
INSERT INTO passkey_credentials(credential_id,user_id,public_key_cose,transports_json,sign_count,created_at,last_used_at)
|
||||||
|
VALUES('key','user','preserved-public-key','["internal"]',9,'1','2');
|
||||||
|
"#).unwrap();
|
||||||
|
if !job_variant {
|
||||||
|
conn.execute_batch("UPDATE passkey_metadata SET key_reference='stable-reference',normal_login_verified_at='verified';
|
||||||
|
INSERT INTO passkey_addition_receipts(challenge_id,credential_id,user_id,authentication_reference)
|
||||||
|
VALUES('challenge','key','user','saved-authentication-reference');").unwrap();
|
||||||
|
}
|
||||||
|
for disposition in ["available", "reclaimed", "erased", "invalidated"] {
|
||||||
|
let request = resource_job_request(disposition, disposition, "candidate");
|
||||||
|
conn.execute("INSERT INTO backend_jobs(workspace_id,job_id,purpose,input_digest,input_ref,request_json,intent_fingerprint,state,current_attempt,result_json,result_digest,created_at,updated_at,completed_at)
|
||||||
|
VALUES('workspace',?1,?2,?3,?4,?5,?6,'completed',1,'{\"saved\":true}','result-digest','1','2','2')",
|
||||||
|
params![request.job_id, request.purpose, request.input_digest().unwrap(), request.input_ref, serde_json::to_string(&request).unwrap(), request.fingerprint().unwrap()]).unwrap();
|
||||||
|
conn.execute("INSERT INTO backend_job_attempts(workspace_id,job_id,attempt_id,attempt,input_digest,state,deadline_at,result_json,result_digest,created_at,updated_at,completed_at)
|
||||||
|
VALUES('workspace',?1,?1,1,?2,'completed','deadline','{\"saved\":true}','result-digest','1','2','2')",
|
||||||
|
params![request.job_id, request.input_digest().unwrap()]).unwrap();
|
||||||
|
if job_variant && disposition != "available" {
|
||||||
|
conn.execute("UPDATE backend_jobs SET request_json=json_set(request_json,'$.input',json('null')),input_disposition=?1 WHERE job_id=?1", [disposition]).unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
verify_schema_history(&conn, 103).unwrap();
|
||||||
|
(dir, path)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn feature_rows(conn: &Connection, job_variant: bool) -> Vec<Vec<Vec<rusqlite::types::Value>>> {
|
||||||
|
let mut tables = vec!["passkey_credentials", "backend_job_attempts"];
|
||||||
|
if !job_variant {
|
||||||
|
tables.extend(["passkey_metadata", "passkey_addition_receipts"]);
|
||||||
|
}
|
||||||
|
let mut rows: Vec<_> = tables
|
||||||
|
.into_iter()
|
||||||
|
.map(|table| stored_fixture_rows(conn, table))
|
||||||
|
.collect();
|
||||||
|
// A Passkey-branch Job gains only input_disposition; all old columns must
|
||||||
|
// remain byte-for-byte equal, including body, digest and result receipts.
|
||||||
|
rows.push(fixture_query_rows(conn, "SELECT workspace_id,job_id,purpose,input_digest,input_ref,resource_key,request_json,intent_fingerprint,state,current_attempt,result_json,result_digest,failure_category,failure_detail,created_at,updated_at,completed_at FROM backend_jobs ORDER BY job_id"));
|
||||||
|
if job_variant {
|
||||||
|
rows.push(fixture_query_rows(
|
||||||
|
conn,
|
||||||
|
"SELECT job_id,input_disposition FROM backend_jobs ORDER BY job_id",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
rows
|
||||||
|
}
|
||||||
|
|
||||||
|
fn schema_rows(conn: &Connection) -> Vec<Vec<rusqlite::types::Value>> {
|
||||||
|
fixture_query_rows(
|
||||||
|
conn,
|
||||||
|
"SELECT type,name,tbl_name,sql FROM sqlite_schema ORDER BY type,name",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn schema_103_variants_converge_preserving_history_values_and_constraints() {
|
||||||
|
for job_variant in [false, true] {
|
||||||
|
for baseline_only in [false, true] {
|
||||||
|
let (_dir, path) = variant_fixture(job_variant, baseline_only);
|
||||||
|
let conn = Connection::open(&path).unwrap();
|
||||||
|
let history = workspace_schema_migration_history(&conn).unwrap();
|
||||||
|
let rows = feature_rows(&conn, job_variant);
|
||||||
|
let schema = schema_rows(&conn);
|
||||||
|
let plan = SqliteWorkspaceStore::migration_plan(&path).unwrap();
|
||||||
|
assert_eq!(plan.current_schema_version, 103);
|
||||||
|
assert_eq!(
|
||||||
|
plan.migrations,
|
||||||
|
vec![WorkspaceSchemaMigrationStep {
|
||||||
|
version: 104,
|
||||||
|
name: "Converge Passkey results and Job input retention".into(),
|
||||||
|
}]
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
schema_rows(&conn),
|
||||||
|
schema,
|
||||||
|
"preflight must not mutate the source"
|
||||||
|
);
|
||||||
|
assert_eq!(feature_rows(&conn, job_variant), rows);
|
||||||
|
drop(conn);
|
||||||
|
let upgraded = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||||
|
let fresh = SqliteWorkspaceStore::in_memory_for_test().unwrap();
|
||||||
|
upgraded.with_conn(|conn| {
|
||||||
|
assert_eq!(feature_rows(conn, job_variant), rows);
|
||||||
|
let mut expected = history.clone();
|
||||||
|
expected.push((104, "Converge Passkey results and Job input retention".into()));
|
||||||
|
assert_eq!(workspace_schema_migration_history(conn)?, expected);
|
||||||
|
for table in ["passkey_credentials", "passkey_metadata", "passkey_addition_receipts", "backend_jobs", "backend_job_attempts"] {
|
||||||
|
assert_eq!(state_contract_schema_signature(conn, table), fresh.with_conn(|c| Ok(state_contract_schema_signature(c, table)))?, "{table}");
|
||||||
|
}
|
||||||
|
assert_eq!(conn.query_row("SELECT count(*) FROM pragma_foreign_key_check", [], |r| r.get::<_, i64>(0))?, 0);
|
||||||
|
assert!(conn.execute("UPDATE backend_jobs SET input_digest='changed'", []).is_err());
|
||||||
|
assert!(conn.execute("UPDATE backend_jobs SET input_disposition='unsupported'", []).is_err());
|
||||||
|
assert!(conn.execute("UPDATE backend_jobs SET request_json=json_set(request_json,'$.input','replacement')", []).is_err());
|
||||||
|
assert!(conn.execute("INSERT INTO passkey_metadata(credential_id,key_reference) VALUES('missing','new')", []).is_err());
|
||||||
|
assert!(conn.execute("INSERT INTO passkey_addition_receipts VALUES('other','missing','user','reference')", []).is_err());
|
||||||
|
let reference: (String, Option<String>) = conn.query_row("SELECT key_reference,normal_login_verified_at FROM passkey_metadata WHERE credential_id='key'", [], |r| Ok((r.get(0)?,r.get(1)?)))?;
|
||||||
|
if job_variant {
|
||||||
|
assert_eq!(reference.0.len(), 32);
|
||||||
|
assert_eq!(reference.1, None);
|
||||||
|
} else {
|
||||||
|
assert_eq!(reference, ("stable-reference".into(), Some("verified".into())));
|
||||||
|
assert_eq!(conn.query_row("SELECT count(*) FROM backend_jobs WHERE input_disposition='available'", [], |r| r.get::<_, i64>(0))?, 4);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}).unwrap();
|
||||||
|
let summaries = upgraded.list_passkey_summaries("user").unwrap();
|
||||||
|
drop(upgraded);
|
||||||
|
let reopened = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||||
|
assert_eq!(reopened.list_passkey_summaries("user").unwrap(), summaries);
|
||||||
|
reopened
|
||||||
|
.with_conn(|conn| {
|
||||||
|
assert_eq!(feature_rows(conn, job_variant), rows);
|
||||||
|
assert_eq!(
|
||||||
|
workspace_schema_migration_history(conn)?.len(),
|
||||||
|
history.len() + 1
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn schema_103_convergence_failure_rolls_back_both_variants_and_retries() {
|
||||||
|
for job_variant in [false, true] {
|
||||||
|
for baseline_only in [false, true] {
|
||||||
|
let (_dir, path) = variant_fixture(job_variant, baseline_only);
|
||||||
|
let conn = Connection::open(&path).unwrap();
|
||||||
|
conn.execute_batch("CREATE TRIGGER interrupt_convergence BEFORE INSERT ON __yoi_schema_migrations WHEN NEW.version=104 BEGIN SELECT RAISE(ABORT,'injected convergence failure'); END;").unwrap();
|
||||||
|
let schema = schema_rows(&conn);
|
||||||
|
let history = workspace_schema_migration_history(&conn).unwrap();
|
||||||
|
let rows = feature_rows(&conn, job_variant);
|
||||||
|
drop(conn);
|
||||||
|
let error = SqliteWorkspaceStore::open_for_test(&path)
|
||||||
|
.err()
|
||||||
|
.expect("migration must fail");
|
||||||
|
assert!(
|
||||||
|
error.to_string().contains("injected convergence failure"),
|
||||||
|
"{error}"
|
||||||
|
);
|
||||||
|
let conn = Connection::open(&path).unwrap();
|
||||||
|
assert_eq!(schema_rows(&conn), schema);
|
||||||
|
assert_eq!(workspace_schema_migration_history(&conn).unwrap(), history);
|
||||||
|
assert_eq!(feature_rows(&conn, job_variant), rows);
|
||||||
|
conn.execute_batch("DROP TRIGGER interrupt_convergence")
|
||||||
|
.unwrap();
|
||||||
|
drop(conn);
|
||||||
|
let upgraded = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||||
|
upgraded
|
||||||
|
.with_conn(|conn| {
|
||||||
|
assert_eq!(feature_rows(conn, job_variant), rows);
|
||||||
|
assert_eq!(current_schema_version(conn)?, 104);
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn schema_103_unknown_migration_identity_is_rejected_without_changes() {
|
||||||
|
for job_variant in [false, true] {
|
||||||
|
let (_dir, path) = variant_fixture(job_variant, false);
|
||||||
|
let conn = Connection::open(&path).unwrap();
|
||||||
|
conn.execute("UPDATE __yoi_schema_migrations SET name='unrecognized schema 103' WHERE version=103", []).unwrap();
|
||||||
|
let schema = schema_rows(&conn);
|
||||||
|
let history = workspace_schema_migration_history(&conn).unwrap();
|
||||||
|
let rows = feature_rows(&conn, job_variant);
|
||||||
|
assert!(SqliteWorkspaceStore::migration_plan(&path).is_err());
|
||||||
|
assert!(SqliteWorkspaceStore::open_for_test(&path).is_err());
|
||||||
|
assert_eq!(schema_rows(&conn), schema);
|
||||||
|
assert_eq!(workspace_schema_migration_history(&conn).unwrap(), history);
|
||||||
|
assert_eq!(feature_rows(&conn, job_variant), rows);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn schema_103_mismatched_identity_or_incomplete_features_are_rejected_atomically() {
|
||||||
|
for job_variant in [false, true] {
|
||||||
|
for mismatched_identity in [false, true] {
|
||||||
|
let (_dir, path) = variant_fixture(job_variant, false);
|
||||||
|
let conn = Connection::open(&path).unwrap();
|
||||||
|
if mismatched_identity {
|
||||||
|
let other = if job_variant {
|
||||||
|
"Passkey metadata and atomic addition results"
|
||||||
|
} else {
|
||||||
|
"Job input retirement independent of immutable receipts"
|
||||||
|
};
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE __yoi_schema_migrations SET name=?1 WHERE version=103",
|
||||||
|
[other],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
} else {
|
||||||
|
let trigger = if job_variant {
|
||||||
|
"backend_job_identity_immutable"
|
||||||
|
} else {
|
||||||
|
"passkey_metadata_insert"
|
||||||
|
};
|
||||||
|
conn.execute_batch(&format!("DROP TRIGGER {trigger}"))
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
let schema = schema_rows(&conn);
|
||||||
|
let history = workspace_schema_migration_history(&conn).unwrap();
|
||||||
|
let rows = feature_rows(&conn, job_variant);
|
||||||
|
assert!(SqliteWorkspaceStore::open_for_test(&path).is_err());
|
||||||
|
assert_eq!(schema_rows(&conn), schema);
|
||||||
|
assert_eq!(workspace_schema_migration_history(&conn).unwrap(), history);
|
||||||
|
assert_eq!(feature_rows(&conn, job_variant), rows);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,40 +1,41 @@
|
|||||||
// Included in store::tests so both paths use the real retained migration chain.
|
// Included in store::tests so both paths use the real retained migration chain.
|
||||||
|
fn state_contract_schema_signature(conn: &Connection, table: &str) -> Vec<Vec<String>> {
|
||||||
|
let queries = [
|
||||||
|
"SELECT name, type, CAST(\"notnull\" AS TEXT), COALESCE(dflt_value, '<NULL>'), CAST(pk AS TEXT) FROM pragma_table_info(?1) ORDER BY name",
|
||||||
|
"SELECT \"table\", \"from\", \"to\", on_update, on_delete, match, CAST(seq AS TEXT) FROM pragma_foreign_key_list(?1) ORDER BY \"table\", \"from\", \"to\", seq",
|
||||||
|
"SELECT CAST(i.\"unique\" AS TEXT), CAST(i.partial AS TEXT), (SELECT group_concat(name, ',') FROM (SELECT name FROM pragma_index_info(i.name) ORDER BY seqno)) FROM pragma_index_list(?1) AS i ORDER BY 1, 2, 3",
|
||||||
|
"SELECT name, sql FROM sqlite_schema WHERE type='trigger' AND tbl_name=?1 ORDER BY name",
|
||||||
|
];
|
||||||
|
queries
|
||||||
|
.into_iter()
|
||||||
|
.flat_map(|sql| {
|
||||||
|
let mut statement = conn.prepare(sql).unwrap();
|
||||||
|
let count = statement.column_count();
|
||||||
|
statement
|
||||||
|
.query_map([table], |row| {
|
||||||
|
(0..count)
|
||||||
|
.map(|i| row.get::<_, String>(i))
|
||||||
|
.collect::<rusqlite::Result<Vec<_>>>()
|
||||||
|
})
|
||||||
|
.unwrap()
|
||||||
|
.map(|row| {
|
||||||
|
row.unwrap()
|
||||||
|
.into_iter()
|
||||||
|
.map(|value| {
|
||||||
|
value
|
||||||
|
.chars()
|
||||||
|
.filter(|c| !c.is_whitespace() && *c != '"')
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn fresh_and_upgraded_state_contract_schemas_enforce_the_same_constraints() {
|
fn fresh_and_upgraded_state_contract_schemas_enforce_the_same_constraints() {
|
||||||
fn signature(conn: &Connection, table: &str) -> Vec<Vec<String>> {
|
|
||||||
let queries = [
|
|
||||||
"SELECT name, type, CAST(\"notnull\" AS TEXT), COALESCE(dflt_value, '<NULL>'), CAST(pk AS TEXT) FROM pragma_table_info(?1) ORDER BY name",
|
|
||||||
"SELECT \"table\", \"from\", \"to\", on_update, on_delete, match, CAST(seq AS TEXT) FROM pragma_foreign_key_list(?1) ORDER BY \"table\", \"from\", \"to\", seq",
|
|
||||||
"SELECT CAST(i.\"unique\" AS TEXT), CAST(i.partial AS TEXT), (SELECT group_concat(name, ',') FROM (SELECT name FROM pragma_index_info(i.name) ORDER BY seqno)) FROM pragma_index_list(?1) AS i ORDER BY 1, 2, 3",
|
|
||||||
"SELECT name, sql FROM sqlite_schema WHERE type='trigger' AND tbl_name=?1 ORDER BY name",
|
|
||||||
];
|
|
||||||
queries
|
|
||||||
.into_iter()
|
|
||||||
.flat_map(|sql| {
|
|
||||||
let mut statement = conn.prepare(sql).unwrap();
|
|
||||||
let count = statement.column_count();
|
|
||||||
statement
|
|
||||||
.query_map([table], |row| {
|
|
||||||
(0..count)
|
|
||||||
.map(|i| row.get::<_, String>(i))
|
|
||||||
.collect::<rusqlite::Result<Vec<_>>>()
|
|
||||||
})
|
|
||||||
.unwrap()
|
|
||||||
.map(|row| {
|
|
||||||
row.unwrap()
|
|
||||||
.into_iter()
|
|
||||||
.map(|value| {
|
|
||||||
value
|
|
||||||
.chars()
|
|
||||||
.filter(|c| !c.is_whitespace() && *c != '"')
|
|
||||||
.collect()
|
|
||||||
})
|
|
||||||
.collect()
|
|
||||||
})
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
})
|
|
||||||
.collect()
|
|
||||||
}
|
|
||||||
let temp = tempfile::tempdir().unwrap();
|
let temp = tempfile::tempdir().unwrap();
|
||||||
let path = temp.path().join("upgraded.db");
|
let path = temp.path().join("upgraded.db");
|
||||||
prepare_retained_schema(&path, 85);
|
prepare_retained_schema(&path, 85);
|
||||||
@@ -58,6 +59,9 @@ fn fresh_and_upgraded_state_contract_schemas_enforce_the_same_constraints() {
|
|||||||
"workspace_memory_settings",
|
"workspace_memory_settings",
|
||||||
"worker_create_reservations",
|
"worker_create_reservations",
|
||||||
"subject_worker_releases",
|
"subject_worker_releases",
|
||||||
|
"passkey_credentials",
|
||||||
|
"passkey_metadata",
|
||||||
|
"passkey_addition_receipts",
|
||||||
"backend_jobs",
|
"backend_jobs",
|
||||||
"backend_job_attempts",
|
"backend_job_attempts",
|
||||||
"flow_sources",
|
"flow_sources",
|
||||||
@@ -92,9 +96,11 @@ fn fresh_and_upgraded_state_contract_schemas_enforce_the_same_constraints() {
|
|||||||
"workdir_create_legacy_ssh_archives",
|
"workdir_create_legacy_ssh_archives",
|
||||||
"repository_secret_legacy_receipts",
|
"repository_secret_legacy_receipts",
|
||||||
] {
|
] {
|
||||||
let expected = fresh.with_conn(|conn| Ok(signature(conn, table))).unwrap();
|
let expected = fresh
|
||||||
|
.with_conn(|conn| Ok(state_contract_schema_signature(conn, table)))
|
||||||
|
.unwrap();
|
||||||
let actual = upgraded
|
let actual = upgraded
|
||||||
.with_conn(|conn| Ok(signature(conn, table)))
|
.with_conn(|conn| Ok(state_contract_schema_signature(conn, table)))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert!(!expected.is_empty(), "missing canonical table {table}");
|
assert!(!expected.is_empty(), "missing canonical table {table}");
|
||||||
if actual != expected {
|
if actual != expected {
|
||||||
|
|||||||
+1462
-180
File diff suppressed because it is too large
Load Diff
@@ -92,6 +92,20 @@ pub(super) fn prepare(api: &WorkspaceApi, reservation: &BackendJobReservation) -
|
|||||||
};
|
};
|
||||||
let store = store(api)?;
|
let store = store(api)?;
|
||||||
let host = host(api, &store, grant)?;
|
let host = host(api, &store, grant)?;
|
||||||
|
let attr = store
|
||||||
|
.session_attribution(&grant.session_id)
|
||||||
|
.map_err(|e| Error::Store(e.to_string()))?
|
||||||
|
.ok_or_else(|| {
|
||||||
|
Error::WorkspacePermissionDenied("Journal Session attribution missing".into())
|
||||||
|
})?;
|
||||||
|
if store
|
||||||
|
.journal_worker_was_released(&attr.subject_id, &attr.runtime_id, &attr.worker_id)
|
||||||
|
.map_err(domain)?
|
||||||
|
{
|
||||||
|
return Err(Error::WorkspacePermissionDenied(
|
||||||
|
"released Journal body cannot dispatch another attempt".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
let input = saved_input(&reservation.job.request, grant)?;
|
let input = saved_input(&reservation.job.request, grant)?;
|
||||||
let g = store
|
let g = store
|
||||||
.prepare_journal_input(
|
.prepare_journal_input(
|
||||||
@@ -584,6 +598,18 @@ fn dispatch(
|
|||||||
.lock()
|
.lock()
|
||||||
.map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))?;
|
.map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))?;
|
||||||
let store = store(api)?;
|
let store = store(api)?;
|
||||||
|
if store
|
||||||
|
.journal_worker_was_released(
|
||||||
|
&attribution.subject_id,
|
||||||
|
&attribution.runtime_id,
|
||||||
|
&attribution.worker_id,
|
||||||
|
)
|
||||||
|
.map_err(domain)?
|
||||||
|
{
|
||||||
|
// Deletion can retain an unfinished prefix. It must not mint attempts or
|
||||||
|
// transfer the removed body's write capability to its successor.
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
let host =
|
let host =
|
||||||
JournalHostContext::validated_session(&store, attribution.clone()).map_err(domain)?;
|
JournalHostContext::validated_session(&store, attribution.clone()).map_err(domain)?;
|
||||||
if let Some(g) = store
|
if let Some(g) = store
|
||||||
@@ -1740,8 +1766,8 @@ fn finish_erased_release(
|
|||||||
.release_subject_singleton(api.workspace_id(), &lease, binding_id, &operation)
|
.release_subject_singleton(api.workspace_id(), &lease, binding_id, &operation)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Normal deletion cannot substitute for Subject release, even for a stopped or disabled body.
|
/// Ordinary removal owns writer revocation and occupancy release. Before execution
|
||||||
/// Explicit erasure remains a separate monotonic source/diagnostics destruction path.
|
/// only preserve/erase source requirements are checked; an active writer is not a blocker.
|
||||||
pub(super) fn prepare_worker_removal(
|
pub(super) fn prepare_worker_removal(
|
||||||
service: &WorkerRemovalService,
|
service: &WorkerRemovalService,
|
||||||
runtime: &RuntimeRegistry,
|
runtime: &RuntimeRegistry,
|
||||||
@@ -1762,7 +1788,6 @@ pub(super) fn prepare_worker_removal(
|
|||||||
&service.journal_registration,
|
&service.journal_registration,
|
||||||
)
|
)
|
||||||
.map_err(|e| Error::Store(e.to_string()))?;
|
.map_err(|e| Error::Store(e.to_string()))?;
|
||||||
let host = JournalHostContext::validated_session(&store, attr.clone()).map_err(domain)?;
|
|
||||||
let source = runtime_api::SessionPublicSource::Retained {
|
let source = runtime_api::SessionPublicSource::Retained {
|
||||||
worker_id: worker_runtime::identity::WorkerId::parse(&target.worker_id)
|
worker_id: worker_runtime::identity::WorkerId::parse(&target.worker_id)
|
||||||
.ok_or_else(|| Error::Store("Worker identity invalid".into()))?,
|
.ok_or_else(|| Error::Store("Worker identity invalid".into()))?,
|
||||||
@@ -1779,25 +1804,9 @@ pub(super) fn prepare_worker_removal(
|
|||||||
finish_erased_release(&api, &attr)?;
|
finish_erased_release(&api, &attr)?;
|
||||||
return Ok(Some(proof));
|
return Ok(Some(proof));
|
||||||
}
|
}
|
||||||
let Some(binding_id) = host.writer_binding_id() else {
|
// Do not require a live Controller, complete Journal, or prior release here.
|
||||||
return Err(Error::RepositoryConflict(
|
// Runtime removal is the physical fence; finish_worker_removal revokes the
|
||||||
"Subject writer evidence is missing; ordinary deletion cannot bypass release".into(),
|
// exact old body only after its successful, matching retention receipt.
|
||||||
));
|
|
||||||
};
|
|
||||||
let writer = JournalWriterBinding {
|
|
||||||
subject_id: attr.subject_id.clone(),
|
|
||||||
runtime_id: attr.runtime_id.clone(),
|
|
||||||
worker_id: attr.worker_id.clone(),
|
|
||||||
binding_id: binding_id.into(),
|
|
||||||
};
|
|
||||||
if store.journal_writer_active(&writer).map_err(domain)? {
|
|
||||||
return Err(Error::RepositoryConflict("Subject remains occupied: restore to actual Idle, finish Journal recording, then explicitly release before ordinary deletion".into()));
|
|
||||||
}
|
|
||||||
if store.journal_retention_required(&host).map_err(domain)? {
|
|
||||||
return Err(Error::RepositoryConflict(
|
|
||||||
"Released Subject has unresolved Journal material; retain occupancy and source".into(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
Ok(None)
|
Ok(None)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1886,20 +1895,6 @@ pub(super) fn validate_removal_retention(
|
|||||||
// its retained permanent fence, including recovery after the aggregate is gone.
|
// its retained permanent fence, including recovery after the aggregate is gone.
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
let binding_id = host.writer_binding_id().ok_or_else(|| {
|
|
||||||
Error::RepositoryConflict("Subject writer evidence missing during removal recovery".into())
|
|
||||||
})?;
|
|
||||||
let binding = JournalWriterBinding {
|
|
||||||
subject_id: attr.subject_id.clone(),
|
|
||||||
runtime_id: attr.runtime_id.clone(),
|
|
||||||
worker_id: attr.worker_id.clone(),
|
|
||||||
binding_id: binding_id.into(),
|
|
||||||
};
|
|
||||||
if store.journal_writer_active(&binding).map_err(domain)? {
|
|
||||||
return Err(Error::RepositoryConflict(
|
|
||||||
"Prepared removal cannot bypass explicit Subject release".into(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
let source = runtime_api::SessionPublicSource::Retained {
|
let source = runtime_api::SessionPublicSource::Retained {
|
||||||
worker_id: worker_runtime::identity::WorkerId::parse(&target.worker_id)
|
worker_id: worker_runtime::identity::WorkerId::parse(&target.worker_id)
|
||||||
.ok_or_else(|| Error::Store("Worker identity invalid".into()))?,
|
.ok_or_else(|| Error::Store("Worker identity invalid".into()))?,
|
||||||
@@ -1907,37 +1902,140 @@ pub(super) fn validate_removal_retention(
|
|||||||
let runtime = service.runtime.upgrade().ok_or_else(|| {
|
let runtime = service.runtime.upgrade().ok_or_else(|| {
|
||||||
Error::Store("Runtime unavailable for source retention confirmation".into())
|
Error::Store("Runtime unavailable for source retention confirmation".into())
|
||||||
})?;
|
})?;
|
||||||
if store.journal_retention_required(&host).map_err(domain)? {
|
let retention_required = store.journal_retention_required(&host).map_err(domain)?;
|
||||||
if prepared.runtime_request.session_disposition
|
if prepared.runtime_request.session_disposition
|
||||||
!= worker_runtime::retention::SessionDisposition::Archive
|
== worker_runtime::retention::SessionDisposition::Purge
|
||||||
|| prepared.runtime_request.archive_id.is_none()
|
&& prepared.runtime_request.archive_id.is_none()
|
||||||
{
|
&& !retention_required
|
||||||
return Err(Error::RepositoryConflict(
|
&& store
|
||||||
"unreflected Journal requires a retained original Session archive".into(),
|
.journal_worker_was_released(&attr.subject_id, &attr.runtime_id, &attr.worker_id)
|
||||||
));
|
.map_err(domain)?
|
||||||
|
{
|
||||||
|
// Compatibility for an already issued, history-complete ordinary Purge
|
||||||
|
// plan. New Subject plans always Archive. Keep the old fingerprint and
|
||||||
|
// disposition; never turn this into an erasure or allow an active writer.
|
||||||
|
// A missing Runtime Worker is NOT success: execute_worker_retention still
|
||||||
|
// has to return the exact persisted receipt before anything can commit.
|
||||||
|
// A pending receipt may outlive the physical aggregate while Runtime's
|
||||||
|
// in-memory catalog still contains the Worker. Pin release is best effort
|
||||||
|
// in this historical branch; exact Runtime retention replay still enforces
|
||||||
|
// pins if the source exists, and requires a receipt if it is already gone.
|
||||||
|
if let Err(error) = retain_source(&runtime, &service.workspace_id, &attr, source, true) {
|
||||||
|
tracing::debug!(%error, "Historical Purge source unavailable; reconcile exact Runtime receipt");
|
||||||
}
|
}
|
||||||
if let Err(original) = retain_source(&runtime, &service.workspace_id, &attr, source, false)
|
return Ok(());
|
||||||
{
|
}
|
||||||
// Recovery may already have transferred the physical source before its Backend
|
if prepared.runtime_request.session_disposition
|
||||||
// catalog commit; authenticate the exact reserved archive, never another Session.
|
!= worker_runtime::retention::SessionDisposition::Archive
|
||||||
let archive = runtime_api::SessionPublicSource::Archived {
|
|| prepared.runtime_request.archive_id.is_none()
|
||||||
archive_id: prepared
|
{
|
||||||
.runtime_request
|
return Err(Error::RepositoryConflict(
|
||||||
.archive_id
|
"Subject removal retains its original Session; explicit erasure is required for Purge"
|
||||||
.clone()
|
.into(),
|
||||||
.expect("checked"),
|
));
|
||||||
};
|
}
|
||||||
retain_source(&runtime, &service.workspace_id, &attr, archive, false)
|
let release = !retention_required;
|
||||||
.map_err(|_| original)?;
|
if let Err(original) = retain_source(&runtime, &service.workspace_id, &attr, source, release) {
|
||||||
}
|
// A lost reply may follow the physical transfer. Reconcile only the exact
|
||||||
} else {
|
// reserved archive and Session, whether or not an admitted remainder exists.
|
||||||
// This stopped/removal-fenced Session has no admitted remainder; purge need not
|
let archive = runtime_api::SessionPublicSource::Archived {
|
||||||
// force a generation. A failed release conservatively blocks ordinary deletion.
|
archive_id: prepared
|
||||||
retain_source(&runtime, &service.workspace_id, &attr, source, true)?;
|
.runtime_request
|
||||||
|
.archive_id
|
||||||
|
.clone()
|
||||||
|
.expect("checked"),
|
||||||
|
};
|
||||||
|
retain_source(&runtime, &service.workspace_id, &attr, archive, release)
|
||||||
|
.map_err(|_| original)?;
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A successful Runtime receipt, not catalog liveness, authorizes ending the old
|
||||||
|
/// writer. The ordinary removal commit releases occupancy atomically with catalog
|
||||||
|
/// deletion. Until then a crash leaves the singleton occupied and the same receipt
|
||||||
|
/// can safely repeat this step; no Controller restore or Journal generation is needed.
|
||||||
|
pub(super) fn finish_worker_removal<T>(
|
||||||
|
service: &WorkerRemovalService,
|
||||||
|
target: &RuntimeWorkerRef,
|
||||||
|
prepared: &crate::retention::PreparedWorkerRemoval,
|
||||||
|
result: &worker_runtime::retention::WorkerRetentionExecutionResult,
|
||||||
|
commit: impl FnOnce() -> T,
|
||||||
|
) -> Result<T> {
|
||||||
|
let Some(attr) = removal_attribution(service, target)? else {
|
||||||
|
return Ok(commit());
|
||||||
|
};
|
||||||
|
let request = &prepared.runtime_request;
|
||||||
|
if prepared.plan.worker != *target
|
||||||
|
|| request.workspace_id != service.workspace_id
|
||||||
|
|| request.source_runtime_id != target.runtime_id
|
||||||
|
|| request.worker_id.to_string() != target.worker_id
|
||||||
|
|| !result.source_removed
|
||||||
|
|| result.worker_id != request.worker_id
|
||||||
|
|| result.operation_id != request.operation_id
|
||||||
|
|| result.input_fingerprint != request.input_fingerprint
|
||||||
|
|| result.session_disposition != request.session_disposition
|
||||||
|
|| result.diagnostics_disposition != request.diagnostics_disposition
|
||||||
|
|| result.diagnostics_retained
|
||||||
|
!= (request.diagnostics_disposition
|
||||||
|
== worker_runtime::retention::DiagnosticsDisposition::Retain)
|
||||||
|
{
|
||||||
|
return Err(Error::RepositoryConflict(
|
||||||
|
"Subject removal receipt does not match its plan".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if request.session_disposition == worker_runtime::retention::SessionDisposition::Archive {
|
||||||
|
let archive = result.archive.as_ref().ok_or_else(|| {
|
||||||
|
Error::RepositoryConflict("Subject removal archive receipt is missing".into())
|
||||||
|
})?;
|
||||||
|
if Some(&archive.archive_id) != request.archive_id.as_ref()
|
||||||
|
|| archive.workspace_id != request.workspace_id
|
||||||
|
|| archive.source_runtime_id != request.source_runtime_id
|
||||||
|
|| archive.source_worker_id != request.worker_id
|
||||||
|
|| archive.source_session_id != attr.session_id
|
||||||
|
|| archive.operation_id != request.operation_id
|
||||||
|
|| archive.input_fingerprint != request.input_fingerprint
|
||||||
|
|| archive.policy_id != request.policy_id
|
||||||
|
{
|
||||||
|
return Err(Error::RepositoryConflict(
|
||||||
|
"Subject removal archive identity mismatch".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let api = service.journal_dispatch_api.upgrade().ok_or_else(|| {
|
||||||
|
Error::Store("Workspace API unavailable for Subject removal completion".into())
|
||||||
|
})?;
|
||||||
|
let gate = lifecycle_lock(&api, &attr.subject_id)?;
|
||||||
|
let _guard = gate
|
||||||
|
.lock()
|
||||||
|
.map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))?;
|
||||||
|
let store = store(&api)?;
|
||||||
|
let host = JournalHostContext::validated_session(&store, attr.clone()).map_err(domain)?;
|
||||||
|
if store.journal_session_erased(&host).map_err(domain)? {
|
||||||
|
if request.session_disposition == worker_runtime::retention::SessionDisposition::Archive {
|
||||||
|
// Erasure may win after Runtime archive transfer. Do not publish that
|
||||||
|
// archive or skip writer revocation: the same removal retry must take
|
||||||
|
// erased_archive_removal_proof and its physical-erasure commit path.
|
||||||
|
return Err(Error::RepositoryConflict(
|
||||||
|
"Session erased during removal; reconcile the same removal operation".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
return Ok(commit());
|
||||||
|
}
|
||||||
|
store.release_removed_journal_body(&host).map_err(domain)?;
|
||||||
|
for session in store
|
||||||
|
.subject_session_attributions(&attr.subject_id)
|
||||||
|
.map_err(|e| Error::Store(e.to_string()))?
|
||||||
|
.into_iter()
|
||||||
|
.filter(|a| a.runtime_id == target.runtime_id && a.worker_id == target.worker_id)
|
||||||
|
{
|
||||||
|
reconcile_session_jobs(&api, &session, false)?;
|
||||||
|
}
|
||||||
|
// Keep erasure excluded through the catalog commit: otherwise a tombstone
|
||||||
|
// could win between revocation and publication of the retained archive.
|
||||||
|
Ok(commit())
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Deserialize)]
|
#[derive(Deserialize)]
|
||||||
#[serde(deny_unknown_fields)]
|
#[serde(deny_unknown_fields)]
|
||||||
pub(super) struct Empty {}
|
pub(super) struct Empty {}
|
||||||
@@ -2089,7 +2187,28 @@ pub(super) fn reconcile_unknown(
|
|||||||
.as_ref()
|
.as_ref()
|
||||||
.ok_or_else(|| Error::InvalidInput("Journal recovery requires a grant".into()))?;
|
.ok_or_else(|| Error::InvalidInput("Journal recovery requires a grant".into()))?;
|
||||||
let store = store(api)?;
|
let store = store(api)?;
|
||||||
let host = host(api, &store, grant)?;
|
let attr = store
|
||||||
|
.session_attribution(&grant.session_id)
|
||||||
|
.map_err(|e| Error::Store(e.to_string()))?
|
||||||
|
.filter(|a| a.subject_id == grant.subject_id)
|
||||||
|
.ok_or_else(|| Error::WorkspacePermissionDenied("Journal attribution mismatch".into()))?;
|
||||||
|
let released = store
|
||||||
|
.journal_worker_was_released(&attr.subject_id, &attr.runtime_id, &attr.worker_id)
|
||||||
|
.map_err(domain)?;
|
||||||
|
let host = if released {
|
||||||
|
// Recovery reads immutable evidence, never renews a legacy writer grant.
|
||||||
|
let host = JournalHostContext::validated_session(&store, attr.clone()).map_err(domain)?;
|
||||||
|
if !grant.binding_id.is_empty()
|
||||||
|
&& host.writer_binding_id() != Some(grant.binding_id.as_str())
|
||||||
|
{
|
||||||
|
return Err(Error::WorkspacePermissionDenied(
|
||||||
|
"Journal recovery binding mismatch".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
host
|
||||||
|
} else {
|
||||||
|
host(api, &store, grant)?
|
||||||
|
};
|
||||||
let erased = store.journal_session_erased(&host).map_err(domain)?;
|
let erased = store.journal_session_erased(&host).map_err(domain)?;
|
||||||
if !erased {
|
if !erased {
|
||||||
job.load_executable_request()?;
|
job.load_executable_request()?;
|
||||||
@@ -2097,70 +2216,93 @@ pub(super) fn reconcile_unknown(
|
|||||||
job_id: job.request.job_id.clone(),
|
job_id: job.request.job_id.clone(),
|
||||||
attempt_id: attempt.attempt_id.clone(),
|
attempt_id: attempt.attempt_id.clone(),
|
||||||
};
|
};
|
||||||
let g = if legacy_descriptor(&job.request) {
|
let generation = if released {
|
||||||
// Only reconcile already-saved legacy receipts; never redispatch descriptors.
|
|
||||||
store
|
store
|
||||||
.prepare_journal_generation(
|
.retained_journal_generation(&host, &grant.journal_id, &binding)
|
||||||
&host,
|
|
||||||
&grant.journal_id,
|
|
||||||
binding,
|
|
||||||
grant.selected_ranges.clone(),
|
|
||||||
)
|
|
||||||
.map_err(domain)?
|
.map_err(domain)?
|
||||||
|
} else if legacy_descriptor(&job.request) {
|
||||||
|
Some(
|
||||||
|
store
|
||||||
|
.prepare_journal_generation(
|
||||||
|
&host,
|
||||||
|
&grant.journal_id,
|
||||||
|
binding,
|
||||||
|
grant.selected_ranges.clone(),
|
||||||
|
)
|
||||||
|
.map_err(domain)?,
|
||||||
|
)
|
||||||
} else {
|
} else {
|
||||||
let input = saved_input(&job.request, grant)?;
|
// Preserve active-body recovery: its saved intent can still establish
|
||||||
let generation = store
|
// the original generation fence. A released body must only read it.
|
||||||
.prepare_journal_input(&host, &input, binding)
|
Some(
|
||||||
.map_err(domain)?;
|
store
|
||||||
input.validate_generation(&generation).map_err(domain)?;
|
.prepare_journal_input(&host, &saved_input(&job.request, grant)?, binding)
|
||||||
generation
|
.map_err(domain)?,
|
||||||
|
)
|
||||||
};
|
};
|
||||||
let ctx = JournalJobContext::validated_recovery(&store, &host, &g.id, g.job.clone())
|
if let Some(g) = generation {
|
||||||
.map_err(domain)?;
|
if g.selected_ranges != grant.selected_ranges {
|
||||||
match journal::service::finish(&store, &ctx) {
|
return Err(Error::WorkspacePermissionDenied(
|
||||||
Ok(completion) => {
|
"Journal recovery selection mismatch".into(),
|
||||||
let submission = BackendJobResultSubmission {
|
));
|
||||||
job_id: job.request.job_id.clone(),
|
}
|
||||||
attempt_id: attempt.attempt_id.clone(),
|
if !legacy_descriptor(&job.request) {
|
||||||
input_digest: attempt.input_digest.clone(),
|
saved_input(&job.request, grant)?
|
||||||
result: serde_json::to_value(completion)
|
.validate_generation(&g)
|
||||||
.map_err(|e| Error::Store(e.to_string()))?,
|
.map_err(domain)?;
|
||||||
};
|
}
|
||||||
let current = api.store.reconcile_backend_journal_attempt(
|
let ctx = JournalJobContext::validated_recovery(&store, &host, &g.id, g.job.clone())
|
||||||
api.workspace_id(),
|
.map_err(domain)?;
|
||||||
attempt,
|
match journal::service::finish(&store, &ctx) {
|
||||||
Some(&submission),
|
Ok(completion) => {
|
||||||
&Utc::now().to_rfc3339(),
|
let submission = BackendJobResultSubmission {
|
||||||
)?;
|
job_id: job.request.job_id.clone(),
|
||||||
let saved = api
|
attempt_id: attempt.attempt_id.clone(),
|
||||||
.store
|
input_digest: attempt.input_digest.clone(),
|
||||||
.get_backend_job(api.workspace_id(), &job.request.job_id)?
|
result: serde_json::to_value(completion)
|
||||||
.ok_or_else(|| Error::Store("reconciled Job missing".into()))?;
|
.map_err(|e| Error::Store(e.to_string()))?,
|
||||||
notification_delivery::deliver_job_result(
|
};
|
||||||
api,
|
let current = api.store.reconcile_backend_journal_attempt(
|
||||||
&BackendJobResultAcceptance {
|
api.workspace_id(),
|
||||||
job: saved,
|
attempt,
|
||||||
attempt: current.clone(),
|
Some(&submission),
|
||||||
replayed: false,
|
&Utc::now().to_rfc3339(),
|
||||||
},
|
)?;
|
||||||
);
|
let saved = api
|
||||||
if current.worker_cleanup_state == Some(BackendJobWorkerCleanupState::Completed) {
|
.store
|
||||||
drain_after_completion(api, &job.request);
|
.get_backend_job(api.workspace_id(), &job.request.job_id)?
|
||||||
if job.input_disposition == crate::backend_job::JobInputDisposition::Available {
|
.ok_or_else(|| Error::Store("reconciled Job missing".into()))?;
|
||||||
api.store.reclaim_backend_job_input(
|
notification_delivery::deliver_job_result(
|
||||||
api.workspace_id(),
|
api,
|
||||||
&job.request.job_id,
|
&BackendJobResultAcceptance {
|
||||||
&job.input_digest,
|
job: saved,
|
||||||
crate::backend_job::JobInputRetirementReason::Completed,
|
attempt: current.clone(),
|
||||||
)?;
|
replayed: false,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
if current.worker_cleanup_state == Some(BackendJobWorkerCleanupState::Completed)
|
||||||
|
{
|
||||||
|
if !released {
|
||||||
|
drain_after_completion(api, &job.request);
|
||||||
|
}
|
||||||
|
if job.input_disposition
|
||||||
|
== crate::backend_job::JobInputDisposition::Available
|
||||||
|
{
|
||||||
|
api.store.reclaim_backend_job_input(
|
||||||
|
api.workspace_id(),
|
||||||
|
&job.request.job_id,
|
||||||
|
&job.input_digest,
|
||||||
|
crate::backend_job::JobInputRetirementReason::Completed,
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
api.schedule_backend_job_worker_cleanup(¤t);
|
||||||
}
|
}
|
||||||
} else {
|
return Ok(());
|
||||||
api.schedule_backend_job_worker_cleanup(¤t);
|
|
||||||
}
|
}
|
||||||
return Ok(());
|
Err(JournalError::Conflict(_) | JournalError::PermissionDenied(_)) => {}
|
||||||
|
Err(error) => return Err(domain(error)),
|
||||||
}
|
}
|
||||||
Err(JournalError::Conflict(_) | JournalError::PermissionDenied(_)) => {}
|
|
||||||
Err(error) => return Err(domain(error)),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if attempt.worker_cleanup_state != Some(BackendJobWorkerCleanupState::Completed) {
|
if attempt.worker_cleanup_state != Some(BackendJobWorkerCleanupState::Completed) {
|
||||||
@@ -2179,11 +2321,17 @@ pub(super) fn reconcile_unknown(
|
|||||||
false
|
false
|
||||||
};
|
};
|
||||||
if !confirmed_purge {
|
if !confirmed_purge {
|
||||||
|
if released {
|
||||||
|
// Source deletion already fenced this generation. An unbound Unknown
|
||||||
|
// still lacks execution-cleanup proof: preserve Unknown, but it no
|
||||||
|
// longer owns the Subject or blocks removal of the original body.
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
return Err(Error::RepositoryConflict(
|
return Err(Error::RepositoryConflict(
|
||||||
"Unknown Journal attempt has no committed stop/Purge fence; not retried".into(),
|
"Unknown Journal attempt has no committed stop/Purge fence; not retried".into(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
if !erased {
|
if !erased && !released {
|
||||||
close(api, &job.request, attempt)?;
|
close(api, &job.request, attempt)?;
|
||||||
}
|
}
|
||||||
api.store.reconcile_backend_journal_attempt(
|
api.store.reconcile_backend_journal_attempt(
|
||||||
@@ -2192,7 +2340,7 @@ pub(super) fn reconcile_unknown(
|
|||||||
None,
|
None,
|
||||||
&Utc::now().to_rfc3339(),
|
&Utc::now().to_rfc3339(),
|
||||||
)?;
|
)?;
|
||||||
if !erased {
|
if !erased && !released {
|
||||||
// Resume the same saved body input only after canonical execution/domain fences.
|
// Resume the same saved body input only after canonical execution/domain fences.
|
||||||
// dispatch selects its retained intent before it can capture any newer source.
|
// dispatch selects its retained intent before it can capture any newer source.
|
||||||
let attr = store
|
let attr = store
|
||||||
@@ -2209,6 +2357,10 @@ fn reconcile_session_jobs(
|
|||||||
attr: &crate::subjektiv::SubjectSessionAttribution,
|
attr: &crate::subjektiv::SubjectSessionAttribution,
|
||||||
erased: bool,
|
erased: bool,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
|
let released = !erased
|
||||||
|
&& store(api)?
|
||||||
|
.journal_worker_was_released(&attr.subject_id, &attr.runtime_id, &attr.worker_id)
|
||||||
|
.map_err(domain)?;
|
||||||
let mut after: Option<String> = None;
|
let mut after: Option<String> = None;
|
||||||
loop {
|
loop {
|
||||||
let jobs = api.store.list_backend_journal_jobs(
|
let jobs = api.store.list_backend_journal_jobs(
|
||||||
@@ -2223,6 +2375,29 @@ fn reconcile_session_jobs(
|
|||||||
.store
|
.store
|
||||||
.get_backend_job_attempt(api.workspace_id(), &job.request.job_id, &id)?
|
.get_backend_job_attempt(api.workspace_id(), &job.request.job_id, &id)?
|
||||||
.ok_or_else(|| Error::Store("Journal current attempt missing".into()))?;
|
.ok_or_else(|| Error::Store("Journal current attempt missing".into()))?;
|
||||||
|
if released {
|
||||||
|
// Writer revocation has already fenced all Journal mutations. End
|
||||||
|
// pending/Unknown attempts without claiming their input was reflected;
|
||||||
|
// saved input, operations and original source remain retained.
|
||||||
|
if !attempt.state.terminal() {
|
||||||
|
attempt = api.store.finish_backend_job_attempt(
|
||||||
|
api.workspace_id(),
|
||||||
|
&job.request.job_id,
|
||||||
|
&id,
|
||||||
|
BackendJobAttemptState::Failed,
|
||||||
|
"journal_body_released",
|
||||||
|
"Source Worker was removed; unfinished Journal history retained",
|
||||||
|
&Utc::now().to_rfc3339(),
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
api.schedule_backend_job_worker_cleanup(&attempt);
|
||||||
|
if attempt.state == BackendJobAttemptState::Unknown {
|
||||||
|
// A saved checkpoint may settle success; otherwise retain Unknown
|
||||||
|
// until canonical Job-worker cleanup has supplied its Purge fence.
|
||||||
|
reconcile_unknown(api, job, &attempt)?;
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
if !erased
|
if !erased
|
||||||
&& legacy_descriptor(&job.request)
|
&& legacy_descriptor(&job.request)
|
||||||
&& job.input_disposition == crate::backend_job::JobInputDisposition::Available
|
&& job.input_disposition == crate::backend_job::JobInputDisposition::Available
|
||||||
|
|||||||
@@ -11,6 +11,20 @@ use worker_runtime::worker_backend::{
|
|||||||
WorkerRuntimeExecutionBackend,
|
WorkerRuntimeExecutionBackend,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
pub(super) fn connected_restore_tempdir() -> tempfile::TempDir {
|
||||||
|
crate::restore_test_environment::require_restore_test_environment();
|
||||||
|
let checkout = Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||||
|
.parent()
|
||||||
|
.unwrap()
|
||||||
|
.parent()
|
||||||
|
.unwrap();
|
||||||
|
tempfile::Builder::new()
|
||||||
|
.prefix("c")
|
||||||
|
.rand_bytes(3)
|
||||||
|
.tempdir_in(checkout.join("target"))
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
const WATCHDOG: Duration = Duration::from_secs(45);
|
const WATCHDOG: Duration = Duration::from_secs(45);
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
@@ -82,24 +96,91 @@ impl RuntimeWorkerFactory for CapturingFactory {
|
|||||||
) -> std::result::Result<(), String> {
|
) -> std::result::Result<(), String> {
|
||||||
self.inner.preflight_restore(request).await
|
self.inner.preflight_restore(request).await
|
||||||
}
|
}
|
||||||
async fn restore_controller(
|
async fn recover_restore_source(
|
||||||
|
&self,
|
||||||
|
request: &worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||||
|
) -> std::result::Result<bool, worker_runtime::worker_backend::RuntimeRestoreFailure> {
|
||||||
|
self.inner.recover_restore_source(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn prepare_restore(
|
||||||
&self,
|
&self,
|
||||||
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||||
) -> std::result::Result<RuntimeWorkerController, String> {
|
) -> std::result::Result<
|
||||||
|
Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||||
|
> {
|
||||||
let worker_ref = request.worker_ref.clone();
|
let worker_ref = request.worker_ref.clone();
|
||||||
let controller = self.inner.restore_controller(request).await?;
|
let inner = self.inner.prepare_restore(request).await?;
|
||||||
|
Ok(Box::new(CapturingCandidate {
|
||||||
|
inner,
|
||||||
|
worker_ref,
|
||||||
|
captures: self.captures.clone(),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn cleanup_failed_restore(
|
||||||
|
&self,
|
||||||
|
request: &worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||||
|
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
|
||||||
|
self.inner.cleanup_failed_restore(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn reconcile_stopped_worker(
|
||||||
|
&self,
|
||||||
|
worker_ref: &worker_runtime::identity::WorkerRef,
|
||||||
|
) -> std::result::Result<(), String> {
|
||||||
|
self.inner.reconcile_stopped_worker(worker_ref).await
|
||||||
|
}
|
||||||
|
|
||||||
|
fn activate_restored_controller(
|
||||||
|
&self,
|
||||||
|
worker_ref: &worker_runtime::identity::WorkerRef,
|
||||||
|
workspace_id: Option<&str>,
|
||||||
|
handle: &worker::WorkerHandle,
|
||||||
|
) -> std::result::Result<(), String> {
|
||||||
|
self.inner
|
||||||
|
.activate_restored_controller(worker_ref, workspace_id, handle)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct CapturingCandidate {
|
||||||
|
inner: Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
|
||||||
|
worker_ref: worker_runtime::identity::WorkerRef,
|
||||||
|
captures: Arc<Captures>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait::async_trait]
|
||||||
|
impl worker_runtime::worker_backend::RuntimeRestoreCandidate for CapturingCandidate {
|
||||||
|
fn initial_state(&self) -> protocol::WorkerStateSnapshot {
|
||||||
|
self.inner.initial_state()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn activate(
|
||||||
|
&mut self,
|
||||||
|
) -> std::result::Result<
|
||||||
|
RuntimeWorkerController,
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||||
|
> {
|
||||||
|
let controller = self.inner.activate().await?;
|
||||||
self.captures
|
self.captures
|
||||||
.handles
|
.handles
|
||||||
.lock()
|
.lock()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.push((worker_ref.clone(), controller.handle.clone()));
|
.push((self.worker_ref.clone(), controller.handle.clone()));
|
||||||
self.captures
|
self.captures
|
||||||
.restored_clients
|
.restored_clients
|
||||||
.lock()
|
.lock()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.push((worker_ref, controller.workspace_client.clone()));
|
.push((self.worker_ref.clone(), controller.workspace_client.clone()));
|
||||||
Ok(controller)
|
Ok(controller)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn discard(
|
||||||
|
&mut self,
|
||||||
|
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
|
||||||
|
self.inner.discard().await
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
struct Resources {
|
struct Resources {
|
||||||
@@ -259,7 +340,7 @@ async fn call(tools: &BTreeMap<String, Arc<dyn Tool>>, name: &str, args: Value)
|
|||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||||
async fn restored_runtime_parent_lists_and_inputs_existing_child() {
|
async fn restored_runtime_parent_lists_and_inputs_existing_child() {
|
||||||
let root = tempfile::tempdir().unwrap();
|
let root = connected_restore_tempdir();
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
let base = format!("http://{}", listener.local_addr().unwrap());
|
let base = format!("http://{}", listener.local_addr().unwrap());
|
||||||
let mut config = test_server_config(root.path());
|
let mut config = test_server_config(root.path());
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -18,6 +18,57 @@ const EVIDENCE_MR: &str = "ticket-evidence-mr";
|
|||||||
const EVIDENCE_SOURCE: &str = "immutable-approved-source";
|
const EVIDENCE_SOURCE: &str = "immutable-approved-source";
|
||||||
const EVIDENCE_TARGET: &str = "immutable-target-result";
|
const EVIDENCE_TARGET: &str = "immutable-target-result";
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn integration_preserves_legacy_empty_post_ref_and_exposes_new_ref_with_server_schema() {
|
||||||
|
let fixture = EvidenceApiFixture::new().await;
|
||||||
|
let approval = fixture.approve("integration-post-ref");
|
||||||
|
let connection = Connection::open(&fixture.api.config.database_path).unwrap();
|
||||||
|
let legacy_index: i64 = connection.query_row(
|
||||||
|
"SELECT COALESCE(MAX(event_index),-1)+1 FROM typed_ticket_events WHERE workspace_id=?1 AND ticket_id=?2",
|
||||||
|
params![TEST_WORKSPACE_ID, fixture.ticket_id], |row| row.get(0),
|
||||||
|
).unwrap();
|
||||||
|
connection.execute(
|
||||||
|
"INSERT INTO typed_ticket_events(workspace_id,ticket_id,event_index,event_ref,kind,body)
|
||||||
|
VALUES(?1,?2,?3,'','comment','Legacy integration')",
|
||||||
|
params![TEST_WORKSPACE_ID, fixture.ticket_id, legacy_index],
|
||||||
|
).unwrap();
|
||||||
|
|
||||||
|
let merged = fixture.integrate(&approval);
|
||||||
|
assert_eq!(fixture.integrate(&approval), merged);
|
||||||
|
let detail = fixture
|
||||||
|
.backend
|
||||||
|
.show(fixture.ticket_id.clone().into())
|
||||||
|
.unwrap();
|
||||||
|
let legacy = detail
|
||||||
|
.events
|
||||||
|
.iter()
|
||||||
|
.find(|event| event.body.as_str() == "Legacy integration")
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(legacy.event_ref, "");
|
||||||
|
let integrations = detail
|
||||||
|
.events
|
||||||
|
.iter()
|
||||||
|
.filter(|event| {
|
||||||
|
event.attributes.get("merge_request_id").map(String::as_str) == Some(EVIDENCE_MR)
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
assert_eq!(
|
||||||
|
integrations.len(),
|
||||||
|
1,
|
||||||
|
"completion replay must not duplicate the Ticket post"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
integrations[0].event_ref,
|
||||||
|
format!("{}:{}", fixture.ticket_id, legacy_index + 1)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
integrations[0].attributes["operation_id"],
|
||||||
|
"evidence-integration"
|
||||||
|
);
|
||||||
|
let shown = fixture.show().await;
|
||||||
|
assert_eq!(shown.merge_requests[0].state, "merged");
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn ticket_browse_does_not_observe_sources_but_authoritative_query_and_show_still_do() {
|
async fn ticket_browse_does_not_observe_sources_but_authoritative_query_and_show_still_do() {
|
||||||
struct CountingObserver(Arc<std::sync::atomic::AtomicUsize>);
|
struct CountingObserver(Arc<std::sync::atomic::AtomicUsize>);
|
||||||
@@ -341,6 +392,33 @@ impl EvidenceApiFixture {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn mr_summary_route_omits_history_but_keeps_page_external_approval() {
|
||||||
|
let fixture = EvidenceApiFixture::new().await;
|
||||||
|
let approval = fixture.approve("summary-route");
|
||||||
|
fixture.integrate(&approval);
|
||||||
|
let value = get_json(
|
||||||
|
fixture.app.clone(),
|
||||||
|
&format!("/api/w/{TEST_WORKSPACE_ID}/merge-requests/{EVIDENCE_MR}?view=summary"),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let summary: MergeRequestDetailResponse = serde_json::from_value(value).unwrap();
|
||||||
|
assert!(summary.merge_request.thread.is_empty());
|
||||||
|
assert!(summary.event_page.omitted);
|
||||||
|
assert!(summary.event_page.has_more);
|
||||||
|
assert_eq!(summary.summary.review_status, "approved");
|
||||||
|
assert_eq!(
|
||||||
|
summary.summary.review.as_ref().unwrap().event.event_id,
|
||||||
|
approval.event_id
|
||||||
|
);
|
||||||
|
assert!(summary.summary.readiness.review.is_none());
|
||||||
|
let page = fixture.detail(0, 1).await;
|
||||||
|
assert_eq!(page.merge_request.thread.len(), 1);
|
||||||
|
assert!(page.event_page.has_more);
|
||||||
|
assert!(page.event_page.next_after.is_some());
|
||||||
|
assert_eq!(page.summary, summary.summary);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn closed_merged_remote_ticket_uses_immutable_result_in_show_query_list_and_paged_detail() {
|
async fn closed_merged_remote_ticket_uses_immutable_result_in_show_query_list_and_paged_detail() {
|
||||||
let fixture = EvidenceApiFixture::new().await;
|
let fixture = EvidenceApiFixture::new().await;
|
||||||
|
|||||||
@@ -652,6 +652,7 @@ async fn source_workdir_removal_keeps_published_drive_bytes_and_receipt() {
|
|||||||
&removal_id,
|
&removal_id,
|
||||||
"account:drive-test",
|
"account:drive-test",
|
||||||
"source no longer needed",
|
"source no longer needed",
|
||||||
|
"drive-source-removal",
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
|
|||||||
@@ -476,6 +476,7 @@ impl AttentionWorkerFactory {
|
|||||||
shutdown: Arc::new(tokio::sync::Mutex::new(Some(started.shutdown))),
|
shutdown: Arc::new(tokio::sync::Mutex::new(Some(started.shutdown))),
|
||||||
controller_task: started.controller_task,
|
controller_task: started.controller_task,
|
||||||
workspace_client,
|
workspace_client,
|
||||||
|
restore_streams: None,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -489,12 +490,61 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for AttentionWorkerFac
|
|||||||
self.controller(request.worker_ref.worker_id.to_string(), false)
|
self.controller(request.worker_ref.worker_id.to_string(), false)
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
async fn restore_controller(
|
async fn prepare_restore(
|
||||||
&self,
|
&self,
|
||||||
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||||
) -> std::result::Result<worker_runtime::worker_backend::RuntimeWorkerController, String> {
|
) -> std::result::Result<
|
||||||
self.controller(request.worker_ref.worker_id.to_string(), true)
|
Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
|
||||||
.await
|
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||||
|
> {
|
||||||
|
Ok(Box::new(AttentionRestoreCandidate {
|
||||||
|
inputs: Some((
|
||||||
|
AttentionWorkerFactory {
|
||||||
|
root: self.root.clone(),
|
||||||
|
},
|
||||||
|
request.worker_ref.worker_id.to_string(),
|
||||||
|
)),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Preparation owns only fixture inputs; reconstruction and Controller startup
|
||||||
|
// remain lazy so discard never needs to shut down a started Worker.
|
||||||
|
struct AttentionRestoreCandidate {
|
||||||
|
inputs: Option<(AttentionWorkerFactory, String)>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait::async_trait]
|
||||||
|
impl worker_runtime::worker_backend::RuntimeRestoreCandidate for AttentionRestoreCandidate {
|
||||||
|
fn initial_state(&self) -> protocol::WorkerStateSnapshot {
|
||||||
|
protocol::WorkerStateSnapshot::initial()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn activate(
|
||||||
|
&mut self,
|
||||||
|
) -> std::result::Result<
|
||||||
|
worker_runtime::worker_backend::RuntimeWorkerController,
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||||
|
> {
|
||||||
|
let (factory, worker_id) = self.inputs.take().ok_or_else(|| {
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreFailure::refused(
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreStage::Controller,
|
||||||
|
"attention fixture candidate already consumed",
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
factory.controller(worker_id, true).await.map_err(|_| {
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreFailure::unproven(
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreStage::Reconstruction,
|
||||||
|
"attention fixture reconstruction failed",
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn discard(
|
||||||
|
&mut self,
|
||||||
|
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
|
||||||
|
self.inputs.take();
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -601,7 +651,7 @@ async fn orchestrator_attention_real_worker_lost_response_retry_commits_once() {
|
|||||||
|
|
||||||
async fn real_attention_worker_start_restore(lose_response: bool) {
|
async fn real_attention_worker_start_restore(lose_response: bool) {
|
||||||
use session_store::{LogEntry, Store, WorkerSessionStore};
|
use session_store::{LogEntry, Store, WorkerSessionStore};
|
||||||
let dir = tempfile::tempdir().unwrap();
|
let dir = restored_parent_control_tests::connected_restore_tempdir();
|
||||||
init_clean_git_workspace(dir.path());
|
init_clean_git_workspace(dir.path());
|
||||||
let config = test_server_config(dir.path());
|
let config = test_server_config(dir.path());
|
||||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap());
|
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap());
|
||||||
|
|||||||
@@ -14,17 +14,37 @@ async fn notification_diagnostics_http_auth_scope_and_readonly_evidence() {
|
|||||||
seed_test_api_token(api.store.as_ref(), TEST_WORKSPACE_ID);
|
seed_test_api_token(api.store.as_ref(), TEST_WORKSPACE_ID);
|
||||||
let ticket = delivery_ticket(&api, "Notification diagnostics auth fixture");
|
let ticket = delivery_ticket(&api, "Notification diagnostics auth fixture");
|
||||||
delivery_assign(&api, ticket.id.as_str(), "diagnostics-assignment").await;
|
delivery_assign(&api, ticket.id.as_str(), "diagnostics-assignment").await;
|
||||||
let path = format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?ticket={}&limit=1",ticket.id.as_str());
|
let path = format!(
|
||||||
let page = delivery_http(&api,"GET",&path,None,StatusCode::OK).await;
|
"/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?ticket={}&limit=1",
|
||||||
assert_eq!(page["items"].as_array().unwrap().len(),1);
|
ticket.id.as_str()
|
||||||
assert_eq!(page["items"][0]["model_processing_confirmed"],false);
|
);
|
||||||
|
let page = delivery_http(&api, "GET", &path, None, StatusCode::OK).await;
|
||||||
|
assert_eq!(page["items"].as_array().unwrap().len(), 1);
|
||||||
|
assert_eq!(page["items"][0]["model_processing_confirmed"], false);
|
||||||
assert!(page["items"][0].get("content").is_none());
|
assert!(page["items"][0].get("content").is_none());
|
||||||
let again=delivery_http(&api,"GET",&path,None,StatusCode::OK).await;
|
let again = delivery_http(&api, "GET", &path, None, StatusCode::OK).await;
|
||||||
assert_eq!(again,page);
|
assert_eq!(again, page);
|
||||||
let response=build_router(api.clone()).oneshot(Request::builder().uri(&path).body(Body::empty()).unwrap()).await.unwrap();
|
let response = build_router(api.clone())
|
||||||
assert_eq!(response.status(),StatusCode::UNAUTHORIZED);
|
.oneshot(Request::builder().uri(&path).body(Body::empty()).unwrap())
|
||||||
delivery_http(&api,"GET","/api/w/other-workspace/notification-diagnostics",None,StatusCode::NOT_FOUND).await;
|
.await
|
||||||
delivery_http(&api,"GET",&format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?limit=101"),None,StatusCode::BAD_REQUEST).await;
|
.unwrap();
|
||||||
|
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
|
||||||
|
delivery_http(
|
||||||
|
&api,
|
||||||
|
"GET",
|
||||||
|
"/api/w/other-workspace/notification-diagnostics",
|
||||||
|
None,
|
||||||
|
StatusCode::NOT_FOUND,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
delivery_http(
|
||||||
|
&api,
|
||||||
|
"GET",
|
||||||
|
&format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?limit=101"),
|
||||||
|
None,
|
||||||
|
StatusCode::BAD_REQUEST,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
|
|
||||||
fn delivery_assignment_path(ticket: &str) -> String {
|
fn delivery_assignment_path(ticket: &str) -> String {
|
||||||
@@ -814,6 +834,7 @@ impl DeliveryWorkerFactory {
|
|||||||
shutdown: Arc::new(tokio::sync::Mutex::new(Some(started.shutdown))),
|
shutdown: Arc::new(tokio::sync::Mutex::new(Some(started.shutdown))),
|
||||||
controller_task: started.controller_task,
|
controller_task: started.controller_task,
|
||||||
workspace_client,
|
workspace_client,
|
||||||
|
restore_streams: None,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -827,12 +848,62 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for DeliveryWorkerFact
|
|||||||
self.controller(request.worker_ref.worker_id.to_string(), false)
|
self.controller(request.worker_ref.worker_id.to_string(), false)
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
async fn restore_controller(
|
async fn prepare_restore(
|
||||||
&self,
|
&self,
|
||||||
request: delivery_execution::WorkerExecutionRestoreRequest,
|
request: delivery_execution::WorkerExecutionRestoreRequest,
|
||||||
) -> std::result::Result<worker_runtime::worker_backend::RuntimeWorkerController, String> {
|
) -> std::result::Result<
|
||||||
self.controller(request.worker_ref.worker_id.to_string(), true)
|
Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
|
||||||
.await
|
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||||
|
> {
|
||||||
|
Ok(Box::new(DeliveryRestoreCandidate {
|
||||||
|
inputs: Some((
|
||||||
|
DeliveryWorkerFactory {
|
||||||
|
root: self.root.clone(),
|
||||||
|
client: self.client.clone(),
|
||||||
|
},
|
||||||
|
request.worker_ref.worker_id.to_string(),
|
||||||
|
)),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Retain the same scripted client/receipt store without opening either store or
|
||||||
|
// starting a Controller until activation. Discard drops only these owned inputs.
|
||||||
|
struct DeliveryRestoreCandidate {
|
||||||
|
inputs: Option<(DeliveryWorkerFactory, String)>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait::async_trait]
|
||||||
|
impl worker_runtime::worker_backend::RuntimeRestoreCandidate for DeliveryRestoreCandidate {
|
||||||
|
fn initial_state(&self) -> protocol::WorkerStateSnapshot {
|
||||||
|
protocol::WorkerStateSnapshot::initial()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn activate(
|
||||||
|
&mut self,
|
||||||
|
) -> std::result::Result<
|
||||||
|
worker_runtime::worker_backend::RuntimeWorkerController,
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||||
|
> {
|
||||||
|
let (factory, worker_id) = self.inputs.take().ok_or_else(|| {
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreFailure::refused(
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreStage::Controller,
|
||||||
|
"delivery fixture candidate already consumed",
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
factory.controller(worker_id, true).await.map_err(|_| {
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreFailure::unproven(
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreStage::Reconstruction,
|
||||||
|
"delivery fixture reconstruction failed",
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn discard(
|
||||||
|
&mut self,
|
||||||
|
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
|
||||||
|
self.inputs.take();
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -931,6 +1002,7 @@ async fn delivery_real_fixture(
|
|||||||
Arc<DeliveryResponseLossBackend>,
|
Arc<DeliveryResponseLossBackend>,
|
||||||
DeliveryScriptedClient,
|
DeliveryScriptedClient,
|
||||||
) {
|
) {
|
||||||
|
let _environment = restored_parent_control_tests::connected_restore_tempdir();
|
||||||
let config = test_server_config(root);
|
let config = test_server_config(root);
|
||||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap());
|
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap());
|
||||||
seed_test_registered_workspace(store.as_ref(), &config)
|
seed_test_registered_workspace(store.as_ref(), &config)
|
||||||
@@ -1136,9 +1208,22 @@ async fn assignment_real_worker_response_loss_restore_and_concurrent_scans_commi
|
|||||||
"uncertain delivery changed request id or payload: {attempts:?}"
|
"uncertain delivery changed request id or payload: {attempts:?}"
|
||||||
);
|
);
|
||||||
assert_eq!(delivery_receipts(&api, &worker).len(), 1);
|
assert_eq!(delivery_receipts(&api, &worker).len(), 1);
|
||||||
let unknown = api.store.notification_diagnostics(TEST_WORKSPACE_ID, &server_api::NotificationDiagnosticsQuery {
|
let unknown = api
|
||||||
assignment_id: Some(assignment.into()), ..Default::default()
|
.store
|
||||||
}).unwrap().items.into_iter().find(|item| item.stage == server_api::NotificationDiagnosticStage::AcceptanceUnknown).expect("saved unknown acceptance must remain visible while Runtime already has its receipt");
|
.notification_diagnostics(
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
&server_api::NotificationDiagnosticsQuery {
|
||||||
|
assignment_id: Some(assignment.into()),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
.items
|
||||||
|
.into_iter()
|
||||||
|
.find(|item| item.stage == server_api::NotificationDiagnosticStage::AcceptanceUnknown)
|
||||||
|
.expect(
|
||||||
|
"saved unknown acceptance must remain visible while Runtime already has its receipt",
|
||||||
|
);
|
||||||
assert!(!unknown.model_processing_confirmed);
|
assert!(!unknown.model_processing_confirmed);
|
||||||
// Change mutable rendering context while the acknowledgement is unknown.
|
// Change mutable rendering context while the acknowledgement is unknown.
|
||||||
// A new self-report is not a new input and cannot change the frozen retry.
|
// A new self-report is not a new input and cannot change the frozen retry.
|
||||||
@@ -1199,11 +1284,24 @@ async fn assignment_real_worker_response_loss_restore_and_concurrent_scans_commi
|
|||||||
}
|
}
|
||||||
delivery_wait_idle(&reopened, &worker, 2).await;
|
delivery_wait_idle(&reopened, &worker, 2).await;
|
||||||
assert_eq!(delivery_receipts(&reopened, &worker).len(), 2);
|
assert_eq!(delivery_receipts(&reopened, &worker).len(), 2);
|
||||||
let evidence = delivery_http(&reopened,"GET",&format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?source_id={}",unknown.source_id),None,StatusCode::OK).await;
|
let evidence = delivery_http(
|
||||||
assert_eq!(evidence["items"][0]["stage"],"runtime_accepted");
|
&reopened,
|
||||||
assert_eq!(evidence["items"][0]["delivery_id"],json!(unknown.delivery_id));
|
"GET",
|
||||||
assert_eq!(evidence["items"][0]["model_processing_confirmed"],false);
|
&format!(
|
||||||
assert_eq!(evidence["items"][0]["retry_candidate"],false);
|
"/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?source_id={}",
|
||||||
|
unknown.source_id
|
||||||
|
),
|
||||||
|
None,
|
||||||
|
StatusCode::OK,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(evidence["items"][0]["stage"], "runtime_accepted");
|
||||||
|
assert_eq!(
|
||||||
|
evidence["items"][0]["delivery_id"],
|
||||||
|
json!(unknown.delivery_id)
|
||||||
|
);
|
||||||
|
assert_eq!(evidence["items"][0]["model_processing_confirmed"], false);
|
||||||
|
assert_eq!(evidence["items"][0]["retry_candidate"], false);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
delivery_commits(&reopened, &worker)
|
delivery_commits(&reopened, &worker)
|
||||||
.iter()
|
.iter()
|
||||||
|
|||||||
@@ -0,0 +1,399 @@
|
|||||||
|
// Integration boundary: generated HTTP admission -> real WebAuthn verifier ->
|
||||||
|
// SQLite credential/receipt transaction -> real session issuance. Only the
|
||||||
|
// authenticator is synthetic; failures are SQLite triggers, not success fakes.
|
||||||
|
|
||||||
|
impl Fixture {
|
||||||
|
async fn addition_result(&self, cookie: &str, challenge_id: &str) -> (StatusCode, Value) {
|
||||||
|
let mut request = self.expected_owner(Some(cookie), None).await;
|
||||||
|
request["challenge_id"] = challenge_id.into();
|
||||||
|
let (status, _, body) = self
|
||||||
|
.post(
|
||||||
|
"/api/auth/passkeys/addition/result",
|
||||||
|
Some(cookie),
|
||||||
|
Some(ORIGIN_URL),
|
||||||
|
request,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
(status, body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn addition_receipt_is_atomic_and_recovers_credential_commit_not_session_success() {
|
||||||
|
for failure in [
|
||||||
|
"before_credential",
|
||||||
|
"receipt_insert",
|
||||||
|
"session_insert",
|
||||||
|
"lost_response",
|
||||||
|
"lost_body_accepted_cookie",
|
||||||
|
] {
|
||||||
|
let f = Fixture::new().await;
|
||||||
|
let mut original = authenticator();
|
||||||
|
let (cookie, owner) = f.enroll("registration", None, "owner", &mut original).await;
|
||||||
|
let original_expectation = f.expected_owner(Some(&cookie), None).await;
|
||||||
|
let options = f.options("addition", Some(&cookie), "").await;
|
||||||
|
let challenge_id = options.challenge_id.clone();
|
||||||
|
let mut added = authenticator();
|
||||||
|
let request = Fixture::credential(options, &mut added);
|
||||||
|
let (status, result) = f.addition_result(&cookie, &challenge_id).await;
|
||||||
|
assert_eq!(status, StatusCode::OK);
|
||||||
|
assert_eq!(result["status"], "unknown");
|
||||||
|
let table = match failure {
|
||||||
|
"before_credential" => Some("passkey_credentials"),
|
||||||
|
"receipt_insert" => Some("passkey_addition_receipts"),
|
||||||
|
"session_insert" => Some("browser_sessions"),
|
||||||
|
_ => None,
|
||||||
|
};
|
||||||
|
if let Some(table) = table {
|
||||||
|
f.api.config_store.with_conn(|conn| {
|
||||||
|
conn.execute_batch(&format!("CREATE TRIGGER inject_passkey_failure BEFORE INSERT ON {table} BEGIN SELECT RAISE(ABORT, 'synthetic failure'); END;"))?;
|
||||||
|
Ok(())
|
||||||
|
}).unwrap();
|
||||||
|
}
|
||||||
|
// Discard the completion body; one variant still accepts Set-Cookie.
|
||||||
|
let (status, headers, _) = f
|
||||||
|
.post(
|
||||||
|
"/api/auth/passkeys/addition/complete",
|
||||||
|
Some(&cookie),
|
||||||
|
Some(ORIGIN_URL),
|
||||||
|
request.clone(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(
|
||||||
|
status,
|
||||||
|
if matches!(failure, "lost_response" | "lost_body_accepted_cookie") {
|
||||||
|
StatusCode::OK
|
||||||
|
} else {
|
||||||
|
StatusCode::INTERNAL_SERVER_ERROR
|
||||||
|
}
|
||||||
|
);
|
||||||
|
if !matches!(failure, "lost_response" | "lost_body_accepted_cookie") {
|
||||||
|
assert!(!headers.contains_key(axum::http::header::SET_COOKIE));
|
||||||
|
}
|
||||||
|
let saved = matches!(
|
||||||
|
failure,
|
||||||
|
"session_insert" | "lost_response" | "lost_body_accepted_cookie"
|
||||||
|
);
|
||||||
|
if failure == "lost_body_accepted_cookie" {
|
||||||
|
let accepted_cookie = headers[axum::http::header::SET_COOKIE]
|
||||||
|
.to_str()
|
||||||
|
.unwrap()
|
||||||
|
.split(';')
|
||||||
|
.next()
|
||||||
|
.unwrap();
|
||||||
|
let current = f.expected_owner(Some(accepted_cookie), None).await;
|
||||||
|
assert!(
|
||||||
|
current["expected_authentication_reference"]
|
||||||
|
!= original_expectation["expected_authentication_reference"],
|
||||||
|
"addition issued a distinct session"
|
||||||
|
);
|
||||||
|
let mut recovery = original_expectation.clone();
|
||||||
|
recovery["challenge_id"] = challenge_id.clone().into();
|
||||||
|
let (status, _, result) = f
|
||||||
|
.post(
|
||||||
|
"/api/auth/passkeys/addition/result",
|
||||||
|
Some(accepted_cookie),
|
||||||
|
Some(ORIGIN_URL),
|
||||||
|
recovery,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(status, StatusCode::OK);
|
||||||
|
assert_eq!(result["status"], "saved");
|
||||||
|
assert_eq!(result["user"]["user_id"], owner.user_id);
|
||||||
|
}
|
||||||
|
let (status, result) = f.addition_result(&cookie, &challenge_id).await;
|
||||||
|
assert_eq!(status, StatusCode::OK);
|
||||||
|
assert_eq!(result["status"], if saved { "saved" } else { "unknown" });
|
||||||
|
assert_eq!(f.credentials(&owner).len(), if saved { 2 } else { 1 });
|
||||||
|
let receipt_count = f
|
||||||
|
.api
|
||||||
|
.config_store
|
||||||
|
.with_conn(|conn| {
|
||||||
|
Ok(conn.query_row(
|
||||||
|
"SELECT COUNT(*) FROM passkey_addition_receipts",
|
||||||
|
[],
|
||||||
|
|row| row.get::<_, u64>(0),
|
||||||
|
)?)
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(receipt_count, u64::from(saved));
|
||||||
|
if saved {
|
||||||
|
assert_eq!(result["user"]["user_id"], owner.user_id);
|
||||||
|
let (_, listed) = f
|
||||||
|
.get_authenticated("/api/auth/passkeys", Some(&cookie), None)
|
||||||
|
.await;
|
||||||
|
let reference = result["key"]["key_reference"].as_str().unwrap();
|
||||||
|
assert!(
|
||||||
|
listed["keys"]
|
||||||
|
.as_array()
|
||||||
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.any(|key| key["key_reference"] == reference)
|
||||||
|
);
|
||||||
|
assert!(result["key"]["normal_login_verified_at"].is_null());
|
||||||
|
}
|
||||||
|
if table.is_some() {
|
||||||
|
f.api
|
||||||
|
.config_store
|
||||||
|
.with_conn(|conn| {
|
||||||
|
conn.execute_batch("DROP TRIGGER inject_passkey_failure")?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
// Consumed completion is not retry authority, even after a transaction failure.
|
||||||
|
let (status, headers, _) = f
|
||||||
|
.post(
|
||||||
|
"/api/auth/passkeys/addition/complete",
|
||||||
|
Some(&cookie),
|
||||||
|
Some(ORIGIN_URL),
|
||||||
|
request,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(status, StatusCode::BAD_REQUEST);
|
||||||
|
assert!(!headers.contains_key(axum::http::header::SET_COOKIE));
|
||||||
|
assert_eq!(f.credentials(&owner).len(), if saved { 2 } else { 1 });
|
||||||
|
if saved {
|
||||||
|
assert_eq!(f.discoverable_login(&mut added).await, owner);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn stale_displayed_owner_or_session_cannot_start_addition_or_read_another_binding_result() {
|
||||||
|
let f = Fixture::new().await;
|
||||||
|
let mut alice_key = authenticator();
|
||||||
|
let (alice_cookie, alice) = f
|
||||||
|
.enroll("registration", None, "alice", &mut alice_key)
|
||||||
|
.await;
|
||||||
|
let (bob_cookie, _) = f
|
||||||
|
.enroll("registration", None, "bob", &mut authenticator())
|
||||||
|
.await;
|
||||||
|
let expected = f.expected_owner(Some(&alice_cookie), None).await;
|
||||||
|
let (replacement_cookie, _) = f.login_session("alice", &mut alice_key).await;
|
||||||
|
let count_challenges = || {
|
||||||
|
f.api
|
||||||
|
.config_store
|
||||||
|
.with_conn(|conn| {
|
||||||
|
Ok(
|
||||||
|
conn.query_row("SELECT COUNT(*) FROM auth_challenges", [], |row| {
|
||||||
|
row.get::<_, u64>(0)
|
||||||
|
})?,
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.unwrap()
|
||||||
|
};
|
||||||
|
let before = count_challenges();
|
||||||
|
for cookie in [&bob_cookie, &replacement_cookie] {
|
||||||
|
let (status, _, _) = f
|
||||||
|
.post(
|
||||||
|
"/api/auth/passkeys/addition/options",
|
||||||
|
Some(cookie),
|
||||||
|
Some(ORIGIN_URL),
|
||||||
|
expected.clone(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(status, StatusCode::BAD_REQUEST);
|
||||||
|
assert_eq!(
|
||||||
|
count_challenges(),
|
||||||
|
before,
|
||||||
|
"stale options must not persist a ceremony"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let options = f.options("addition", Some(&alice_cookie), "").await;
|
||||||
|
let challenge_id = options.challenge_id.clone();
|
||||||
|
let request = Fixture::credential(options, &mut authenticator());
|
||||||
|
assert_eq!(
|
||||||
|
f.post(
|
||||||
|
"/api/auth/passkeys/addition/complete",
|
||||||
|
Some(&alice_cookie),
|
||||||
|
Some(ORIGIN_URL),
|
||||||
|
request
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.0,
|
||||||
|
StatusCode::OK
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
f.addition_result(&alice_cookie, &challenge_id).await.1["status"],
|
||||||
|
"saved"
|
||||||
|
);
|
||||||
|
// Current owner authorization is separate from original operation binding.
|
||||||
|
let mut recovery = expected.clone();
|
||||||
|
recovery["challenge_id"] = challenge_id.clone().into();
|
||||||
|
let (status, _, result) = f
|
||||||
|
.post(
|
||||||
|
"/api/auth/passkeys/addition/result",
|
||||||
|
Some(&replacement_cookie),
|
||||||
|
Some(ORIGIN_URL),
|
||||||
|
recovery.clone(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(status, StatusCode::OK);
|
||||||
|
assert_eq!(result["status"], "saved");
|
||||||
|
assert_eq!(
|
||||||
|
f.post(
|
||||||
|
"/api/auth/passkeys/addition/result",
|
||||||
|
Some(&bob_cookie),
|
||||||
|
Some(ORIGIN_URL),
|
||||||
|
recovery.clone()
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.0,
|
||||||
|
StatusCode::BAD_REQUEST
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
f.post(
|
||||||
|
"/api/auth/passkeys/addition/result",
|
||||||
|
None,
|
||||||
|
Some(ORIGIN_URL),
|
||||||
|
recovery.clone()
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.0,
|
||||||
|
StatusCode::FORBIDDEN
|
||||||
|
);
|
||||||
|
recovery["expected_authentication_reference"] = "changed-original-reference".into();
|
||||||
|
assert_eq!(
|
||||||
|
f.post(
|
||||||
|
"/api/auth/passkeys/addition/result",
|
||||||
|
Some(&replacement_cookie),
|
||||||
|
Some(ORIGIN_URL),
|
||||||
|
recovery
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.0,
|
||||||
|
StatusCode::BAD_REQUEST
|
||||||
|
);
|
||||||
|
// Bob's own expected fields cannot discover Alice's operation either.
|
||||||
|
let (status, result) = f.addition_result(&bob_cookie, &challenge_id).await;
|
||||||
|
assert_eq!(status, StatusCode::OK);
|
||||||
|
assert_eq!(result, serde_json::json!({"status":"unknown"}));
|
||||||
|
assert_eq!(f.credentials(&alice).len(), 2);
|
||||||
|
assert_eq!(
|
||||||
|
f.post(
|
||||||
|
"/api/auth/logout",
|
||||||
|
Some(&alice_cookie),
|
||||||
|
Some(ORIGIN_URL),
|
||||||
|
serde_json::json!({})
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.0,
|
||||||
|
StatusCode::OK
|
||||||
|
);
|
||||||
|
let mut request = expected;
|
||||||
|
request["challenge_id"] = challenge_id.into();
|
||||||
|
assert_eq!(
|
||||||
|
f.post(
|
||||||
|
"/api/auth/passkeys/addition/result",
|
||||||
|
Some(&alice_cookie),
|
||||||
|
Some(ORIGIN_URL),
|
||||||
|
request
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.0,
|
||||||
|
StatusCode::FORBIDDEN
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn own_key_list_is_safe_and_only_verified_normal_login_marks_confirmation() {
|
||||||
|
let f = Fixture::new().await;
|
||||||
|
let mut key = authenticator();
|
||||||
|
let (cookie, owner) = f.enroll("registration", None, "owner", &mut key).await;
|
||||||
|
let (other_cookie, _) = f
|
||||||
|
.enroll("registration", None, "other", &mut authenticator())
|
||||||
|
.await;
|
||||||
|
assert_eq!(
|
||||||
|
f.get_authenticated("/api/auth/passkeys", None, None)
|
||||||
|
.await
|
||||||
|
.0,
|
||||||
|
StatusCode::FORBIDDEN
|
||||||
|
);
|
||||||
|
let (status, list) = f
|
||||||
|
.get_authenticated("/api/auth/passkeys", Some(&cookie), None)
|
||||||
|
.await;
|
||||||
|
assert_eq!(status, StatusCode::OK);
|
||||||
|
assert_eq!(list["user"]["user_id"], owner.user_id);
|
||||||
|
assert_eq!(list["keys"].as_array().unwrap().len(), 1);
|
||||||
|
let summary = &list["keys"][0];
|
||||||
|
assert!(summary["normal_login_verified_at"].is_null());
|
||||||
|
assert!(summary["last_used_at"].is_null());
|
||||||
|
let reference = summary["key_reference"].as_str().unwrap();
|
||||||
|
assert_eq!(reference.len(), 32);
|
||||||
|
assert_ne!(reference, f.credentials(&owner)[0].credential_id);
|
||||||
|
assert_eq!(
|
||||||
|
summary
|
||||||
|
.as_object()
|
||||||
|
.unwrap()
|
||||||
|
.keys()
|
||||||
|
.map(String::as_str)
|
||||||
|
.collect::<std::collections::BTreeSet<_>>(),
|
||||||
|
[
|
||||||
|
"key_reference",
|
||||||
|
"created_at",
|
||||||
|
"last_used_at",
|
||||||
|
"normal_login_verified_at"
|
||||||
|
]
|
||||||
|
.into_iter()
|
||||||
|
.collect()
|
||||||
|
);
|
||||||
|
let (_, other) = f
|
||||||
|
.get_authenticated("/api/auth/passkeys", Some(&other_cookie), None)
|
||||||
|
.await;
|
||||||
|
assert_ne!(other["keys"][0]["key_reference"], reference);
|
||||||
|
assert_eq!(f.login("owner", &mut key).await, owner);
|
||||||
|
let (_, legacy) = f
|
||||||
|
.get_authenticated("/api/auth/passkeys", Some(&cookie), None)
|
||||||
|
.await;
|
||||||
|
assert!(legacy["keys"][0]["last_used_at"].is_string());
|
||||||
|
assert!(legacy["keys"][0]["normal_login_verified_at"].is_null());
|
||||||
|
// Rejected normal assertion does not produce confirmation metadata.
|
||||||
|
let mut invalid = f.discoverable_request(&mut key).await;
|
||||||
|
invalid.credential.response.signature = vec![0u8; 64].into();
|
||||||
|
assert_eq!(
|
||||||
|
f.post(
|
||||||
|
"/api/auth/passkeys/login/complete",
|
||||||
|
None,
|
||||||
|
Some(ORIGIN_URL),
|
||||||
|
serde_json::to_value(invalid).unwrap()
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.0,
|
||||||
|
StatusCode::BAD_REQUEST
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
f.api.store.list_passkey_summaries(&owner.user_id).unwrap()[0]
|
||||||
|
.normal_login_verified_at
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert_eq!(f.discoverable_login(&mut key).await, owner);
|
||||||
|
let (_, confirmed) = f
|
||||||
|
.get_authenticated("/api/auth/passkeys", Some(&cookie), None)
|
||||||
|
.await;
|
||||||
|
assert_eq!(confirmed["keys"][0]["key_reference"], reference);
|
||||||
|
assert!(confirmed["keys"][0]["normal_login_verified_at"].is_string());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn unknown_transport_metadata_does_not_discard_verified_registration() {
|
||||||
|
let f = Fixture::new().await;
|
||||||
|
let options = f.options("registration", None, "future-transport").await;
|
||||||
|
let mut key = authenticator();
|
||||||
|
let mut request = Fixture::credential(options, &mut key);
|
||||||
|
request["credential"]["response"]["transports"] =
|
||||||
|
serde_json::json!(["internal", "future-transport", "test"]);
|
||||||
|
let (status, _, body) = f
|
||||||
|
.post(
|
||||||
|
"/api/auth/passkeys/registration/complete",
|
||||||
|
None,
|
||||||
|
Some(ORIGIN_URL),
|
||||||
|
request,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(status, StatusCode::OK);
|
||||||
|
let user = serde_json::from_value::<AuthUserResponse>(body)
|
||||||
|
.unwrap()
|
||||||
|
.user;
|
||||||
|
assert_eq!(f.discoverable_login(&mut key).await, user);
|
||||||
|
}
|
||||||
@@ -120,6 +120,14 @@ struct TicketSurface {
|
|||||||
}
|
}
|
||||||
impl TicketSurface {
|
impl TicketSurface {
|
||||||
async fn new(root: &Path, client: Arc<TicketRouterClient>, wip: bool) -> Self {
|
async fn new(root: &Path, client: Arc<TicketRouterClient>, wip: bool) -> Self {
|
||||||
|
Self::with_access(root, client, wip, TicketFeatureAccess::workflow()).await
|
||||||
|
}
|
||||||
|
async fn with_access(
|
||||||
|
root: &Path,
|
||||||
|
client: Arc<dyn WorkspaceClient>,
|
||||||
|
wip: bool,
|
||||||
|
access: TicketFeatureAccess,
|
||||||
|
) -> Self {
|
||||||
let manifest = worker::WorkerManifest::from_toml("[worker]\nname=\"ticket-tools\"\n[model]\nscheme=\"anthropic\"\nmodel_id=\"unused\"\n[engine]\n[scope]\nallow=[]\n").unwrap();
|
let manifest = worker::WorkerManifest::from_toml("[worker]\nname=\"ticket-tools\"\n[model]\nscheme=\"anthropic\"\nmodel_id=\"unused\"\n[engine]\n[scope]\nallow=[]\n").unwrap();
|
||||||
let engine =
|
let engine =
|
||||||
agen::Engine::<_, agen::state::Mutable, worker::SessionHistoryMetadata>::new_annotated(
|
agen::Engine::<_, agen::state::Mutable, worker::SessionHistoryMetadata>::new_annotated(
|
||||||
@@ -138,22 +146,18 @@ impl TicketSurface {
|
|||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let report =
|
let report = host.install_features(
|
||||||
host.install_features(worker::feature::FeatureRegistryBuilder::new().with_module(
|
worker::feature::FeatureRegistryBuilder::new()
|
||||||
TicketFeature::new(client.clone(), TicketFeatureAccess::workflow()),
|
.with_module(TicketFeature::new(client.clone(), access)),
|
||||||
));
|
);
|
||||||
assert!(!report.has_errors());
|
assert!(!report.has_errors());
|
||||||
let handle = host.engine().tool_server_handle();
|
let handle = host.engine().tool_server_handle();
|
||||||
handle.flush_pending();
|
handle.flush_pending();
|
||||||
let tools = [
|
let tools = access
|
||||||
"TicketAssignmentTrackingQuery",
|
.tool_names()
|
||||||
"TicketAssignmentAnswer",
|
.into_iter()
|
||||||
"TicketImplementationResult",
|
.map(|name| (name.into(), handle.get_tool(name).unwrap().1))
|
||||||
"TicketImplementationResultQuery",
|
.collect();
|
||||||
]
|
|
||||||
.into_iter()
|
|
||||||
.map(|name| (name.into(), handle.get_tool(name).unwrap().1))
|
|
||||||
.collect();
|
|
||||||
if !wip {
|
if !wip {
|
||||||
return Self { tools, wip };
|
return Self { tools, wip };
|
||||||
}
|
}
|
||||||
@@ -162,7 +166,7 @@ impl TicketSurface {
|
|||||||
worker::feature::builtin::ticket::mount_workspace_http_ticket_wip(
|
worker::feature::builtin::ticket::mount_workspace_http_ticket_wip(
|
||||||
&mut mounts,
|
&mut mounts,
|
||||||
client,
|
client,
|
||||||
TicketFeatureAccess::workflow(),
|
access,
|
||||||
None,
|
None,
|
||||||
&route,
|
&route,
|
||||||
)
|
)
|
||||||
@@ -189,8 +193,9 @@ impl TicketSurface {
|
|||||||
if !self.wip {
|
if !self.wip {
|
||||||
return execute_tool(self.tools[name].clone(), input, context).await;
|
return execute_tool(self.tools[name].clone(), input, context).await;
|
||||||
}
|
}
|
||||||
let path = format!("/tickets/{}", input["ticket"].as_str().unwrap());
|
let reference_field = if name == "ShowTicket" { "id" } else { "ticket" };
|
||||||
input.as_object_mut().unwrap().remove("ticket");
|
let path = format!("/tickets/{}", input[reference_field].as_str().unwrap());
|
||||||
|
input.as_object_mut().unwrap().remove(reference_field);
|
||||||
// Observe the real item representation before invoking a mutation. Tracking
|
// Observe the real item representation before invoking a mutation. Tracking
|
||||||
// reads alone do not establish the Ticket WIP object validator. Explicit
|
// reads alone do not establish the Ticket WIP object validator. Explicit
|
||||||
// refresh also reconciles stale observations after an unknown result;
|
// refresh also reconciles stale observations after an unknown result;
|
||||||
@@ -201,6 +206,9 @@ impl TicketSurface {
|
|||||||
ToolExecutionContext::direct(),
|
ToolExecutionContext::direct(),
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
if name == "ShowTicket" {
|
||||||
|
return execute_tool(self.tools["Invoke"].clone(), serde_json::json!({"path":path,"interface":inspected["interfaces"][0]["reference"],"operation":"read","arguments":input}), context).await;
|
||||||
|
}
|
||||||
execute_tool(self.tools["Invoke"].clone(), serde_json::json!({"path":path,"interface":inspected["interfaces"][0]["reference"],"operation":"read","arguments":{}}), ToolExecutionContext::direct()).await?;
|
execute_tool(self.tools["Invoke"].clone(), serde_json::json!({"path":path,"interface":inspected["interfaces"][0]["reference"],"operation":"read","arguments":{}}), ToolExecutionContext::direct()).await?;
|
||||||
let inspected = execute_tool(
|
let inspected = execute_tool(
|
||||||
self.tools["Inspect"].clone(),
|
self.tools["Inspect"].clone(),
|
||||||
@@ -364,6 +372,210 @@ async fn ticket_public_roundtrip_observed_answer_replays_without_changing_waitin
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Reviewer policy, tool projection, reference resolution, authentication and
|
||||||
|
// persistence remain production code. Only the synchronous HTTP transport is
|
||||||
|
// replaced by signed in-process router dispatch. Child spawning and MR attestation
|
||||||
|
// have their own suites; this fixture does not claim to exercise those boundaries.
|
||||||
|
fn reviewer_client(inner: Arc<TicketRouterClient>, bound_ticket: &str) -> Arc<dyn WorkspaceClient> {
|
||||||
|
Arc::new(worker::ReviewerChildWorkspaceClient::new(
|
||||||
|
inner,
|
||||||
|
worker::ReviewerContext {
|
||||||
|
ticket_id: bound_ticket.into(),
|
||||||
|
merge_request_id: "unsubmitted-review-candidate".into(),
|
||||||
|
},
|
||||||
|
"unused-review-capability".into(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
async fn reviewer_reads_saved_related_tickets_by_key_and_id_and_pages_their_threads() {
|
||||||
|
for wip in [false, true] {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let (api, _runtime, identity, source, bound_id) = tracking_fixture(dir.path()).await;
|
||||||
|
let backend = browser_ticket_backend(&api).unwrap();
|
||||||
|
let mut related = ticket::NewTicket::new("Related requirements");
|
||||||
|
related.body = ticket::MarkdownText::new("Requirements from a different Ticket");
|
||||||
|
let related = backend.create(related).unwrap();
|
||||||
|
backend
|
||||||
|
.add_ticket_relation(
|
||||||
|
bound_id.clone().into(),
|
||||||
|
ticket::NewTicketRelation {
|
||||||
|
kind: ticket::TicketRelationKind::DependsOn,
|
||||||
|
target: related.id.clone(),
|
||||||
|
note: None,
|
||||||
|
author: None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
for body in ["Earlier decision", "Latest decision"] {
|
||||||
|
backend
|
||||||
|
.add_event(
|
||||||
|
related.id.clone().into(),
|
||||||
|
ticket::NewTicketEvent::new(ticket::TicketEventKind::Comment, body),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
let bound = backend.show(bound_id.clone().into()).unwrap();
|
||||||
|
let related_before = backend.show(related.id.clone().into()).unwrap();
|
||||||
|
let bound_key = bound.meta.resource_key.as_deref().unwrap();
|
||||||
|
let related_key = related.resource_key.as_deref().unwrap();
|
||||||
|
let transport = ticket_client(&api, identity, &source);
|
||||||
|
// One client must retain its review binding while following related Tickets.
|
||||||
|
let client = reviewer_client(transport.clone(), &bound_id);
|
||||||
|
let surface =
|
||||||
|
TicketSurface::with_access(dir.path(), client, wip, TicketFeatureAccess::review())
|
||||||
|
.await;
|
||||||
|
for (reference, expected_key, expected_title, expected_body) in [
|
||||||
|
(
|
||||||
|
bound_key,
|
||||||
|
bound_key,
|
||||||
|
"Tracking authority",
|
||||||
|
bound.document.body.as_str(),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
related_key,
|
||||||
|
related_key,
|
||||||
|
"Related requirements",
|
||||||
|
related_before.document.body.as_str(),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
related.id.as_str(),
|
||||||
|
related_key,
|
||||||
|
"Related requirements",
|
||||||
|
related_before.document.body.as_str(),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
bound_id.as_str(),
|
||||||
|
bound_key,
|
||||||
|
"Tracking authority",
|
||||||
|
bound.document.body.as_str(),
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
let shown = surface
|
||||||
|
.call(
|
||||||
|
"ShowTicket",
|
||||||
|
serde_json::json!({"id":reference}),
|
||||||
|
ToolExecutionContext::direct(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(shown["ticket"], expected_key);
|
||||||
|
assert_eq!(shown["title"], expected_title);
|
||||||
|
assert_eq!(shown["body"], expected_body);
|
||||||
|
if expected_key == bound_key {
|
||||||
|
assert!(
|
||||||
|
shown["relations"]["outgoing"]
|
||||||
|
.as_array()
|
||||||
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.any(|relation| relation["ticket"] == related_key
|
||||||
|
&& relation["kind"] == "depends_on")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let first = surface
|
||||||
|
.call(
|
||||||
|
"ShowTicket",
|
||||||
|
serde_json::json!({"id":related_key,"event_limit":1}),
|
||||||
|
ToolExecutionContext::direct(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(first["thread"].as_array().unwrap().len(), 1);
|
||||||
|
assert_eq!(first["thread"][0]["body"], "Latest decision");
|
||||||
|
assert_eq!(first["event_page"]["has_more"], true);
|
||||||
|
let second = surface.call("ShowTicket", serde_json::json!({"id":related_key,"event_limit":1,"event_cursor":first["event_page"]["next_cursor"]}), ToolExecutionContext::direct()).await.unwrap();
|
||||||
|
assert_eq!(second["thread"].as_array().unwrap().len(), 1);
|
||||||
|
assert_eq!(second["thread"][0]["body"], "Earlier decision");
|
||||||
|
assert_eq!(backend.show(bound_id.into()).unwrap(), bound);
|
||||||
|
assert_eq!(backend.show(related.id.into()).unwrap(), related_before);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
async fn reviewer_read_uses_live_backend_membership_and_cannot_mutate_saved_ticket() {
|
||||||
|
for wip in [false, true] {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let (api, _runtime, identity, source, ticket) = tracking_fixture(dir.path()).await;
|
||||||
|
let backend = browser_ticket_backend(&api).unwrap();
|
||||||
|
let before = backend.show(ticket.clone().into()).unwrap();
|
||||||
|
let transport = ticket_client(&api, identity, &source);
|
||||||
|
let client = reviewer_client(transport.clone(), &ticket);
|
||||||
|
let surface = TicketSurface::with_access(
|
||||||
|
dir.path(),
|
||||||
|
client.clone(),
|
||||||
|
wip,
|
||||||
|
TicketFeatureAccess::review(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
surface
|
||||||
|
.call(
|
||||||
|
"ShowTicket",
|
||||||
|
serde_json::json!({"id":ticket}),
|
||||||
|
ToolExecutionContext::direct(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let request_count = transport.requests.lock().unwrap().len();
|
||||||
|
for (method, path, body) in [
|
||||||
|
(
|
||||||
|
WorkspaceRequestMethod::Post,
|
||||||
|
format!("/api/w/{TEST_WORKSPACE_ID}/tickets/{ticket}/comments"),
|
||||||
|
serde_json::json!({"body":"unauthorized comment"}),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
WorkspaceRequestMethod::Patch,
|
||||||
|
format!("/api/w/{TEST_WORKSPACE_ID}/tickets/{ticket}"),
|
||||||
|
serde_json::json!({"title":"unauthorized title"}),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
WorkspaceRequestMethod::Post,
|
||||||
|
format!("/api/w/another-workspace/tickets/{ticket}/show"),
|
||||||
|
serde_json::json!({}),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
WorkspaceRequestMethod::Post,
|
||||||
|
format!("/api/w/{TEST_WORKSPACE_ID}/merge-requests/other-candidate/reviews"),
|
||||||
|
serde_json::json!({"verdict":"approved"}),
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
let client = client.clone();
|
||||||
|
let error = tokio::task::spawn_blocking(move || {
|
||||||
|
client.execute(WorkspaceRequest::json(method, path, body.to_string()))
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(error.to_string().contains("read-only"), "{error}");
|
||||||
|
}
|
||||||
|
assert_eq!(transport.requests.lock().unwrap().len(), request_count);
|
||||||
|
assert_eq!(backend.show(ticket.clone().into()).unwrap(), before);
|
||||||
|
// Keep a valid Runtime signature but select an unregistered Worker.
|
||||||
|
// This must fail at the real Backend, not the Reviewer route filter.
|
||||||
|
*transport.source.lock().unwrap() =
|
||||||
|
RuntimeWorkerRef::new(source.runtime_id, uuid::Uuid::now_v7().to_string());
|
||||||
|
let error = surface
|
||||||
|
.call(
|
||||||
|
"ShowTicket",
|
||||||
|
serde_json::json!({"id":ticket}),
|
||||||
|
ToolExecutionContext::direct(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(error.to_string().contains("401"), "{error}");
|
||||||
|
assert!(
|
||||||
|
transport
|
||||||
|
.requests
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.skip(request_count)
|
||||||
|
.any(|(_, status, _)| *status == 401)
|
||||||
|
);
|
||||||
|
assert_eq!(backend.show(ticket.into()).unwrap(), before);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn report_counts(api: &WorkspaceApi) -> (i64, i64) {
|
fn report_counts(api: &WorkspaceApi) -> (i64, i64) {
|
||||||
let conn = rusqlite::Connection::open(&api.config.database_path).unwrap();
|
let conn = rusqlite::Connection::open(&api.config.database_path).unwrap();
|
||||||
(
|
(
|
||||||
|
|||||||
@@ -819,6 +819,7 @@ mod workspace_config_integration {
|
|||||||
},
|
},
|
||||||
"owner".into(),
|
"owner".into(),
|
||||||
WorkingDirectoryRemovalRequest {
|
WorkingDirectoryRemovalRequest {
|
||||||
|
operation_id: "unsupported-logical-cleanup".into(),
|
||||||
reason: "unsupported logical cleanup".into()
|
reason: "unsupported logical cleanup".into()
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -187,7 +187,14 @@ async fn config_call(
|
|||||||
runtime
|
runtime
|
||||||
.invoke(
|
.invoke(
|
||||||
path.into(),
|
path.into(),
|
||||||
if matches!(operation,"edit"|"write") {wip_protocol::InterfaceReference {scope:"/".into(),name:format!("yoi.file/{operation}-text/v1")}} else {config_interface(path)},
|
if matches!(operation, "edit" | "write") {
|
||||||
|
wip_protocol::InterfaceReference {
|
||||||
|
scope: "/".into(),
|
||||||
|
name: format!("yoi.file/{operation}-text/v1"),
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
config_interface(path)
|
||||||
|
},
|
||||||
operation.into(),
|
operation.into(),
|
||||||
args,
|
args,
|
||||||
agen::tool::ToolExecutionContext::direct(),
|
agen::tool::ToolExecutionContext::direct(),
|
||||||
@@ -581,11 +588,88 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for ConfigCaptureFacto
|
|||||||
*self.controller_abort.lock().unwrap() = Some(controller.controller_task.abort_handle());
|
*self.controller_abort.lock().unwrap() = Some(controller.controller_task.abort_handle());
|
||||||
Ok(controller)
|
Ok(controller)
|
||||||
}
|
}
|
||||||
async fn restore_controller(
|
async fn preflight_restore(
|
||||||
|
&self,
|
||||||
|
request: &worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||||
|
) -> std::result::Result<(), String> {
|
||||||
|
self.inner.preflight_restore(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn recover_restore_source(
|
||||||
|
&self,
|
||||||
|
request: &worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||||
|
) -> std::result::Result<bool, worker_runtime::worker_backend::RuntimeRestoreFailure> {
|
||||||
|
self.inner.recover_restore_source(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn prepare_restore(
|
||||||
&self,
|
&self,
|
||||||
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||||
) -> std::result::Result<worker_runtime::worker_backend::RuntimeWorkerController, String> {
|
) -> std::result::Result<
|
||||||
self.inner.restore_controller(request).await
|
Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||||
|
> {
|
||||||
|
let inner = self.inner.prepare_restore(request).await?;
|
||||||
|
Ok(Box::new(ConfigCaptureCandidate {
|
||||||
|
inner,
|
||||||
|
client: self.client.clone(),
|
||||||
|
controller_abort: self.controller_abort.clone(),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn cleanup_failed_restore(
|
||||||
|
&self,
|
||||||
|
request: &worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||||
|
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
|
||||||
|
self.inner.cleanup_failed_restore(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn reconcile_stopped_worker(
|
||||||
|
&self,
|
||||||
|
worker_ref: &worker_runtime::identity::WorkerRef,
|
||||||
|
) -> std::result::Result<(), String> {
|
||||||
|
self.inner.reconcile_stopped_worker(worker_ref).await
|
||||||
|
}
|
||||||
|
|
||||||
|
fn activate_restored_controller(
|
||||||
|
&self,
|
||||||
|
worker_ref: &worker_runtime::identity::WorkerRef,
|
||||||
|
workspace_id: Option<&str>,
|
||||||
|
handle: &worker::WorkerHandle,
|
||||||
|
) -> std::result::Result<(), String> {
|
||||||
|
self.inner
|
||||||
|
.activate_restored_controller(worker_ref, workspace_id, handle)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct ConfigCaptureCandidate {
|
||||||
|
inner: Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
|
||||||
|
client: Arc<Mutex<Option<Arc<dyn worker::WorkspaceClient>>>>,
|
||||||
|
controller_abort: Arc<Mutex<Option<tokio::task::AbortHandle>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl worker_runtime::worker_backend::RuntimeRestoreCandidate for ConfigCaptureCandidate {
|
||||||
|
fn initial_state(&self) -> protocol::WorkerStateSnapshot {
|
||||||
|
self.inner.initial_state()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn activate(
|
||||||
|
&mut self,
|
||||||
|
) -> std::result::Result<
|
||||||
|
worker_runtime::worker_backend::RuntimeWorkerController,
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||||
|
> {
|
||||||
|
let controller = self.inner.activate().await?;
|
||||||
|
*self.client.lock().unwrap() = Some(controller.workspace_client.clone());
|
||||||
|
*self.controller_abort.lock().unwrap() = Some(controller.controller_task.abort_handle());
|
||||||
|
Ok(controller)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn discard(
|
||||||
|
&mut self,
|
||||||
|
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
|
||||||
|
self.inner.discard().await
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -714,6 +798,7 @@ async fn workspace_config_actual_remote_and_embedded_clients_use_authenticated_w
|
|||||||
RUNTIME_REQUEST_SOURCE_PROOF_HEADER, WORKSPACE_REQUEST_PERMISSION,
|
RUNTIME_REQUEST_SOURCE_PROOF_HEADER, WORKSPACE_REQUEST_PERMISSION,
|
||||||
decode_runtime_request_source_claims,
|
decode_runtime_request_source_claims,
|
||||||
};
|
};
|
||||||
|
let root = restored_parent_control_tests::connected_restore_tempdir();
|
||||||
// Remote Runtime-owned HTTP adapter, including its real request signer.
|
// Remote Runtime-owned HTTP adapter, including its real request signer.
|
||||||
let mut fixture = manual_worker_assignment_fixture().await;
|
let mut fixture = manual_worker_assignment_fixture().await;
|
||||||
let identity = RuntimeIdentityMaterial::generate(&fixture.worker.runtime_id).unwrap();
|
let identity = RuntimeIdentityMaterial::generate(&fixture.worker.runtime_id).unwrap();
|
||||||
@@ -793,7 +878,6 @@ async fn workspace_config_actual_remote_and_embedded_clients_use_authenticated_w
|
|||||||
|
|
||||||
// Actual embedded Runtime factory -> real Worker controller -> captured
|
// Actual embedded Runtime factory -> real Worker controller -> captured
|
||||||
// production WorkspaceClient -> signed HTTP -> production Backend middleware.
|
// production WorkspaceClient -> signed HTTP -> production Backend middleware.
|
||||||
let root = tempfile::tempdir().unwrap();
|
|
||||||
init_clean_git_workspace(root.path());
|
init_clean_git_workspace(root.path());
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
let addr = listener.local_addr().unwrap();
|
let addr = listener.local_addr().unwrap();
|
||||||
|
|||||||
@@ -119,7 +119,7 @@ pub use ticket::{
|
|||||||
};
|
};
|
||||||
|
|
||||||
const OLDEST_SCHEMA_VERSION: i64 = 50;
|
const OLDEST_SCHEMA_VERSION: i64 = 50;
|
||||||
const LATEST_SCHEMA_VERSION: i64 = 103;
|
const LATEST_SCHEMA_VERSION: i64 = 104;
|
||||||
const WORKSPACE_DRIVE_GRANTS_MIGRATION_NAME: &str = "durable Workspace Drive grants";
|
const WORKSPACE_DRIVE_GRANTS_MIGRATION_NAME: &str = "durable Workspace Drive grants";
|
||||||
const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs";
|
const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs";
|
||||||
const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline";
|
const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline";
|
||||||
@@ -439,11 +439,24 @@ const MIGRATIONS: &[Migration] = &[
|
|||||||
},
|
},
|
||||||
Migration {
|
Migration {
|
||||||
version: 103,
|
version: 103,
|
||||||
name: "Job input retirement independent of immutable receipts",
|
name: "Passkey metadata and atomic addition results",
|
||||||
apply: migrate_job_input_retention_v102_to_v103,
|
apply: migrate_passkey_results_v102_to_v103,
|
||||||
|
},
|
||||||
|
Migration {
|
||||||
|
version: 104,
|
||||||
|
name: "Converge Passkey results and Job input retention",
|
||||||
|
apply: migrate_schema_103_variants_to_v104,
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
|
// Both schema-103 identities shipped. Keep the alternate implementation and accept
|
||||||
|
// its exact receipt without renaming or replacing any applied history.
|
||||||
|
const JOB_INPUT_RETENTION_SCHEMA_103: Migration = Migration {
|
||||||
|
version: 103,
|
||||||
|
name: "Job input retirement independent of immutable receipts",
|
||||||
|
apply: migrate_job_input_retention_v102_to_v103,
|
||||||
|
};
|
||||||
|
|
||||||
#[derive(Clone, Copy)]
|
#[derive(Clone, Copy)]
|
||||||
struct Migration {
|
struct Migration {
|
||||||
version: i64,
|
version: i64,
|
||||||
@@ -754,6 +767,15 @@ pub enum PasskeyAuthenticationBinding {
|
|||||||
ApiToken(String),
|
ApiToken(String),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl PasskeyAuthenticationBinding {
|
||||||
|
pub(crate) fn reference(&self) -> String {
|
||||||
|
let hash = match self {
|
||||||
|
Self::BrowserSession(hash) | Self::ApiToken(hash) => hash,
|
||||||
|
};
|
||||||
|
crate::auth::authentication_reference(hash)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
pub struct PasskeyCredentialRecord {
|
pub struct PasskeyCredentialRecord {
|
||||||
pub credential_id: String,
|
pub credential_id: String,
|
||||||
@@ -1718,6 +1740,27 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore {
|
|||||||
&self,
|
&self,
|
||||||
target: &PasskeyRegistrationTarget,
|
target: &PasskeyRegistrationTarget,
|
||||||
credential: &PasskeyCredentialRecord,
|
credential: &PasskeyCredentialRecord,
|
||||||
|
) -> Result<()> {
|
||||||
|
self.register_verified_passkey_with_result(target, credential, None)
|
||||||
|
}
|
||||||
|
/// Receipt and credential commit together; no receipt claims session issuance.
|
||||||
|
fn register_verified_passkey_with_result(
|
||||||
|
&self,
|
||||||
|
target: &PasskeyRegistrationTarget,
|
||||||
|
credential: &PasskeyCredentialRecord,
|
||||||
|
addition_challenge_id: Option<&str>,
|
||||||
|
) -> Result<()>;
|
||||||
|
fn list_passkey_summaries(&self, user_id: &str) -> Result<Vec<server_api::PasskeySummary>>;
|
||||||
|
fn read_passkey_addition_result(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
original_authentication_reference: &str,
|
||||||
|
challenge_id: &str,
|
||||||
|
) -> Result<Option<server_api::PasskeySummary>>;
|
||||||
|
fn mark_passkey_normal_login_verified(
|
||||||
|
&self,
|
||||||
|
credential_id: &str,
|
||||||
|
verified_at: &str,
|
||||||
) -> Result<()>;
|
) -> Result<()>;
|
||||||
fn upsert_passkey_credential(&self, record: &PasskeyCredentialRecord) -> Result<()>;
|
fn upsert_passkey_credential(&self, record: &PasskeyCredentialRecord) -> Result<()>;
|
||||||
fn get_passkey_credential(
|
fn get_passkey_credential(
|
||||||
@@ -2603,6 +2646,10 @@ pub struct SqliteWorkspaceStore {
|
|||||||
db_executor: crate::db_executor::DbExecutor,
|
db_executor: crate::db_executor::DbExecutor,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "test-support")]
|
||||||
|
#[path = "test_support.rs"]
|
||||||
|
pub mod fixture_support;
|
||||||
|
|
||||||
impl SqliteWorkspaceStore {
|
impl SqliteWorkspaceStore {
|
||||||
pub fn open(path: impl AsRef<Path>) -> Result<Self> {
|
pub fn open(path: impl AsRef<Path>) -> Result<Self> {
|
||||||
let conn = Connection::open(path)?;
|
let conn = Connection::open(path)?;
|
||||||
@@ -2914,6 +2961,33 @@ impl SqliteWorkspaceStore {
|
|||||||
request_fingerprint: &str,
|
request_fingerprint: &str,
|
||||||
singleton_key: Option<&str>,
|
singleton_key: Option<&str>,
|
||||||
current_memory_settings: &WorkspaceMemorySettingsRecord,
|
current_memory_settings: &WorkspaceMemorySettingsRecord,
|
||||||
|
) -> Result<WorkerCreateReservation> {
|
||||||
|
self.reserve_worker_create_with(
|
||||||
|
workspace_id,
|
||||||
|
runtime_id,
|
||||||
|
allocation_key,
|
||||||
|
request_fingerprint,
|
||||||
|
singleton_key,
|
||||||
|
current_memory_settings,
|
||||||
|
|_| Ok(None),
|
||||||
|
|_, _| Ok(()),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Keep reservation validation, Memory binding and writes in one transaction. The
|
||||||
|
// nondefault test-support module supplies fixed identity / offline catalog hooks;
|
||||||
|
// ordinary creation always supplies a newly generated ID and no completion hook.
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
fn reserve_worker_create_with(
|
||||||
|
&self,
|
||||||
|
workspace_id: &str,
|
||||||
|
runtime_id: &str,
|
||||||
|
allocation_key: &str,
|
||||||
|
request_fingerprint: &str,
|
||||||
|
singleton_key: Option<&str>,
|
||||||
|
current_memory_settings: &WorkspaceMemorySettingsRecord,
|
||||||
|
prepare: impl FnOnce(&rusqlite::Transaction<'_>) -> Result<Option<WorkerId>>,
|
||||||
|
complete: impl FnOnce(&rusqlite::Transaction<'_>, &WorkerCreateReservation) -> Result<()>,
|
||||||
) -> Result<WorkerCreateReservation> {
|
) -> Result<WorkerCreateReservation> {
|
||||||
if allocation_key.trim().is_empty() || request_fingerprint.trim().is_empty() {
|
if allocation_key.trim().is_empty() || request_fingerprint.trim().is_empty() {
|
||||||
return Err(Error::InvalidInput(
|
return Err(Error::InvalidInput(
|
||||||
@@ -2926,6 +3000,7 @@ impl SqliteWorkspaceStore {
|
|||||||
validate_workspace_memory_settings_record(current_memory_settings, workspace_id)?;
|
validate_workspace_memory_settings_record(current_memory_settings, workspace_id)?;
|
||||||
self.with_conn_mut(|conn| {
|
self.with_conn_mut(|conn| {
|
||||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||||
|
let fixed_worker_id = prepare(&tx)?;
|
||||||
let existing = tx
|
let existing = tx
|
||||||
.query_row(
|
.query_row(
|
||||||
"SELECT worker_id, runtime_id, request_fingerprint, create_fingerprint, \
|
"SELECT worker_id, runtime_id, request_fingerprint, create_fingerprint, \
|
||||||
@@ -3037,8 +3112,8 @@ impl SqliteWorkspaceStore {
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
.optional()?;
|
.optional()?;
|
||||||
// Subject occupancy survives removal until an explicit release
|
// An unconfirmed legacy removal cannot release Subject occupancy.
|
||||||
// proof. It must not resurrect the removed creation reservation:
|
// Completed removal or explicit release does; never resurrect a reservation:
|
||||||
// Runtime callbacks require a live reservation or catalog member.
|
// Runtime callbacks require a live reservation or catalog member.
|
||||||
if stored.as_ref().is_some_and(|row| row.6 == "removed") {
|
if stored.as_ref().is_some_and(|row| row.6 == "removed") {
|
||||||
return Err(Error::RepositoryConflict(format!(
|
return Err(Error::RepositoryConflict(format!(
|
||||||
@@ -3108,7 +3183,7 @@ impl SqliteWorkspaceStore {
|
|||||||
validate_workspace_memory_settings_snapshot(&snapshot, workspace_id)?;
|
validate_workspace_memory_settings_snapshot(&snapshot, workspace_id)?;
|
||||||
let create_fingerprint =
|
let create_fingerprint =
|
||||||
bound_worker_create_fingerprint(request_fingerprint, &snapshot);
|
bound_worker_create_fingerprint(request_fingerprint, &snapshot);
|
||||||
let worker_id = WorkerId::now_v7();
|
let worker_id = fixed_worker_id.unwrap_or_else(WorkerId::now_v7);
|
||||||
let worker_id_text = worker_id.to_string();
|
let worker_id_text = worker_id.to_string();
|
||||||
let now = chrono::Utc::now().to_rfc3339();
|
let now = chrono::Utc::now().to_rfc3339();
|
||||||
let singleton_generation = match singleton_owner.as_ref() {
|
let singleton_generation = match singleton_owner.as_ref() {
|
||||||
@@ -3197,13 +3272,15 @@ impl SqliteWorkspaceStore {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
tx.commit()?;
|
let reservation = WorkerCreateReservation {
|
||||||
Ok(WorkerCreateReservation {
|
|
||||||
worker_id,
|
worker_id,
|
||||||
create_fingerprint,
|
create_fingerprint,
|
||||||
memory_settings: snapshot,
|
memory_settings: snapshot,
|
||||||
singleton,
|
singleton,
|
||||||
})
|
};
|
||||||
|
complete(&tx, &reservation)?;
|
||||||
|
tx.commit()?;
|
||||||
|
Ok(reservation)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -6952,10 +7029,11 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn register_verified_passkey(
|
fn register_verified_passkey_with_result(
|
||||||
&self,
|
&self,
|
||||||
target: &PasskeyRegistrationTarget,
|
target: &PasskeyRegistrationTarget,
|
||||||
credential: &PasskeyCredentialRecord,
|
credential: &PasskeyCredentialRecord,
|
||||||
|
addition_challenge_id: Option<&str>,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
self.with_immediate_transaction(|tx| {
|
self.with_immediate_transaction(|tx| {
|
||||||
let user = match target {
|
let user = match target {
|
||||||
@@ -7022,6 +7100,15 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
|||||||
"INSERT INTO passkey_credentials (credential_id, user_id, public_key_cose, transports_json, sign_count, created_at, last_used_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)",
|
"INSERT INTO passkey_credentials (credential_id, user_id, public_key_cose, transports_json, sign_count, created_at, last_used_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)",
|
||||||
params![credential.credential_id, credential.user_id, credential.public_key_cose, credential.transports_json, credential.sign_count, credential.created_at, credential.last_used_at],
|
params![credential.credential_id, credential.user_id, credential.public_key_cose, credential.transports_json, credential.sign_count, credential.created_at, credential.last_used_at],
|
||||||
)?;
|
)?;
|
||||||
|
if let Some(challenge_id) = addition_challenge_id {
|
||||||
|
let PasskeyRegistrationTarget::Addition { user, authentication } = target else {
|
||||||
|
return Err(crate::auth::auth_error("passkey_target_mismatch", "receipt requires an addition"));
|
||||||
|
};
|
||||||
|
tx.execute(
|
||||||
|
"INSERT INTO passkey_addition_receipts(challenge_id, credential_id, user_id, authentication_reference) VALUES (?1, ?2, ?3, ?4)",
|
||||||
|
params![challenge_id, credential.credential_id, user.user_id, authentication.reference()],
|
||||||
|
)?;
|
||||||
|
}
|
||||||
if let PasskeyRegistrationTarget::Activation { user, issuance_id } = target {
|
if let PasskeyRegistrationTarget::Activation { user, issuance_id } = target {
|
||||||
if tx.execute("DELETE FROM user_activations WHERE user_id=?1 AND issuance_id=?2", params![user.user_id, issuance_id])? != 1 {
|
if tx.execute("DELETE FROM user_activations WHERE user_id=?1 AND issuance_id=?2", params![user.user_id, issuance_id])? != 1 {
|
||||||
return Err(crate::auth::auth_error("activation_code_invalid", "activation issuance no longer exists"));
|
return Err(crate::auth::auth_error("activation_code_invalid", "activation issuance no longer exists"));
|
||||||
@@ -7031,6 +7118,57 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn list_passkey_summaries(&self, user_id: &str) -> Result<Vec<server_api::PasskeySummary>> {
|
||||||
|
self.with_conn(|conn| {
|
||||||
|
let mut stmt = conn.prepare(
|
||||||
|
"SELECT m.key_reference, c.created_at, c.last_used_at, m.normal_login_verified_at
|
||||||
|
FROM passkey_credentials c JOIN passkey_metadata m USING(credential_id)
|
||||||
|
WHERE c.user_id = ?1 ORDER BY c.created_at, m.key_reference",
|
||||||
|
)?;
|
||||||
|
let rows = stmt.query_map([user_id], read_passkey_summary)?;
|
||||||
|
rows.collect::<std::result::Result<Vec<_>, _>>()
|
||||||
|
.map_err(Error::from)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_passkey_addition_result(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
original_authentication_reference: &str,
|
||||||
|
challenge_id: &str,
|
||||||
|
) -> Result<Option<server_api::PasskeySummary>> {
|
||||||
|
self.with_conn(|conn| {
|
||||||
|
// Current authenticated ownership authorizes reading. The original
|
||||||
|
// binding reference identifies the saved operation, not the current
|
||||||
|
// session: Set-Cookie may have arrived even if its body was lost.
|
||||||
|
let result = conn.query_row(
|
||||||
|
"SELECT m.key_reference, c.created_at, c.last_used_at, m.normal_login_verified_at, r.authentication_reference
|
||||||
|
FROM passkey_addition_receipts r
|
||||||
|
JOIN passkey_credentials c USING(credential_id)
|
||||||
|
JOIN passkey_metadata m USING(credential_id)
|
||||||
|
WHERE r.challenge_id = ?1 AND r.user_id = ?2 AND c.user_id = ?2",
|
||||||
|
params![challenge_id, user_id],
|
||||||
|
|row| Ok((read_passkey_summary(row)?, row.get::<_, String>(4)?)),
|
||||||
|
).optional()?;
|
||||||
|
match result {
|
||||||
|
Some((key, reference)) if reference == original_authentication_reference => Ok(Some(key)),
|
||||||
|
Some(_) => Err(crate::auth::auth_error("passkey_result_binding_mismatch", "original operation authentication reference does not match")),
|
||||||
|
None => Ok(None),
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn mark_passkey_normal_login_verified(
|
||||||
|
&self,
|
||||||
|
credential_id: &str,
|
||||||
|
verified_at: &str,
|
||||||
|
) -> Result<()> {
|
||||||
|
self.with_conn(|conn| {
|
||||||
|
conn.execute("UPDATE passkey_metadata SET normal_login_verified_at = ?2 WHERE credential_id = ?1", params![credential_id, verified_at])?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
fn upsert_passkey_credential(&self, record: &PasskeyCredentialRecord) -> Result<()> {
|
fn upsert_passkey_credential(&self, record: &PasskeyCredentialRecord) -> Result<()> {
|
||||||
self.with_immediate_transaction(|conn| {
|
self.with_immediate_transaction(|conn| {
|
||||||
activation::require_active(conn, &record.user_id)?;
|
activation::require_active(conn, &record.user_id)?;
|
||||||
@@ -10981,49 +11119,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
|||||||
record.workdir_id
|
record.workdir_id
|
||||||
)));
|
)));
|
||||||
}
|
}
|
||||||
let active_for_alias = tx
|
// Ownership is keyed by Workdir, not by the requested local name.
|
||||||
.query_row(
|
|
||||||
r#"SELECT workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, linked_at, unlinked_at, connection_id
|
|
||||||
FROM worker_workdir_links
|
|
||||||
WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3
|
|
||||||
AND alias = ?4 AND unlinked_at IS NULL"#,
|
|
||||||
params![
|
|
||||||
record.workspace_id,
|
|
||||||
record.worker.runtime_id,
|
|
||||||
record.worker.worker_id,
|
|
||||||
record.alias,
|
|
||||||
],
|
|
||||||
read_worker_workdir_link_record,
|
|
||||||
)
|
|
||||||
.optional()?;
|
|
||||||
if let Some(mut active) = active_for_alias {
|
|
||||||
if active.workdir_id == record.workdir_id {
|
|
||||||
let capabilities = active.capabilities.intersection(record.capabilities);
|
|
||||||
if capabilities != active.capabilities {
|
|
||||||
tx.execute(
|
|
||||||
r#"UPDATE worker_workdir_links
|
|
||||||
SET capabilities = ?1
|
|
||||||
WHERE workspace_id = ?2 AND runtime_id = ?3 AND worker_id = ?4
|
|
||||||
AND workdir_id = ?5 AND alias = ?6 AND unlinked_at IS NULL"#,
|
|
||||||
params![
|
|
||||||
encode_workdir_link_capabilities(capabilities)?,
|
|
||||||
record.workspace_id,
|
|
||||||
record.worker.runtime_id,
|
|
||||||
record.worker.worker_id,
|
|
||||||
record.workdir_id,
|
|
||||||
record.alias,
|
|
||||||
],
|
|
||||||
)?;
|
|
||||||
active.capabilities = capabilities;
|
|
||||||
}
|
|
||||||
tx.commit()?;
|
|
||||||
return Ok(active);
|
|
||||||
}
|
|
||||||
return Err(Error::WorkdirAttachmentConflict(format!(
|
|
||||||
"Worker {}:{} attachment alias `{}` is already bound to Workdir {}",
|
|
||||||
record.worker.runtime_id, record.worker.worker_id, record.alias, active.workdir_id
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
let active_for_workdir = tx
|
let active_for_workdir = tx
|
||||||
.query_row(
|
.query_row(
|
||||||
r#"SELECT workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, linked_at, unlinked_at, connection_id
|
r#"SELECT workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, linked_at, unlinked_at, connection_id
|
||||||
@@ -11033,10 +11129,45 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
|||||||
read_worker_workdir_link_record,
|
read_worker_workdir_link_record,
|
||||||
)
|
)
|
||||||
.optional()?;
|
.optional()?;
|
||||||
if let Some(active) = active_for_workdir {
|
if let Some(mut active) = active_for_workdir {
|
||||||
|
if active.worker != record.worker {
|
||||||
|
return Err(Error::WorkdirOccupied {
|
||||||
|
workdir_id: record.workdir_id.clone(),
|
||||||
|
runtime_id: active.worker.runtime_id,
|
||||||
|
worker_id: active.worker.worker_id,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
let capabilities = active.capabilities.intersection(record.capabilities);
|
||||||
|
if capabilities != active.capabilities {
|
||||||
|
tx.execute(
|
||||||
|
r#"UPDATE worker_workdir_links SET capabilities = ?1
|
||||||
|
WHERE workspace_id = ?2 AND runtime_id = ?3 AND worker_id = ?4
|
||||||
|
AND workdir_id = ?5 AND alias = ?6 AND unlinked_at IS NULL"#,
|
||||||
|
params![
|
||||||
|
encode_workdir_link_capabilities(capabilities)?,
|
||||||
|
active.workspace_id, active.worker.runtime_id, active.worker.worker_id,
|
||||||
|
active.workdir_id, active.alias,
|
||||||
|
],
|
||||||
|
)?;
|
||||||
|
active.capabilities = capabilities;
|
||||||
|
}
|
||||||
|
tx.commit()?;
|
||||||
|
return Ok(active);
|
||||||
|
}
|
||||||
|
let active_for_alias = tx
|
||||||
|
.query_row(
|
||||||
|
r#"SELECT workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, linked_at, unlinked_at, connection_id
|
||||||
|
FROM worker_workdir_links
|
||||||
|
WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3
|
||||||
|
AND alias = ?4 AND unlinked_at IS NULL"#,
|
||||||
|
params![record.workspace_id, record.worker.runtime_id, record.worker.worker_id, record.alias],
|
||||||
|
read_worker_workdir_link_record,
|
||||||
|
)
|
||||||
|
.optional()?;
|
||||||
|
if let Some(active) = active_for_alias {
|
||||||
return Err(Error::WorkdirAttachmentConflict(format!(
|
return Err(Error::WorkdirAttachmentConflict(format!(
|
||||||
"Workdir {} is already attached to Worker {}:{}",
|
"Worker {}:{} attachment alias `{}` is already bound to Workdir {}",
|
||||||
record.workdir_id, active.worker.runtime_id, active.worker.worker_id
|
record.worker.runtime_id, record.worker.worker_id, record.alias, active.workdir_id
|
||||||
)));
|
)));
|
||||||
}
|
}
|
||||||
// Direct attachments (including ordinary recovery/reconnection)
|
// Direct attachments (including ordinary recovery/reconnection)
|
||||||
@@ -11939,6 +12070,15 @@ fn passkey_select_sql(where_clause: &str) -> String {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn read_passkey_summary(row: &rusqlite::Row<'_>) -> rusqlite::Result<server_api::PasskeySummary> {
|
||||||
|
Ok(server_api::PasskeySummary {
|
||||||
|
key_reference: row.get(0)?,
|
||||||
|
created_at: row.get(1)?,
|
||||||
|
last_used_at: row.get(2)?,
|
||||||
|
normal_login_verified_at: row.get(3)?,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
fn read_passkey_credential_record(
|
fn read_passkey_credential_record(
|
||||||
row: &rusqlite::Row<'_>,
|
row: &rusqlite::Row<'_>,
|
||||||
) -> rusqlite::Result<PasskeyCredentialRecord> {
|
) -> rusqlite::Result<PasskeyCredentialRecord> {
|
||||||
@@ -15218,6 +15358,22 @@ fn migrate_result_reports_v97_to_v98(conn: &Connection) -> Result<()> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn migrate_passkey_results_v102_to_v103(conn: &Connection) -> Result<()> {
|
||||||
|
if current_schema_version(conn)? != 102 {
|
||||||
|
return Err(Error::Store(
|
||||||
|
"Passkey results migration requires schema 102".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let tx = conn.unchecked_transaction()?;
|
||||||
|
tx.execute_batch(include_str!("passkey_results_schema.sql"))?;
|
||||||
|
tx.execute(
|
||||||
|
"INSERT INTO __yoi_schema_migrations(version,name) VALUES(103,?1)",
|
||||||
|
["Passkey metadata and atomic addition results"],
|
||||||
|
)?;
|
||||||
|
tx.commit()?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
fn migrate_job_input_retention_v102_to_v103(conn: &Connection) -> Result<()> {
|
fn migrate_job_input_retention_v102_to_v103(conn: &Connection) -> Result<()> {
|
||||||
if current_schema_version(conn)? != 102 {
|
if current_schema_version(conn)? != 102 {
|
||||||
return Err(Error::Store(
|
return Err(Error::Store(
|
||||||
@@ -15236,6 +15392,73 @@ fn migrate_job_input_retention_v102_to_v103(conn: &Connection) -> Result<()> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn migrate_schema_103_variants_to_v104(conn: &Connection) -> Result<()> {
|
||||||
|
if current_schema_version(conn)? != 103 {
|
||||||
|
return Err(Error::Store(
|
||||||
|
"Schema convergence migration requires schema 103".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
verify_schema_history(conn, 103)?;
|
||||||
|
let tx = conn.unchecked_transaction()?;
|
||||||
|
// Fresh databases on either branch recorded only a schema-103 baseline, so
|
||||||
|
// the receipt alone cannot identify which feature needs to be installed.
|
||||||
|
// Never rebuild existing metadata or receipts: key references and retired
|
||||||
|
// Job input dispositions are already externally observable saved values.
|
||||||
|
let passkey_results = table_exists(&tx, "passkey_metadata")?;
|
||||||
|
let job_retention = column_exists(&tx, "backend_jobs", "input_disposition")?;
|
||||||
|
let identity: String = tx.query_row(
|
||||||
|
"SELECT name FROM __yoi_schema_migrations WHERE version=103",
|
||||||
|
[],
|
||||||
|
|row| row.get(0),
|
||||||
|
)?;
|
||||||
|
let matches_identity = match identity.as_str() {
|
||||||
|
SCHEMA_BASELINE_NAME => passkey_results != job_retention,
|
||||||
|
"Passkey metadata and atomic addition results" => passkey_results && !job_retention,
|
||||||
|
"Job input retirement independent of immutable receipts" => {
|
||||||
|
job_retention && !passkey_results
|
||||||
|
}
|
||||||
|
_ => false,
|
||||||
|
};
|
||||||
|
if !matches_identity {
|
||||||
|
return Err(Error::Store(
|
||||||
|
"Schema-103 features do not match the applied migration identity".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if !passkey_results {
|
||||||
|
tx.execute_batch(include_str!("passkey_results_schema.sql"))?;
|
||||||
|
}
|
||||||
|
if !job_retention {
|
||||||
|
tx.execute_batch("ALTER TABLE backend_jobs ADD COLUMN input_disposition TEXT NOT NULL DEFAULT 'available' CHECK (input_disposition IN ('available','reclaimed','erased','invalidated'));")?;
|
||||||
|
tx.execute_batch(crate::backend_job::INPUT_RETENTION_TRIGGERS)?;
|
||||||
|
}
|
||||||
|
let objects: i64 = tx.query_row(
|
||||||
|
"SELECT count(*) FROM sqlite_schema WHERE
|
||||||
|
(type='table' AND name IN ('passkey_metadata','passkey_addition_receipts')) OR
|
||||||
|
(type='trigger' AND name IN ('passkey_metadata_insert','backend_job_identity_immutable','backend_job_input_retirement'))",
|
||||||
|
[],
|
||||||
|
|row| row.get(0),
|
||||||
|
)?;
|
||||||
|
if objects != 5 {
|
||||||
|
return Err(Error::Store("Incomplete schema-103 feature schema".into()));
|
||||||
|
}
|
||||||
|
verify_workspace_resource_constraints(&tx)?;
|
||||||
|
let violations: i64 =
|
||||||
|
tx.query_row("SELECT count(*) FROM pragma_foreign_key_check", [], |row| {
|
||||||
|
row.get(0)
|
||||||
|
})?;
|
||||||
|
if violations != 0 {
|
||||||
|
return Err(Error::Store(
|
||||||
|
"Schema convergence foreign-key violation".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
tx.execute(
|
||||||
|
"INSERT INTO __yoi_schema_migrations(version,name) VALUES(104,?1)",
|
||||||
|
["Converge Passkey results and Job input retention"],
|
||||||
|
)?;
|
||||||
|
tx.commit()?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
fn migrate_subject_create_recovery_v101_to_v102(conn: &Connection) -> Result<()> {
|
fn migrate_subject_create_recovery_v101_to_v102(conn: &Connection) -> Result<()> {
|
||||||
if current_schema_version(conn)? != 101 {
|
if current_schema_version(conn)? != 101 {
|
||||||
return Err(Error::Store(
|
return Err(Error::Store(
|
||||||
@@ -15327,6 +15550,7 @@ fn create_latest_workspace_schema(conn: &Connection) -> Result<()> {
|
|||||||
fn create_workspace_schema_structure(conn: &Connection) -> Result<()> {
|
fn create_workspace_schema_structure(conn: &Connection) -> Result<()> {
|
||||||
create_workspace_schema_structure_from_sql(conn, include_str!("latest_schema.sql"), true)?;
|
create_workspace_schema_structure_from_sql(conn, include_str!("latest_schema.sql"), true)?;
|
||||||
conn.execute_batch(include_str!("activation_schema.sql"))?;
|
conn.execute_batch(include_str!("activation_schema.sql"))?;
|
||||||
|
conn.execute_batch(include_str!("passkey_results_schema.sql"))?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -15992,7 +16216,7 @@ fn current_worker_singleton_owner(
|
|||||||
CASE WHEN substr(owner.singleton_key,1,10)='subjektiv:' AND reservation.state='removed' \
|
CASE WHEN substr(owner.singleton_key,1,10)='subjektiv:' AND reservation.state='removed' \
|
||||||
AND NOT EXISTS(SELECT 1 FROM subject_worker_releases r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.completed=1) \
|
AND NOT EXISTS(SELECT 1 FROM subject_worker_releases r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.completed=1) \
|
||||||
AND NOT EXISTS(SELECT 1 FROM subject_create_recoveries r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.create_fingerprint=reservation.create_fingerprint AND r.completed=1) \
|
AND NOT EXISTS(SELECT 1 FROM subject_create_recoveries r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.create_fingerprint=reservation.create_fingerprint AND r.completed=1) \
|
||||||
THEN 'created' ELSE reservation.state END \
|
AND NOT EXISTS(SELECT 1 FROM worker_removal_operations r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.state='succeeded') THEN 'created' ELSE reservation.state END \
|
||||||
FROM worker_singleton_owners owner \
|
FROM worker_singleton_owners owner \
|
||||||
LEFT JOIN worker_create_reservations reservation \
|
LEFT JOIN worker_create_reservations reservation \
|
||||||
ON reservation.workspace_id = owner.workspace_id \
|
ON reservation.workspace_id = owner.workspace_id \
|
||||||
@@ -16198,6 +16422,17 @@ fn verify_schema_history(conn: &Connection, current: i64) -> Result<()> {
|
|||||||
.iter()
|
.iter()
|
||||||
.filter(|migration| migration.version > *baseline_version && migration.version <= current)
|
.filter(|migration| migration.version > *baseline_version && migration.version <= current)
|
||||||
{
|
{
|
||||||
|
// The two published schema-103 migrations are alternatives, not aliases.
|
||||||
|
// Match the exact saved identity while retaining every other history check.
|
||||||
|
let migration = if migration.version == JOB_INPUT_RETENTION_SCHEMA_103.version
|
||||||
|
&& rows.iter().any(|(version, name)| {
|
||||||
|
*version == JOB_INPUT_RETENTION_SCHEMA_103.version
|
||||||
|
&& name == JOB_INPUT_RETENTION_SCHEMA_103.name
|
||||||
|
}) {
|
||||||
|
&JOB_INPUT_RETENTION_SCHEMA_103
|
||||||
|
} else {
|
||||||
|
migration
|
||||||
|
};
|
||||||
expected.push((migration.version, migration.name.to_string()));
|
expected.push((migration.version, migration.name.to_string()));
|
||||||
}
|
}
|
||||||
if rows != expected || expected.last().map(|row| row.0) != Some(current) {
|
if rows != expected || expected.last().map(|row| row.0) != Some(current) {
|
||||||
@@ -18608,6 +18843,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
|||||||
|
|
||||||
include!("activation_tests.rs");
|
include!("activation_tests.rs");
|
||||||
include!("schema_equivalence_tests.rs");
|
include!("schema_equivalence_tests.rs");
|
||||||
|
include!("schema_convergence_tests.rs");
|
||||||
include!("ticket_tag_upgrade_tests.rs");
|
include!("ticket_tag_upgrade_tests.rs");
|
||||||
include!("ticket_post_upgrade_tests.rs");
|
include!("ticket_post_upgrade_tests.rs");
|
||||||
include!("migration_entrypoint_tests.rs");
|
include!("migration_entrypoint_tests.rs");
|
||||||
@@ -19020,7 +19256,8 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
|||||||
WorkspaceSchemaMigrationStep {version:100, name:"Provisional users and scoped passkey activation".into()},
|
WorkspaceSchemaMigrationStep {version:100, name:"Provisional users and scoped passkey activation".into()},
|
||||||
WorkspaceSchemaMigrationStep {version:101, name:"Durable explicit Subject body release".into()},
|
WorkspaceSchemaMigrationStep {version:101, name:"Durable explicit Subject body release".into()},
|
||||||
WorkspaceSchemaMigrationStep {version:102, name:"Durable failed Subject creation recovery".into()},
|
WorkspaceSchemaMigrationStep {version:102, name:"Durable failed Subject creation recovery".into()},
|
||||||
WorkspaceSchemaMigrationStep {version:103, name:"Job input retirement independent of immutable receipts".into()},
|
WorkspaceSchemaMigrationStep {version:103, name:"Passkey metadata and atomic addition results".into()},
|
||||||
|
WorkspaceSchemaMigrationStep {version:104, name:"Converge Passkey results and Job input retention".into()},
|
||||||
]
|
]
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -19133,7 +19370,8 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
|||||||
(100, "Provisional users and scoped passkey activation".to_string()),
|
(100, "Provisional users and scoped passkey activation".to_string()),
|
||||||
(101, "Durable explicit Subject body release".to_string()),
|
(101, "Durable explicit Subject body release".to_string()),
|
||||||
(102, "Durable failed Subject creation recovery".to_string()),
|
(102, "Durable failed Subject creation recovery".to_string()),
|
||||||
(103, "Job input retirement independent of immutable receipts".to_string()),
|
(103, "Passkey metadata and atomic addition results".to_string()),
|
||||||
|
(104, "Converge Passkey results and Job input retention".to_string()),
|
||||||
]
|
]
|
||||||
);
|
);
|
||||||
assert!(!table_exists(conn, "trusted_runtime_records")?);
|
assert!(!table_exists(conn, "trusted_runtime_records")?);
|
||||||
@@ -21362,6 +21600,83 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn passkey_results_migration_preserves_old_keys_unknown_confirmation_and_rolls_back() {
|
||||||
|
for baseline in [50, 102] {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let database = dir.path().join("server.db");
|
||||||
|
if baseline == 50 {
|
||||||
|
prepare_schema_v50(&database, None);
|
||||||
|
} else {
|
||||||
|
prepare_retained_schema(&database, baseline);
|
||||||
|
}
|
||||||
|
let conn = Connection::open(&database).unwrap();
|
||||||
|
configure_sqlite(&conn).unwrap();
|
||||||
|
conn.execute_batch(r#"INSERT INTO accounts(account_id, kind, handle, display_name, created_at, updated_at)
|
||||||
|
VALUES('account-old', 'user', 'old', 'Old', 'created', 'updated');
|
||||||
|
INSERT INTO users(user_id, account_id, handle, display_name, created_at, updated_at)
|
||||||
|
VALUES('user-old', 'account-old', 'old', 'Old', 'created', 'updated');
|
||||||
|
INSERT INTO passkey_credentials(credential_id, user_id, public_key_cose, transports_json, sign_count, created_at, last_used_at)
|
||||||
|
VALUES('old-key', 'user-old', 'unchanged-key-material', '["internal"]', 7, 'created-key', 'used-key');"#)
|
||||||
|
.unwrap();
|
||||||
|
if baseline == 102 {
|
||||||
|
conn.execute_batch("CREATE TRIGGER interrupt_passkey_migration BEFORE INSERT ON __yoi_schema_migrations
|
||||||
|
WHEN NEW.version=103 BEGIN SELECT RAISE(ABORT, 'synthetic migration interruption'); END;").unwrap();
|
||||||
|
assert!(migrate_passkey_results_v102_to_v103(&conn).is_err());
|
||||||
|
assert_eq!(current_schema_version(&conn).unwrap(), 102);
|
||||||
|
assert!(!table_exists(&conn, "passkey_metadata").unwrap());
|
||||||
|
assert!(!table_exists(&conn, "passkey_addition_receipts").unwrap());
|
||||||
|
conn.execute_batch("DROP TRIGGER interrupt_passkey_migration")
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
drop(conn);
|
||||||
|
let preflight = SqliteWorkspaceStore::migration_plan(&database).unwrap();
|
||||||
|
assert!(preflight.migration_required());
|
||||||
|
// Preflight must not install metadata in the source database.
|
||||||
|
let conn = Connection::open(&database).unwrap();
|
||||||
|
assert!(!table_exists(&conn, "passkey_metadata").unwrap());
|
||||||
|
drop(conn);
|
||||||
|
let migrated = SqliteWorkspaceStore::open_for_test(&database).unwrap();
|
||||||
|
let record = migrated.get_passkey_credential("old-key").unwrap().unwrap();
|
||||||
|
assert_eq!(record.public_key_cose, "unchanged-key-material");
|
||||||
|
assert_eq!(record.sign_count, 7);
|
||||||
|
assert_eq!(record.last_used_at.as_deref(), Some("used-key"));
|
||||||
|
let summary = migrated
|
||||||
|
.list_passkey_summaries("user-old")
|
||||||
|
.unwrap()
|
||||||
|
.remove(0);
|
||||||
|
assert_eq!(summary.created_at, "created-key");
|
||||||
|
assert_eq!(summary.last_used_at.as_deref(), Some("used-key"));
|
||||||
|
assert!(summary.normal_login_verified_at.is_none());
|
||||||
|
assert_eq!(summary.key_reference.len(), 32);
|
||||||
|
let fresh = SqliteWorkspaceStore::in_memory_for_test().unwrap();
|
||||||
|
let schema = |conn: &Connection| -> Result<Vec<(String, String)>> {
|
||||||
|
let mut stmt = conn.prepare("SELECT name, sql FROM sqlite_schema WHERE name IN
|
||||||
|
('passkey_metadata', 'passkey_metadata_insert', 'passkey_addition_receipts') ORDER BY name")?;
|
||||||
|
Ok(stmt
|
||||||
|
.query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
|
||||||
|
.collect::<std::result::Result<Vec<_>, _>>()?)
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
migrated.with_conn(schema).unwrap(),
|
||||||
|
fresh.with_conn(schema).unwrap()
|
||||||
|
);
|
||||||
|
migrated.with_conn(|conn| {
|
||||||
|
assert!(conn.execute("INSERT INTO passkey_metadata(credential_id,key_reference) VALUES('missing-key','ref')", []).is_err());
|
||||||
|
assert!(conn.execute("INSERT INTO passkey_addition_receipts(challenge_id,credential_id,user_id,authentication_reference)
|
||||||
|
VALUES('operation','missing-key','user-old','{}')", []).is_err());
|
||||||
|
assert_eq!(conn.query_row("SELECT count(*) FROM pragma_foreign_key_check", [], |row| row.get::<_, u64>(0))?, 0);
|
||||||
|
Ok(())
|
||||||
|
}).unwrap();
|
||||||
|
drop(migrated);
|
||||||
|
let reopened = SqliteWorkspaceStore::open_for_test(&database).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
reopened.list_passkey_summaries("user-old").unwrap(),
|
||||||
|
vec![summary]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn subject_create_recovery_migration_is_atomic_and_matches_fresh_schema() {
|
fn subject_create_recovery_migration_is_atomic_and_matches_fresh_schema() {
|
||||||
let dir = tempfile::tempdir().unwrap();
|
let dir = tempfile::tempdir().unwrap();
|
||||||
@@ -22341,7 +22656,8 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
|||||||
(100, "Provisional users and scoped passkey activation".into()),
|
(100, "Provisional users and scoped passkey activation".into()),
|
||||||
(101, "Durable explicit Subject body release".into()),
|
(101, "Durable explicit Subject body release".into()),
|
||||||
(102, "Durable failed Subject creation recovery".into()),
|
(102, "Durable failed Subject creation recovery".into()),
|
||||||
(103, "Job input retirement independent of immutable receipts".into())]);
|
(103, "Passkey metadata and atomic addition results".into()),
|
||||||
|
(104, "Converge Passkey results and Job input retention".into())]);
|
||||||
for (table, rows) in unchanged.iter().zip(before.iter()) {
|
for (table, rows) in unchanged.iter().zip(before.iter()) {
|
||||||
assert_eq!(&stored_fixture_rows(conn, table), rows, "preserve {table}");
|
assert_eq!(&stored_fixture_rows(conn, table), rows, "preserve {table}");
|
||||||
}
|
}
|
||||||
@@ -25173,6 +25489,24 @@ INSERT INTO worker_registry (
|
|||||||
2
|
2
|
||||||
);
|
);
|
||||||
|
|
||||||
|
for alias in ["another-name", "docs"] {
|
||||||
|
let requested = WorkerWorkdirLinkRecord {
|
||||||
|
alias: alias.into(),
|
||||||
|
..link.clone()
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
store.attach_worker_workdir(&requested).unwrap(),
|
||||||
|
downgraded_link
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
store
|
||||||
|
.list_worker_workdir_links("local-dev", &worker.worker)
|
||||||
|
.unwrap()
|
||||||
|
.len(),
|
||||||
|
2
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
let second_worker = WorkerRegistryRecord {
|
let second_worker = WorkerRegistryRecord {
|
||||||
worker: RuntimeWorkerRef::new("embedded", "2"),
|
worker: RuntimeWorkerRef::new("embedded", "2"),
|
||||||
display_name: "Browser 2".to_string(),
|
display_name: "Browser 2".to_string(),
|
||||||
@@ -25188,7 +25522,7 @@ INSERT INTO worker_registry (
|
|||||||
};
|
};
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
store.attach_worker_workdir(&workdir_conflict),
|
store.attach_worker_workdir(&workdir_conflict),
|
||||||
Err(Error::WorkdirAttachmentConflict(_))
|
Err(Error::WorkdirOccupied { .. })
|
||||||
));
|
));
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
store.detach_worker_workdir("local-dev", &worker.worker, Some("wrong-workdir"), "6",),
|
store.detach_worker_workdir("local-dev", &worker.worker, Some("wrong-workdir"), "6",),
|
||||||
@@ -26536,16 +26870,21 @@ INSERT INTO worker_registry (
|
|||||||
store.with_conn(|conn| {
|
store.with_conn(|conn| {
|
||||||
conn.execute_batch("DROP TRIGGER backend_job_input_retirement; DROP TRIGGER backend_job_identity_immutable;
|
conn.execute_batch("DROP TRIGGER backend_job_input_retirement; DROP TRIGGER backend_job_identity_immutable;
|
||||||
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
|
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
|
||||||
UPDATE __yoi_schema_migrations SET version=102 WHERE version=103;
|
DROP TRIGGER passkey_metadata_insert;
|
||||||
|
DROP TABLE passkey_addition_receipts;
|
||||||
|
DROP TABLE passkey_metadata;
|
||||||
|
UPDATE __yoi_schema_migrations SET version=102 WHERE version=104;
|
||||||
CREATE TRIGGER backend_job_input_retirement BEFORE UPDATE ON backend_job_attempts BEGIN SELECT RAISE(ABORT,'collision'); END;")?; Ok(())
|
CREATE TRIGGER backend_job_input_retirement BEFORE UPDATE ON backend_job_attempts BEGIN SELECT RAISE(ABORT,'collision'); END;")?; Ok(())
|
||||||
}).unwrap();
|
}).unwrap();
|
||||||
drop(store);
|
drop(store);
|
||||||
assert!(SqliteWorkspaceStore::open_for_test(&path).is_err());
|
|
||||||
let conn = Connection::open(&path).unwrap();
|
let conn = Connection::open(&path).unwrap();
|
||||||
|
configure_sqlite(&conn).unwrap();
|
||||||
|
assert!(migrate_job_input_retention_v102_to_v103(&conn).is_err());
|
||||||
assert_eq!(current_schema_version(&conn).unwrap(), 102);
|
assert_eq!(current_schema_version(&conn).unwrap(), 102);
|
||||||
assert!(!column_exists(&conn, "backend_jobs", "input_disposition").unwrap());
|
assert!(!column_exists(&conn, "backend_jobs", "input_disposition").unwrap());
|
||||||
conn.execute_batch("DROP TRIGGER backend_job_input_retirement")
|
conn.execute_batch("DROP TRIGGER backend_job_input_retirement")
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
migrate_job_input_retention_v102_to_v103(&conn).unwrap();
|
||||||
drop(conn);
|
drop(conn);
|
||||||
let reopened = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
let reopened = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
|
|||||||
@@ -0,0 +1,688 @@
|
|||||||
|
//! Offline Backend fixture preparation, available only with `test-support`.
|
||||||
|
//!
|
||||||
|
//! Use a new isolated database and the existing account, Workspace signing-identity /
|
||||||
|
//! `ControlPlaneStore::create_workspace_bootstrap`, and Runtime binding APIs first.
|
||||||
|
//! Generate fresh account credentials and signing keys for each environment; no fixture
|
||||||
|
//! credentials are accepted here. Install Profile configuration through the existing
|
||||||
|
//! config-source mechanisms before starting the product processes.
|
||||||
|
//!
|
||||||
|
//! This module seeds Backend ownership only. It neither writes Runtime storage nor
|
||||||
|
//! launches a Worker, fabricates Runtime observations, changes authorization, or sends
|
||||||
|
//! initial input. The caller must separately install the matching stopped Runtime
|
||||||
|
//! fixture. Build product binaries in a separate Cargo invocation **without** this
|
||||||
|
//! feature (Cargo features unify within an invocation).
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
use crate::hosts::{WorkerSpawnIntent, WorkerSpawnRequest, worker_spawn_create_fingerprint};
|
||||||
|
use worker_runtime::catalog::ProfileSelector;
|
||||||
|
|
||||||
|
/// One ordinary, attachment-free Worker from an externally validated fixture.
|
||||||
|
///
|
||||||
|
/// IDs must match the Runtime fixture. `request` is the unresolved canonical create
|
||||||
|
/// intent, not a Runtime launch request containing credentials or execution grants.
|
||||||
|
/// The Profile selector must already be installed by the harness. `initial_submit`
|
||||||
|
/// must be empty: saved Session contents belong to the Runtime fixture, not new input.
|
||||||
|
/// Singleton/service, Ticket-assigned and attachment-bearing Workers are deliberately
|
||||||
|
/// unsupported.
|
||||||
|
pub struct FixedWorkerSeed<'a> {
|
||||||
|
pub workspace_id: &'a str,
|
||||||
|
pub runtime_id: &'a str,
|
||||||
|
pub worker_id: WorkerId,
|
||||||
|
pub allocation_key: &'a str,
|
||||||
|
pub request: &'a WorkerSpawnRequest,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Insert a fixed Worker catalog entry and completed creation reservation atomically.
|
||||||
|
///
|
||||||
|
/// Uses the normal reservation validators, authoritative Memory snapshot, fingerprint
|
||||||
|
/// binding, resource-key allocator and immediate transaction. Even an exact repeat is
|
||||||
|
/// rejected: this is not an upsert/import/recovery API. Existing allocations, Worker
|
||||||
|
/// identities and residual lifecycle/attachment records are never adopted or repaired.
|
||||||
|
/// Returns the creation binding needed to check the corresponding Runtime fixture.
|
||||||
|
pub fn seed_fixed_worker(
|
||||||
|
store: &SqliteWorkspaceStore,
|
||||||
|
seed: FixedWorkerSeed<'_>,
|
||||||
|
) -> Result<WorkerCreateReservation> {
|
||||||
|
validate_identifier("workspace_id", seed.workspace_id)?;
|
||||||
|
validate_identifier("runtime_id", seed.runtime_id)?;
|
||||||
|
let request = seed.request;
|
||||||
|
if !request.initial_submit.is_empty() {
|
||||||
|
return Err(Error::InvalidInput(
|
||||||
|
"fixture Worker cannot contain initial_submit; offline seeding accepts no new input"
|
||||||
|
.into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if !matches!(request.intent, WorkerSpawnIntent::WorkspaceCoding)
|
||||||
|
|| request.singleton_key.is_some()
|
||||||
|
|| request.ticket_assignment.is_some()
|
||||||
|
|| request.resolved_subjektiv_attached
|
||||||
|
|| !request.workdir_attachment_requests.is_empty()
|
||||||
|
|| !request.resolved_workdir_attachment_requests.is_empty()
|
||||||
|
|| !request.resolved_workdir_attachments.is_empty()
|
||||||
|
|| request.resolved_config_bundle.is_some()
|
||||||
|
|| request.resolved_workspace_api.is_some()
|
||||||
|
|| request.resolved_memory_settings.is_some()
|
||||||
|
|| request.resolved_worker_observation_enabled
|
||||||
|
|| !request.resolved_worker_observation_grants.is_empty()
|
||||||
|
|| request.resolved_control_operation.is_some()
|
||||||
|
|| request.resolved_execution_grant.is_some()
|
||||||
|
{
|
||||||
|
return Err(Error::InvalidInput(
|
||||||
|
"fixture Worker must be ordinary unresolved create intent without singleton, assignments, attachments or grants".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
// Builtin selectors already carry their canonical `builtin:` prefix.
|
||||||
|
let profile = match &request.profile {
|
||||||
|
ProfileSelector::Builtin(name) | ProfileSelector::Named(name) => name,
|
||||||
|
};
|
||||||
|
validate_non_empty("fixture Profile selector", profile)?;
|
||||||
|
let worker_id = seed.worker_id.to_string();
|
||||||
|
let display_name = request
|
||||||
|
.requested_worker_name
|
||||||
|
.as_deref()
|
||||||
|
.unwrap_or(&worker_id);
|
||||||
|
validate_non_empty("fixture Worker display name", display_name)?;
|
||||||
|
let fingerprint = worker_spawn_create_fingerprint(request).map_err(Error::InvalidInput)?;
|
||||||
|
let memory = store.get_workspace_memory_settings(seed.workspace_id)?;
|
||||||
|
store.reserve_worker_create_with(
|
||||||
|
seed.workspace_id,
|
||||||
|
seed.runtime_id,
|
||||||
|
seed.allocation_key,
|
||||||
|
&fingerprint,
|
||||||
|
None,
|
||||||
|
&memory,
|
||||||
|
|tx| {
|
||||||
|
require_seed_target(tx, &seed)?;
|
||||||
|
Ok(Some(seed.worker_id))
|
||||||
|
},
|
||||||
|
|tx, reservation| {
|
||||||
|
let now = chrono::Utc::now().to_rfc3339();
|
||||||
|
tx.execute(
|
||||||
|
"INSERT INTO worker_registry (
|
||||||
|
workspace_id, runtime_id, worker_id, display_name, profile,
|
||||||
|
retention_state, created_at, updated_at
|
||||||
|
) VALUES (?1, ?2, ?3, ?4, ?5, 'normal', ?6, ?6)",
|
||||||
|
params![seed.workspace_id, seed.runtime_id, worker_id, display_name, profile, now],
|
||||||
|
)?;
|
||||||
|
let changed = tx.execute(
|
||||||
|
"UPDATE worker_create_reservations SET state = 'created', updated_at = ?4
|
||||||
|
WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3 AND state = 'reserved'",
|
||||||
|
params![seed.workspace_id, seed.runtime_id, reservation.worker_id.to_string(), now],
|
||||||
|
)?;
|
||||||
|
if changed != 1 {
|
||||||
|
return Err(Error::Store("fixture creation reservation disappeared".into()));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn require_seed_target(tx: &rusqlite::Transaction<'_>, seed: &FixedWorkerSeed<'_>) -> Result<()> {
|
||||||
|
let active: bool = tx.query_row(
|
||||||
|
"SELECT EXISTS(SELECT 1 FROM workspaces w
|
||||||
|
JOIN workspace_runtime_bindings r ON r.workspace_id = w.workspace_id
|
||||||
|
JOIN workspace_signing_identities s ON s.workspace_id = w.workspace_id
|
||||||
|
WHERE w.workspace_id = ?1 AND w.state = 'active'
|
||||||
|
AND r.runtime_id = ?2 AND r.revoked_at IS NULL
|
||||||
|
AND r.authentication_mode = 'workspace_identity' AND s.state = 'active')",
|
||||||
|
params![seed.workspace_id, seed.runtime_id],
|
||||||
|
|row| row.get(0),
|
||||||
|
)?;
|
||||||
|
if !active {
|
||||||
|
return Err(Error::InvalidInput(
|
||||||
|
"fixture requires an active bootstrapped Workspace and authenticated Runtime binding"
|
||||||
|
.into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let worker_id = seed.worker_id.to_string();
|
||||||
|
let allocation_exists: bool = tx.query_row(
|
||||||
|
"SELECT EXISTS(SELECT 1 FROM worker_create_reservations
|
||||||
|
WHERE workspace_id = ?1 AND allocation_key = ?2)",
|
||||||
|
params![seed.workspace_id, seed.allocation_key],
|
||||||
|
|row| row.get(0),
|
||||||
|
)?;
|
||||||
|
if allocation_exists {
|
||||||
|
return Err(Error::InvalidInput(
|
||||||
|
"fixture Worker allocation already exists".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
// Check by WorkerId, not just the proposed Runtime tuple: orphaned and terminal
|
||||||
|
// ownership is still a collision. These tables are owned by this store schema.
|
||||||
|
for table in [
|
||||||
|
"worker_create_reservations",
|
||||||
|
"worker_registry",
|
||||||
|
"worker_registry_observations",
|
||||||
|
"worker_registry_projection_removals",
|
||||||
|
"worker_removal_operations",
|
||||||
|
"worker_singleton_owners",
|
||||||
|
"worker_workdir_links",
|
||||||
|
"worker_restore_intents",
|
||||||
|
"worker_tombstones",
|
||||||
|
"worker_orphan_diagnostics",
|
||||||
|
"worker_registry_projection_diagnostics",
|
||||||
|
"worker_session_archives",
|
||||||
|
"worker_diagnostics_archives",
|
||||||
|
"ticket_assignment_worker_tombstones",
|
||||||
|
] {
|
||||||
|
let exists: bool = tx.query_row(
|
||||||
|
&format!("SELECT EXISTS(SELECT 1 FROM {table} WHERE worker_id = ?1)"),
|
||||||
|
[&worker_id],
|
||||||
|
|row| row.get(0),
|
||||||
|
)?;
|
||||||
|
if exists {
|
||||||
|
return Err(Error::InvalidInput(format!(
|
||||||
|
"fixture Worker identity already exists in {table}"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A resource key without a catalog/reservation must not be silently reused.
|
||||||
|
let mut statement = tx.prepare(
|
||||||
|
"SELECT resource_id FROM workspace_resource_keys WHERE resource_kind = 'worker'",
|
||||||
|
)?;
|
||||||
|
let resources = statement.query_map([], |row| row.get::<_, String>(0))?;
|
||||||
|
for resource in resources {
|
||||||
|
let resource = resource?;
|
||||||
|
let existing = resource
|
||||||
|
.is_ascii()
|
||||||
|
.then(|| parse_worker_resource_id(&resource))
|
||||||
|
.flatten()
|
||||||
|
.ok_or_else(|| {
|
||||||
|
Error::InvalidInput("unknown fixture Worker resource identity collision".into())
|
||||||
|
})?;
|
||||||
|
if existing.worker_id == worker_id {
|
||||||
|
return Err(Error::InvalidInput(
|
||||||
|
"fixture Worker resource identity already exists".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::hosts::{
|
||||||
|
WorkerSpawnAcceptanceRequirement, WorkerSpawnWorkingDirectoryAttachmentRequest,
|
||||||
|
WorkerSpawnWorkingDirectoryRequest,
|
||||||
|
};
|
||||||
|
use crate::workspace_signing_identity::{
|
||||||
|
InMemoryWorkspaceSigningMaterialStore, WorkspaceSigningIdentityService,
|
||||||
|
};
|
||||||
|
|
||||||
|
const WORKSPACE: &str = "0197a949-4b6b-7f2a-9d9a-1f87e3a4c501";
|
||||||
|
const RUNTIME: &str = "fixture-runtime";
|
||||||
|
const WORKER: &str = "0197a949-4b6b-7f2a-9d9a-1f87e3a4c502";
|
||||||
|
|
||||||
|
// Exercises the same public bootstrap APIs the offline harness uses. All account
|
||||||
|
// and key identities are newly generated, even across stores with fixed fixture IDs.
|
||||||
|
fn bootstrap() -> SqliteWorkspaceStore {
|
||||||
|
let store = SqliteWorkspaceStore::in_memory_for_test().unwrap();
|
||||||
|
let account = AccountRecord {
|
||||||
|
account_id: format!("fixture-account-{}", Uuid::now_v7()),
|
||||||
|
kind: "user".into(),
|
||||||
|
handle: format!("fixture-{}", Uuid::now_v7()),
|
||||||
|
display_name: "Fixture owner".into(),
|
||||||
|
created_at: "2026-10-24T00:00:00Z".into(),
|
||||||
|
updated_at: "2026-10-24T00:00:00Z".into(),
|
||||||
|
};
|
||||||
|
store.upsert_account(&account).unwrap();
|
||||||
|
let identities = WorkspaceSigningIdentityService::new(
|
||||||
|
Arc::new(store.clone()),
|
||||||
|
Arc::new(InMemoryWorkspaceSigningMaterialStore::default()),
|
||||||
|
);
|
||||||
|
let (identity, operation) = identities
|
||||||
|
.prepare_workspace_creation(
|
||||||
|
"fixture-bootstrap",
|
||||||
|
"fixture-bootstrap",
|
||||||
|
WORKSPACE,
|
||||||
|
&account.account_id,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
store
|
||||||
|
.create_workspace_bootstrap(
|
||||||
|
&WorkspaceBootstrapRecord {
|
||||||
|
operation_key: "fixture-bootstrap".into(),
|
||||||
|
request_fingerprint: "fixture-bootstrap".into(),
|
||||||
|
workspace: WorkspaceRecord {
|
||||||
|
workspace_id: WORKSPACE.into(),
|
||||||
|
owner_account_id: account.account_id,
|
||||||
|
display_name: "Fixture".into(),
|
||||||
|
state: "active".into(),
|
||||||
|
created_at: account.created_at,
|
||||||
|
updated_at: account.updated_at,
|
||||||
|
},
|
||||||
|
repository: None,
|
||||||
|
},
|
||||||
|
&identity,
|
||||||
|
&operation,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let runtime_key = worker_runtime::auth::RuntimeIdentityMaterial::generate(RUNTIME).unwrap();
|
||||||
|
store
|
||||||
|
.upsert_workspace_runtime_binding(
|
||||||
|
WorkspaceRuntimeBinding {
|
||||||
|
workspace_id: WORKSPACE.into(),
|
||||||
|
runtime_id: RUNTIME.into(),
|
||||||
|
display_name: "Fixture Runtime".into(),
|
||||||
|
base_url: "http://127.0.0.1:12345".into(),
|
||||||
|
public_key: runtime_key.public_key,
|
||||||
|
public_key_fingerprint: String::new(),
|
||||||
|
binding_id: String::new(),
|
||||||
|
authentication_mode: WorkspaceRuntimeAuthenticationMode::WorkspaceIdentity,
|
||||||
|
created_at: "2026-10-24T00:00:00Z".into(),
|
||||||
|
updated_at: "2026-10-24T00:00:00Z".into(),
|
||||||
|
revoked_at: None,
|
||||||
|
},
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
store
|
||||||
|
}
|
||||||
|
|
||||||
|
fn request() -> WorkerSpawnRequest {
|
||||||
|
WorkerSpawnRequest {
|
||||||
|
intent: WorkerSpawnIntent::WorkspaceCoding,
|
||||||
|
requested_worker_name: Some("Fixture Worker".into()),
|
||||||
|
singleton_key: None,
|
||||||
|
acceptance: WorkerSpawnAcceptanceRequirement::SocketReady,
|
||||||
|
profile: ProfileSelector::Builtin("builtin:coder".into()),
|
||||||
|
ticket_assignment: None,
|
||||||
|
initial_submit: Vec::new(),
|
||||||
|
workdir_attachment_requests: Vec::new(),
|
||||||
|
resolved_workdir_attachment_requests: Vec::new(),
|
||||||
|
resolved_workdir_attachments: Vec::new(),
|
||||||
|
resolved_config_bundle: None,
|
||||||
|
resolved_workspace_api: None,
|
||||||
|
resolved_memory_settings: None,
|
||||||
|
resolved_subjektiv_attached: false,
|
||||||
|
resolved_worker_observation_enabled: false,
|
||||||
|
resolved_worker_observation_grants: Vec::new(),
|
||||||
|
resolved_control_operation: None,
|
||||||
|
resolved_execution_grant: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn seed(request: &WorkerSpawnRequest) -> FixedWorkerSeed<'_> {
|
||||||
|
FixedWorkerSeed {
|
||||||
|
workspace_id: WORKSPACE,
|
||||||
|
runtime_id: RUNTIME,
|
||||||
|
worker_id: WORKER.parse().unwrap(),
|
||||||
|
allocation_key: "fixture-allocation",
|
||||||
|
request,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn worker() -> RuntimeWorkerRef {
|
||||||
|
RuntimeWorkerRef::new(RUNTIME, WORKER)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn fixed_ids_are_isolated_and_seed_has_no_runtime_observation() {
|
||||||
|
let a = bootstrap();
|
||||||
|
let b = bootstrap();
|
||||||
|
let request = request();
|
||||||
|
let first = seed_fixed_worker(&a, seed(&request)).unwrap();
|
||||||
|
let second = seed_fixed_worker(&b, seed(&request)).unwrap();
|
||||||
|
assert_eq!(first, second);
|
||||||
|
assert_eq!(first.worker_id.to_string(), WORKER);
|
||||||
|
assert_eq!(first.memory_settings.workspace_id, WORKSPACE);
|
||||||
|
assert_eq!(first.memory_settings.language, "English");
|
||||||
|
assert!(first.singleton.is_none());
|
||||||
|
assert_ne!(
|
||||||
|
a.get_workspace_signing_identity(WORKSPACE)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
.public_key,
|
||||||
|
b.get_workspace_signing_identity(WORKSPACE)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
.public_key
|
||||||
|
);
|
||||||
|
assert_ne!(
|
||||||
|
a.get_workspace_runtime_binding(WORKSPACE, RUNTIME)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
.public_key,
|
||||||
|
b.get_workspace_runtime_binding(WORKSPACE, RUNTIME)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
.public_key
|
||||||
|
);
|
||||||
|
for store in [&a, &b] {
|
||||||
|
assert_eq!(
|
||||||
|
store
|
||||||
|
.worker_resource_key(WORKSPACE, &worker())
|
||||||
|
.unwrap()
|
||||||
|
.as_deref(),
|
||||||
|
Some("W-1")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
store
|
||||||
|
.resolve_worker_resource_reference(WORKSPACE, "W-1")
|
||||||
|
.unwrap(),
|
||||||
|
Some(worker())
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!store
|
||||||
|
.has_active_worker_create_reservation(WORKSPACE, &worker())
|
||||||
|
.unwrap()
|
||||||
|
);
|
||||||
|
let registry = store
|
||||||
|
.get_worker_registry(WORKSPACE, &worker())
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(registry.profile.as_deref(), Some("builtin:coder"));
|
||||||
|
assert_eq!(registry.display_name, "Fixture Worker");
|
||||||
|
assert_eq!(registry.retention_state, "normal");
|
||||||
|
let projection = store
|
||||||
|
.with_conn(|conn| read_worker_registry_projection(conn, WORKSPACE, &worker()))
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert!(projection.observation.is_none());
|
||||||
|
assert!(projection.job.is_none());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn initial_submit_is_refused_without_registration_or_allocation() {
|
||||||
|
let store = bootstrap();
|
||||||
|
let mut request = request();
|
||||||
|
// Even an empty text segment is new input; the vector itself must be empty.
|
||||||
|
for text in ["new input must not be seeded", ""] {
|
||||||
|
request.initial_submit = vec![protocol::Segment::text(text)];
|
||||||
|
let error = seed_fixed_worker(&store, seed(&request)).unwrap_err();
|
||||||
|
assert!(
|
||||||
|
matches!(error, Error::InvalidInput(ref message) if message.contains("initial_submit"))
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.get_worker_registry(WORKSPACE, &worker())
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.worker_resource_key(WORKSPACE, &worker())
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!store
|
||||||
|
.has_active_worker_create_reservation(WORKSPACE, &worker())
|
||||||
|
.unwrap()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// The rejected attempts must not consume the allocation or resource counter.
|
||||||
|
request.initial_submit.clear();
|
||||||
|
seed_fixed_worker(&store, seed(&request)).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
store
|
||||||
|
.worker_resource_key(WORKSPACE, &worker())
|
||||||
|
.unwrap()
|
||||||
|
.as_deref(),
|
||||||
|
Some("W-1")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn exact_replay_changed_input_and_cross_runtime_identity_are_refused() {
|
||||||
|
let store = bootstrap();
|
||||||
|
let mut request = request();
|
||||||
|
seed_fixed_worker(&store, seed(&request)).unwrap();
|
||||||
|
let before = store.get_worker_registry(WORKSPACE, &worker()).unwrap();
|
||||||
|
assert!(seed_fixed_worker(&store, seed(&request)).is_err());
|
||||||
|
request.requested_worker_name = Some("Must not overwrite".into());
|
||||||
|
assert!(seed_fixed_worker(&store, seed(&request)).is_err());
|
||||||
|
let mut new_allocation = seed(&request);
|
||||||
|
new_allocation.allocation_key = "another-allocation";
|
||||||
|
assert!(seed_fixed_worker(&store, new_allocation).is_err());
|
||||||
|
let mut same_allocation = seed(&request);
|
||||||
|
same_allocation.worker_id = WorkerId::now_v7();
|
||||||
|
assert!(seed_fixed_worker(&store, same_allocation).is_err());
|
||||||
|
assert_eq!(
|
||||||
|
store.get_worker_registry(WORKSPACE, &worker()).unwrap(),
|
||||||
|
before
|
||||||
|
);
|
||||||
|
|
||||||
|
let other = bootstrap();
|
||||||
|
let mut record = before.unwrap();
|
||||||
|
record.worker.runtime_id = "different-runtime".into();
|
||||||
|
other.upsert_worker_registry(&record).unwrap();
|
||||||
|
assert!(seed_fixed_worker(&other, seed(&request)).is_err());
|
||||||
|
assert!(
|
||||||
|
other
|
||||||
|
.get_worker_registry(WORKSPACE, &worker())
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn reserved_and_orphaned_resource_identities_are_not_adopted() {
|
||||||
|
let store = bootstrap();
|
||||||
|
let request = request();
|
||||||
|
let memory = store.get_workspace_memory_settings(WORKSPACE).unwrap();
|
||||||
|
let ordinary = store
|
||||||
|
.reserve_worker_create(WORKSPACE, RUNTIME, "ordinary", "fingerprint", None, &memory)
|
||||||
|
.unwrap();
|
||||||
|
let mut collision = seed(&request);
|
||||||
|
collision.worker_id = ordinary.worker_id;
|
||||||
|
assert!(seed_fixed_worker(&store, collision).is_err());
|
||||||
|
// Store-level fault setup only: the external harness never writes SQL.
|
||||||
|
store
|
||||||
|
.with_conn(|conn| {
|
||||||
|
allocate_resource_key(
|
||||||
|
conn,
|
||||||
|
WORKSPACE,
|
||||||
|
WorkspaceResourceKind::Worker,
|
||||||
|
&worker_resource_id(&worker()),
|
||||||
|
"now",
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert!(seed_fixed_worker(&store, seed(&request)).is_err());
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.get_worker_registry(WORKSPACE, &worker())
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unknown_resource_identity_and_retained_removal_fence_are_refused() {
|
||||||
|
let request = request();
|
||||||
|
let malformed = bootstrap();
|
||||||
|
malformed
|
||||||
|
.with_conn(|conn| {
|
||||||
|
allocate_resource_key(
|
||||||
|
conn,
|
||||||
|
WORKSPACE,
|
||||||
|
WorkspaceResourceKind::Worker,
|
||||||
|
"unknown-identity",
|
||||||
|
"now",
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert!(matches!(
|
||||||
|
seed_fixed_worker(&malformed, seed(&request)),
|
||||||
|
Err(Error::InvalidInput(_))
|
||||||
|
));
|
||||||
|
assert!(
|
||||||
|
malformed
|
||||||
|
.get_worker_registry(WORKSPACE, &worker())
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
|
||||||
|
let removed = bootstrap();
|
||||||
|
removed.with_conn(|conn| {
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO worker_registry_projection_removals(workspace_id, runtime_id, worker_id) VALUES (?1, ?2, ?3)",
|
||||||
|
params![WORKSPACE, RUNTIME, WORKER],
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
}).unwrap();
|
||||||
|
assert!(matches!(
|
||||||
|
seed_fixed_worker(&removed, seed(&request)),
|
||||||
|
Err(Error::InvalidInput(_))
|
||||||
|
));
|
||||||
|
assert!(
|
||||||
|
removed
|
||||||
|
.worker_resource_key(WORKSPACE, &worker())
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unknown_workspace_runtime_and_revoked_binding_are_refused() {
|
||||||
|
let store = bootstrap();
|
||||||
|
let request = request();
|
||||||
|
let mut unknown = seed(&request);
|
||||||
|
unknown.workspace_id = "unknown";
|
||||||
|
assert!(seed_fixed_worker(&store, unknown).is_err());
|
||||||
|
let mut unknown = seed(&request);
|
||||||
|
unknown.runtime_id = "unknown";
|
||||||
|
assert!(seed_fixed_worker(&store, unknown).is_err());
|
||||||
|
store
|
||||||
|
.revoke_workspace_runtime_binding(WORKSPACE, RUNTIME, "2026-10-24T01:00:00Z")
|
||||||
|
.unwrap();
|
||||||
|
assert!(seed_fixed_worker(&store, seed(&request)).is_err());
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.worker_resource_key(WORKSPACE, &worker())
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unsupported_intent_is_refused_without_partial_registration() {
|
||||||
|
let store = bootstrap();
|
||||||
|
let base = request();
|
||||||
|
let mut requests = Vec::new();
|
||||||
|
let mut request = base.clone();
|
||||||
|
request.singleton_key = Some("fixture-singleton".into());
|
||||||
|
requests.push(request);
|
||||||
|
let mut request = base.clone();
|
||||||
|
request.intent = WorkerSpawnIntent::WorkspaceOrchestrator;
|
||||||
|
requests.push(request);
|
||||||
|
let mut request = base.clone();
|
||||||
|
request.resolved_subjektiv_attached = true;
|
||||||
|
requests.push(request);
|
||||||
|
let mut request = base.clone();
|
||||||
|
request
|
||||||
|
.workdir_attachment_requests
|
||||||
|
.push(WorkerSpawnWorkingDirectoryAttachmentRequest {
|
||||||
|
alias: workdir::WorkdirAttachmentAlias::new("repo").unwrap(),
|
||||||
|
working_directory: WorkerSpawnWorkingDirectoryRequest {
|
||||||
|
repository_key: "repo".into(),
|
||||||
|
selector: None,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
requests.push(request);
|
||||||
|
for request in requests {
|
||||||
|
assert!(seed_fixed_worker(&store, seed(&request)).is_err());
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.get_worker_registry(WORKSPACE, &worker())
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.worker_resource_key(WORKSPACE, &worker())
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
seed_fixed_worker(&store, seed(&base))
|
||||||
|
.unwrap()
|
||||||
|
.worker_id
|
||||||
|
.to_string(),
|
||||||
|
WORKER
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn late_catalog_failure_rolls_back_reservation_and_resource_allocation() {
|
||||||
|
let store = bootstrap();
|
||||||
|
let request = request();
|
||||||
|
store
|
||||||
|
.with_conn(|conn| {
|
||||||
|
conn.execute_batch(
|
||||||
|
"CREATE TEMP TRIGGER fail_fixture_catalog BEFORE INSERT ON worker_registry
|
||||||
|
BEGIN SELECT RAISE(ABORT, 'injected catalog failure'); END",
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert!(seed_fixed_worker(&store, seed(&request)).is_err());
|
||||||
|
assert!(
|
||||||
|
!store
|
||||||
|
.has_active_worker_create_reservation(WORKSPACE, &worker())
|
||||||
|
.unwrap()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.worker_resource_key(WORKSPACE, &worker())
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
store
|
||||||
|
.with_conn(|conn| {
|
||||||
|
conn.execute_batch("DROP TRIGGER fail_fixture_catalog")?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
seed_fixed_worker(&store, seed(&request)).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
store
|
||||||
|
.worker_resource_key(WORKSPACE, &worker())
|
||||||
|
.unwrap()
|
||||||
|
.as_deref(),
|
||||||
|
Some("W-1")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn ordinary_creation_still_allocates_and_replays_without_catalog_insertion() {
|
||||||
|
let store = bootstrap();
|
||||||
|
let memory = store.get_workspace_memory_settings(WORKSPACE).unwrap();
|
||||||
|
let reserve = |key| {
|
||||||
|
store
|
||||||
|
.reserve_worker_create(WORKSPACE, RUNTIME, key, "fingerprint", None, &memory)
|
||||||
|
.unwrap()
|
||||||
|
};
|
||||||
|
let first = reserve("ordinary-one");
|
||||||
|
assert_eq!(first, reserve("ordinary-one"));
|
||||||
|
let second = reserve("ordinary-two");
|
||||||
|
assert_ne!(first.worker_id, second.worker_id);
|
||||||
|
assert_ne!(first.worker_id.to_string(), WORKER);
|
||||||
|
assert_eq!(
|
||||||
|
first.worker_id.as_uuid().get_version(),
|
||||||
|
Some(uuid::Version::SortRand)
|
||||||
|
);
|
||||||
|
let worker = RuntimeWorkerRef::new(RUNTIME, first.worker_id.to_string());
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.has_active_worker_create_reservation(WORKSPACE, &worker)
|
||||||
|
.unwrap()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.get_worker_registry(WORKSPACE, &worker)
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.reserve_worker_create(WORKSPACE, RUNTIME, "ordinary-one", "changed", None, &memory)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,24 +1,15 @@
|
|||||||
//! Persists durable deletion intent and attempts for Repository-backed Workdirs.
|
//! Records one Repository-backed Workdir removal request and its result.
|
||||||
//!
|
//!
|
||||||
//! Deletion intent fixes Workspace, Workdir, owning Runtime, Repository/materialization, actor,
|
//! Caller-generated request identities distinguish an exact resend from a new explicit request;
|
||||||
//! and reason. Endpoints, host paths, and provider handles do not grant caller deletion authority.
|
//! reason text is audit input, never a replay key. Failed and completed records are terminal,
|
||||||
|
//! including legacy failed records marked retryable. Retryability only advises a new explicit
|
||||||
|
//! request after resolving its cause; it is not continuing authority to delete.
|
||||||
|
//!
|
||||||
|
//! Only one unresolved request per Workdir is allowed. Attempts are claimed transactionally and
|
||||||
|
//! fenced by process/start marker and attempt identity. Recovery observes uncertain external
|
||||||
|
//! effects but never reissues deletion. Confirmed absence and the matching materialization permit
|
||||||
|
//! registry removal and the saved result to commit together. Unconfirmed effects remain pending.
|
||||||
//! External Workdirs follow grant revocation, not this operation.
|
//! External Workdirs follow grant revocation, not this operation.
|
||||||
//!
|
|
||||||
//! Only one pending operation per Workdir is allowed; attempts are claimed transactionally.
|
|
||||||
//! Process IDs and startup markers are retained. Recovery reclaims the exact owner/attempt by
|
|
||||||
//! compare-and-set only after confirming the previous owner is gone; live or unobservable owners
|
|
||||||
//! are not displaced. In-process locks alone are insufficient.
|
|
||||||
//!
|
|
||||||
//! Server execution revalidates stored materialization and current attachments, reservations,
|
|
||||||
//! assignments, retention, and creation operations, then observes occupancy and cleanliness via
|
|
||||||
//! Runtime. Dirty resources remain; unknown observations or provider failures retain the registry
|
|
||||||
//! and record attention/retryability. Only confirmed deletion or exact working_directory_not_found
|
|
||||||
//! is deletion evidence; registry removal and completed/removed commit in the same transaction.
|
|
||||||
//!
|
|
||||||
//! Normal deletion, Workspace deletion, cleanup, and startup recovery use the same reserved Server
|
|
||||||
//! operation execution. Failed DB commits after provider deletion are reobserved through that
|
|
||||||
//! operation. Recovery relies on provider identity and results, not an exactly-once guarantee
|
|
||||||
//! across all interruption points.
|
|
||||||
|
|
||||||
use chrono::Utc;
|
use chrono::Utc;
|
||||||
use rusqlite::{Connection, OptionalExtension, TransactionBehavior, params};
|
use rusqlite::{Connection, OptionalExtension, TransactionBehavior, params};
|
||||||
@@ -118,10 +109,22 @@ pub fn workdir_materialization_fingerprint(record: &WorkdirRegistryRecord) -> St
|
|||||||
hex_sha256(&bytes)
|
hex_sha256(&bytes)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn workdir_removal_request_id(
|
||||||
|
workspace_id: &str,
|
||||||
|
source_actor: &str,
|
||||||
|
request_id: &str,
|
||||||
|
) -> Result<String> {
|
||||||
|
validate_bounded("operation id", request_id, 256)?;
|
||||||
|
let bytes = serde_json::to_vec(&(workspace_id, source_actor, request_id))
|
||||||
|
.map_err(|error| Error::Store(error.to_string()))?;
|
||||||
|
Ok(format!("wdr_{}", &hex_sha256(&bytes)[..32]))
|
||||||
|
}
|
||||||
|
|
||||||
pub fn workdir_removal_intent(
|
pub fn workdir_removal_intent(
|
||||||
record: &WorkdirRegistryRecord,
|
record: &WorkdirRegistryRecord,
|
||||||
source_actor: &str,
|
source_actor: &str,
|
||||||
reason: &str,
|
reason: &str,
|
||||||
|
request_id: &str,
|
||||||
) -> Result<WorkdirRemovalIntent> {
|
) -> Result<WorkdirRemovalIntent> {
|
||||||
validate_bounded("source actor", source_actor, MAX_ACTOR_BYTES)?;
|
validate_bounded("source actor", source_actor, MAX_ACTOR_BYTES)?;
|
||||||
validate_bounded("reason", reason, MAX_REASON_BYTES)?;
|
validate_bounded("reason", reason, MAX_REASON_BYTES)?;
|
||||||
@@ -146,7 +149,7 @@ pub fn workdir_removal_intent(
|
|||||||
));
|
));
|
||||||
};
|
};
|
||||||
Ok(WorkdirRemovalIntent {
|
Ok(WorkdirRemovalIntent {
|
||||||
operation_id: format!("wdr_{}", &request_fingerprint[..32]),
|
operation_id: workdir_removal_request_id(&record.workspace_id, source_actor, request_id)?,
|
||||||
request_fingerprint,
|
request_fingerprint,
|
||||||
workspace_id: record.workspace_id.clone(),
|
workspace_id: record.workspace_id.clone(),
|
||||||
working_directory_id: record.workdir_id.clone(),
|
working_directory_id: record.workdir_id.clone(),
|
||||||
@@ -285,17 +288,10 @@ impl SqliteWorkspaceStore {
|
|||||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||||
let operation =
|
let operation =
|
||||||
require_operation(&tx, workspace_id, operation_id, request_fingerprint)?;
|
require_operation(&tx, workspace_id, operation_id, request_fingerprint)?;
|
||||||
if operation.state == WorkdirRemovalOperationState::Completed
|
if operation.state != WorkdirRemovalOperationState::Pending {
|
||||||
&& operation.disposition == Some(WorkdirRemovalDisposition::Removed)
|
|
||||||
{
|
|
||||||
tx.commit()?;
|
tx.commit()?;
|
||||||
return Ok(operation);
|
return Ok(operation);
|
||||||
}
|
}
|
||||||
if operation.state == WorkdirRemovalOperationState::Failed && !operation.retryable {
|
|
||||||
return Err(Error::InvalidInput(format!(
|
|
||||||
"Workdir removal operation `{operation_id}` is not retryable"
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
if let Some((expected_owner, expected_attempt_id)) = recovery_expected {
|
if let Some((expected_owner, expected_attempt_id)) = recovery_expected {
|
||||||
if operation.state != WorkdirRemovalOperationState::Pending
|
if operation.state != WorkdirRemovalOperationState::Pending
|
||||||
|| operation.attempt_owner != expected_owner
|
|| operation.attempt_owner != expected_owner
|
||||||
@@ -473,7 +469,7 @@ impl SqliteWorkspaceStore {
|
|||||||
&operation.operation_id,
|
&operation.operation_id,
|
||||||
&operation.request_fingerprint,
|
&operation.request_fingerprint,
|
||||||
)?;
|
)?;
|
||||||
if current.state == WorkdirRemovalOperationState::Completed {
|
if current.state != WorkdirRemovalOperationState::Pending {
|
||||||
tx.commit()?;
|
tx.commit()?;
|
||||||
return Ok(current);
|
return Ok(current);
|
||||||
}
|
}
|
||||||
@@ -502,6 +498,39 @@ impl SqliteWorkspaceStore {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Keep an uncertain external outcome for observation, not automatic re-execution.
|
||||||
|
pub fn record_workdir_removal_unknown(
|
||||||
|
&self,
|
||||||
|
operation: &WorkdirRemovalOperation,
|
||||||
|
) -> Result<WorkdirRemovalOperation> {
|
||||||
|
self.with_conn_mut(|conn| {
|
||||||
|
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||||
|
let current = require_operation(
|
||||||
|
&tx,
|
||||||
|
&operation.workspace_id,
|
||||||
|
&operation.operation_id,
|
||||||
|
&operation.request_fingerprint,
|
||||||
|
)?;
|
||||||
|
if current.state != WorkdirRemovalOperationState::Pending {
|
||||||
|
tx.commit()?;
|
||||||
|
return Ok(current);
|
||||||
|
}
|
||||||
|
require_matching_attempt(operation, ¤t)?;
|
||||||
|
tx.execute(
|
||||||
|
"UPDATE workdir_removal_operations SET retryable=0, disposition='attention_required', failure_category='provider_cleanup_outcome_unknown', attempt_owner_pid=NULL, attempt_owner_start_marker=NULL, updated_at=?1 WHERE workspace_id=?2 AND operation_id=?3",
|
||||||
|
params![Utc::now().to_rfc3339(), operation.workspace_id, operation.operation_id],
|
||||||
|
)?;
|
||||||
|
let updated = require_operation(
|
||||||
|
&tx,
|
||||||
|
&operation.workspace_id,
|
||||||
|
&operation.operation_id,
|
||||||
|
&operation.request_fingerprint,
|
||||||
|
)?;
|
||||||
|
tx.commit()?;
|
||||||
|
Ok(updated)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
pub fn commit_workdir_removal_removed(
|
pub fn commit_workdir_removal_removed(
|
||||||
&self,
|
&self,
|
||||||
operation: &WorkdirRemovalOperation,
|
operation: &WorkdirRemovalOperation,
|
||||||
@@ -531,7 +560,7 @@ impl SqliteWorkspaceStore {
|
|||||||
&operation.operation_id,
|
&operation.operation_id,
|
||||||
&operation.request_fingerprint,
|
&operation.request_fingerprint,
|
||||||
)?;
|
)?;
|
||||||
if current.state == WorkdirRemovalOperationState::Completed {
|
if current.state != WorkdirRemovalOperationState::Pending {
|
||||||
tx.commit()?;
|
tx.commit()?;
|
||||||
return Ok(current);
|
return Ok(current);
|
||||||
}
|
}
|
||||||
@@ -541,24 +570,29 @@ impl SqliteWorkspaceStore {
|
|||||||
&tx,
|
&tx,
|
||||||
&operation.workspace_id,
|
&operation.workspace_id,
|
||||||
&operation.working_directory_id,
|
&operation.working_directory_id,
|
||||||
)?
|
|
||||||
.ok_or_else(|| Error::RegistryInconsistency(
|
|
||||||
"Workdir registry row disappeared before durable removal commit".to_string(),
|
|
||||||
))?;
|
|
||||||
require_operation_materialization(operation, ®istry)?;
|
|
||||||
require_no_removal_blockers(
|
|
||||||
&tx,
|
|
||||||
&operation.workspace_id,
|
|
||||||
&operation.working_directory_id,
|
|
||||||
)?;
|
)?;
|
||||||
let deleted = tx.execute(
|
if let Some(registry) = registry {
|
||||||
"DELETE FROM workdir_registry WHERE workspace_id=?1 AND workdir_id=?2",
|
require_operation_materialization(operation, ®istry)?;
|
||||||
params![operation.workspace_id, operation.working_directory_id],
|
require_no_removal_blockers(
|
||||||
)?;
|
&tx,
|
||||||
if deleted != 1 {
|
&operation.workspace_id,
|
||||||
return Err(Error::RegistryInconsistency(
|
&operation.working_directory_id,
|
||||||
"Workdir registry deletion did not remove exactly one row".to_string(),
|
)?;
|
||||||
));
|
tx.execute(
|
||||||
|
"DELETE FROM workdir_registry WHERE workspace_id=?1 AND workdir_id=?2",
|
||||||
|
params![operation.workspace_id, operation.working_directory_id],
|
||||||
|
)?;
|
||||||
|
} else {
|
||||||
|
let removed: bool = tx.query_row(
|
||||||
|
"SELECT EXISTS(SELECT 1 FROM workdir_removal_operations WHERE workspace_id=?1 AND workdir_id=?2 AND materialization_fingerprint=?3 AND state='completed' AND disposition='removed')",
|
||||||
|
params![operation.workspace_id, operation.working_directory_id, operation.materialization_fingerprint],
|
||||||
|
|row| row.get(0),
|
||||||
|
)?;
|
||||||
|
if !removed {
|
||||||
|
return Err(Error::RegistryInconsistency(
|
||||||
|
"Workdir registry disappeared without confirmed removal".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let now = Utc::now().to_rfc3339();
|
let now = Utc::now().to_rfc3339();
|
||||||
@@ -585,20 +619,18 @@ impl SqliteWorkspaceStore {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn find_workdir_removal_operation_by_intent(
|
pub fn completed_workdir_removal(
|
||||||
&self,
|
&self,
|
||||||
workspace_id: &str,
|
workspace_id: &str,
|
||||||
working_directory_id: &str,
|
working_directory_id: &str,
|
||||||
source_actor: &str,
|
|
||||||
reason: &str,
|
|
||||||
) -> Result<Option<WorkdirRemovalOperation>> {
|
) -> Result<Option<WorkdirRemovalOperation>> {
|
||||||
self.with_conn(|conn| {
|
self.with_conn(|conn| {
|
||||||
conn.query_row(
|
conn.query_row(
|
||||||
&format!(
|
&format!(
|
||||||
"{} WHERE workspace_id=?1 AND workdir_id=?2 AND source_actor=?3 AND reason=?4 ORDER BY created_at DESC, operation_id DESC LIMIT 1",
|
"{} WHERE workspace_id=?1 AND workdir_id=?2 AND state='completed' AND disposition='removed' ORDER BY completed_at DESC, operation_id DESC LIMIT 1",
|
||||||
operation_select_sql()
|
operation_select_sql()
|
||||||
),
|
),
|
||||||
params![workspace_id, working_directory_id, source_actor, reason],
|
params![workspace_id, working_directory_id],
|
||||||
read_operation,
|
read_operation,
|
||||||
)
|
)
|
||||||
.optional()
|
.optional()
|
||||||
@@ -614,7 +646,7 @@ impl SqliteWorkspaceStore {
|
|||||||
self.with_conn(|conn| {
|
self.with_conn(|conn| {
|
||||||
let mut statement = conn.prepare(
|
let mut statement = conn.prepare(
|
||||||
&format!(
|
&format!(
|
||||||
"{} WHERE workspace_id=?1 AND (state='pending' OR (state='failed' AND retryable=1)) ORDER BY updated_at ASC, operation_id ASC LIMIT ?2",
|
"{} WHERE workspace_id=?1 AND state='pending' ORDER BY updated_at ASC, operation_id ASC LIMIT ?2",
|
||||||
operation_select_sql()
|
operation_select_sql()
|
||||||
),
|
),
|
||||||
)?;
|
)?;
|
||||||
@@ -1007,8 +1039,13 @@ mod tests {
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn exact_replay_reuses_operation_and_conflicting_intent_fails() {
|
async fn exact_replay_reuses_operation_and_conflicting_intent_fails() {
|
||||||
let (store, workdir) = seeded_store().await;
|
let (store, workdir) = seeded_store().await;
|
||||||
let intent =
|
let intent = workdir_removal_intent(
|
||||||
workdir_removal_intent(&workdir, "worker:W-1", "remove stale Workdir").unwrap();
|
&workdir,
|
||||||
|
"worker:W-1",
|
||||||
|
"remove stale Workdir",
|
||||||
|
"test-removal-request",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
let first = store.reserve_workdir_removal_operation(&intent).unwrap();
|
let first = store.reserve_workdir_removal_operation(&intent).unwrap();
|
||||||
let replay = store.reserve_workdir_removal_operation(&intent).unwrap();
|
let replay = store.reserve_workdir_removal_operation(&intent).unwrap();
|
||||||
assert_eq!(replay, first);
|
assert_eq!(replay, first);
|
||||||
@@ -1023,10 +1060,15 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn failed_attempt_is_retryable_and_completed_retry_replays() {
|
async fn failed_request_is_terminal_and_new_request_has_independent_result() {
|
||||||
let (store, workdir) = seeded_store().await;
|
let (store, workdir) = seeded_store().await;
|
||||||
let intent =
|
let intent = workdir_removal_intent(
|
||||||
workdir_removal_intent(&workdir, "workspace-api", "remove clean Workdir").unwrap();
|
&workdir,
|
||||||
|
"workspace-api",
|
||||||
|
"remove clean Workdir",
|
||||||
|
"test-removal-request",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
let reserved = store.reserve_workdir_removal_operation(&intent).unwrap();
|
let reserved = store.reserve_workdir_removal_operation(&intent).unwrap();
|
||||||
let first = store
|
let first = store
|
||||||
.begin_workdir_removal_attempt(
|
.begin_workdir_removal_attempt(
|
||||||
@@ -1049,7 +1091,32 @@ mod tests {
|
|||||||
attempt_owner(),
|
attempt_owner(),
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_ne!(retry.attempt_id, first.attempt_id);
|
assert_eq!(retry, failed);
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.recoverable_workdir_removal_operations("workspace-a", 100)
|
||||||
|
.unwrap()
|
||||||
|
.is_empty()
|
||||||
|
);
|
||||||
|
let next_intent = workdir_removal_intent(
|
||||||
|
&workdir,
|
||||||
|
"workspace-api",
|
||||||
|
"remove clean Workdir",
|
||||||
|
"new-request",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let next = store
|
||||||
|
.reserve_workdir_removal_operation(&next_intent)
|
||||||
|
.unwrap();
|
||||||
|
let retry = store
|
||||||
|
.begin_workdir_removal_attempt(
|
||||||
|
&next.workspace_id,
|
||||||
|
&next.operation_id,
|
||||||
|
&next.request_fingerprint,
|
||||||
|
attempt_owner(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_ne!(retry.operation_id, failed.operation_id);
|
||||||
let completed = store.commit_workdir_removal_removed(&retry).unwrap();
|
let completed = store.commit_workdir_removal_removed(&retry).unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
completed.disposition,
|
completed.disposition,
|
||||||
@@ -1075,7 +1142,13 @@ mod tests {
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn recovery_and_completion_are_fenced_by_attempt_identity() {
|
async fn recovery_and_completion_are_fenced_by_attempt_identity() {
|
||||||
let (store, workdir) = seeded_store().await;
|
let (store, workdir) = seeded_store().await;
|
||||||
let intent = workdir_removal_intent(&workdir, "workspace-api", "remove Workdir").unwrap();
|
let intent = workdir_removal_intent(
|
||||||
|
&workdir,
|
||||||
|
"workspace-api",
|
||||||
|
"remove Workdir",
|
||||||
|
"test-removal-request",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
let reserved = store.reserve_workdir_removal_operation(&intent).unwrap();
|
let reserved = store.reserve_workdir_removal_operation(&intent).unwrap();
|
||||||
let first = store
|
let first = store
|
||||||
.begin_workdir_removal_attempt(
|
.begin_workdir_removal_attempt(
|
||||||
@@ -1130,8 +1203,13 @@ mod tests {
|
|||||||
use std::sync::{Arc, Barrier};
|
use std::sync::{Arc, Barrier};
|
||||||
|
|
||||||
let (store, workdir) = seeded_store().await;
|
let (store, workdir) = seeded_store().await;
|
||||||
let intent =
|
let intent = workdir_removal_intent(
|
||||||
workdir_removal_intent(&workdir, "workspace-api", "remove clean Workdir").unwrap();
|
&workdir,
|
||||||
|
"workspace-api",
|
||||||
|
"remove clean Workdir",
|
||||||
|
"test-removal-request",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
let operation = store.reserve_workdir_removal_operation(&intent).unwrap();
|
let operation = store.reserve_workdir_removal_operation(&intent).unwrap();
|
||||||
let barrier = Arc::new(Barrier::new(3));
|
let barrier = Arc::new(Barrier::new(3));
|
||||||
let provider_calls = Arc::new(AtomicUsize::new(0));
|
let provider_calls = Arc::new(AtomicUsize::new(0));
|
||||||
@@ -1218,8 +1296,13 @@ mod tests {
|
|||||||
"2",
|
"2",
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let intent =
|
let intent = workdir_removal_intent(
|
||||||
workdir_removal_intent(&workdir, "workspace-api", "remove clean Workdir").unwrap();
|
&workdir,
|
||||||
|
"workspace-api",
|
||||||
|
"remove clean Workdir",
|
||||||
|
"test-removal-request",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
store.reserve_workdir_removal_operation(&intent).unwrap();
|
store.reserve_workdir_removal_operation(&intent).unwrap();
|
||||||
|
|
||||||
let error = store
|
let error = store
|
||||||
@@ -1236,8 +1319,13 @@ mod tests {
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn pending_removal_fences_new_attachment_and_retry_rereads_live_reservation() {
|
async fn pending_removal_fences_new_attachment_and_retry_rereads_live_reservation() {
|
||||||
let (store, workdir) = seeded_store().await;
|
let (store, workdir) = seeded_store().await;
|
||||||
let intent =
|
let intent = workdir_removal_intent(
|
||||||
workdir_removal_intent(&workdir, "workspace-api", "remove clean Workdir").unwrap();
|
&workdir,
|
||||||
|
"workspace-api",
|
||||||
|
"remove clean Workdir",
|
||||||
|
"test-removal-request",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
let pending = store.reserve_workdir_removal_operation(&intent).unwrap();
|
let pending = store.reserve_workdir_removal_operation(&intent).unwrap();
|
||||||
|
|
||||||
let error = store
|
let error = store
|
||||||
@@ -1251,11 +1339,22 @@ mod tests {
|
|||||||
store
|
store
|
||||||
.reserve_worker_workdir_attachment("workspace-a", "workdir-a", "reservation-b", "3")
|
.reserve_worker_workdir_attachment("workspace-a", "workdir-a", "reservation-b", "3")
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
let next_intent = workdir_removal_intent(
|
||||||
|
&workdir,
|
||||||
|
"workspace-api",
|
||||||
|
"remove clean Workdir",
|
||||||
|
"new-request",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let next = store
|
||||||
|
.reserve_workdir_removal_operation(&next_intent)
|
||||||
|
.unwrap();
|
||||||
|
assert_ne!(next.operation_id, failed.operation_id);
|
||||||
let retry = store
|
let retry = store
|
||||||
.begin_workdir_removal_attempt(
|
.begin_workdir_removal_attempt(
|
||||||
&failed.workspace_id,
|
&next.workspace_id,
|
||||||
&failed.operation_id,
|
&next.operation_id,
|
||||||
&failed.request_fingerprint,
|
&next.request_fingerprint,
|
||||||
attempt_owner(),
|
attempt_owner(),
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
@@ -1324,7 +1423,13 @@ mod tests {
|
|||||||
Ok(())
|
Ok(())
|
||||||
})
|
})
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let intent = workdir_removal_intent(&workdir, "workspace-api", "remove Workdir").unwrap();
|
let intent = workdir_removal_intent(
|
||||||
|
&workdir,
|
||||||
|
"workspace-api",
|
||||||
|
"remove Workdir",
|
||||||
|
"test-removal-request",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
let operation = store.reserve_workdir_removal_operation(&intent).unwrap();
|
let operation = store.reserve_workdir_removal_operation(&intent).unwrap();
|
||||||
let categories = |guards: Vec<WorkdirRemovalGuard>| {
|
let categories = |guards: Vec<WorkdirRemovalGuard>| {
|
||||||
guards
|
guards
|
||||||
@@ -1447,8 +1552,13 @@ mod tests {
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn retained_completion_keeps_registry_and_is_auditable() {
|
async fn retained_completion_keeps_registry_and_is_auditable() {
|
||||||
let (store, workdir) = seeded_store().await;
|
let (store, workdir) = seeded_store().await;
|
||||||
let intent =
|
let intent = workdir_removal_intent(
|
||||||
workdir_removal_intent(&workdir, "workspace-api", "inspect dirty Workdir").unwrap();
|
&workdir,
|
||||||
|
"workspace-api",
|
||||||
|
"inspect dirty Workdir",
|
||||||
|
"test-removal-request",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
let operation = store.reserve_workdir_removal_operation(&intent).unwrap();
|
let operation = store.reserve_workdir_removal_operation(&intent).unwrap();
|
||||||
let retained = store
|
let retained = store
|
||||||
.complete_workdir_removal_retained(
|
.complete_workdir_removal_retained(
|
||||||
@@ -1477,10 +1587,12 @@ mod tests {
|
|||||||
attempt_owner(),
|
attempt_owner(),
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(retry.state, WorkdirRemovalOperationState::Pending);
|
assert_eq!(retry, retained);
|
||||||
assert!(retry.attempt_id.is_some());
|
assert!(
|
||||||
assert_eq!(retry.disposition, None);
|
store
|
||||||
assert_eq!(retry.failure_category, None);
|
.recoverable_workdir_removal_operations("workspace-a", 100)
|
||||||
assert!(retry.retryable);
|
.unwrap()
|
||||||
|
.is_empty()
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
{
|
{
|
||||||
"tasks": {
|
"tasks": {
|
||||||
"test": "deno run --allow-read --allow-write=target --allow-run --allow-env tools/test.ts",
|
"test": "deno run --allow-read --allow-write=target --allow-run --allow-env --allow-sys=uid tools/test.ts",
|
||||||
"test:ticket:real-api": "cargo test -p yoi-workspace-server --lib ticket_web_roundtrip_tests -- --ignored",
|
"test:ticket:real-api": "cargo test -p yoi-workspace-server --lib ticket_web_roundtrip_tests -- --ignored",
|
||||||
"test:runner": "deno test --allow-read --allow-write --allow-run --allow-env tools/test.test.ts"
|
"test:runner": "deno test --allow-read --allow-write --allow-run --allow-env --allow-sys=uid tools/test.test.ts"
|
||||||
},
|
},
|
||||||
"fmt": {
|
"fmt": {
|
||||||
"proseWrap": "never"
|
"proseWrap": "never"
|
||||||
|
|||||||
@@ -22,6 +22,8 @@ Assignは担当への委譲であり、readyやQueueの操作を前提にしな
|
|||||||
|
|
||||||
実装・routing・レビュー・終了の判断前には、現行Ticketと必要な関連記録を読む。通知、要約、過去の実装報告を現行の要求や承認の証拠にしない。Worker内の操作は、提供されたtyped toolsとsystem instructionsに従う。CLIや保存先の直接編集で欠けた権限を代替しない。typed Ticket toolsのないクライアントでは、Ticketの作成・更新をYoi Workerへ委ねる。
|
実装・routing・レビュー・終了の判断前には、現行Ticketと必要な関連記録を読む。通知、要約、過去の実装報告を現行の要求や承認の証拠にしない。Worker内の操作は、提供されたtyped toolsとsystem instructionsに従う。CLIや保存先の直接編集で欠けた権限を代替しない。typed Ticket toolsのないクライアントでは、Ticketの作成・更新をYoi Workerへ委ねる。
|
||||||
|
|
||||||
|
通常の進捗確認は`ShowTicket(view="summary")`を使い、要求初読・`content_digest`の変化・内容を要する判断では`detail`へ進む。モデル向け要求本文は`initial_post.body`に一度だけ置き、MRは`merge_requests`を正規の一覧とする。`history`は本文を反復しないページ、`post`は観測済み`event_ref`による全文取得であり、切詰め本文を編集CASに使わない。全文post読取りは1 MiBまでで、それを超える投稿は切詰めずエラーにする(分割全文読取りは未対応)。`ShowMergeRequest`のTool/WIPはsummaryを既定とし、履歴・review本文・findingは明示的な`detail`とcursorで読む。MR状態要約は完全な証拠から計算し、表示中の履歴ページから承認・失効を推測しない。これらはモデルへの出力を分ける契約であり、保存正本やDB読込み量の削減を意味しない。
|
||||||
|
|
||||||
成果には、実際の変更、検証結果、未確認事項、判断に必要な証拠を残す。すべての作業にコメントやartifactを義務付けず、利用者が必要とする返却方法を選ぶ。長い調査結果、生ログ、秘密情報はTicket本文へ転載しない。
|
成果には、実際の変更、検証結果、未確認事項、判断に必要な証拠を残す。すべての作業にコメントやartifactを義務付けず、利用者が必要とする返却方法を選ぶ。長い調査結果、生ログ、秘密情報はTicket本文へ転載しない。
|
||||||
|
|
||||||
対応終了と案件終了はそれぞれ明示的に判断する。結果を返して案件を開いたままにすることもできる。要求充足の判断は担当Worker・利用者の責任であり、Backendの認可・競合検査の成功や、成果物・承認済みMRの存在が代わりに判定するものではない。
|
対応終了と案件終了はそれぞれ明示的に判断する。結果を返して案件を開いたままにすることもできる。要求充足の判断は担当Worker・利用者の責任であり、Backendの認可・競合検査の成功や、成果物・承認済みMRの存在が代わりに判定するものではない。
|
||||||
@@ -32,4 +34,8 @@ Assignは担当への委譲であり、readyやQueueの操作を前提にしな
|
|||||||
|
|
||||||
source変更後は変更後のsourceを再レビューする。targetだけの変化では、未変更sourceへの承認と統合可否の再確認を区別する。通常の修正・再レビュー証拠はMerge Requestに置き、Ticketには調整が必要な阻害要因、決定、最終引継ぎを残す。
|
source変更後は変更後のsourceを再レビューする。targetだけの変化では、未変更sourceへの承認と統合可否の再確認を区別する。通常の修正・再レビュー証拠はMerge Requestに置き、Ticketには調整が必要な阻害要因、決定、最終引継ぎを残す。
|
||||||
|
|
||||||
|
再レビューの判断対象は現在の候補全体だが、全試験の再実行とは区別する。同一tree・同等条件の証拠は再利用し、新しい差分では依存・lockfile・feature・生成物・共有初期化・migration・環境への影響から必要な試験を選ぶ。元のsource/dirty snapshot、command、条件、exit/timeoutと到達範囲、ログ参照、未解決事項を一度記録し、後続報告では参照する。再利用した証拠を新sourceでの実行成功と呼ばず、旧承認を新候補へ移さない。
|
||||||
|
|
||||||
|
組込みPrompt/Flowの変更は、保存済みWorker指示や既存Flow定義を自動更新しない。新規Profileからの指示構成・Flowコンパイルの試験と、実モデルでの待機・再試験・出力量の測定を区別する。前者だけで運用改善を実証済みとしない。
|
||||||
|
|
||||||
統合、Ticketの結論、Worker停止、接続解除、資源削除は別の判断である。対応終了後の担当表示の保持と、未完作業による資源拘束も区別する。終了通知や最終回答だけから後続操作を開始せず、依頼範囲と現在の正本を確認する。未反映の変更を統合済みと報告しない。
|
統合、Ticketの結論、Worker停止、接続解除、資源削除は別の判断である。対応終了後の担当表示の保持と、未完作業による資源拘束も区別する。終了通知や最終回答だけから後続操作を開始せず、依頼範囲と現在の正本を確認する。未反映の変更を統合済みと報告しない。
|
||||||
|
|||||||
+455
-12
@@ -298,12 +298,17 @@
|
|||||||
"AuthUserResponse": {
|
"AuthUserResponse": {
|
||||||
"additionalProperties": false,
|
"additionalProperties": false,
|
||||||
"properties": {
|
"properties": {
|
||||||
|
"authentication_reference": {
|
||||||
|
"description": "Comparison evidence for the exact newly issued session, not a credential.",
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
"user": {
|
"user": {
|
||||||
"$ref": "#/components/schemas/AuthenticatedUser"
|
"$ref": "#/components/schemas/AuthenticatedUser"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"required": [
|
"required": [
|
||||||
"user"
|
"user",
|
||||||
|
"authentication_reference"
|
||||||
],
|
],
|
||||||
"type": "object"
|
"type": "object"
|
||||||
},
|
},
|
||||||
@@ -4928,6 +4933,9 @@
|
|||||||
"created_at": {
|
"created_at": {
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
|
"event_page": {
|
||||||
|
"$ref": "#/components/schemas/MergeRequestEventPage"
|
||||||
|
},
|
||||||
"linked_tickets": {
|
"linked_tickets": {
|
||||||
"items": {
|
"items": {
|
||||||
"$ref": "#/components/schemas/MergeRequestLinkedTicketResponse"
|
"$ref": "#/components/schemas/MergeRequestLinkedTicketResponse"
|
||||||
@@ -4955,6 +4963,9 @@
|
|||||||
"state": {
|
"state": {
|
||||||
"$ref": "#/components/schemas/MergeRequestState"
|
"$ref": "#/components/schemas/MergeRequestState"
|
||||||
},
|
},
|
||||||
|
"summary": {
|
||||||
|
"$ref": "#/components/schemas/MergeRequestStatusSummary"
|
||||||
|
},
|
||||||
"target": {
|
"target": {
|
||||||
"$ref": "#/components/schemas/MergeRequestRefResponse"
|
"$ref": "#/components/schemas/MergeRequestRefResponse"
|
||||||
},
|
},
|
||||||
@@ -4973,6 +4984,9 @@
|
|||||||
"updated_at": {
|
"updated_at": {
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
|
"view": {
|
||||||
|
"$ref": "#/components/schemas/MergeRequestReadView"
|
||||||
|
},
|
||||||
"workspace_id": {
|
"workspace_id": {
|
||||||
"type": "string"
|
"type": "string"
|
||||||
}
|
}
|
||||||
@@ -4989,7 +5003,63 @@
|
|||||||
"thread",
|
"thread",
|
||||||
"source",
|
"source",
|
||||||
"target",
|
"target",
|
||||||
"linked_tickets"
|
"linked_tickets",
|
||||||
|
"view",
|
||||||
|
"summary",
|
||||||
|
"event_page"
|
||||||
|
],
|
||||||
|
"type": "object"
|
||||||
|
},
|
||||||
|
"MergeRequestEventPage": {
|
||||||
|
"additionalProperties": false,
|
||||||
|
"properties": {
|
||||||
|
"has_more": {
|
||||||
|
"type": "boolean"
|
||||||
|
},
|
||||||
|
"limit": {
|
||||||
|
"format": "uint",
|
||||||
|
"maximum": 200,
|
||||||
|
"minimum": 1,
|
||||||
|
"type": "integer"
|
||||||
|
},
|
||||||
|
"next_after": {
|
||||||
|
"format": "uint64",
|
||||||
|
"maximum": 9007199254740991,
|
||||||
|
"minimum": 0,
|
||||||
|
"type": [
|
||||||
|
"integer",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"omitted": {
|
||||||
|
"description": "Summary deliberately omits events. Read detail from the beginning to obtain history.",
|
||||||
|
"type": "boolean"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"omitted",
|
||||||
|
"has_more",
|
||||||
|
"limit"
|
||||||
|
],
|
||||||
|
"type": "object"
|
||||||
|
},
|
||||||
|
"MergeRequestEventReference": {
|
||||||
|
"additionalProperties": false,
|
||||||
|
"properties": {
|
||||||
|
"event_id": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"sequence": {
|
||||||
|
"description": "Exact event: read detail with after = sequence - 1 and limit = 1.",
|
||||||
|
"format": "uint64",
|
||||||
|
"maximum": 9007199254740991,
|
||||||
|
"minimum": 0,
|
||||||
|
"type": "integer"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"event_id",
|
||||||
|
"sequence"
|
||||||
],
|
],
|
||||||
"type": "object"
|
"type": "object"
|
||||||
},
|
},
|
||||||
@@ -5115,6 +5185,13 @@
|
|||||||
],
|
],
|
||||||
"type": "object"
|
"type": "object"
|
||||||
},
|
},
|
||||||
|
"MergeRequestReadView": {
|
||||||
|
"enum": [
|
||||||
|
"summary",
|
||||||
|
"detail"
|
||||||
|
],
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
"MergeRequestReadinessResponse": {
|
"MergeRequestReadinessResponse": {
|
||||||
"additionalProperties": false,
|
"additionalProperties": false,
|
||||||
"properties": {
|
"properties": {
|
||||||
@@ -5214,6 +5291,42 @@
|
|||||||
],
|
],
|
||||||
"type": "object"
|
"type": "object"
|
||||||
},
|
},
|
||||||
|
"MergeRequestReviewSummary": {
|
||||||
|
"additionalProperties": false,
|
||||||
|
"properties": {
|
||||||
|
"decision": {
|
||||||
|
"$ref": "#/components/schemas/ReviewDecision"
|
||||||
|
},
|
||||||
|
"event": {
|
||||||
|
"$ref": "#/components/schemas/MergeRequestEventReference"
|
||||||
|
},
|
||||||
|
"findings_count": {
|
||||||
|
"description": "Findings retain their order in the exact review event; no finding body is truncated here.",
|
||||||
|
"format": "uint",
|
||||||
|
"maximum": 9007199254740991,
|
||||||
|
"minimum": 0,
|
||||||
|
"type": "integer"
|
||||||
|
},
|
||||||
|
"request_event_id": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"subject_ref": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"ticket_content_digest": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"event",
|
||||||
|
"request_event_id",
|
||||||
|
"subject_ref",
|
||||||
|
"ticket_content_digest",
|
||||||
|
"decision",
|
||||||
|
"findings_count"
|
||||||
|
],
|
||||||
|
"type": "object"
|
||||||
|
},
|
||||||
"MergeRequestState": {
|
"MergeRequestState": {
|
||||||
"enum": [
|
"enum": [
|
||||||
"open",
|
"open",
|
||||||
@@ -5222,6 +5335,58 @@
|
|||||||
],
|
],
|
||||||
"type": "string"
|
"type": "string"
|
||||||
},
|
},
|
||||||
|
"MergeRequestStatusSummary": {
|
||||||
|
"additionalProperties": false,
|
||||||
|
"description": "Computed from complete persisted evidence, never from the returned history page.\nThis observation is not integration authority or current-requirements approval.",
|
||||||
|
"properties": {
|
||||||
|
"latest_event": {
|
||||||
|
"anyOf": [
|
||||||
|
{
|
||||||
|
"$ref": "#/components/schemas/MergeRequestEventReference"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "null"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"latest_review_subject_ref": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"readiness": {
|
||||||
|
"$ref": "#/components/schemas/MergeRequestReadinessResponse",
|
||||||
|
"description": "Existing readiness semantics; review bodies are omitted in this compact projection."
|
||||||
|
},
|
||||||
|
"review": {
|
||||||
|
"anyOf": [
|
||||||
|
{
|
||||||
|
"$ref": "#/components/schemas/MergeRequestReviewSummary"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "null"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"review_status": {
|
||||||
|
"description": "unknown, none, pending, approved, or request_changes for the observed source.",
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"thread_event_count": {
|
||||||
|
"format": "uint",
|
||||||
|
"maximum": 9007199254740991,
|
||||||
|
"minimum": 0,
|
||||||
|
"type": "integer"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"review_status",
|
||||||
|
"thread_event_count",
|
||||||
|
"readiness"
|
||||||
|
],
|
||||||
|
"type": "object"
|
||||||
|
},
|
||||||
"MergeRequestThreadEvent": {
|
"MergeRequestThreadEvent": {
|
||||||
"oneOf": [
|
"oneOf": [
|
||||||
{
|
{
|
||||||
@@ -5543,6 +5708,17 @@
|
|||||||
"integer",
|
"integer",
|
||||||
"null"
|
"null"
|
||||||
]
|
]
|
||||||
|
},
|
||||||
|
"view": {
|
||||||
|
"anyOf": [
|
||||||
|
{
|
||||||
|
"$ref": "#/components/schemas/MergeRequestReadView"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"type": "null"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"description": "Show only: summary omits history; detail returns an exact bounded thread page.\nOmission retains the existing HTTP detail response."
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"type": "object"
|
"type": "object"
|
||||||
@@ -6709,15 +6885,112 @@
|
|||||||
},
|
},
|
||||||
"PasskeyAdditionOptionsRequest": {
|
"PasskeyAdditionOptionsRequest": {
|
||||||
"additionalProperties": false,
|
"additionalProperties": false,
|
||||||
"description": "Adds a credential to the current authenticated actor; target identity is never public input.",
|
"description": "Adds to the current actor. Expected identity/binding fields only fence stale UI;\nthey never select an owner or authorize an addition.",
|
||||||
"properties": {
|
"properties": {
|
||||||
"browser_origin": {
|
"browser_origin": {
|
||||||
"type": [
|
"type": [
|
||||||
"string",
|
"string",
|
||||||
"null"
|
"null"
|
||||||
]
|
]
|
||||||
|
},
|
||||||
|
"expected_account_id": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"expected_authentication_reference": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"expected_user_id": {
|
||||||
|
"type": "string"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"required": [
|
||||||
|
"expected_user_id",
|
||||||
|
"expected_account_id",
|
||||||
|
"expected_authentication_reference"
|
||||||
|
],
|
||||||
|
"type": "object"
|
||||||
|
},
|
||||||
|
"PasskeyAdditionResultRequest": {
|
||||||
|
"additionalProperties": false,
|
||||||
|
"properties": {
|
||||||
|
"challenge_id": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"expected_account_id": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"expected_authentication_reference": {
|
||||||
|
"description": "Original options authentication reference; the current same-owner session may differ.",
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"expected_user_id": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"challenge_id",
|
||||||
|
"expected_user_id",
|
||||||
|
"expected_account_id",
|
||||||
|
"expected_authentication_reference"
|
||||||
|
],
|
||||||
|
"type": "object"
|
||||||
|
},
|
||||||
|
"PasskeyAdditionResultResponse": {
|
||||||
|
"description": "Unknown is not proof that registration failed: completion may still be in flight.\nSaved proves the credential commit, not session issuance or current session health.",
|
||||||
|
"oneOf": [
|
||||||
|
{
|
||||||
|
"additionalProperties": false,
|
||||||
|
"properties": {
|
||||||
|
"status": {
|
||||||
|
"const": "unknown",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"status"
|
||||||
|
],
|
||||||
|
"type": "object"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"additionalProperties": false,
|
||||||
|
"properties": {
|
||||||
|
"key": {
|
||||||
|
"$ref": "#/components/schemas/PasskeySummary"
|
||||||
|
},
|
||||||
|
"status": {
|
||||||
|
"const": "saved",
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"user": {
|
||||||
|
"$ref": "#/components/schemas/AuthenticatedUser"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"status",
|
||||||
|
"user",
|
||||||
|
"key"
|
||||||
|
],
|
||||||
|
"type": "object"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"PasskeyListResponse": {
|
||||||
|
"additionalProperties": false,
|
||||||
|
"properties": {
|
||||||
|
"keys": {
|
||||||
|
"items": {
|
||||||
|
"$ref": "#/components/schemas/PasskeySummary"
|
||||||
|
},
|
||||||
|
"type": "array"
|
||||||
|
},
|
||||||
|
"user": {
|
||||||
|
"$ref": "#/components/schemas/AuthenticatedUser"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"user",
|
||||||
|
"keys"
|
||||||
|
],
|
||||||
"type": "object"
|
"type": "object"
|
||||||
},
|
},
|
||||||
"PasskeyLoginCompleteRequest": {
|
"PasskeyLoginCompleteRequest": {
|
||||||
@@ -6820,6 +7093,35 @@
|
|||||||
],
|
],
|
||||||
"type": "object"
|
"type": "object"
|
||||||
},
|
},
|
||||||
|
"PasskeySummary": {
|
||||||
|
"additionalProperties": false,
|
||||||
|
"description": "Safe account metadata. Neither the WebAuthn credential ID nor key material is exposed.\nNormal-login verification is observed verifier success, not inferred discoverability.",
|
||||||
|
"properties": {
|
||||||
|
"created_at": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"key_reference": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"last_used_at": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"normal_login_verified_at": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"key_reference",
|
||||||
|
"created_at"
|
||||||
|
],
|
||||||
|
"type": "object"
|
||||||
|
},
|
||||||
"PasteArtifactAvailability": {
|
"PasteArtifactAvailability": {
|
||||||
"enum": [
|
"enum": [
|
||||||
"available",
|
"available",
|
||||||
@@ -15143,6 +15445,13 @@
|
|||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
"authentication_reference": {
|
||||||
|
"description": "Non-secret comparison evidence, never an authentication credential.",
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
"session": {
|
"session": {
|
||||||
"$ref": "#/components/schemas/BrowserSessionState"
|
"$ref": "#/components/schemas/BrowserSessionState"
|
||||||
}
|
}
|
||||||
@@ -15697,14 +16006,23 @@
|
|||||||
"type": "object"
|
"type": "object"
|
||||||
},
|
},
|
||||||
"WorkerRestoreState": {
|
"WorkerRestoreState": {
|
||||||
"description": "Cross-layer outcome of an explicit Worker restore operation.\n\n`Rejected` is reserved for read-only preflight failures. Once live restore\nwork starts, failure is either a confirmed `RolledBack` operation or a\n`ReconciliationRequired` result whose commit state must be reread/retried.",
|
"description": "Cross-layer outcome of an explicit Worker restore operation.\n\n`Rejected` is reserved for read-only preflight failures. Once live restore\nwork starts, distinguish confirmed `RolledBack`, unresolved\n`ReconciliationRequired`, and a later Stop's `Superseded` outcome.",
|
||||||
"enum": [
|
"oneOf": [
|
||||||
"accepted",
|
{
|
||||||
"rejected",
|
"enum": [
|
||||||
"rolled_back",
|
"accepted",
|
||||||
"reconciliation_required"
|
"rejected",
|
||||||
],
|
"rolled_back",
|
||||||
"type": "string"
|
"reconciliation_required"
|
||||||
|
],
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"const": "superseded",
|
||||||
|
"description": "A later Stop superseded this restore; neither rollback nor prior Accepted is asserted.",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
]
|
||||||
},
|
},
|
||||||
"WorkerRetentionResponse": {
|
"WorkerRetentionResponse": {
|
||||||
"additionalProperties": false,
|
"additionalProperties": false,
|
||||||
@@ -16143,11 +16461,16 @@
|
|||||||
"WorkingDirectoryRemovalRequest": {
|
"WorkingDirectoryRemovalRequest": {
|
||||||
"additionalProperties": false,
|
"additionalProperties": false,
|
||||||
"properties": {
|
"properties": {
|
||||||
|
"operation_id": {
|
||||||
|
"description": "Stable for an exact resend; a new explicit request uses a new identity.",
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
"reason": {
|
"reason": {
|
||||||
"type": "string"
|
"type": "string"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"required": [
|
"required": [
|
||||||
|
"operation_id",
|
||||||
"reason"
|
"reason"
|
||||||
],
|
],
|
||||||
"type": "object"
|
"type": "object"
|
||||||
@@ -18155,7 +18478,7 @@
|
|||||||
"info": {
|
"info": {
|
||||||
"title": "Yoi Server API",
|
"title": "Yoi Server API",
|
||||||
"version": "0.1.0",
|
"version": "0.1.0",
|
||||||
"x-yoi-source-digest": "sha256:5de3f3b716555717d3274b8c6a385fb66624585941628a53b8b3050cb58f8ab7"
|
"x-yoi-source-digest": "sha256:d5817a2c8527f1859b68e9824b3b277987980f228b0692a2569708a57035fd10"
|
||||||
},
|
},
|
||||||
"openapi": "3.1.0",
|
"openapi": "3.1.0",
|
||||||
"paths": {
|
"paths": {
|
||||||
@@ -18589,6 +18912,61 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"/api/auth/passkeys": {
|
||||||
|
"get": {
|
||||||
|
"operationId": "auth_passkey_list",
|
||||||
|
"responses": {
|
||||||
|
"200": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/PasskeyListResponse"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "Successful response"
|
||||||
|
},
|
||||||
|
"400": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/RepositoryApiError"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "Error response"
|
||||||
|
},
|
||||||
|
"403": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/RepositoryApiError"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "Error response"
|
||||||
|
},
|
||||||
|
"500": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/RepositoryApiError"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "Error response"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"security": [
|
||||||
|
{
|
||||||
|
"bearerAuth": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"browserSession": []
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
"/api/auth/passkeys/addition/complete": {
|
"/api/auth/passkeys/addition/complete": {
|
||||||
"post": {
|
"post": {
|
||||||
"operationId": "auth_passkey_addition_complete",
|
"operationId": "auth_passkey_addition_complete",
|
||||||
@@ -18736,6 +19114,71 @@
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"/api/auth/passkeys/addition/result": {
|
||||||
|
"post": {
|
||||||
|
"operationId": "auth_passkey_addition_result",
|
||||||
|
"requestBody": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/PasskeyAdditionResultRequest"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": true
|
||||||
|
},
|
||||||
|
"responses": {
|
||||||
|
"200": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/PasskeyAdditionResultResponse"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "Successful response"
|
||||||
|
},
|
||||||
|
"400": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/RepositoryApiError"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "Error response"
|
||||||
|
},
|
||||||
|
"403": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/RepositoryApiError"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "Error response"
|
||||||
|
},
|
||||||
|
"500": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/RepositoryApiError"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "Error response"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"security": [
|
||||||
|
{
|
||||||
|
"bearerAuth": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"browserSession": []
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
"/api/auth/passkeys/login/complete": {
|
"/api/auth/passkeys/login/complete": {
|
||||||
"post": {
|
"post": {
|
||||||
"operationId": "auth_passkey_login_complete",
|
"operationId": "auth_passkey_login_complete",
|
||||||
|
|||||||
@@ -33,7 +33,7 @@
|
|||||||
};
|
};
|
||||||
|
|
||||||
fix = {
|
fix = {
|
||||||
instructions = "Resolve every open Reviewer finding on the same Ticket work branch and existing Merge Request source selector, rerun validation affected by the fixes, and commit the corrected implementation. Publish only that updated source ref with a normal non-force push and verify that the provider resolves it to exact new HEAD. Source movement invalidates the prior verdict, so request a fresh Reviewer child after publication. Do not open a replacement Merge Request, invent an operation to append another source candidate, create a fresh integration branch, rewrite previously reviewed commits, reuse the prior request_changes result as approval, push the target branch, push tags or unrelated refs, force-push, merge, delete branches, or discard pre-existing changes. Request a Flow transition only after the corrected source ref is published and ready for fresh independent review.";
|
instructions = "Resolve every open Reviewer finding on the same Ticket work branch and existing Merge Request source selector, rerun validation affected by the fixes, and commit the corrected implementation. Cite reusable evidence for unaffected scope under equivalent conditions, including its original source and execution conditions; do not claim those checks were rerun on the new source. Assess dependencies, features, generated code, shared initialization, migrations, and environment when choosing the affected checks. Fresh independent judgment covers the complete current candidate without automatically rerunning every prior suite. Publish only that updated source ref with a normal non-force push and verify that the provider resolves it to exact new HEAD. Source movement invalidates the prior verdict, so request a fresh Reviewer child after publication. Do not open a replacement Merge Request, invent an operation to append another source candidate, create a fresh integration branch, rewrite previously reviewed commits, reuse the prior request_changes result as approval, push the target branch, push tags or unrelated refs, force-push, merge, delete branches, or discard pre-existing changes. Request a Flow transition only after the corrected source ref is published and ready for fresh independent review.";
|
||||||
transitions = {
|
transitions = {
|
||||||
review = {
|
review = {
|
||||||
target = "review";
|
target = "review";
|
||||||
@@ -43,7 +43,7 @@
|
|||||||
};
|
};
|
||||||
|
|
||||||
complete = {
|
complete = {
|
||||||
instructions = "Call `ShowMergeRequest` and verify that authoritative approval still matches the exact current source ref, Ticket content digest, and complete linked result snapshot, keep that reviewed source ref immutable, save concise implementation and validation evidence with `TicketImplementationResult` using the Ticket, stable operation_key, kind, bounded body and optional retained MR/source evidence. Backend authenticates an active Workspace Worker; no current or pinned implementation assignment or Orchestrator delegation is required. Other or previously assigned Workers may report facts without fulfilling an assignment or gaining authority. Do not supply assignment identities or model actor claims. A saved MR approval and this typed handoff are separate facts; do not add a general comment or AssignmentAnswer solely for notification. Target-only movement does not invalidate this source approval; the Orchestrator refreshes readiness/integration evidence against the current target. Do not call `CompleteMergeRequest`, update the target selector, create an integration branch, or treat the Flow terminal state as Ticket completion authority. After the typed result receipt exists, request a Flow transition. Retry unknown outcomes only with the same key and full input.";
|
instructions = "Call `ShowMergeRequest` and verify that authoritative approval still matches the exact current source ref, Ticket content digest, and complete linked result snapshot, keep that reviewed source ref immutable, save concise implementation and validation evidence with `TicketImplementationResult` using the Ticket, kind, bounded body and optional retained MR/source evidence. The Host owns transport retry identity; do not supply operation_key. Backend authenticates an active Workspace Worker; no current or pinned implementation assignment or Orchestrator delegation is required. Other or previously assigned Workers may report facts without fulfilling an assignment or gaining authority. Do not supply assignment identities or model actor claims. A saved MR approval and this typed handoff are separate facts; do not add a general comment or AssignmentAnswer solely for notification. Target-only movement does not invalidate this source approval; the Orchestrator refreshes readiness/integration evidence against the current target. Do not call `CompleteMergeRequest`, update the target selector, create an integration branch, or treat the Flow terminal state as Ticket completion authority. After the typed result receipt exists, request a Flow transition. If the report outcome is uncertain, reconcile saved evidence before a new invocation; a new invocation is a new report, not a transport retry.";
|
||||||
transitions = {
|
transitions = {
|
||||||
completed = {
|
completed = {
|
||||||
target = "done";
|
target = "done";
|
||||||
|
|||||||
@@ -0,0 +1,344 @@
|
|||||||
|
# Workspace Plugin APIの設計契約
|
||||||
|
|
||||||
|
本書はWorkspace管理のPluginに採用する設計を定める。実行形式はWebAssembly Component Model、engineはWasmtime、guest/Host境界はWITとする。実装済みAPIの利用ガイドではない。既存の[v1 Tool world](wit/yoi-plugin-tool-v1.wit)、[v1 instance world](wit/yoi-plugin-instance-v1.wit)、[v1 Host imports](wit/deps/yoi-host/yoi-host-v1.wit)、PDK、templateは旧v1のoffline authoring資産であり、本書のv2実行契約をまだ提供していない。
|
||||||
|
|
||||||
|
管理、guest実行、Server上の拡張データを横断する判断をWIT資産の近くで一か所に保持する。実装時には各wire型と境界testを契約の機械的な根拠にし、ここには所有境界・失敗保証・採用理由を残す。作業順と進捗の正本はO-4、T-469、T-465、T-533とする。
|
||||||
|
|
||||||
|
## 1. 三つのAPIと所有境界
|
||||||
|
|
||||||
|
Pluginを管理する権限とPluginコードに渡す権限を分離する。Serverに保存するからServer operatorしか導入できない、とはしない。
|
||||||
|
|
||||||
|
| 境界 | 呼び出し方向 | 責任 |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Workspace管理API | 認可された管理client → Server | 導入・更新・失効・削除、設定、検証結果、診断 |
|
||||||
|
| Component exports | Runtime Host → guest | Tool呼び出し、Service/event処理 |
|
||||||
|
| Host imports | guest → Runtime Host → 必要なdomain owner | 外部通信、拡張データ、診断等の限定された作用 |
|
||||||
|
|
||||||
|
ServerはWorkspace所属、設定、installation、artifact、永続操作の正本を持つ。Runtimeはplanを適用し、guestと接続・一時handleを所有する。公開HTTP DTOは`server-api`、domain認可と永続化は`workspace-server`、guest ABIは`resources/plugin/wit`とPDKの責任とする。Rust traitやServer RESTを汎用proxyとしてguestへ渡さない。
|
||||||
|
|
||||||
|
Workspace管理者は自Workspaceに導入・更新できる。読取りcallerにはWorkspace読取り権限、mutationにはWorkspace管理権限を要求し、既存の認証済みactorから判断する。operator権限は通常導入の前提にしない。Worker/model/repository/packageの自己申告は管理権限にならない。
|
||||||
|
|
||||||
|
Hostが実行する全操作の上限は、packageの要求、Workspaceの付与、Profileの縮小、呼出元から委譲可能な範囲、現在の対象資源・運用制約の積集合とする。Serviceはlaunch時に明示されたService用grantを使い、管理者のsession credentialを借用しない。SubWorkerへのPlugin継承も再解決された委譲範囲内のみとし、親のhandleを移送しない。
|
||||||
|
|
||||||
|
## 2. identityと保存するもの
|
||||||
|
|
||||||
|
installationはWorkspaceに属し、artifactはimmutable bytesで識別する。同じpackage名やversionは権限でも実体の一意性の証拠でもない。
|
||||||
|
|
||||||
|
- `installation_id`はServerが発行するWorkspace内の導入identity。package作者の`id`は名前であり、このidentityを置換しない。
|
||||||
|
- 一つのinstallationに複数のimmutable artifactを登録できる。updateは新しいdigestの登録であり既存bytesを置換しない。同じinstallation内のpackage id/versionに異なる内容を登録する場合はversion衝突として拒否する。他Workspaceの同名versionは検査・存在開示の対象にしない。
|
||||||
|
- 同一package idの別installationは許すが、設定aliasを一意にする。同じpackageを別credentialで使う用途をversion名の改変で代替しない。
|
||||||
|
- Tool公開名はinstallationの不変ID由来namespaceと宣言Tool名からHostが構成する。表示aliasの変更で既存planのTool名を変えない。衝突・長さ超過はplan検証時に拒否し、登録時に黙ってrenameしない。
|
||||||
|
- artifact blobの共有は物理dedupeに限定する。他Workspaceのcatalog、設定、導入履歴、data、grantは共有されない。hashを知っているだけでは取得できない。
|
||||||
|
|
||||||
|
### package digestと配送digest
|
||||||
|
|
||||||
|
既存`manifest::plugin`のdigestは、path順のfile集合をhashするpackage identityであり、ZIP全体のSHA-256ではない。両者を同じfieldとして検証しない。
|
||||||
|
|
||||||
|
- `package_digest`は既存のfile集合digestを維持する。schema/manifest/componentを含む論理内容の同一性を検証する。
|
||||||
|
- `artifact_digest`はServerが保存・配送するcanonical archive bytesのSHA-256とする。`artifact_size`と共に配送・cacheを検証する。
|
||||||
|
- Serverは検証済みfile集合から既存canonical pack規則でarchiveを生成し、そのbytesを一度保存する。取得時に再packしない。upload元ZIPのbytesと保存bytesが異なり得るため、両digestと保存sizeを応答する。
|
||||||
|
- planは両digest、package schema、exact Component world/API versionを固定する。Runtimeは受信bytesのhashに加え、package validatorで論理digestを検証する。
|
||||||
|
|
||||||
|
## 3. Workspace管理API
|
||||||
|
|
||||||
|
管理操作は既存のtyped `ServerApi`へ追加する。以下は採用するoperationの意味であり、別のPlugin専用HTTP frameworkは作らない。HTTP prefixは既存の`/api/w/{workspace}`とし、route/DTO/OpenAPI/clientは同じ宣言から生成する。
|
||||||
|
|
||||||
|
### 操作と競合
|
||||||
|
|
||||||
|
| 操作 | 契約 |
|
||||||
|
| --- | --- |
|
||||||
|
| `POST /plugin-uploads` | bounded binary `.yoi-plugin`を受け、caller/Workspaceに束縛した一時upload identityと検証結果を返す。install・enableはしない |
|
||||||
|
| `POST /plugins` | upload identity、Workspace内alias、operation keyでinstallationを作成。結果はinstallation/artifact identityと確定operationの識別子 |
|
||||||
|
| `GET /plugins` / `GET /plugins/{installation}` | Workspace-scoped bounded一覧・詳細。opaque cursor、page sizeの上限あり。guest codeは実行しない |
|
||||||
|
| `POST /plugins/{installation}/artifacts` | 新しい検証済みuploadからartifact追加。観測した最終catalog変更operationと新しいoperation keyが必要。既存plan/configは変更しない |
|
||||||
|
| `POST /plugins/{installation}/revoke` | 指定artifact digestの導入許可を失効。観測した最終catalog変更operation、理由、新しいoperation key。新規実行と既存実行の操作権限を撤回 |
|
||||||
|
| `POST /plugins/{installation}/uninstall` | 観測した最終catalog変更operationと新しいoperation keyで除去を要求。参照があれば拒否し、安全な阻害情報を返す |
|
||||||
|
| `GET /plugin-operations/{operation}` | 認可された同Workspace callerが保存結果を照合。結果不明のmutationを新しいkeyでやり直さない |
|
||||||
|
|
||||||
|
uploadはmanagement operationと分ける。巨大bytesをJSON/idempotency記録に埋めず、検証に失敗したuploadからcatalogが生じないようにするためである。upload identityは24時間で失効する一時資源で、同じcontentの再uploadは可能。失効後も既にcommitしたinstallationを消さない。usage/quotaは未確定uploadも含む。
|
||||||
|
|
||||||
|
catalog mutationのoperation keyはWorkspace・認証主体・操作kindに束縛し、canonical request fingerprintと保存結果をdomain更新と同じtransactionで確定する。fingerprintは対象、観測した最終catalog変更operation、内容digest、alias、理由など意味を変える入力を含み、生成された結果IDや秘密値を含まない。同じkey/入力には元の結果、異なる入力にはconflictを返す。replay時も現在の読取り権限を確認し、権限失効後に以前の結果を漏らさない。保存済みreplayは最新stateの観測ではない。
|
||||||
|
|
||||||
|
catalogの追加・失効・除去では、そのinstallationの最終変更を確定したoperation identityをtransaction内で比較してから更新する。時刻やstateだけでは、同じstateのままartifactが追加された後の古い削除判断を拒否できないためである。設定内容はconfig treeの実内容digestで比較し、別の汎用更新カウンタは導入しない。
|
||||||
|
|
||||||
|
configのenable/disable、artifact選択、接続binding、grant変更は既存Workspace config Preview/Commit/CASへ統合する。別のenable REST endpointを作って設定正本を二重化しない。Preview後のinstallation/schema変更はcommit時にも照合する。
|
||||||
|
|
||||||
|
### install・disable・revoke・uninstall
|
||||||
|
|
||||||
|
- installは導入可能なartifactを登録するだけで、Workerの開始やTool登録を行わない。
|
||||||
|
- updateとconfig disableは新規/replacementのplanに反映する。既存Workerのpinをhot reloadしない。即時停止が必要なら明示revokeを使う。
|
||||||
|
- revokeはinstallationとartifact digestの組に対して不可逆とする。同じ組を再許可して古い実行を復活させない。管理者が同じbytesを再利用する場合は新しいinstallationを明示作成し、新しいconfig/planを生成する。古いinstallationの失効tombstoneとpinは保持する。
|
||||||
|
- revokeは新規Host操作を拒否し、稼働中guestの取消・接続closeを要求する。既に外部へ送った効果の取り消しは保証しない。Remote Runtimeへ失効通知を送るだけでは認可が撤回された証拠にならない。保護操作のadmissionはownerでlive確認し、確認不能なら拒否する。stream接続はHostの失効監視で閉じ、監視を維持できない場合も継続しない。
|
||||||
|
- uninstallはconfig、active/pending/restorable Worker pinが参照する間は拒否する。revokeしてもpinを勝手に消さない。Worker retention解除・config変更・uninstallを別操作として扱う。
|
||||||
|
- installationの拡張データが残る場合もuninstallは拒否する。後述の明示purgeで消すか、データを保持したdisabled installationとして残す。
|
||||||
|
- artifact GCは全Workspaceの参照を確認する。DB参照より先に永続blobをpublishし、crash時の未参照blobは後で回収する。未publish blobをcatalogから参照させない。
|
||||||
|
|
||||||
|
参照検査・新規pin/config参照の作成・削除fenceは同じdomain transaction境界で競合を解決する。削除直前の単なるlist結果では判断しない。
|
||||||
|
|
||||||
|
HTTP errorは既存共通error envelopeへ合わせる。invalid=400、未認証=401、権限不足=403、不可視対象=404、最終catalog変更operation不一致/operation-key/参照競合=409、size=413、quota=429、owner unavailable=503を使う。他Workspace対象の存在をerror詳細から漏らさない。errorの`retryable`だけでmutationの新規再送を許可しない。
|
||||||
|
|
||||||
|
## 4. package宣言とABI version
|
||||||
|
|
||||||
|
新しい実行契約は`yoi:plugin@2.0.0`と`yoi:host@2.0.0`にする。JSON文字列だけのHost呼出し、数値handle、guest側WebSocket recvを変えるため、v1の意味を黙って差し替えない。
|
||||||
|
|
||||||
|
- 新しいmanifest schemaはv2とし、required worlds/interfaces、Tool schema、接続slot、拡張data schema、event subscriptionを静的に宣言する。schema contributionを得るためにguestを実行しない。
|
||||||
|
- Workspace設定はDCDLのclosed typed schemaで検証する。Toolのmodel-facing入力はJSON Schemaで定義する。JSON SchemaとDCDL schemaを同じものと仮定しない。
|
||||||
|
- Componentの実imports/exportsを静的に検査してmanifestと照合する。manifestの申告だけで未公開importをlinkしない。
|
||||||
|
- 初期実装はexact world/interface versionだけを受け入れる。semver rangeでHost APIを黙って置換しない。互換な追加は別versionとして明示サポートし、既存pinには元の契約を提供するかunsupportedで止める。
|
||||||
|
- 旧v1 packageはofflineで読めても新Runtimeでは実行対象にしない。v2向けrebuild/explicit installを要求し、汎用JSON adapterで旧権限経路を復活させない。
|
||||||
|
- PDK/template/validator/WITを一緒にv2へ更新する。今回の設計文書だけを根拠に既存v1 templateが新Runtimeで動くとは扱わない。
|
||||||
|
|
||||||
|
## 5. Component exports
|
||||||
|
|
||||||
|
Tool payloadだけはPluginごとにschemaが異なるため、schema検証済みUTF-8 JSONを運ぶ。それ以外の共通context、error、event envelope、Host request/resultはWIT record/variant/resourceで定義する。
|
||||||
|
|
||||||
|
以下は採用するinterfaceの形を説明する最小例で、現行WITへ追加済みという意味ではない。
|
||||||
|
|
||||||
|
```wit
|
||||||
|
package yoi:plugin@2.0.0;
|
||||||
|
|
||||||
|
interface tool {
|
||||||
|
record invocation {
|
||||||
|
invocation-id: string,
|
||||||
|
tool-name: string,
|
||||||
|
input-json: string,
|
||||||
|
config-json: string,
|
||||||
|
remaining-ms: u32,
|
||||||
|
}
|
||||||
|
record output {
|
||||||
|
summary: string,
|
||||||
|
content: option<string>,
|
||||||
|
structured-json: option<string>,
|
||||||
|
}
|
||||||
|
record tool-error {
|
||||||
|
code: string,
|
||||||
|
message: string,
|
||||||
|
}
|
||||||
|
invoke: func(input: invocation) -> result<output, tool-error>;
|
||||||
|
}
|
||||||
|
|
||||||
|
world tool-plugin {
|
||||||
|
export tool;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`invocation-id`はHost生成の追跡identityであり権限ではない。contextにServer token、host path、任意Workspace selectorを含めない。configはplanのnon-secret snapshotで、secret値は含めない。structured outputには宣言されたoutput schemaを要求し、Hostが検証する。
|
||||||
|
|
||||||
|
Tool worldは呼出しごとにfresh Store/instanceとし、compiled Componentは共有できる。Toolのglobal/static memoryを永続化や他呼出しとの通信に使えない。v2初期Toolは応答一回のbounded callとし、guestがmodelへ直接streamを注入するAPIは設けない。
|
||||||
|
|
||||||
|
Service/event worldには`on-start(config) -> result`、`on-event(envelope) -> event-result`、`on-stop(reason) -> result`を設ける。いずれもfresh Storeで呼び、HostがWorker/installation/planに束縛したService resourceを保持する。ToolとServiceで共有する状態はHost経由の明示dataとし、guest memoryの残存を契約にしない。`status()`を呼んで診断のためにguestを実行せず、Hostが実行・接続・失敗の観測を返す。
|
||||||
|
|
||||||
|
`on-start`は設定検証とローカル初期化だけを行い、logging以外のHost作用を認めない。GETも副作用を持ち得るため、汎用HTTPや付随data書込みを初期化中に許可しない。外部event sourceの接続は全required Pluginの検証・導入成功後にHostが開始する。initialization失敗時に外部効果のrollbackを装わない。`on-stop`もloggingだけのbest effort通知で、resource解放の前提にしない。
|
||||||
|
|
||||||
|
## 6. Host importsとcapability lifetime
|
||||||
|
|
||||||
|
公開するinterfaceは機能ごとに分ける。汎用`invoke(method, json)`、汎用Server HTTP proxy、DB connection、host path、任意process実行APIは提供しない。
|
||||||
|
|
||||||
|
| interface | 操作の意味 |
|
||||||
|
| --- | --- |
|
||||||
|
| `connections` | 宣言slot名から、その呼出しに許可されたconnection resourceを取得 |
|
||||||
|
| `http` | connectionに束縛されたbounded request/response |
|
||||||
|
| `extension-data` | Plugin所有schemaのdataを対象object capability経由でread/CAS update |
|
||||||
|
| `logging` | bounded structured診断。model contextへの注入ではない |
|
||||||
|
| `workdir` | 後続で既存attachment-scoped operationを型付きにadapt。ambient filesystemは不可 |
|
||||||
|
| `websocket` | 後続のHost-owned driverへのsend/close。recvはevent配送で行う |
|
||||||
|
|
||||||
|
`connections.open(slot)`はWIT resourceを返す。slotはmanifestの宣言名で、Workspace設定が具体的な接続先とsecret参照にbindする。guestは任意credential IDや別Workspace IDを指定しない。resourceはStore/呼出し内だけで有効で、serialize・persist・別Workerへtransferできない。Serviceの長寿命接続自体はHost所有とし、各event処理時にfresh handleを発行する。
|
||||||
|
|
||||||
|
Hostは各importでcurrent invocation、installation、plan、付与範囲、対象資源、deadline/cancellationを照合する。lookup済みhandleを後続操作の無条件許可にしない。guestのComponent start sectionからの外部I/Oも禁止し、export呼出し用contextのないimportを拒否する。
|
||||||
|
|
||||||
|
planはconfig treeの内容digest、Profile projection digest、installation/artifact identities、接続binding identityとpolicyの内容digest、宣言schemaと実効grant、実行主体と委譲identityを固定する。管理操作を行った人の資格情報はguestの実行主体ではない。
|
||||||
|
|
||||||
|
| 変更・状態 | 既存plan/new export/restoreへの作用 |
|
||||||
|
| --- | --- |
|
||||||
|
| artifact追加、config enable/disable、Profileの選択・grant設定変更 | 新規/replacement planだけに適用。既存pinの集合や付与snapshotは変更しない |
|
||||||
|
| Workspace/対象資源のlive grant上限の縮小、明示的grant revoke、実行主体の委譲失効 | 既存snapshotとの積集合を直ちに縮小。次のadmission/Host操作を拒否し、進行中処理へ取消要求 |
|
||||||
|
| live grant拡大 | 古いsnapshotを広げない。追加利用には新規planが必要 |
|
||||||
|
| 接続binding失効、installation+artifactのrevoke | 新しいexport・接続・restoreを拒否。既存guest/driverを取消対象にする |
|
||||||
|
| 管理者の管理権限喪失 | 以後の管理操作を拒否。独立したWorker実行権限まで導入者の権限と同一視しない |
|
||||||
|
| ownerへ確認できない | fresh export、restore、保護操作・保存結果読取りを拒否。cached planを認可にしない |
|
||||||
|
|
||||||
|
各fresh exportの開始とrestoreにはServer ownerでinstallation/plan/実行主体のlive admissionを行う。pure Toolも例外にしない。既にadmit済みのローカル計算は取消・deadlineまで進み得るが、失効後の結果公開と追加Host作用は拒否する。初期Toolにoffline実行leaseは設けない。設定でのgrant変更とlive security grantの失効を混同しないよう、管理画面/APIの結果も適用対象を明示する。
|
||||||
|
|
||||||
|
WASIを一括linkしない。v2 worldに宣言したimportだけをallowlistし、ambient filesystem/network/environment/stdin/stdoutを与えない。clock/random等を追加する場合も個別のversioned interfaceとして明示する。PDKはこのimport集合だけでcompileできるようにする。
|
||||||
|
|
||||||
|
Wasmtimeのasync Host実装を使って待機中のexecutor threadを占有しない。ただしguestにexperimental async Component ABIを要求せず、初期WITは通常のcall/resultとする。Storeを並行利用せず、同一Service consumerへのevent処理を直列化する。
|
||||||
|
|
||||||
|
## 7. 外部HTTP接続
|
||||||
|
|
||||||
|
HTTPはURLを自由に受けるAPIではなく、connection resourceに対する相対pathのrequestとする。Server/Runtimeの資格情報を外部requestへ転用しない。
|
||||||
|
|
||||||
|
requestはmethod enum、absolute-path形式の相対path、query pair、許可されたheader pair、body bytes、要求timeoutを持つ。responseはstatus、許可されたheader、body bytesを持つ。4xx/5xxもHTTP応答として返し、transport errorと区別する。binaryはbyte listで運び、base64やJSON stringで二重に包まない。
|
||||||
|
|
||||||
|
初期Host interfaceの型は次の形にする。Tool worldにはこれらの必要なinterfaceを明示importし、未使用の機能を一括importするworldにはしない。
|
||||||
|
|
||||||
|
```wit
|
||||||
|
package yoi:host@2.0.0;
|
||||||
|
|
||||||
|
interface errors {
|
||||||
|
enum code {
|
||||||
|
invalid-input, denied, not-found, conflict, unsupported,
|
||||||
|
quota-exceeded, deadline-exceeded, cancelled, unavailable,
|
||||||
|
indeterminate, receipt-expired,
|
||||||
|
}
|
||||||
|
enum effect-observation { not-dispatched, response-observed, indeterminate }
|
||||||
|
record host-error {
|
||||||
|
code: code,
|
||||||
|
message: string,
|
||||||
|
diagnostic-id: string,
|
||||||
|
effect: effect-observation,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
interface connections {
|
||||||
|
use errors.{host-error};
|
||||||
|
resource connection;
|
||||||
|
open: func(slot: string) -> result<connection, host-error>;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface http {
|
||||||
|
use connections.{connection};
|
||||||
|
use errors.{host-error};
|
||||||
|
enum method { get, head, post, put, patch, delete, options }
|
||||||
|
record pair { name: string, value: string }
|
||||||
|
record request {
|
||||||
|
method: method,
|
||||||
|
path: string,
|
||||||
|
query: list<pair>,
|
||||||
|
headers: list<pair>,
|
||||||
|
body: list<u8>,
|
||||||
|
timeout-ms: u32,
|
||||||
|
effect-key: string,
|
||||||
|
}
|
||||||
|
record response {
|
||||||
|
status: u16,
|
||||||
|
headers: list<pair>,
|
||||||
|
body: list<u8>,
|
||||||
|
}
|
||||||
|
send: func(target: borrow<connection>, input: request) -> result<response, host-error>;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
queryはHostがpercent-encodeして送る。path中への`?`埋込みを拒否し、query/headerの重複・順序は許可されたprotocol semanticsを保持する。guest指定の接続/転送制御headerは拒否し、Hostが必要なものを生成する。effect-keyは最大128 UTF-8 bytesの非秘密のopaque文字列とし、Hostのinvocationに束縛する。
|
||||||
|
|
||||||
|
- Workspace bindingはexact origin(scheme/host/port)、許可method、segment境界のpath範囲、secret参照、資源上限を持つ。packageはslotと要求範囲を宣言できるが、bindingを自己発行できない。
|
||||||
|
- 初期APIはHTTPSのみ。TLS検証を無効にできない。raw socket、任意proxy、Unix socket、CONNECT、upgrade、redirect自動追従は提供しない。
|
||||||
|
- pathはHostが一度canonicalizeし検査・送信に同じ値を使う。scheme/authority変更、userinfo、fragment、backslash、encoded separator/dot traversalを拒否する。prefixの文字列一致で`/api`に`/apix`を含めない。
|
||||||
|
- HostがDNS解決した全候補を検査し、許可したaddressへ接続を固定する。接続時の再解決で検査を迂回しない。TLSの名前検証とSNIは許可hostnameを使う。
|
||||||
|
- loopback、link-local、metadata endpoint、Server/Runtime管理endpointを拒否する。private LANはServer運用policyが許す範囲内でWorkspaceが明示bindingできる。Workspace管理権限だけで共有Hostの内部network全体を開かない。
|
||||||
|
- `Authorization`、Cookie、Host、proxy/hop-by-hop header等はguestが指定できない。接続policyで許可したnon-secret headerだけを受け付ける。credentialはHostが指定origin/pathへだけ注入し、responseのSet-Cookie等を返さない。
|
||||||
|
- credentialの値をguestへ返すAPIはない。ただし相手serverがsecretをechoする可能性まで完全に除去できるとは主張しない。credentialの配送先はWorkspaceが管理する信頼境界として扱う。
|
||||||
|
- planはsecret参照と接続policyの内容digestを固定し、secret値は保存しない。通常のsecret rotationは同じbindingのHost解決で反映できるが、宛先/権限変更は新規planにする。live失効・policy縮小は古いplanにも効く。
|
||||||
|
- request/responseはbuffered boundedとし、上限到達時は切断してlimit errorを返す。切断されたresponseを成功に見せない。圧縮展開後の上限も検査する。
|
||||||
|
|
||||||
|
### 外部副作用と再試行
|
||||||
|
|
||||||
|
HostはHTTP requestを自動再送しない。GETであってもprovider固有semanticsがあるため、transport層が勝手に再試行しない。redirectもguestにresponseとして返すだけで、別originへの追跡は新しい認可済みbindingが必要になる。
|
||||||
|
|
||||||
|
全HTTP requestに`(Workspace, installation, Worker, consumer-or-tool-call, logical-invocation, effect-key)`のscopeを使い、request fingerprint、送信admission、保存結果を記録する。consumerはServiceの場合の固定identity、Toolの場合は永続Tool-call identityであり、別consumerで同じevent-idを処理してもreceiptを共有しない。GETも例外にせず、methodだけから副作用なしを推定しない。同じkey/内容は保存結果を返し、異なる内容はconflict。guestがkeyを変えれば新しい作用なので、package内の再試行も自動的に安全になるわけではない。
|
||||||
|
|
||||||
|
fingerprintはinstallation/plan、接続binding identityとslotとpolicyの内容digest、method、正規化path、順序を保つquery/header、body hashを含む。secretの値、残りdeadline、attemptは含めない。saved responseはboundedでWorkspace保護データとして保持し、diagnostic logとは分離する。request body/secret headerの生値をreceiptへ保存しない。同一requestの保存結果を返す際にも現在のinstallation/接続/読取り権限を検証する。
|
||||||
|
|
||||||
|
invocation identityはHostの永続Tool-call/event記録に結び付け、同じTool-callの回復や同じconsumer/eventの再配送では固定する。guestがidentityを指定・変更できない。新しいユーザーの呼出しは新identityとする。返却済みTool結果は再実行せず既存結果を使い、未完了の回復を行う場合だけ同じeffect-key領域を使う。
|
||||||
|
|
||||||
|
送信前にdurable admissionを記録し、記録不能なら送らない。送信した可能性があるままtimeout/切断/crashした場合は`indeterminate`とし、同じkeyでも汎用Hostは再送しない。provider固有の照合・idempotency APIがある場合だけ、そのadapterの契約で回復できる。Tool結果が失敗でも外部効果が取り消されたとは表示しない。
|
||||||
|
|
||||||
|
保持期限を過ぎたeffect keyは未使用keyとして扱わず`receipt-expired`で拒否する。古いinvocation/eventの再開も同様で、再実行は明示的な新しい依頼にする。invocationの受理時にHostがreceipt照合期限を固定し、期限の過ぎたidentityには新しいeffect-keyを追加できない。期限情報/tombstoneは関連Worker/eventの保持期間中残し、bodyを削除しても未実行扱いに戻さない。receipt保持とevent再配送可能期間をHostで連動させ、exactly-onceを約束しない。
|
||||||
|
|
||||||
|
## 8. 共通error、出力、診断
|
||||||
|
|
||||||
|
Host errorは閉じたWIT variantで返し、少なくともinvalid-input、denied、not-found、conflict、unsupported、quota-exceeded、deadline-exceeded、cancelled、unavailable、indeterminate、receipt-expiredを区別する。guest business errorは宣言されたcodeとbounded messageで返す。未知guest codeはplugin-failedへ正規化し、Host errorの権限をguestが偽装できないようにする。
|
||||||
|
|
||||||
|
Host errorには安全なmessage、診断ID、外部効果の観測(not-dispatched / response-observed / indeterminate)を付ける。`response-observed`はbusiness成功や外部transaction commitの証明ではない。再試行の可否はerror文字列でなくoperation契約で判断する。trap、不正output、resource超過はHostが実行失敗として記録する。
|
||||||
|
|
||||||
|
Tool outputは通常のWorker Tool結果・履歴経路へ流す。Pluginがhidden prompt、System instruction、任意Session entryを直接差し込むAPIはない。structured JSONはschema検証し、不正・超過時に切り詰めたJSONを返さない。textの表示短縮はHost projectionで明示する。
|
||||||
|
|
||||||
|
loggingはlevel、event code、安全なmessageとscalar fieldsに限定する。raw request/response、secret、prompt、host pathを標準診断に保存しない。Pluginが受け取った外部データ自体はuntrusted contentであり、命令や権限として扱わない。
|
||||||
|
|
||||||
|
## 9. 付随データの契約
|
||||||
|
|
||||||
|
Server拡張はdomain objectに関連付けるPlugin所有のtyped dataとし、既存tableへの任意column追加では実現しない。物理保存schemaはHostが管理する。`feature-storage`のnative migration/SQLite handleは信頼済みHost用でありguestへ公開しない。
|
||||||
|
|
||||||
|
addressはHost contextのWorkspace/installationと、認可済みobject resource、宣言collection、schema id/version、keyから決まる。schema id/versionも物理・論理namespaceに含め、新旧artifactが同じcellを別schemaで読み書きしない。guestにWorkspace selectorやraw table名を与えない。最初のobject kindはTicketとし、Hostが存在・Workspace・callerのread/write権限を検証したTicket resourceだけを使う。一般objectへの拡張はadapterと権限を追加して行う。
|
||||||
|
|
||||||
|
後続の`objects.resolve-ticket(ticket-ref, access)`はreadまたはextension-writeを要求し、現在のWorker主体・呼出元からの委譲とPlugin grantを検証して呼出し内だけ有効なWIT `ticket` resourceを返す。Tool入力のTicket refもevent envelopeのrefも単なる選択子であり、どちらも同じ解決を通す。ServiceにはService用grantを適用し、eventを受けたことだけでwriteを許可しない。削除済み・不可視・別Workspaceの対象はnot-foundとする。
|
||||||
|
|
||||||
|
`ticket-read.read(borrow<ticket>)`はread権限を再検査し、title、case state、content digest、bounded bodyとtruncated flagだけを返す専用read APIとする。thread、添付、認証・管理情報、将来増えるfieldを汎用serializeで公開しない。上限は64 KiB、完全本文が必要なら本APIではunsupportedとする。eventの削除通知自体は安全なmetadataとして配送できるが、削除済みTicket resourceをresolve/readできる意味ではない。
|
||||||
|
|
||||||
|
- packageはcollectionごとにschema id/versionとclosed typed schemaを宣言する。data valueのJSONはそのschemaで検証する。
|
||||||
|
- APIは`get`、`put(expected_last_write | never_written, operation_key, value)`、`delete(expected_last_write, operation_key)`、bounded `list(cursor, limit)`。任意query、SQL、cross-object transactionは公開しない。
|
||||||
|
- 同じcellへのput/deleteを確定した最終operation identityを、書込みtransaction内で観測値と比較する。CAS成功とoperation receiptを同じtransactionで保存し、自動読直しによるlast-write-winsへ変えない。同じ内容の再書込みや削除・再作成を時刻/内容hashだけで区別できないため、このoperationを比較する。
|
||||||
|
- deleteは最終delete operationのtombstoneを保持する。再作成はそのoperationを期待値にし、`never_written`を通さない。これによりABAを防ぐ。`get`は対象objectの認可成功後に限り、present(value, last-write)、never-written、deleted(last-delete)を区別する。不可視objectと不存在は同じnot-foundにする。
|
||||||
|
- data mutationのreceiptはHTTPと同じ実行scope(Workspace/installation/Worker/consumer-or-tool-call/logical-invocation)とguest operation-keyに束縛する。fingerprintに完全なcell address、put/delete種別、期待last-write、schemaとvalueのdigestを含める。異入力でkeyを再利用すればconflictとする。現在の対象認可確認後にreceipt照合をCAS検査より先に行い、成功済み同一要求を最新cellとの差で失敗にしない。照合期限・expired・tombstone保持はHTTPと同じinvocation規則を使う。
|
||||||
|
- `lookup-mutation(operation-key)`で同じinvocationの保存結果(committed: mutation identityと結果digest / not-recorded / expired)を現在の対象認可の下で照合できる。event再配送では先にreceiptを照合し、成功済みなら新しい値を読み直して同じkeyで別のputを作らない。ownerがnot-recordedを返した場合だけ現在のcellから要求を作れる。照合通信失敗はnot-recordedではない。外部HTTP・data・event ackを跨ぐ原子性は保証せず、handlerは段階ごとの保存結果で回復する。
|
||||||
|
- listはstable key順のbounded探索でありsnapshot全体の一貫性は保証しない。cursorはWorkspace/installation/object/queryに束縛し、不正・失効cursorを最初のpageに黙って戻さない。
|
||||||
|
- Ticketの通常fieldやworkflow判断をこのdata APIで変更できない。必要ならそのdomainの専用Host APIを別に設計する。
|
||||||
|
- 親object削除とPlugin data書込みはowner側の削除fenceで競合を解決する。削除後に新しい子dataを作らず、retention後にHostが回収する。
|
||||||
|
- schemaの破壊的変更は新しいschema version/namespaceとする。既存dataをguest起動時にmigrationしない。移行は新旧schemaを明示した認可付き操作として後続で設計し、初期APIでは自動変換しない。
|
||||||
|
- rollbackで旧artifactを使う場合も元schemaだけを読む。最新schemaへ読み替えない。
|
||||||
|
- revoke/disableではdataを消さない。Workspace管理者の明示purgeをoperation keyと期待data-generationで行い、先に実行をrevoke/fenceする。purge receiptを残し、uninstallとの一操作への暗黙統合をしない。
|
||||||
|
|
||||||
|
## 10. commit済みeventとService
|
||||||
|
|
||||||
|
最初のhookはcommit済みeventの観察とする。PluginをServer transaction内で実行せず、元操作の成功をPluginの応答待ちにしない。domain ownerがtransactional outboxへeventを記録し、Hostが認可・投影して配送する。共有の無制限event busは提供しない。
|
||||||
|
|
||||||
|
最初のeventはTicket変更通知とし、envelopeはevent-id、event-type/version、occurred-at、対象resource reference、変更を確定したdomain eventのidentityと対象内容digest、correlation/causation identity、bounded typed payloadを持つ。全文や秘密fieldは載せず、必要な詳細は認可付きAPIで取得する。削除eventは削除済みreferenceと安全なmetadataのみとし、resource accessを復活させない。
|
||||||
|
|
||||||
|
subscriptionはmanifest宣言・Workspace grant・Profile選択から確定し、guestが任意event名を動的subscribeしない。consumerはWorker/installation/plan内のservice単位であり、別Workerに同じPluginを入れると別consumerになる。Workspace-wide singletonや分散consumer groupは暗黙に作らない。
|
||||||
|
|
||||||
|
- event-idは再配送でも固定、attemptはHost生成の観測値。consumer内はHost保存順に一件ずつ処理する。Workspace全体や別consumer間の全順序は約束しない。
|
||||||
|
- `event-result`はack、retry-after、rejectのvariant。ackはHostのcursor commitで確定し、guestのreturnだけでは確定しない。
|
||||||
|
- deliveryは保持期間内のat-least-once。crash/timeoutで再配送され得るため、外部作用はevent-id由来invocationとstable effect-keyを使う。payload/keyを変えた再送を同一作用として扱わない。
|
||||||
|
- retry-afterはeffectが結果不明でない場合のみ再配送を予約できる。indeterminateな外部効果を含むeventは要確認でconsumerを止め、汎用retryで再送しない。
|
||||||
|
- reject、繰返し失敗、queue上限、retention gapではconsumerをpausedとして可視化する。silent drop、無限retry、詰まったeventの無断skipはしない。復旧・明示skipはWorkspace管理操作として監査する。
|
||||||
|
- cursorが保持範囲から外れたらgapを返し、最新位置へ自動復旧しない。payload取得時にも対象への現在のread権限を再検査する。
|
||||||
|
- disableは新規planに作用し、既存consumerの即時停止はrevoke/Worker stopで行う。restoreは同じconsumer identity/pin/cursorを使い、replacementは新しいconsumerとして開始する。開始時点以前のeventを自動backfillしない。
|
||||||
|
- 同じconsumerの旧・新Runtimeが同時に処理しないようHost実行leaseでfenceする。receipt/cursor mutationもleaseを検証する。
|
||||||
|
|
||||||
|
WebSocketは別のHost-owned source driverがeventへ変換する。guestの`recv(timeout)` loopは復活させない。network frame自体にdurable delivery保証はないため、domain outbox eventと同じ保証を表示しない。必要なら外部protocolのsequence/resumeをbridge側で使う。
|
||||||
|
|
||||||
|
処理前の検証・拒否hookは初期APIに含めない。将来導入時は対象操作ごとに入力/返却値/timeout/failure policy/再入・副作用を契約化し、汎用interceptorとしてServer内部を公開しない。
|
||||||
|
|
||||||
|
## 11. 資源上限と終了
|
||||||
|
|
||||||
|
以下は新APIの初期defaultとして採用する。既存実装値の転載ではない。Hostは実効limitを起動前に固定して診断へ公開し、運用上限は縮小できる。guestから無制限や上限引上げを要求できない。
|
||||||
|
|
||||||
|
| 資源 | 初期default |
|
||||||
|
| --- | --- |
|
||||||
|
| Tool input / output、event payload | 各64 KiB(UTF-8/serialized bytes) |
|
||||||
|
| config、付随dataの1 cell | 各64 KiB |
|
||||||
|
| guest linear memory | 256 MiB / Store |
|
||||||
|
| Wasmtime fuel | 10,000,000 / export call |
|
||||||
|
| Tool/event export wall deadline | 30秒、上位caller deadlineが短ければそちら |
|
||||||
|
| start / stop hook | 5秒 / 2秒 |
|
||||||
|
| HTTP request body / response body | 各1 MiB |
|
||||||
|
| HTTP header総量 | 16 KiB、64 pairまで |
|
||||||
|
| HTTP request deadline | 15秒、残りcall deadline以下 |
|
||||||
|
| guest発行Host操作 | 同時4、累計32 / call |
|
||||||
|
| log総量 | 16 KiB / call、超過はdrop件数のみ記録 |
|
||||||
|
| Service待機queue | 256 event、満杯時は配送をpause |
|
||||||
|
| event再試行 | 最大5 attempt、待機上限5分、その後pause |
|
||||||
|
| event自動再配送・effect receipt照合 | 最低7日、期限後は明示expired/gap |
|
||||||
|
| 管理list | default 50、上限100件 |
|
||||||
|
|
||||||
|
fuelはCPU時間の精密尺度ではない。wall deadline、memory/table/instance/resource数のWasmtime limiterと併用する。HTTPとguest実行で別々に30秒ずつ延長せず、同じcall deadlineから残り時間を計算する。package最大size/file数は既存validator上限を再利用し、値の第二正本を作らない。Workspace全体の保存/queue/実行同時数はHostの有限quotaでadmissionし、個別上限だけで総量を無制限にしない。
|
||||||
|
|
||||||
|
終了順は新規admission停止、invocation取消、接続/driver停止、期限付きon-stop、登録lease解除、Store/一時resource解放とする。取消中に届いた結果で終了済みinvocationへ追加作用を発行しない。失効後のon-stopは外部write禁止。guestがstopに応答しなくてもHostはresourceを回収する。既に送信された外部効果はrollbackしない。
|
||||||
|
|
||||||
|
## 12. 初期範囲と契約検証
|
||||||
|
|
||||||
|
最初の実行APIはTool、connection-bound HTTPS、loggingに絞る。付随データとcommit済みeventは本書の設計を採用する後続interfaceであり、初期worldへ動かないstubを登録しない。WebSocket、Workdir、MCP transport、判断hookも未対応importとして明示拒否する。MCP専用のprocess/credential APIを先行して作らず、必要なHost capabilityを個別に追加する。
|
||||||
|
|
||||||
|
最初のvertical contract testは、Server operatorでないWorkspace管理者によるupload/install/config、plan生成、Remote/Embedded双方のWasmtime Tool実行、Host HTTPS、通常Tool履歴への結果、同一pin restoreを通す。mock guestだけでHost境界の証明を代替しない。
|
||||||
|
|
||||||
|
境界testでは少なくとも次を固定する。
|
||||||
|
|
||||||
|
- 同じblobでも別Workspaceのcatalog/plan/secret/dataを読めず、digestだけのfetchができない。
|
||||||
|
- v1/未対応world、manifestと実importの不一致、未宣言Tool、invalid input/outputを実行前または結果公開前に拒否する。
|
||||||
|
- path canonicalization、DNS rebinding、redirect、credential header、private/control-plane宛先を実際のHost adapterで検証する。
|
||||||
|
- upload/publish/DB commitの部分失敗、CAS競合、同一key再送、異入力key reuse、uninstallとpin作成の競合を検証する。
|
||||||
|
- effectの送信前失敗、送信後応答喪失、receipt期限、cancel/revokeを区別し、indeterminateから自動再送しない。
|
||||||
|
- fuel/memory/wall deadline、Host待機中の取消、trap、不応答on-stopでもresourceが回収される。
|
||||||
|
- 後続data/eventではCAS/ABA、親削除fence、schema rollback、再配送、retention gap、lease競合、外部効果不明時pauseを検証する。
|
||||||
|
|
||||||
|
採用する契約と実装手段を区別する。DBのtable名、内部trait名、物理blob配置、adapterのmodule分割は実装裁量だが、Workspace分離、pin、APIの型・失敗保証・再試行・拡張境界はこの契約を満たす必要がある。
|
||||||
@@ -2,6 +2,10 @@
|
|||||||
|
|
||||||
MR bookkeeping is separate from selecting and adopting changes. `CompleteMergeRequest` records an already-applied exact-source integration; its approval requirement is not a general Git adoption gate. Partial or adapted adoption is not full MR integration.
|
MR bookkeeping is separate from selecting and adopting changes. `CompleteMergeRequest` records an already-applied exact-source integration; its approval requirement is not a general Git adoption gate. Partial or adapted adoption is not full MR integration.
|
||||||
|
|
||||||
|
`ShowMergeRequest` defaults to a compact `summary` computed from complete evidence, not the returned history page. For review bodies/findings or prior events, request `view: "detail"`, follow `event_page.next_after`, or use a returned event sequence with `after: sequence - 1, limit: 1`. Summary readiness is an observation, not permission to integrate; unknown source observations remain unknown.
|
||||||
|
|
||||||
When using MRs, address each by `merge_request_id`. Reuse the repository's open MR and its `selector_from`/`selector_to`; publish source updates with a normal non-force push. Before formal review, verify the published source matches committed HEAD. Keep review/fix evidence on the MR, not repeated Ticket comments.
|
When using MRs, address each by `merge_request_id`. Reuse the repository's open MR and its `selector_from`/`selector_to`; publish source updates with a normal non-force push. Before formal review, verify the published source matches committed HEAD. Keep review/fix evidence on the MR, not repeated Ticket comments.
|
||||||
|
|
||||||
|
Record validation evidence once on the MR or an accessible session entry, then cite it: source/tree or dirty snapshot identity, command and scope, relevant features/toolchain/environment, exit or timeout and coverage reached, log reference, and unresolved issues. Reuse evidence for unchanged code under equivalent conditions. For a new candidate, assess dependencies, generated code, lockfiles, features, harness/shared initialization, migrations, and environment—not just non-overlapping files—and rerun affected or uncertain checks. Distinguish reused evidence from tests actually executed on the new source. Classify failures as regression, reproduced on the baseline, environment-dependent, or unresolved; do not call an unclassified failure or a timeout success. Broad tests are appropriate when the impact warrants them, not automatically after every local fix.
|
||||||
|
|
||||||
Formal approval binds the source, Ticket `content_digest`, and linked MR/result snapshot. Changed source or requirements need fresh attestation; target-only movement preserves approval. `CheckMergeRequestReadiness` checks this recording workflow. Ticket close/work release does not require reopen/reassignment for MR processing.
|
Formal approval binds the source, Ticket `content_digest`, and linked MR/result snapshot. Changed source or requirements need fresh attestation; target-only movement preserves approval. `CheckMergeRequestReadiness` checks this recording workflow. Ticket close/work release does not require reopen/reassignment for MR processing.
|
||||||
|
|||||||
@@ -1,11 +1,15 @@
|
|||||||
## Ticket workflow
|
## Ticket workflow
|
||||||
|
|
||||||
Use the available typed Ticket tools as the authority for Ticket reads and mutations. Use `QueryTicket` for bounded discovery and filtering, then `ShowTicket` for the authoritative item content and `content_digest`, thread/evidence, relations, linked Objectives, and any current Merge Request context before routing, review, or conclusion decisions. Do not invoke a Ticket CLI or edit backend storage directly as an alternative implementation of those tools.
|
Use the available typed Ticket tools as the authority for Ticket reads and mutations. Use `QueryTicket` for bounded discovery and filtering, then `ShowTicket` for the authoritative item content and `content_digest`, thread/evidence, relations, linked Objectives, and any current Merge Request context before routing, review, or conclusion decisions.
|
||||||
|
|
||||||
|
For routine progress or notification checks, use `ShowTicket` with `view: "summary"`; it omits prose and history, not authority checks. Read `view: "detail"` for the initial requirements, changed `content_digest`, or decisions needing content. Use `view: "history"` with the returned cursor for older events, and `view: "post"` with an observed `event_ref` for exact text before editing a truncated preview. The canonical requirement body is `initial_post.body`; linked MRs are in `merge_requests`. A summary or cached observation never substitutes for required rereads, edit CAS, or approval.
|
||||||
|
|
||||||
Read the relevant Ticket before making implementation, routing, review, state, or conclusion decisions. Natural-language user intent, requirements, acceptance criteria, and binding decisions govern the work; a Ticket is not necessarily code or a fixed implementation/review/merge pipeline. Do not infer the current contract from an id, title, notification, or remembered summary alone. Check related or potentially duplicate Tickets when creating or materially rescoping work.
|
Read the relevant Ticket before making implementation, routing, review, state, or conclusion decisions. Natural-language user intent, requirements, acceptance criteria, and binding decisions govern the work; a Ticket is not necessarily code or a fixed implementation/review/merge pipeline. Do not infer the current contract from an id, title, notification, or remembered summary alone. Check related or potentially duplicate Tickets when creating or materially rescoping work.
|
||||||
|
|
||||||
Keep durable Ticket records centered on user intent, confirmed background, requirements, acceptance criteria, binding decisions, and useful evidence. Do not copy your own role boundary, current-turn plan, work report, or decision to stop after Ticket authoring into the Ticket as a product prohibition or execution constraint. Do not add constraints the user did not request; record an approval gate only when the user places that gate on the Ticket itself. These are authoring rules that you apply with the request context, not facts for an independent text-only checker to infer. Separate confirmed facts from user claims, hypotheses, and open questions. Avoid prematurely turning implementation tactics into requirements. Use `QueryObjective` for bounded Objective discovery and `ShowObjective` for authoritative content and linked-Ticket context when coordinating broader work.
|
Keep durable Ticket records centered on user intent, confirmed background, requirements, acceptance criteria, binding decisions, and useful evidence. Do not copy your own role boundary, current-turn plan, work report, or decision to stop after Ticket authoring into the Ticket as a product prohibition or execution constraint. Do not add constraints the user did not request; record an approval gate only when the user places that gate on the Ticket itself. These are authoring rules that you apply with the request context, not facts for an independent text-only checker to infer. Separate confirmed facts from user claims, hypotheses, and open questions. Avoid prematurely turning implementation tactics into requirements. Use `QueryObjective` for bounded Objective discovery and `ShowObjective` for authoritative content and linked-Ticket context when coordinating broader work.
|
||||||
|
|
||||||
|
Record a Ticket dependency when delivering its outcome requires another Ticket's outcome. Editing the same file on separate branches does not by itself create a dependency or require exclusive editing. Keep independent changes self-contained on their own branches and reconcile them during merge.
|
||||||
|
|
||||||
Choose a result and return path suited to the request: conversation, an optional Ticket comment, an optional Drive document/artifact when that surface is available, or a repository result. Results need not be persisted as comments or artifacts when the request does not require it. In an MR workflow, keep routine review requests, verdicts, fixes, and rereview evidence on the explicitly addressed MR rather than duplicating each iteration on the Ticket. Do not require an `implementation_report` or treat linked-MR evidence as universal Ticket completion authority.
|
Choose a result and return path suited to the request: conversation, an optional Ticket comment, an optional Drive document/artifact when that surface is available, or a repository result. Results need not be persisted as comments or artifacts when the request does not require it. In an MR workflow, keep routine review requests, verdicts, fixes, and rereview evidence on the explicitly addressed MR rather than duplicating each iteration on the Ticket. Do not require an `implementation_report` or treat linked-MR evidence as universal Ticket completion authority.
|
||||||
|
|
||||||
Ticket cases have only `open` (still handled) and `closed` (handling ended for now); open is not execution readiness and closed is not necessarily fulfilled. Use authorized `TicketCaseUpdate` with `close`, `reopen`, or `correct_closure`, an explanation and typed closure reason when the request calls for it. Observe `content_digest`, case state, and nullable `closure_operation_key` from `ShowTicket` for CAS, and use a stable operation key for exact replay. Closure reasons and immutable history are separate from classification tags; tag changes never alter the closure judgment. Reread and reconcile after conflict; never guess a newer observation. Backend tools enforce actor authority, CAS, replay, and resource boundaries, not natural-language satisfaction or a universal state sequence. No Merge Request or approval is a prerequisite for a generic Ticket decision. Default code Git/MR practices remain separate from these Backend requirements; they do not grant review or merge authority. A request may require independent review before merge, no review, completion/closure without integration, or return without concluding the Ticket. Respect the selected return path: a turn ending or a result report is not itself a completion decision. Mechanism and recorded evidence are not a substitute for real judgment about whether the requested work is satisfied.
|
Ticket cases have only `open` (still handled) and `closed` (handling ended for now); open is not execution readiness and closed is not necessarily fulfilled. Use authorized `TicketCaseUpdate` with `close`, `reopen`, or `correct_closure`, an explanation and typed closure reason when the request calls for it. Observe `content_digest`, case state, and nullable `closure_operation_key` from `ShowTicket` for CAS, and use a stable operation key for exact replay. Closure reasons and immutable history are separate from classification tags; tag changes never alter the closure judgment. Reread and reconcile after conflict; never guess a newer observation. Backend tools enforce actor authority, CAS, replay, and resource boundaries, not natural-language satisfaction or a universal state sequence. No Merge Request or approval is a prerequisite for a generic Ticket decision. Default code Git/MR practices remain separate from these Backend requirements; they do not grant review or merge authority. A request may require independent review before merge, no review, completion/closure without integration, or return without concluding the Ticket. Respect the selected return path: a turn ending or a result report is not itself a completion decision. Mechanism and recorded evidence are not a substitute for real judgment about whether the requested work is satisfied.
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ Worker-session tools are a read-only exploration surface over host-granted activ
|
|||||||
- Use `WorkerList` to discover known Workers and reuse its returned structured `subject` exactly. Runtime Workers use `{ kind: "runtime_worker", runtime_id, worker_id }`, while parent-owned children use `{ kind: "sub_worker", name }`. Do not guess subject identifiers.
|
- Use `WorkerList` to discover known Workers and reuse its returned structured `subject` exactly. Runtime Workers use `{ kind: "runtime_worker", runtime_id, worker_id }`, while parent-owned children use `{ kind: "sub_worker", name }`. Do not guess subject identifiers.
|
||||||
- Use `ViewSessionOverview` for sparse orientation, `SearchSessionEntries` for bounded range/filter queries, and `ReadSessionEntry` for one bounded entry.
|
- Use `ViewSessionOverview` for sparse orientation, `SearchSessionEntries` for bounded range/filter queries, and `ReadSessionEntry` for one bounded entry.
|
||||||
- `SessionEntryRef` is the common entry identity across overview, search, reads, and evidence conversion. Reuse returned `E...` values; never invent them.
|
- `SessionEntryRef` is the common entry identity across overview, search, reads, and evidence conversion. Reuse returned `E...` values; never invent them.
|
||||||
|
- For routine progress, read new results, failures, and blockers with their stable evidence references rather than reopening and transcribing all earlier logs. Before adoption or completion, inspect the concrete evidence relevant to that decision; an inaccessible reference remains unverified. Keep handling updates focused on current status, next action, and material result references.
|
||||||
- Every operation rereads the latest committed capture. Existing references remain stable when entries append.
|
- Every operation rereads the latest committed capture. Existing references remain stable when entries append.
|
||||||
- A missing subject and an unauthorized subject intentionally produce the same result. Treat either as inaccessible.
|
- A missing subject and an unauthorized subject intentionally produce the same result. Treat either as inaccessible.
|
||||||
- Observation is not mutation authority and does not prove approval, completion, or Ticket state. Reread the relevant domain authority before acting.
|
- Observation is not mutation authority and does not prove approval, completion, or Ticket state. Reread the relevant domain authority before acting.
|
||||||
|
|||||||
@@ -4,6 +4,10 @@ The host gives you one immutable attempt snapshot containing the current state,
|
|||||||
|
|
||||||
Use ShowOverview, SearchEntries, and ReadEntry to inspect bounded committed parent-session evidence. When read-only Workdir tools are available, use Read, Glob, and Grep only as needed to check current repository evidence. You have no authority to mutate files, Tickets, Memory, Workers, Workdirs, Flow state, or any other domain.
|
Use ShowOverview, SearchEntries, and ReadEntry to inspect bounded committed parent-session evidence. When read-only Workdir tools are available, use Read, Glob, and Grep only as needed to check current repository evidence. You have no authority to mutate files, Tickets, Memory, Workers, Workdirs, Flow state, or any other domain.
|
||||||
|
|
||||||
|
Use stable entry references in the capture to inspect already-recorded validation rather than requiring the parent to repeat the same logs or commands. Evidence from an unchanged tree under equivalent conditions can remain applicable; for a changed candidate, inspect the impact on dependencies, generated code, features, shared initialization, migrations, and environment before reusing unaffected evidence. Reuse is not a claim that checks ran on the new source, and never transfers an old approval or authority.
|
||||||
|
|
||||||
|
A reference alone is not verification: inaccessible evidence, timeouts, and checks that did not reach their target remain unverified. You cannot perform live MR or other-session reads with tools that are not provided. If a condition requires a current authoritative observation that the captured evidence cannot establish, return `indeterminate`; do not invent freshness or demand copied prose as a substitute.
|
||||||
|
|
||||||
For every supplied transition, decide exactly one verdict:
|
For every supplied transition, decide exactly one verdict:
|
||||||
|
|
||||||
- `met`: the available evidence establishes the condition.
|
- `met`: the available evidence establishes the condition.
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user