Compare commits
57
Commits
897c025b85
...
dogfooding
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e7351c16b0 | ||
|
|
00d96b7163 | ||
|
|
2b4bf4b7bd | ||
|
|
e7ebc92f83 | ||
|
|
181b0d637e | ||
|
|
b0733e53b2 | ||
|
|
618880d38d | ||
|
|
79b76c8789 | ||
|
|
5b12c3e085 | ||
|
|
623c31615d | ||
|
|
af719e387d | ||
|
|
dddb21bd70 | ||
|
|
e8d4e964cb | ||
|
|
eb24cb04dc | ||
|
|
e84057f402 | ||
|
|
15c3006d09 | ||
|
|
b3209c2012 | ||
|
|
6f5a424661 | ||
|
|
8d47d2670b | ||
|
|
b19de45a82 | ||
|
|
1e88893ef9 | ||
|
|
1fd2e09971 | ||
|
|
f85d831104 | ||
|
|
2b24db5a15 | ||
|
|
e1945d977c | ||
|
|
a10875d68e | ||
|
|
f35dcef4d0 | ||
|
|
f5ebae1134 | ||
|
|
6be4e8388b | ||
|
|
9e3f87ac52 | ||
|
|
ceb01f6eaa | ||
|
|
1b13563e19 | ||
|
|
848a1e9a5b | ||
|
|
a77453338c | ||
|
|
d12bc2ea0b | ||
|
|
a1df3742b1 | ||
|
|
d8818b776b | ||
|
|
40c36ccb1d | ||
|
|
80c151d084 | ||
|
|
c6a8759e7b | ||
|
|
f17b5212c7 | ||
|
|
85f70d4bc1 | ||
|
|
b284934e34 | ||
|
|
2d33321fab | ||
|
|
0975471986 | ||
|
|
214d6e21dd | ||
|
|
fca5c23345 | ||
|
|
f9b1500ba2 | ||
|
|
931bf3b27d | ||
|
|
6bb671be21 | ||
|
|
b8f2fa479c | ||
|
|
1639be9841 | ||
|
|
907392cce8 | ||
|
|
73a3e674c9 | ||
|
|
b5c105545c | ||
|
|
61c20b0995 | ||
|
|
b7a22146a0 |
Generated
+6
@@ -6712,15 +6712,21 @@ name = "yoi-e2e"
|
|||||||
version = "0.0.0"
|
version = "0.0.0"
|
||||||
dependencies = [
|
dependencies = [
|
||||||
"chrono",
|
"chrono",
|
||||||
|
"config-source",
|
||||||
"libc",
|
"libc",
|
||||||
|
"manifest",
|
||||||
|
"protocol",
|
||||||
"reqwest",
|
"reqwest",
|
||||||
"serde",
|
"serde",
|
||||||
"serde_json",
|
"serde_json",
|
||||||
"server-api",
|
"server-api",
|
||||||
|
"session-store",
|
||||||
"sha2 0.11.0",
|
"sha2 0.11.0",
|
||||||
"tempfile",
|
"tempfile",
|
||||||
|
"tungstenite 0.29.0",
|
||||||
"webauthn-authenticator-rs",
|
"webauthn-authenticator-rs",
|
||||||
"webauthn-rs-proto",
|
"webauthn-rs-proto",
|
||||||
|
"worker-runtime",
|
||||||
"yoi-workspace-server",
|
"yoi-workspace-server",
|
||||||
]
|
]
|
||||||
|
|
||||||
|
|||||||
+184
-83
@@ -617,7 +617,8 @@ impl fmt::Display for McpRequestStage {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// A running initialized stdio MCP client.
|
/// An owned stdio MCP process, initialized by `connect` or explicitly after
|
||||||
|
/// `spawn_uninitialized` when startup cleanup must retain it before any await.
|
||||||
pub struct McpStdioClient {
|
pub struct McpStdioClient {
|
||||||
server_name: String,
|
server_name: String,
|
||||||
limits: McpStdioLimits,
|
limits: McpStdioLimits,
|
||||||
@@ -636,6 +637,9 @@ pub struct McpStdioClient {
|
|||||||
|
|
||||||
impl McpStdioClient {
|
impl McpStdioClient {
|
||||||
/// Spawn, initialize, negotiate capabilities, and send notifications/initialized.
|
/// Spawn, initialize, negotiate capabilities, and send notifications/initialized.
|
||||||
|
/// Initialization errors attempt shutdown, but this convenience API cannot
|
||||||
|
/// return an unreaped client on failure/cancellation. Its error is not proof
|
||||||
|
/// of cleanup; use `spawn_uninitialized` with retained ownership when needed.
|
||||||
pub async fn connect(
|
pub async fn connect(
|
||||||
spec: McpStdioServerSpec,
|
spec: McpStdioServerSpec,
|
||||||
limits: McpStdioLimits,
|
limits: McpStdioLimits,
|
||||||
@@ -667,6 +671,21 @@ impl McpStdioClient {
|
|||||||
async fn spawn(
|
async fn spawn(
|
||||||
spec: McpStdioServerSpec,
|
spec: McpStdioServerSpec,
|
||||||
limits: McpStdioLimits,
|
limits: McpStdioLimits,
|
||||||
|
) -> Result<Self, McpClientError> {
|
||||||
|
Self::spawn_uninitialized(spec, limits)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Spawn without awaiting the handshake, so a cleanup owner can retain the
|
||||||
|
/// client before initialization can fail or be cancelled. Call
|
||||||
|
/// `initialize_with_timeout` before discovery; on error/cancellation, shut
|
||||||
|
/// down this same client rather than spawning a replacement.
|
||||||
|
///
|
||||||
|
/// Once the process spawns, this returns its owner even if an expected pipe
|
||||||
|
/// is missing. Initialization then fails, but cleanup can still reap the
|
||||||
|
/// actual child rather than relying on kill-on-drop for a post-spawn error.
|
||||||
|
pub fn spawn_uninitialized(
|
||||||
|
spec: McpStdioServerSpec,
|
||||||
|
limits: McpStdioLimits,
|
||||||
) -> Result<Self, McpClientError> {
|
) -> Result<Self, McpClientError> {
|
||||||
let redactor = spec.redactor();
|
let redactor = spec.redactor();
|
||||||
let mut command = Command::new(&spec.command);
|
let mut command = Command::new(&spec.command);
|
||||||
@@ -688,29 +707,23 @@ impl McpStdioClient {
|
|||||||
McpErrorKind::Io(redactor.redact(&err.to_string())),
|
McpErrorKind::Io(redactor.redact(&err.to_string())),
|
||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
let stdin = child.stdin.take().ok_or_else(|| {
|
let stdin = child.stdin.take();
|
||||||
McpClientError::new(
|
let stdout = child.stdout.take();
|
||||||
&spec.name,
|
let stderr = child.stderr.take();
|
||||||
McpPhase::Spawn,
|
let missing_pipe = if stdin.is_none() {
|
||||||
McpErrorKind::Protocol("child stdin was not piped".into()),
|
Some("stdin")
|
||||||
)
|
} else if stdout.is_none() {
|
||||||
})?;
|
Some("stdout")
|
||||||
let stdout = child.stdout.take().ok_or_else(|| {
|
} else if stderr.is_none() {
|
||||||
McpClientError::new(
|
Some("stderr")
|
||||||
&spec.name,
|
} else {
|
||||||
McpPhase::Spawn,
|
None
|
||||||
McpErrorKind::Protocol("child stdout was not piped".into()),
|
};
|
||||||
)
|
|
||||||
})?;
|
|
||||||
let stderr = child.stderr.take().ok_or_else(|| {
|
|
||||||
McpClientError::new(
|
|
||||||
&spec.name,
|
|
||||||
McpPhase::Spawn,
|
|
||||||
McpErrorKind::Protocol("child stderr was not piped".into()),
|
|
||||||
)
|
|
||||||
})?;
|
|
||||||
|
|
||||||
let stdin: SharedStdin = Arc::new(Mutex::new(Some(Box::new(stdin))));
|
let stdin: SharedStdin =
|
||||||
|
Arc::new(Mutex::new(stdin.map(|stdin| {
|
||||||
|
Box::new(stdin) as Box<dyn AsyncWrite + Send + Unpin>
|
||||||
|
})));
|
||||||
let diagnostics = Arc::new(Mutex::new(BoundedDiagnostics::new(
|
let diagnostics = Arc::new(Mutex::new(BoundedDiagnostics::new(
|
||||||
spec.name.clone(),
|
spec.name.clone(),
|
||||||
limits.max_diagnostic_lines,
|
limits.max_diagnostic_lines,
|
||||||
@@ -718,16 +731,32 @@ impl McpStdioClient {
|
|||||||
)));
|
)));
|
||||||
let list_changes = Arc::new(Mutex::new(BoundedListChanged::new(spec.name.clone())));
|
let list_changes = Arc::new(Mutex::new(BoundedListChanged::new(spec.name.clone())));
|
||||||
let (tx, rx) = mpsc::channel(16);
|
let (tx, rx) = mpsc::channel(16);
|
||||||
let reader_task = spawn_stdout_reader(
|
// Do not return an error that discards an already spawned child. Queue
|
||||||
spec.name.clone(),
|
// setup failure before reader events so initialization cannot succeed.
|
||||||
stdout,
|
if let Some(pipe) = missing_pipe {
|
||||||
stdin.clone(),
|
tx.try_send(ReaderEvent::Error(McpClientError::new(
|
||||||
tx,
|
&spec.name,
|
||||||
limits.clone(),
|
McpPhase::Spawn,
|
||||||
redactor.clone(),
|
McpErrorKind::Protocol(format!("child {pipe} was not piped")),
|
||||||
list_changes.clone(),
|
)))
|
||||||
);
|
.expect("new response channel has capacity");
|
||||||
let stderr_task = spawn_stderr_reader(stderr, diagnostics.clone(), limits.clone());
|
}
|
||||||
|
let reader_task = match stdout {
|
||||||
|
Some(stdout) => spawn_stdout_reader(
|
||||||
|
spec.name.clone(),
|
||||||
|
stdout,
|
||||||
|
stdin.clone(),
|
||||||
|
tx,
|
||||||
|
limits.clone(),
|
||||||
|
redactor.clone(),
|
||||||
|
list_changes.clone(),
|
||||||
|
),
|
||||||
|
None => tokio::spawn(async {}),
|
||||||
|
};
|
||||||
|
let stderr_task = match stderr {
|
||||||
|
Some(stderr) => spawn_stderr_reader(stderr, diagnostics.clone(), limits.clone()),
|
||||||
|
None => tokio::spawn(async {}),
|
||||||
|
};
|
||||||
|
|
||||||
Ok(Self {
|
Ok(Self {
|
||||||
server_name: spec.name,
|
server_name: spec.name,
|
||||||
@@ -746,6 +775,25 @@ impl McpStdioClient {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Perform the bounded handshake on an already retained client. Failure or
|
||||||
|
/// cancellation does not drop or shut down the client: its owner must do so.
|
||||||
|
/// Do not retry initialization on a partially initialized connection.
|
||||||
|
pub async fn initialize_with_timeout(&mut self) -> Result<(), McpClientError> {
|
||||||
|
let started = Instant::now();
|
||||||
|
match timeout(self.limits.startup_timeout, self.initialize()).await {
|
||||||
|
Ok(result) => result,
|
||||||
|
Err(_) => Err(McpClientError::new(
|
||||||
|
&self.server_name,
|
||||||
|
McpPhase::Initialize,
|
||||||
|
McpErrorKind::Timeout {
|
||||||
|
operation: "startup".to_string(),
|
||||||
|
elapsed: started.elapsed(),
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.with_diagnostics(self.snapshot_diagnostics().await)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn initialize(&mut self) -> Result<(), McpClientError> {
|
async fn initialize(&mut self) -> Result<(), McpClientError> {
|
||||||
let result: InitializeResult = self
|
let result: InitializeResult = self
|
||||||
.request(
|
.request(
|
||||||
@@ -1080,6 +1128,8 @@ impl McpStdioClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/// Close stdin and wait for process exit, falling back to terminate and kill.
|
/// Close stdin and wait for process exit, falling back to terminate and kill.
|
||||||
|
/// Errors and cancellation retain the child for another attempt on this
|
||||||
|
/// client. Only confirmed reaping releases the process handle.
|
||||||
pub async fn shutdown(&mut self) -> Result<ShutdownReport, McpClientError> {
|
pub async fn shutdown(&mut self) -> Result<ShutdownReport, McpClientError> {
|
||||||
self.shutdown_started = true;
|
self.shutdown_started = true;
|
||||||
let deadline = Instant::now() + self.limits.shutdown_timeout;
|
let deadline = Instant::now() + self.limits.shutdown_timeout;
|
||||||
@@ -1091,64 +1141,55 @@ impl McpStdioClient {
|
|||||||
.await
|
.await
|
||||||
.is_ok();
|
.is_ok();
|
||||||
|
|
||||||
let mut child = match self.child.take() {
|
// Keep the actual process handle in the client across every await and
|
||||||
Some(child) => child,
|
// error. Taking it before wait loses retry/reaping authority on cancel.
|
||||||
None => return Ok(ShutdownReport::already_finished()),
|
if self.child.is_none() {
|
||||||
};
|
return Ok(ShutdownReport::already_finished());
|
||||||
|
}
|
||||||
if !closed {
|
if !closed {
|
||||||
self.reader_task.abort();
|
self.reader_task.abort();
|
||||||
return self.terminate_then_kill(child).await;
|
return self.terminate_then_kill().await;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
let child = self.child.as_mut().expect("checked child");
|
||||||
if let Ok(Some(status)) = child.try_wait() {
|
if let Ok(Some(status)) = child.try_wait() {
|
||||||
self.reader_task.abort();
|
return Ok(self.finish_shutdown(status, false, false));
|
||||||
self.stderr_task.abort();
|
|
||||||
return Ok(ShutdownReport {
|
|
||||||
exit_status: Some(status),
|
|
||||||
terminated: false,
|
|
||||||
killed: false,
|
|
||||||
});
|
|
||||||
}
|
}
|
||||||
|
|
||||||
match timeout_at(deadline, child.wait()).await {
|
match timeout_at(deadline, child.wait()).await {
|
||||||
Ok(Ok(status)) => {
|
Ok(Ok(status)) => Ok(self.finish_shutdown(status, false, false)),
|
||||||
self.reader_task.abort();
|
|
||||||
self.stderr_task.abort();
|
|
||||||
Ok(ShutdownReport {
|
|
||||||
exit_status: Some(status),
|
|
||||||
terminated: false,
|
|
||||||
killed: false,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
Ok(Err(err)) => Err(McpClientError::new(
|
Ok(Err(err)) => Err(McpClientError::new(
|
||||||
&self.server_name,
|
&self.server_name,
|
||||||
McpPhase::Shutdown,
|
McpPhase::Shutdown,
|
||||||
McpErrorKind::Io(self.redactor.redact(&err.to_string())),
|
McpErrorKind::Io(self.redactor.redact(&err.to_string())),
|
||||||
)
|
)
|
||||||
.with_diagnostics(self.snapshot_diagnostics().await)),
|
.with_diagnostics(self.snapshot_diagnostics().await)),
|
||||||
Err(_) => self.terminate_then_kill(child).await,
|
Err(_) => self.terminate_then_kill().await,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn terminate_then_kill(
|
fn finish_shutdown(
|
||||||
&mut self,
|
&mut self,
|
||||||
mut child: Child,
|
status: ExitStatus,
|
||||||
) -> Result<ShutdownReport, McpClientError> {
|
terminated: bool,
|
||||||
let mut terminated = false;
|
killed: bool,
|
||||||
let mut killed = false;
|
) -> ShutdownReport {
|
||||||
if send_terminate(&mut child).is_ok() {
|
// No await separates confirmed reaping from releasing the handle.
|
||||||
terminated = true;
|
drop(self.child.take());
|
||||||
|
self.reader_task.abort();
|
||||||
|
self.stderr_task.abort();
|
||||||
|
ShutdownReport {
|
||||||
|
exit_status: Some(status),
|
||||||
|
terminated,
|
||||||
|
killed,
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn terminate_then_kill(&mut self) -> Result<ShutdownReport, McpClientError> {
|
||||||
|
let child = self.child.as_mut().expect("shutdown retains child");
|
||||||
|
let terminated = send_terminate(child).is_ok();
|
||||||
match timeout(self.limits.kill_timeout, child.wait()).await {
|
match timeout(self.limits.kill_timeout, child.wait()).await {
|
||||||
Ok(Ok(status)) => {
|
Ok(Ok(status)) => Ok(self.finish_shutdown(status, terminated, false)),
|
||||||
self.reader_task.abort();
|
|
||||||
self.stderr_task.abort();
|
|
||||||
Ok(ShutdownReport {
|
|
||||||
exit_status: Some(status),
|
|
||||||
terminated,
|
|
||||||
killed,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
Ok(Err(err)) => Err(McpClientError::new(
|
Ok(Err(err)) => Err(McpClientError::new(
|
||||||
&self.server_name,
|
&self.server_name,
|
||||||
McpPhase::Shutdown,
|
McpPhase::Shutdown,
|
||||||
@@ -1163,7 +1204,6 @@ impl McpStdioClient {
|
|||||||
McpErrorKind::Io(self.redactor.redact(&err.to_string())),
|
McpErrorKind::Io(self.redactor.redact(&err.to_string())),
|
||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
killed = true;
|
|
||||||
let status = timeout(self.limits.kill_timeout, child.wait())
|
let status = timeout(self.limits.kill_timeout, child.wait())
|
||||||
.await
|
.await
|
||||||
.map_err(|_| {
|
.map_err(|_| {
|
||||||
@@ -1183,13 +1223,7 @@ impl McpStdioClient {
|
|||||||
McpErrorKind::Io(self.redactor.redact(&err.to_string())),
|
McpErrorKind::Io(self.redactor.redact(&err.to_string())),
|
||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
self.reader_task.abort();
|
Ok(self.finish_shutdown(status, terminated, true))
|
||||||
self.stderr_task.abort();
|
|
||||||
Ok(ShutdownReport {
|
|
||||||
exit_status: Some(status),
|
|
||||||
terminated,
|
|
||||||
killed,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1197,10 +1231,10 @@ impl McpStdioClient {
|
|||||||
|
|
||||||
impl Drop for McpStdioClient {
|
impl Drop for McpStdioClient {
|
||||||
fn drop(&mut self) {
|
fn drop(&mut self) {
|
||||||
if !self.shutdown_started {
|
// An attempted shutdown is not proof of exit. Errors and cancellation
|
||||||
if let Some(child) = &mut self.child {
|
// leave the child here so Drop must still make a best-effort kill.
|
||||||
let _ = child.start_kill();
|
if let Some(child) = &mut self.child {
|
||||||
}
|
let _ = child.start_kill();
|
||||||
}
|
}
|
||||||
self.reader_task.abort();
|
self.reader_task.abort();
|
||||||
self.stderr_task.abort();
|
self.stderr_task.abort();
|
||||||
@@ -1784,3 +1818,70 @@ fn send_terminate(child: &mut Child) -> Result<(), ()> {
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod deadline_tests;
|
mod deadline_tests;
|
||||||
|
|
||||||
|
#[cfg(all(test, unix))]
|
||||||
|
mod shutdown_retention_tests {
|
||||||
|
//! The real client and OS process are used; only the external server is a
|
||||||
|
//! local shell that acknowledges initialization and ignores TERM/EOF.
|
||||||
|
use super::*;
|
||||||
|
use std::future::{Future, poll_fn};
|
||||||
|
use std::task::Poll;
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn cancelled_graceful_or_terminate_wait_retries_reaping_the_same_child() {
|
||||||
|
for cancel_in_fallback in [false, true] {
|
||||||
|
let script = r#"trap '' TERM
|
||||||
|
read init || exit 1
|
||||||
|
printf '%s\n' '{"jsonrpc":"2.0","id":1,"result":{"protocolVersion":"2025-11-25","capabilities":{},"serverInfo":{"name":"retained","version":"1"}}}'
|
||||||
|
while :; do :; done
|
||||||
|
"#;
|
||||||
|
let mut client = McpStdioClient::connect(
|
||||||
|
McpStdioServerSpec::new("retained", "/bin/sh").args(["-c", script]),
|
||||||
|
McpStdioLimits {
|
||||||
|
shutdown_timeout: if cancel_in_fallback {
|
||||||
|
Duration::ZERO
|
||||||
|
} else {
|
||||||
|
Duration::from_secs(30)
|
||||||
|
},
|
||||||
|
kill_timeout: Duration::from_secs(30),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("initialize after installing TERM handler");
|
||||||
|
let pid = client.child.as_ref().unwrap().id();
|
||||||
|
let mut shutdown = Box::pin(client.shutdown());
|
||||||
|
poll_fn(|cx| {
|
||||||
|
assert!(shutdown.as_mut().poll(cx).is_pending());
|
||||||
|
Poll::Ready(())
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
drop(shutdown);
|
||||||
|
assert_eq!(client.child.as_ref().and_then(Child::id), pid);
|
||||||
|
assert!(client.shutdown_started);
|
||||||
|
|
||||||
|
client.limits.shutdown_timeout = Duration::from_millis(20);
|
||||||
|
client.limits.kill_timeout = Duration::from_millis(50);
|
||||||
|
let report = timeout(Duration::from_secs(2), client.shutdown())
|
||||||
|
.await
|
||||||
|
.expect("retry bounded")
|
||||||
|
.expect("retry reaps original child");
|
||||||
|
assert!(
|
||||||
|
report.exit_status.is_some(),
|
||||||
|
"not an already-finished receipt"
|
||||||
|
);
|
||||||
|
assert!(report.killed, "TERM was ignored");
|
||||||
|
assert!(client.child.is_none());
|
||||||
|
let mut status = 0;
|
||||||
|
assert_eq!(
|
||||||
|
unsafe { libc::waitpid(pid.unwrap() as libc::pid_t, &mut status, libc::WNOHANG) },
|
||||||
|
-1
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
std::io::Error::last_os_error().raw_os_error(),
|
||||||
|
Some(libc::ECHILD),
|
||||||
|
"shutdown, not a later observer, reaped the original process"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -210,12 +210,7 @@ impl MergeRequestThreadEvent {
|
|||||||
}
|
}
|
||||||
fn bound_bodies(&mut self) {
|
fn bound_bodies(&mut self) {
|
||||||
match self {
|
match self {
|
||||||
Self::Review(value) => {
|
Self::Review(_) => {}
|
||||||
truncate_body(&mut value.body);
|
|
||||||
for finding in &mut value.findings {
|
|
||||||
truncate_body(&mut finding.body);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
Self::ReviewRevoked(value) => truncate_body(&mut value.reason),
|
Self::ReviewRevoked(value) => truncate_body(&mut value.reason),
|
||||||
Self::ReviewCancelled(value) => truncate_body(&mut value.reason),
|
Self::ReviewCancelled(value) => truncate_body(&mut value.reason),
|
||||||
Self::Comment(value) => truncate_body(&mut value.body),
|
Self::Comment(value) => truncate_body(&mut value.body),
|
||||||
@@ -939,10 +934,6 @@ impl MergeRequestStore {
|
|||||||
&self,
|
&self,
|
||||||
i: SubmitMergeRequestReview,
|
i: SubmitMergeRequestReview,
|
||||||
) -> Result<ReviewEvent, MergeRequestError> {
|
) -> Result<ReviewEvent, MergeRequestError> {
|
||||||
bounded_body("review body", &i.body)?;
|
|
||||||
for finding in &i.findings {
|
|
||||||
bounded_body("review finding", &finding.body)?;
|
|
||||||
}
|
|
||||||
let mut c = self.lock()?;
|
let mut c = self.lock()?;
|
||||||
let t = c.transaction()?;
|
let t = c.transaction()?;
|
||||||
let g: Option<(String, String, String, String, String, String)> = t
|
let g: Option<(String, String, String, String, String, String)> = t
|
||||||
@@ -1747,7 +1738,23 @@ fn ticket_event(
|
|||||||
) -> Result<(), MergeRequestError> {
|
) -> Result<(), MergeRequestError> {
|
||||||
let ticket = &mr.ticket_ids[0];
|
let ticket = &mr.ticket_ids[0];
|
||||||
let n:i64=t.query_row("SELECT COALESCE(MAX(event_index),-1)+1 FROM typed_ticket_events WHERE workspace_id=?1 AND ticket_id=?2",params![mr.workspace_id,ticket],|r|r.get(0))?;
|
let n:i64=t.query_row("SELECT COALESCE(MAX(event_index),-1)+1 FROM typed_ticket_events WHERE workspace_id=?1 AND ticket_id=?2",params![mr.workspace_id,ticket],|r|r.get(0))?;
|
||||||
t.execute("INSERT INTO typed_ticket_events(workspace_id,ticket_id,event_index,kind,author,at,from_state,to_state,heading,body)VALUES(?1,?2,?3,'comment',?4,?5,NULL,NULL,'Merge Request integrated',?6)",params![mr.workspace_id,ticket,n,format!("worker:{}:{}",e.merged_by.runtime_id,e.merged_by.worker_id),e.created_at.to_rfc3339(),format!("Merge Request `{}` integrated approved source ref `{}`; Ticket remains in progress until guarded Ticket completion.",mr.merge_request_id,e.approved_source_ref)])?;
|
t.execute(
|
||||||
|
"INSERT INTO typed_ticket_events(
|
||||||
|
workspace_id,ticket_id,event_index,kind,author,at,from_state,to_state,heading,body,event_ref
|
||||||
|
) VALUES(?1,?2,?3,'comment',?4,?5,NULL,NULL,'Merge Request integrated',?6,?7)",
|
||||||
|
params![
|
||||||
|
mr.workspace_id,
|
||||||
|
ticket,
|
||||||
|
n,
|
||||||
|
format!("worker:{}:{}", e.merged_by.runtime_id, e.merged_by.worker_id),
|
||||||
|
e.created_at.to_rfc3339(),
|
||||||
|
format!(
|
||||||
|
"Merge Request `{}` integrated approved source ref `{}`; Ticket remains in progress until guarded Ticket completion.",
|
||||||
|
mr.merge_request_id, e.approved_source_ref
|
||||||
|
),
|
||||||
|
format!("{ticket}:{n}"),
|
||||||
|
],
|
||||||
|
)?;
|
||||||
for (k, v) in [
|
for (k, v) in [
|
||||||
("implementation_assignment_id", a),
|
("implementation_assignment_id", a),
|
||||||
("merge_request_id", &mr.merge_request_id),
|
("merge_request_id", &mr.merge_request_id),
|
||||||
|
|||||||
@@ -53,8 +53,8 @@ fn fixture() -> (tempfile::TempDir, MergeRequestStore) {
|
|||||||
let d = tempfile::tempdir().unwrap();
|
let d = tempfile::tempdir().unwrap();
|
||||||
let p = d.path().join("db");
|
let p = d.path().join("db");
|
||||||
let c = Connection::open(&p).unwrap();
|
let c = Connection::open(&p).unwrap();
|
||||||
c.execute_batch("CREATE TABLE workspaces(workspace_id TEXT PRIMARY KEY);CREATE TABLE repositories(workspace_id TEXT,repository_id TEXT,PRIMARY KEY(workspace_id,repository_id));CREATE TABLE ticket_current_worker_assignments(workspace_id TEXT,ticket_id TEXT,assignment_id TEXT,runtime_id TEXT,worker_id TEXT,updated_at TEXT,PRIMARY KEY(workspace_id,ticket_id));CREATE TABLE typed_tickets(workspace_id TEXT,ticket_id TEXT,workflow_state TEXT,workflow_state_explicit INTEGER,updated_at TEXT,title TEXT NOT NULL,body TEXT NOT NULL,PRIMARY KEY(workspace_id,ticket_id));CREATE TABLE typed_ticket_targets(workspace_id TEXT,ticket_id TEXT,ordinal INTEGER,repository_key TEXT,ref_selector TEXT,access TEXT);CREATE TABLE typed_ticket_events(workspace_id TEXT,ticket_id TEXT,event_index INTEGER,kind TEXT,author TEXT,at TEXT,from_state TEXT,to_state TEXT,heading TEXT,body TEXT,PRIMARY KEY(workspace_id,ticket_id,event_index));CREATE TABLE typed_ticket_event_attributes(workspace_id TEXT,ticket_id TEXT,event_index INTEGER,key TEXT,value TEXT,PRIMARY KEY(workspace_id,ticket_id,event_index,key));INSERT INTO workspaces VALUES('W');INSERT INTO repositories VALUES('W','R');INSERT INTO repositories VALUES('W','R2');INSERT INTO ticket_current_worker_assignments VALUES('W','T','A','runtime','coder','t');INSERT INTO typed_tickets VALUES('W','T','inprogress',1,'t','Title','Requirements');CREATE VIEW ticket_active_worker_assignments AS SELECT current.* FROM ticket_current_worker_assignments current JOIN typed_tickets ticket ON ticket.workspace_id=current.workspace_id AND ticket.ticket_id=current.ticket_id WHERE ticket.workflow_state NOT IN ('done','closed');").unwrap();
|
c.execute_batch("CREATE TABLE workspaces(workspace_id TEXT PRIMARY KEY);CREATE TABLE repositories(workspace_id TEXT,repository_id TEXT,PRIMARY KEY(workspace_id,repository_id));CREATE TABLE ticket_current_worker_assignments(workspace_id TEXT,ticket_id TEXT,assignment_id TEXT,runtime_id TEXT,worker_id TEXT,updated_at TEXT,PRIMARY KEY(workspace_id,ticket_id));CREATE TABLE typed_tickets(workspace_id TEXT,ticket_id TEXT,workflow_state TEXT,workflow_state_explicit INTEGER,updated_at TEXT,title TEXT NOT NULL,body TEXT NOT NULL,PRIMARY KEY(workspace_id,ticket_id));CREATE TABLE typed_ticket_targets(workspace_id TEXT,ticket_id TEXT,ordinal INTEGER,repository_key TEXT,ref_selector TEXT,access TEXT);CREATE TABLE typed_ticket_events(workspace_id TEXT,ticket_id TEXT,event_index INTEGER,kind TEXT,author TEXT,at TEXT,from_state TEXT,to_state TEXT,heading TEXT,body TEXT,event_ref TEXT NOT NULL DEFAULT '',PRIMARY KEY(workspace_id,ticket_id,event_index),UNIQUE(workspace_id,ticket_id,event_ref));CREATE TABLE typed_ticket_event_attributes(workspace_id TEXT,ticket_id TEXT,event_index INTEGER,key TEXT,value TEXT,PRIMARY KEY(workspace_id,ticket_id,event_index,key));INSERT INTO workspaces VALUES('W');INSERT INTO repositories VALUES('W','R');INSERT INTO repositories VALUES('W','R2');INSERT INTO ticket_current_worker_assignments VALUES('W','T','A','runtime','coder','t');INSERT INTO typed_tickets VALUES('W','T','inprogress',1,'t','Title','Requirements');CREATE VIEW ticket_active_worker_assignments AS SELECT current.* FROM ticket_current_worker_assignments current JOIN typed_tickets ticket ON ticket.workspace_id=current.workspace_id AND ticket.ticket_id=current.ticket_id WHERE ticket.workflow_state NOT IN ('done','closed');").unwrap();
|
||||||
c.execute("INSERT INTO typed_ticket_events(workspace_id,ticket_id,event_index,kind,body) VALUES('W','T',-1,'comment','Requirements')",[]).unwrap();
|
c.execute("INSERT INTO typed_ticket_events(workspace_id,ticket_id,event_index,kind,body,event_ref) VALUES('W','T',-1,'comment','Requirements','T:-1')",[]).unwrap();
|
||||||
drop(c);
|
drop(c);
|
||||||
let a = Assignments(Arc::new(Mutex::new(CurrentAssignment {
|
let a = Assignments(Arc::new(Mutex::new(CurrentAssignment {
|
||||||
assignment_id: "A".into(),
|
assignment_id: "A".into(),
|
||||||
@@ -376,6 +376,49 @@ fn approve_for(
|
|||||||
fn approve(s: &MergeRequestStore, subject: &str, token: &str) -> ReviewEvent {
|
fn approve(s: &MergeRequestStore, subject: &str, token: &str) -> ReviewEvent {
|
||||||
approve_for(s, "MR", "R", subject, token)
|
approve_for(s, "MR", "R", subject, token)
|
||||||
}
|
}
|
||||||
|
#[test]
|
||||||
|
fn review_submission_preserves_long_body_and_findings_on_reload() {
|
||||||
|
for (body, finding_body) in [
|
||||||
|
("b".repeat(16 * 1024 + 1), "finding".into()),
|
||||||
|
("review".into(), "指摘".repeat(4096)),
|
||||||
|
] {
|
||||||
|
let (_dir, store) = fixture();
|
||||||
|
open(&store);
|
||||||
|
request(&store, "source", "token");
|
||||||
|
let findings = vec![ReviewFinding {
|
||||||
|
severity: FindingSeverity::Major,
|
||||||
|
code: None,
|
||||||
|
path: None,
|
||||||
|
line: None,
|
||||||
|
body: finding_body,
|
||||||
|
}];
|
||||||
|
let submitted = store
|
||||||
|
.submit_review(SubmitMergeRequestReview {
|
||||||
|
merge_request_id: "MR".into(),
|
||||||
|
ticket_id: "T".into(),
|
||||||
|
current_subject_ref: "source".into(),
|
||||||
|
capability_token: "token".into(),
|
||||||
|
decision: ReviewDecision::RequestChanges,
|
||||||
|
body: body.clone(),
|
||||||
|
findings: findings.clone(),
|
||||||
|
now: at(4),
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(submitted.body, body);
|
||||||
|
assert_eq!(submitted.findings, findings);
|
||||||
|
let reloaded = store.get_by_id("W", "MR").unwrap();
|
||||||
|
let saved = reloaded
|
||||||
|
.thread
|
||||||
|
.iter()
|
||||||
|
.find_map(|event| match event {
|
||||||
|
MergeRequestThreadEvent::Review(review) => Some(review),
|
||||||
|
_ => None,
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(saved, &submitted);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn review_submission_authorization_rejects_invalid_grants_before_side_effects() {
|
fn review_submission_authorization_rejects_invalid_grants_before_side_effects() {
|
||||||
let (_d, store) = fixture();
|
let (_d, store) = fixture();
|
||||||
@@ -457,6 +500,18 @@ fn integration_preserves_selectors_ticket_state_and_assignment_and_replays_its_m
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(replayed, merged);
|
assert_eq!(replayed, merged);
|
||||||
let connection = Connection::open(d.path().join("db")).unwrap();
|
let connection = Connection::open(d.path().join("db")).unwrap();
|
||||||
|
let ticket_refs = connection
|
||||||
|
.prepare("SELECT event_ref FROM typed_ticket_events WHERE event_index >= 0")
|
||||||
|
.unwrap()
|
||||||
|
.query_map([], |row| row.get::<_, String>(0))
|
||||||
|
.unwrap()
|
||||||
|
.collect::<Result<Vec<_>, _>>()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
ticket_refs,
|
||||||
|
vec!["T:0"],
|
||||||
|
"replay must not append another post"
|
||||||
|
);
|
||||||
let (state, active): (String, bool) = connection.query_row(
|
let (state, active): (String, bool) = connection.query_row(
|
||||||
"SELECT workflow_state, EXISTS(SELECT 1 FROM ticket_active_worker_assignments WHERE workspace_id='W' AND ticket_id='T') FROM typed_tickets WHERE workspace_id='W' AND ticket_id='T'",
|
"SELECT workflow_state, EXISTS(SELECT 1 FROM ticket_active_worker_assignments WHERE workspace_id='W' AND ticket_id='T') FROM typed_tickets WHERE workspace_id='W' AND ticket_id='T'",
|
||||||
[],
|
[],
|
||||||
@@ -477,6 +532,87 @@ fn integration_preserves_selectors_ticket_state_and_assignment_and_replays_its_m
|
|||||||
assert!(!json.contains(banned), "{banned} in {json}")
|
assert!(!json.contains(banned), "{banned} in {json}")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
#[test]
|
||||||
|
fn integration_post_failure_rolls_back_and_same_operation_recovers_with_legacy_empty_ref() {
|
||||||
|
let (dir, store) = fixture();
|
||||||
|
open(&store);
|
||||||
|
let review = approve(&store, "source", "approved");
|
||||||
|
request(&store, "source", "pending");
|
||||||
|
let completion = CompleteMergeRequest {
|
||||||
|
merge_request_id: "MR".into(),
|
||||||
|
ticket_id: "T".into(),
|
||||||
|
operation_id: "recover-integration".into(),
|
||||||
|
approval_event_id: review.event_id,
|
||||||
|
current_subject_ref: "source".into(),
|
||||||
|
target_ref_before: "before".into(),
|
||||||
|
target_ref_after: "after".into(),
|
||||||
|
strategy: MergeStrategy::FastForward,
|
||||||
|
resolution: ConflictResolution::None,
|
||||||
|
auth: auth(),
|
||||||
|
now: at(5),
|
||||||
|
};
|
||||||
|
let connection = Connection::open(dir.path().join("db")).unwrap();
|
||||||
|
// Retain a comment written before integration populated event_ref.
|
||||||
|
connection.execute_batch(
|
||||||
|
"INSERT INTO typed_ticket_events(workspace_id,ticket_id,event_index,kind,body,event_ref)
|
||||||
|
VALUES('W','T',0,'comment','Legacy integration','');
|
||||||
|
CREATE TRIGGER reject_integration_post BEFORE INSERT ON typed_ticket_events
|
||||||
|
WHEN NEW.heading='Merge Request integrated'
|
||||||
|
BEGIN SELECT RAISE(ABORT,'injected integration post failure'); END;",
|
||||||
|
).unwrap();
|
||||||
|
let before = store.get_by_id("W", "MR").unwrap();
|
||||||
|
let error = store.complete(completion.clone()).unwrap_err();
|
||||||
|
assert!(
|
||||||
|
error
|
||||||
|
.to_string()
|
||||||
|
.contains("injected integration post failure")
|
||||||
|
);
|
||||||
|
let failed = store.get_by_id("W", "MR").unwrap();
|
||||||
|
assert_eq!(failed.state, MergeRequestState::Open);
|
||||||
|
assert_eq!(failed.thread, before.thread);
|
||||||
|
assert!(store.authorize_review_submission("MR", "pending").is_ok());
|
||||||
|
assert_eq!(
|
||||||
|
connection
|
||||||
|
.query_row(
|
||||||
|
"SELECT COUNT(*) FROM typed_ticket_event_attributes",
|
||||||
|
[],
|
||||||
|
|row| row.get::<_, i64>(0),
|
||||||
|
)
|
||||||
|
.unwrap(),
|
||||||
|
0
|
||||||
|
);
|
||||||
|
|
||||||
|
connection
|
||||||
|
.execute_batch("DROP TRIGGER reject_integration_post")
|
||||||
|
.unwrap();
|
||||||
|
let merged = store.complete(completion.clone()).unwrap();
|
||||||
|
assert_eq!(store.complete(completion).unwrap(), merged);
|
||||||
|
assert_eq!(
|
||||||
|
store.get_by_id("W", "MR").unwrap().state,
|
||||||
|
MergeRequestState::Merged
|
||||||
|
);
|
||||||
|
assert!(store.authorize_review_submission("MR", "pending").is_err());
|
||||||
|
let posts = connection
|
||||||
|
.prepare("SELECT event_ref,body FROM typed_ticket_events WHERE event_index >= 0 ORDER BY event_index")
|
||||||
|
.unwrap()
|
||||||
|
.query_map([], |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)))
|
||||||
|
.unwrap()
|
||||||
|
.collect::<Result<Vec<_>, _>>()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
posts.len(),
|
||||||
|
2,
|
||||||
|
"retry and replay must append exactly one post"
|
||||||
|
);
|
||||||
|
assert_eq!(posts[0], (String::new(), "Legacy integration".into()));
|
||||||
|
assert_eq!(posts[1].0, "T:1");
|
||||||
|
assert!(posts[1].1.contains("`MR`"));
|
||||||
|
assert_eq!(connection.query_row(
|
||||||
|
"SELECT value FROM typed_ticket_event_attributes WHERE event_index=1 AND key='operation_id'",
|
||||||
|
[], |row| row.get::<_, String>(0),
|
||||||
|
).unwrap(), "recover-integration");
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn source_move_cancels_submission_and_old_approval_is_reusable_when_source_returns() {
|
fn source_move_cancels_submission_and_old_approval_is_reusable_when_source_returns() {
|
||||||
let (_d, s) = fixture();
|
let (_d, s) = fixture();
|
||||||
@@ -955,7 +1091,7 @@ fn ticket_rescope_requires_fresh_review_before_merge_and_remains_recoverable() {
|
|||||||
let connection = Connection::open(dir.path().join("db")).unwrap();
|
let connection = Connection::open(dir.path().join("db")).unwrap();
|
||||||
connection
|
connection
|
||||||
.execute_batch(
|
.execute_batch(
|
||||||
"UPDATE typed_tickets SET title='Rescoped' WHERE workspace_id='W' AND ticket_id='T'; INSERT INTO typed_ticket_events VALUES('W','T',1,'item_edit','user','t2',NULL,NULL,NULL,NULL)",
|
"UPDATE typed_tickets SET title='Rescoped' WHERE workspace_id='W' AND ticket_id='T'; INSERT INTO typed_ticket_events VALUES('W','T',1,'item_edit','user','t2',NULL,NULL,NULL,NULL,'T:1')",
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
connection
|
connection
|
||||||
@@ -1082,7 +1218,7 @@ fn ticket_rescope_after_integration_accepts_fresh_requirement_attestation() {
|
|||||||
let connection = Connection::open(dir.path().join("db")).unwrap();
|
let connection = Connection::open(dir.path().join("db")).unwrap();
|
||||||
connection
|
connection
|
||||||
.execute_batch(
|
.execute_batch(
|
||||||
"UPDATE typed_tickets SET title='Rescoped' WHERE workspace_id='W' AND ticket_id='T'; INSERT INTO typed_ticket_events VALUES('W','T',1,'item_edit','user','t2',NULL,NULL,NULL,NULL)",
|
"UPDATE typed_tickets SET title='Rescoped' WHERE workspace_id='W' AND ticket_id='T'; INSERT INTO typed_ticket_events VALUES('W','T',1,'item_edit','user','t2',NULL,NULL,NULL,NULL,'T:1')",
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
connection
|
connection
|
||||||
@@ -1307,6 +1443,24 @@ fn partial_and_full_integration_retain_ticket_state_and_active_assignment() {
|
|||||||
assert_eq!(second_request.merged_result().unwrap(), &second_merge);
|
assert_eq!(second_request.merged_result().unwrap(), &second_merge);
|
||||||
assert_eq!(first_request.integration_approval().unwrap(), &first);
|
assert_eq!(first_request.integration_approval().unwrap(), &first);
|
||||||
assert_eq!(second_request.integration_approval().unwrap(), &second);
|
assert_eq!(second_request.integration_approval().unwrap(), &second);
|
||||||
|
let posts = connection
|
||||||
|
.prepare(
|
||||||
|
"SELECT event.event_ref, attribute.value
|
||||||
|
FROM typed_ticket_events event
|
||||||
|
JOIN typed_ticket_event_attributes attribute USING(workspace_id,ticket_id,event_index)
|
||||||
|
WHERE attribute.key='merge_request_id' ORDER BY event.event_index",
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
.query_map([], |row| {
|
||||||
|
Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
|
||||||
|
})
|
||||||
|
.unwrap()
|
||||||
|
.collect::<Result<Vec<_>, _>>()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
posts,
|
||||||
|
vec![("T:0".into(), "MR".into()), ("T:1".into(), "MR-2".into())]
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -1324,7 +1478,7 @@ fn historical_ticket_completion_payload_remains_readable_and_unchanged_on_migrat
|
|||||||
}"#;
|
}"#;
|
||||||
connection.execute_batch(
|
connection.execute_batch(
|
||||||
"UPDATE typed_tickets SET workflow_state='done' WHERE workspace_id='W' AND ticket_id='T';
|
"UPDATE typed_tickets SET workflow_state='done' WHERE workspace_id='W' AND ticket_id='T';
|
||||||
INSERT INTO typed_ticket_events VALUES('W','T',0,'state_changed','worker:runtime:coder','2026-07-26T12:00:07Z','inprogress','done','Ticket requirements completed','historical body');"
|
INSERT INTO typed_ticket_events VALUES('W','T',0,'state_changed','worker:runtime:coder','2026-07-26T12:00:07Z','inprogress','done','Ticket requirements completed','historical body','T:0');"
|
||||||
).unwrap();
|
).unwrap();
|
||||||
connection
|
connection
|
||||||
.execute(
|
.execute(
|
||||||
|
|||||||
@@ -86,6 +86,8 @@ pub enum WorkerRestoreState {
|
|||||||
Accepted,
|
Accepted,
|
||||||
Rejected,
|
Rejected,
|
||||||
RolledBack,
|
RolledBack,
|
||||||
|
/// A later Stop superseded this restore; neither rollback nor prior Accepted is asserted.
|
||||||
|
Superseded,
|
||||||
ReconciliationRequired,
|
ReconciliationRequired,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1578,13 +1580,24 @@ mod tests {
|
|||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert!(request.preparation.is_none());
|
assert!(request.preparation.is_none());
|
||||||
for state in [
|
for (state, wire) in [
|
||||||
"accepted",
|
(WorkerRestoreState::Accepted, "accepted"),
|
||||||
"rejected",
|
(WorkerRestoreState::Rejected, "rejected"),
|
||||||
"rolled_back",
|
(WorkerRestoreState::RolledBack, "rolled_back"),
|
||||||
"reconciliation_required",
|
(WorkerRestoreState::Superseded, "superseded"),
|
||||||
|
(
|
||||||
|
WorkerRestoreState::ReconciliationRequired,
|
||||||
|
"reconciliation_required",
|
||||||
|
),
|
||||||
] {
|
] {
|
||||||
assert!(serde_json::from_value::<WorkerRestoreState>(serde_json::json!(state)).is_ok());
|
assert_eq!(
|
||||||
|
serde_json::from_value::<WorkerRestoreState>(serde_json::json!(wire)).unwrap(),
|
||||||
|
state
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(state).unwrap(),
|
||||||
|
serde_json::json!(wire)
|
||||||
|
);
|
||||||
}
|
}
|
||||||
assert!(
|
assert!(
|
||||||
serde_json::from_value::<WorkerRestoreState>(serde_json::json!(
|
serde_json::from_value::<WorkerRestoreState>(serde_json::json!(
|
||||||
|
|||||||
+128
-10
@@ -229,6 +229,10 @@ impl_openapi_schema!(
|
|||||||
PasskeyLoginOptionsResponse,
|
PasskeyLoginOptionsResponse,
|
||||||
PasskeyLoginCompleteRequest,
|
PasskeyLoginCompleteRequest,
|
||||||
PasskeyAdditionOptionsRequest,
|
PasskeyAdditionOptionsRequest,
|
||||||
|
PasskeySummary,
|
||||||
|
PasskeyListResponse,
|
||||||
|
PasskeyAdditionResultRequest,
|
||||||
|
PasskeyAdditionResultResponse,
|
||||||
PasskeyRegistrationOptionsRequest,
|
PasskeyRegistrationOptionsRequest,
|
||||||
PasskeyRegistrationOptionsResponse,
|
PasskeyRegistrationOptionsResponse,
|
||||||
PasskeyRegistrationCompleteRequest,
|
PasskeyRegistrationCompleteRequest,
|
||||||
@@ -552,8 +556,9 @@ pub struct WorkspaceWorkerSessionHistoryResponse {
|
|||||||
/// comparisons do not replace admission checks under the Runtime operation lock; conflicts do
|
/// comparisons do not replace admission checks under the Runtime operation lock; conflicts do
|
||||||
/// not trigger automatic token replacement.
|
/// not trigger automatic token replacement.
|
||||||
/// [`WorkerRestoreState`] Accepted means Restore committed, not that the subsequent run succeeded.
|
/// [`WorkerRestoreState`] Accepted means Restore committed, not that the subsequent run succeeded.
|
||||||
/// Distinguish read-only preflight Rejected, confirmed RolledBack after side effects, and
|
/// Distinguish read-only preflight Rejected, confirmed RolledBack after side effects,
|
||||||
/// ReconciliationRequired needing settled-state checks. Response loss does not imply rollback.
|
/// Superseded by a later Stop, and ReconciliationRequired needing settled-state checks.
|
||||||
|
/// Response loss does not imply rollback.
|
||||||
#[api(reqwest, axum, openapi)]
|
#[api(reqwest, axum, openapi)]
|
||||||
pub trait ServerApi {
|
pub trait ServerApi {
|
||||||
#[get("/health", status = 200, error_status = 400)]
|
#[get("/health", status = 200, error_status = 400)]
|
||||||
@@ -652,6 +657,33 @@ pub trait ServerApi {
|
|||||||
#[body] request: PasskeyRegistrationCompleteRequest,
|
#[body] request: PasskeyRegistrationCompleteRequest,
|
||||||
) -> Result<server_api_responses::AuthPasskeyRegistrationComplete, RepositoryApiError>;
|
) -> Result<server_api_responses::AuthPasskeyRegistrationComplete, RepositoryApiError>;
|
||||||
|
|
||||||
|
#[get(
|
||||||
|
"/api/auth/passkeys",
|
||||||
|
status = 200,
|
||||||
|
error_status = 400,
|
||||||
|
additional_error_statuses = [403, 500],
|
||||||
|
bearer_auth = true,
|
||||||
|
browser_auth = true
|
||||||
|
)]
|
||||||
|
async fn auth_passkey_list(
|
||||||
|
&self,
|
||||||
|
#[extension] context: ServerRequestContext,
|
||||||
|
) -> Result<PasskeyListResponse, RepositoryApiError>;
|
||||||
|
|
||||||
|
#[post(
|
||||||
|
"/api/auth/passkeys/addition/result",
|
||||||
|
status = 200,
|
||||||
|
error_status = 400,
|
||||||
|
additional_error_statuses = [403, 500],
|
||||||
|
bearer_auth = true,
|
||||||
|
browser_auth = true
|
||||||
|
)]
|
||||||
|
async fn auth_passkey_addition_result(
|
||||||
|
&self,
|
||||||
|
#[extension] context: ServerRequestContext,
|
||||||
|
#[body] request: PasskeyAdditionResultRequest,
|
||||||
|
) -> Result<PasskeyAdditionResultResponse, RepositoryApiError>;
|
||||||
|
|
||||||
#[post(
|
#[post(
|
||||||
"/api/auth/passkeys/addition/options",
|
"/api/auth/passkeys/addition/options",
|
||||||
status = 200,
|
status = 200,
|
||||||
@@ -4119,6 +4151,8 @@ impl RequestActor {
|
|||||||
pub struct WhoamiResponse {
|
pub struct WhoamiResponse {
|
||||||
pub actor: Option<RequestActor>,
|
pub actor: Option<RequestActor>,
|
||||||
pub session: BrowserSessionState,
|
pub session: BrowserSessionState,
|
||||||
|
/// Non-secret comparison evidence, never an authentication credential.
|
||||||
|
pub authentication_reference: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Browser navigation evidence, not an authorization grant or a generic 401 classification.
|
/// Browser navigation evidence, not an authorization grant or a generic 401 classification.
|
||||||
@@ -4146,6 +4180,8 @@ pub struct AuthBootstrapUserRequest {
|
|||||||
#[serde(deny_unknown_fields)]
|
#[serde(deny_unknown_fields)]
|
||||||
pub struct AuthUserResponse {
|
pub struct AuthUserResponse {
|
||||||
pub user: AuthenticatedUser,
|
pub user: AuthenticatedUser,
|
||||||
|
/// Comparison evidence for the exact newly issued session, not a credential.
|
||||||
|
pub authentication_reference: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Starts new-account registration only; an existing handle is never interpreted as key addition.
|
/// Starts new-account registration only; an existing handle is never interpreted as key addition.
|
||||||
@@ -4162,16 +4198,64 @@ pub struct PasskeyRegistrationOptionsRequest {
|
|||||||
pub browser_origin: Option<String>,
|
pub browser_origin: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Adds a credential to the current authenticated actor; target identity is never public input.
|
/// Adds to the current actor. Expected identity/binding fields only fence stale UI;
|
||||||
|
/// they never select an owner or authorize an addition.
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
#[serde(deny_unknown_fields)]
|
#[serde(deny_unknown_fields)]
|
||||||
pub struct PasskeyAdditionOptionsRequest {
|
pub struct PasskeyAdditionOptionsRequest {
|
||||||
|
pub expected_user_id: String,
|
||||||
|
pub expected_account_id: String,
|
||||||
|
pub expected_authentication_reference: String,
|
||||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||||
#[cfg_attr(feature = "typescript", ts(optional = nullable))]
|
#[cfg_attr(feature = "typescript", ts(optional = nullable))]
|
||||||
pub browser_origin: Option<String>,
|
pub browser_origin: Option<String>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Safe account metadata. Neither the WebAuthn credential ID nor key material is exposed.
|
||||||
|
/// Normal-login verification is observed verifier success, not inferred discoverability.
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct PasskeySummary {
|
||||||
|
pub key_reference: String,
|
||||||
|
pub created_at: String,
|
||||||
|
pub last_used_at: Option<String>,
|
||||||
|
pub normal_login_verified_at: Option<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct PasskeyListResponse {
|
||||||
|
pub user: AuthenticatedUser,
|
||||||
|
pub keys: Vec<PasskeySummary>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct PasskeyAdditionResultRequest {
|
||||||
|
pub challenge_id: String,
|
||||||
|
pub expected_user_id: String,
|
||||||
|
pub expected_account_id: String,
|
||||||
|
/// Original options authentication reference; the current same-owner session may differ.
|
||||||
|
pub expected_authentication_reference: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Unknown is not proof that registration failed: completion may still be in flight.
|
||||||
|
/// Saved proves the credential commit, not session issuance or current session health.
|
||||||
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
|
#[serde(tag = "status", rename_all = "snake_case", deny_unknown_fields)]
|
||||||
|
pub enum PasskeyAdditionResultResponse {
|
||||||
|
Unknown {},
|
||||||
|
Saved {
|
||||||
|
user: AuthenticatedUser,
|
||||||
|
key: PasskeySummary,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize, JsonSchema)]
|
#[derive(Debug, Serialize, Deserialize, JsonSchema)]
|
||||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
#[serde(deny_unknown_fields)]
|
#[serde(deny_unknown_fields)]
|
||||||
@@ -6980,6 +7064,8 @@ pub struct WorkingDirectoryCleanupTarget {
|
|||||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
#[serde(deny_unknown_fields)]
|
#[serde(deny_unknown_fields)]
|
||||||
pub struct WorkingDirectoryRemovalRequest {
|
pub struct WorkingDirectoryRemovalRequest {
|
||||||
|
/// Stable for an exact resend; a new explicit request uses a new identity.
|
||||||
|
pub operation_id: String,
|
||||||
pub reason: String,
|
pub reason: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -7518,6 +7604,9 @@ pub struct CurrentWorkerWorkdirAttachmentResponse {
|
|||||||
#[serde(skip_serializing_if = "Option::is_none")]
|
#[serde(skip_serializing_if = "Option::is_none")]
|
||||||
pub environment: Option<workdir::WorkdirSessionEnvironment>,
|
pub environment: Option<workdir::WorkdirSessionEnvironment>,
|
||||||
pub attached: bool,
|
pub attached: bool,
|
||||||
|
/// The Workdir was already attached to this Worker; alias is its existing name.
|
||||||
|
#[serde(default)]
|
||||||
|
pub already_attached: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
@@ -10579,8 +10668,8 @@ pub enum WorkerOperationState {
|
|||||||
/// Cross-layer outcome of an explicit Worker restore operation.
|
/// Cross-layer outcome of an explicit Worker restore operation.
|
||||||
///
|
///
|
||||||
/// `Rejected` is reserved for read-only preflight failures. Once live restore
|
/// `Rejected` is reserved for read-only preflight failures. Once live restore
|
||||||
/// work starts, failure is either a confirmed `RolledBack` operation or a
|
/// work starts, distinguish confirmed `RolledBack`, unresolved
|
||||||
/// `ReconciliationRequired` result whose commit state must be reread/retried.
|
/// `ReconciliationRequired`, and a later Stop's `Superseded` outcome.
|
||||||
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||||
#[serde(rename_all = "snake_case")]
|
#[serde(rename_all = "snake_case")]
|
||||||
@@ -10588,6 +10677,8 @@ pub enum WorkerRestoreState {
|
|||||||
Accepted,
|
Accepted,
|
||||||
Rejected,
|
Rejected,
|
||||||
RolledBack,
|
RolledBack,
|
||||||
|
/// A later Stop superseded this restore; neither rollback nor prior Accepted is asserted.
|
||||||
|
Superseded,
|
||||||
ReconciliationRequired,
|
ReconciliationRequired,
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -11651,6 +11742,10 @@ pub fn auth_api_typescript() -> String {
|
|||||||
AuthBootstrapUserRequest::decl(&config),
|
AuthBootstrapUserRequest::decl(&config),
|
||||||
AuthUserResponse::decl(&config),
|
AuthUserResponse::decl(&config),
|
||||||
PasskeyAdditionOptionsRequest::decl(&config),
|
PasskeyAdditionOptionsRequest::decl(&config),
|
||||||
|
PasskeySummary::decl(&config),
|
||||||
|
PasskeyListResponse::decl(&config),
|
||||||
|
PasskeyAdditionResultRequest::decl(&config),
|
||||||
|
PasskeyAdditionResultResponse::decl(&config),
|
||||||
PasskeyRegistrationOptionsRequest::decl(&config),
|
PasskeyRegistrationOptionsRequest::decl(&config),
|
||||||
PasskeyRegistrationOptionsResponse::decl(&config),
|
PasskeyRegistrationOptionsResponse::decl(&config),
|
||||||
PasskeyRegistrationCompleteRequest::decl(&config),
|
PasskeyRegistrationCompleteRequest::decl(&config),
|
||||||
@@ -12681,6 +12776,7 @@ mod tests {
|
|||||||
(WorkerRestoreState::Accepted, "accepted"),
|
(WorkerRestoreState::Accepted, "accepted"),
|
||||||
(WorkerRestoreState::Rejected, "rejected"),
|
(WorkerRestoreState::Rejected, "rejected"),
|
||||||
(WorkerRestoreState::RolledBack, "rolled_back"),
|
(WorkerRestoreState::RolledBack, "rolled_back"),
|
||||||
|
(WorkerRestoreState::Superseded, "superseded"),
|
||||||
(
|
(
|
||||||
WorkerRestoreState::ReconciliationRequired,
|
WorkerRestoreState::ReconciliationRequired,
|
||||||
"reconciliation_required",
|
"reconciliation_required",
|
||||||
@@ -13291,9 +13387,11 @@ mod tests {
|
|||||||
capabilities: workdir::WorkdirSessionCapabilities::EMPTY,
|
capabilities: workdir::WorkdirSessionCapabilities::EMPTY,
|
||||||
environment: None,
|
environment: None,
|
||||||
attached: false,
|
attached: false,
|
||||||
|
already_attached: false,
|
||||||
};
|
};
|
||||||
let value = serde_json::to_value(&response).unwrap();
|
let value = serde_json::to_value(&response).unwrap();
|
||||||
assert_eq!(value.as_object().unwrap().len(), 5);
|
assert_eq!(value.as_object().unwrap().len(), 6);
|
||||||
|
assert_eq!(value["already_attached"], false);
|
||||||
assert!(value.get("connection_id").is_none());
|
assert!(value.get("connection_id").is_none());
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
serde_json::from_value::<CurrentWorkerWorkdirAttachmentResponse>(value).unwrap(),
|
serde_json::from_value::<CurrentWorkerWorkdirAttachmentResponse>(value).unwrap(),
|
||||||
@@ -14079,7 +14177,8 @@ mod tests {
|
|||||||
"display_name": "Hare",
|
"display_name": "Hare",
|
||||||
"auth_method": "browser_session"
|
"auth_method": "browser_session"
|
||||||
},
|
},
|
||||||
"session": { "kind": "authenticated", "expires_at": "2026-10-20T00:00:00Z" }
|
"session": { "kind": "authenticated", "expires_at": "2026-10-20T00:00:00Z" },
|
||||||
|
"authentication_reference": "synthetic-session-reference"
|
||||||
});
|
});
|
||||||
let decoded = serde_json::from_value::<WhoamiResponse>(whoami.clone())
|
let decoded = serde_json::from_value::<WhoamiResponse>(whoami.clone())
|
||||||
.expect("server whoami fixture should match shared DTO");
|
.expect("server whoami fixture should match shared DTO");
|
||||||
@@ -14101,7 +14200,8 @@ mod tests {
|
|||||||
"account_id": "account-1",
|
"account_id": "account-1",
|
||||||
"handle": "hare",
|
"handle": "hare",
|
||||||
"display_name": "Hare"
|
"display_name": "Hare"
|
||||||
}
|
},
|
||||||
|
"authentication_reference": "synthetic-session-reference"
|
||||||
});
|
});
|
||||||
let decoded = serde_json::from_value::<AuthUserResponse>(auth_user.clone())
|
let decoded = serde_json::from_value::<AuthUserResponse>(auth_user.clone())
|
||||||
.expect("server auth-user fixture should match shared DTO");
|
.expect("server auth-user fixture should match shared DTO");
|
||||||
@@ -14274,7 +14374,7 @@ mod tests {
|
|||||||
serde_json::json!({"kind":"expired"}),
|
serde_json::json!({"kind":"expired"}),
|
||||||
serde_json::json!({"kind":"authenticated", "expires_at":null}),
|
serde_json::json!({"kind":"authenticated", "expires_at":null}),
|
||||||
] {
|
] {
|
||||||
let value = serde_json::json!({"actor":null, "session":session});
|
let value = serde_json::json!({"actor":null, "session":session, "authentication_reference": null});
|
||||||
let response: WhoamiResponse = serde_json::from_value(value.clone()).unwrap();
|
let response: WhoamiResponse = serde_json::from_value(value.clone()).unwrap();
|
||||||
assert_eq!(serde_json::to_value(response).unwrap(), value);
|
assert_eq!(serde_json::to_value(response).unwrap(), value);
|
||||||
}
|
}
|
||||||
@@ -14812,10 +14912,28 @@ mod openapi_artifact_tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
assert!(
|
assert!(
|
||||||
serde_json::from_value::<PasskeyAdditionOptionsRequest>(serde_json::json!({})).is_ok()
|
serde_json::from_value::<PasskeyAdditionOptionsRequest>(serde_json::json!({})).is_err()
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn passkey_addition_requires_expected_owner_and_binding() {
|
||||||
|
let expected = serde_json::json!({
|
||||||
|
"expected_user_id": "user-a", "expected_account_id": "account-a",
|
||||||
|
"expected_authentication_reference": "reference"
|
||||||
|
});
|
||||||
|
assert!(serde_json::from_value::<PasskeyAdditionOptionsRequest>(expected.clone()).is_ok());
|
||||||
|
for field in [
|
||||||
|
"expected_user_id",
|
||||||
|
"expected_account_id",
|
||||||
|
"expected_authentication_reference",
|
||||||
|
] {
|
||||||
|
let mut missing = expected.clone();
|
||||||
|
missing.as_object_mut().unwrap().remove(field);
|
||||||
|
assert!(serde_json::from_value::<PasskeyAdditionOptionsRequest>(missing).is_err());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn auth_workspace_catalog_and_typed_header_contract_is_strict() {
|
fn auth_workspace_catalog_and_typed_header_contract_is_strict() {
|
||||||
let document = canonical_openapi_document().expect("canonical OpenAPI contract must build");
|
let document = canonical_openapi_document().expect("canonical OpenAPI contract must build");
|
||||||
|
|||||||
@@ -194,6 +194,21 @@ impl WorkerSessionStore {
|
|||||||
Ok(store)
|
Ok(store)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Open a current retained Session without creating or migrating anything.
|
||||||
|
/// The live constraint is checked here and again by every later publication;
|
||||||
|
/// attaching the control store itself does not acquire publication authority.
|
||||||
|
pub fn open_existing_with_retention(
|
||||||
|
root: impl Into<PathBuf>,
|
||||||
|
retention: crate::retention::SessionRetentionStore,
|
||||||
|
) -> Result<Self, StoreError> {
|
||||||
|
let mut store = Self::open_read_only(root)?;
|
||||||
|
if let Some(session) = *store.session_id.lock().unwrap() {
|
||||||
|
retention.ensure_live(session)?;
|
||||||
|
}
|
||||||
|
store.retention = Some(retention);
|
||||||
|
Ok(store)
|
||||||
|
}
|
||||||
|
|
||||||
fn lock_publication(
|
fn lock_publication(
|
||||||
&self,
|
&self,
|
||||||
session: SessionId,
|
session: SessionId,
|
||||||
|
|||||||
@@ -194,6 +194,29 @@ impl SubjektivStore {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// End a body after the Host has confirmed its Runtime removal. This is not a
|
||||||
|
/// completed Journal handoff: source coverage, positions and saved receipts stay
|
||||||
|
/// unchanged. Revocation and closing generations are atomic, so no retained Job
|
||||||
|
/// can write after deletion even while Backend Job cleanup is being retried.
|
||||||
|
pub fn release_removed_journal_body(&self, host: &JournalHostContext) -> JournalResult<()> {
|
||||||
|
self.database.try_transaction(|tx| {
|
||||||
|
super::store::check_session_host(self, tx, host)?;
|
||||||
|
let attr = host.attribution.as_ref().expect("checked Session authority");
|
||||||
|
let binding: Option<String> = tx.query_row(
|
||||||
|
"SELECT binding_id FROM journal_writer_bindings WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3 ORDER BY rowid DESC LIMIT 1",
|
||||||
|
params![host.subject_id, attr.runtime_id, attr.worker_id], |r| r.get(0),
|
||||||
|
).optional()?;
|
||||||
|
let binding = binding.unwrap_or_else(|| crate::issued_id("journal-removed-body"));
|
||||||
|
tx.execute("INSERT OR IGNORE INTO journal_writer_bindings(binding_id,subject_id,runtime_id,worker_id,active) VALUES(?1,?2,?3,?4,0)",params![binding,host.subject_id,attr.runtime_id,attr.worker_id])?;
|
||||||
|
tx.execute("UPDATE journal_writer_bindings SET active=0 WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3",params![host.subject_id,attr.runtime_id,attr.worker_id])?;
|
||||||
|
// Bind legacy attributed Sessions to an inactive lifetime, never to the
|
||||||
|
// successor. Retained unfinished history is not an active writer.
|
||||||
|
tx.execute("INSERT OR IGNORE INTO journal_session_bindings(session_id,binding_id) SELECT session_id,?4 FROM subject_session_attributions WHERE subject_id=?1 AND runtime_id=?2 AND worker_id=?3",params![host.subject_id,attr.runtime_id,attr.worker_id,binding])?;
|
||||||
|
tx.execute("UPDATE journal_generations SET active=0 WHERE journal_id IN (SELECT j.id FROM journals j JOIN subject_session_attributions a ON a.session_id=j.source_session_id WHERE a.subject_id=?1 AND a.runtime_id=?2 AND a.worker_id=?3)",params![host.subject_id,attr.runtime_id,attr.worker_id])?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
/// The Host must hold its lifecycle fence, confirm Idle, reconcile all committed original
|
/// The Host must hold its lifecycle fence, confirm Idle, reconcile all committed original
|
||||||
/// Sessions (including the save/notification gap), and exclude queued/running/Unknown Jobs.
|
/// Sessions (including the save/notification gap), and exclude queued/running/Unknown Jobs.
|
||||||
/// This transaction rejects retained writers, pending source bytes and active generations.
|
/// This transaction rejects retained writers, pending source bytes and active generations.
|
||||||
|
|||||||
@@ -651,6 +651,32 @@ impl SubjektivStore {
|
|||||||
Ok(())
|
Ok(())
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
/// Read the exact retained Job/attempt snapshot without granting writer
|
||||||
|
/// authority or creating a generation. Used for outcome recovery after removal.
|
||||||
|
pub fn retained_journal_generation(
|
||||||
|
&self,
|
||||||
|
host: &JournalHostContext,
|
||||||
|
id: &str,
|
||||||
|
binding: &JobAttemptBinding,
|
||||||
|
) -> JournalResult<Option<JournalGeneration>> {
|
||||||
|
self.database.try_with_connection(|tx| {
|
||||||
|
check_host(self, tx, host, false)?;
|
||||||
|
check_session_host(self, tx, host)?;
|
||||||
|
journal(tx, host, id, true)?;
|
||||||
|
let raw: Option<String> = tx.query_row(
|
||||||
|
"SELECT snapshot_json FROM journal_generations WHERE journal_id=?1 AND job_id=?2 AND attempt_id=?3",
|
||||||
|
params![id, binding.job_id, binding.attempt_id], |r| r.get(0),
|
||||||
|
).optional()?;
|
||||||
|
raw.map(|raw| {
|
||||||
|
let generation: JournalGeneration = serde_json::from_str(&raw)?;
|
||||||
|
if generation.journal_id != id || &generation.job != binding {
|
||||||
|
return Err(conflict("retained generation identity mismatch"));
|
||||||
|
}
|
||||||
|
Ok(generation)
|
||||||
|
}).transpose()
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
pub fn read_journal_generation(
|
pub fn read_journal_generation(
|
||||||
&self,
|
&self,
|
||||||
host: &JournalHostContext,
|
host: &JournalHostContext,
|
||||||
|
|||||||
@@ -785,6 +785,12 @@ mod tests {
|
|||||||
#[derive(Default)]
|
#[derive(Default)]
|
||||||
struct PendingRuntimeBackend {
|
struct PendingRuntimeBackend {
|
||||||
operations: std::sync::Mutex<Vec<worker_runtime::execution::WorkerLifecycleOperationId>>,
|
operations: std::sync::Mutex<Vec<worker_runtime::execution::WorkerLifecycleOperationId>>,
|
||||||
|
owners: std::sync::Mutex<
|
||||||
|
std::collections::HashMap<
|
||||||
|
worker_runtime::execution::WorkerLifecycleOperationId,
|
||||||
|
worker_runtime::identity::WorkerRef,
|
||||||
|
>,
|
||||||
|
>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl worker_runtime::execution::WorkerExecutionBackend for PendingRuntimeBackend {
|
impl worker_runtime::execution::WorkerExecutionBackend for PendingRuntimeBackend {
|
||||||
@@ -806,6 +812,21 @@ mod tests {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn restore_recovery_disposition(
|
||||||
|
&self,
|
||||||
|
operation_id: worker_runtime::execution::WorkerLifecycleOperationId,
|
||||||
|
worker_ref: &worker_runtime::identity::WorkerRef,
|
||||||
|
) -> Option<worker_runtime::execution::RestoreRecoveryDisposition> {
|
||||||
|
use worker_runtime::execution::RestoreRecoveryDisposition as D;
|
||||||
|
Some(
|
||||||
|
if self.owners.lock().unwrap().get(&operation_id) == Some(worker_ref) {
|
||||||
|
D::Continue
|
||||||
|
} else {
|
||||||
|
D::Absent
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
fn restore_worker(
|
fn restore_worker(
|
||||||
&self,
|
&self,
|
||||||
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||||
@@ -813,6 +834,10 @@ mod tests {
|
|||||||
use worker_runtime::execution::*;
|
use worker_runtime::execution::*;
|
||||||
let mut operations = self.operations.lock().unwrap();
|
let mut operations = self.operations.lock().unwrap();
|
||||||
operations.push(request.operation_id);
|
operations.push(request.operation_id);
|
||||||
|
self.owners
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.insert(request.operation_id, request.worker_ref.clone());
|
||||||
if operations.len() == 1 {
|
if operations.len() == 1 {
|
||||||
WorkerExecutionSpawnResult::ReconciliationRequired {
|
WorkerExecutionSpawnResult::ReconciliationRequired {
|
||||||
result: WorkerExecutionResult::errored(
|
result: WorkerExecutionResult::errored(
|
||||||
@@ -1119,39 +1144,45 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn rejected_resume_does_not_observe_or_open_retained_state() {
|
async fn settled_nonaccepted_resume_does_not_retry_observe_or_open_retained_state() {
|
||||||
let lifecycle = FakeBackendWorkerLifecycle {
|
for state in [
|
||||||
restore_response: restore_response(BackendWorkerRestoreState::Rejected),
|
BackendWorkerRestoreState::Rejected,
|
||||||
observation: session_observation(serde_json::json!({
|
BackendWorkerRestoreState::RolledBack,
|
||||||
"availability": "unavailable",
|
BackendWorkerRestoreState::Superseded,
|
||||||
"reason": "retention_missing",
|
] {
|
||||||
"message": "must not be opened"
|
let lifecycle = FakeBackendWorkerLifecycle {
|
||||||
})),
|
restore_response: restore_response(state),
|
||||||
calls: std::sync::Mutex::new(Vec::new()),
|
observation: session_observation(serde_json::json!({
|
||||||
};
|
"availability": "unavailable",
|
||||||
let target = client::BackendRuntimeTarget::new(
|
"reason": "retention_missing",
|
||||||
"http://127.0.0.1:3000",
|
"message": "must not be opened"
|
||||||
"workspace-a",
|
})),
|
||||||
"runtime-a",
|
calls: std::sync::Mutex::new(Vec::new()),
|
||||||
"worker-a",
|
};
|
||||||
);
|
let target = client::BackendRuntimeTarget::new(
|
||||||
|
"http://127.0.0.1:3000",
|
||||||
|
"workspace-a",
|
||||||
|
"runtime-a",
|
||||||
|
"worker-a",
|
||||||
|
);
|
||||||
|
|
||||||
let error = prepare_selected_worker(
|
let error = prepare_selected_worker(
|
||||||
&lifecycle,
|
&lifecycle,
|
||||||
target,
|
target,
|
||||||
BackendWorkerPickerIntent::Resume,
|
BackendWorkerPickerIntent::Resume,
|
||||||
Some(client::BackendWorkerRestoreRequest {
|
Some(client::BackendWorkerRestoreRequest {
|
||||||
expected_observation_token: "observed-worker-state".into(),
|
expected_observation_token: "observed-worker-state".into(),
|
||||||
request_id: "resume-operation".into(),
|
request_id: "resume-operation".into(),
|
||||||
}),
|
}),
|
||||||
|_| Ok(false),
|
|_| panic!("settled nonaccepted restore must not offer recovery retry"),
|
||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.unwrap_err()
|
.unwrap_err()
|
||||||
.to_string();
|
.to_string();
|
||||||
|
|
||||||
assert!(error.contains("Rejected"));
|
assert!(error.contains(&format!("{state:?}")));
|
||||||
assert_eq!(*lifecycle.calls.lock().unwrap(), ["restore"]);
|
assert_eq!(*lifecycle.calls.lock().unwrap(), ["restore"]);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -1176,6 +1207,7 @@ mod tests {
|
|||||||
for state in [
|
for state in [
|
||||||
BackendWorkerRestoreState::Rejected,
|
BackendWorkerRestoreState::Rejected,
|
||||||
BackendWorkerRestoreState::RolledBack,
|
BackendWorkerRestoreState::RolledBack,
|
||||||
|
BackendWorkerRestoreState::Superseded,
|
||||||
BackendWorkerRestoreState::ReconciliationRequired,
|
BackendWorkerRestoreState::ReconciliationRequired,
|
||||||
] {
|
] {
|
||||||
let mut response = restore_response(state);
|
let mut response = restore_response(state);
|
||||||
|
|||||||
@@ -19,15 +19,19 @@ required-features = ["ws-server", "fs-store"]
|
|||||||
[features]
|
[features]
|
||||||
default = ["ws-server", "fs-store"]
|
default = ["ws-server", "fs-store"]
|
||||||
fs-store = ["dep:fs4"]
|
fs-store = ["dep:fs4"]
|
||||||
|
# Offline fixture tooling only; never enabled by the product defaults.
|
||||||
|
test-support = ["fs-store", "ws-server"]
|
||||||
|
# Explicit process-test synchronization; independent of offline fixture tooling.
|
||||||
|
restore-checkpoints = ["fs-store"]
|
||||||
http-server = ["dep:axum", "dep:tower", "dep:reqwest"]
|
http-server = ["dep:axum", "dep:tower", "dep:reqwest"]
|
||||||
ws-server = ["http-server", "axum/ws", "dep:futures", "tokio/sync"]
|
ws-server = ["http-server", "axum/ws", "tokio/sync"]
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
agen.workspace = true
|
agen.workspace = true
|
||||||
async-trait.workspace = true
|
async-trait.workspace = true
|
||||||
base64.workspace = true
|
base64.workspace = true
|
||||||
axum = { workspace = true, optional = true }
|
axum = { workspace = true, optional = true }
|
||||||
futures = { workspace = true, optional = true }
|
futures.workspace = true
|
||||||
decodal.workspace = true
|
decodal.workspace = true
|
||||||
flow = { path = "../flow" }
|
flow = { path = "../flow" }
|
||||||
fs4 = { workspace = true, optional = true }
|
fs4 = { workspace = true, optional = true }
|
||||||
|
|||||||
@@ -8,3 +8,61 @@ Manages Worker creation, stopping, and Restore, execution connections within the
|
|||||||
- [`execution`] — execution backends and operation-specific results.
|
- [`execution`] — execution backends and operation-specific results.
|
||||||
- [`working_directory`] — repository materialization, session resources, and safe deletion.
|
- [`working_directory`] — repository materialization, session resources, and safe deletion.
|
||||||
- [`workspace_issuer`] — Workspace signature verification and replay prevention.
|
- [`workspace_issuer`] — Workspace signature verification and replay prevention.
|
||||||
|
|
||||||
|
## Offline stopped-Worker fixtures
|
||||||
|
|
||||||
|
The nondefault `test-support` feature exposes `test_support::StoppedWorkerTemplate`.
|
||||||
|
It is intentionally limited to current-schema, stopped, attachment-free artificial
|
||||||
|
conversations with one Segment, text history, and no pending execution/activation.
|
||||||
|
It does not add a product endpoint or change create/restore authorization.
|
||||||
|
|
||||||
|
1. With the source Runtime offline, call `StoppedWorkerTemplate::extract(worker_dir)`.
|
||||||
|
The source must have its existing Runtime owner lock and identity. Extraction is
|
||||||
|
read-only; the returned serde template excludes source auth, saved Manifest,
|
||||||
|
request/config references, run directories, and Runtime operation history.
|
||||||
|
2. Initialize **new** Runtime identity and Workspace trust in a fresh destination
|
||||||
|
using normal setup. Preserve the template's Runtime and Workspace IDs.
|
||||||
|
3. Call `template.install(&options, request, &resolved_manifest)` before starting
|
||||||
|
the Runtime. Supply a fresh typed `CreateWorkerRequest` (including config/profile
|
||||||
|
references and trusted memory settings) and `manifest::WorkerManifest` resolved
|
||||||
|
for that environment. Its Worker name must equal `template.worker_name()`.
|
||||||
|
4. Start normally and use the ordinary explicit restore operation. Installation
|
||||||
|
itself creates no execution and leaves the Worker stopped with a fresh guard.
|
||||||
|
|
||||||
|
The template preserves the original Session manifest/JSONL bytes, including stable
|
||||||
|
entry IDs, timestamps, and settled submission receipts. It rejects resumable queues
|
||||||
|
and Feature invocation history. A saved system prompt is required for restore;
|
||||||
|
checkpoint receipts must match the Worker's typed receipt schema. Preserved records
|
||||||
|
must use canonical writer field shapes: unknown/duplicate fields (including nested
|
||||||
|
fields), discarded optional fields, and opaque Feature history payloads are rejected,
|
||||||
|
not stripped or rewritten. Portability/secret-marker checks do not replace
|
||||||
|
review of the supplied artificial conversation or its independently verified hashes.
|
||||||
|
The installer refuses any destination content other than initialized `auth.toml`;
|
||||||
|
interrupted publication is fail-closed, not an overwrite/recovery facility.
|
||||||
|
|
||||||
|
## Owned-process Restore checkpoints
|
||||||
|
|
||||||
|
The independent, nondefault `restore-checkpoints` feature exists only for explicit
|
||||||
|
process tests in the shared `tests/e2e` harness. Offline `test-support` does not enable
|
||||||
|
it. Normal binaries reject `--test-restore-checkpoint`; no environment variable or
|
||||||
|
HTTP endpoint activates this facility. Do not deploy a checkpoint-enabled binary
|
||||||
|
as a replacement for the default product.
|
||||||
|
|
||||||
|
A test-enabled Runtime accepts `--test-restore-checkpoint <config.json>` with a
|
||||||
|
bounded regular JSON file selecting a loopback socket, Worker ID, one stage and a
|
||||||
|
diagnostic nonce. The nonce is only a handshake label, never execution identity,
|
||||||
|
request authority or routing state. The one-shot stages are real prepared candidate
|
||||||
|
with `NotCommitted`, and durable `Committed` before activation/terminal receipt.
|
||||||
|
Before acknowledging, the existing bounded canonical reader validates the exact
|
||||||
|
Worker/operation/decision, syncs its file/directories and requires no saved receipt.
|
||||||
|
No pending JSON is fabricated and no lifecycle work is skipped.
|
||||||
|
|
||||||
|
The Runtime sends bounded newline-delimited JSON evidence, then waits at most eight
|
||||||
|
seconds for byte `R`. EOF, invalid reply, unavailable peer or deadline releases the
|
||||||
|
wait into the original operation. The Runtime-state mutex is not held across that
|
||||||
|
wait; existing per-Worker ownership still covers the actual candidate and cleanup.
|
||||||
|
The harness verifies the ack, kills/reaps only its owned child and inspects the
|
||||||
|
canonical aggregate offline before restarting the same roots/identity without the
|
||||||
|
checkpoint flag or another fixture install. These tests are a separate cfg/binary
|
||||||
|
lane from featureless product regression; they do not certify arbitrary OS failure,
|
||||||
|
external power loss or a real model call.
|
||||||
|
|||||||
@@ -215,6 +215,38 @@ impl WorkerExecutionResult {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Private one-shot synchronization at real publication boundaries. Disconnect
|
||||||
|
// releases the waiter during unwinding; timeout bounds a forgotten test owner.
|
||||||
|
#[cfg(test)]
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub(crate) struct RestoreTestGate<T> {
|
||||||
|
pub(crate) entered: std::sync::mpsc::Sender<T>,
|
||||||
|
pub(crate) release: std::sync::mpsc::Receiver<()>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
impl<T> RestoreTestGate<T> {
|
||||||
|
pub(crate) fn wait(self, evidence: T) {
|
||||||
|
let _ = self.entered.send(evidence);
|
||||||
|
assert!(
|
||||||
|
!matches!(
|
||||||
|
self.release
|
||||||
|
.recv_timeout(std::time::Duration::from_secs(30)),
|
||||||
|
Err(std::sync::mpsc::RecvTimeoutError::Timeout)
|
||||||
|
),
|
||||||
|
"publication test gate timed out"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(all(test, feature = "ws-server"))]
|
||||||
|
#[derive(Debug)]
|
||||||
|
pub(crate) struct RestoreFlushFault {
|
||||||
|
pub(crate) remaining_successes: usize,
|
||||||
|
pub(crate) gate: RestoreTestGate<protocol::Event>,
|
||||||
|
pub(crate) restored: std::sync::mpsc::Sender<Vec<protocol::Event>>,
|
||||||
|
}
|
||||||
|
|
||||||
/// Runtime hooks available to an execution backend for one Worker.
|
/// Runtime hooks available to an execution backend for one Worker.
|
||||||
#[cfg(feature = "ws-server")]
|
#[cfg(feature = "ws-server")]
|
||||||
type WorkerObservationPublisher = Arc<
|
type WorkerObservationPublisher = Arc<
|
||||||
@@ -242,10 +274,15 @@ enum WorkerExecutionPublicationState {
|
|||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct WorkerExecutionContext {
|
pub struct WorkerExecutionContext {
|
||||||
worker_ref: WorkerRef,
|
worker_ref: WorkerRef,
|
||||||
|
restore_progress: Option<Arc<dyn Fn(WorkerLifecycleOperationId) + Send + Sync>>,
|
||||||
|
restore_repair_authorizer:
|
||||||
|
Option<Arc<dyn Fn(WorkerLifecycleOperationId) -> Result<(), RuntimeError> + Send + Sync>>,
|
||||||
#[cfg(feature = "ws-server")]
|
#[cfg(feature = "ws-server")]
|
||||||
observation_publisher: WorkerObservationPublisher,
|
observation_publisher: WorkerObservationPublisher,
|
||||||
#[cfg(feature = "ws-server")]
|
#[cfg(feature = "ws-server")]
|
||||||
publication_state: Arc<Mutex<WorkerExecutionPublicationState>>,
|
publication_state: Arc<Mutex<WorkerExecutionPublicationState>>,
|
||||||
|
#[cfg(all(test, feature = "ws-server"))]
|
||||||
|
flush_fault: Arc<Mutex<Option<RestoreFlushFault>>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
impl WorkerExecutionContext {
|
impl WorkerExecutionContext {
|
||||||
@@ -256,7 +293,11 @@ impl WorkerExecutionContext {
|
|||||||
) -> Self {
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
worker_ref,
|
worker_ref,
|
||||||
|
restore_progress: None,
|
||||||
|
restore_repair_authorizer: None,
|
||||||
observation_publisher,
|
observation_publisher,
|
||||||
|
#[cfg(test)]
|
||||||
|
flush_fault: Arc::default(),
|
||||||
publication_state: Arc::new(Mutex::new(WorkerExecutionPublicationState::Active)),
|
publication_state: Arc::new(Mutex::new(WorkerExecutionPublicationState::Active)),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -268,7 +309,11 @@ impl WorkerExecutionContext {
|
|||||||
) -> Self {
|
) -> Self {
|
||||||
Self {
|
Self {
|
||||||
worker_ref,
|
worker_ref,
|
||||||
|
restore_progress: None,
|
||||||
|
restore_repair_authorizer: None,
|
||||||
observation_publisher,
|
observation_publisher,
|
||||||
|
#[cfg(test)]
|
||||||
|
flush_fault: Arc::default(),
|
||||||
publication_state: Arc::new(Mutex::new(WorkerExecutionPublicationState::Pending(
|
publication_state: Arc::new(Mutex::new(WorkerExecutionPublicationState::Pending(
|
||||||
VecDeque::new(),
|
VecDeque::new(),
|
||||||
))),
|
))),
|
||||||
@@ -277,12 +322,56 @@ impl WorkerExecutionContext {
|
|||||||
|
|
||||||
#[cfg(not(feature = "ws-server"))]
|
#[cfg(not(feature = "ws-server"))]
|
||||||
pub(crate) fn new(worker_ref: WorkerRef) -> Self {
|
pub(crate) fn new(worker_ref: WorkerRef) -> Self {
|
||||||
Self { worker_ref }
|
Self {
|
||||||
|
worker_ref,
|
||||||
|
restore_progress: None,
|
||||||
|
restore_repair_authorizer: None,
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(not(feature = "ws-server"))]
|
#[cfg(not(feature = "ws-server"))]
|
||||||
pub(crate) fn candidate(worker_ref: WorkerRef) -> Self {
|
pub(crate) fn candidate(worker_ref: WorkerRef) -> Self {
|
||||||
Self { worker_ref }
|
Self {
|
||||||
|
worker_ref,
|
||||||
|
restore_progress: None,
|
||||||
|
restore_repair_authorizer: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn with_restore_repair_authorizer(
|
||||||
|
mut self,
|
||||||
|
authorizer: Arc<
|
||||||
|
dyn Fn(WorkerLifecycleOperationId) -> Result<(), RuntimeError> + Send + Sync,
|
||||||
|
>,
|
||||||
|
) -> Self {
|
||||||
|
self.restore_repair_authorizer = Some(authorizer);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn authorize_restore_source_repair(
|
||||||
|
&self,
|
||||||
|
operation: WorkerLifecycleOperationId,
|
||||||
|
) -> Result<(), RuntimeError> {
|
||||||
|
match &self.restore_repair_authorizer {
|
||||||
|
Some(authorizer) => authorizer(operation),
|
||||||
|
// Direct factory/component users own their source repair separately;
|
||||||
|
// only Runtime contexts carry a Runtime aggregate authority callback.
|
||||||
|
None => Ok(()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn with_restore_progress(
|
||||||
|
mut self,
|
||||||
|
progress: Arc<dyn Fn(WorkerLifecycleOperationId) + Send + Sync>,
|
||||||
|
) -> Self {
|
||||||
|
self.restore_progress = Some(progress);
|
||||||
|
self
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn restore_progressed(&self, operation_id: WorkerLifecycleOperationId) {
|
||||||
|
if let Some(progress) = &self.restore_progress {
|
||||||
|
progress(operation_id);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn worker_ref(&self) -> &WorkerRef {
|
pub fn worker_ref(&self) -> &WorkerRef {
|
||||||
@@ -335,20 +424,65 @@ impl WorkerExecutionContext {
|
|||||||
let Some(event) = event else {
|
let Some(event) = event else {
|
||||||
return Ok(());
|
return Ok(());
|
||||||
};
|
};
|
||||||
if let Err(error) = (self.observation_publisher)(self.worker_ref.clone(), event.clone())
|
#[cfg(test)]
|
||||||
{
|
let fault = {
|
||||||
|
let mut fault = self.flush_fault.lock().unwrap();
|
||||||
|
if fault.as_ref().is_some_and(|f| f.remaining_successes == 0) {
|
||||||
|
fault.take()
|
||||||
|
} else {
|
||||||
|
if let Some(fault) = fault.as_mut() {
|
||||||
|
fault.remaining_successes -= 1;
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
};
|
||||||
|
#[cfg(test)]
|
||||||
|
let mut restored = None;
|
||||||
|
#[cfg(test)]
|
||||||
|
let publication = if let Some(fault) = fault {
|
||||||
|
restored = Some(fault.restored);
|
||||||
|
fault.gate.wait(event.clone());
|
||||||
|
Err(RuntimeError::InvalidRequest(
|
||||||
|
"injected buffered publication failure".into(),
|
||||||
|
))
|
||||||
|
} else {
|
||||||
|
(self.observation_publisher)(self.worker_ref.clone(), event.clone())
|
||||||
|
};
|
||||||
|
#[cfg(not(test))]
|
||||||
|
let publication = (self.observation_publisher)(self.worker_ref.clone(), event.clone());
|
||||||
|
if let Err(error) = publication {
|
||||||
let mut state = self
|
let mut state = self
|
||||||
.publication_state
|
.publication_state
|
||||||
.lock()
|
.lock()
|
||||||
.map_err(|_| RuntimeError::StatePoisoned)?;
|
.map_err(|_| RuntimeError::StatePoisoned)?;
|
||||||
if let WorkerExecutionPublicationState::Activating(events) = &mut *state {
|
if let WorkerExecutionPublicationState::Activating(events) = &mut *state {
|
||||||
events.push_front(event);
|
events.push_front(event);
|
||||||
|
#[cfg(test)]
|
||||||
|
if let Some(restored) = restored {
|
||||||
|
let _ = restored.send(events.iter().cloned().collect());
|
||||||
|
}
|
||||||
}
|
}
|
||||||
return Err(error);
|
return Err(error);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(all(test, feature = "ws-server"))]
|
||||||
|
pub(crate) fn test_fail_flush(&self, fault: RestoreFlushFault) {
|
||||||
|
assert!(self.flush_fault.lock().unwrap().replace(fault).is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(all(test, feature = "ws-server"))]
|
||||||
|
pub(crate) fn test_buffered_events(&self) -> Vec<protocol::Event> {
|
||||||
|
match &*self.publication_state.lock().unwrap() {
|
||||||
|
WorkerExecutionPublicationState::Pending(events)
|
||||||
|
| WorkerExecutionPublicationState::Activating(events) => {
|
||||||
|
events.iter().cloned().collect()
|
||||||
|
}
|
||||||
|
_ => Vec::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(not(feature = "ws-server"))]
|
#[cfg(not(feature = "ws-server"))]
|
||||||
pub(crate) fn activate_candidate(&self) -> Result<(), RuntimeError> {
|
pub(crate) fn activate_candidate(&self) -> Result<(), RuntimeError> {
|
||||||
Ok(())
|
Ok(())
|
||||||
@@ -424,6 +558,7 @@ pub struct WorkerExecutionSpawnRequest {
|
|||||||
/// Request passed to a [`WorkerExecutionBackend`] when restoring a persisted Worker.
|
/// Request passed to a [`WorkerExecutionBackend`] when restoring a persisted Worker.
|
||||||
#[derive(Clone, Debug)]
|
#[derive(Clone, Debug)]
|
||||||
pub struct WorkerExecutionRestoreRequest {
|
pub struct WorkerExecutionRestoreRequest {
|
||||||
|
pub source_repair_possible: bool,
|
||||||
pub operation_id: WorkerLifecycleOperationId,
|
pub operation_id: WorkerLifecycleOperationId,
|
||||||
pub worker_ref: WorkerRef,
|
pub worker_ref: WorkerRef,
|
||||||
pub request: crate::catalog::CreateWorkerRequest,
|
pub request: crate::catalog::CreateWorkerRequest,
|
||||||
@@ -565,6 +700,19 @@ impl fmt::Debug for WorkerProtocolTransport {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Derived resource disposition for an existing restore owner. Not durable
|
||||||
|
/// authority: Runtime must reread its decision before choosing any action.
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||||
|
pub enum RestoreRecoveryDisposition {
|
||||||
|
/// No live or pending owner for this exact operation. Not cleanup evidence.
|
||||||
|
Absent,
|
||||||
|
InFlight,
|
||||||
|
Continue,
|
||||||
|
Discard,
|
||||||
|
/// A retained cleanup-only owner; never reconstruct or activate it.
|
||||||
|
Reclaim,
|
||||||
|
}
|
||||||
|
|
||||||
/// The connection selects this backend; WorkerRef identifies an execution,
|
/// The connection selects this backend; WorkerRef identifies an execution,
|
||||||
/// never a capability or authorization token.
|
/// never a capability or authorization token.
|
||||||
pub trait WorkerExecutionBackend: Send + Sync + 'static {
|
pub trait WorkerExecutionBackend: Send + Sync + 'static {
|
||||||
@@ -633,6 +781,33 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Completed resource-owner evidence retained while Runtime settlement may
|
||||||
|
/// still need retry. This is not a new request or execution authorization.
|
||||||
|
fn completed_restore_failure(
|
||||||
|
&self,
|
||||||
|
_operation_id: WorkerLifecycleOperationId,
|
||||||
|
_worker_ref: &WorkerRef,
|
||||||
|
) -> Option<WorkerExecutionSpawnResult> {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Opt in to Runtime-owned autonomous restore continuation. Pending tasks
|
||||||
|
/// signal completion through the original execution context; callbacks must
|
||||||
|
/// not wait for Runtime locks. Independent of resource disposition support.
|
||||||
|
fn supports_restore_continuation(&self) -> bool {
|
||||||
|
false
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Describe exact operation resource ownership for warm/cold recovery
|
||||||
|
/// dispatch. This does not opt in to callbacks or autonomous continuation.
|
||||||
|
fn restore_recovery_disposition(
|
||||||
|
&self,
|
||||||
|
_operation_id: WorkerLifecycleOperationId,
|
||||||
|
_worker_ref: &WorkerRef,
|
||||||
|
) -> Option<RestoreRecoveryDisposition> {
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
fn reconcile_restore(
|
fn reconcile_restore(
|
||||||
&self,
|
&self,
|
||||||
request: WorkerExecutionRestoreRequest,
|
request: WorkerExecutionRestoreRequest,
|
||||||
@@ -640,6 +815,42 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static {
|
|||||||
self.restore_worker(request)
|
self.restore_worker(request)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Reclaim an uncommitted operation after restart without preparing,
|
||||||
|
/// repairing, or activating a replacement candidate. Runtime must establish
|
||||||
|
/// the durable uncommitted decision before calling this seam. Unsupported
|
||||||
|
/// cleanup is uncertainty, never a successful rollback.
|
||||||
|
fn reclaim_uncommitted_restore(
|
||||||
|
&self,
|
||||||
|
_request: WorkerExecutionRestoreRequest,
|
||||||
|
) -> WorkerExecutionSpawnResult {
|
||||||
|
WorkerExecutionSpawnResult::ReconciliationRequired {
|
||||||
|
result: WorkerExecutionResult::unsupported(
|
||||||
|
WorkerExecutionOperation::Restore,
|
||||||
|
"backend cannot reclaim an uncommitted restore after restart",
|
||||||
|
),
|
||||||
|
worker_state: None,
|
||||||
|
workdir_attachments: Vec::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Discard only the exact unpublished operation after Runtime has proved
|
||||||
|
/// its durable decision uncommitted. This is not Stop and must not affect a
|
||||||
|
/// connected execution. RolledBack requires confirmed cleanup and no repair.
|
||||||
|
fn discard_uncommitted_restore(
|
||||||
|
&self,
|
||||||
|
_operation_id: WorkerLifecycleOperationId,
|
||||||
|
_worker_ref: &WorkerRef,
|
||||||
|
) -> WorkerExecutionSpawnResult {
|
||||||
|
WorkerExecutionSpawnResult::ReconciliationRequired {
|
||||||
|
result: WorkerExecutionResult::unsupported(
|
||||||
|
WorkerExecutionOperation::Restore,
|
||||||
|
"backend cannot confirm unpublished restore cleanup",
|
||||||
|
),
|
||||||
|
worker_state: None,
|
||||||
|
workdir_attachments: Vec::new(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn activate_restored_worker(
|
fn activate_restored_worker(
|
||||||
&self,
|
&self,
|
||||||
_operation_id: WorkerLifecycleOperationId,
|
_operation_id: WorkerLifecycleOperationId,
|
||||||
@@ -901,6 +1112,28 @@ impl WorkerExecutionBackendRef {
|
|||||||
self.backend.preflight_restore(request)
|
self.backend.preflight_restore(request)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn completed_restore_failure(
|
||||||
|
&self,
|
||||||
|
operation_id: WorkerLifecycleOperationId,
|
||||||
|
worker_ref: &WorkerRef,
|
||||||
|
) -> Option<WorkerExecutionSpawnResult> {
|
||||||
|
self.backend
|
||||||
|
.completed_restore_failure(operation_id, worker_ref)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn supports_restore_continuation(&self) -> bool {
|
||||||
|
self.backend.supports_restore_continuation()
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn restore_recovery_disposition(
|
||||||
|
&self,
|
||||||
|
operation_id: WorkerLifecycleOperationId,
|
||||||
|
worker_ref: &WorkerRef,
|
||||||
|
) -> Option<RestoreRecoveryDisposition> {
|
||||||
|
self.backend
|
||||||
|
.restore_recovery_disposition(operation_id, worker_ref)
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn reconcile_restore(
|
pub(crate) fn reconcile_restore(
|
||||||
&self,
|
&self,
|
||||||
request: WorkerExecutionRestoreRequest,
|
request: WorkerExecutionRestoreRequest,
|
||||||
@@ -908,6 +1141,22 @@ impl WorkerExecutionBackendRef {
|
|||||||
self.backend.reconcile_restore(request)
|
self.backend.reconcile_restore(request)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub(crate) fn reclaim_uncommitted_restore(
|
||||||
|
&self,
|
||||||
|
request: WorkerExecutionRestoreRequest,
|
||||||
|
) -> WorkerExecutionSpawnResult {
|
||||||
|
self.backend.reclaim_uncommitted_restore(request)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn discard_uncommitted_restore(
|
||||||
|
&self,
|
||||||
|
operation_id: WorkerLifecycleOperationId,
|
||||||
|
worker_ref: &WorkerRef,
|
||||||
|
) -> WorkerExecutionSpawnResult {
|
||||||
|
self.backend
|
||||||
|
.discard_uncommitted_restore(operation_id, worker_ref)
|
||||||
|
}
|
||||||
|
|
||||||
pub(crate) fn activate_restored_worker(
|
pub(crate) fn activate_restored_worker(
|
||||||
&self,
|
&self,
|
||||||
operation_id: WorkerLifecycleOperationId,
|
operation_id: WorkerLifecycleOperationId,
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -25,6 +25,7 @@ use crate::interaction::{WorkerInput, WorkerInteractionAck};
|
|||||||
use crate::management::{RuntimeSummary, WorkerDeleteResult};
|
use crate::management::{RuntimeSummary, WorkerDeleteResult};
|
||||||
#[cfg(feature = "ws-server")]
|
#[cfg(feature = "ws-server")]
|
||||||
use crate::observation::WorkerObservationCursor;
|
use crate::observation::WorkerObservationCursor;
|
||||||
|
#[cfg(feature = "fs-store")]
|
||||||
use crate::retention::{
|
use crate::retention::{
|
||||||
WorkerRetentionExecutionRequest, WorkerRetentionExecutionResult, WorkerRetentionInventory,
|
WorkerRetentionExecutionRequest, WorkerRetentionExecutionResult, WorkerRetentionInventory,
|
||||||
};
|
};
|
||||||
@@ -1568,6 +1569,7 @@ fn protocol_error_event(message: impl Into<String>) -> protocol::Event {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "fs-store")]
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
async fn worker_retention_inventory(
|
async fn worker_retention_inventory(
|
||||||
State(state): State<RuntimeHttpState>,
|
State(state): State<RuntimeHttpState>,
|
||||||
@@ -1589,6 +1591,7 @@ async fn worker_retention_inventory(
|
|||||||
.map_err(RuntimeHttpRestError::runtime)
|
.map_err(RuntimeHttpRestError::runtime)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "fs-store")]
|
||||||
#[allow(dead_code)]
|
#[allow(dead_code)]
|
||||||
async fn execute_worker_retention(
|
async fn execute_worker_retention(
|
||||||
State(state): State<RuntimeHttpState>,
|
State(state): State<RuntimeHttpState>,
|
||||||
@@ -2654,6 +2657,87 @@ mod tests {
|
|||||||
.unwrap()
|
.unwrap()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Generated management routes and real Workspace capability middleware stay
|
||||||
|
// connected even when this process has no filesystem retention provider.
|
||||||
|
#[cfg(not(feature = "fs-store"))]
|
||||||
|
#[tokio::test]
|
||||||
|
async fn memory_retention_routes_preserve_auth_and_return_typed_unavailable() {
|
||||||
|
let identity = RuntimeIdentityMaterial::generate("workspace-key").unwrap();
|
||||||
|
let trust = workspace_trust(&identity);
|
||||||
|
let runtime = Runtime::new_memory();
|
||||||
|
let app = runtime_http_router_with_workspace_auth(
|
||||||
|
runtime,
|
||||||
|
None,
|
||||||
|
workspace_test_auth(vec![trust.clone()]),
|
||||||
|
);
|
||||||
|
let worker_id = WorkerId::now_v7();
|
||||||
|
let body = runtime_api::WorkerRetentionExecutionRequest {
|
||||||
|
operation_id: "memory-retention".into(),
|
||||||
|
input_fingerprint: "input".into(),
|
||||||
|
archive_id: None,
|
||||||
|
workspace_id: "workspace-a".into(),
|
||||||
|
source_runtime_id: "runtime-test".into(),
|
||||||
|
worker_id,
|
||||||
|
source_created_at: "2026-01-01T00:00:00Z".into(),
|
||||||
|
removed_at: "2026-01-02T00:00:00Z".into(),
|
||||||
|
effective_profile: None,
|
||||||
|
retention_class: None,
|
||||||
|
policy_id: "policy".into(),
|
||||||
|
session_disposition: runtime_api::SessionDisposition::Purge,
|
||||||
|
diagnostics_disposition: runtime_api::DiagnosticsDisposition::Purge,
|
||||||
|
};
|
||||||
|
for (method, path, body) in [
|
||||||
|
(
|
||||||
|
"GET",
|
||||||
|
format!("/v1/workers/{worker_id}/retention/inventory"),
|
||||||
|
vec![],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"POST",
|
||||||
|
format!("/v1/workers/{worker_id}/retention/execute"),
|
||||||
|
serde_json::to_vec(&body).unwrap(),
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
let unsigned = app
|
||||||
|
.clone()
|
||||||
|
.oneshot(
|
||||||
|
Request::builder()
|
||||||
|
.method(method)
|
||||||
|
.uri(&path)
|
||||||
|
.header(header::CONTENT_TYPE, "application/json")
|
||||||
|
.body(Body::from(body.clone()))
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(unsigned.status(), StatusCode::UNAUTHORIZED);
|
||||||
|
let claims = signed_claims(&trust, method, &path, &body);
|
||||||
|
let response =
|
||||||
|
signed_request(app.clone(), &identity, &claims, body, "workspace-a").await;
|
||||||
|
assert_eq!(response.status(), StatusCode::SERVICE_UNAVAILABLE);
|
||||||
|
let error: RuntimeHttpErrorResponse = read_json(response).await;
|
||||||
|
assert_eq!(error.error.code, "worker_retention_unavailable");
|
||||||
|
}
|
||||||
|
for (mut value, expected_status, expected_code) in [
|
||||||
|
(body.clone(), StatusCode::NOT_FOUND, "worker_not_found"),
|
||||||
|
(body.clone(), StatusCode::BAD_REQUEST, "worker_id_mismatch"),
|
||||||
|
] {
|
||||||
|
if expected_status == StatusCode::NOT_FOUND {
|
||||||
|
value.workspace_id = "workspace-b".into();
|
||||||
|
} else {
|
||||||
|
value.worker_id = WorkerId::now_v7();
|
||||||
|
}
|
||||||
|
let path = format!("/v1/workers/{worker_id}/retention/execute");
|
||||||
|
let bytes = serde_json::to_vec(&value).unwrap();
|
||||||
|
let claims = signed_claims(&trust, "POST", &path, &bytes);
|
||||||
|
let response =
|
||||||
|
signed_request(app.clone(), &identity, &claims, bytes, "workspace-a").await;
|
||||||
|
assert_eq!(response.status(), expected_status);
|
||||||
|
let error: RuntimeHttpErrorResponse = read_json(response).await;
|
||||||
|
assert_eq!(error.error.code, expected_code);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn ordinary_signed_ping_succeeds_without_handshake_and_rejects_replay() {
|
async fn ordinary_signed_ping_succeeds_without_handshake_and_rejects_replay() {
|
||||||
let identity = RuntimeIdentityMaterial::generate("workspace-key").unwrap();
|
let identity = RuntimeIdentityMaterial::generate("workspace-key").unwrap();
|
||||||
|
|||||||
@@ -719,6 +719,9 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
|
|||||||
server_api::WorkerRestoreState::RolledBack => {
|
server_api::WorkerRestoreState::RolledBack => {
|
||||||
runtime_api::WorkerRestoreState::RolledBack
|
runtime_api::WorkerRestoreState::RolledBack
|
||||||
}
|
}
|
||||||
|
server_api::WorkerRestoreState::Superseded => {
|
||||||
|
runtime_api::WorkerRestoreState::Superseded
|
||||||
|
}
|
||||||
server_api::WorkerRestoreState::ReconciliationRequired => {
|
server_api::WorkerRestoreState::ReconciliationRequired => {
|
||||||
runtime_api::WorkerRestoreState::ReconciliationRequired
|
runtime_api::WorkerRestoreState::ReconciliationRequired
|
||||||
}
|
}
|
||||||
@@ -769,6 +772,9 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
|
|||||||
server_api::WorkerRestoreState::RolledBack => {
|
server_api::WorkerRestoreState::RolledBack => {
|
||||||
runtime_api::WorkerRestoreState::RolledBack
|
runtime_api::WorkerRestoreState::RolledBack
|
||||||
}
|
}
|
||||||
|
server_api::WorkerRestoreState::Superseded => {
|
||||||
|
runtime_api::WorkerRestoreState::Superseded
|
||||||
|
}
|
||||||
server_api::WorkerRestoreState::ReconciliationRequired => {
|
server_api::WorkerRestoreState::ReconciliationRequired => {
|
||||||
runtime_api::WorkerRestoreState::ReconciliationRequired
|
runtime_api::WorkerRestoreState::ReconciliationRequired
|
||||||
}
|
}
|
||||||
@@ -881,13 +887,25 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
|
|||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
let worker_ref = worker_ref_for(&self.state.runtime, worker_id).map_err(api_error)?;
|
let worker_ref = worker_ref_for(&self.state.runtime, worker_id).map_err(api_error)?;
|
||||||
let inventory = self
|
#[cfg(feature = "fs-store")]
|
||||||
.state
|
{
|
||||||
.runtime
|
let inventory = self
|
||||||
.worker_retention_inventory(&scope.workspace_id, &worker_ref)
|
.state
|
||||||
.map_err(RuntimeHttpRestError::runtime)
|
.runtime
|
||||||
.map_err(api_error)?;
|
.worker_retention_inventory(&scope.workspace_id, &worker_ref)
|
||||||
response(inventory)
|
.map_err(RuntimeHttpRestError::runtime)
|
||||||
|
.map_err(api_error)?;
|
||||||
|
response(inventory)
|
||||||
|
}
|
||||||
|
#[cfg(not(feature = "fs-store"))]
|
||||||
|
{
|
||||||
|
let _ = (scope, worker_ref);
|
||||||
|
Err(runtime_api::RuntimeApiError::new(
|
||||||
|
503,
|
||||||
|
"worker_retention_unavailable",
|
||||||
|
"Worker retention requires filesystem storage",
|
||||||
|
))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async fn execute_retention(
|
async fn execute_retention(
|
||||||
@@ -905,27 +923,36 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi {
|
|||||||
)
|
)
|
||||||
})?;
|
})?;
|
||||||
let worker_ref = worker_ref_for(&self.state.runtime, worker_id).map_err(api_error)?;
|
let worker_ref = worker_ref_for(&self.state.runtime, worker_id).map_err(api_error)?;
|
||||||
let request: WorkerRetentionExecutionRequest = request(value)?;
|
if value.workspace_id != scope.workspace_id {
|
||||||
if request.workspace_id != scope.workspace_id {
|
|
||||||
return Err(runtime_api::RuntimeApiError::new(
|
return Err(runtime_api::RuntimeApiError::new(
|
||||||
StatusCode::NOT_FOUND.as_u16(),
|
StatusCode::NOT_FOUND.as_u16(),
|
||||||
"worker_not_found",
|
"worker_not_found",
|
||||||
"worker does not belong to the authenticated Workspace",
|
"worker does not belong to the authenticated Workspace",
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
if request.worker_id != worker_ref.worker_id {
|
if value.worker_id != worker_ref.worker_id {
|
||||||
return Err(runtime_api::RuntimeApiError::new(
|
return Err(runtime_api::RuntimeApiError::new(
|
||||||
StatusCode::BAD_REQUEST.as_u16(),
|
StatusCode::BAD_REQUEST.as_u16(),
|
||||||
"worker_id_mismatch",
|
"worker_id_mismatch",
|
||||||
"Worker retention request worker_id does not match route worker_id",
|
"Worker retention request worker_id does not match route worker_id",
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
let result = self
|
#[cfg(feature = "fs-store")]
|
||||||
.state
|
{
|
||||||
.runtime
|
let request: WorkerRetentionExecutionRequest = request(value)?;
|
||||||
.execute_worker_retention(&request)
|
let result = self
|
||||||
.map_err(RuntimeHttpRestError::runtime)
|
.state
|
||||||
.map_err(api_error)?;
|
.runtime
|
||||||
response(result)
|
.execute_worker_retention(&request)
|
||||||
|
.map_err(RuntimeHttpRestError::runtime)
|
||||||
|
.map_err(api_error)?;
|
||||||
|
response(result)
|
||||||
|
}
|
||||||
|
#[cfg(not(feature = "fs-store"))]
|
||||||
|
Err(runtime_api::RuntimeApiError::new(
|
||||||
|
503,
|
||||||
|
"worker_retention_unavailable",
|
||||||
|
"Worker retention requires filesystem storage",
|
||||||
|
))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -16,6 +16,11 @@ pub mod management;
|
|||||||
pub mod observation;
|
pub mod observation;
|
||||||
pub mod profile_archive;
|
pub mod profile_archive;
|
||||||
pub mod resource;
|
pub mod resource;
|
||||||
|
#[cfg(feature = "restore-checkpoints")]
|
||||||
|
pub mod restore_checkpoints;
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "../../../tests/support/restore_environment.rs"]
|
||||||
|
mod restore_test_environment;
|
||||||
#[cfg(feature = "fs-store")]
|
#[cfg(feature = "fs-store")]
|
||||||
pub mod retention;
|
pub mod retention;
|
||||||
mod runtime;
|
mod runtime;
|
||||||
@@ -32,6 +37,8 @@ pub mod workspace_issuer;
|
|||||||
#[cfg(feature = "http-server")]
|
#[cfg(feature = "http-server")]
|
||||||
pub mod workspace_request;
|
pub mod workspace_request;
|
||||||
|
|
||||||
|
#[cfg(feature = "test-support")]
|
||||||
|
pub use fs_store::test_support;
|
||||||
#[cfg(feature = "fs-store")]
|
#[cfg(feature = "fs-store")]
|
||||||
pub use fs_store::{FsRuntimeStore, FsRuntimeStoreOptions};
|
pub use fs_store::{FsRuntimeStore, FsRuntimeStoreOptions};
|
||||||
pub use management::RuntimeOptions;
|
pub use management::RuntimeOptions;
|
||||||
|
|||||||
@@ -117,6 +117,12 @@ fn run() -> Result<(), ProcessError> {
|
|||||||
let listener = tokio::net::TcpListener::bind(config.http.bind_addr).await?;
|
let listener = tokio::net::TcpListener::bind(config.http.bind_addr).await?;
|
||||||
let local_addr = listener.local_addr()?;
|
let local_addr = listener.local_addr()?;
|
||||||
let worker_runtime = build_runtime(&config)?;
|
let worker_runtime = build_runtime(&config)?;
|
||||||
|
#[cfg(feature = "restore-checkpoints")]
|
||||||
|
if let Some(checkpoint) = &config.test_restore_checkpoint {
|
||||||
|
worker_runtime
|
||||||
|
.install_test_restore_checkpoint(checkpoint.clone())
|
||||||
|
.map_err(ProcessError::Runtime)?;
|
||||||
|
}
|
||||||
eprintln!(
|
eprintln!(
|
||||||
"yoi-runtime listening on {local_addr}; intended client is a trusted backend/proxy, not a browser"
|
"yoi-runtime listening on {local_addr}; intended client is a trusted backend/proxy, not a browser"
|
||||||
);
|
);
|
||||||
@@ -391,6 +397,21 @@ where
|
|||||||
let (flag, inline_value) = split_flag_value(arg)?;
|
let (flag, inline_value) = split_flag_value(arg)?;
|
||||||
match flag.as_str() {
|
match flag.as_str() {
|
||||||
"--help" | "-h" => return Ok(None),
|
"--help" | "-h" => return Ok(None),
|
||||||
|
#[cfg(feature = "restore-checkpoints")]
|
||||||
|
"--test-restore-checkpoint" => {
|
||||||
|
let path = take_value(&flag, inline_value, &mut args)?;
|
||||||
|
if config.test_restore_checkpoint.is_some() {
|
||||||
|
return Err(ProcessError::usage(
|
||||||
|
"--test-restore-checkpoint may only be specified once".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
config.test_restore_checkpoint = Some(
|
||||||
|
worker_runtime::restore_checkpoints::RestoreCheckpointConfig::from_path(
|
||||||
|
Path::new(&path),
|
||||||
|
)
|
||||||
|
.map_err(ProcessError::Runtime)?,
|
||||||
|
);
|
||||||
|
}
|
||||||
"--bind" => {
|
"--bind" => {
|
||||||
let value = take_value(&flag, inline_value, &mut args)?;
|
let value = take_value(&flag, inline_value, &mut args)?;
|
||||||
config.http.bind_addr = value.parse::<SocketAddr>().map_err(|error| {
|
config.http.bind_addr = value.parse::<SocketAddr>().map_err(|error| {
|
||||||
@@ -465,6 +486,12 @@ where
|
|||||||
}
|
}
|
||||||
|
|
||||||
apply_store_selection(&mut config);
|
apply_store_selection(&mut config);
|
||||||
|
#[cfg(feature = "restore-checkpoints")]
|
||||||
|
if config.test_restore_checkpoint.is_some() && config.no_store {
|
||||||
|
return Err(ProcessError::usage(
|
||||||
|
"--test-restore-checkpoint requires an fs Runtime store".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
Ok(Some(config))
|
Ok(Some(config))
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -521,6 +548,8 @@ struct ProcessConfig {
|
|||||||
no_store: bool,
|
no_store: bool,
|
||||||
backend_resource_endpoint: Option<String>,
|
backend_resource_endpoint: Option<String>,
|
||||||
backend_resource_token: Option<String>,
|
backend_resource_token: Option<String>,
|
||||||
|
#[cfg(feature = "restore-checkpoints")]
|
||||||
|
test_restore_checkpoint: Option<worker_runtime::restore_checkpoints::RestoreCheckpointConfig>,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||||
@@ -542,6 +571,8 @@ impl ProcessConfig {
|
|||||||
no_store: false,
|
no_store: false,
|
||||||
backend_resource_endpoint: None,
|
backend_resource_endpoint: None,
|
||||||
backend_resource_token: None,
|
backend_resource_token: None,
|
||||||
|
#[cfg(feature = "restore-checkpoints")]
|
||||||
|
test_restore_checkpoint: None,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1227,6 +1258,47 @@ Auth commands:
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn checkpoint_cli_is_absent_by_default() {
|
||||||
|
let config = parse_args([] as [&str; 0]).unwrap().unwrap();
|
||||||
|
#[cfg(feature = "restore-checkpoints")]
|
||||||
|
assert!(config.test_restore_checkpoint.is_none());
|
||||||
|
#[cfg(not(feature = "restore-checkpoints"))]
|
||||||
|
{
|
||||||
|
let _ = config;
|
||||||
|
let error = parse_args(["--test-restore-checkpoint", "never-opened.json"]).unwrap_err();
|
||||||
|
assert!(error.to_string().contains("unknown argument"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "restore-checkpoints")]
|
||||||
|
#[test]
|
||||||
|
fn checkpoint_cli_requires_valid_explicit_file_and_fs_store() {
|
||||||
|
use worker_runtime::restore_checkpoints::{
|
||||||
|
RestoreCheckpointConfig, RestoreCheckpointStage,
|
||||||
|
};
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = dir.path().join("checkpoint.json");
|
||||||
|
let checkpoint = RestoreCheckpointConfig {
|
||||||
|
endpoint: "127.0.0.1:1234".parse().unwrap(),
|
||||||
|
worker_id: protocol::WorkerId::now_v7(),
|
||||||
|
stage: RestoreCheckpointStage::CommittedPreterminal,
|
||||||
|
nonce: "cli-test".into(),
|
||||||
|
};
|
||||||
|
std::fs::write(&path, serde_json::to_vec(&checkpoint).unwrap()).unwrap();
|
||||||
|
let args = ["--test-restore-checkpoint", path.to_str().unwrap()];
|
||||||
|
let config = parse_args(args).unwrap().unwrap();
|
||||||
|
assert_eq!(config.test_restore_checkpoint, Some(checkpoint));
|
||||||
|
assert!(parse_args([args[0], args[1], "--no-store"]).is_err());
|
||||||
|
assert!(parse_args([args[0], args[1], args[0], args[1]]).is_err());
|
||||||
|
assert!(parse_args([args[0]]).is_err());
|
||||||
|
std::fs::write(&path, b"private-invalid-body").unwrap();
|
||||||
|
let error = parse_args(args).unwrap_err().to_string();
|
||||||
|
assert!(!error.contains("private-invalid-body"));
|
||||||
|
std::fs::remove_file(&path).unwrap();
|
||||||
|
assert!(parse_args(args).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn help_does_not_start_server() {
|
fn help_does_not_start_server() {
|
||||||
assert!(parse_args(["--help"]).unwrap().is_none());
|
assert!(parse_args(["--help"]).unwrap().is_none());
|
||||||
|
|||||||
@@ -1,6 +1,8 @@
|
|||||||
|
#[cfg(feature = "http-server")]
|
||||||
use crate::auth::BACKEND_RESOURCE_FETCH_PERMISSION;
|
use crate::auth::BACKEND_RESOURCE_FETCH_PERMISSION;
|
||||||
use crate::identity::WorkerId;
|
use crate::identity::WorkerId;
|
||||||
use crate::profile_archive::{ProfileSourceArchive, ProfileSourceArchiveRef, sha256_hex};
|
use crate::profile_archive::{ProfileSourceArchive, ProfileSourceArchiveRef, sha256_hex};
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
use crate::workspace_request::RuntimeWorkspaceRequestClient;
|
use crate::workspace_request::RuntimeWorkspaceRequestClient;
|
||||||
use async_trait::async_trait;
|
use async_trait::async_trait;
|
||||||
use serde::{Deserialize, Serialize};
|
use serde::{Deserialize, Serialize};
|
||||||
|
|||||||
@@ -0,0 +1,144 @@
|
|||||||
|
//! Explicit, ephemeral synchronization for owned Runtime process tests only.
|
||||||
|
//!
|
||||||
|
//! The nonce authenticates no Runtime action and is never a routing identity.
|
||||||
|
//! Socket failure, EOF and the fixed deadline only release the gate; the lifecycle
|
||||||
|
//! owner continues its original operation and remains responsible for cleanup.
|
||||||
|
|
||||||
|
use std::fs::OpenOptions;
|
||||||
|
use std::io::{Read, Write};
|
||||||
|
use std::net::{SocketAddr, TcpStream};
|
||||||
|
use std::path::Path;
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use protocol::WorkerId;
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
|
||||||
|
use crate::error::RuntimeError;
|
||||||
|
use crate::execution::WorkerLifecycleOperationId;
|
||||||
|
|
||||||
|
pub const RESTORE_CHECKPOINT_TIMEOUT: Duration = Duration::from_secs(8);
|
||||||
|
pub const MAX_RESTORE_CHECKPOINT_CONFIG_BYTES: usize = 4096;
|
||||||
|
pub const MAX_RESTORE_CHECKPOINT_ACK_BYTES: usize = 4096;
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum RestoreCheckpointStage {
|
||||||
|
PreparedNotCommitted,
|
||||||
|
CommittedPreterminal,
|
||||||
|
ActivatedPreterminal,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(rename_all = "snake_case")]
|
||||||
|
pub enum RestoreCheckpointCommit {
|
||||||
|
NotCommitted,
|
||||||
|
Committed,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct RestoreCheckpointConfig {
|
||||||
|
pub endpoint: SocketAddr,
|
||||||
|
pub worker_id: WorkerId,
|
||||||
|
pub stage: RestoreCheckpointStage,
|
||||||
|
pub nonce: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct RestoreCheckpointAck {
|
||||||
|
pub nonce: String,
|
||||||
|
pub stage: RestoreCheckpointStage,
|
||||||
|
pub runtime_id: String,
|
||||||
|
pub worker_id: WorkerId,
|
||||||
|
pub operation_id: WorkerLifecycleOperationId,
|
||||||
|
pub commit: RestoreCheckpointCommit,
|
||||||
|
pub receipt_absent: bool,
|
||||||
|
pub pid: u32,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn invalid_config() -> RuntimeError {
|
||||||
|
// Do not include serde errors, paths or input contents in process diagnostics.
|
||||||
|
RuntimeError::InvalidRequest("invalid test restore checkpoint configuration".into())
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RestoreCheckpointConfig {
|
||||||
|
pub fn validate(&self) -> Result<(), RuntimeError> {
|
||||||
|
if !self.endpoint.ip().is_loopback()
|
||||||
|
|| self.endpoint.port() == 0
|
||||||
|
|| self.nonce.is_empty()
|
||||||
|
|| self.nonce.len() > 256
|
||||||
|
|| self.nonce.chars().any(char::is_control)
|
||||||
|
{
|
||||||
|
return Err(invalid_config());
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn from_path(path: &Path) -> Result<Self, RuntimeError> {
|
||||||
|
let mut options = OpenOptions::new();
|
||||||
|
options.read(true);
|
||||||
|
#[cfg(unix)]
|
||||||
|
{
|
||||||
|
use std::os::unix::fs::OpenOptionsExt;
|
||||||
|
// A FIFO must not stall startup before its regular-file check.
|
||||||
|
options.custom_flags(libc::O_NONBLOCK | libc::O_NOFOLLOW);
|
||||||
|
}
|
||||||
|
let file = options.open(path).map_err(|_| invalid_config())?;
|
||||||
|
let metadata = file.metadata().map_err(|_| invalid_config())?;
|
||||||
|
if !metadata.is_file() || metadata.len() > MAX_RESTORE_CHECKPOINT_CONFIG_BYTES as u64 {
|
||||||
|
return Err(invalid_config());
|
||||||
|
}
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
file.take(MAX_RESTORE_CHECKPOINT_CONFIG_BYTES as u64 + 1)
|
||||||
|
.read_to_end(&mut bytes)
|
||||||
|
.map_err(|_| invalid_config())?;
|
||||||
|
if bytes.len() > MAX_RESTORE_CHECKPOINT_CONFIG_BYTES {
|
||||||
|
return Err(invalid_config());
|
||||||
|
}
|
||||||
|
let config: Self = serde_json::from_slice(&bytes).map_err(|_| invalid_config())?;
|
||||||
|
config.validate()?;
|
||||||
|
Ok(config)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn wait_for_release(config: &RestoreCheckpointConfig, ack: &RestoreCheckpointAck) {
|
||||||
|
let _ = exchange(config, ack);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn remaining(deadline: Instant) -> std::io::Result<Duration> {
|
||||||
|
deadline
|
||||||
|
.checked_duration_since(Instant::now())
|
||||||
|
.filter(|remaining| !remaining.is_zero())
|
||||||
|
.ok_or_else(|| std::io::Error::from(std::io::ErrorKind::TimedOut))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn exchange(config: &RestoreCheckpointConfig, ack: &RestoreCheckpointAck) -> std::io::Result<()> {
|
||||||
|
let deadline = Instant::now() + RESTORE_CHECKPOINT_TIMEOUT;
|
||||||
|
let mut bytes = serde_json::to_vec(ack)?;
|
||||||
|
bytes.push(b'\n');
|
||||||
|
if bytes.len() > MAX_RESTORE_CHECKPOINT_ACK_BYTES {
|
||||||
|
return Err(std::io::Error::from(std::io::ErrorKind::InvalidData));
|
||||||
|
}
|
||||||
|
let mut stream = TcpStream::connect_timeout(&config.endpoint, remaining(deadline)?)?;
|
||||||
|
// Each syscall shares one budget; partial writes cannot reset the deadline.
|
||||||
|
let mut pending = bytes.as_slice();
|
||||||
|
while !pending.is_empty() {
|
||||||
|
stream.set_write_timeout(Some(remaining(deadline)?))?;
|
||||||
|
let written = stream.write(pending)?;
|
||||||
|
if written == 0 {
|
||||||
|
return Err(std::io::Error::from(std::io::ErrorKind::WriteZero));
|
||||||
|
}
|
||||||
|
pending = &pending[written..];
|
||||||
|
}
|
||||||
|
stream.set_read_timeout(Some(remaining(deadline)?))?;
|
||||||
|
let mut release = [0];
|
||||||
|
stream.read_exact(&mut release)?;
|
||||||
|
if release != [b'R'] {
|
||||||
|
return Err(std::io::Error::from(std::io::ErrorKind::InvalidData));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests;
|
||||||
@@ -0,0 +1,139 @@
|
|||||||
|
use super::*;
|
||||||
|
use std::io::{BufRead, BufReader};
|
||||||
|
use std::net::TcpListener;
|
||||||
|
use std::thread;
|
||||||
|
|
||||||
|
fn config(endpoint: SocketAddr) -> RestoreCheckpointConfig {
|
||||||
|
RestoreCheckpointConfig {
|
||||||
|
endpoint,
|
||||||
|
worker_id: WorkerId::now_v7(),
|
||||||
|
stage: RestoreCheckpointStage::PreparedNotCommitted,
|
||||||
|
nonce: "checkpoint-test-handshake".into(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn ack(config: &RestoreCheckpointConfig) -> RestoreCheckpointAck {
|
||||||
|
RestoreCheckpointAck {
|
||||||
|
nonce: config.nonce.clone(),
|
||||||
|
stage: config.stage,
|
||||||
|
runtime_id: "checkpoint-runtime".into(),
|
||||||
|
worker_id: config.worker_id,
|
||||||
|
operation_id: WorkerLifecycleOperationId::new(),
|
||||||
|
commit: RestoreCheckpointCommit::NotCommitted,
|
||||||
|
receipt_absent: true,
|
||||||
|
pid: std::process::id(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn config_rejects_nonliteral_nonloopback_bad_stage_unknown_fields_and_invalid_worker() {
|
||||||
|
let valid = serde_json::to_value(config("127.0.0.1:1234".parse().unwrap())).unwrap();
|
||||||
|
for (field, value) in [
|
||||||
|
("endpoint", "localhost:1234"),
|
||||||
|
("endpoint", "192.0.2.1:1234"),
|
||||||
|
("endpoint", "0.0.0.0:1234"),
|
||||||
|
("endpoint", "127.0.0.1:0"),
|
||||||
|
("endpoint", "[::ffff:127.0.0.1]:1234"),
|
||||||
|
("worker_id", "not-a-worker-id"),
|
||||||
|
("stage", "after_terminal"),
|
||||||
|
("nonce", ""),
|
||||||
|
("nonce", "secret\nbody"),
|
||||||
|
("timeout", "unbounded"),
|
||||||
|
] {
|
||||||
|
let mut invalid = valid.clone();
|
||||||
|
invalid[field] = value.into();
|
||||||
|
let parsed = serde_json::from_value::<RestoreCheckpointConfig>(invalid);
|
||||||
|
assert!(
|
||||||
|
parsed.is_err() || parsed.unwrap().validate().is_err(),
|
||||||
|
"{field}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for endpoint in ["127.0.0.1:1234", "[::1]:1234"] {
|
||||||
|
config(endpoint.parse().unwrap()).validate().unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn config_file_bounds_and_errors_do_not_disclose_contents() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = dir.path().join("config.json");
|
||||||
|
assert!(RestoreCheckpointConfig::from_path(&path).is_err());
|
||||||
|
std::fs::write(&path, vec![b'x'; MAX_RESTORE_CHECKPOINT_CONFIG_BYTES + 1]).unwrap();
|
||||||
|
assert!(RestoreCheckpointConfig::from_path(&path).is_err());
|
||||||
|
std::fs::write(
|
||||||
|
&path,
|
||||||
|
br#"{"nonce":"private-config-body","stage":"secret"}"#,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let error = RestoreCheckpointConfig::from_path(&path)
|
||||||
|
.unwrap_err()
|
||||||
|
.to_string();
|
||||||
|
assert!(!error.contains("private-config-body"));
|
||||||
|
assert!(!error.contains("secret"));
|
||||||
|
let mut valid = config("127.0.0.1:1234".parse().unwrap());
|
||||||
|
valid.nonce = "x".repeat(257);
|
||||||
|
assert!(valid.validate().is_err());
|
||||||
|
valid.nonce = "valid".into();
|
||||||
|
std::fs::write(&path, serde_json::to_vec(&valid).unwrap()).unwrap();
|
||||||
|
assert_eq!(RestoreCheckpointConfig::from_path(&path).unwrap(), valid);
|
||||||
|
assert!(RestoreCheckpointConfig::from_path(dir.path()).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn explicit_release_eof_and_invalid_release_close_the_owned_stream() {
|
||||||
|
for release in [Some(b'R'), None, Some(b'X')] {
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||||
|
let config = config(listener.local_addr().unwrap());
|
||||||
|
let expected = ack(&config);
|
||||||
|
let sent = expected.clone();
|
||||||
|
let task = thread::spawn(move || exchange(&config, &sent));
|
||||||
|
let (mut stream, _) = listener.accept().unwrap();
|
||||||
|
stream
|
||||||
|
.set_read_timeout(Some(Duration::from_secs(2)))
|
||||||
|
.unwrap();
|
||||||
|
let mut line = String::new();
|
||||||
|
BufReader::new(&mut stream).read_line(&mut line).unwrap();
|
||||||
|
assert!(line.len() <= MAX_RESTORE_CHECKPOINT_ACK_BYTES);
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::from_str::<RestoreCheckpointAck>(&line).unwrap(),
|
||||||
|
expected
|
||||||
|
);
|
||||||
|
if let Some(byte) = release {
|
||||||
|
stream.write_all(&[byte]).unwrap();
|
||||||
|
} else {
|
||||||
|
stream.shutdown(std::net::Shutdown::Write).unwrap();
|
||||||
|
}
|
||||||
|
assert_eq!(task.join().unwrap().is_ok(), release == Some(b'R'));
|
||||||
|
let mut byte = [0];
|
||||||
|
assert_eq!(stream.read(&mut byte).unwrap(), 0);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn absent_listener_resumes_without_waiting_or_retrying() {
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||||
|
let config = config(listener.local_addr().unwrap());
|
||||||
|
drop(listener);
|
||||||
|
let start = Instant::now();
|
||||||
|
wait_for_release(&config, &ack(&config));
|
||||||
|
assert!(start.elapsed() < Duration::from_secs(2));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn silent_peer_hits_fixed_deadline_and_stream_is_dropped() {
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||||
|
let config = config(listener.local_addr().unwrap());
|
||||||
|
let start = Instant::now();
|
||||||
|
let task = thread::spawn(move || exchange(&config, &ack(&config)));
|
||||||
|
let (mut stream, _) = listener.accept().unwrap();
|
||||||
|
stream
|
||||||
|
.set_read_timeout(Some(Duration::from_secs(12)))
|
||||||
|
.unwrap();
|
||||||
|
let mut reader = BufReader::new(&mut stream);
|
||||||
|
let mut line = String::new();
|
||||||
|
reader.read_line(&mut line).unwrap();
|
||||||
|
assert!(task.join().unwrap().is_err());
|
||||||
|
assert!(start.elapsed() >= RESTORE_CHECKPOINT_TIMEOUT - Duration::from_millis(100));
|
||||||
|
assert!(start.elapsed() < RESTORE_CHECKPOINT_TIMEOUT + Duration::from_secs(3));
|
||||||
|
assert_eq!(reader.read(&mut [0]).unwrap(), 0);
|
||||||
|
}
|
||||||
@@ -3,7 +3,7 @@
|
|||||||
//! only the empty segment envelope is built with current APIs.
|
//! only the empty segment envelope is built with current APIs.
|
||||||
use super::tests::{request, write_json};
|
use super::tests::{request, write_json};
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::fs_store::{FsRuntimeStore, FsRuntimeStoreOptions};
|
use crate::fs_store::{FsRuntimeStore, FsRuntimeStoreOptions, SCHEMA_VERSION};
|
||||||
use serde_json::{Value, json};
|
use serde_json::{Value, json};
|
||||||
use session_store::{LogEntry, SegmentId, SessionId, Store, WorkerSessionStore};
|
use session_store::{LogEntry, SegmentId, SessionId, Store, WorkerSessionStore};
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
@@ -226,7 +226,7 @@ fn completed_v4_archive(
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() {
|
fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() {
|
||||||
for runtime_schema in [11, 12] {
|
for runtime_schema in [11, 12, SCHEMA_VERSION] {
|
||||||
let temp = tempfile::tempdir().unwrap();
|
let temp = tempfile::tempdir().unwrap();
|
||||||
let root = temp.path().join("runtime");
|
let root = temp.path().join("runtime");
|
||||||
let (worker, session, segment) = runtime_fixture(&root, runtime_schema);
|
let (worker, session, segment) = runtime_fixture(&root, runtime_schema);
|
||||||
@@ -329,7 +329,7 @@ fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() {
|
|||||||
(storage.checksum_sha256, storage.bytes, storage.file_count),
|
(storage.checksum_sha256, storage.bytes, storage.file_count),
|
||||||
(checksum, bytes, count)
|
(checksum, bytes, count)
|
||||||
);
|
);
|
||||||
if runtime_schema == 12 {
|
if runtime_schema == SCHEMA_VERSION {
|
||||||
for name in [
|
for name in [
|
||||||
PathBuf::from("runtime.json"),
|
PathBuf::from("runtime.json"),
|
||||||
PathBuf::from(format!("workers/{worker}/worker.json")),
|
PathBuf::from(format!("workers/{worker}/worker.json")),
|
||||||
@@ -346,8 +346,8 @@ fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() {
|
|||||||
);
|
);
|
||||||
drop(FsRuntimeStore::open_or_create(root.clone(), "runtime-a").unwrap());
|
drop(FsRuntimeStore::open_or_create(root.clone(), "runtime-a").unwrap());
|
||||||
assert_eq!(tree_bytes(&root), once, "retry rewrote migrated evidence");
|
assert_eq!(tree_bytes(&root), once, "retry rewrote migrated evidence");
|
||||||
assert!(!temp.path().join(".runtime.schema-v12-staging").exists());
|
assert!(!temp.path().join(".runtime.schema-v13-staging").exists());
|
||||||
assert!(!temp.path().join(".runtime.pre-schema-v12-backup").exists());
|
assert!(!temp.path().join(".runtime.pre-schema-v13-backup").exists());
|
||||||
// session-storage, not the immutable receipt's old checksum, attests
|
// session-storage, not the immutable receipt's old checksum, attests
|
||||||
// current bytes. Tampering with those bytes must remain detectable.
|
// current bytes. Tampering with those bytes must remain detectable.
|
||||||
let path = archive_session
|
let path = archive_session
|
||||||
@@ -360,7 +360,7 @@ fn runtime_v11_and_current_route_v4_stopped_orphan_archive_through_staging() {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn corrupt_archive_does_not_block_startup_or_healthy_session_migration_and_repair_converges() {
|
fn corrupt_archive_does_not_block_startup_or_healthy_session_migration_and_repair_converges() {
|
||||||
for runtime_schema in [11, 12] {
|
for runtime_schema in [11, 12, SCHEMA_VERSION] {
|
||||||
let temp = tempfile::tempdir().unwrap();
|
let temp = tempfile::tempdir().unwrap();
|
||||||
let root = temp.path().join("runtime");
|
let root = temp.path().join("runtime");
|
||||||
let (worker, session, segment) = runtime_fixture(&root, runtime_schema);
|
let (worker, session, segment) = runtime_fixture(&root, runtime_schema);
|
||||||
@@ -440,8 +440,8 @@ fn unmigratable_archive_preserves_evidence_across_startup_and_reopen() {
|
|||||||
] {
|
] {
|
||||||
let temp = tempfile::tempdir().unwrap();
|
let temp = tempfile::tempdir().unwrap();
|
||||||
let root = temp.path().join("runtime");
|
let root = temp.path().join("runtime");
|
||||||
let (live_worker, live_session, live_segment) = runtime_fixture(&root, 12);
|
let (live_worker, live_session, live_segment) = runtime_fixture(&root, SCHEMA_VERSION);
|
||||||
let (worker, _, segment) = stopped_worker(&root, 12);
|
let (worker, _, segment) = stopped_worker(&root, SCHEMA_VERSION);
|
||||||
let provider = FsWorkerRetentionProvider::new(&root);
|
let provider = FsWorkerRetentionProvider::new(&root);
|
||||||
let path = provider
|
let path = provider
|
||||||
.worker_dir(worker)
|
.worker_dir(worker)
|
||||||
@@ -542,8 +542,8 @@ fn unreadable_archive_metadata_does_not_block_archive_only_runtime_startup() {
|
|||||||
] {
|
] {
|
||||||
let temp = tempfile::tempdir().unwrap();
|
let temp = tempfile::tempdir().unwrap();
|
||||||
let root = temp.path().join("runtime");
|
let root = temp.path().join("runtime");
|
||||||
let (worker, _, _) = runtime_fixture(&root, 12);
|
let (worker, _, _) = runtime_fixture(&root, SCHEMA_VERSION);
|
||||||
completed_v4_archive(&root, 12);
|
completed_v4_archive(&root, SCHEMA_VERSION);
|
||||||
let provider = FsWorkerRetentionProvider::new(&root);
|
let provider = FsWorkerRetentionProvider::new(&root);
|
||||||
fs::remove_dir_all(provider.worker_dir(worker)).unwrap();
|
fs::remove_dir_all(provider.worker_dir(worker)).unwrap();
|
||||||
let archive = provider.archive_dir("archive-a").unwrap();
|
let archive = provider.archive_dir("archive-a").unwrap();
|
||||||
@@ -575,8 +575,8 @@ fn unreadable_archive_metadata_does_not_block_archive_only_runtime_startup() {
|
|||||||
fn archive_isolation_does_not_bypass_corrupt_erasure_authority() {
|
fn archive_isolation_does_not_bypass_corrupt_erasure_authority() {
|
||||||
let temp = tempfile::tempdir().unwrap();
|
let temp = tempfile::tempdir().unwrap();
|
||||||
let root = temp.path().join("runtime");
|
let root = temp.path().join("runtime");
|
||||||
runtime_fixture(&root, 12);
|
runtime_fixture(&root, SCHEMA_VERSION);
|
||||||
let (session, _, _, _) = completed_v4_archive(&root, 12);
|
let (session, _, _, _) = completed_v4_archive(&root, SCHEMA_VERSION);
|
||||||
let ledger = root.join("retention/session-sources");
|
let ledger = root.join("retention/session-sources");
|
||||||
fs::create_dir_all(&ledger).unwrap();
|
fs::create_dir_all(&ledger).unwrap();
|
||||||
fs::write(ledger.join(format!("{session}.json")), b"{broken\n").unwrap();
|
fs::write(ledger.join(format!("{session}.json")), b"{broken\n").unwrap();
|
||||||
@@ -591,7 +591,7 @@ fn archive_isolation_does_not_bypass_corrupt_erasure_authority() {
|
|||||||
fn runtime_v5_probe_and_staging_skip_erased_originals() {
|
fn runtime_v5_probe_and_staging_skip_erased_originals() {
|
||||||
let temp = tempfile::tempdir().unwrap();
|
let temp = tempfile::tempdir().unwrap();
|
||||||
let root = temp.path().join("runtime");
|
let root = temp.path().join("runtime");
|
||||||
let (worker, live_session, live_segment) = runtime_fixture(&root, 12);
|
let (worker, live_session, live_segment) = runtime_fixture(&root, SCHEMA_VERSION);
|
||||||
let erased = root
|
let erased = root
|
||||||
.join("orphaned-workers")
|
.join("orphaned-workers")
|
||||||
.join(WorkerId::now_v7().to_string());
|
.join(WorkerId::now_v7().to_string());
|
||||||
@@ -643,7 +643,7 @@ fn runtime_v5_rejects_broken_or_future_source_without_rewriting_evidence() {
|
|||||||
for corruption in ["broken-json", "future-schema"] {
|
for corruption in ["broken-json", "future-schema"] {
|
||||||
let temp = tempfile::tempdir().unwrap();
|
let temp = tempfile::tempdir().unwrap();
|
||||||
let root = temp.path().join("runtime");
|
let root = temp.path().join("runtime");
|
||||||
let (worker, session, segment) = runtime_fixture(&root, 12);
|
let (worker, session, segment) = runtime_fixture(&root, SCHEMA_VERSION);
|
||||||
let session_root = root
|
let session_root = root
|
||||||
.join("workers")
|
.join("workers")
|
||||||
.join(worker.to_string())
|
.join(worker.to_string())
|
||||||
@@ -674,19 +674,19 @@ fn runtime_v5_rejects_broken_or_future_source_without_rewriting_evidence() {
|
|||||||
assert_eq!(tree_bytes(&root), before);
|
assert_eq!(tree_bytes(&root), before);
|
||||||
assert!(FsRuntimeStore::migrate(&options).is_err());
|
assert!(FsRuntimeStore::migrate(&options).is_err());
|
||||||
assert_eq!(tree_bytes(&root), before);
|
assert_eq!(tree_bytes(&root), before);
|
||||||
assert!(!temp.path().join(".runtime.schema-v12-staging").exists());
|
assert!(!temp.path().join(".runtime.schema-v13-staging").exists());
|
||||||
assert!(!temp.path().join(".runtime.pre-schema-v12-backup").exists());
|
assert!(!temp.path().join(".runtime.pre-schema-v13-backup").exists());
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(unix)]
|
#[cfg(unix)]
|
||||||
#[test]
|
#[test]
|
||||||
fn current_runtime_session_migration_still_rejects_live_worker_sockets() {
|
fn current_runtime_session_migration_still_rejects_live_worker_sockets() {
|
||||||
// Relative socket names keep this fixture independent of the checkout/TMPDIR
|
// The same TempDir owns the relative bind/connect reference. No cwd change
|
||||||
// length. The TempDir still owns the same tree; no process cwd is changed.
|
// or alias is needed, and the real live-socket migration refusal stays tested.
|
||||||
let temp = tempfile::tempdir_in(".").unwrap();
|
let temp = tempfile::tempdir_in(".").unwrap();
|
||||||
let root = Path::new(temp.path().file_name().unwrap()).join("runtime");
|
let root = Path::new(temp.path().file_name().unwrap()).join("runtime");
|
||||||
let (worker, _, _) = runtime_fixture(&root, 12);
|
let (worker, _, _) = runtime_fixture(&root, SCHEMA_VERSION);
|
||||||
let before = tree_bytes(&root);
|
let before = tree_bytes(&root);
|
||||||
let runs = root.join("workers").join(worker.to_string()).join("runs");
|
let runs = root.join("workers").join(worker.to_string()).join("runs");
|
||||||
fs::create_dir_all(runs.join("1")).unwrap();
|
fs::create_dir_all(runs.join("1")).unwrap();
|
||||||
@@ -697,7 +697,7 @@ fn current_runtime_session_migration_still_rejects_live_worker_sockets() {
|
|||||||
drop(listener);
|
drop(listener);
|
||||||
fs::remove_dir_all(runs).unwrap();
|
fs::remove_dir_all(runs).unwrap();
|
||||||
assert_eq!(tree_bytes(&root), before);
|
assert_eq!(tree_bytes(&root), before);
|
||||||
assert!(!temp.path().join(".runtime.schema-v12-staging").exists());
|
assert!(!temp.path().join(".runtime.schema-v13-staging").exists());
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
@@ -705,7 +705,7 @@ fn current_runtime_upgrades_v5_role_metadata_through_staging_without_reserializi
|
|||||||
{
|
{
|
||||||
let temp = tempfile::tempdir().unwrap();
|
let temp = tempfile::tempdir().unwrap();
|
||||||
let root = temp.path().join("runtime");
|
let root = temp.path().join("runtime");
|
||||||
let (worker, session, segment) = runtime_fixture(&root, 12);
|
let (worker, session, segment) = runtime_fixture(&root, SCHEMA_VERSION);
|
||||||
let aggregate = root.join("workers").join(worker.to_string());
|
let aggregate = root.join("workers").join(worker.to_string());
|
||||||
let path = aggregate
|
let path = aggregate
|
||||||
.join("session/segments")
|
.join("session/segments")
|
||||||
|
|||||||
+1945
-496
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,112 @@
|
|||||||
|
use super::*;
|
||||||
|
use crate::restore_checkpoints::{
|
||||||
|
RestoreCheckpointAck, RestoreCheckpointCommit, RestoreCheckpointConfig, RestoreCheckpointStage,
|
||||||
|
wait_for_release,
|
||||||
|
};
|
||||||
|
|
||||||
|
impl Runtime {
|
||||||
|
/// Install an explicitly selected, one-shot process-test gate after normal startup.
|
||||||
|
/// It is not persisted or consulted by admission, routing or recovery authority.
|
||||||
|
pub fn install_test_restore_checkpoint(
|
||||||
|
&self,
|
||||||
|
config: RestoreCheckpointConfig,
|
||||||
|
) -> Result<(), RuntimeError> {
|
||||||
|
config.validate()?;
|
||||||
|
let mut state = self.lock()?;
|
||||||
|
if !matches!(state.persistence, RuntimePersistence::Fs(_)) {
|
||||||
|
return Err(RuntimeError::InvalidRequest(
|
||||||
|
"test restore checkpoint requires an fs Runtime store".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
state.restore_checkpoint = Some(config);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
// The lifecycle caller holds the per-Worker operation lock. No Runtime-state
|
||||||
|
// mutex is retained while waiting; the real backend candidate remains owned by
|
||||||
|
// the same operation throughout this synchronous call.
|
||||||
|
pub(super) fn test_restore_checkpoint(
|
||||||
|
&self,
|
||||||
|
worker_ref: &WorkerRef,
|
||||||
|
operation_id: WorkerLifecycleOperationId,
|
||||||
|
stage: RestoreCheckpointStage,
|
||||||
|
) {
|
||||||
|
let config = {
|
||||||
|
let Ok(mut state) = self.lock() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
let slot = &mut state.restore_checkpoint;
|
||||||
|
if !slot.as_ref().is_some_and(|config| {
|
||||||
|
config.worker_id == worker_ref.worker_id && config.stage == stage
|
||||||
|
}) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
// Consume even if canonical verification or the handshake fails.
|
||||||
|
slot.take().expect("matched checkpoint")
|
||||||
|
};
|
||||||
|
let Ok(ack) = self.restore_checkpoint_ack(&config, worker_ref, operation_id) else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
wait_for_release(&config, &ack);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn restore_checkpoint_ack(
|
||||||
|
&self,
|
||||||
|
config: &RestoreCheckpointConfig,
|
||||||
|
worker_ref: &WorkerRef,
|
||||||
|
operation_id: WorkerLifecycleOperationId,
|
||||||
|
) -> Result<RestoreCheckpointAck, RuntimeError> {
|
||||||
|
let invalid = || {
|
||||||
|
RuntimeError::InvalidRequest("test restore checkpoint authority did not match".into())
|
||||||
|
};
|
||||||
|
let state = self.lock()?;
|
||||||
|
let RuntimePersistence::Fs(store) = &state.persistence else {
|
||||||
|
return Err(invalid());
|
||||||
|
};
|
||||||
|
// The existing helper validates the exact bounded canonical aggregate and
|
||||||
|
// syncs that same file plus its owning directories. Never manufacture a
|
||||||
|
// journal marker or infer commit from an in-memory projection.
|
||||||
|
let record = store.read_durable_worker_record(worker_ref)?;
|
||||||
|
let PersistedWorkerExecutionState::ReconciliationRequired(
|
||||||
|
PersistedWorkerLifecycleOperation::Restore(pending),
|
||||||
|
) = &record.execution_state
|
||||||
|
else {
|
||||||
|
return Err(invalid());
|
||||||
|
};
|
||||||
|
let expected = match config.stage {
|
||||||
|
RestoreCheckpointStage::PreparedNotCommitted => RestoreCommitFact::NotCommitted,
|
||||||
|
RestoreCheckpointStage::CommittedPreterminal
|
||||||
|
| RestoreCheckpointStage::ActivatedPreterminal => RestoreCommitFact::Committed,
|
||||||
|
};
|
||||||
|
if config.worker_id != worker_ref.worker_id
|
||||||
|
|| pending.operation_id != operation_id
|
||||||
|
|| pending.commit != expected
|
||||||
|
{
|
||||||
|
return Err(invalid());
|
||||||
|
}
|
||||||
|
let guard = &record.restore_guard;
|
||||||
|
let id = guard.active_request_id.as_ref().ok_or_else(invalid)?;
|
||||||
|
let owner = guard.owners.get(id).ok_or_else(invalid)?;
|
||||||
|
if owner.operation_id != operation_id
|
||||||
|
|| owner.request.request_id != *id
|
||||||
|
|| owner.receipt.is_some()
|
||||||
|
{
|
||||||
|
return Err(invalid());
|
||||||
|
}
|
||||||
|
let runtime_id = state.runtime_identity.clone().ok_or_else(invalid)?;
|
||||||
|
Ok(RestoreCheckpointAck {
|
||||||
|
nonce: config.nonce.clone(),
|
||||||
|
stage: config.stage,
|
||||||
|
runtime_id,
|
||||||
|
worker_id: record.worker_id,
|
||||||
|
operation_id,
|
||||||
|
commit: match expected {
|
||||||
|
RestoreCommitFact::NotCommitted => RestoreCheckpointCommit::NotCommitted,
|
||||||
|
RestoreCommitFact::Committed => RestoreCheckpointCommit::Committed,
|
||||||
|
RestoreCommitFact::LegacyUnknown => unreachable!(),
|
||||||
|
},
|
||||||
|
receipt_absent: true,
|
||||||
|
pid: std::process::id(),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,304 @@
|
|||||||
|
use super::*;
|
||||||
|
use crate::restore_checkpoints::{
|
||||||
|
RestoreCheckpointAck, RestoreCheckpointCommit, RestoreCheckpointConfig, RestoreCheckpointStage,
|
||||||
|
};
|
||||||
|
use std::io::{BufRead, BufReader, Write};
|
||||||
|
use std::net::TcpListener;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn checkpoint_stages_ack_canonical_decision_and_eof_resumes_same_operation_once() {
|
||||||
|
// Component scope: durable Runtime with a substituted execution backend.
|
||||||
|
// Real prepared Controllers and process death are owned by the E2E suite.
|
||||||
|
for (stage, release) in [
|
||||||
|
(RestoreCheckpointStage::PreparedNotCommitted, "release"),
|
||||||
|
(RestoreCheckpointStage::CommittedPreterminal, "eof"),
|
||||||
|
(RestoreCheckpointStage::CommittedPreterminal, "timeout"),
|
||||||
|
(RestoreCheckpointStage::ActivatedPreterminal, "release"),
|
||||||
|
] {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let backend = Arc::new(TestExecutionBackend::default());
|
||||||
|
let runtime = Runtime::with_fs_store_and_execution_backend(
|
||||||
|
FsRuntimeStoreOptions::new(dir.path().join("runtime"))
|
||||||
|
.with_runtime_id("checkpoint-runtime"),
|
||||||
|
backend.clone(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
runtime.bind_runtime_identity("checkpoint-runtime").unwrap();
|
||||||
|
runtime.store_config_bundle(test_bundle()).unwrap();
|
||||||
|
let worker = runtime
|
||||||
|
.create_worker(task_request("checkpoint original operation"))
|
||||||
|
.unwrap();
|
||||||
|
runtime.stop_worker(&worker.worker_ref, None).unwrap();
|
||||||
|
let request = runtime.test_restore_request(&worker.worker_ref);
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||||
|
runtime
|
||||||
|
.install_test_restore_checkpoint(RestoreCheckpointConfig {
|
||||||
|
endpoint: listener.local_addr().unwrap(),
|
||||||
|
worker_id: worker.worker_id,
|
||||||
|
stage,
|
||||||
|
nonce: "runtime-stage-test".into(),
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
let cloned = runtime.clone();
|
||||||
|
let worker_ref = worker.worker_ref.clone();
|
||||||
|
let sent_request = request.clone();
|
||||||
|
let (sent, received) = std::sync::mpsc::channel();
|
||||||
|
let task = std::thread::spawn(move || {
|
||||||
|
sent.send(cloned.restore_worker_operation(&worker_ref, sent_request))
|
||||||
|
.unwrap();
|
||||||
|
});
|
||||||
|
listener.set_nonblocking(true).unwrap();
|
||||||
|
let stream = tokio::runtime::Builder::new_current_thread()
|
||||||
|
.enable_all()
|
||||||
|
.build()
|
||||||
|
.unwrap()
|
||||||
|
.block_on(async {
|
||||||
|
let listener = tokio::net::TcpListener::from_std(listener).unwrap();
|
||||||
|
let (stream, _) = tokio::time::timeout(Duration::from_secs(5), listener.accept())
|
||||||
|
.await
|
||||||
|
.expect("checkpoint did not connect")
|
||||||
|
.unwrap();
|
||||||
|
stream.into_std().unwrap()
|
||||||
|
});
|
||||||
|
stream.set_nonblocking(false).unwrap();
|
||||||
|
stream
|
||||||
|
.set_read_timeout(Some(Duration::from_secs(3)))
|
||||||
|
.unwrap();
|
||||||
|
let mut reader = BufReader::new(stream);
|
||||||
|
let mut line = String::new();
|
||||||
|
reader.read_line(&mut line).unwrap();
|
||||||
|
let ack: RestoreCheckpointAck = serde_json::from_str(&line).unwrap();
|
||||||
|
assert_eq!(ack.stage, stage);
|
||||||
|
assert_eq!(ack.nonce, "runtime-stage-test");
|
||||||
|
assert_eq!(ack.runtime_id, "checkpoint-runtime");
|
||||||
|
assert_eq!(ack.worker_id, worker.worker_id);
|
||||||
|
assert_eq!(ack.pid, std::process::id());
|
||||||
|
assert!(ack.receipt_absent);
|
||||||
|
assert_eq!(
|
||||||
|
ack.commit,
|
||||||
|
match stage {
|
||||||
|
RestoreCheckpointStage::PreparedNotCommitted =>
|
||||||
|
RestoreCheckpointCommit::NotCommitted,
|
||||||
|
RestoreCheckpointStage::CommittedPreterminal
|
||||||
|
| RestoreCheckpointStage::ActivatedPreterminal => RestoreCheckpointCommit::Committed,
|
||||||
|
}
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
backend.restore_operation_ids.lock().unwrap().as_slice(),
|
||||||
|
&[ack.operation_id]
|
||||||
|
);
|
||||||
|
if stage == RestoreCheckpointStage::ActivatedPreterminal {
|
||||||
|
assert_eq!(backend.activation_operation_ids.lock().unwrap().as_slice(), &[ack.operation_id]);
|
||||||
|
} else {
|
||||||
|
assert!(backend.activation_operation_ids.lock().unwrap().is_empty());
|
||||||
|
}
|
||||||
|
assert!(received.try_recv().is_err());
|
||||||
|
let canonical = runtime_store(&runtime)
|
||||||
|
.read_durable_worker_record(&worker.worker_ref)
|
||||||
|
.unwrap();
|
||||||
|
let id = canonical.restore_guard.active_request_id.as_ref().unwrap();
|
||||||
|
assert_eq!(id, &request.request_id);
|
||||||
|
assert!(canonical.restore_guard.owners[id].receipt.is_none());
|
||||||
|
// Release, EOF and timeout all resume the original lifecycle owner.
|
||||||
|
let start = std::time::Instant::now();
|
||||||
|
if release == "release" {
|
||||||
|
reader.get_mut().write_all(b"R").unwrap();
|
||||||
|
} else if release == "eof" {
|
||||||
|
reader
|
||||||
|
.get_mut()
|
||||||
|
.shutdown(std::net::Shutdown::Write)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
let result = received
|
||||||
|
.recv_timeout(Duration::from_secs(12))
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert!(start.elapsed() < Duration::from_secs(11));
|
||||||
|
task.join().unwrap();
|
||||||
|
assert_eq!(result.state, WorkerRestoreState::Accepted);
|
||||||
|
assert_eq!(
|
||||||
|
backend.activation_operation_ids.lock().unwrap().as_slice(),
|
||||||
|
&[ack.operation_id]
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.restore_worker_operation(&worker.worker_ref, request)
|
||||||
|
.unwrap(),
|
||||||
|
result
|
||||||
|
);
|
||||||
|
assert_eq!(*backend.restore_count.lock().unwrap(), 1);
|
||||||
|
assert!(runtime.lock().unwrap().restore_checkpoint.is_none());
|
||||||
|
runtime.stop_worker(&worker.worker_ref, None).unwrap();
|
||||||
|
let next = runtime.test_restore_request(&worker.worker_ref);
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.restore_worker_operation(&worker.worker_ref, next)
|
||||||
|
.unwrap()
|
||||||
|
.state,
|
||||||
|
WorkerRestoreState::Accepted
|
||||||
|
);
|
||||||
|
assert_eq!(*backend.restore_count.lock().unwrap(), 2);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn checkpoint_disabled_mismatched_or_unreachable_does_not_change_restore_result() {
|
||||||
|
for mode in ["disabled", "mismatch", "unreachable"] {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let backend = Arc::new(TestExecutionBackend::default());
|
||||||
|
let runtime = Runtime::with_fs_store_and_execution_backend(
|
||||||
|
FsRuntimeStoreOptions::new(dir.path().join("runtime"))
|
||||||
|
.with_runtime_id("checkpoint-runtime"),
|
||||||
|
backend.clone(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
runtime.bind_runtime_identity("checkpoint-runtime").unwrap();
|
||||||
|
runtime.store_config_bundle(test_bundle()).unwrap();
|
||||||
|
let worker = runtime
|
||||||
|
.create_worker(task_request("checkpoint absent"))
|
||||||
|
.unwrap();
|
||||||
|
runtime.stop_worker(&worker.worker_ref, None).unwrap();
|
||||||
|
let request = runtime.test_restore_request(&worker.worker_ref);
|
||||||
|
if mode != "disabled" {
|
||||||
|
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||||
|
let endpoint = listener.local_addr().unwrap();
|
||||||
|
drop(listener);
|
||||||
|
runtime
|
||||||
|
.install_test_restore_checkpoint(RestoreCheckpointConfig {
|
||||||
|
endpoint,
|
||||||
|
worker_id: if mode == "mismatch" {
|
||||||
|
WorkerId::now_v7()
|
||||||
|
} else {
|
||||||
|
worker.worker_id
|
||||||
|
},
|
||||||
|
stage: RestoreCheckpointStage::PreparedNotCommitted,
|
||||||
|
nonce: "absent-test".into(),
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
let result = runtime
|
||||||
|
.restore_worker_operation(&worker.worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(result.state, WorkerRestoreState::Accepted, "{mode}");
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.restore_worker_operation(&worker.worker_ref, request)
|
||||||
|
.unwrap(),
|
||||||
|
result
|
||||||
|
);
|
||||||
|
assert_eq!(*backend.restore_count.lock().unwrap(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
runtime.lock().unwrap().restore_checkpoint.is_some(),
|
||||||
|
mode == "mismatch"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn checkpoint_verification_requires_canonical_worker_operation_commit_and_absent_receipt() {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let root = dir.path().join("runtime");
|
||||||
|
let backend = Arc::new(TestExecutionBackend::default());
|
||||||
|
let runtime = Runtime::with_fs_store_and_execution_backend(
|
||||||
|
FsRuntimeStoreOptions::new(&root).with_runtime_id("checkpoint-runtime"),
|
||||||
|
backend.clone(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
runtime.bind_runtime_identity("checkpoint-runtime").unwrap();
|
||||||
|
runtime.store_config_bundle(test_bundle()).unwrap();
|
||||||
|
let worker = runtime
|
||||||
|
.create_worker(task_request("checkpoint validation"))
|
||||||
|
.unwrap();
|
||||||
|
runtime.stop_worker(&worker.worker_ref, None).unwrap();
|
||||||
|
let request = runtime.test_restore_request(&worker.worker_ref);
|
||||||
|
*backend.restore_result.lock().unwrap() = Some(WorkerExecutionSpawnResult::Errored(
|
||||||
|
WorkerExecutionResult::errored(WorkerExecutionOperation::Restore, "not yet connected"),
|
||||||
|
));
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.restore_worker_operation(&worker.worker_ref, request.clone())
|
||||||
|
.unwrap()
|
||||||
|
.state,
|
||||||
|
WorkerRestoreState::ReconciliationRequired
|
||||||
|
);
|
||||||
|
let operation = backend.restore_operation_ids.lock().unwrap()[0];
|
||||||
|
let mut config = RestoreCheckpointConfig {
|
||||||
|
endpoint: "127.0.0.1:1234".parse().unwrap(),
|
||||||
|
worker_id: worker.worker_id,
|
||||||
|
stage: RestoreCheckpointStage::PreparedNotCommitted,
|
||||||
|
nonce: "verification-test".into(),
|
||||||
|
};
|
||||||
|
let path = root
|
||||||
|
.join("workers")
|
||||||
|
.join(worker.worker_id.to_string())
|
||||||
|
.join("worker.json");
|
||||||
|
let before = std::fs::read(&path).unwrap();
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.restore_checkpoint_ack(&config, &worker.worker_ref, operation)
|
||||||
|
.is_ok()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.restore_checkpoint_ack(
|
||||||
|
&config,
|
||||||
|
&worker.worker_ref,
|
||||||
|
WorkerLifecycleOperationId::new()
|
||||||
|
)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
config.worker_id = WorkerId::now_v7();
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.restore_checkpoint_ack(&config, &worker.worker_ref, operation)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
config.worker_id = worker.worker_id;
|
||||||
|
config.stage = RestoreCheckpointStage::CommittedPreterminal;
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.restore_checkpoint_ack(&config, &worker.worker_ref, operation)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
config.stage = RestoreCheckpointStage::ActivatedPreterminal;
|
||||||
|
assert!(runtime.restore_checkpoint_ack(&config, &worker.worker_ref, operation).is_err());
|
||||||
|
config.stage = RestoreCheckpointStage::PreparedNotCommitted;
|
||||||
|
runtime_store(&runtime).fail_next_worker_reconcile_sync();
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.restore_checkpoint_ack(&config, &worker.worker_ref, operation)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read(&path).unwrap(),
|
||||||
|
before,
|
||||||
|
"verification must not write authority"
|
||||||
|
);
|
||||||
|
// In-memory equality must not hide an unreadable canonical aggregate.
|
||||||
|
std::fs::write(&path, b"unreadable canonical aggregate").unwrap();
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.restore_checkpoint_ack(&config, &worker.worker_ref, operation)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read(&path).unwrap(),
|
||||||
|
b"unreadable canonical aggregate"
|
||||||
|
);
|
||||||
|
std::fs::write(&path, before).unwrap();
|
||||||
|
backend.restore_result.lock().unwrap().take();
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.restore_worker_operation(&worker.worker_ref, request)
|
||||||
|
.unwrap()
|
||||||
|
.state,
|
||||||
|
WorkerRestoreState::Accepted
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
runtime
|
||||||
|
.restore_checkpoint_ack(&config, &worker.worker_ref, operation)
|
||||||
|
.is_err(),
|
||||||
|
"terminal operation cannot ack a preterminal checkpoint"
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,522 @@
|
|||||||
|
//! Runtime-owned continuation of an admitted restore, independent of its caller.
|
||||||
|
//!
|
||||||
|
//! Completion callbacks only signal a condition variable. They never acquire a
|
||||||
|
//! Runtime/lifecycle lock and are never joined by this driver while holding one.
|
||||||
|
//! Each step revalidates the canonical owner under the existing Worker lock;
|
||||||
|
//! waiting/backoff and driver teardown happen outside it. A repeated obstruction
|
||||||
|
//! parks this owner with its reason, rather than spinning or inventing rollback.
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
use crate::execution::RestoreRecoveryDisposition;
|
||||||
|
use std::collections::HashMap;
|
||||||
|
use std::sync::{Condvar, Weak};
|
||||||
|
use std::thread::JoinHandle;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
type Key = (WorkerRef, WorkerLifecycleOperationId);
|
||||||
|
|
||||||
|
#[derive(Debug, Default)]
|
||||||
|
pub(super) struct RestoreContinuations {
|
||||||
|
owners: Mutex<HashMap<Key, Arc<Continuation>>>,
|
||||||
|
#[cfg(test)]
|
||||||
|
paused: std::sync::atomic::AtomicBool,
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(super) terminal_receipt_gate:
|
||||||
|
Mutex<Option<crate::execution::RestoreTestGate<WorkerLifecycleOperationId>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Default)]
|
||||||
|
struct Schedule {
|
||||||
|
signaled: bool,
|
||||||
|
cancelled: bool,
|
||||||
|
finished: bool,
|
||||||
|
blocked_reason: Option<String>,
|
||||||
|
parked: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug, Default)]
|
||||||
|
struct Continuation {
|
||||||
|
schedule: Mutex<Schedule>,
|
||||||
|
wake: Condvar,
|
||||||
|
task: Mutex<Option<JoinHandle<()>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl RestoreContinuations {
|
||||||
|
pub(super) fn signal(&self, worker_ref: &WorkerRef, operation: WorkerLifecycleOperationId) {
|
||||||
|
if let Some(owner) = self
|
||||||
|
.owners
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||||
|
.get(&(worker_ref.clone(), operation))
|
||||||
|
{
|
||||||
|
let mut schedule = owner
|
||||||
|
.schedule
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||||
|
schedule.signaled = true;
|
||||||
|
owner.wake.notify_all();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for RestoreContinuations {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
let owners = self
|
||||||
|
.owners
|
||||||
|
.get_mut()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||||
|
for owner in owners.values() {
|
||||||
|
let mut schedule = owner
|
||||||
|
.schedule
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||||
|
schedule.cancelled = true;
|
||||||
|
owner.wake.notify_all();
|
||||||
|
}
|
||||||
|
for owner in owners.values() {
|
||||||
|
if let Some(task) = owner
|
||||||
|
.task
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||||
|
.take()
|
||||||
|
{
|
||||||
|
if task.thread().id() != std::thread::current().id() {
|
||||||
|
let _ = task.join();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) struct ContinuationPause(Arc<RestoreContinuations>);
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
impl Drop for ContinuationPause {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
self.0
|
||||||
|
.paused
|
||||||
|
.store(false, std::sync::atomic::Ordering::Release);
|
||||||
|
for owner in self.0.owners.lock().unwrap().values() {
|
||||||
|
owner.schedule.lock().unwrap().signaled = true;
|
||||||
|
owner.wake.notify_all();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
enum Step {
|
||||||
|
Finished,
|
||||||
|
Waiting,
|
||||||
|
Contended,
|
||||||
|
Retry(String),
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Runtime {
|
||||||
|
pub(super) fn restore_repair_authorizer(
|
||||||
|
&self,
|
||||||
|
worker_ref: WorkerRef,
|
||||||
|
) -> Arc<dyn Fn(WorkerLifecycleOperationId) -> Result<(), RuntimeError> + Send + Sync> {
|
||||||
|
let inner = Arc::downgrade(&self.inner);
|
||||||
|
Arc::new(move |operation| {
|
||||||
|
let inner = inner.upgrade().ok_or(RuntimeError::RuntimeStopped)?;
|
||||||
|
// The factory caller owns the lifecycle lock. Never reacquire it
|
||||||
|
// here: it may be awaiting this exact factory task.
|
||||||
|
let mut state = inner.lock().map_err(|_| RuntimeError::StatePoisoned)?;
|
||||||
|
state.ensure_running()?;
|
||||||
|
state.refresh_uncertain_worker_write(&worker_ref)?;
|
||||||
|
let worker = state.worker(&worker_ref)?;
|
||||||
|
state.ensure_subject_not_reserved(&worker_ref, false)?;
|
||||||
|
if worker.pending_stop.is_some()
|
||||||
|
|| worker.pending_restore.is_none_or(|pending| {
|
||||||
|
pending.operation_id != operation
|
||||||
|
|| pending.commit == RestoreCommitFact::LegacyUnknown
|
||||||
|
})
|
||||||
|
{
|
||||||
|
return Err(RuntimeError::RestoreObservationConflict {
|
||||||
|
worker_id: worker_ref.worker_id,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if let Some(owner) = worker
|
||||||
|
.restore_guard
|
||||||
|
.active_request_id
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|id| worker.restore_guard.owners.get(id))
|
||||||
|
{
|
||||||
|
owner.validate_workdir_preparation_authority(worker)?;
|
||||||
|
}
|
||||||
|
if worker
|
||||||
|
.pending_restore
|
||||||
|
.is_some_and(|pending| pending.source_repair_possible)
|
||||||
|
{
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let mut candidate = worker.clone();
|
||||||
|
candidate
|
||||||
|
.pending_restore
|
||||||
|
.as_mut()
|
||||||
|
.expect("validated restore owner")
|
||||||
|
.source_repair_possible = true;
|
||||||
|
// Grant before touching another canonical store. Unknown durability
|
||||||
|
// denies repair until authoritative reconciliation; no old-memory write.
|
||||||
|
state.persist_restore_worker_record(&candidate)?;
|
||||||
|
state.workers.insert(worker_ref.worker_id, candidate);
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) fn test_wait_restore_continuation_parked(
|
||||||
|
&self,
|
||||||
|
worker_ref: &WorkerRef,
|
||||||
|
operation: WorkerLifecycleOperationId,
|
||||||
|
timeout: Duration,
|
||||||
|
) -> String {
|
||||||
|
let owner = self
|
||||||
|
.restore_continuations
|
||||||
|
.owners
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.get(&(worker_ref.clone(), operation))
|
||||||
|
.cloned()
|
||||||
|
.unwrap();
|
||||||
|
let (schedule, waited) = owner
|
||||||
|
.wake
|
||||||
|
.wait_timeout_while(owner.schedule.lock().unwrap(), timeout, |s| {
|
||||||
|
!s.parked && !s.finished
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
!waited.timed_out() && schedule.parked,
|
||||||
|
"continuation did not park: {:?}",
|
||||||
|
schedule.blocked_reason
|
||||||
|
);
|
||||||
|
schedule.blocked_reason.clone().expect("parked reason")
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Gate autonomous steps when a component test observes a single fault
|
||||||
|
/// boundary. Dedicated continuation tests run without this gate.
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) fn test_pause_restore_continuation(&self) -> ContinuationPause {
|
||||||
|
assert!(
|
||||||
|
!self
|
||||||
|
.restore_continuations
|
||||||
|
.paused
|
||||||
|
.swap(true, std::sync::atomic::Ordering::AcqRel)
|
||||||
|
);
|
||||||
|
ContinuationPause(self.restore_continuations.clone())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
pub(crate) fn test_wait_restore_continuation(
|
||||||
|
&self,
|
||||||
|
worker_ref: &WorkerRef,
|
||||||
|
operation: WorkerLifecycleOperationId,
|
||||||
|
timeout: Duration,
|
||||||
|
) {
|
||||||
|
let owner = self
|
||||||
|
.restore_continuations
|
||||||
|
.owners
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.get(&(worker_ref.clone(), operation))
|
||||||
|
.cloned()
|
||||||
|
.expect("continuation owner");
|
||||||
|
let schedule = owner.schedule.lock().unwrap();
|
||||||
|
let (schedule, waited) = owner
|
||||||
|
.wake
|
||||||
|
.wait_timeout_while(schedule, timeout, |s| !s.finished && !s.cancelled)
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
!waited.timed_out() && schedule.finished,
|
||||||
|
"restore continuation did not settle: {:?}",
|
||||||
|
schedule.blocked_reason
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn ensure_restore_continuation(
|
||||||
|
&self,
|
||||||
|
worker_ref: &WorkerRef,
|
||||||
|
) -> Result<(), RuntimeError> {
|
||||||
|
let operation = {
|
||||||
|
let state = self.lock()?;
|
||||||
|
let worker = state.worker(worker_ref)?;
|
||||||
|
worker
|
||||||
|
.pending_restore
|
||||||
|
.map(|pending| pending.operation_id)
|
||||||
|
.or_else(|| {
|
||||||
|
worker
|
||||||
|
.restore_guard
|
||||||
|
.active_request_id
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|id| worker.restore_guard.owners.get(id))
|
||||||
|
.filter(|owner| owner.receipt.is_none())
|
||||||
|
.map(|owner| owner.operation_id)
|
||||||
|
})
|
||||||
|
};
|
||||||
|
if let Some(operation) = operation {
|
||||||
|
self.start_restore_continuation(worker_ref.clone(), operation)?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn start_restore_continuation(
|
||||||
|
&self,
|
||||||
|
worker_ref: WorkerRef,
|
||||||
|
operation: WorkerLifecycleOperationId,
|
||||||
|
) -> Result<(), RuntimeError> {
|
||||||
|
{
|
||||||
|
let state = self.lock()?;
|
||||||
|
let Some(backend) = &state.execution_backend else {
|
||||||
|
return Ok(());
|
||||||
|
};
|
||||||
|
if !backend.supports_restore_continuation()
|
||||||
|
|| backend
|
||||||
|
.restore_recovery_disposition(operation, &worker_ref)
|
||||||
|
.is_none()
|
||||||
|
{
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let key = (worker_ref.clone(), operation);
|
||||||
|
let mut owners = self
|
||||||
|
.restore_continuations
|
||||||
|
.owners
|
||||||
|
.lock()
|
||||||
|
.map_err(|_| RuntimeError::StatePoisoned)?;
|
||||||
|
if owners.contains_key(&key) {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let owner = Arc::new(Continuation::default());
|
||||||
|
let retained = owner.clone();
|
||||||
|
let state = Arc::downgrade(&self.inner);
|
||||||
|
let operations = Arc::downgrade(&self.worker_operations);
|
||||||
|
let continuations = Arc::downgrade(&self.restore_continuations);
|
||||||
|
let task = std::thread::Builder::new()
|
||||||
|
.name("restore-continuation".into())
|
||||||
|
.spawn(move || {
|
||||||
|
run(
|
||||||
|
state,
|
||||||
|
operations,
|
||||||
|
continuations,
|
||||||
|
retained,
|
||||||
|
worker_ref,
|
||||||
|
operation,
|
||||||
|
);
|
||||||
|
})
|
||||||
|
.map_err(|error| {
|
||||||
|
RuntimeError::InvalidRequest(format!(
|
||||||
|
"Restore continuation could not start: {error}"
|
||||||
|
))
|
||||||
|
})?;
|
||||||
|
*owner.task.lock().map_err(|_| RuntimeError::StatePoisoned)? = Some(task);
|
||||||
|
owners.insert(key, owner);
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn continue_restore_operation(
|
||||||
|
&self,
|
||||||
|
worker_ref: &WorkerRef,
|
||||||
|
operation: WorkerLifecycleOperationId,
|
||||||
|
) -> Result<Step, RuntimeError> {
|
||||||
|
#[cfg(test)]
|
||||||
|
if self
|
||||||
|
.restore_continuations
|
||||||
|
.paused
|
||||||
|
.load(std::sync::atomic::Ordering::Acquire)
|
||||||
|
{
|
||||||
|
return Ok(Step::Contended);
|
||||||
|
}
|
||||||
|
let lease = self.worker_operation_lock(worker_ref.worker_id)?;
|
||||||
|
// Another caller may be waiting for a backend task. Do not queue a join
|
||||||
|
// of that caller/driver while retaining the lock it needs to complete.
|
||||||
|
let _guard = match lease.try_lock() {
|
||||||
|
Ok(guard) => guard,
|
||||||
|
Err(std::sync::TryLockError::WouldBlock) => return Ok(Step::Contended),
|
||||||
|
Err(_) => return Err(RuntimeError::StatePoisoned),
|
||||||
|
};
|
||||||
|
let stop_is_owned = {
|
||||||
|
let mut state = self.lock()?;
|
||||||
|
state.ensure_running()?;
|
||||||
|
state.refresh_uncertain_worker_write(worker_ref)?;
|
||||||
|
let worker = state.worker(worker_ref)?;
|
||||||
|
worker.pending_stop.is_some()
|
||||||
|
&& worker
|
||||||
|
.pending_restore
|
||||||
|
.is_some_and(|pending| pending.operation_id == operation)
|
||||||
|
};
|
||||||
|
if stop_is_owned {
|
||||||
|
// Stop supersedes publication, not ownership of late resources.
|
||||||
|
// Retain this same continuation until its cleanup is confirmed.
|
||||||
|
self.reconcile_pending_stop_under_lock(worker_ref)?;
|
||||||
|
return Ok(Step::Finished);
|
||||||
|
}
|
||||||
|
let (backend, pending) = {
|
||||||
|
let mut state = self.lock()?;
|
||||||
|
state.ensure_running()?;
|
||||||
|
state.refresh_uncertain_worker_write(worker_ref)?;
|
||||||
|
let worker = state.worker(worker_ref)?;
|
||||||
|
if worker.pending_stop.is_some() {
|
||||||
|
return Ok(Step::Finished);
|
||||||
|
}
|
||||||
|
let owner = worker
|
||||||
|
.restore_guard
|
||||||
|
.active_request_id
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|id| worker.restore_guard.owners.get(id));
|
||||||
|
if owner.is_some_and(|owner| owner.operation_id != operation || owner.receipt.is_some())
|
||||||
|
{
|
||||||
|
return Ok(Step::Finished);
|
||||||
|
}
|
||||||
|
let Some(pending) = worker.pending_restore else {
|
||||||
|
return Ok(Step::Finished);
|
||||||
|
};
|
||||||
|
if pending.operation_id != operation {
|
||||||
|
return Ok(Step::Finished);
|
||||||
|
}
|
||||||
|
if pending.commit == RestoreCommitFact::LegacyUnknown {
|
||||||
|
return Ok(Step::Retry(
|
||||||
|
"Saved Restore has no authoritative commit evidence".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
state.ensure_subject_not_reserved(worker_ref, false)?;
|
||||||
|
if let Some(owner) = owner {
|
||||||
|
owner.validate_workdir_preparation_authority(worker)?;
|
||||||
|
}
|
||||||
|
let Some(backend) = state.execution_backend.clone() else {
|
||||||
|
return Ok(Step::Finished);
|
||||||
|
};
|
||||||
|
(backend, pending)
|
||||||
|
};
|
||||||
|
match backend.restore_recovery_disposition(operation, worker_ref) {
|
||||||
|
None => return Ok(Step::Finished),
|
||||||
|
Some(RestoreRecoveryDisposition::InFlight) => return Ok(Step::Waiting),
|
||||||
|
Some(RestoreRecoveryDisposition::Discard)
|
||||||
|
if pending.commit == RestoreCommitFact::NotCommitted =>
|
||||||
|
{
|
||||||
|
match backend.discard_uncommitted_restore(operation, worker_ref) {
|
||||||
|
WorkerExecutionSpawnResult::RolledBack(result) => {
|
||||||
|
let receipt = RuntimeWorkerRestoreResult::failed(
|
||||||
|
WorkerRestoreState::RolledBack,
|
||||||
|
"worker_restore_rolled_back",
|
||||||
|
result.message_or_default(),
|
||||||
|
);
|
||||||
|
#[cfg(feature = "fs-store")]
|
||||||
|
self.lock()?
|
||||||
|
.record_restore_failure(worker_ref, &result, &receipt)?;
|
||||||
|
#[cfg(not(feature = "fs-store"))]
|
||||||
|
self.settle_restore_rollback(worker_ref, &receipt)?;
|
||||||
|
return Ok(Step::Finished);
|
||||||
|
}
|
||||||
|
result => {
|
||||||
|
return Ok(Step::Retry(format!(
|
||||||
|
"Restore cleanup remains unresolved: {result:?}"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Some(_) => {}
|
||||||
|
}
|
||||||
|
let result = self.restore_admitted_worker_under_lock(worker_ref, pending.mode)?;
|
||||||
|
if backend.restore_recovery_disposition(operation, worker_ref)
|
||||||
|
== Some(RestoreRecoveryDisposition::InFlight)
|
||||||
|
{
|
||||||
|
return Ok(Step::Waiting);
|
||||||
|
}
|
||||||
|
if result.state == WorkerRestoreState::ReconciliationRequired {
|
||||||
|
Ok(Step::Retry(result.message.unwrap_or_else(|| {
|
||||||
|
"Restore reconciliation remains pending".into()
|
||||||
|
})))
|
||||||
|
} else {
|
||||||
|
Ok(Step::Finished)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn run(
|
||||||
|
inner: Weak<Mutex<RuntimeState>>,
|
||||||
|
operations: Weak<Mutex<BTreeMap<WorkerId, Arc<Mutex<()>>>>>,
|
||||||
|
continuations: Weak<RestoreContinuations>,
|
||||||
|
owner: Arc<Continuation>,
|
||||||
|
worker_ref: WorkerRef,
|
||||||
|
operation: WorkerLifecycleOperationId,
|
||||||
|
) {
|
||||||
|
let mut repeated = 0u32;
|
||||||
|
let mut last_reason = None;
|
||||||
|
loop {
|
||||||
|
let (Some(inner), Some(worker_operations), Some(restore_continuations)) = (
|
||||||
|
inner.upgrade(),
|
||||||
|
operations.upgrade(),
|
||||||
|
continuations.upgrade(),
|
||||||
|
) else {
|
||||||
|
break;
|
||||||
|
};
|
||||||
|
owner
|
||||||
|
.schedule
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||||
|
.parked = false;
|
||||||
|
let runtime = Runtime {
|
||||||
|
inner,
|
||||||
|
worker_operations,
|
||||||
|
restore_continuations,
|
||||||
|
};
|
||||||
|
let step = runtime.continue_restore_operation(&worker_ref, operation);
|
||||||
|
// No lifecycle lock or strong Runtime/registry reference survives waiting.
|
||||||
|
drop(runtime);
|
||||||
|
let mut schedule = owner
|
||||||
|
.schedule
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||||
|
if schedule.cancelled {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
let contended = matches!(&step, Ok(Step::Contended));
|
||||||
|
let reason = match step {
|
||||||
|
Ok(Step::Finished) => {
|
||||||
|
schedule.finished = true;
|
||||||
|
owner.wake.notify_all();
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
Ok(Step::Waiting) | Ok(Step::Contended) => None,
|
||||||
|
Ok(Step::Retry(reason)) => Some(reason),
|
||||||
|
Err(error) => Some(sanitize_worker_create_failure_message(&error.to_string())),
|
||||||
|
};
|
||||||
|
if let Some(reason) = &reason {
|
||||||
|
repeated = repeated.saturating_add(1);
|
||||||
|
last_reason = Some(reason.clone());
|
||||||
|
schedule.blocked_reason = Some(reason.clone());
|
||||||
|
}
|
||||||
|
// Completed failed attempts can have signaled while this step joined
|
||||||
|
// them. Do not let our own unchanged failure reset the retry budget.
|
||||||
|
if reason.is_some() {
|
||||||
|
schedule.signaled = false;
|
||||||
|
}
|
||||||
|
if (!contended && reason.is_none()) || repeated >= 4 {
|
||||||
|
if repeated >= 4 {
|
||||||
|
schedule.parked = true;
|
||||||
|
owner.wake.notify_all();
|
||||||
|
tracing::warn!(worker_id = %worker_ref.worker_id, operation_id = %operation, reason = ?last_reason,
|
||||||
|
"Restore continuation parked after bounded attempts; operation remains unresolved");
|
||||||
|
}
|
||||||
|
schedule = owner
|
||||||
|
.wake
|
||||||
|
.wait_while(schedule, |s| !s.signaled && !s.cancelled)
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner);
|
||||||
|
} else {
|
||||||
|
let delay = Duration::from_millis(100 * (1u64 << repeated.min(4)));
|
||||||
|
schedule = owner
|
||||||
|
.wake
|
||||||
|
.wait_timeout_while(schedule, delay, |s| !s.signaled && !s.cancelled)
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||||
|
.0;
|
||||||
|
}
|
||||||
|
if schedule.cancelled {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
if schedule.signaled {
|
||||||
|
schedule.signaled = false;
|
||||||
|
// A completion wakes this same owner, not a new request. Its next
|
||||||
|
// observed result, not the signal itself, determines progress.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,872 @@
|
|||||||
|
//! Offline support for the approved, attachment-free stopped Worker fixture.
|
||||||
|
//!
|
||||||
|
//! Enabled only by `test-support`. This is not an import API: extraction accepts a
|
||||||
|
//! current-schema, settled, single-Segment aggregate; installation accepts only a
|
||||||
|
//! fresh Runtime directory containing its newly initialized `auth.toml`. No
|
||||||
|
//! Runtime or execution backend is constructed. Original Session bytes are never
|
||||||
|
//! rewritten. The caller must separately review/attest the artificial conversation;
|
||||||
|
//! structural checks and secret-marker checks cannot prove arbitrary prose safe.
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
use crate::catalog::ProfileSourceArchiveSource;
|
||||||
|
use crate::workspace_issuer::{
|
||||||
|
WorkspaceIssuerTrustRecord, WorkspaceIssuerTrustState, validate_workspace_issuer_trust_records,
|
||||||
|
};
|
||||||
|
use session_store::{
|
||||||
|
SegmentId, SessionId, WorkerActiveSegmentRef, WorkerMetadata, WorkerSessionStore,
|
||||||
|
};
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "test_support_tests.rs"]
|
||||||
|
mod tests;
|
||||||
|
|
||||||
|
const TEMPLATE_VERSION: u32 = 1;
|
||||||
|
const MAX_SESSION_BYTES: usize = 8 * 1024 * 1024;
|
||||||
|
|
||||||
|
/// Portable fixture, intentionally without Debug (Session material is private).
|
||||||
|
/// Fields are private and every deserialized template is validated on install.
|
||||||
|
/// Serialize this value to retain the fixture; do not serialize its source aggregate.
|
||||||
|
#[derive(Clone, Serialize, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
pub struct StoppedWorkerTemplate {
|
||||||
|
template_version: u32,
|
||||||
|
runtime_schema_version: u32,
|
||||||
|
runtime_id: String,
|
||||||
|
workspace_id: String,
|
||||||
|
worker_id: WorkerId,
|
||||||
|
created_at_ms: Option<u64>,
|
||||||
|
session_id: SessionId,
|
||||||
|
segment_id: SegmentId,
|
||||||
|
session_manifest: Vec<u8>,
|
||||||
|
segment_jsonl: Vec<u8>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
struct SessionManifest {
|
||||||
|
schema_version: u32,
|
||||||
|
session_id: SessionId,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read-only projection: never retain source key material in a template.
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
struct AuthIdentity {
|
||||||
|
identity_id: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
struct SourceAuth {
|
||||||
|
identity: AuthIdentity,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
struct InitializedAuth {
|
||||||
|
identity: crate::auth::RuntimeIdentityMaterial,
|
||||||
|
workspace_issuers: Vec<WorkspaceIssuerTrustRecord>,
|
||||||
|
#[serde(default)]
|
||||||
|
archived_legacy_issuers: Vec<String>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
struct GuardState {
|
||||||
|
active_request_id: Option<String>,
|
||||||
|
owners: BTreeMap<String, GuardOwner>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
struct GuardOwner {
|
||||||
|
receipt: Option<serde::de::IgnoredAny>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reject(message: &'static str) -> RuntimeError {
|
||||||
|
RuntimeError::InvalidRequest(format!("stopped Worker fixture: {message}"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn io<T>(result: std::io::Result<T>) -> Result<T, RuntimeError> {
|
||||||
|
result.map_err(|_| reject("filesystem operation failed"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn decode<T: serde::de::DeserializeOwned>(bytes: &[u8]) -> Result<T, RuntimeError> {
|
||||||
|
serde_json::from_slice(bytes).map_err(|_| reject("invalid typed document"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn no_symlink_components(path: &Path) -> Result<(), RuntimeError> {
|
||||||
|
let mut current = PathBuf::new();
|
||||||
|
for part in path.components() {
|
||||||
|
if matches!(part, std::path::Component::ParentDir) {
|
||||||
|
return Err(reject("parent traversal is unsupported"));
|
||||||
|
}
|
||||||
|
current.push(part);
|
||||||
|
let metadata = io(fs::symlink_metadata(¤t))?;
|
||||||
|
if metadata.file_type().is_symlink() {
|
||||||
|
return Err(reject("symlinks are unsupported"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn check_tree(path: &Path, budget: &mut usize) -> Result<(), RuntimeError> {
|
||||||
|
if *budget == 0 {
|
||||||
|
return Err(reject("file count exceeds fixture limit"));
|
||||||
|
}
|
||||||
|
*budget -= 1;
|
||||||
|
let metadata = io(fs::symlink_metadata(path))?;
|
||||||
|
if metadata.is_dir() {
|
||||||
|
for entry in io(fs::read_dir(path))? {
|
||||||
|
check_tree(&io(entry)?.path(), budget)?;
|
||||||
|
}
|
||||||
|
} else if !metadata.is_file() {
|
||||||
|
return Err(reject("symlinks and special files are unsupported"));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn regular_file(path: &Path) -> Result<(), RuntimeError> {
|
||||||
|
no_symlink_components(path)?;
|
||||||
|
if !io(fs::symlink_metadata(path))?.is_file() {
|
||||||
|
return Err(reject("expected a regular file"));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn bytes(path: &Path) -> Result<Vec<u8>, RuntimeError> {
|
||||||
|
regular_file(path)?;
|
||||||
|
let mut options = OpenOptions::new();
|
||||||
|
options.read(true);
|
||||||
|
#[cfg(target_os = "linux")]
|
||||||
|
{
|
||||||
|
use std::os::unix::fs::OpenOptionsExt;
|
||||||
|
options.custom_flags(libc::O_NOATIME | libc::O_NOFOLLOW);
|
||||||
|
}
|
||||||
|
let file = io(options.open(path))?;
|
||||||
|
let mut result = Vec::new();
|
||||||
|
io(file
|
||||||
|
.take(MAX_SESSION_BYTES as u64 + 1)
|
||||||
|
.read_to_end(&mut result))?;
|
||||||
|
if result.len() > MAX_SESSION_BYTES {
|
||||||
|
return Err(reject("file exceeds fixture limit"));
|
||||||
|
}
|
||||||
|
Ok(result)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn text(path: &Path) -> Result<String, RuntimeError> {
|
||||||
|
String::from_utf8(bytes(path)?).map_err(|_| reject("invalid UTF-8 document"))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn portable_id(id: &str) -> bool {
|
||||||
|
!id.is_empty()
|
||||||
|
&& id.len() <= 256
|
||||||
|
&& id
|
||||||
|
.bytes()
|
||||||
|
.all(|c| c.is_ascii_alphanumeric() || b"-_".contains(&c))
|
||||||
|
}
|
||||||
|
|
||||||
|
impl StoppedWorkerTemplate {
|
||||||
|
pub fn runtime_id(&self) -> &str {
|
||||||
|
&self.runtime_id
|
||||||
|
}
|
||||||
|
pub fn workspace_id(&self) -> &str {
|
||||||
|
&self.workspace_id
|
||||||
|
}
|
||||||
|
pub fn worker_id(&self) -> WorkerId {
|
||||||
|
self.worker_id
|
||||||
|
}
|
||||||
|
pub fn worker_name(&self) -> String {
|
||||||
|
runtime_worker_name(self.worker_id)
|
||||||
|
}
|
||||||
|
pub fn session_id(&self) -> SessionId {
|
||||||
|
self.session_id
|
||||||
|
}
|
||||||
|
pub fn segment_id(&self) -> SegmentId {
|
||||||
|
self.segment_id
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Extract without migration, rewriting metadata, or copying credentials,
|
||||||
|
/// resolved manifests, execution-operation receipts, or run directories.
|
||||||
|
/// The source Runtime must be offline; its existing owner lock is held for
|
||||||
|
/// the entire read. The path must be `<runtime>/workers/<WorkerId>`.
|
||||||
|
pub fn extract(worker_dir: &Path) -> Result<Self, RuntimeError> {
|
||||||
|
no_symlink_components(worker_dir)?;
|
||||||
|
let workers = worker_dir
|
||||||
|
.parent()
|
||||||
|
.ok_or_else(|| reject("invalid aggregate path"))?;
|
||||||
|
if workers.file_name().and_then(|s| s.to_str()) != Some(WORKERS_DIR) {
|
||||||
|
return Err(reject("invalid aggregate path"));
|
||||||
|
}
|
||||||
|
let root = workers
|
||||||
|
.parent()
|
||||||
|
.ok_or_else(|| reject("invalid Runtime path"))?;
|
||||||
|
// Require the existing lock, rather than creating a file in the source.
|
||||||
|
let lock_path = root.with_file_name(format!(
|
||||||
|
".{}.runtime-owner.lock",
|
||||||
|
root.file_name()
|
||||||
|
.and_then(|s| s.to_str())
|
||||||
|
.ok_or_else(|| reject("invalid Runtime path"))?
|
||||||
|
));
|
||||||
|
regular_file(&lock_path)?;
|
||||||
|
let lock = io(OpenOptions::new().read(true).open(lock_path))?;
|
||||||
|
if !io(FileExt::try_lock_shared(&lock))? {
|
||||||
|
return Err(reject("source Runtime is not offline"));
|
||||||
|
}
|
||||||
|
let _lock = RuntimeStoreOwnerLock { file: lock };
|
||||||
|
check_tree(worker_dir, &mut 256)?;
|
||||||
|
let snapshot: RuntimeSnapshot = decode(&bytes(&root.join(RUNTIME_FILE))?)?;
|
||||||
|
snapshot
|
||||||
|
.validate(&root.join(RUNTIME_FILE))
|
||||||
|
.map_err(|_| reject("unsupported Runtime schema"))?;
|
||||||
|
if !snapshot.subject_create_recovery_fences.is_empty()
|
||||||
|
|| !snapshot.subject_releases.is_empty()
|
||||||
|
{
|
||||||
|
return Err(reject("source has lifecycle operations"));
|
||||||
|
}
|
||||||
|
let record: WorkerAggregateRecord = decode(&bytes(&worker_dir.join(WORKER_FILE))?)?;
|
||||||
|
let record = record
|
||||||
|
.validate(&worker_dir.join(WORKER_FILE))
|
||||||
|
.map_err(|_| reject("invalid Worker schema or identity"))?;
|
||||||
|
let guard: GuardState = decode(
|
||||||
|
&serde_json::to_vec(&record.restore_guard)
|
||||||
|
.map_err(|_| reject("invalid restore guard"))?,
|
||||||
|
)?;
|
||||||
|
if record.status != WorkerStatus::Stopped
|
||||||
|
|| guard.active_request_id.is_some()
|
||||||
|
|| guard.owners.values().any(|owner| owner.receipt.is_none())
|
||||||
|
{
|
||||||
|
return Err(reject("Worker is not settled and stopped"));
|
||||||
|
}
|
||||||
|
let PersistedWorkerExecutionState::Available(execution) = &record.execution_state else {
|
||||||
|
return Err(reject("pending or unavailable execution metadata"));
|
||||||
|
};
|
||||||
|
// A stopped aggregate may retain the empty historical binding marker.
|
||||||
|
// It is deliberately not copied to the installed Worker.
|
||||||
|
if execution.restore_intent != WorkerRestoreIntent::Explicit {
|
||||||
|
return Err(reject("Worker requires explicit restore"));
|
||||||
|
}
|
||||||
|
if !record.workdir_attachments.is_empty() || !record.logical_workdir_attachments.is_empty()
|
||||||
|
{
|
||||||
|
return Err(reject("attachments are unsupported"));
|
||||||
|
}
|
||||||
|
check_request(&execution.request, false)?;
|
||||||
|
let workspace_id = record
|
||||||
|
.workspace_id
|
||||||
|
.ok_or_else(|| reject("missing Workspace identity"))?;
|
||||||
|
let metadata: WorkerMetadata = decode(&bytes(&worker_dir.join(WORKER_METADATA_FILE))?)?;
|
||||||
|
crate::worker_backend::validate_fixture_restore_metadata(&metadata, &execution.request)
|
||||||
|
.map_err(|_| reject("invalid source restore Manifest or settings"))?;
|
||||||
|
if metadata.worker_name != runtime_worker_name(record.worker_id)
|
||||||
|
|| metadata.workspace_id.as_deref() != Some(&workspace_id)
|
||||||
|
|| !metadata.spawned_children.is_empty()
|
||||||
|
|| !metadata.reclaimed_children.is_empty()
|
||||||
|
|| !metadata.peers.is_empty()
|
||||||
|
|| metadata.subjektiv_session_attribution.is_some()
|
||||||
|
{
|
||||||
|
return Err(reject("unsupported or inconsistent Worker metadata"));
|
||||||
|
}
|
||||||
|
let active = metadata
|
||||||
|
.active
|
||||||
|
.ok_or_else(|| reject("missing active Session"))?;
|
||||||
|
let segment_id = active
|
||||||
|
.segment_id
|
||||||
|
.ok_or_else(|| reject("unmaterialized Session"))?;
|
||||||
|
let source_auth: SourceAuth = toml::from_str(&text(&root.join("auth.toml"))?)
|
||||||
|
.map_err(|_| reject("missing Runtime identity"))?;
|
||||||
|
let session = worker_dir.join("session");
|
||||||
|
let fixture = Self {
|
||||||
|
template_version: TEMPLATE_VERSION,
|
||||||
|
runtime_schema_version: SCHEMA_VERSION,
|
||||||
|
runtime_id: source_auth.identity.identity_id,
|
||||||
|
workspace_id,
|
||||||
|
worker_id: record.worker_id,
|
||||||
|
created_at_ms: record.created_at_ms,
|
||||||
|
session_id: active.session_id,
|
||||||
|
segment_id,
|
||||||
|
session_manifest: bytes(&session.join("session.json"))?,
|
||||||
|
segment_jsonl: bytes(&session.join("segments").join(format!("{segment_id}.jsonl")))?,
|
||||||
|
};
|
||||||
|
fixture.validate_shape()?;
|
||||||
|
fixture.validate_session(&session)?;
|
||||||
|
Ok(fixture)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Install after normal identity/trust initialization, before Runtime start.
|
||||||
|
/// `request` and `manifest` supply NEW backend/config/profile references and
|
||||||
|
/// freshly resolved policy, not a launch. The saved source Manifest is never
|
||||||
|
/// reused. `manifest.worker.name` must equal `worker_name()`.
|
||||||
|
/// Only `auth.toml` may already exist. The owner lock fences live Runtimes;
|
||||||
|
/// staged files are synced and published without overwriting any Worker.
|
||||||
|
/// An interrupted publication fails closed as an incomplete Runtime layout.
|
||||||
|
pub fn install(
|
||||||
|
&self,
|
||||||
|
options: &FsRuntimeStoreOptions,
|
||||||
|
request: CreateWorkerRequest,
|
||||||
|
manifest: &manifest::WorkerManifest,
|
||||||
|
) -> Result<(), RuntimeError> {
|
||||||
|
self.validate_shape()?;
|
||||||
|
check_request(&request, true)?;
|
||||||
|
if manifest.worker.name != self.worker_name() {
|
||||||
|
return Err(reject("resolved Manifest Worker name mismatch"));
|
||||||
|
}
|
||||||
|
let mut metadata = WorkerMetadata::new(
|
||||||
|
self.worker_name(),
|
||||||
|
Some(WorkerActiveSegmentRef {
|
||||||
|
session_id: self.session_id,
|
||||||
|
segment_id: Some(self.segment_id),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.with_workspace_id(&self.workspace_id);
|
||||||
|
metadata.resolved_manifest_snapshot = Some(
|
||||||
|
manifest::write_persisted_worker_manifest_snapshot(manifest)
|
||||||
|
.map_err(|_| reject("invalid resolved Manifest"))?,
|
||||||
|
);
|
||||||
|
crate::worker_backend::validate_fixture_restore_metadata(&metadata, &request)
|
||||||
|
.map_err(|_| reject("resolved Manifest does not match fresh request settings"))?;
|
||||||
|
if options.runtime_id != self.runtime_id
|
||||||
|
|| request.worker_id != self.worker_id
|
||||||
|
|| request
|
||||||
|
.workspace_api
|
||||||
|
.as_ref()
|
||||||
|
.map(|api| api.workspace_id.as_str())
|
||||||
|
!= Some(&self.workspace_id)
|
||||||
|
{
|
||||||
|
return Err(reject("installation identity mismatch"));
|
||||||
|
}
|
||||||
|
no_symlink_components(&options.root)?;
|
||||||
|
check_tree(&options.root, &mut 256)?;
|
||||||
|
// Validate the lock location before the ordinary owner-lock opener.
|
||||||
|
let lock_path = options.root.with_file_name(format!(
|
||||||
|
".{}.runtime-owner.lock",
|
||||||
|
options
|
||||||
|
.root
|
||||||
|
.file_name()
|
||||||
|
.and_then(|s| s.to_str())
|
||||||
|
.ok_or_else(|| reject("invalid Runtime path"))?
|
||||||
|
));
|
||||||
|
if fs::symlink_metadata(&lock_path).is_ok() {
|
||||||
|
regular_file(&lock_path)?;
|
||||||
|
}
|
||||||
|
let lock = acquire_runtime_store_owner_lock(&options.root)?;
|
||||||
|
validate_fresh_runtime_store_layout(&options.root)
|
||||||
|
.map_err(|_| reject("destination is not fresh; refusing overwrite"))?;
|
||||||
|
let auth: InitializedAuth = toml::from_str(&text(&options.root.join("auth.toml"))?)
|
||||||
|
.map_err(|_| reject("initialize Runtime identity and trust before install"))?;
|
||||||
|
if auth.identity.identity_id != self.runtime_id
|
||||||
|
|| !auth.archived_legacy_issuers.is_empty()
|
||||||
|
|| auth.identity.signing_key().is_err()
|
||||||
|
{
|
||||||
|
return Err(reject("invalid destination Runtime identity"));
|
||||||
|
}
|
||||||
|
validate_workspace_issuer_trust_records(&auth.workspace_issuers)
|
||||||
|
.map_err(|_| reject("invalid destination Workspace trust"))?;
|
||||||
|
let api = request
|
||||||
|
.workspace_api
|
||||||
|
.as_ref()
|
||||||
|
.expect("validated Workspace binding");
|
||||||
|
if !auth.workspace_issuers.iter().any(|trust| {
|
||||||
|
trust.workspace_id == self.workspace_id
|
||||||
|
&& trust.backend_url == api.base_url
|
||||||
|
&& trust.state == WorkspaceIssuerTrustState::Active
|
||||||
|
}) {
|
||||||
|
return Err(reject("request does not match initialized Workspace trust"));
|
||||||
|
}
|
||||||
|
let stage = options
|
||||||
|
.root
|
||||||
|
.join(format!(".fixture-{}", uuid::Uuid::now_v7()));
|
||||||
|
io(fs::create_dir(&stage))?;
|
||||||
|
let store = FsRuntimeStore {
|
||||||
|
root: stage.clone(),
|
||||||
|
_owner_lock: Some(lock),
|
||||||
|
worker_write_fault: Arc::new(AtomicU8::new(AtomicWriteFault::None as u8)),
|
||||||
|
#[cfg(test)]
|
||||||
|
worker_reconcile_sync_fault: Default::default(),
|
||||||
|
#[cfg(test)]
|
||||||
|
worker_reconcile_read_fault: Default::default(),
|
||||||
|
#[cfg(test)]
|
||||||
|
worker_reconcile_counts: Default::default(),
|
||||||
|
};
|
||||||
|
let result = (|| {
|
||||||
|
let worker = PersistedWorkerRecord {
|
||||||
|
restore_guard: crate::runtime::RestoreGuard::default(),
|
||||||
|
worker_ref: WorkerRef::new(self.worker_id),
|
||||||
|
worker_id: self.worker_id,
|
||||||
|
profile: request.profile.clone(),
|
||||||
|
display_name: request.display_name.clone(),
|
||||||
|
profile_source: request.profile_source.reference(),
|
||||||
|
config_bundle: request.config_bundle.clone(),
|
||||||
|
created_at_ms: self.created_at_ms,
|
||||||
|
status: WorkerStatus::Stopped,
|
||||||
|
last_finished_submission_request_id: None,
|
||||||
|
workspace_id: Some(self.workspace_id.clone()),
|
||||||
|
workdir_attachments: Vec::new(),
|
||||||
|
logical_workdir_attachments: Vec::new(),
|
||||||
|
execution_state: PersistedWorkerExecutionState::Available(
|
||||||
|
PersistedWorkerExecution {
|
||||||
|
request,
|
||||||
|
binding: None,
|
||||||
|
restore_intent: WorkerRestoreIntent::Explicit,
|
||||||
|
},
|
||||||
|
),
|
||||||
|
};
|
||||||
|
WorkerAggregateRecord::from_persisted(&worker)
|
||||||
|
.validate(&store.worker_dir(&self.worker_id).join(WORKER_FILE))
|
||||||
|
.map_err(|_| reject("invalid installation request"))?;
|
||||||
|
store.write_worker_record(&worker)?;
|
||||||
|
let worker_dir = store.worker_dir(&self.worker_id);
|
||||||
|
atomic_write_json(
|
||||||
|
&worker_dir.join(WORKER_METADATA_FILE),
|
||||||
|
&metadata,
|
||||||
|
"write fixture metadata",
|
||||||
|
)?;
|
||||||
|
let session = worker_dir.join("session");
|
||||||
|
io(fs::create_dir_all(session.join("segments")))?;
|
||||||
|
write_new(&session.join("session.json"), &self.session_manifest)?;
|
||||||
|
write_new(
|
||||||
|
&session
|
||||||
|
.join("segments")
|
||||||
|
.join(format!("{}.jsonl", self.segment_id)),
|
||||||
|
&self.segment_jsonl,
|
||||||
|
)?;
|
||||||
|
self.validate_session(&session)?;
|
||||||
|
store.write_runtime_snapshot(&PersistedRuntimeState {
|
||||||
|
subject_create_recovery_fences: BTreeMap::new(),
|
||||||
|
subject_releases: Vec::new(),
|
||||||
|
// Match normal store initialization: this is catalog availability,
|
||||||
|
// not a process or Worker execution start.
|
||||||
|
display_name: options.display_name.clone(),
|
||||||
|
status: RuntimeStatus::Running,
|
||||||
|
next_diagnostic_id: 1,
|
||||||
|
workers: BTreeMap::new(),
|
||||||
|
workspace_owners: BTreeMap::new(),
|
||||||
|
diagnostics: Vec::new(),
|
||||||
|
})?;
|
||||||
|
let loaded = store.load_runtime_state()?;
|
||||||
|
if loaded.workers.len() != 1 || !loaded.diagnostics.is_empty() {
|
||||||
|
return Err(reject("staged aggregate failed ordinary store roundtrip"));
|
||||||
|
}
|
||||||
|
sync_runtime_tree(&stage)?;
|
||||||
|
// No destination existed when checked under the owner lock. Hard-link
|
||||||
|
// the snapshot last with create-new semantics; never replace a file.
|
||||||
|
// Reserve the destination catalog with create-new semantics. Even
|
||||||
|
// an empty preexisting workers directory must never be replaced.
|
||||||
|
let destination_workers = options.root.join(WORKERS_DIR);
|
||||||
|
io(fs::create_dir(&destination_workers))?;
|
||||||
|
io(fs::rename(
|
||||||
|
store.worker_dir(&self.worker_id),
|
||||||
|
destination_workers.join(self.worker_id.to_string()),
|
||||||
|
))?;
|
||||||
|
sync_directory(&destination_workers, "publish stopped Worker aggregate")?;
|
||||||
|
io(fs::hard_link(
|
||||||
|
stage.join(RUNTIME_FILE),
|
||||||
|
options.root.join(RUNTIME_FILE),
|
||||||
|
))?;
|
||||||
|
sync_directory(&options.root, "publish stopped fixture")?;
|
||||||
|
Ok(())
|
||||||
|
})();
|
||||||
|
// Only remove our private staging directory, never existing destination data.
|
||||||
|
let cleanup = fs::remove_dir_all(&stage);
|
||||||
|
result?;
|
||||||
|
io(cleanup)?;
|
||||||
|
sync_directory(&options.root, "finish stopped fixture")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_shape(&self) -> Result<(), RuntimeError> {
|
||||||
|
// Version 1 templates carry only settled Session material, never a
|
||||||
|
// lifecycle operation. Schema 12 -> 13 changes pending decision evidence,
|
||||||
|
// not that payload. Accept the fixed schema-12 artifact read-only and
|
||||||
|
// install through the current aggregate owner; never rewrite its bytes.
|
||||||
|
if self.template_version != TEMPLATE_VERSION
|
||||||
|
|| !matches!(
|
||||||
|
self.runtime_schema_version,
|
||||||
|
RESTORE_ADMISSION_SCHEMA_VERSION | SCHEMA_VERSION
|
||||||
|
)
|
||||||
|
{
|
||||||
|
return Err(reject("unsupported template schema"));
|
||||||
|
}
|
||||||
|
if !portable_id(&self.runtime_id)
|
||||||
|
|| !portable_id(&self.workspace_id)
|
||||||
|
|| self.session_id.get_version_num() != 7
|
||||||
|
|| self.segment_id.get_version_num() != 7
|
||||||
|
|| self.session_manifest.len() > MAX_SESSION_BYTES
|
||||||
|
|| self.segment_jsonl.len() > MAX_SESSION_BYTES
|
||||||
|
{
|
||||||
|
return Err(reject("invalid portable identity or size"));
|
||||||
|
}
|
||||||
|
let manifest: SessionManifest = decode(&self.session_manifest)?;
|
||||||
|
if manifest.session_id != self.session_id
|
||||||
|
|| manifest.schema_version == 0
|
||||||
|
|| !self.segment_jsonl.ends_with(b"\n")
|
||||||
|
{
|
||||||
|
return Err(reject("Session identity or committed boundary mismatch"));
|
||||||
|
}
|
||||||
|
// Read-only JSON traversal, never mutation/reconstruction of Session bytes.
|
||||||
|
// Reject known secret/host markers even in otherwise valid typed records.
|
||||||
|
for line in self
|
||||||
|
.segment_jsonl
|
||||||
|
.split(|b| *b == b'\n')
|
||||||
|
.filter(|line| !line.is_empty())
|
||||||
|
{
|
||||||
|
let _: PortableJson = decode(line)?;
|
||||||
|
validate_preserved_record(line)?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_history_entry(
|
||||||
|
&self,
|
||||||
|
entry: &session_store::LoggedHistoryEntry,
|
||||||
|
ids: &mut std::collections::BTreeSet<String>,
|
||||||
|
) -> Result<(), RuntimeError> {
|
||||||
|
if !portable_id(&entry.metadata.entry_id.0)
|
||||||
|
|| !ids.insert(entry.metadata.entry_id.0.clone())
|
||||||
|
{
|
||||||
|
return Err(reject("duplicate or invalid entry identity"));
|
||||||
|
}
|
||||||
|
if entry
|
||||||
|
.metadata
|
||||||
|
.input_metadata
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|metadata| {
|
||||||
|
metadata.feature_payload().is_some() || metadata.public_item().is_some()
|
||||||
|
})
|
||||||
|
{
|
||||||
|
// These are opaque owner-specific payloads, not part of the narrow
|
||||||
|
// artificial conversation contract. Never preserve unchecked JSON.
|
||||||
|
return Err(reject("Feature history payloads are unsupported"));
|
||||||
|
}
|
||||||
|
if !matches!(
|
||||||
|
entry.item,
|
||||||
|
session_store::LoggedItem::Message { .. }
|
||||||
|
| session_store::LoggedItem::AssistantResponseBoundary { .. }
|
||||||
|
) {
|
||||||
|
return Err(reject("only text conversation records are supported"));
|
||||||
|
}
|
||||||
|
if let session_store::LoggedSessionHistoryOrigin::ModelOutput { worker } =
|
||||||
|
&entry.metadata.origin
|
||||||
|
{
|
||||||
|
// Session producer IDs identify the original execution, not the
|
||||||
|
// Workspace catalog Worker. Preserve them; do not rebind/rewrite them.
|
||||||
|
if !portable_id(&worker.worker_id)
|
||||||
|
|| worker
|
||||||
|
.workspace_id
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|id| id != &self.workspace_id)
|
||||||
|
|| worker
|
||||||
|
.runtime_id
|
||||||
|
.as_ref()
|
||||||
|
.is_some_and(|id| id != &self.runtime_id)
|
||||||
|
{
|
||||||
|
return Err(reject("Session origin identity mismatch"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn validate_session(&self, root: &Path) -> Result<(), RuntimeError> {
|
||||||
|
for entry in io(fs::read_dir(root))? {
|
||||||
|
let entry = io(entry)?;
|
||||||
|
match entry.file_name().to_str() {
|
||||||
|
Some("session.json" | "segments") => {}
|
||||||
|
Some("artifacts") => check_empty_artifacts(&entry.path())?,
|
||||||
|
_ => return Err(reject("unsupported Session files")),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let entries = io(fs::read_dir(root.join("segments")))?.collect::<Result<Vec<_>, _>>();
|
||||||
|
let entries = io(entries)?;
|
||||||
|
if entries.len() != 1
|
||||||
|
|| entries[0].file_name() != format!("{}.jsonl", self.segment_id).as_str()
|
||||||
|
{
|
||||||
|
return Err(reject("only the fixed single-Segment fixture is supported"));
|
||||||
|
}
|
||||||
|
let store = WorkerSessionStore::open_read_only(root)
|
||||||
|
.map_err(|_| reject("unsupported Session schema"))?;
|
||||||
|
let logs = store
|
||||||
|
.read_all_read_only_bounded(self.session_id, self.segment_id, MAX_SESSION_BYTES as u64)
|
||||||
|
.map_err(|_| reject("invalid canonical Session records"))?
|
||||||
|
.0;
|
||||||
|
if !matches!(logs.first(), Some(session_store::LogEntry::AnnotatedSegmentStart {
|
||||||
|
session_id, system_prompt: Some(_), forked_from: None, compacted_from: None, ..
|
||||||
|
}) if *session_id == self.session_id)
|
||||||
|
{
|
||||||
|
return Err(reject(
|
||||||
|
"missing system prompt, invalid Segment seed, or unsupported lineage",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let mut ids = std::collections::BTreeSet::new();
|
||||||
|
for (index, log) in logs.iter().enumerate() {
|
||||||
|
use session_store::LogEntry;
|
||||||
|
match log {
|
||||||
|
LogEntry::AnnotatedSegmentStart { history, .. }
|
||||||
|
if index == 0 && history.is_empty() => {}
|
||||||
|
LogEntry::AnnotatedUserInput {
|
||||||
|
history,
|
||||||
|
extensions,
|
||||||
|
..
|
||||||
|
} => {
|
||||||
|
for extension in extensions {
|
||||||
|
validate_settled_activation(extension)?;
|
||||||
|
}
|
||||||
|
for entry in history {
|
||||||
|
self.validate_history_entry(entry, &mut ids)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
LogEntry::AnnotatedAssistantItem { entry, .. } => {
|
||||||
|
self.validate_history_entry(entry, &mut ids)?
|
||||||
|
}
|
||||||
|
LogEntry::RunCompleted {
|
||||||
|
entry_id: Some(id), ..
|
||||||
|
} => {
|
||||||
|
if !portable_id(&id.0) || !ids.insert(id.0.clone()) {
|
||||||
|
return Err(reject("duplicate or invalid entry identity"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
LogEntry::Invoke { .. } | LogEntry::LlmUsage { .. } | LogEntry::TurnEnd { .. } => {}
|
||||||
|
_ => {
|
||||||
|
return Err(reject(
|
||||||
|
"unsupported conversation record or pending operation",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let state = session_store::collect_state(&logs);
|
||||||
|
if state.last_run_interrupted || state.active_run_turn_count.is_some() {
|
||||||
|
return Err(reject("Session has pending execution"));
|
||||||
|
}
|
||||||
|
for segments in &state.user_segments {
|
||||||
|
if segments
|
||||||
|
.iter()
|
||||||
|
.any(|segment| !matches!(segment, protocol::Segment::Text { .. }))
|
||||||
|
{
|
||||||
|
return Err(reject("Session attachments are unsupported"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Compare *fields*, not re-encoded bytes or scalar values (e.g. f32 JSON
|
||||||
|
// normalization). Serde's permissive/untagged/flattened compatibility readers
|
||||||
|
// can drop unknown data without surfacing it to an ignored-field callback. A
|
||||||
|
// retained field must survive the owner's typed projection at every depth.
|
||||||
|
// This intentionally accepts only canonical writer shapes: even an explicit
|
||||||
|
// empty optional field omitted by that writer is rejected, not repaired.
|
||||||
|
// The projection is validation-only; installation always writes original bytes.
|
||||||
|
fn validate_preserved_record(bytes: &[u8]) -> Result<(), RuntimeError> {
|
||||||
|
let original: serde_json::Value = decode(bytes)?;
|
||||||
|
let record: session_store::LogEntry = decode(bytes)?;
|
||||||
|
let projection =
|
||||||
|
serde_json::to_value(&record).map_err(|_| reject("invalid typed Session record"))?;
|
||||||
|
require_preserved_fields(&original, &projection)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn require_preserved_fields(
|
||||||
|
original: &serde_json::Value,
|
||||||
|
projection: &serde_json::Value,
|
||||||
|
) -> Result<(), RuntimeError> {
|
||||||
|
use serde_json::Value;
|
||||||
|
match (original, projection) {
|
||||||
|
(Value::Object(original), Value::Object(projection)) => {
|
||||||
|
for (key, value) in original {
|
||||||
|
let projected = projection
|
||||||
|
.get(key)
|
||||||
|
.ok_or_else(|| reject("unknown or noncanonical Session field"))?;
|
||||||
|
require_preserved_fields(value, projected)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
(Value::Array(original), Value::Array(projection))
|
||||||
|
if original.len() == projection.len() =>
|
||||||
|
{
|
||||||
|
for (value, projected) in original.iter().zip(projection) {
|
||||||
|
require_preserved_fields(value, projected)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
(Value::Null, Value::Null)
|
||||||
|
| (Value::Bool(_), Value::Bool(_))
|
||||||
|
| (Value::Number(_), Value::Number(_))
|
||||||
|
| (Value::String(_), Value::String(_)) => {}
|
||||||
|
_ => return Err(reject("noncanonical Session field shape")),
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
// Exact receipt wire fields owned by worker::PendingActivationState. Keep these
|
||||||
|
// fixture-only mirrors strict; do not change the production compatibility reader.
|
||||||
|
#[derive(Serialize, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
struct SubmissionReceipt {
|
||||||
|
submission_request_id: String,
|
||||||
|
source_namespace: String,
|
||||||
|
submission_id: String,
|
||||||
|
payload_digest: String,
|
||||||
|
disposition: protocol::SubmissionDisposition,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Serialize, Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
struct NotificationReceipt {
|
||||||
|
notification_request_id: String,
|
||||||
|
source_namespace: String,
|
||||||
|
payload_digest: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
// The approved conversation has committed submission checkpoints in its original
|
||||||
|
// Session records. Keep their receipt bytes, but reject any resumable queue or
|
||||||
|
// Feature invocation history. This projection confers no execution authority.
|
||||||
|
fn validate_settled_activation(
|
||||||
|
extension: &session_store::SessionExtension,
|
||||||
|
) -> Result<(), RuntimeError> {
|
||||||
|
#[derive(Deserialize)]
|
||||||
|
#[serde(deny_unknown_fields)]
|
||||||
|
struct Checkpoint {
|
||||||
|
#[serde(rename = "next_activation_sequence")]
|
||||||
|
_next_activation_sequence: u64,
|
||||||
|
activating: Option<serde::de::IgnoredAny>,
|
||||||
|
activating_notification: Vec<serde::de::IgnoredAny>,
|
||||||
|
pending: Vec<serde::de::IgnoredAny>,
|
||||||
|
pending_notifications: Vec<serde::de::IgnoredAny>,
|
||||||
|
#[serde(rename = "receipts")]
|
||||||
|
_receipts: Vec<SubmissionReceipt>,
|
||||||
|
#[serde(rename = "notification_receipts")]
|
||||||
|
_notification_receipts: Vec<NotificationReceipt>,
|
||||||
|
invocation_receipts: Vec<serde::de::IgnoredAny>,
|
||||||
|
invocation_recovery_blocked: bool,
|
||||||
|
}
|
||||||
|
if extension.domain != "worker.pending_activations.v1" {
|
||||||
|
return Err(reject("unsupported Session extension"));
|
||||||
|
}
|
||||||
|
let checkpoint: Checkpoint = serde_json::from_value(extension.payload.clone())
|
||||||
|
.map_err(|_| reject("invalid Session activation checkpoint"))?;
|
||||||
|
if checkpoint.activating.is_some()
|
||||||
|
|| !checkpoint.activating_notification.is_empty()
|
||||||
|
|| !checkpoint.pending.is_empty()
|
||||||
|
|| !checkpoint.pending_notifications.is_empty()
|
||||||
|
|| !checkpoint.invocation_receipts.is_empty()
|
||||||
|
|| checkpoint.invocation_recovery_blocked
|
||||||
|
{
|
||||||
|
return Err(reject(
|
||||||
|
"Session has pending activation or invocation history",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn check_request(request: &CreateWorkerRequest, install: bool) -> Result<(), RuntimeError> {
|
||||||
|
if !request.workdir_attachment_requests.is_empty()
|
||||||
|
|| !request.workdir_attachments.is_empty()
|
||||||
|
|| request.backend_job.is_some()
|
||||||
|
|| request.subjektiv_attached
|
||||||
|
|| request.worker_observation_enabled
|
||||||
|
|| !request.worker_observation_grants.is_empty()
|
||||||
|
|| (install && request.initial_input.is_some())
|
||||||
|
{
|
||||||
|
return Err(reject(
|
||||||
|
"unsupported capabilities, attachments, or initial input",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if install
|
||||||
|
&& (request.create_fingerprint.trim().is_empty()
|
||||||
|
|| request.config_bundle.is_none()
|
||||||
|
|| !matches!(
|
||||||
|
request.profile_source,
|
||||||
|
ProfileSourceArchiveSource::WorkspaceConfig { .. }
|
||||||
|
))
|
||||||
|
{
|
||||||
|
return Err(reject(
|
||||||
|
"fresh config/profile references and create fingerprint are required",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn check_empty_artifacts(root: &Path) -> Result<(), RuntimeError> {
|
||||||
|
for entry in io(fs::read_dir(root))? {
|
||||||
|
let entry = io(entry)?;
|
||||||
|
let metadata = io(entry.metadata())?;
|
||||||
|
if metadata.is_dir() {
|
||||||
|
check_empty_artifacts(&entry.path())?;
|
||||||
|
} else if entry.file_name() != ".aggregate.lock" || metadata.len() != 0 {
|
||||||
|
return Err(reject("Session artifacts are unsupported"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn write_new(path: &Path, bytes: &[u8]) -> Result<(), RuntimeError> {
|
||||||
|
let mut file = io(OpenOptions::new().write(true).create_new(true).open(path))?;
|
||||||
|
io(file.write_all(bytes))?;
|
||||||
|
io(file.sync_all())
|
||||||
|
}
|
||||||
|
|
||||||
|
// A streaming validation visitor, not a generic document available to callers.
|
||||||
|
struct PortableJson;
|
||||||
|
impl<'de> Deserialize<'de> for PortableJson {
|
||||||
|
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||||
|
struct Visitor;
|
||||||
|
impl<'de> serde::de::Visitor<'de> for Visitor {
|
||||||
|
type Value = PortableJson;
|
||||||
|
fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
f.write_str("portable fixture JSON")
|
||||||
|
}
|
||||||
|
fn visit_bool<E: serde::de::Error>(self, _: bool) -> Result<PortableJson, E> {
|
||||||
|
Ok(PortableJson)
|
||||||
|
}
|
||||||
|
fn visit_i64<E: serde::de::Error>(self, _: i64) -> Result<PortableJson, E> {
|
||||||
|
Ok(PortableJson)
|
||||||
|
}
|
||||||
|
fn visit_u64<E: serde::de::Error>(self, _: u64) -> Result<PortableJson, E> {
|
||||||
|
Ok(PortableJson)
|
||||||
|
}
|
||||||
|
fn visit_f64<E: serde::de::Error>(self, _: f64) -> Result<PortableJson, E> {
|
||||||
|
Ok(PortableJson)
|
||||||
|
}
|
||||||
|
fn visit_unit<E: serde::de::Error>(self) -> Result<PortableJson, E> {
|
||||||
|
Ok(PortableJson)
|
||||||
|
}
|
||||||
|
fn visit_str<E: serde::de::Error>(self, value: &str) -> Result<PortableJson, E> {
|
||||||
|
let lower = value.to_ascii_lowercase();
|
||||||
|
if [
|
||||||
|
"://",
|
||||||
|
"private key",
|
||||||
|
"bearer ",
|
||||||
|
"sk-",
|
||||||
|
"api_key",
|
||||||
|
"access_token",
|
||||||
|
"refresh_token",
|
||||||
|
"private_key",
|
||||||
|
"authorization",
|
||||||
|
]
|
||||||
|
.iter()
|
||||||
|
.any(|marker| lower.contains(marker))
|
||||||
|
|| value
|
||||||
|
.split_whitespace()
|
||||||
|
.any(|word| word.starts_with('/') || word.contains(":\\"))
|
||||||
|
{
|
||||||
|
return Err(E::custom("nonportable or sensitive fixture material"));
|
||||||
|
}
|
||||||
|
Ok(PortableJson)
|
||||||
|
}
|
||||||
|
fn visit_seq<A: serde::de::SeqAccess<'de>>(
|
||||||
|
self,
|
||||||
|
mut seq: A,
|
||||||
|
) -> Result<PortableJson, A::Error> {
|
||||||
|
while seq.next_element::<PortableJson>()?.is_some() {}
|
||||||
|
Ok(PortableJson)
|
||||||
|
}
|
||||||
|
fn visit_map<A: serde::de::MapAccess<'de>>(
|
||||||
|
self,
|
||||||
|
mut map: A,
|
||||||
|
) -> Result<PortableJson, A::Error> {
|
||||||
|
let mut keys = std::collections::BTreeSet::new();
|
||||||
|
while let Some(key) = map.next_key::<String>()? {
|
||||||
|
// A duplicate key can hide discarded data even inside an
|
||||||
|
// otherwise opaque extension payload. Never retain it.
|
||||||
|
if !keys.insert(key.clone()) {
|
||||||
|
return Err(serde::de::Error::custom("duplicate fixture JSON field"));
|
||||||
|
}
|
||||||
|
serde::de::Visitor::visit_str::<A::Error>(Visitor, &key)?;
|
||||||
|
map.next_value::<PortableJson>()?;
|
||||||
|
}
|
||||||
|
Ok(PortableJson)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
deserializer.deserialize_any(Visitor)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,919 @@
|
|||||||
|
use super::*;
|
||||||
|
use crate::auth::RuntimeIdentityMaterial;
|
||||||
|
use crate::catalog::{ProfileSelector, WorkspaceApiRef};
|
||||||
|
use crate::profile_archive::ProfileSourceGraphSummary;
|
||||||
|
use session_store::{LogEntry, Store};
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
|
||||||
|
const RUNTIME: &str = "fixture-runtime";
|
||||||
|
const WORKSPACE: &str = "fixture-workspace";
|
||||||
|
const NEW_URL: &str = "http://127.0.0.1:34567";
|
||||||
|
|
||||||
|
#[derive(Serialize)]
|
||||||
|
#[serde(untagged)]
|
||||||
|
enum ReceiptFixture<T> {
|
||||||
|
Valid(T),
|
||||||
|
Malformed(u8),
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Default, Serialize)]
|
||||||
|
struct CheckpointFixture {
|
||||||
|
next_activation_sequence: u64,
|
||||||
|
activating: Option<u8>,
|
||||||
|
activating_notification: Vec<u8>,
|
||||||
|
pending: Vec<u8>,
|
||||||
|
pending_notifications: Vec<u8>,
|
||||||
|
receipts: Vec<ReceiptFixture<SubmissionReceipt>>,
|
||||||
|
notification_receipts: Vec<ReceiptFixture<NotificationReceipt>>,
|
||||||
|
invocation_receipts: Vec<u8>,
|
||||||
|
invocation_recovery_blocked: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn settled_checkpoint() -> CheckpointFixture {
|
||||||
|
CheckpointFixture {
|
||||||
|
next_activation_sequence: 2,
|
||||||
|
receipts: vec![ReceiptFixture::Valid(SubmissionReceipt {
|
||||||
|
submission_request_id: "completed-submission-request".into(),
|
||||||
|
source_namespace: "fixture-source".into(),
|
||||||
|
submission_id: "completed-submission".into(),
|
||||||
|
payload_digest: "fixture-payload-digest".into(),
|
||||||
|
disposition: protocol::SubmissionDisposition::Started,
|
||||||
|
})],
|
||||||
|
notification_receipts: vec![ReceiptFixture::Valid(NotificationReceipt {
|
||||||
|
notification_request_id: "completed-notification-request".into(),
|
||||||
|
source_namespace: "fixture-source".into(),
|
||||||
|
payload_digest: "fixture-notification-digest".into(),
|
||||||
|
})],
|
||||||
|
..Default::default()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn checkpoint_extension(checkpoint: &CheckpointFixture) -> session_store::SessionExtension {
|
||||||
|
session_store::SessionExtension::new(
|
||||||
|
"worker.pending_activations.v1",
|
||||||
|
serde_json::to_value(checkpoint).unwrap(),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn worker_id() -> WorkerId {
|
||||||
|
"01a1657d-b762-7791-87ae-a49f28cadbce".parse().unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn resolved() -> manifest::WorkerManifest {
|
||||||
|
let mut config = manifest::WorkerManifestConfig::resolution_defaults();
|
||||||
|
config.worker.name = Some(runtime_worker_name(worker_id()));
|
||||||
|
manifest::WorkerManifest::try_from(config).unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn request() -> CreateWorkerRequest {
|
||||||
|
CreateWorkerRequest {
|
||||||
|
worker_id: worker_id(),
|
||||||
|
create_fingerprint: "new-create-fingerprint".into(),
|
||||||
|
profile: ProfileSelector::Builtin("default".into()),
|
||||||
|
display_name: Some("fixture".into()),
|
||||||
|
profile_source: ProfileSourceArchiveSource::WorkspaceConfig {
|
||||||
|
archive: ProfileSourceArchiveRef {
|
||||||
|
id: "new-profile".into(),
|
||||||
|
digest: "new-profile-digest".into(),
|
||||||
|
size_bytes: 1,
|
||||||
|
source_graph: ProfileSourceGraphSummary {
|
||||||
|
source_count: 1,
|
||||||
|
total_source_bytes: 1,
|
||||||
|
entrypoints: BTreeMap::from([("builtin:default".into(), "default.dcl".into())]),
|
||||||
|
import_count: 0,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
config_bundle: Some(ConfigBundleRef {
|
||||||
|
id: "new-config".into(),
|
||||||
|
digest: "new-config-digest".into(),
|
||||||
|
}),
|
||||||
|
workspace_api: Some(WorkspaceApiRef {
|
||||||
|
workspace_id: WORKSPACE.into(),
|
||||||
|
base_url: NEW_URL.into(),
|
||||||
|
}),
|
||||||
|
initial_input: None,
|
||||||
|
workdir_attachment_requests: Vec::new(),
|
||||||
|
workdir_attachments: Vec::new(),
|
||||||
|
worker_observation_enabled: false,
|
||||||
|
worker_observation_grants: Vec::new(),
|
||||||
|
memory_settings: Some(manifest::WorkspaceMemorySettingsSnapshot {
|
||||||
|
workspace_id: WORKSPACE.into(),
|
||||||
|
language: "en".into(),
|
||||||
|
}),
|
||||||
|
subjektiv_attached: false,
|
||||||
|
backend_job: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn initialize(root: &Path, runtime_id: &str, workspace_id: &str, url: &str) {
|
||||||
|
#[derive(Serialize)]
|
||||||
|
struct Auth {
|
||||||
|
identity: RuntimeIdentityMaterial,
|
||||||
|
workspace_issuers: Vec<WorkspaceIssuerTrustRecord>,
|
||||||
|
}
|
||||||
|
let issuer = RuntimeIdentityMaterial::generate("fixture-issuer").unwrap();
|
||||||
|
let digest = Sha256::digest(crate::auth::decode_public_key(&issuer.public_key).unwrap());
|
||||||
|
let fingerprint = format!(
|
||||||
|
"sha256:{}",
|
||||||
|
digest
|
||||||
|
.iter()
|
||||||
|
.map(|byte| format!("{byte:02x}"))
|
||||||
|
.collect::<String>()
|
||||||
|
);
|
||||||
|
let trust = WorkspaceIssuerTrustRecord::from_bundle(
|
||||||
|
server_api::WorkspacePublicIdentityBundle {
|
||||||
|
workspace_id: workspace_id.into(),
|
||||||
|
backend_url: url.into(),
|
||||||
|
key_id: "fixture-issuer".into(),
|
||||||
|
algorithm: "ed25519".into(),
|
||||||
|
public_key: issuer.public_key,
|
||||||
|
public_key_fingerprint: fingerprint,
|
||||||
|
},
|
||||||
|
"fixture-trust".into(),
|
||||||
|
100,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
fs::create_dir_all(root).unwrap();
|
||||||
|
fs::write(
|
||||||
|
root.join("auth.toml"),
|
||||||
|
toml::to_string(&Auth {
|
||||||
|
identity: RuntimeIdentityMaterial::generate(runtime_id).unwrap(),
|
||||||
|
workspace_issuers: vec![trust],
|
||||||
|
})
|
||||||
|
.unwrap(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn record(request: CreateWorkerRequest) -> PersistedWorkerRecord {
|
||||||
|
PersistedWorkerRecord {
|
||||||
|
restore_guard: crate::runtime::RestoreGuard::default(),
|
||||||
|
worker_ref: WorkerRef::new(request.worker_id),
|
||||||
|
worker_id: request.worker_id,
|
||||||
|
profile: request.profile.clone(),
|
||||||
|
display_name: request.display_name.clone(),
|
||||||
|
profile_source: request.profile_source.reference(),
|
||||||
|
config_bundle: request.config_bundle.clone(),
|
||||||
|
created_at_ms: Some(123456),
|
||||||
|
status: WorkerStatus::Stopped,
|
||||||
|
last_finished_submission_request_id: Some("old-submission-operation".into()),
|
||||||
|
workspace_id: Some(WORKSPACE.into()),
|
||||||
|
workdir_attachments: vec![],
|
||||||
|
logical_workdir_attachments: vec![],
|
||||||
|
execution_state: PersistedWorkerExecutionState::Available(PersistedWorkerExecution {
|
||||||
|
request,
|
||||||
|
binding: Some(PersistedWorkerExecutionBinding {}),
|
||||||
|
restore_intent: WorkerRestoreIntent::Explicit,
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn source() -> (tempfile::TempDir, PathBuf, StoppedWorkerTemplate) {
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let root = temp.path().join("runtime");
|
||||||
|
initialize(&root, RUNTIME, WORKSPACE, "http://127.0.0.1:23456");
|
||||||
|
drop(
|
||||||
|
crate::Runtime::with_fs_store(FsRuntimeStoreOptions::new(&root).with_runtime_id(RUNTIME))
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
let opened = FsRuntimeStore::open_or_create(root.clone(), RUNTIME).unwrap();
|
||||||
|
let mut old = request();
|
||||||
|
old.workspace_api.as_mut().unwrap().base_url = "http://127.0.0.1:23456".into();
|
||||||
|
old.config_bundle.as_mut().unwrap().id = "old-config".into();
|
||||||
|
opened.store.write_worker_record(&record(old)).unwrap();
|
||||||
|
let worker = opened.store.worker_dir(&worker_id());
|
||||||
|
let session_id = "01a1657d-b800-7777-8888-111111111111".parse().unwrap();
|
||||||
|
let segment_id = "01a1657d-b818-77c1-bd73-176172a4bd82".parse().unwrap();
|
||||||
|
let mut metadata = WorkerMetadata::new(
|
||||||
|
runtime_worker_name(worker_id()),
|
||||||
|
Some(WorkerActiveSegmentRef {
|
||||||
|
session_id,
|
||||||
|
segment_id: Some(segment_id),
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
.with_workspace_id(WORKSPACE)
|
||||||
|
.with_workspace_root(PathBuf::from("/old/host/path"));
|
||||||
|
metadata.resolved_manifest_snapshot =
|
||||||
|
Some(manifest::write_persisted_worker_manifest_snapshot(&resolved()).unwrap());
|
||||||
|
atomic_write_json(
|
||||||
|
&worker.join(WORKER_METADATA_FILE),
|
||||||
|
&metadata,
|
||||||
|
"test metadata",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let sessions = WorkerSessionStore::new(worker.join("session")).unwrap();
|
||||||
|
let history = session_store::LoggedHistoryEntry {
|
||||||
|
item: session_store::LoggedItem::Message {
|
||||||
|
role: session_store::LoggedRole::Assistant,
|
||||||
|
content: vec![session_store::LoggedContentPart::Text {
|
||||||
|
text: "artificial fixture reply".into(),
|
||||||
|
}],
|
||||||
|
},
|
||||||
|
metadata: session_store::LoggedSessionHistoryMetadata {
|
||||||
|
entry_id: session_store::LoggedSessionHistoryEntryId("fixed-entry-id".into()),
|
||||||
|
origin: session_store::LoggedSessionHistoryOrigin::ModelOutput {
|
||||||
|
worker: session_store::LoggedWorkerSubject {
|
||||||
|
workspace_id: Some(WORKSPACE.into()),
|
||||||
|
runtime_id: Some(RUNTIME.into()),
|
||||||
|
worker_id: worker_id().to_string(),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
derivation: None,
|
||||||
|
input_metadata: None,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
sessions
|
||||||
|
.create_segment(
|
||||||
|
session_id,
|
||||||
|
segment_id,
|
||||||
|
&[
|
||||||
|
LogEntry::AnnotatedSegmentStart {
|
||||||
|
ts: 123,
|
||||||
|
session_id,
|
||||||
|
system_prompt: Some("Harmless artificial fixture instructions.".into()),
|
||||||
|
config: Default::default(),
|
||||||
|
history: vec![],
|
||||||
|
forked_from: None,
|
||||||
|
compacted_from: None,
|
||||||
|
},
|
||||||
|
LogEntry::Invoke {
|
||||||
|
ts: 124,
|
||||||
|
trigger: protocol::InvokeKind::UserSend,
|
||||||
|
},
|
||||||
|
LogEntry::AnnotatedUserInput {
|
||||||
|
ts: 124,
|
||||||
|
segments: vec![protocol::Segment::text(
|
||||||
|
"Harmless artificial fixture input.",
|
||||||
|
)],
|
||||||
|
extensions: vec![checkpoint_extension(&settled_checkpoint())],
|
||||||
|
history: vec![session_store::LoggedHistoryEntry {
|
||||||
|
item: session_store::LoggedItem::Message {
|
||||||
|
role: session_store::LoggedRole::User,
|
||||||
|
content: vec![session_store::LoggedContentPart::Text {
|
||||||
|
text: "Harmless artificial fixture input.".into(),
|
||||||
|
}],
|
||||||
|
},
|
||||||
|
metadata: session_store::LoggedSessionHistoryMetadata {
|
||||||
|
entry_id: session_store::LoggedSessionHistoryEntryId(
|
||||||
|
"fixed-user-id".into(),
|
||||||
|
),
|
||||||
|
origin: session_store::LoggedSessionHistoryOrigin::BackendInstruction {
|
||||||
|
operation_id: None,
|
||||||
|
},
|
||||||
|
derivation: None,
|
||||||
|
input_metadata: None,
|
||||||
|
},
|
||||||
|
}],
|
||||||
|
},
|
||||||
|
LogEntry::AnnotatedAssistantItem {
|
||||||
|
ts: 125,
|
||||||
|
entry: history,
|
||||||
|
},
|
||||||
|
LogEntry::LlmUsage {
|
||||||
|
ts: 125,
|
||||||
|
history_len: 2,
|
||||||
|
input_total_tokens: 1,
|
||||||
|
cache_read_tokens: 0,
|
||||||
|
cache_write_tokens: 0,
|
||||||
|
output_tokens: 1,
|
||||||
|
},
|
||||||
|
LogEntry::TurnEnd {
|
||||||
|
ts: 126,
|
||||||
|
turn_count: 1,
|
||||||
|
},
|
||||||
|
LogEntry::RunCompleted {
|
||||||
|
entry_id: Some(session_store::LoggedSessionHistoryEntryId(
|
||||||
|
"fixed-terminal-id".into(),
|
||||||
|
)),
|
||||||
|
ts: 126,
|
||||||
|
interrupted: false,
|
||||||
|
result: agen::EngineResult::Finished,
|
||||||
|
active_run_turn_count: None,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
fs::create_dir_all(worker.join("runs/old-run")).unwrap();
|
||||||
|
fs::write(worker.join("runs/old-run/private.log"), "not exported").unwrap();
|
||||||
|
drop(opened);
|
||||||
|
let template = StoppedWorkerTemplate::extract(&worker).unwrap();
|
||||||
|
(temp, worker, template)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn destination(template: &StoppedWorkerTemplate) -> (tempfile::TempDir, FsRuntimeStoreOptions) {
|
||||||
|
let temp = tempfile::tempdir().unwrap();
|
||||||
|
let root = temp.path().join("runtime");
|
||||||
|
initialize(
|
||||||
|
&root,
|
||||||
|
template.runtime_id(),
|
||||||
|
template.workspace_id(),
|
||||||
|
NEW_URL,
|
||||||
|
);
|
||||||
|
(
|
||||||
|
temp,
|
||||||
|
FsRuntimeStoreOptions::new(root).with_runtime_id(template.runtime_id()),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_fresh(options: &FsRuntimeStoreOptions) {
|
||||||
|
assert_eq!(fs::read_dir(&options.root).unwrap().count(), 1);
|
||||||
|
assert!(options.root.join("auth.toml").is_file());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn extraction_install_roundtrip_preserves_bytes_and_is_inactive() {
|
||||||
|
let (_source, worker, template) = source();
|
||||||
|
let original_worker = fs::read(worker.join(WORKER_FILE)).unwrap();
|
||||||
|
let original_metadata = fs::read(worker.join(WORKER_METADATA_FILE)).unwrap();
|
||||||
|
let exported = serde_json::to_vec(&template).unwrap();
|
||||||
|
let exported_text = String::from_utf8(exported.clone()).unwrap();
|
||||||
|
for forbidden in [
|
||||||
|
"old-config",
|
||||||
|
"old-submission-operation",
|
||||||
|
"restore_guard",
|
||||||
|
"private_key",
|
||||||
|
"http://",
|
||||||
|
"/old/host",
|
||||||
|
"private.log",
|
||||||
|
"resolved_manifest_snapshot",
|
||||||
|
] {
|
||||||
|
assert!(!exported_text.contains(forbidden));
|
||||||
|
}
|
||||||
|
let template: StoppedWorkerTemplate = serde_json::from_slice(&exported).unwrap();
|
||||||
|
let (_target, options) = destination(&template);
|
||||||
|
let auth_before = fs::read(options.root.join("auth.toml")).unwrap();
|
||||||
|
template.install(&options, request(), &resolved()).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
fs::read(options.root.join("auth.toml")).unwrap(),
|
||||||
|
auth_before
|
||||||
|
);
|
||||||
|
assert_eq!(fs::read(worker.join(WORKER_FILE)).unwrap(), original_worker);
|
||||||
|
assert_eq!(
|
||||||
|
fs::read(worker.join(WORKER_METADATA_FILE)).unwrap(),
|
||||||
|
original_metadata
|
||||||
|
);
|
||||||
|
let installed = options
|
||||||
|
.root
|
||||||
|
.join(WORKERS_DIR)
|
||||||
|
.join(template.worker_id().to_string());
|
||||||
|
assert_eq!(
|
||||||
|
fs::read(installed.join("session/session.json")).unwrap(),
|
||||||
|
template.session_manifest
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
fs::read(installed.join(format!("session/segments/{}.jsonl", template.segment_id())))
|
||||||
|
.unwrap(),
|
||||||
|
template.segment_jsonl
|
||||||
|
);
|
||||||
|
assert!(!installed.join("runs").exists());
|
||||||
|
let metadata: WorkerMetadata =
|
||||||
|
decode(&fs::read(installed.join(WORKER_METADATA_FILE)).unwrap()).unwrap();
|
||||||
|
assert_eq!(metadata.active.unwrap().session_id, template.session_id());
|
||||||
|
assert!(metadata.workspace_root.is_none() && metadata.resolved_manifest_snapshot.is_some());
|
||||||
|
|
||||||
|
let opened = FsRuntimeStore::open_or_create(options.root.clone(), RUNTIME).unwrap();
|
||||||
|
let state = opened.state.unwrap();
|
||||||
|
assert!(state.diagnostics.is_empty());
|
||||||
|
let saved = state.workers.get(&worker_id()).unwrap();
|
||||||
|
assert_eq!(saved.status, WorkerStatus::Stopped);
|
||||||
|
assert!(saved.last_finished_submission_request_id.is_none());
|
||||||
|
let PersistedWorkerExecutionState::Available(execution) = &saved.execution_state else {
|
||||||
|
panic!("not available")
|
||||||
|
};
|
||||||
|
assert_eq!(execution.restore_intent, WorkerRestoreIntent::Explicit);
|
||||||
|
assert!(execution.binding.is_none());
|
||||||
|
assert_eq!(execution.request, request());
|
||||||
|
// Fresh restore guard, no operation owner/receipt history.
|
||||||
|
assert_ne!(
|
||||||
|
serde_json::to_vec(&saved.restore_guard).unwrap(),
|
||||||
|
serde_json::to_vec(
|
||||||
|
&decode::<WorkerAggregateRecord>(&original_worker)
|
||||||
|
.unwrap()
|
||||||
|
.restore_guard
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
);
|
||||||
|
let guard: GuardState = decode(&serde_json::to_vec(&saved.restore_guard).unwrap()).unwrap();
|
||||||
|
assert!(guard.active_request_id.is_none() && guard.owners.is_empty());
|
||||||
|
drop(opened.store);
|
||||||
|
|
||||||
|
struct NoExecution;
|
||||||
|
impl crate::execution::WorkerExecutionBackend for NoExecution {
|
||||||
|
fn backend_id(&self) -> &str {
|
||||||
|
"fixture-no-execution"
|
||||||
|
}
|
||||||
|
fn spawn_worker(
|
||||||
|
&self,
|
||||||
|
_: crate::execution::WorkerExecutionSpawnRequest,
|
||||||
|
) -> crate::execution::WorkerExecutionSpawnResult {
|
||||||
|
panic!("fixture executed")
|
||||||
|
}
|
||||||
|
fn dispatch_input(
|
||||||
|
&self,
|
||||||
|
_: &WorkerRef,
|
||||||
|
_: crate::interaction::WorkerInput,
|
||||||
|
) -> crate::execution::WorkerExecutionResult {
|
||||||
|
panic!("fixture submitted")
|
||||||
|
}
|
||||||
|
fn restore_worker(
|
||||||
|
&self,
|
||||||
|
_: crate::execution::WorkerExecutionRestoreRequest,
|
||||||
|
) -> crate::execution::WorkerExecutionSpawnResult {
|
||||||
|
panic!("fixture auto-restored")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let runtime =
|
||||||
|
crate::Runtime::with_fs_store_and_execution_backend(options.clone(), Arc::new(NoExecution))
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(runtime.list_workers().unwrap().len(), 1);
|
||||||
|
assert_eq!(runtime.summary().unwrap().active_worker_count, 0);
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.worker_detail(&WorkerRef::new(worker_id()))
|
||||||
|
.unwrap()
|
||||||
|
.status,
|
||||||
|
WorkerStatus::Stopped
|
||||||
|
);
|
||||||
|
drop(runtime);
|
||||||
|
let runtime = crate::Runtime::with_fs_store(options).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
runtime.list_workers().unwrap()[0].status,
|
||||||
|
WorkerStatus::Stopped
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn install_rejects_schema_identity_input_and_overwrite() {
|
||||||
|
let (_source, _, template) = source();
|
||||||
|
for change in [
|
||||||
|
"template",
|
||||||
|
"runtime-schema",
|
||||||
|
"session-schema",
|
||||||
|
"session-id",
|
||||||
|
"segment-id",
|
||||||
|
"runtime-id",
|
||||||
|
"workspace-id",
|
||||||
|
"worker-id",
|
||||||
|
"origin-id",
|
||||||
|
"duplicate-id",
|
||||||
|
"truncated",
|
||||||
|
] {
|
||||||
|
let (_target, options) = destination(&template);
|
||||||
|
let mut invalid = template.clone();
|
||||||
|
match change {
|
||||||
|
"template" => invalid.template_version += 1,
|
||||||
|
"runtime-schema" => invalid.runtime_schema_version += 1,
|
||||||
|
"session-schema" => {
|
||||||
|
invalid.session_manifest = format!(
|
||||||
|
"{{\"schema_version\":999,\"session_id\":\"{}\"}}",
|
||||||
|
invalid.session_id
|
||||||
|
)
|
||||||
|
.into_bytes()
|
||||||
|
}
|
||||||
|
"session-id" => invalid.session_id = uuid::Uuid::now_v7(),
|
||||||
|
"segment-id" => invalid.segment_id = uuid::Uuid::nil(), // no external attestation: valid new Segment ID is otherwise portable
|
||||||
|
"runtime-id" => invalid.runtime_id = "other-runtime".into(),
|
||||||
|
"workspace-id" => invalid.workspace_id = "other-workspace".into(),
|
||||||
|
"worker-id" => invalid.worker_id = WorkerId::now_v7(),
|
||||||
|
"origin-id" => {
|
||||||
|
invalid.segment_jsonl = String::from_utf8(invalid.segment_jsonl)
|
||||||
|
.unwrap()
|
||||||
|
.replace(RUNTIME, "other-runtime")
|
||||||
|
.into_bytes()
|
||||||
|
}
|
||||||
|
"duplicate-id" => {
|
||||||
|
let last = invalid
|
||||||
|
.segment_jsonl
|
||||||
|
.split_inclusive(|b| *b == b'\n')
|
||||||
|
.last()
|
||||||
|
.unwrap()
|
||||||
|
.to_vec();
|
||||||
|
invalid.segment_jsonl.extend(last);
|
||||||
|
}
|
||||||
|
"truncated" => {
|
||||||
|
invalid.segment_jsonl.pop();
|
||||||
|
}
|
||||||
|
_ => unreachable!(),
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
invalid.install(&options, request(), &resolved()).is_err(),
|
||||||
|
"accepted {change}"
|
||||||
|
);
|
||||||
|
assert_fresh(&options);
|
||||||
|
}
|
||||||
|
let (_target, options) = destination(&template);
|
||||||
|
let mut launch = request();
|
||||||
|
launch.initial_input = Some(crate::interaction::WorkerInput::user("do not execute"));
|
||||||
|
assert!(template.install(&options, launch, &resolved()).is_err());
|
||||||
|
assert_fresh(&options);
|
||||||
|
template.install(&options, request(), &resolved()).unwrap();
|
||||||
|
let worker_path = options
|
||||||
|
.root
|
||||||
|
.join(WORKERS_DIR)
|
||||||
|
.join(worker_id().to_string())
|
||||||
|
.join(WORKER_FILE);
|
||||||
|
let before = fs::read(&worker_path).unwrap();
|
||||||
|
assert!(template.install(&options, request(), &resolved()).is_err());
|
||||||
|
assert_eq!(fs::read(worker_path).unwrap(), before);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rejects_uninitialized_untrusted_live_and_partial_destination() {
|
||||||
|
let (_source, _, template) = source();
|
||||||
|
let (_target, options) = destination(&template);
|
||||||
|
let held = acquire_runtime_store_owner_lock(&options.root).unwrap();
|
||||||
|
assert!(template.install(&options, request(), &resolved()).is_err());
|
||||||
|
assert_fresh(&options);
|
||||||
|
drop(held);
|
||||||
|
initialize(&options.root, "wrong-runtime", WORKSPACE, NEW_URL);
|
||||||
|
assert!(template.install(&options, request(), &resolved()).is_err());
|
||||||
|
initialize(&options.root, RUNTIME, WORKSPACE, "http://127.0.0.1:23456");
|
||||||
|
assert!(template.install(&options, request(), &resolved()).is_err());
|
||||||
|
fs::remove_file(options.root.join("auth.toml")).unwrap();
|
||||||
|
assert!(template.install(&options, request(), &resolved()).is_err());
|
||||||
|
initialize(&options.root, RUNTIME, WORKSPACE, NEW_URL);
|
||||||
|
fs::create_dir(options.root.join(WORKERS_DIR)).unwrap();
|
||||||
|
assert!(template.install(&options, request(), &resolved()).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn extraction_rejects_nonstopped_pending_schema_and_metadata_mismatch() {
|
||||||
|
for change in [
|
||||||
|
"running",
|
||||||
|
"pending",
|
||||||
|
"guard-pending",
|
||||||
|
"attachment",
|
||||||
|
"schema",
|
||||||
|
"worker-id",
|
||||||
|
"metadata-workspace",
|
||||||
|
"metadata-session",
|
||||||
|
"artifact",
|
||||||
|
"secret",
|
||||||
|
"url",
|
||||||
|
"path",
|
||||||
|
] {
|
||||||
|
let (_source, worker, _) = source();
|
||||||
|
let mut aggregate: WorkerAggregateRecord =
|
||||||
|
decode(&fs::read(worker.join(WORKER_FILE)).unwrap()).unwrap();
|
||||||
|
match change {
|
||||||
|
"running" => aggregate.status = WorkerStatus::Running,
|
||||||
|
"pending" => {
|
||||||
|
aggregate.execution_state = WorkerExecutionStateRecord::ReconciliationRequired(
|
||||||
|
PersistedWorkerLifecycleOperation::Restore(PersistedWorkerRestoreOperation {
|
||||||
|
operation_id: WorkerLifecycleOperationId::new(),
|
||||||
|
mode: PersistedWorkerRestoreMode::Explicit,
|
||||||
|
commit: crate::runtime::RestoreCommitFact::NotCommitted,
|
||||||
|
source_repair_possible: false,
|
||||||
|
request: request(),
|
||||||
|
binding: None,
|
||||||
|
last_settled_status: WorkerStatus::Stopped,
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
"guard-pending" => aggregate.restore_guard = serde_json::from_str(r#"{"observation_token":"old-token","active_request_id":"pending-restore","owners":{}}"#).unwrap(),
|
||||||
|
"attachment" => aggregate.logical_workdir_attachments.push(LogicalWorkdirAttachment {
|
||||||
|
alias: workdir::WorkdirAttachmentAlias::new("source").unwrap(),
|
||||||
|
working_directory_id: "old-workdir".into(),
|
||||||
|
capabilities: workdir::WorkdirSessionCapabilities::READ_ONLY,
|
||||||
|
}),
|
||||||
|
"schema" => aggregate.schema_version += 1,
|
||||||
|
"worker-id" => aggregate.worker_id = WorkerId::now_v7(),
|
||||||
|
"metadata-workspace" | "metadata-session" => {
|
||||||
|
let mut metadata: WorkerMetadata =
|
||||||
|
decode(&fs::read(worker.join(WORKER_METADATA_FILE)).unwrap()).unwrap();
|
||||||
|
if change == "metadata-workspace" {
|
||||||
|
metadata.workspace_id = Some("wrong-workspace".into());
|
||||||
|
} else {
|
||||||
|
metadata.active.as_mut().unwrap().session_id = uuid::Uuid::now_v7();
|
||||||
|
}
|
||||||
|
atomic_write_json(
|
||||||
|
&worker.join(WORKER_METADATA_FILE),
|
||||||
|
&metadata,
|
||||||
|
"test mismatch",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
"artifact" => {
|
||||||
|
fs::create_dir_all(worker.join("session/artifacts/paste")).unwrap();
|
||||||
|
fs::write(
|
||||||
|
worker.join("session/artifacts/paste/payload"),
|
||||||
|
"unsupported",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
"secret" | "url" | "path" => {
|
||||||
|
let segment = fs::read_dir(worker.join("session/segments"))
|
||||||
|
.unwrap()
|
||||||
|
.next()
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
.path();
|
||||||
|
let replacement = match change {
|
||||||
|
"secret" => "Bearer secret",
|
||||||
|
"url" => "https://old-host.invalid",
|
||||||
|
_ => "/old/host/path",
|
||||||
|
};
|
||||||
|
let text = fs::read_to_string(&segment)
|
||||||
|
.unwrap()
|
||||||
|
.replace("artificial fixture reply", replacement);
|
||||||
|
fs::write(segment, text).unwrap();
|
||||||
|
}
|
||||||
|
_ => unreachable!(),
|
||||||
|
}
|
||||||
|
atomic_write_json(&worker.join(WORKER_FILE), &aggregate, "test aggregate").unwrap();
|
||||||
|
assert!(
|
||||||
|
StoppedWorkerTemplate::extract(&worker).is_err(),
|
||||||
|
"accepted {change}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let (_source, worker, _) = source();
|
||||||
|
let held =
|
||||||
|
acquire_runtime_store_owner_lock(worker.parent().unwrap().parent().unwrap()).unwrap();
|
||||||
|
assert!(StoppedWorkerTemplate::extract(&worker).is_err());
|
||||||
|
drop(held);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn schema_v12_settled_template_installs_current_aggregate_without_rewriting_session() {
|
||||||
|
let (_source, _, mut template) = source();
|
||||||
|
template.runtime_schema_version = RESTORE_ADMISSION_SCHEMA_VERSION;
|
||||||
|
let bytes = serde_json::to_vec(&template).unwrap();
|
||||||
|
let (_target, options) = destination(&template);
|
||||||
|
template.install(&options, request(), &resolved()).unwrap();
|
||||||
|
assert_eq!(serde_json::to_vec(&template).unwrap(), bytes);
|
||||||
|
let worker = options.root.join(WORKERS_DIR).join(worker_id().to_string());
|
||||||
|
assert_eq!(
|
||||||
|
fs::read(worker.join("session/session.json")).unwrap(),
|
||||||
|
template.session_manifest
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
fs::read(worker.join(format!("session/segments/{}.jsonl", template.segment_id))).unwrap(),
|
||||||
|
template.segment_jsonl
|
||||||
|
);
|
||||||
|
let aggregate: WorkerAggregateRecord =
|
||||||
|
read_bounded_json(&worker.join(WORKER_FILE), "read installed fixture").unwrap();
|
||||||
|
let saved = aggregate.validate(&worker.join(WORKER_FILE)).unwrap();
|
||||||
|
assert_eq!(saved.status, WorkerStatus::Stopped);
|
||||||
|
assert!(matches!(
|
||||||
|
saved.execution_state,
|
||||||
|
PersistedWorkerExecutionState::Available(_)
|
||||||
|
));
|
||||||
|
// Unsupported template schemas do not enter the versioned reader.
|
||||||
|
template.runtime_schema_version = RESTORE_ADMISSION_SCHEMA_VERSION - 1;
|
||||||
|
assert!(template.validate_shape().is_err());
|
||||||
|
template.runtime_schema_version = SCHEMA_VERSION + 1;
|
||||||
|
assert!(template.validate_shape().is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn fresh_manifest_and_memory_settings_are_required() {
|
||||||
|
let (_source, _, template) = source();
|
||||||
|
let (_target, options) = destination(&template);
|
||||||
|
let mut manifest = resolved();
|
||||||
|
manifest.worker.name = "wrong-worker".into();
|
||||||
|
assert!(template.install(&options, request(), &manifest).is_err());
|
||||||
|
let mut binding = request();
|
||||||
|
binding.memory_settings = None;
|
||||||
|
assert!(template.install(&options, binding, &resolved()).is_err());
|
||||||
|
let mut binding = request();
|
||||||
|
binding.memory_settings.as_mut().unwrap().workspace_id = "other-workspace".into();
|
||||||
|
assert!(template.install(&options, binding, &resolved()).is_err());
|
||||||
|
assert_fresh(&options);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn settled_checkpoints_preserve_receipts_but_refuse_pending_work() {
|
||||||
|
let mut checkpoint = settled_checkpoint();
|
||||||
|
assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_ok());
|
||||||
|
checkpoint.pending.push(1);
|
||||||
|
assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_err());
|
||||||
|
checkpoint.pending.clear();
|
||||||
|
checkpoint.activating = Some(1);
|
||||||
|
assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_err());
|
||||||
|
checkpoint.activating = None;
|
||||||
|
checkpoint.invocation_receipts.push(1);
|
||||||
|
assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_err());
|
||||||
|
checkpoint.invocation_receipts.clear();
|
||||||
|
checkpoint.invocation_recovery_blocked = true;
|
||||||
|
assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
// Deliberately corrupt only test-owned synthetic source files, never the lab.
|
||||||
|
fn assert_rejected_session_bytes(worker: &Path, template: &StoppedWorkerTemplate, bytes: Vec<u8>) {
|
||||||
|
let path = worker.join(format!("session/segments/{}.jsonl", template.segment_id()));
|
||||||
|
fs::write(path, &bytes).unwrap();
|
||||||
|
let extraction_error = StoppedWorkerTemplate::extract(worker)
|
||||||
|
.err()
|
||||||
|
.expect("invalid source accepted");
|
||||||
|
assert!(
|
||||||
|
!extraction_error
|
||||||
|
.to_string()
|
||||||
|
.contains("synthetic-discarded-material")
|
||||||
|
);
|
||||||
|
let (_target, options) = destination(template);
|
||||||
|
let mut invalid = template.clone();
|
||||||
|
invalid.segment_jsonl = bytes;
|
||||||
|
let installation_error = invalid
|
||||||
|
.install(&options, request(), &resolved())
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(
|
||||||
|
!installation_error
|
||||||
|
.to_string()
|
||||||
|
.contains("synthetic-discarded-material")
|
||||||
|
);
|
||||||
|
assert_fresh(&options);
|
||||||
|
}
|
||||||
|
|
||||||
|
fn synthetic_logs(template: &StoppedWorkerTemplate) -> Vec<LogEntry> {
|
||||||
|
template
|
||||||
|
.segment_jsonl
|
||||||
|
.split(|byte| *byte == b'\n')
|
||||||
|
.filter(|line| !line.is_empty())
|
||||||
|
.map(|line| serde_json::from_slice(line).unwrap())
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn encode_synthetic_logs(logs: &[LogEntry]) -> Vec<u8> {
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
for log in logs {
|
||||||
|
serde_json::to_writer(&mut bytes, log).unwrap();
|
||||||
|
bytes.push(b'\n');
|
||||||
|
}
|
||||||
|
bytes
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn missing_system_prompt_matches_worker_restore_rejection() {
|
||||||
|
let (_source, worker_dir, template) = source();
|
||||||
|
let mut logs = synthetic_logs(&template);
|
||||||
|
assert!(session_store::collect_state(&logs).system_prompt.is_some());
|
||||||
|
let LogEntry::AnnotatedSegmentStart { system_prompt, .. } = &mut logs[0] else {
|
||||||
|
panic!("missing seed")
|
||||||
|
};
|
||||||
|
*system_prompt = None;
|
||||||
|
assert_rejected_session_bytes(&worker_dir, &template, encode_synthetic_logs(&logs));
|
||||||
|
// The owning Worker constructor rejects this before model-client creation,
|
||||||
|
// allocation, or execution. Do not start a controller or submit any input.
|
||||||
|
let store = session_store::CombinedStore::new(
|
||||||
|
WorkerSessionStore::open_read_only(worker_dir.join("session")).unwrap(),
|
||||||
|
session_store::WorkerAggregateStore::open_read_only(&worker_dir, template.worker_name())
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
let restored = worker::Worker::restore_from_manifest(
|
||||||
|
template.session_id(),
|
||||||
|
template.segment_id(),
|
||||||
|
resolved(),
|
||||||
|
store,
|
||||||
|
worker::PromptCatalogSource::builtins_only(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert!(matches!(
|
||||||
|
restored,
|
||||||
|
Err(worker::WorkerError::SegmentSystemPromptMissing { .. })
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn malformed_submission_and_notification_receipts_fail_extraction_and_install() {
|
||||||
|
let (_source, worker, template) = source();
|
||||||
|
for notification in [false, true] {
|
||||||
|
let mut checkpoint = settled_checkpoint();
|
||||||
|
if notification {
|
||||||
|
checkpoint.notification_receipts =
|
||||||
|
vec![ReceiptFixture::Malformed(1), ReceiptFixture::Malformed(2)];
|
||||||
|
} else {
|
||||||
|
checkpoint.receipts = vec![ReceiptFixture::Malformed(1), ReceiptFixture::Malformed(2)];
|
||||||
|
}
|
||||||
|
let mut logs = synthetic_logs(&template);
|
||||||
|
for log in &mut logs {
|
||||||
|
if let LogEntry::AnnotatedUserInput { extensions, .. } = log {
|
||||||
|
*extensions = vec![checkpoint_extension(&checkpoint)];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert_rejected_session_bytes(&worker, &template, encode_synthetic_logs(&logs));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unknown_fields_at_every_retained_record_depth_are_rejected_without_secret_diagnostics() {
|
||||||
|
let (_source, worker, template) = source();
|
||||||
|
let original = String::from_utf8(template.segment_jsonl.clone()).unwrap();
|
||||||
|
// These values are harmless synthetic markers, not lab/private material.
|
||||||
|
// The compatibility decoder normally discards many of these fields.
|
||||||
|
let locations = [
|
||||||
|
r#""kind":"annotated_segment_start""#,
|
||||||
|
r#""kind":"invoke""#,
|
||||||
|
r#""kind":"annotated_user_input""#,
|
||||||
|
r#""kind":"annotated_assistant_item""#,
|
||||||
|
r#""kind":"llm_usage""#,
|
||||||
|
r#""kind":"turn_end""#,
|
||||||
|
r#""kind":"run_completed""#,
|
||||||
|
r#""max_tokens":null"#,
|
||||||
|
r#""kind":"message","role":"assistant""#,
|
||||||
|
r#""kind":"text","text":"artificial fixture reply""#,
|
||||||
|
r#""kind":"text","content":"Harmless artificial fixture input.""#,
|
||||||
|
r#""entry_id":"fixed-entry-id""#,
|
||||||
|
r#""kind":"model_output""#,
|
||||||
|
r#""worker_id":"01a1657d-b762-7791-87ae-a49f28cadbce""#,
|
||||||
|
r#""domain":"worker.pending_activations.v1""#,
|
||||||
|
r#""next_activation_sequence":2"#,
|
||||||
|
r#""submission_request_id":"completed-submission-request""#,
|
||||||
|
r#""notification_request_id":"completed-notification-request""#,
|
||||||
|
];
|
||||||
|
for location in locations {
|
||||||
|
assert!(original.contains(location));
|
||||||
|
for extra in [
|
||||||
|
r#", "reasoning":{"text":"synthetic-discarded-material"}"#,
|
||||||
|
r#", "encrypted_content":"synthetic-discarded-material""#,
|
||||||
|
] {
|
||||||
|
let altered = original.replacen(location, &format!("{location}{extra}"), 1);
|
||||||
|
assert_rejected_session_bytes(&worker, &template, altered.into_bytes());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let duplicate = original.replacen(
|
||||||
|
r#""role":"assistant""#,
|
||||||
|
r#""role":"assistant","role":"assistant""#,
|
||||||
|
1,
|
||||||
|
);
|
||||||
|
assert_rejected_session_bytes(&worker, &template, duplicate.into_bytes());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn opaque_feature_payloads_cannot_bypass_strict_session_validation() {
|
||||||
|
let (_source, worker, template) = source();
|
||||||
|
let mut logs = synthetic_logs(&template);
|
||||||
|
for log in &mut logs {
|
||||||
|
if let LogEntry::AnnotatedUserInput { history, .. } = log {
|
||||||
|
if let session_store::LoggedItem::Message { role, .. } = &mut history[0].item {
|
||||||
|
*role = session_store::LoggedRole::System;
|
||||||
|
}
|
||||||
|
history[0].metadata.input_metadata = Some(
|
||||||
|
session_store::HistoryInputMetadata::new(
|
||||||
|
session_store::HistoryInputUsePolicy {
|
||||||
|
recording_material: session_store::InputEvidenceUse::Exclude,
|
||||||
|
memory_evidence: session_store::InputEvidenceUse::Exclude,
|
||||||
|
observation: session_store::InputObservationUse::Hidden,
|
||||||
|
compaction: session_store::InputCompactionUse::Retain,
|
||||||
|
},
|
||||||
|
Some(
|
||||||
|
session_store::FeatureInputPayload::encode(
|
||||||
|
"fixture.test",
|
||||||
|
1,
|
||||||
|
&BTreeMap::from([("reasoning", "harmless synthetic payload")]),
|
||||||
|
)
|
||||||
|
.unwrap(),
|
||||||
|
),
|
||||||
|
None,
|
||||||
|
)
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert_rejected_session_bytes(&worker, &template, encode_synthetic_logs(&logs));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
#[test]
|
||||||
|
fn refuses_source_and_destination_symlinks() {
|
||||||
|
let (_source, worker, template) = source();
|
||||||
|
let (_target, options) = destination(&template);
|
||||||
|
std::os::unix::fs::symlink(worker.join(WORKER_FILE), options.root.join("linked")).unwrap();
|
||||||
|
assert!(template.install(&options, request(), &resolved()).is_err());
|
||||||
|
fs::remove_file(options.root.join("linked")).unwrap();
|
||||||
|
std::os::unix::fs::symlink(worker.join(WORKER_FILE), worker.join("runs/link")).unwrap();
|
||||||
|
assert!(StoppedWorkerTemplate::extract(&worker).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Opt-in only: source path supplied explicitly; assertions never display content.
|
||||||
|
#[test]
|
||||||
|
#[ignore = "requires the approved artificial lab aggregate via YOI_STOPPED_FIXTURE_SOURCE"]
|
||||||
|
fn approved_lab_extract_install() {
|
||||||
|
let path = PathBuf::from(
|
||||||
|
std::env::var_os("YOI_STOPPED_FIXTURE_SOURCE").expect("explicit source required"),
|
||||||
|
);
|
||||||
|
let template = StoppedWorkerTemplate::extract(&path).unwrap();
|
||||||
|
let (_target, options) = destination(&template);
|
||||||
|
let mut bind = request();
|
||||||
|
bind.worker_id = template.worker_id();
|
||||||
|
bind.workspace_api.as_mut().unwrap().workspace_id = template.workspace_id().into();
|
||||||
|
bind.memory_settings.as_mut().unwrap().workspace_id = template.workspace_id().into();
|
||||||
|
let mut manifest = resolved();
|
||||||
|
manifest.worker.name = template.worker_name();
|
||||||
|
template.install(&options, bind, &manifest).unwrap();
|
||||||
|
let installed = options
|
||||||
|
.root
|
||||||
|
.join(WORKERS_DIR)
|
||||||
|
.join(template.worker_id().to_string());
|
||||||
|
for relative in [
|
||||||
|
"session/session.json".to_string(),
|
||||||
|
format!("session/segments/{}.jsonl", template.segment_id()),
|
||||||
|
] {
|
||||||
|
assert!(
|
||||||
|
fs::read(path.join(&relative)).unwrap() == fs::read(installed.join(relative)).unwrap()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let runtime = crate::Runtime::with_fs_store(options).unwrap();
|
||||||
|
assert_eq!(runtime.list_workers().unwrap().len(), 1);
|
||||||
|
assert_eq!(runtime.summary().unwrap().active_worker_count, 0);
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -193,6 +193,21 @@ mod tests {
|
|||||||
use super::*;
|
use super::*;
|
||||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||||
|
|
||||||
|
fn short_tempdir() -> tempfile::TempDir {
|
||||||
|
let target = std::path::Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||||
|
.parent()
|
||||||
|
.unwrap()
|
||||||
|
.parent()
|
||||||
|
.unwrap()
|
||||||
|
.join("target");
|
||||||
|
std::fs::create_dir_all(&target).unwrap();
|
||||||
|
tempfile::Builder::new()
|
||||||
|
.prefix("d")
|
||||||
|
.rand_bytes(3)
|
||||||
|
.tempdir_in(target)
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
fn read_request() -> ReadRequest {
|
fn read_request() -> ReadRequest {
|
||||||
ReadRequest {
|
ReadRequest {
|
||||||
path: WorkdirPath::new("README.md").unwrap(),
|
path: WorkdirPath::new("README.md").unwrap(),
|
||||||
@@ -225,9 +240,60 @@ mod tests {
|
|||||||
assert_eq!(attempts.load(Ordering::SeqCst), 0);
|
assert_eq!(attempts.load(Ordering::SeqCst), 0);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn router_cleanup_of_unopened_session_never_connects_or_follows_reused_alias() {
|
||||||
|
let attempts = Arc::new(AtomicUsize::new(0));
|
||||||
|
let observed = attempts.clone();
|
||||||
|
let session = DeferredWorkdirSession::handle(
|
||||||
|
"old",
|
||||||
|
WorkdirSessionCapabilities::READ_ONLY,
|
||||||
|
move || {
|
||||||
|
observed.fetch_add(1, Ordering::SeqCst);
|
||||||
|
Err("offline".into())
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let router = WorkdirSessionRouter::new();
|
||||||
|
let alias = WorkdirAttachmentAlias::new("w").unwrap();
|
||||||
|
let routed = router.attach(alias.clone(), session.clone()).unwrap();
|
||||||
|
assert!(routed.subscribe_command_events().is_some());
|
||||||
|
assert!(routed.command_snapshot().is_empty());
|
||||||
|
router.close_all().await.unwrap();
|
||||||
|
router.close_all().await.unwrap();
|
||||||
|
assert!(router.aliases().is_empty());
|
||||||
|
assert!(matches!(
|
||||||
|
session.read(read_request()).await,
|
||||||
|
Err(WorkdirError::SessionClosed)
|
||||||
|
));
|
||||||
|
assert_eq!(attempts.load(Ordering::SeqCst), 0);
|
||||||
|
|
||||||
|
let new_attempts = Arc::new(AtomicUsize::new(0));
|
||||||
|
let new_observed = new_attempts.clone();
|
||||||
|
let replacement = DeferredWorkdirSession::handle(
|
||||||
|
"new",
|
||||||
|
WorkdirSessionCapabilities::READ_ONLY,
|
||||||
|
move || {
|
||||||
|
new_observed.fetch_add(1, Ordering::SeqCst);
|
||||||
|
Err("offline".into())
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let new_routed = router.attach(alias, replacement).unwrap();
|
||||||
|
// A retained old connection cannot open the replacement provider.
|
||||||
|
assert!(routed.read(read_request()).await.is_err());
|
||||||
|
assert_eq!(new_attempts.load(Ordering::SeqCst), 0);
|
||||||
|
assert!(matches!(
|
||||||
|
new_routed.read(read_request()).await,
|
||||||
|
Err(ref error) if matches!(error.classification_source(), WorkdirError::Unavailable(_))
|
||||||
|
));
|
||||||
|
assert_eq!(new_attempts.load(Ordering::SeqCst), 1);
|
||||||
|
// Even after a failed open, cleanup must not try connecting again.
|
||||||
|
router.close_all().await.unwrap();
|
||||||
|
assert_eq!(new_attempts.load(Ordering::SeqCst), 1);
|
||||||
|
assert_eq!(attempts.load(Ordering::SeqCst), 0);
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn failed_open_retries_on_use_and_keeps_read_only_capabilities() {
|
async fn failed_open_retries_on_use_and_keeps_read_only_capabilities() {
|
||||||
let temp = tempfile::tempdir().unwrap();
|
let temp = short_tempdir();
|
||||||
std::fs::write(temp.path().join("README.md"), "restored").unwrap();
|
std::fs::write(temp.path().join("README.md"), "restored").unwrap();
|
||||||
let root = temp.path().to_path_buf();
|
let root = temp.path().to_path_buf();
|
||||||
let attempts = Arc::new(AtomicUsize::new(0));
|
let attempts = Arc::new(AtomicUsize::new(0));
|
||||||
|
|||||||
@@ -0,0 +1,185 @@
|
|||||||
|
//! Explicit, operation-owned repair of saved Session material. This is not
|
||||||
|
//! read-only candidate preparation: the canonical Session/attribution is the
|
||||||
|
//! receipt for replay after a lost response, and effects are reported as such.
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
impl ProfileRuntimeWorkerFactory {
|
||||||
|
pub(super) async fn recover_saved_restore_source(
|
||||||
|
&self,
|
||||||
|
request: &WorkerExecutionRestoreRequest,
|
||||||
|
) -> Result<bool, RuntimeRestoreFailure> {
|
||||||
|
let cause = |cause| RuntimeRestoreFailure::started(RuntimeRestoreStage::Recovery, cause);
|
||||||
|
let name = Self::runtime_worker_name_for_ref(&request.worker_ref);
|
||||||
|
let root = self
|
||||||
|
.worker_aggregate_dir(&request.worker_ref)
|
||||||
|
.map_err(|_| cause("restore recovery aggregate unavailable"))?;
|
||||||
|
let metadata_store = WorkerAggregateStore::open_read_only(&root, &name)
|
||||||
|
.map_err(|_| cause("restore recovery metadata open failed"))?;
|
||||||
|
let metadata = metadata_store
|
||||||
|
.read_by_name(&name)
|
||||||
|
.map_err(|_| cause("restore recovery metadata read failed"))?
|
||||||
|
.ok_or_else(|| cause("restore recovery metadata missing"))?;
|
||||||
|
let active = metadata
|
||||||
|
.active
|
||||||
|
.as_ref()
|
||||||
|
.ok_or_else(|| cause("restore recovery Session missing"))?;
|
||||||
|
let pending = active.segment_id.is_none();
|
||||||
|
let incomplete = request.request.subjektiv_attached
|
||||||
|
&& !matches!(
|
||||||
|
metadata.subjektiv_session_attribution,
|
||||||
|
Some(session_store::SubjektivSessionAttributionState::Confirmed { .. })
|
||||||
|
);
|
||||||
|
if !pending && !incomplete {
|
||||||
|
return Ok(false);
|
||||||
|
}
|
||||||
|
request
|
||||||
|
.context
|
||||||
|
.authorize_restore_source_repair(request.operation_id)
|
||||||
|
.map_err(|_| cause("canonical source repair authority is not durably confirmed"))?;
|
||||||
|
// The accepted Runtime operation already owns this Session. Its stable
|
||||||
|
// Session identity and sole committed seed, not a new restore request,
|
||||||
|
// are used to reconcile an interrupted first-head publication.
|
||||||
|
let session_store = WorkerSessionStore::new_with_retention(
|
||||||
|
root.join("session"),
|
||||||
|
self.session_retention()
|
||||||
|
.map_err(|_| cause("restore recovery retention unavailable"))?,
|
||||||
|
)
|
||||||
|
.map_err(|_| cause("restore recovery Session open failed"))?;
|
||||||
|
let (manifest, mut loader) = Self::manifest_for_restore(&metadata, &request.request)
|
||||||
|
.map_err(|_| cause("restore recovery saved manifest invalid"))?;
|
||||||
|
let workspace = RuntimeWorkspaceBackendRef::from_worker_request(
|
||||||
|
&request.request,
|
||||||
|
self.runtime_id.as_deref(),
|
||||||
|
)
|
||||||
|
.map_err(cause)?;
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
|
let request_client = request
|
||||||
|
.request
|
||||||
|
.workspace_api
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|api| self.workspace_request_clients.get(&api.workspace_id));
|
||||||
|
let context = workspace.worker_context(
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
|
&request.worker_ref,
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
|
request.workspace_scope.as_ref(),
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
|
self.worker_mutation_identity.as_ref(),
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
|
request_client,
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
|
self.embedded_worker_mutation_dispatcher.as_ref(),
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
|
Some(self.prompt_projection_cache.clone()),
|
||||||
|
);
|
||||||
|
if pending && context.workspace_id().is_some() {
|
||||||
|
let bundle = request
|
||||||
|
.config_bundle
|
||||||
|
.as_ref()
|
||||||
|
.ok_or_else(|| cause("pending recovery launch material missing"))?;
|
||||||
|
let projection = self
|
||||||
|
.observe_bundle_prompt_projection(
|
||||||
|
bundle,
|
||||||
|
request
|
||||||
|
.request
|
||||||
|
.workspace_api
|
||||||
|
.as_ref()
|
||||||
|
.map(|api| api.workspace_id.as_str()),
|
||||||
|
)
|
||||||
|
.map_err(|_| cause("pending recovery Prompt projection invalid"))?
|
||||||
|
.ok_or_else(|| cause("pending recovery Prompt projection missing"))?;
|
||||||
|
loader = loader.with_effective_catalog(projection.projection.catalog.clone());
|
||||||
|
}
|
||||||
|
self.failed_restore_resources
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||||
|
.insert(
|
||||||
|
request.worker_ref.clone(),
|
||||||
|
FailedRestoreResources {
|
||||||
|
operation_id: request.operation_id,
|
||||||
|
sessions: Vec::new(),
|
||||||
|
candidate_workdir_cleanup: None,
|
||||||
|
allocation: None,
|
||||||
|
feature_tasks: None,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let store = CombinedStore::new(session_store, metadata_store);
|
||||||
|
let worker_result = Worker::restore_pending_from_worker_metadata_with_context(
|
||||||
|
&name,
|
||||||
|
manifest,
|
||||||
|
store,
|
||||||
|
loader,
|
||||||
|
context,
|
||||||
|
WorkerFilesystemAuthority::None,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
let mut worker = worker_result.map_err(|error| {
|
||||||
|
self.retain_construction_failure(request, &error);
|
||||||
|
cause(restore_worker_cause(&error))
|
||||||
|
})?;
|
||||||
|
let tasks = worker.retain_startup_feature_cleanup();
|
||||||
|
let sessions = worker.workdir_sessions();
|
||||||
|
let allocation = worker.restore_allocation_cleanup_handle();
|
||||||
|
self.failed_restore_resources
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||||
|
.insert(
|
||||||
|
request.worker_ref.clone(),
|
||||||
|
FailedRestoreResources {
|
||||||
|
operation_id: request.operation_id,
|
||||||
|
sessions: vec![sessions.clone()],
|
||||||
|
candidate_workdir_cleanup: None,
|
||||||
|
allocation: allocation.clone(),
|
||||||
|
feature_tasks: Some(tasks.clone()),
|
||||||
|
},
|
||||||
|
);
|
||||||
|
let result = async {
|
||||||
|
if let Some(binding) = request.request.backend_job.as_ref() {
|
||||||
|
worker
|
||||||
|
.bind_backend_job(binding.clone())
|
||||||
|
.map_err(|_| cause("recovery Backend Job binding failed"))?;
|
||||||
|
}
|
||||||
|
worker
|
||||||
|
.finalize_subjektiv_session_attribution(
|
||||||
|
request.request.subjektiv_attached,
|
||||||
|
SubjektivSessionAttributionLifecycle::RestoredSession,
|
||||||
|
)
|
||||||
|
.map_err(|error| cause(restore_worker_cause(&error)))?;
|
||||||
|
if pending {
|
||||||
|
worker::WorkerController::recover_pending_head(&mut worker, &root)
|
||||||
|
.await
|
||||||
|
.map_err(|_| cause("pending Session head recovery failed"))?;
|
||||||
|
}
|
||||||
|
Ok::<_, RuntimeRestoreFailure>(())
|
||||||
|
}
|
||||||
|
.await;
|
||||||
|
// Never release allocation while the Worker guard is attached. Retained
|
||||||
|
// handles in the same operation remain available on every cleanup error.
|
||||||
|
drop(worker);
|
||||||
|
let cleanup = async {
|
||||||
|
tasks
|
||||||
|
.shutdown()
|
||||||
|
.await
|
||||||
|
.map_err(|_| cause("recovery Feature settlement failed"))?;
|
||||||
|
sessions
|
||||||
|
.close_all()
|
||||||
|
.await
|
||||||
|
.map_err(|_| cause("recovery Workdir close failed"))?;
|
||||||
|
if let Some(allocation) = allocation {
|
||||||
|
allocation
|
||||||
|
.release()
|
||||||
|
.map_err(|_| cause("recovery allocation release failed"))?;
|
||||||
|
}
|
||||||
|
Ok::<_, RuntimeRestoreFailure>(())
|
||||||
|
}
|
||||||
|
.await;
|
||||||
|
result?;
|
||||||
|
cleanup?;
|
||||||
|
self.failed_restore_resources
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||||
|
.remove(&request.worker_ref);
|
||||||
|
Ok(true)
|
||||||
|
}
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,927 @@
|
|||||||
|
// Included by the production Profile publication suite. Each fault is at a
|
||||||
|
// distinct partial-publication boundary; no factory result is substituted.
|
||||||
|
|
||||||
|
fn publication_gate<T>() -> (
|
||||||
|
mpsc::Sender<()>,
|
||||||
|
mpsc::Receiver<T>,
|
||||||
|
crate::execution::RestoreTestGate<T>,
|
||||||
|
) {
|
||||||
|
let (release, receiver) = mpsc::channel();
|
||||||
|
let (entered, evidence) = mpsc::channel();
|
||||||
|
(
|
||||||
|
release,
|
||||||
|
evidence,
|
||||||
|
crate::execution::RestoreTestGate {
|
||||||
|
entered,
|
||||||
|
release: receiver,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_single_publication(fixture: &Fixture, expected: usize) {
|
||||||
|
let faults = &fixture.backend.factory.restore_faults;
|
||||||
|
assert_eq!(
|
||||||
|
faults.executions.load(Ordering::SeqCst),
|
||||||
|
expected,
|
||||||
|
"construction reran"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
faults.effects_calls.load(Ordering::SeqCst),
|
||||||
|
expected,
|
||||||
|
"restore effects reran"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
faults.publication_attempts.load(Ordering::SeqCst),
|
||||||
|
expected,
|
||||||
|
"Controller start reran"
|
||||||
|
);
|
||||||
|
fixture.assert_quiet();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn assert_real_attach(
|
||||||
|
fixture: &Fixture,
|
||||||
|
handle: &WorkerHandle,
|
||||||
|
) -> crate::execution::WorkerProtocolTransport {
|
||||||
|
let worker_ref = &fixture.request.worker_ref;
|
||||||
|
assert!(matches!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.worker_session(crate::execution::WorkerSessionObservationRequest {
|
||||||
|
worker_ref: worker_ref.clone(),
|
||||||
|
}),
|
||||||
|
runtime_api::WorkerSessionAvailability::LiveProtocol
|
||||||
|
));
|
||||||
|
let transport = fixture
|
||||||
|
.backend
|
||||||
|
.clone()
|
||||||
|
.attach_worker_protocol(worker_ref)
|
||||||
|
.unwrap();
|
||||||
|
transport.validate().unwrap();
|
||||||
|
let paired = subscribe_worker_protocol_session(handle);
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(&transport.snapshot).unwrap(),
|
||||||
|
serde_json::to_value(paired.snapshot_event).unwrap()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
matches!(&transport.snapshot, Event::Snapshot { session, greeting, state, in_flight, .. }
|
||||||
|
if session == &session_store::public_snapshot::project_current_session_snapshot(&fixture.entries())
|
||||||
|
&& !greeting.tools.is_empty() && state.state == protocol::WorkerState::Idle && in_flight.is_empty())
|
||||||
|
);
|
||||||
|
transport
|
||||||
|
}
|
||||||
|
|
||||||
|
fn observe_publication_markers(
|
||||||
|
fixture: &Fixture,
|
||||||
|
observations: tokio::sync::broadcast::Receiver<crate::observation::WorkerObservationEvent>,
|
||||||
|
markers: &'static [&'static str],
|
||||||
|
) -> tokio::sync::broadcast::Receiver<crate::observation::WorkerObservationEvent> {
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.run_on_adapter_runtime(async move {
|
||||||
|
let mut observations = observations;
|
||||||
|
let mut counts = vec![0; markers.len()];
|
||||||
|
let mut snapshots = 0;
|
||||||
|
tokio::time::timeout(WAIT, async {
|
||||||
|
loop {
|
||||||
|
let observed = observations.recv().await.unwrap();
|
||||||
|
if matches!(observed.payload, Event::Snapshot { .. }) {
|
||||||
|
snapshots += 1;
|
||||||
|
}
|
||||||
|
for (index, marker) in markers.iter().enumerate() {
|
||||||
|
if is_marker(&observed.payload, marker) {
|
||||||
|
counts[index] += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if counts.last() == Some(&1) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(|_| format!("lost original subscription markers: {counts:?}"))?;
|
||||||
|
assert_eq!(
|
||||||
|
counts,
|
||||||
|
vec![1; markers.len()],
|
||||||
|
"lost or duplicated prefix on original receiver"
|
||||||
|
);
|
||||||
|
assert_eq!(snapshots, 1, "initial full snapshot was lost or replayed");
|
||||||
|
while let Ok(event) = observations.try_recv() {
|
||||||
|
assert!(
|
||||||
|
!markers
|
||||||
|
.iter()
|
||||||
|
.any(|marker| is_marker(&event.payload, marker)),
|
||||||
|
"duplicate after tail fence"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Ok(observations)
|
||||||
|
})
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn partial_registration_recovers_same_controller(hub: bool) {
|
||||||
|
let mut fixture = Fixture::no_model();
|
||||||
|
Arc::get_mut(&mut fixture.backend)
|
||||||
|
.unwrap()
|
||||||
|
.spawn_restore_timeout = Duration::from_millis(250);
|
||||||
|
let runtime = stopped_runtime(&fixture);
|
||||||
|
let pause = runtime.test_pause_restore_continuation();
|
||||||
|
let worker_ref = fixture.request.worker_ref.clone();
|
||||||
|
let scope = fixture.request.workspace_scope.as_ref().unwrap();
|
||||||
|
let request = runtime.test_restore_request(&worker_ref);
|
||||||
|
let observations = runtime.subscribe_worker_observation().unwrap();
|
||||||
|
let (started_release, started_entered) = activation_gate(&fixture, true);
|
||||||
|
let first = runtime
|
||||||
|
.restore_worker_operation_scoped(scope, &worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
first.state,
|
||||||
|
server_api::WorkerRestoreState::ReconciliationRequired
|
||||||
|
);
|
||||||
|
let started = started_entered.recv_timeout(WAIT).unwrap();
|
||||||
|
let pending = assert_activation_owned(&fixture, &runtime, &request.request_id, &started);
|
||||||
|
let old = started.handle.as_ref().unwrap().clone();
|
||||||
|
let canonical = fixture.canonical_files();
|
||||||
|
alert_marker(&old, "registration-original-paired-prefix");
|
||||||
|
let (release, entered, fault) = publication_gate();
|
||||||
|
if hub {
|
||||||
|
*fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.observation_hub
|
||||||
|
.registration_gate
|
||||||
|
.lock()
|
||||||
|
.unwrap() = Some(RestoreHubRegistrationGate {
|
||||||
|
gate: fault,
|
||||||
|
fail: true,
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
*fixture.backend.bridge_fault.lock().unwrap() = Some(fault);
|
||||||
|
}
|
||||||
|
started_release.release();
|
||||||
|
drop(pause);
|
||||||
|
let boundary_handle = entered.recv_timeout(WAIT).unwrap();
|
||||||
|
assert!(boundary_handle.same_controller(&old));
|
||||||
|
let durable = durable_operation(&runtime, &worker_ref).unwrap();
|
||||||
|
assert_eq!(durable.operation_id, started.operation_id);
|
||||||
|
assert_eq!(durable.commit, crate::runtime::RestoreCommitFact::Committed);
|
||||||
|
assert!(restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"].is_null());
|
||||||
|
if hub {
|
||||||
|
let (_, shared, _) = fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.observation_hub
|
||||||
|
.get(&worker_ref)
|
||||||
|
.unwrap();
|
||||||
|
assert!(
|
||||||
|
Arc::ptr_eq(&shared, &old.shared_state),
|
||||||
|
"real hub insert preceded fault"
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
assert!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.observation_hub
|
||||||
|
.get(&worker_ref)
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let (operation, context, uncertain) = runtime.test_pending_restore_context(&worker_ref);
|
||||||
|
assert_eq!(operation, started.operation_id);
|
||||||
|
assert!(!uncertain);
|
||||||
|
assert!(
|
||||||
|
context
|
||||||
|
.test_buffered_events()
|
||||||
|
.iter()
|
||||||
|
.any(|event| matches!(event, Event::Snapshot { .. }))
|
||||||
|
);
|
||||||
|
alert_marker(&old, "registration-during-fault");
|
||||||
|
let pause = runtime.test_pause_restore_continuation();
|
||||||
|
release.send(()).unwrap();
|
||||||
|
// This lock also observes the end of the actual registration attempt, not
|
||||||
|
// merely the test's entered notification. Its retained scope owns cleanup.
|
||||||
|
let execution = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
|
||||||
|
assert!(old.same_controller(&execution.handle));
|
||||||
|
assert_eq!(execution.restore_operation_id, Some(operation));
|
||||||
|
assert_eq!(
|
||||||
|
execution.tasks.tasks.lock().unwrap().len(),
|
||||||
|
if hub { 2 } else { 1 }
|
||||||
|
);
|
||||||
|
let bridge = execution.tasks.protocol_bridge.lock().unwrap();
|
||||||
|
if hub {
|
||||||
|
assert!(bridge.is_none());
|
||||||
|
} else {
|
||||||
|
let bridge = bridge.as_ref().unwrap();
|
||||||
|
assert!(
|
||||||
|
!bridge.snapshot_pending,
|
||||||
|
"snapshot already registered, do not replay it"
|
||||||
|
);
|
||||||
|
let streams = bridge.streams.lock().unwrap();
|
||||||
|
assert!(
|
||||||
|
streams.is_some(),
|
||||||
|
"original paired streams survived failed task admission"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(&streams.as_ref().unwrap().snapshot_event).unwrap(),
|
||||||
|
serde_json::to_value(&started.snapshot).unwrap()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
drop(bridge);
|
||||||
|
assert!(
|
||||||
|
!old.protocol_is_closed(),
|
||||||
|
"publication failure must not clean a committed Controller"
|
||||||
|
);
|
||||||
|
assert_single_publication(&fixture, 1);
|
||||||
|
drop(pause);
|
||||||
|
runtime.test_wait_restore_continuation(&worker_ref, operation, WAIT);
|
||||||
|
assert!(durable_operation(&runtime, &worker_ref).is_none());
|
||||||
|
let owner = restore_owner(&runtime, &worker_ref, &request.request_id);
|
||||||
|
assert_eq!(
|
||||||
|
owner["operation_id"],
|
||||||
|
serde_json::to_value(operation).unwrap()
|
||||||
|
);
|
||||||
|
assert_eq!(owner["receipt"]["state"], "accepted");
|
||||||
|
assert!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.pending_restore(&worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert!(!pending.task_in_flight.load(Ordering::Acquire));
|
||||||
|
assert!(pending.activated.lock().unwrap().is_none());
|
||||||
|
assert!(pending.result.lock().unwrap().is_none());
|
||||||
|
assert!(pending.task.try_lock().unwrap().is_none());
|
||||||
|
assert_eq!(fixture.backend.workers.lock().unwrap().len(), 1);
|
||||||
|
assert!(old.same_controller(&fixture.backend.workers.lock().unwrap()[&worker_ref].handle));
|
||||||
|
let transport = assert_real_attach(&fixture, &old);
|
||||||
|
assert_eq!(fixture.canonical_files(), canonical);
|
||||||
|
assert_single_publication(&fixture, 1);
|
||||||
|
alert_marker(&old, "registration-tail-fence");
|
||||||
|
let observations = observe_publication_markers(
|
||||||
|
&fixture,
|
||||||
|
observations,
|
||||||
|
&[
|
||||||
|
"registration-original-paired-prefix",
|
||||||
|
"registration-during-fault",
|
||||||
|
"registration-tail-fence",
|
||||||
|
],
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.restore_worker_operation_scoped(scope, &worker_ref, request.clone())
|
||||||
|
.unwrap()
|
||||||
|
.state,
|
||||||
|
server_api::WorkerRestoreState::Accepted
|
||||||
|
);
|
||||||
|
assert_single_publication(&fixture, 1);
|
||||||
|
// Retain the original transport/handle and scope through Stop/replacement.
|
||||||
|
runtime
|
||||||
|
.stop_worker_scoped(scope, &worker_ref, None)
|
||||||
|
.unwrap();
|
||||||
|
assert!(old.protocol_is_closed());
|
||||||
|
assert!(transport.validate().is_err());
|
||||||
|
let replacement_request = runtime.test_restore_request(&worker_ref);
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.restore_worker_operation_scoped(scope, &worker_ref, replacement_request.clone())
|
||||||
|
.unwrap()
|
||||||
|
.state,
|
||||||
|
server_api::WorkerRestoreState::Accepted
|
||||||
|
);
|
||||||
|
let replacement = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
|
||||||
|
assert!(!old.same_controller(&replacement.handle));
|
||||||
|
assert_ne!(replacement.restore_operation_id, Some(operation));
|
||||||
|
assert!(
|
||||||
|
transport
|
||||||
|
.dispatch(Method::Pause {
|
||||||
|
command: protocol::WorkerCommandEnvelope::new(801)
|
||||||
|
})
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
transport
|
||||||
|
.dispatch(Method::Shutdown {
|
||||||
|
command: protocol::WorkerCommandEnvelope::new(802)
|
||||||
|
})
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
for method in [
|
||||||
|
Method::Pause {
|
||||||
|
command: protocol::WorkerCommandEnvelope::new(803),
|
||||||
|
},
|
||||||
|
Method::Shutdown {
|
||||||
|
command: protocol::WorkerCommandEnvelope::new(804),
|
||||||
|
},
|
||||||
|
] {
|
||||||
|
let old_command = old.clone();
|
||||||
|
assert!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.run_on_adapter_runtime(async move {
|
||||||
|
old_command
|
||||||
|
.send(method)
|
||||||
|
.await
|
||||||
|
.map_err(|error| error.to_string())
|
||||||
|
})
|
||||||
|
.is_err(),
|
||||||
|
"closed original Controller command must not route to replacement"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.deactivate_stopped_controller(&worker_ref, &old)
|
||||||
|
.unwrap();
|
||||||
|
let (_, shared, _) = fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.observation_hub
|
||||||
|
.get(&worker_ref)
|
||||||
|
.unwrap();
|
||||||
|
assert!(Arc::ptr_eq(&shared, &replacement.handle.shared_state));
|
||||||
|
alert_marker(&old, "registration-stopped-event-must-not-leak");
|
||||||
|
alert_marker(&replacement.handle, "registration-replacement-fence");
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.run_on_adapter_runtime(async move {
|
||||||
|
let mut observations = observations;
|
||||||
|
tokio::time::timeout(WAIT, async {
|
||||||
|
loop {
|
||||||
|
let event = observations.recv().await.unwrap();
|
||||||
|
assert!(!is_marker(
|
||||||
|
&event.payload,
|
||||||
|
"registration-stopped-event-must-not-leak"
|
||||||
|
));
|
||||||
|
if is_marker(&event.payload, "registration-replacement-fence") {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(|_| "replacement observation fence missing".to_string())?;
|
||||||
|
drop(transport);
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_single_publication(&fixture, 2);
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.restore_worker_operation_scoped(scope, &worker_ref, request.clone())
|
||||||
|
.unwrap()
|
||||||
|
.state,
|
||||||
|
server_api::WorkerRestoreState::Accepted
|
||||||
|
);
|
||||||
|
assert_eq!(restore_owner_count(&runtime, &worker_ref), 2);
|
||||||
|
let current = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
|
||||||
|
assert!(current.handle.same_controller(&replacement.handle));
|
||||||
|
assert_eq!(
|
||||||
|
current.restore_operation_id,
|
||||||
|
replacement.restore_operation_id
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
replacement.handle.shared_state.snapshot().state,
|
||||||
|
protocol::WorkerState::Idle
|
||||||
|
);
|
||||||
|
drop(current);
|
||||||
|
drop(execution);
|
||||||
|
drop(replacement);
|
||||||
|
drop(pending);
|
||||||
|
runtime
|
||||||
|
.stop_worker_scoped(scope, &worker_ref, None)
|
||||||
|
.unwrap();
|
||||||
|
fixture.assert_released();
|
||||||
|
}
|
||||||
|
|
||||||
|
fn restore_owner_count(runtime: &EmbeddedRuntime, worker_ref: &WorkerRef) -> usize {
|
||||||
|
let saved = runtime
|
||||||
|
.test_runtime_store()
|
||||||
|
.read_worker_record(worker_ref)
|
||||||
|
.unwrap();
|
||||||
|
serde_json::to_value(saved.restore_guard).unwrap()["owners"]
|
||||||
|
.as_object()
|
||||||
|
.unwrap()
|
||||||
|
.len()
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn bridge_partial_snapshot_registration_retains_original_streams_and_autonomously_recovers() {
|
||||||
|
partial_registration_recovers_same_controller(false);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn hub_partial_insert_registration_retains_original_controller_and_autonomously_recovers() {
|
||||||
|
partial_registration_recovers_same_controller(true);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn context_partial_flush_retains_only_undelivered_queue_and_recovers_without_repost() {
|
||||||
|
let mut fixture = Fixture::no_model();
|
||||||
|
Arc::get_mut(&mut fixture.backend)
|
||||||
|
.unwrap()
|
||||||
|
.spawn_restore_timeout = Duration::from_millis(250);
|
||||||
|
let runtime = stopped_runtime(&fixture);
|
||||||
|
let pause = runtime.test_pause_restore_continuation();
|
||||||
|
let worker_ref = fixture.request.worker_ref.clone();
|
||||||
|
let scope = fixture.request.workspace_scope.as_ref().unwrap();
|
||||||
|
let request = runtime.test_restore_request(&worker_ref);
|
||||||
|
let observations = runtime.subscribe_worker_observation().unwrap();
|
||||||
|
let (started_release, started_entered) = activation_gate(&fixture, true);
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.restore_worker_operation_scoped(scope, &worker_ref, request.clone())
|
||||||
|
.unwrap()
|
||||||
|
.state,
|
||||||
|
server_api::WorkerRestoreState::ReconciliationRequired
|
||||||
|
);
|
||||||
|
let started = started_entered.recv_timeout(WAIT).unwrap();
|
||||||
|
let pending = assert_activation_owned(&fixture, &runtime, &request.request_id, &started);
|
||||||
|
let old = started.handle.unwrap();
|
||||||
|
let (operation, context, _) = runtime.test_pending_restore_context(&worker_ref);
|
||||||
|
// Stop after the actual bridge and hub registration, without failing that
|
||||||
|
// boundary, so the flush begins with its real original full snapshot.
|
||||||
|
let (hub_release, hub_entered, hub_gate) = publication_gate();
|
||||||
|
*fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.observation_hub
|
||||||
|
.registration_gate
|
||||||
|
.lock()
|
||||||
|
.unwrap() = Some(RestoreHubRegistrationGate {
|
||||||
|
gate: hub_gate,
|
||||||
|
fail: false,
|
||||||
|
});
|
||||||
|
started_release.release();
|
||||||
|
drop(pause);
|
||||||
|
assert!(
|
||||||
|
hub_entered
|
||||||
|
.recv_timeout(WAIT)
|
||||||
|
.unwrap()
|
||||||
|
.same_controller(&old)
|
||||||
|
);
|
||||||
|
let buffered_prefix = context.test_buffered_events();
|
||||||
|
assert!(matches!(
|
||||||
|
buffered_prefix.first(),
|
||||||
|
Some(Event::Snapshot { .. })
|
||||||
|
));
|
||||||
|
let prefix_length = buffered_prefix.len();
|
||||||
|
let marker = |text: &str| Event::Error {
|
||||||
|
code: protocol::ErrorCode::Internal,
|
||||||
|
message: text.into(),
|
||||||
|
};
|
||||||
|
context
|
||||||
|
.publish_protocol_event(marker("flush-delivered"))
|
||||||
|
.unwrap();
|
||||||
|
context
|
||||||
|
.publish_protocol_event(marker("flush-failed"))
|
||||||
|
.unwrap();
|
||||||
|
context
|
||||||
|
.publish_protocol_event(marker("flush-pending"))
|
||||||
|
.unwrap();
|
||||||
|
let (release, entered, gate) = publication_gate();
|
||||||
|
let (restored, restored_queue) = mpsc::channel();
|
||||||
|
context.test_fail_flush(crate::execution::RestoreFlushFault {
|
||||||
|
remaining_successes: prefix_length + 1,
|
||||||
|
gate,
|
||||||
|
restored,
|
||||||
|
});
|
||||||
|
drop(hub_release);
|
||||||
|
assert!(
|
||||||
|
matches!(entered.recv_timeout(WAIT).unwrap(), Event::Error { message, .. } if message == "flush-failed")
|
||||||
|
);
|
||||||
|
let partial_execution = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
|
||||||
|
assert!(old.same_controller(&partial_execution.handle));
|
||||||
|
assert_eq!(partial_execution.restore_operation_id, Some(operation));
|
||||||
|
assert_eq!(partial_execution.tasks.tasks.lock().unwrap().len(), 2);
|
||||||
|
let (_, shared, _) = fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.observation_hub
|
||||||
|
.get(&worker_ref)
|
||||||
|
.unwrap();
|
||||||
|
assert!(Arc::ptr_eq(&shared, &old.shared_state));
|
||||||
|
let pause = runtime.test_pause_restore_continuation();
|
||||||
|
context
|
||||||
|
.publish_protocol_event(marker("flush-arrived-during-error"))
|
||||||
|
.unwrap();
|
||||||
|
release.send(()).unwrap();
|
||||||
|
let restored = restored_queue.recv_timeout(WAIT).unwrap();
|
||||||
|
let undelivered: Vec<_> = restored
|
||||||
|
.iter()
|
||||||
|
.filter_map(|event| match event {
|
||||||
|
Event::Error { message, .. } if message.starts_with("flush-") => Some(message.as_str()),
|
||||||
|
_ => None,
|
||||||
|
})
|
||||||
|
.collect();
|
||||||
|
assert_eq!(
|
||||||
|
undelivered,
|
||||||
|
[
|
||||||
|
"flush-failed",
|
||||||
|
"flush-pending",
|
||||||
|
"flush-arrived-during-error"
|
||||||
|
]
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!restored
|
||||||
|
.iter()
|
||||||
|
.any(|event| matches!(event, Event::Snapshot { .. })),
|
||||||
|
"delivered snapshot must not rejoin the failed suffix"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(context.test_buffered_events()).unwrap(),
|
||||||
|
serde_json::to_value(&restored).unwrap()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
durable_operation(&runtime, &worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.operation_id,
|
||||||
|
operation
|
||||||
|
);
|
||||||
|
assert!(restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"].is_null());
|
||||||
|
// The next terminal gate observes successful retry only after the same
|
||||||
|
// queued owner has repaired its failed callback and remaining suffix.
|
||||||
|
let (terminal_release, terminal_entered, gate) = publication_gate();
|
||||||
|
runtime.test_terminal_receipt_gate(gate);
|
||||||
|
drop(pause);
|
||||||
|
assert_eq!(terminal_entered.recv_timeout(WAIT).unwrap(), operation);
|
||||||
|
assert!(context.test_buffered_events().is_empty());
|
||||||
|
assert!(restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"].is_null());
|
||||||
|
assert_eq!(
|
||||||
|
durable_operation(&runtime, &worker_ref).unwrap().commit,
|
||||||
|
crate::runtime::RestoreCommitFact::Committed
|
||||||
|
);
|
||||||
|
let execution = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
|
||||||
|
assert!(old.same_controller(&execution.handle));
|
||||||
|
assert_eq!(execution.restore_operation_id, Some(operation));
|
||||||
|
drop(terminal_release); // RAII disconnect, not a second request.
|
||||||
|
runtime.test_wait_restore_continuation(&worker_ref, operation, WAIT);
|
||||||
|
assert_eq!(
|
||||||
|
restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"]["state"],
|
||||||
|
"accepted"
|
||||||
|
);
|
||||||
|
assert_real_attach(&fixture, &old);
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.run_on_adapter_runtime(async move {
|
||||||
|
let mut observations = observations;
|
||||||
|
let mut messages = Vec::new();
|
||||||
|
let mut snapshots = 0;
|
||||||
|
while let Ok(event) = observations.try_recv() {
|
||||||
|
match event.payload {
|
||||||
|
Event::Error { message, .. } if message.starts_with("flush-") => {
|
||||||
|
messages.push(message)
|
||||||
|
}
|
||||||
|
Event::Snapshot { .. } => snapshots += 1,
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert_eq!(
|
||||||
|
messages,
|
||||||
|
[
|
||||||
|
"flush-delivered",
|
||||||
|
"flush-failed",
|
||||||
|
"flush-pending",
|
||||||
|
"flush-arrived-during-error"
|
||||||
|
]
|
||||||
|
);
|
||||||
|
assert_eq!(snapshots, 1);
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_single_publication(&fixture, 1);
|
||||||
|
drop(partial_execution);
|
||||||
|
drop(execution);
|
||||||
|
drop(pending);
|
||||||
|
runtime
|
||||||
|
.stop_worker_scoped(scope, &worker_ref, None)
|
||||||
|
.unwrap();
|
||||||
|
fixture.assert_released();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Copy, Debug)]
|
||||||
|
enum TerminalFault {
|
||||||
|
BeforeRename,
|
||||||
|
AfterRename,
|
||||||
|
SyncUnknown,
|
||||||
|
ReadUnknown,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn published_terminal_receipt_fault(fault: TerminalFault) {
|
||||||
|
let mut fixture = Fixture::no_model();
|
||||||
|
// Reuse the existing ordinary non-network model boundary, not the product
|
||||||
|
// Codex profile. All fault/convergence assertions precede the sole input.
|
||||||
|
let script = MockClient::sequential(vec![MockResponse::Complete(simple_text_events())]);
|
||||||
|
let injected = script.clone();
|
||||||
|
let factory = fixture
|
||||||
|
.backend
|
||||||
|
.factory
|
||||||
|
.as_ref()
|
||||||
|
.clone()
|
||||||
|
.with_spawn_model_client_factory(move || Box::new(injected.clone()));
|
||||||
|
fixture.backend = Arc::new(WorkerRuntimeExecutionBackend::new(factory).unwrap());
|
||||||
|
let runtime = Arc::new(stopped_runtime(&fixture));
|
||||||
|
let pause = runtime.test_pause_restore_continuation();
|
||||||
|
let worker_ref = fixture.request.worker_ref.clone();
|
||||||
|
let scope = fixture.request.workspace_scope.clone().unwrap();
|
||||||
|
let request = runtime.test_restore_request(&worker_ref);
|
||||||
|
let observations = runtime.subscribe_worker_observation().unwrap();
|
||||||
|
let mut catalog = runtime
|
||||||
|
.subscribe_event_selector_scoped(
|
||||||
|
&scope,
|
||||||
|
protocol::subscription::EventSubscriptionSelector::RuntimeWorkers,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let (release, entered, gate) = publication_gate();
|
||||||
|
runtime.test_terminal_receipt_gate(gate);
|
||||||
|
let restoring = {
|
||||||
|
let runtime = runtime.clone();
|
||||||
|
let worker_ref = worker_ref.clone();
|
||||||
|
let scope = scope.clone();
|
||||||
|
let request = request.clone();
|
||||||
|
std::thread::spawn(move || {
|
||||||
|
runtime.restore_worker_operation_scoped(&scope, &worker_ref, request)
|
||||||
|
})
|
||||||
|
};
|
||||||
|
let operation = entered.recv_timeout(WAIT).unwrap();
|
||||||
|
// Restore legitimately repairs derived metadata before publication. Receipt
|
||||||
|
// failures must not rerun those effects; compare from this actual boundary.
|
||||||
|
let canonical = fixture.canonical_files();
|
||||||
|
let execution = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
|
||||||
|
let old = execution.handle.clone();
|
||||||
|
assert_eq!(execution.restore_operation_id, Some(operation));
|
||||||
|
assert!(
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.pending_restore(&worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert_eq!(execution.tasks.tasks.lock().unwrap().len(), 2);
|
||||||
|
assert!(execution.tasks.protocol_bridge.lock().unwrap().is_none());
|
||||||
|
assert_eq!(
|
||||||
|
durable_operation(&runtime, &worker_ref)
|
||||||
|
.unwrap()
|
||||||
|
.operation_id,
|
||||||
|
operation
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
durable_operation(&runtime, &worker_ref).unwrap().commit,
|
||||||
|
crate::runtime::RestoreCommitFact::Committed
|
||||||
|
);
|
||||||
|
let owner = restore_owner(&runtime, &worker_ref, &request.request_id);
|
||||||
|
assert!(owner["receipt"].is_null());
|
||||||
|
let (same_operation, context, uncertain) = runtime.test_pending_restore_context(&worker_ref);
|
||||||
|
assert_eq!(same_operation, operation);
|
||||||
|
assert!(!uncertain);
|
||||||
|
assert!(context.test_buffered_events().is_empty());
|
||||||
|
let transport = assert_real_attach(&fixture, &old);
|
||||||
|
assert_single_publication(&fixture, 1);
|
||||||
|
alert_marker(&old, "terminal-before-receipt");
|
||||||
|
let store = runtime.test_runtime_store();
|
||||||
|
let reconcile_before = store.test_worker_reconcile_counts();
|
||||||
|
store.fail_next_worker_write(if matches!(fault, TerminalFault::BeforeRename) {
|
||||||
|
AtomicWriteFault::BeforeRename
|
||||||
|
} else {
|
||||||
|
AtomicWriteFault::AfterRename
|
||||||
|
});
|
||||||
|
match fault {
|
||||||
|
TerminalFault::SyncUnknown => store.fail_next_worker_reconcile_sync(),
|
||||||
|
TerminalFault::ReadUnknown => store.fail_next_worker_reconcile_read(),
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
release.send(()).unwrap();
|
||||||
|
let first = restoring.join().unwrap().unwrap();
|
||||||
|
let reconcile_after = store.test_worker_reconcile_counts();
|
||||||
|
let reconciliation = (
|
||||||
|
reconcile_after.0 - reconcile_before.0,
|
||||||
|
reconcile_after.1 - reconcile_before.1,
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
reconciliation,
|
||||||
|
match fault {
|
||||||
|
TerminalFault::BeforeRename => (0, 0),
|
||||||
|
TerminalFault::AfterRename => (1, 4), // canonical file plus Worker/workers/root directories
|
||||||
|
TerminalFault::SyncUnknown => (1, 1),
|
||||||
|
TerminalFault::ReadUnknown => (1, 0),
|
||||||
|
},
|
||||||
|
"{fault:?}: actual canonical read/sync attempts"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
first.state,
|
||||||
|
if matches!(fault, TerminalFault::AfterRename) {
|
||||||
|
server_api::WorkerRestoreState::Accepted
|
||||||
|
} else {
|
||||||
|
server_api::WorkerRestoreState::ReconciliationRequired
|
||||||
|
},
|
||||||
|
"{fault:?}: {first:?}"
|
||||||
|
);
|
||||||
|
assert_eq!(fixture.canonical_files(), canonical);
|
||||||
|
assert!(!old.protocol_is_closed());
|
||||||
|
transport.validate().unwrap();
|
||||||
|
assert_single_publication(&fixture, 1);
|
||||||
|
let disk_owner = restore_owner(&runtime, &worker_ref, &request.request_id);
|
||||||
|
assert_eq!(disk_owner["operation_id"], owner["operation_id"]);
|
||||||
|
match fault {
|
||||||
|
TerminalFault::BeforeRename => {
|
||||||
|
assert!(disk_owner["receipt"].is_null());
|
||||||
|
assert!(!runtime.test_pending_restore_context(&worker_ref).2);
|
||||||
|
}
|
||||||
|
TerminalFault::AfterRename => {
|
||||||
|
assert_eq!(disk_owner["receipt"]["state"], "accepted");
|
||||||
|
assert!(durable_operation(&runtime, &worker_ref).is_none());
|
||||||
|
}
|
||||||
|
TerminalFault::SyncUnknown | TerminalFault::ReadUnknown => {
|
||||||
|
assert_eq!(
|
||||||
|
disk_owner["receipt"]["state"], "accepted",
|
||||||
|
"visible rename alone is not durable proof"
|
||||||
|
);
|
||||||
|
let (same_operation, _, fenced) = runtime.test_pending_restore_context(&worker_ref);
|
||||||
|
assert_eq!(same_operation, operation);
|
||||||
|
assert!(
|
||||||
|
fenced,
|
||||||
|
"unknown canonical outcome must retain the memory owner/write fence"
|
||||||
|
);
|
||||||
|
// Keep durability unproved for one competing Stop as well: it must
|
||||||
|
// fail before any Shutdown, never manufacture a Stopped judgment.
|
||||||
|
match fault {
|
||||||
|
TerminalFault::SyncUnknown => store.fail_next_worker_reconcile_sync(),
|
||||||
|
TerminalFault::ReadUnknown => store.fail_next_worker_reconcile_read(),
|
||||||
|
_ => unreachable!(),
|
||||||
|
}
|
||||||
|
assert!(matches!(
|
||||||
|
runtime.stop_worker_scoped(&scope, &worker_ref, None),
|
||||||
|
Err(crate::error::RuntimeError::StoreCommitOutcomeUnknown { .. })
|
||||||
|
));
|
||||||
|
assert!(!old.protocol_is_closed());
|
||||||
|
assert!(!execution.shutdown_requested.load(Ordering::Acquire));
|
||||||
|
assert!(runtime.test_pending_restore_context(&worker_ref).2);
|
||||||
|
assert_single_publication(&fixture, 1);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Recovery is autonomous after the failed write / confirmed unknown fence.
|
||||||
|
// No POST, fresh request, or input is needed to make it converge.
|
||||||
|
drop(pause);
|
||||||
|
if !matches!(fault, TerminalFault::AfterRename) {
|
||||||
|
runtime.test_wait_restore_continuation(&worker_ref, operation, WAIT);
|
||||||
|
}
|
||||||
|
assert!(durable_operation(&runtime, &worker_ref).is_none());
|
||||||
|
assert_eq!(
|
||||||
|
restore_owner(&runtime, &worker_ref, &request.request_id)["receipt"]["state"],
|
||||||
|
"accepted"
|
||||||
|
);
|
||||||
|
let current = fixture.backend.workers.lock().unwrap()[&worker_ref].clone();
|
||||||
|
assert!(old.same_controller(¤t.handle));
|
||||||
|
assert_eq!(current.restore_operation_id, Some(operation));
|
||||||
|
assert_eq!(fixture.backend.workers.lock().unwrap().len(), 1);
|
||||||
|
assert_real_attach(&fixture, &old);
|
||||||
|
assert_single_publication(&fixture, 1);
|
||||||
|
// A canonical terminal reread may bypass the final redundant upsert. The
|
||||||
|
// original subscriber must still have the current state and observation token.
|
||||||
|
let fresh_catalog = runtime
|
||||||
|
.subscribe_event_selector_scoped(
|
||||||
|
&scope,
|
||||||
|
protocol::subscription::EventSubscriptionSelector::RuntimeWorkers,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let protocol::subscription::SubscriptionSnapshot::Workers { workers } =
|
||||||
|
fresh_catalog.snapshot()
|
||||||
|
else {
|
||||||
|
panic!("Runtime catalog must return Workers");
|
||||||
|
};
|
||||||
|
assert_eq!(workers.len(), 1);
|
||||||
|
let mut last_applied = None;
|
||||||
|
use futures::FutureExt;
|
||||||
|
while let Some(update) = catalog.recv().now_or_never() {
|
||||||
|
match update.unwrap().payload {
|
||||||
|
protocol::subscription::SubscriptionEventPayload::WorkerUpserted { worker } => {
|
||||||
|
last_applied = Some(worker);
|
||||||
|
}
|
||||||
|
payload => panic!("unexpected catalog update: {payload:?}"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
serde_json::to_value(last_applied.expect("restore must publish its catalog state"))
|
||||||
|
.unwrap()
|
||||||
|
== serde_json::to_value(&workers[0]).unwrap(),
|
||||||
|
"original catalog subscriber diverged after {fault:?}"
|
||||||
|
);
|
||||||
|
drop(fresh_catalog);
|
||||||
|
drop(catalog);
|
||||||
|
alert_marker(&old, "terminal-after-recovery");
|
||||||
|
let _observations = observe_publication_markers(
|
||||||
|
&fixture,
|
||||||
|
observations,
|
||||||
|
&["terminal-before-receipt", "terminal-after-recovery"],
|
||||||
|
);
|
||||||
|
let accepted = runtime
|
||||||
|
.restore_worker_operation_scoped(&scope, &worker_ref, request.clone())
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(accepted.state, server_api::WorkerRestoreState::Accepted);
|
||||||
|
assert_eq!(
|
||||||
|
runtime
|
||||||
|
.restore_worker_operation_scoped(&scope, &worker_ref, request.clone())
|
||||||
|
.unwrap(),
|
||||||
|
accepted
|
||||||
|
);
|
||||||
|
assert_eq!(restore_owner_count(&runtime, &worker_ref), 1);
|
||||||
|
let mut stale = request;
|
||||||
|
stale.request_id = "terminal-fresh-stale-intent".into();
|
||||||
|
assert!(matches!(
|
||||||
|
runtime.restore_worker_operation_scoped(&scope, &worker_ref, stale),
|
||||||
|
Err(crate::error::RuntimeError::RestoreObservationConflict { .. })
|
||||||
|
));
|
||||||
|
assert_single_publication(&fixture, 1);
|
||||||
|
assert_eq!(
|
||||||
|
script.call_count.load(Ordering::SeqCst),
|
||||||
|
0,
|
||||||
|
"receipt recovery cannot trigger input or a model call"
|
||||||
|
);
|
||||||
|
// This transport was paired before the failed receipt write. Exercise its
|
||||||
|
// normal log-entry receiver after autonomous settlement, not a new attach.
|
||||||
|
transport
|
||||||
|
.dispatch(Method::submit_text(
|
||||||
|
"terminal-ordinary-input",
|
||||||
|
"terminal-ordinary-input",
|
||||||
|
))
|
||||||
|
.unwrap();
|
||||||
|
fixture
|
||||||
|
.backend
|
||||||
|
.run_on_adapter_runtime(async move {
|
||||||
|
let mut transport = transport;
|
||||||
|
let (mut admitted, mut committed, mut text, mut ended) = (false, false, false, false);
|
||||||
|
tokio::time::timeout(WAIT, async {
|
||||||
|
while !(admitted && committed && text && ended) {
|
||||||
|
match transport.events.recv().await.unwrap() {
|
||||||
|
Event::SubmissionAccepted {
|
||||||
|
submission_request_id,
|
||||||
|
..
|
||||||
|
} if submission_request_id == "terminal-ordinary-input" => admitted = true,
|
||||||
|
Event::UserMessage { segments, .. } => {
|
||||||
|
assert_eq!(
|
||||||
|
segments,
|
||||||
|
vec![protocol::Segment::text("terminal-ordinary-input")]
|
||||||
|
);
|
||||||
|
assert!(!committed, "paired log receiver duplicated input");
|
||||||
|
committed = true;
|
||||||
|
}
|
||||||
|
Event::TextDone { text: output } => {
|
||||||
|
assert_eq!(output, "hello from worker");
|
||||||
|
text = true;
|
||||||
|
}
|
||||||
|
Event::RunEnd { .. } => ended = true,
|
||||||
|
Event::SubmissionRejected { .. } | Event::Error { .. } => {
|
||||||
|
panic!("ordinary input failed after receipt recovery")
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.map_err(|_| {
|
||||||
|
"ordinary input failed to settle on original paired transport".to_string()
|
||||||
|
})?;
|
||||||
|
drop(transport);
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(script.call_count.load(Ordering::SeqCst), 1);
|
||||||
|
assert_eq!(fixture.entries().iter().filter(|entry| matches!(entry,
|
||||||
|
LogEntry::AnnotatedUserInput { segments, .. } if segments == &vec![protocol::Segment::text("terminal-ordinary-input")]
|
||||||
|
)).count(), 1);
|
||||||
|
assert_single_publication(&fixture, 1);
|
||||||
|
drop(execution);
|
||||||
|
drop(current);
|
||||||
|
runtime
|
||||||
|
.stop_worker_scoped(&scope, &worker_ref, None)
|
||||||
|
.unwrap();
|
||||||
|
assert!(old.protocol_is_closed());
|
||||||
|
fixture.assert_released();
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn published_controller_terminal_receipt_before_rename_autonomously_settles_without_reconstruction()
|
||||||
|
{
|
||||||
|
published_terminal_receipt_fault(TerminalFault::BeforeRename);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn published_controller_terminal_receipt_after_rename_is_canonically_reread_and_synced() {
|
||||||
|
published_terminal_receipt_fault(TerminalFault::AfterRename);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn published_controller_terminal_receipt_sync_unknown_retains_owner_and_write_fence() {
|
||||||
|
published_terminal_receipt_fault(TerminalFault::SyncUnknown);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
#[serial_test::serial(worker_allocation)]
|
||||||
|
fn published_controller_terminal_receipt_read_unknown_retains_owner_and_write_fence() {
|
||||||
|
published_terminal_receipt_fault(TerminalFault::ReadUnknown);
|
||||||
|
}
|
||||||
@@ -1,23 +1,32 @@
|
|||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
#[cfg(feature = "http-server")]
|
||||||
|
use std::time::Duration;
|
||||||
|
use std::time::{SystemTime, UNIX_EPOCH};
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
use crate::workspace_request::{
|
use crate::workspace_request::{
|
||||||
RuntimeWorkspaceRequest, RuntimeWorkspaceRequestClient, RuntimeWorkspaceRequestError,
|
RuntimeWorkspaceRequest, RuntimeWorkspaceRequestClient, RuntimeWorkspaceRequestError,
|
||||||
};
|
};
|
||||||
|
use worker::WorkspaceResponse;
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
use worker::{
|
use worker::{
|
||||||
WorkspaceBinaryRequest, WorkspaceBinaryResponse, WorkspaceClient, WorkspaceClientError,
|
WorkspaceBinaryRequest, WorkspaceBinaryResponse, WorkspaceClient, WorkspaceClientError,
|
||||||
WorkspacePromptCatalogResolution, WorkspacePromptProjection, WorkspaceRequest,
|
WorkspacePromptCatalogResolution, WorkspacePromptProjection, WorkspaceRequest,
|
||||||
WorkspaceRequestMethod, WorkspaceResponse, WorkspaceServerOperation,
|
WorkspaceRequestMethod, WorkspaceServerOperation,
|
||||||
};
|
};
|
||||||
|
|
||||||
use crate::auth::{
|
use crate::auth::{
|
||||||
RuntimeAuthError, RuntimeIdentityMaterial, RuntimeWorkerMutationSourceSigner,
|
RuntimeAuthError, RuntimeIdentityMaterial, RuntimeWorkerMutationSourceSigner,
|
||||||
WORKER_REMOVE_PERMISSION, WORKSPACE_REQUEST_PERMISSION, WORKSPACE_WORKER_DISCOVERY_PERMISSION,
|
WORKER_REMOVE_PERMISSION, WorkerMutationActorKind, WorkerMutationOperation,
|
||||||
WorkerMutationActorKind, WorkerMutationOperation, WorkerMutationSourceClaims, new_token_id,
|
WorkerMutationSourceClaims, new_token_id,
|
||||||
};
|
};
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
|
use crate::auth::{WORKSPACE_REQUEST_PERMISSION, WORKSPACE_WORKER_DISCOVERY_PERMISSION};
|
||||||
use crate::runtime::RuntimeWorkspaceScope;
|
use crate::runtime::RuntimeWorkspaceScope;
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
use crate::worker_backend::WorkspacePromptProjectionCache;
|
use crate::worker_backend::WorkspacePromptProjectionCache;
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
fn ticket_assignment_role_segment(role: server_api::TicketAssignmentRole) -> &'static str {
|
fn ticket_assignment_role_segment(role: server_api::TicketAssignmentRole) -> &'static str {
|
||||||
match role {
|
match role {
|
||||||
server_api::TicketAssignmentRole::Orchestrator => "orchestrator",
|
server_api::TicketAssignmentRole::Orchestrator => "orchestrator",
|
||||||
@@ -144,6 +153,7 @@ pub trait EmbeddedWorkerMutationDispatcher: Send + Sync {
|
|||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
enum RuntimeWorkerMutationTransport {
|
enum RuntimeWorkerMutationTransport {
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
Remote {
|
Remote {
|
||||||
request_client: RuntimeWorkspaceRequestClient,
|
request_client: RuntimeWorkspaceRequestClient,
|
||||||
},
|
},
|
||||||
@@ -161,6 +171,7 @@ pub struct RuntimeWorkerMutationForwarder {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl RuntimeWorkerMutationForwarder {
|
impl RuntimeWorkerMutationForwarder {
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
pub fn remote(
|
pub fn remote(
|
||||||
identity: &RuntimeIdentityMaterial,
|
identity: &RuntimeIdentityMaterial,
|
||||||
scope: RuntimeWorkspaceScope,
|
scope: RuntimeWorkspaceScope,
|
||||||
@@ -206,6 +217,7 @@ impl RuntimeWorkerMutationForwarder {
|
|||||||
target_worker_id,
|
target_worker_id,
|
||||||
)?;
|
)?;
|
||||||
match (&self.transport, proof) {
|
match (&self.transport, proof) {
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
(
|
(
|
||||||
RuntimeWorkerMutationTransport::Remote { request_client },
|
RuntimeWorkerMutationTransport::Remote { request_client },
|
||||||
RuntimeOwnedWorkerMutationProof::Remote(token),
|
RuntimeOwnedWorkerMutationProof::Remote(token),
|
||||||
@@ -231,6 +243,7 @@ impl RuntimeWorkerMutationForwarder {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
struct RemoteWorkerRemoveHttpRequest {
|
struct RemoteWorkerRemoveHttpRequest {
|
||||||
request_client: RuntimeWorkspaceRequestClient,
|
request_client: RuntimeWorkspaceRequestClient,
|
||||||
source_worker_id: String,
|
source_worker_id: String,
|
||||||
@@ -240,6 +253,7 @@ struct RemoteWorkerRemoveHttpRequest {
|
|||||||
reason: String,
|
reason: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
fn execute_remote_worker_remove_http(
|
fn execute_remote_worker_remove_http(
|
||||||
request: RemoteWorkerRemoveHttpRequest,
|
request: RemoteWorkerRemoveHttpRequest,
|
||||||
) -> Result<WorkspaceResponse, RuntimeWorkerMutationForwardError> {
|
) -> Result<WorkspaceResponse, RuntimeWorkerMutationForwardError> {
|
||||||
@@ -263,6 +277,7 @@ fn execute_remote_worker_remove_http(
|
|||||||
execute_remote_worker_remove_http_blocking(request)
|
execute_remote_worker_remove_http_blocking(request)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
fn execute_remote_worker_remove_http_blocking(
|
fn execute_remote_worker_remove_http_blocking(
|
||||||
request: RemoteWorkerRemoveHttpRequest,
|
request: RemoteWorkerRemoveHttpRequest,
|
||||||
) -> Result<WorkspaceResponse, RuntimeWorkerMutationForwardError> {
|
) -> Result<WorkspaceResponse, RuntimeWorkerMutationForwardError> {
|
||||||
@@ -315,6 +330,7 @@ fn execute_remote_worker_remove_http_blocking(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
pub struct RuntimeOwnedWorkspaceClient {
|
pub struct RuntimeOwnedWorkspaceClient {
|
||||||
workspace_id: String,
|
workspace_id: String,
|
||||||
@@ -325,6 +341,7 @@ pub struct RuntimeOwnedWorkspaceClient {
|
|||||||
prompt_projection_cache: Option<Arc<WorkspacePromptProjectionCache>>,
|
prompt_projection_cache: Option<Arc<WorkspacePromptProjectionCache>>,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
impl RuntimeOwnedWorkspaceClient {
|
impl RuntimeOwnedWorkspaceClient {
|
||||||
pub fn new(
|
pub fn new(
|
||||||
workspace_id: impl Into<String>,
|
workspace_id: impl Into<String>,
|
||||||
@@ -495,6 +512,7 @@ impl RuntimeOwnedWorkspaceClient {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
impl std::fmt::Debug for RuntimeOwnedWorkspaceClient {
|
impl std::fmt::Debug for RuntimeOwnedWorkspaceClient {
|
||||||
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
formatter
|
formatter
|
||||||
@@ -510,6 +528,7 @@ impl std::fmt::Debug for RuntimeOwnedWorkspaceClient {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
fn generated_workspace_response<T: serde::Serialize>(
|
fn generated_workspace_response<T: serde::Serialize>(
|
||||||
response: Result<T, server_api::client_support::ClientError<server_api::RepositoryApiError>>,
|
response: Result<T, server_api::client_support::ClientError<server_api::RepositoryApiError>>,
|
||||||
) -> Result<WorkspaceResponse, WorkspaceClientError> {
|
) -> Result<WorkspaceResponse, WorkspaceClientError> {
|
||||||
@@ -530,6 +549,7 @@ fn generated_workspace_response<T: serde::Serialize>(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
impl WorkspaceClient for RuntimeOwnedWorkspaceClient {
|
impl WorkspaceClient for RuntimeOwnedWorkspaceClient {
|
||||||
fn workspace_id(&self) -> Option<&str> {
|
fn workspace_id(&self) -> Option<&str> {
|
||||||
Some(self.request_client.workspace_id())
|
Some(self.request_client.workspace_id())
|
||||||
@@ -929,12 +949,17 @@ fn unix_now_seconds() -> u64 {
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
use crate::auth::{
|
use crate::auth::{
|
||||||
RUNTIME_REQUEST_SOURCE_PROOF_HEADER, WorkerMutationSourceExpectation,
|
RUNTIME_REQUEST_SOURCE_PROOF_HEADER, decode_runtime_request_source_claims,
|
||||||
decode_runtime_request_source_claims, decode_worker_mutation_source_claims,
|
request_body_digest,
|
||||||
request_body_digest, verify_worker_mutation_source_proof,
|
};
|
||||||
|
use crate::auth::{
|
||||||
|
WorkerMutationSourceExpectation, decode_worker_mutation_source_claims,
|
||||||
|
verify_worker_mutation_source_proof,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
fn binary_loopback(response: Vec<u8>) -> (String, std::thread::JoinHandle<(String, Vec<u8>)>) {
|
fn binary_loopback(response: Vec<u8>) -> (String, std::thread::JoinHandle<(String, Vec<u8>)>) {
|
||||||
use std::io::{Read, Write};
|
use std::io::{Read, Write};
|
||||||
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
let listener = std::net::TcpListener::bind("127.0.0.1:0").unwrap();
|
||||||
@@ -972,6 +997,7 @@ mod tests {
|
|||||||
(base_url, server)
|
(base_url, server)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
#[test]
|
#[test]
|
||||||
fn generated_ticket_assignment_transport_signs_exact_authoring_and_bounded_read_subjects() {
|
fn generated_ticket_assignment_transport_signs_exact_authoring_and_bounded_read_subjects() {
|
||||||
use crate::auth::{RuntimeRequestSourceExpectation, verify_runtime_request_source};
|
use crate::auth::{RuntimeRequestSourceExpectation, verify_runtime_request_source};
|
||||||
@@ -1104,6 +1130,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
#[test]
|
#[test]
|
||||||
fn binary_roundtrip_signs_exact_query_method_worker_and_raw_body() {
|
fn binary_roundtrip_signs_exact_query_method_worker_and_raw_body() {
|
||||||
use crate::auth::{RuntimeRequestSourceExpectation, verify_runtime_request_source};
|
use crate::auth::{RuntimeRequestSourceExpectation, verify_runtime_request_source};
|
||||||
@@ -1194,6 +1221,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
#[test]
|
#[test]
|
||||||
fn binary_responses_enforce_declared_and_streamed_limits_with_fixed_errors() {
|
fn binary_responses_enforce_declared_and_streamed_limits_with_fixed_errors() {
|
||||||
for response in [
|
for response in [
|
||||||
@@ -1213,6 +1241,7 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
#[test]
|
#[test]
|
||||||
fn binary_redirect_returns_status_without_forwarding_authentication() {
|
fn binary_redirect_returns_status_without_forwarding_authentication() {
|
||||||
let (base_url, server) = binary_loopback(b"HTTP/1.1 307 Temporary Redirect\r\nLocation: http://127.0.0.1:1/private\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_vec());
|
let (base_url, server) = binary_loopback(b"HTTP/1.1 307 Temporary Redirect\r\nLocation: http://127.0.0.1:1/private\r\nContent-Length: 0\r\nConnection: close\r\n\r\n".to_vec());
|
||||||
@@ -1231,6 +1260,7 @@ mod tests {
|
|||||||
assert!(result.body.is_empty());
|
assert!(result.body.is_empty());
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
#[test]
|
#[test]
|
||||||
fn binary_invalid_or_normalized_targets_fail_with_fixed_errors_before_http() {
|
fn binary_invalid_or_normalized_targets_fail_with_fixed_errors_before_http() {
|
||||||
let client = RuntimeOwnedWorkspaceClient::new(
|
let client = RuntimeOwnedWorkspaceClient::new(
|
||||||
@@ -1277,6 +1307,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
#[test]
|
#[test]
|
||||||
fn current_prompt_projection_uses_the_shared_runtime_cache_without_http() {
|
fn current_prompt_projection_uses_the_shared_runtime_cache_without_http() {
|
||||||
let cache = Arc::new(WorkspacePromptProjectionCache::default());
|
let cache = Arc::new(WorkspacePromptProjectionCache::default());
|
||||||
@@ -1312,6 +1343,7 @@ mod tests {
|
|||||||
assert!(Arc::ptr_eq(&projection.catalog, &second.catalog));
|
assert!(Arc::ptr_eq(&projection.catalog, &second.catalog));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
#[test]
|
#[test]
|
||||||
fn prompt_projection_cache_miss_fetches_content_identity() {
|
fn prompt_projection_cache_miss_fetches_content_identity() {
|
||||||
use std::io::{Read, Write};
|
use std::io::{Read, Write};
|
||||||
@@ -1356,6 +1388,7 @@ mod tests {
|
|||||||
assert_eq!(fetched.projection.source_digest, "source-3");
|
assert_eq!(fetched.projection.source_digest, "source-3");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
#[test]
|
#[test]
|
||||||
fn concurrent_prompt_projection_miss_fetches_once_and_shares_catalog() {
|
fn concurrent_prompt_projection_miss_fetches_once_and_shares_catalog() {
|
||||||
use std::io::{Read, Write};
|
use std::io::{Read, Write};
|
||||||
@@ -1427,6 +1460,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
#[test]
|
#[test]
|
||||||
fn current_prompt_projection_rejects_cross_workspace_response() {
|
fn current_prompt_projection_rejects_cross_workspace_response() {
|
||||||
use std::io::{Read, Write};
|
use std::io::{Read, Write};
|
||||||
@@ -1473,6 +1507,7 @@ mod tests {
|
|||||||
assert!(error.to_string().contains("scope mismatch"));
|
assert!(error.to_string().contains("scope mismatch"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
#[test]
|
#[test]
|
||||||
fn ordinary_workspace_forwarding_stamps_runtime_identity_and_signs_path_and_query() {
|
fn ordinary_workspace_forwarding_stamps_runtime_identity_and_signs_path_and_query() {
|
||||||
use std::io::{Read, Write};
|
use std::io::{Read, Write};
|
||||||
@@ -1531,6 +1566,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
#[test]
|
#[test]
|
||||||
fn workspace_worker_discovery_signs_dedicated_permission_and_encoded_query() {
|
fn workspace_worker_discovery_signs_dedicated_permission_and_encoded_query() {
|
||||||
use std::io::{Read, Write};
|
use std::io::{Read, Write};
|
||||||
@@ -1652,6 +1688,7 @@ mod tests {
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
#[tokio::test(flavor = "multi_thread")]
|
#[tokio::test(flavor = "multi_thread")]
|
||||||
async fn remote_forwarder_is_safe_in_async_runtime_and_stamps_signed_proof() {
|
async fn remote_forwarder_is_safe_in_async_runtime_and_stamps_signed_proof() {
|
||||||
use std::io::{Read, Write};
|
use std::io::{Read, Write};
|
||||||
@@ -1815,6 +1852,7 @@ mod tests {
|
|||||||
assert_eq!(reason, "retire obsolete Worker");
|
assert_eq!(reason, "retire obsolete Worker");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
#[test]
|
#[test]
|
||||||
fn runtime_owned_workspace_client_returns_successful_response_without_request_timeout() {
|
fn runtime_owned_workspace_client_returns_successful_response_without_request_timeout() {
|
||||||
// Effective deadline clearing is exercised with a short inherited deadline in
|
// Effective deadline clearing is exercised with a short inherited deadline in
|
||||||
@@ -1838,6 +1876,7 @@ mod tests {
|
|||||||
server.join().unwrap();
|
server.join().unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
#[test]
|
#[test]
|
||||||
fn runtime_owned_workspace_client_honors_per_request_timeouts() {
|
fn runtime_owned_workspace_client_honors_per_request_timeouts() {
|
||||||
let (base_url, server) = delayed_workspace_response(Duration::from_millis(75));
|
let (base_url, server) = delayed_workspace_response(Duration::from_millis(75));
|
||||||
@@ -1857,6 +1896,7 @@ mod tests {
|
|||||||
server.join().unwrap();
|
server.join().unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
#[test]
|
#[test]
|
||||||
fn runtime_owned_workspace_client_reports_request_timeouts() {
|
fn runtime_owned_workspace_client_reports_request_timeouts() {
|
||||||
let (base_url, server) = delayed_workspace_response(Duration::from_millis(75));
|
let (base_url, server) = delayed_workspace_response(Duration::from_millis(75));
|
||||||
@@ -1878,6 +1918,7 @@ mod tests {
|
|||||||
server.join().unwrap();
|
server.join().unwrap();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "http-server")]
|
||||||
fn delayed_workspace_response(delay: Duration) -> (String, std::thread::JoinHandle<()>) {
|
fn delayed_workspace_response(delay: Duration) -> (String, std::thread::JoinHandle<()>) {
|
||||||
use std::io::{Read, Write};
|
use std::io::{Read, Write};
|
||||||
use std::net::TcpListener;
|
use std::net::TcpListener;
|
||||||
|
|||||||
@@ -3676,6 +3676,7 @@ mod tests {
|
|||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn bound_ssh_workdir_uses_attachment_scoped_agent_and_cleans_it_up() {
|
fn bound_ssh_workdir_uses_attachment_scoped_agent_and_cleans_it_up() {
|
||||||
|
crate::restore_test_environment::require_restore_test_environment();
|
||||||
let repo = create_clean_repo();
|
let repo = create_clean_repo();
|
||||||
// ssh-agent and the broker receive relative socket paths in this owned
|
// ssh-agent and the broker receive relative socket paths in this owned
|
||||||
// fixture tree, so the test does not depend on checkout/TMPDIR length.
|
// fixture tree, so the test does not depend on checkout/TMPDIR length.
|
||||||
|
|||||||
@@ -28,7 +28,19 @@ pub(super) struct Lease {
|
|||||||
id: String,
|
id: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn busy() -> WorkingDirectoryDiagnostic {
|
fn busy(id: &str, action: &str, blocking: &Entry) -> WorkingDirectoryDiagnostic {
|
||||||
|
let (busy_reason, active_use_count) = match blocking {
|
||||||
|
Entry::Use(count) => ("active_use", *count),
|
||||||
|
Entry::Create => ("creation_in_progress", 0),
|
||||||
|
Entry::Cleanup => ("cleanup_in_progress", 0),
|
||||||
|
};
|
||||||
|
tracing::warn!(target: "yoi::workdir_cleanup",
|
||||||
|
working_directory_id = id,
|
||||||
|
action,
|
||||||
|
busy_source = "materializer_occupancy",
|
||||||
|
busy_reason,
|
||||||
|
active_use_count,
|
||||||
|
"Workdir operation blocked by busy resource");
|
||||||
WorkingDirectoryDiagnostic::new(
|
WorkingDirectoryDiagnostic::new(
|
||||||
"working_directory_cleanup_resource_busy",
|
"working_directory_cleanup_resource_busy",
|
||||||
"A filesystem resource is busy or changed during removal. Release the resource and retry.",
|
"A filesystem resource is busy or changed during removal. Release the resource and retry.",
|
||||||
@@ -42,7 +54,9 @@ impl Occupancy {
|
|||||||
.lock()
|
.lock()
|
||||||
.unwrap_or_else(|error| error.into_inner());
|
.unwrap_or_else(|error| error.into_inner());
|
||||||
match entries.get_mut(id) {
|
match entries.get_mut(id) {
|
||||||
Some(Entry::Cleanup | Entry::Create) => return Err(busy()),
|
Some(blocking @ (Entry::Cleanup | Entry::Create)) => {
|
||||||
|
return Err(busy(id, "acquire_use", blocking));
|
||||||
|
}
|
||||||
Some(Entry::Use(count)) => *count += 1,
|
Some(Entry::Use(count)) => *count += 1,
|
||||||
None => {
|
None => {
|
||||||
entries.insert(id.to_string(), Entry::Use(1));
|
entries.insert(id.to_string(), Entry::Use(1));
|
||||||
@@ -101,8 +115,8 @@ impl Occupancy {
|
|||||||
.entries
|
.entries
|
||||||
.lock()
|
.lock()
|
||||||
.unwrap_or_else(|error| error.into_inner());
|
.unwrap_or_else(|error| error.into_inner());
|
||||||
if entries.contains_key(id) {
|
if let Some(blocking) = entries.get(id) {
|
||||||
return Err(busy());
|
return Err(busy(id, "acquire_creation", blocking));
|
||||||
}
|
}
|
||||||
entries.insert(id.to_string(), Entry::Create);
|
entries.insert(id.to_string(), Entry::Create);
|
||||||
Ok(Lease {
|
Ok(Lease {
|
||||||
@@ -116,8 +130,8 @@ impl Occupancy {
|
|||||||
.entries
|
.entries
|
||||||
.lock()
|
.lock()
|
||||||
.unwrap_or_else(|error| error.into_inner());
|
.unwrap_or_else(|error| error.into_inner());
|
||||||
if entries.contains_key(id) {
|
if let Some(blocking) = entries.get(id) {
|
||||||
return Err(busy());
|
return Err(busy(id, "acquire_cleanup", blocking));
|
||||||
}
|
}
|
||||||
entries.insert(id.to_string(), Entry::Cleanup);
|
entries.insert(id.to_string(), Entry::Cleanup);
|
||||||
Ok(Lease {
|
Ok(Lease {
|
||||||
@@ -131,6 +145,71 @@ impl Occupancy {
|
|||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn busy_admission_logs_blocker_and_use_count_without_changing_error() {
|
||||||
|
use std::io::{Read, Seek, SeekFrom};
|
||||||
|
|
||||||
|
let mut output = tempfile::tempfile().unwrap();
|
||||||
|
let writer = output.try_clone().unwrap();
|
||||||
|
let subscriber = tracing_subscriber::fmt()
|
||||||
|
.with_ansi(false)
|
||||||
|
.without_time()
|
||||||
|
.with_writer(move || writer.try_clone().unwrap())
|
||||||
|
.finish();
|
||||||
|
tracing::subscriber::with_default(subscriber, || {
|
||||||
|
tracing::callsite::rebuild_interest_cache();
|
||||||
|
let occupancy = Occupancy::default();
|
||||||
|
for (id, leases) in [
|
||||||
|
(
|
||||||
|
"workdir-used",
|
||||||
|
vec![
|
||||||
|
occupancy.acquire_use("workdir-used").unwrap(),
|
||||||
|
occupancy.acquire_use("workdir-used").unwrap(),
|
||||||
|
],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"workdir-creating",
|
||||||
|
vec![occupancy.acquire_creation("workdir-creating").unwrap()],
|
||||||
|
),
|
||||||
|
(
|
||||||
|
"workdir-removing",
|
||||||
|
vec![occupancy.acquire_cleanup("workdir-removing").unwrap()],
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
let error = occupancy.acquire_cleanup(id).unwrap_err();
|
||||||
|
assert_eq!(error.code, "working_directory_cleanup_resource_busy");
|
||||||
|
assert_eq!(
|
||||||
|
error.message,
|
||||||
|
"A filesystem resource is busy or changed during removal. Release the resource and retry."
|
||||||
|
);
|
||||||
|
drop(leases);
|
||||||
|
drop(occupancy.acquire_cleanup(id).unwrap());
|
||||||
|
}
|
||||||
|
});
|
||||||
|
output.seek(SeekFrom::Start(0)).unwrap();
|
||||||
|
let mut log = String::new();
|
||||||
|
output.read_to_string(&mut log).unwrap();
|
||||||
|
let events: Vec<_> = log.lines().collect();
|
||||||
|
assert_eq!(events.len(), 3, "{log}");
|
||||||
|
for (event, (id, reason, count)) in events.iter().zip([
|
||||||
|
("workdir-used", "active_use", 2),
|
||||||
|
("workdir-creating", "creation_in_progress", 0),
|
||||||
|
("workdir-removing", "cleanup_in_progress", 0),
|
||||||
|
]) {
|
||||||
|
for field in [
|
||||||
|
"WARN".to_string(),
|
||||||
|
"yoi::workdir_cleanup".to_string(),
|
||||||
|
format!("working_directory_id=\"{id}\""),
|
||||||
|
"action=\"acquire_cleanup\"".to_string(),
|
||||||
|
"busy_source=\"materializer_occupancy\"".to_string(),
|
||||||
|
format!("busy_reason=\"{reason}\""),
|
||||||
|
format!("active_use_count={count}"),
|
||||||
|
] {
|
||||||
|
assert!(event.contains(&field), "missing {field}: {event}");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn use_and_cleanup_leases_release_during_unwind() {
|
fn use_and_cleanup_leases_release_during_unwind() {
|
||||||
let occupancy = Occupancy::default();
|
let occupancy = Occupancy::default();
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ use workdir::WorkdirSessionHandle;
|
|||||||
|
|
||||||
use crate::PromptCatalogSource;
|
use crate::PromptCatalogSource;
|
||||||
use crate::controller::{ShutdownReceiver, WorkerController, WorkerHandle};
|
use crate::controller::{ShutdownReceiver, WorkerController, WorkerHandle};
|
||||||
|
pub use crate::worker::{PreparedRestoredWorker, RestoreCandidateWorkdirCleanup};
|
||||||
use crate::worker::{Worker, WorkerError, WorkerFilesystemAuthority, WorkerWorkspaceContext};
|
use crate::worker::{Worker, WorkerError, WorkerFilesystemAuthority, WorkerWorkspaceContext};
|
||||||
use manifest::WorkerManifest;
|
use manifest::WorkerManifest;
|
||||||
|
|
||||||
@@ -161,6 +162,38 @@ where
|
|||||||
Ok(PreparedWorker::new(worker, self.layout))
|
Ok(PreparedWorker::new(worker, self.layout))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Reconstruct a saved materialized Worker without allocation, attribution
|
||||||
|
/// repair, Feature installation, or controller exposure. The caller owns the
|
||||||
|
/// later controller layout and must activate before starting it.
|
||||||
|
pub async fn prepare_restored_candidate(
|
||||||
|
self,
|
||||||
|
worker_name: &str,
|
||||||
|
) -> Result<PreparedRestoredWorker<St>, WorkerBootstrapError> {
|
||||||
|
let mut candidate =
|
||||||
|
Worker::prepare_restored_from_worker_metadata_with_context_and_model_client(
|
||||||
|
worker_name,
|
||||||
|
self.manifest,
|
||||||
|
self.store,
|
||||||
|
self.prompt_catalog,
|
||||||
|
self.workspace_context,
|
||||||
|
self.filesystem_authority,
|
||||||
|
self.model_client,
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.map_err(WorkerBootstrapError::Worker)?;
|
||||||
|
if let Some(workdir_session) = self.workdir_session {
|
||||||
|
candidate
|
||||||
|
.worker_mut()
|
||||||
|
.map_err(WorkerBootstrapError::Worker)?
|
||||||
|
.bind_single_workdir_session(Some(workdir_session));
|
||||||
|
}
|
||||||
|
candidate
|
||||||
|
.worker_mut()
|
||||||
|
.map_err(WorkerBootstrapError::Worker)?
|
||||||
|
.bind_subjektiv_host(self.subjektiv_host);
|
||||||
|
Ok(candidate)
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn prepare_restored(
|
pub async fn prepare_restored(
|
||||||
self,
|
self,
|
||||||
worker_name: &str,
|
worker_name: &str,
|
||||||
@@ -243,6 +276,7 @@ where
|
|||||||
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||||
{
|
{
|
||||||
let cleanup_sessions = worker.workdir_sessions();
|
let cleanup_sessions = worker.workdir_sessions();
|
||||||
|
let factory_owned_cleanup = worker.startup_feature_cleanup_handle().is_some();
|
||||||
let controller = match layout {
|
let controller = match layout {
|
||||||
WorkerBootstrapLayout::Direct { runtime_base } => {
|
WorkerBootstrapLayout::Direct { runtime_base } => {
|
||||||
WorkerController::spawn_owned(worker, &runtime_base).await
|
WorkerController::spawn_owned(worker, &runtime_base).await
|
||||||
@@ -259,7 +293,10 @@ where
|
|||||||
controller_task,
|
controller_task,
|
||||||
}),
|
}),
|
||||||
Err(source) => {
|
Err(source) => {
|
||||||
let cleanup_failed = cleanup_sessions.close_all().await.is_err();
|
// Factory-owned task scopes must settle before sessions close.
|
||||||
|
// Cleanup remains unconfirmed here; preserve the startup source.
|
||||||
|
let cleanup_failed =
|
||||||
|
factory_owned_cleanup || cleanup_sessions.close_all().await.is_err();
|
||||||
Err(WorkerBootstrapError::Controller {
|
Err(WorkerBootstrapError::Controller {
|
||||||
source,
|
source,
|
||||||
cleanup_failed,
|
cleanup_failed,
|
||||||
@@ -267,3 +304,229 @@ where
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Host bindings for one unpublished restored Controller. These do not expose
|
||||||
|
/// Worker execution or permit arbitrary mutation during preparation.
|
||||||
|
pub struct RestoredControllerBindings {
|
||||||
|
pub backend_job: Option<crate::BackendJobExecutionBinding>,
|
||||||
|
pub workdir_sessions: std::sync::Arc<workdir::WorkdirSessionRouter>,
|
||||||
|
pub observation_provider:
|
||||||
|
Option<std::sync::Arc<dyn crate::feature::builtin::WorkerObservationProvider>>,
|
||||||
|
pub subjektiv_attached: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The same real receiver, snapshot sources and paired subscription retained
|
||||||
|
/// privately across the Runtime's durable restore decision.
|
||||||
|
pub struct UnpublishedRestoredController<St: Store> {
|
||||||
|
initialized: Option<crate::controller::InitializedWorkerController<Box<dyn LlmClient>, St>>,
|
||||||
|
activation: Option<crate::worker::RestoredWorkerActivation>,
|
||||||
|
streams: Option<crate::ipc::protocol_session::WorkerProtocolSessionStreams>,
|
||||||
|
subjektiv_attached: bool,
|
||||||
|
effects_applied: bool,
|
||||||
|
activation_failed: bool,
|
||||||
|
closing: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub struct ActivatedRestoredController {
|
||||||
|
pub controller: BootstrappedWorker,
|
||||||
|
pub streams: crate::ipc::protocol_session::WorkerProtocolSessionStreams,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<St> PreparedRestoredWorker<St>
|
||||||
|
where
|
||||||
|
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||||
|
{
|
||||||
|
pub fn workspace_client_handle(
|
||||||
|
&mut self,
|
||||||
|
) -> Result<std::sync::Arc<dyn crate::WorkspaceClient>, WorkerError> {
|
||||||
|
Ok(self.worker_mut()?.workspace_client_handle())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn retain_startup_feature_cleanup(
|
||||||
|
&mut self,
|
||||||
|
) -> Result<crate::WorkerStartupFeatureCleanup, WorkerError> {
|
||||||
|
Ok(self.worker_mut()?.retain_startup_feature_cleanup())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Initialize the real Controller without allocation, canonical restore
|
||||||
|
/// writes, attribution changes, registry reclamation or input consumption.
|
||||||
|
/// MCP discovery and Feature tasks belong to the retained startup cleanup.
|
||||||
|
pub async fn prepare_runtime_controller(
|
||||||
|
mut self,
|
||||||
|
run_dir: &std::path::Path,
|
||||||
|
bindings: RestoredControllerBindings,
|
||||||
|
) -> Result<UnpublishedRestoredController<St>, WorkerBootstrapError> {
|
||||||
|
let worker = self.worker_mut().map_err(WorkerBootstrapError::Worker)?;
|
||||||
|
if let Some(binding) = bindings.backend_job {
|
||||||
|
worker.bind_backend_job(binding).map_err(|error| {
|
||||||
|
WorkerBootstrapError::Worker(WorkerError::FeatureLifecycle(error.into()))
|
||||||
|
})?;
|
||||||
|
}
|
||||||
|
worker
|
||||||
|
.workdir_sessions()
|
||||||
|
.close_all()
|
||||||
|
.await
|
||||||
|
.map_err(|error| {
|
||||||
|
WorkerBootstrapError::Worker(WorkerError::FeatureLifecycle(error.to_string()))
|
||||||
|
})?;
|
||||||
|
worker.bind_workdir_sessions(bindings.workdir_sessions);
|
||||||
|
worker.bind_worker_observation_provider(bindings.observation_provider);
|
||||||
|
let flow_enabled = worker.manifest().feature.flow.enabled;
|
||||||
|
if flow_enabled {
|
||||||
|
let report = worker
|
||||||
|
.install_runtime_flow_transition_feature()
|
||||||
|
.map_err(|error| {
|
||||||
|
WorkerBootstrapError::Worker(WorkerError::FeatureLifecycle(error))
|
||||||
|
})?;
|
||||||
|
if report.reports.iter().any(|report| !report.installed) {
|
||||||
|
return Err(WorkerBootstrapError::Worker(WorkerError::FeatureLifecycle(
|
||||||
|
"Flow transition installation refused".into(),
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let (worker, activation) = self
|
||||||
|
.into_controller_parts()
|
||||||
|
.map_err(WorkerBootstrapError::Worker)?;
|
||||||
|
let initialized = WorkerController::initialize_restored(worker, run_dir)
|
||||||
|
.await
|
||||||
|
.map_err(|source| WorkerBootstrapError::Controller {
|
||||||
|
source,
|
||||||
|
cleanup_failed: true,
|
||||||
|
})?;
|
||||||
|
if flow_enabled {
|
||||||
|
initialized.handle.shared_state.enable_flow_transition();
|
||||||
|
}
|
||||||
|
let streams = {
|
||||||
|
let _guard = initialized.handle.protocol_snapshot_publish_guard();
|
||||||
|
crate::ipc::protocol_session::subscribe_worker_protocol_session(&initialized.handle)
|
||||||
|
};
|
||||||
|
Ok(UnpublishedRestoredController {
|
||||||
|
initialized: Some(initialized),
|
||||||
|
activation: Some(activation),
|
||||||
|
streams: Some(streams),
|
||||||
|
subjektiv_attached: bindings.subjektiv_attached,
|
||||||
|
effects_applied: false,
|
||||||
|
activation_failed: false,
|
||||||
|
closing: false,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<St> UnpublishedRestoredController<St>
|
||||||
|
where
|
||||||
|
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||||
|
{
|
||||||
|
pub fn snapshot(&self) -> &protocol::Event {
|
||||||
|
&self
|
||||||
|
.streams
|
||||||
|
.as_ref()
|
||||||
|
.expect("unpublished restore streams")
|
||||||
|
.snapshot_event
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Only publication failures after confirmed effects can retry on this
|
||||||
|
/// exact owner. Partial activation must be reconciled, never blindly rerun.
|
||||||
|
pub fn can_retry_publication(&self) -> bool {
|
||||||
|
self.initialized.is_some()
|
||||||
|
&& self.effects_applied
|
||||||
|
&& !self.activation_failed
|
||||||
|
&& !self.closing
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Apply canonical restore effects after the host's durable decision, while
|
||||||
|
/// retaining the same private Controller for subsequent publication.
|
||||||
|
pub async fn apply_effects(&mut self) -> Result<(), WorkerBootstrapError> {
|
||||||
|
let initialized = self
|
||||||
|
.initialized
|
||||||
|
.as_mut()
|
||||||
|
.ok_or_else(|| WorkerBootstrapError::Worker(WorkerError::RestoreCandidateClosing))?;
|
||||||
|
if self.activation_failed || self.closing {
|
||||||
|
return Err(WorkerBootstrapError::Worker(
|
||||||
|
WorkerError::RestoreCandidateClosing,
|
||||||
|
));
|
||||||
|
}
|
||||||
|
// Preparation's read-only authorization is not a publication grant.
|
||||||
|
// Revalidate current Host authority after commit, before effects/input,
|
||||||
|
// including a publication retry on this same private Controller.
|
||||||
|
initialized
|
||||||
|
.worker
|
||||||
|
.validate_subjektiv_body_publication()
|
||||||
|
.map_err(WorkerBootstrapError::Worker)?;
|
||||||
|
if !self.effects_applied {
|
||||||
|
self.activation_failed = true;
|
||||||
|
self.activation
|
||||||
|
.as_ref()
|
||||||
|
.expect("restore activation")
|
||||||
|
.activate(&mut initialized.worker)
|
||||||
|
.await
|
||||||
|
.map_err(WorkerBootstrapError::Worker)?;
|
||||||
|
initialized
|
||||||
|
.worker
|
||||||
|
.finalize_subjektiv_session_attribution(
|
||||||
|
self.subjektiv_attached,
|
||||||
|
crate::SubjektivSessionAttributionLifecycle::RestoredSession,
|
||||||
|
)
|
||||||
|
.map_err(WorkerBootstrapError::Worker)?;
|
||||||
|
self.effects_applied = true;
|
||||||
|
self.activation_failed = false;
|
||||||
|
self.activation = None;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn allocation_cleanup(
|
||||||
|
&self,
|
||||||
|
) -> Option<crate::runtime::worker_allocation::ScopeAllocationCleanup> {
|
||||||
|
self.initialized
|
||||||
|
.as_ref()
|
||||||
|
.and_then(|initialized| initialized.worker.restore_allocation_cleanup_handle())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn activate(&mut self) -> Result<ActivatedRestoredController, WorkerBootstrapError> {
|
||||||
|
self.apply_effects().await?;
|
||||||
|
let initialized = self
|
||||||
|
.initialized
|
||||||
|
.as_mut()
|
||||||
|
.expect("same initialized Controller");
|
||||||
|
let publication = initialized.prepare_publication().await.map_err(|source| {
|
||||||
|
WorkerBootstrapError::Controller {
|
||||||
|
source,
|
||||||
|
cleanup_failed: true,
|
||||||
|
}
|
||||||
|
})?;
|
||||||
|
let initialized = self
|
||||||
|
.initialized
|
||||||
|
.take()
|
||||||
|
.expect("same initialized Controller");
|
||||||
|
let (handle, shutdown, controller_task) = initialized.start(publication);
|
||||||
|
Ok(ActivatedRestoredController {
|
||||||
|
controller: BootstrappedWorker {
|
||||||
|
handle,
|
||||||
|
shutdown,
|
||||||
|
controller_task,
|
||||||
|
},
|
||||||
|
streams: self.streams.take().expect("same paired subscription"),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
pub async fn discard(&mut self) -> Result<(), WorkerError> {
|
||||||
|
self.closing = true;
|
||||||
|
if let Some(initialized) = self.initialized.as_mut() {
|
||||||
|
initialized
|
||||||
|
.worker
|
||||||
|
.retain_startup_feature_cleanup()
|
||||||
|
.shutdown()
|
||||||
|
.await
|
||||||
|
.map_err(|error| WorkerError::FeatureLifecycle(error.to_string()))?;
|
||||||
|
initialized
|
||||||
|
.worker
|
||||||
|
.workdir_sessions()
|
||||||
|
.close_all()
|
||||||
|
.await
|
||||||
|
.map_err(|error| WorkerError::FeatureLifecycle(error.to_string()))?;
|
||||||
|
}
|
||||||
|
self.streams = None;
|
||||||
|
self.initialized = None;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
+388
-45
@@ -21,7 +21,7 @@ use crate::shutdown_after_idle::{
|
|||||||
ShutdownAfterIdleRequest, TicketIntakeReadyShutdownHook, is_ticket_intake_role,
|
ShutdownAfterIdleRequest, TicketIntakeReadyShutdownHook, is_ticket_intake_role,
|
||||||
take_shutdown_request_after_status,
|
take_shutdown_request_after_status,
|
||||||
};
|
};
|
||||||
use crate::spawn::registry::SpawnedWorkerRegistry;
|
use crate::spawn::registry::{PreparedSpawnedWorkerRegistryLoad, SpawnedWorkerRegistry};
|
||||||
use crate::spawn::tool::sub_worker_spawn_tool;
|
use crate::spawn::tool::sub_worker_spawn_tool;
|
||||||
use crate::worker::{
|
use crate::worker::{
|
||||||
PendingSubmissionHandle, SystemItemCommitter, Worker, WorkerError, WorkerRunResult,
|
PendingSubmissionHandle, SystemItemCommitter, Worker, WorkerError, WorkerRunResult,
|
||||||
@@ -654,7 +654,56 @@ pub type ShutdownReceiver = oneshot::Receiver<()>;
|
|||||||
|
|
||||||
pub struct WorkerController;
|
pub struct WorkerController;
|
||||||
|
|
||||||
|
// Own the real command receiver and all snapshot sources before starting any
|
||||||
|
// protocol listener or controller loop. Fresh startup may materialize the head;
|
||||||
|
// Runtime restore requires an already materialized source and defers canonical
|
||||||
|
// restore effects until its durable decision. Tool discovery can own external
|
||||||
|
// resources, so private initialization is not synonymous with side-effect-free.
|
||||||
|
pub(crate) struct InitializedWorkerController<C: LlmClient, St: Store> {
|
||||||
|
pub(crate) worker: Worker<C, St>,
|
||||||
|
pub(crate) handle: WorkerHandle,
|
||||||
|
method_rx: mpsc::Receiver<Method>,
|
||||||
|
shutdown_tx: oneshot::Sender<()>,
|
||||||
|
shutdown_rx: ShutdownReceiver,
|
||||||
|
registry_reclaim: PreparedSpawnedWorkerRegistryLoad<St>,
|
||||||
|
shutdown_after_idle: ShutdownAfterIdleRequest,
|
||||||
|
command_events: Option<PreparedWorkdirCommandEvents>,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) struct PreparedControllerPublication {
|
||||||
|
command_observer: Option<tokio::task::JoinHandle<()>>,
|
||||||
|
}
|
||||||
|
|
||||||
impl WorkerController {
|
impl WorkerController {
|
||||||
|
/// Explicit recovery of an unmaterialized Session head. This installs the
|
||||||
|
/// saved Feature composition needed to produce the seed, but creates no
|
||||||
|
/// Controller, input channel, socket, observer, or queue consumer. The caller
|
||||||
|
/// owns the same operation's Feature/session/allocation cleanup on every exit.
|
||||||
|
pub async fn recover_pending_head<C, St>(
|
||||||
|
worker: &mut Worker<C, St>,
|
||||||
|
runtime_base: &Path,
|
||||||
|
) -> std::io::Result<()>
|
||||||
|
where
|
||||||
|
C: LlmClient + Clone + 'static,
|
||||||
|
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||||
|
{
|
||||||
|
if worker.needs_initial_session_head_materialization() {
|
||||||
|
worker
|
||||||
|
.validate_subjektiv_body_publication()
|
||||||
|
.map_err(std::io::Error::other)?;
|
||||||
|
let registry = SpawnedWorkerRegistry::new_internal(
|
||||||
|
worker.manifest().worker.name.clone(),
|
||||||
|
worker.scope().clone(),
|
||||||
|
);
|
||||||
|
register_worker_tools(worker, runtime_base.to_path_buf(), registry, None, None).await?;
|
||||||
|
worker
|
||||||
|
.materialize_durable_session_head()
|
||||||
|
.await
|
||||||
|
.map_err(std::io::Error::other)?;
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
pub async fn spawn<C, St>(
|
pub async fn spawn<C, St>(
|
||||||
worker: Worker<C, St>,
|
worker: Worker<C, St>,
|
||||||
runtime_base: &Path,
|
runtime_base: &Path,
|
||||||
@@ -734,8 +783,12 @@ impl WorkerController {
|
|||||||
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||||
{
|
{
|
||||||
let sessions = worker.workdir_sessions();
|
let sessions = worker.workdir_sessions();
|
||||||
|
let factory_owned_cleanup = worker.startup_feature_cleanup_handle().is_some();
|
||||||
let result = Self::spawn_initialized(worker, runtime_base, runtime_run).await;
|
let result = Self::spawn_initialized(worker, runtime_base, runtime_run).await;
|
||||||
|
// An opted-in factory must quiesce its captured Feature tasks before
|
||||||
|
// closing their Workdir sessions; this wrapper cannot confirm that order.
|
||||||
if result.is_err()
|
if result.is_err()
|
||||||
|
&& !factory_owned_cleanup
|
||||||
&& let Err(error) = sessions.close_all().await
|
&& let Err(error) = sessions.close_all().await
|
||||||
{
|
{
|
||||||
tracing::warn!(%error, "Workdir sessions close after controller startup failure failed");
|
tracing::warn!(%error, "Workdir sessions close after controller startup failure failed");
|
||||||
@@ -744,7 +797,7 @@ impl WorkerController {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn spawn_initialized<C, St>(
|
async fn spawn_initialized<C, St>(
|
||||||
mut worker: Worker<C, St>,
|
worker: Worker<C, St>,
|
||||||
runtime_base: &Path,
|
runtime_base: &Path,
|
||||||
runtime_run: Option<&Path>,
|
runtime_run: Option<&Path>,
|
||||||
) -> Result<(WorkerHandle, ShutdownReceiver, tokio::task::JoinHandle<()>), std::io::Error>
|
) -> Result<(WorkerHandle, ShutdownReceiver, tokio::task::JoinHandle<()>), std::io::Error>
|
||||||
@@ -752,6 +805,52 @@ impl WorkerController {
|
|||||||
C: LlmClient + Clone + 'static,
|
C: LlmClient + Clone + 'static,
|
||||||
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||||
{
|
{
|
||||||
|
Self::initialize(worker, runtime_base, runtime_run)
|
||||||
|
.await?
|
||||||
|
.publish()
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn initialize<C, St>(
|
||||||
|
worker: Worker<C, St>,
|
||||||
|
runtime_base: &Path,
|
||||||
|
runtime_run: Option<&Path>,
|
||||||
|
) -> Result<InitializedWorkerController<C, St>, std::io::Error>
|
||||||
|
where
|
||||||
|
C: LlmClient + Clone + 'static,
|
||||||
|
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||||
|
{
|
||||||
|
Self::initialize_inner(worker, runtime_base, runtime_run, true).await
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn initialize_restored<St>(
|
||||||
|
worker: Worker<Box<dyn LlmClient>, St>,
|
||||||
|
run_dir: &Path,
|
||||||
|
) -> std::io::Result<InitializedWorkerController<Box<dyn LlmClient>, St>>
|
||||||
|
where
|
||||||
|
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||||
|
{
|
||||||
|
let parent = run_dir
|
||||||
|
.parent()
|
||||||
|
.ok_or_else(|| std::io::Error::other("run path has no parent"))?;
|
||||||
|
Self::initialize_inner(worker, parent, Some(run_dir), false).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn initialize_inner<C, St>(
|
||||||
|
mut worker: Worker<C, St>,
|
||||||
|
runtime_base: &Path,
|
||||||
|
runtime_run: Option<&Path>,
|
||||||
|
canonical_startup: bool,
|
||||||
|
) -> Result<InitializedWorkerController<C, St>, std::io::Error>
|
||||||
|
where
|
||||||
|
C: LlmClient + Clone + 'static,
|
||||||
|
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||||
|
{
|
||||||
|
if !canonical_startup && worker.needs_initial_session_head_materialization() {
|
||||||
|
return Err(std::io::Error::other(
|
||||||
|
"restore source requires operation-owned head recovery",
|
||||||
|
));
|
||||||
|
}
|
||||||
worker
|
worker
|
||||||
.validate_subjektiv_body_publication()
|
.validate_subjektiv_body_publication()
|
||||||
.map_err(std::io::Error::other)?;
|
.map_err(std::io::Error::other)?;
|
||||||
@@ -773,21 +872,13 @@ impl WorkerController {
|
|||||||
});
|
});
|
||||||
|
|
||||||
let spawner_name = worker.manifest().worker.name.clone();
|
let spawner_name = worker.manifest().worker.name.clone();
|
||||||
let loaded_registry = SpawnedWorkerRegistry::load_from_worker_state_with_reclaim(
|
let registry_reclaim = SpawnedWorkerRegistry::prepare_from_worker_state_with_reclaim(
|
||||||
runtime_dir.clone(),
|
runtime_dir.clone(),
|
||||||
worker.store().clone(),
|
worker.store().clone(),
|
||||||
spawner_name.clone(),
|
spawner_name.clone(),
|
||||||
Some(worker.scope().clone()),
|
Some(worker.scope().clone()),
|
||||||
)
|
)?;
|
||||||
.await?;
|
let spawned_registry = registry_reclaim.registry.clone();
|
||||||
let reclaimed_unreachable = loaded_registry.reclaimed_unreachable;
|
|
||||||
let spawned_registry = loaded_registry.registry;
|
|
||||||
if reclaimed_unreachable {
|
|
||||||
worker.push_notify(
|
|
||||||
"Restored Worker state contained unreachable delegated child Workers; their delegated write scopes were reclaimed before resume."
|
|
||||||
.to_string(),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Hand the alerter to the Worker so internal operations (compaction,
|
// Hand the alerter to the Worker so internal operations (compaction,
|
||||||
// AGENTS.md ingestion during the first turn) can emit user-facing
|
// AGENTS.md ingestion during the first turn) can emit user-facing
|
||||||
@@ -878,7 +969,6 @@ impl WorkerController {
|
|||||||
if let Some(fs_for_view) = fs_for_view.as_ref() {
|
if let Some(fs_for_view) = fs_for_view.as_ref() {
|
||||||
shared_state.set_fs_view(crate::fs_view::WorkerFsView::new(fs_for_view.clone()));
|
shared_state.set_fs_view(crate::fs_view::WorkerFsView::new(fs_for_view.clone()));
|
||||||
}
|
}
|
||||||
|
|
||||||
let artifact_store: Arc<dyn Store> = Arc::new(worker.store().clone());
|
let artifact_store: Arc<dyn Store> = Arc::new(worker.store().clone());
|
||||||
let session_id = worker.session_id();
|
let session_id = worker.session_id();
|
||||||
let pending_activations = worker.pending_activation_state();
|
let pending_activations = worker.pending_activation_state();
|
||||||
@@ -903,10 +993,71 @@ impl WorkerController {
|
|||||||
pending_activations,
|
pending_activations,
|
||||||
};
|
};
|
||||||
|
|
||||||
// Start the observer only after fallible head/recovery initialization.
|
let command_events = fs_for_view
|
||||||
let command_observer = fs_for_view
|
|
||||||
.as_ref()
|
.as_ref()
|
||||||
.and_then(|session| wire_workdir_command_events(session, &in_flight));
|
.map(|session| PreparedWorkdirCommandEvents::new(session, &handle.in_flight));
|
||||||
|
Ok(InitializedWorkerController {
|
||||||
|
worker,
|
||||||
|
handle,
|
||||||
|
method_rx,
|
||||||
|
shutdown_tx,
|
||||||
|
shutdown_rx,
|
||||||
|
registry_reclaim,
|
||||||
|
shutdown_after_idle,
|
||||||
|
command_events,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<C, St> InitializedWorkerController<C, St>
|
||||||
|
where
|
||||||
|
C: LlmClient + Clone + 'static,
|
||||||
|
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||||
|
{
|
||||||
|
async fn publish(
|
||||||
|
mut self,
|
||||||
|
) -> Result<(WorkerHandle, ShutdownReceiver, tokio::task::JoinHandle<()>), std::io::Error> {
|
||||||
|
let publication = self.prepare_publication().await?;
|
||||||
|
Ok(self.start(publication))
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) async fn prepare_publication(
|
||||||
|
&mut self,
|
||||||
|
) -> std::io::Result<PreparedControllerPublication> {
|
||||||
|
let loaded_registry = self.registry_reclaim.apply().await?;
|
||||||
|
debug_assert!(Arc::ptr_eq(
|
||||||
|
&loaded_registry.registry,
|
||||||
|
&self.handle.spawned_registry
|
||||||
|
));
|
||||||
|
if loaded_registry.reclaimed_unreachable {
|
||||||
|
self.worker.push_notify("Restored Worker state contained unreachable delegated child Workers; their delegated write scopes were reclaimed before resume.".to_string());
|
||||||
|
}
|
||||||
|
let command_observer = self
|
||||||
|
.command_events
|
||||||
|
.take()
|
||||||
|
.and_then(PreparedWorkdirCommandEvents::start);
|
||||||
|
Ok(PreparedControllerPublication { command_observer })
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(crate) fn start(
|
||||||
|
self,
|
||||||
|
publication: PreparedControllerPublication,
|
||||||
|
) -> (WorkerHandle, ShutdownReceiver, tokio::task::JoinHandle<()>) {
|
||||||
|
let Self {
|
||||||
|
mut worker,
|
||||||
|
handle,
|
||||||
|
method_rx,
|
||||||
|
shutdown_tx,
|
||||||
|
shutdown_rx,
|
||||||
|
shutdown_after_idle,
|
||||||
|
..
|
||||||
|
} = self;
|
||||||
|
let PreparedControllerPublication { command_observer } = publication;
|
||||||
|
let runtime_dir = handle.runtime_dir.clone();
|
||||||
|
let shared_state = handle.shared_state.clone();
|
||||||
|
let working_event_tx = handle.working_event_tx.clone();
|
||||||
|
let spawned_registry = handle.spawned_registry.clone();
|
||||||
|
|
||||||
// === 5. controller_loop ===
|
// === 5. controller_loop ===
|
||||||
// Clone cancel sender and notification buffer before moving worker
|
// Clone cancel sender and notification buffer before moving worker
|
||||||
// into the controller task so the in-flight turn can be reached
|
// into the controller task so the in-flight turn can be reached
|
||||||
@@ -915,6 +1066,7 @@ impl WorkerController {
|
|||||||
let pause_tx = worker.engine_mut().pause_sender();
|
let pause_tx = worker.engine_mut().pause_sender();
|
||||||
let notify_buffer = worker.notify_buffer_handle();
|
let notify_buffer = worker.notify_buffer_handle();
|
||||||
|
|
||||||
|
worker.finish_startup_feature_cleanup_capture();
|
||||||
let controller_task = tokio::spawn(controller_loop(
|
let controller_task = tokio::spawn(controller_loop(
|
||||||
worker,
|
worker,
|
||||||
method_rx,
|
method_rx,
|
||||||
@@ -930,7 +1082,7 @@ impl WorkerController {
|
|||||||
command_observer,
|
command_observer,
|
||||||
));
|
));
|
||||||
|
|
||||||
Ok((handle, shutdown_rx, controller_task))
|
(handle, shutdown_rx, controller_task)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -938,30 +1090,51 @@ pub(crate) fn wire_workdir_command_events(
|
|||||||
session: &Arc<dyn WorkdirSession>,
|
session: &Arc<dyn WorkdirSession>,
|
||||||
in_flight: &InFlightEvents,
|
in_flight: &InFlightEvents,
|
||||||
) -> Option<tokio::task::JoinHandle<()>> {
|
) -> Option<tokio::task::JoinHandle<()>> {
|
||||||
in_flight.replace_command_snapshot(protocol_command_snapshots(session.as_ref()));
|
PreparedWorkdirCommandEvents::new(session, in_flight).start()
|
||||||
let Some(mut events) = session.subscribe_command_events() else {
|
}
|
||||||
return None;
|
|
||||||
};
|
struct PreparedWorkdirCommandEvents {
|
||||||
// Keep only a weak reference in the observer task. Holding the session
|
session: std::sync::Weak<dyn WorkdirSession>,
|
||||||
// strongly here would keep its broadcast sender alive forever and prevent
|
in_flight: InFlightEvents,
|
||||||
// the receiver from observing closure during Worker teardown.
|
events: Option<broadcast::Receiver<WorkdirCommandEvent>>,
|
||||||
let session = Arc::downgrade(session);
|
}
|
||||||
let in_flight = in_flight.clone();
|
|
||||||
Some(tokio::spawn(async move {
|
impl PreparedWorkdirCommandEvents {
|
||||||
loop {
|
fn new(session: &Arc<dyn WorkdirSession>, in_flight: &InFlightEvents) -> Self {
|
||||||
match events.recv().await {
|
// Subscribe before reading command state; hold the receiver privately
|
||||||
Ok(event) => in_flight.publish_command_event(protocol_command_event(event)),
|
// across the durable decision instead of creating a publication gap.
|
||||||
Err(broadcast::error::RecvError::Lagged(_)) => {
|
let events = session.subscribe_command_events();
|
||||||
let Some(session) = session.upgrade() else {
|
in_flight.replace_command_snapshot(protocol_command_snapshots(session.as_ref()));
|
||||||
break;
|
Self {
|
||||||
};
|
session: Arc::downgrade(session),
|
||||||
in_flight
|
in_flight: in_flight.clone(),
|
||||||
.replace_command_snapshot(protocol_command_snapshots(session.as_ref()));
|
events,
|
||||||
}
|
|
||||||
Err(broadcast::error::RecvError::Closed) => break,
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
}))
|
}
|
||||||
|
|
||||||
|
fn start(self) -> Option<tokio::task::JoinHandle<()>> {
|
||||||
|
let Self {
|
||||||
|
session,
|
||||||
|
in_flight,
|
||||||
|
events,
|
||||||
|
} = self;
|
||||||
|
let mut events = events?;
|
||||||
|
Some(tokio::spawn(async move {
|
||||||
|
loop {
|
||||||
|
match events.recv().await {
|
||||||
|
Ok(event) => in_flight.publish_command_event(protocol_command_event(event)),
|
||||||
|
Err(broadcast::error::RecvError::Lagged(_)) => {
|
||||||
|
let Some(session) = session.upgrade() else {
|
||||||
|
break;
|
||||||
|
};
|
||||||
|
in_flight
|
||||||
|
.replace_command_snapshot(protocol_command_snapshots(session.as_ref()));
|
||||||
|
}
|
||||||
|
Err(broadcast::error::RecvError::Closed) => break,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fn protocol_command_snapshots(session: &dyn WorkdirSession) -> Vec<ProtocolCommandSnapshot> {
|
fn protocol_command_snapshots(session: &dyn WorkdirSession) -> Vec<ProtocolCommandSnapshot> {
|
||||||
@@ -1687,9 +1860,17 @@ where
|
|||||||
.add_module(crate::feature::builtin::orchestration::orchestration_feature());
|
.add_module(crate::feature::builtin::orchestration::orchestration_feature());
|
||||||
}
|
}
|
||||||
if let Some(workspace_root) = local_workspace_root.as_ref() {
|
if let Some(workspace_root) = local_workspace_root.as_ref() {
|
||||||
if let Some(module) =
|
let module = if let Some(cleanup) = worker.startup_feature_cleanup_handle() {
|
||||||
|
crate::feature::mcp::discover_stdio_tool_feature_with_cleanup(
|
||||||
|
&mcp_config,
|
||||||
|
workspace_root,
|
||||||
|
&cleanup.mcp_discovery(),
|
||||||
|
)
|
||||||
|
.await?
|
||||||
|
} else {
|
||||||
crate::feature::mcp::discover_stdio_tool_feature(&mcp_config, workspace_root).await
|
crate::feature::mcp::discover_stdio_tool_feature(&mcp_config, workspace_root).await
|
||||||
{
|
};
|
||||||
|
if let Some(module) = module {
|
||||||
feature_registry = feature_registry.with_module(module);
|
feature_registry = feature_registry.with_module(module);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -3750,6 +3931,159 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Component connection: real Worker/Store, initialized endpoint, paired
|
||||||
|
// protocol subscription and Controller loop. The model boundary is an
|
||||||
|
// empty non-network script; this test must not request any generation.
|
||||||
|
#[tokio::test]
|
||||||
|
async fn initialized_controller_publishes_the_same_endpoint_and_subscription() {
|
||||||
|
let root = TempDir::new().unwrap();
|
||||||
|
let manifest = manifest::WorkerManifest::from_toml(
|
||||||
|
"[worker]\nname = 'p'\n[model]\nscheme = 'anthropic'\nmodel_id = 'test'\n[engine]\n[scope]\n",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let client = ScriptedClient::new(vec![]);
|
||||||
|
let requests = client.requests.clone();
|
||||||
|
let engine =
|
||||||
|
Engine::<_, agen::state::Mutable, crate::SessionHistoryMetadata>::new_annotated(
|
||||||
|
client,
|
||||||
|
);
|
||||||
|
let store = session_store::CombinedStore::new(
|
||||||
|
session_store::FsStore::new(root.path().join("sessions")).unwrap(),
|
||||||
|
session_store::FsWorkerStore::new(root.path().join("workers")).unwrap(),
|
||||||
|
);
|
||||||
|
let mut worker = Worker::new(
|
||||||
|
manifest,
|
||||||
|
engine,
|
||||||
|
store,
|
||||||
|
crate::worker::WorkerWorkspaceContext::local_filesystem(None),
|
||||||
|
crate::worker::WorkerFilesystemAuthority::None,
|
||||||
|
manifest::Scope::empty(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
worker.set_system_prompt_template(
|
||||||
|
crate::SystemPromptTemplate::parse(
|
||||||
|
"default",
|
||||||
|
crate::PromptCatalogSource::builtins_only(),
|
||||||
|
)
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
let prepared = WorkerController::initialize(worker, root.path(), None)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let endpoint = prepared.handle.clone();
|
||||||
|
let mut streams =
|
||||||
|
crate::ipc::protocol_session::subscribe_worker_protocol_session(&endpoint);
|
||||||
|
let entries = endpoint.committed_entries();
|
||||||
|
assert!(matches!(
|
||||||
|
&streams.snapshot_event,
|
||||||
|
Event::Snapshot { greeting, state, session, .. }
|
||||||
|
if !greeting.tools.is_empty()
|
||||||
|
&& state.state == WorkerState::Idle
|
||||||
|
&& session == &session_store::public_snapshot::project_current_session_snapshot(&entries)
|
||||||
|
));
|
||||||
|
assert!(!endpoint.protocol_is_closed());
|
||||||
|
assert!(!endpoint.runtime_dir.path().join("sock").exists());
|
||||||
|
assert!(!endpoint.runtime_dir.path().join("status.json").exists());
|
||||||
|
assert!(!endpoint.runtime_dir.path().join("manifest.toml").exists());
|
||||||
|
assert!(matches!(
|
||||||
|
streams.events.try_recv(),
|
||||||
|
Err(broadcast::error::TryRecvError::Empty)
|
||||||
|
));
|
||||||
|
assert!(requests.lock().unwrap().is_empty());
|
||||||
|
|
||||||
|
let (handle, shutdown, task) = prepared.publish().await.unwrap();
|
||||||
|
assert!(handle.same_controller(&endpoint));
|
||||||
|
assert!(!handle.runtime_dir.path().join("sock").exists());
|
||||||
|
assert!(!handle.runtime_dir.path().join("status.json").exists());
|
||||||
|
assert!(!handle.runtime_dir.path().join("manifest.toml").exists());
|
||||||
|
handle.send(Method::ListPendingSubmissions).await.unwrap();
|
||||||
|
let pending = tokio::time::timeout(Duration::from_secs(5), async {
|
||||||
|
loop {
|
||||||
|
if let Event::PendingSubmissionsChanged { pending } =
|
||||||
|
streams.events.recv().await.unwrap()
|
||||||
|
{
|
||||||
|
break pending;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(pending, protocol::PendingSubmissionsSnapshot::default());
|
||||||
|
assert_eq!(
|
||||||
|
serde_json::to_value(handle.committed_entries()).unwrap(),
|
||||||
|
serde_json::to_value(entries).unwrap(),
|
||||||
|
);
|
||||||
|
handle
|
||||||
|
.send(Method::Shutdown {
|
||||||
|
command: WorkerCommandEnvelope { command_id: 1 },
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
tokio::time::timeout(Duration::from_secs(5), shutdown)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
tokio::time::timeout(Duration::from_secs(5), task)
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert!(endpoint.protocol_is_closed());
|
||||||
|
assert!(!endpoint.runtime_dir.path().join("sock").exists());
|
||||||
|
assert!(requests.lock().unwrap().is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn tool_initialization_failure_does_not_reclaim_saved_children() {
|
||||||
|
let root = TempDir::new().unwrap();
|
||||||
|
let mut manifest = manifest::WorkerManifest::from_toml(
|
||||||
|
"[worker]\nname = 'p'\n[model]\nscheme = 'anthropic'\nmodel_id = 'test'\n[engine]\n[scope]\n",
|
||||||
|
).unwrap();
|
||||||
|
manifest.feature.ticket.enabled = true;
|
||||||
|
let client = ScriptedClient::new(vec![]);
|
||||||
|
let requests = client.requests.clone();
|
||||||
|
let store = session_store::CombinedStore::new(
|
||||||
|
session_store::FsStore::new(root.path().join("sessions")).unwrap(),
|
||||||
|
session_store::FsWorkerStore::new(root.path().join("workers")).unwrap(),
|
||||||
|
);
|
||||||
|
// New feature activation is refused by the real unavailable host;
|
||||||
|
// legacy child metadata must not be consumed before that refusal.
|
||||||
|
let worker = Worker::new(
|
||||||
|
manifest,
|
||||||
|
Engine::<_, agen::state::Mutable, crate::SessionHistoryMetadata>::new_annotated(
|
||||||
|
client,
|
||||||
|
),
|
||||||
|
store.clone(),
|
||||||
|
crate::worker::WorkerWorkspaceContext::no_workspace(),
|
||||||
|
crate::worker::WorkerFilesystemAuthority::None,
|
||||||
|
manifest::Scope::empty(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let mut metadata = session_store::WorkerMetadata::new(
|
||||||
|
"p",
|
||||||
|
Some(session_store::WorkerActiveSegmentRef {
|
||||||
|
session_id: worker.session_id(),
|
||||||
|
segment_id: Some(worker.segment_id()),
|
||||||
|
}),
|
||||||
|
);
|
||||||
|
metadata
|
||||||
|
.spawned_children
|
||||||
|
.push(session_store::WorkerSpawnedChild {
|
||||||
|
worker_name: "legacy".into(),
|
||||||
|
socket_path: root.path().join("legacy.sock"),
|
||||||
|
callback_address: root.path().join("p/sock"),
|
||||||
|
scope_delegated: Vec::new(),
|
||||||
|
});
|
||||||
|
store.write(&metadata).unwrap();
|
||||||
|
let before = store.read_by_name("p").unwrap();
|
||||||
|
let result = WorkerController::initialize(worker, root.path(), None).await;
|
||||||
|
assert!(result.is_err());
|
||||||
|
assert_eq!(store.read_by_name("p").unwrap(), before);
|
||||||
|
assert!(!root.path().join("p/spawned_workers.json").exists());
|
||||||
|
assert!(requests.lock().unwrap().is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
struct DriveWorkspaceHost {
|
struct DriveWorkspaceHost {
|
||||||
available: bool,
|
available: bool,
|
||||||
@@ -4077,10 +4411,19 @@ mod tests {
|
|||||||
assert!(feature_install < materialize_gate);
|
assert!(feature_install < materialize_gate);
|
||||||
assert!(materialize_gate < materialize);
|
assert!(materialize_gate < materialize);
|
||||||
assert!(materialize < exposure);
|
assert!(materialize < exposure);
|
||||||
let observer = startup.find("let command_observer = fs_for_view").unwrap();
|
let observer_preparation = startup
|
||||||
|
.find("PreparedWorkdirCommandEvents::new(session")
|
||||||
|
.unwrap();
|
||||||
let recover = startup.find(".recover_unfinished_compaction()").unwrap();
|
let recover = startup.find(".recover_unfinished_compaction()").unwrap();
|
||||||
|
let reclaim = startup
|
||||||
|
.find("self.registry_reclaim.apply().await?")
|
||||||
|
.unwrap();
|
||||||
|
let observer_start = startup
|
||||||
|
.find(".and_then(PreparedWorkdirCommandEvents::start)")
|
||||||
|
.unwrap();
|
||||||
let loop_start = startup.find("tokio::spawn(controller_loop(").unwrap();
|
let loop_start = startup.find("tokio::spawn(controller_loop(").unwrap();
|
||||||
assert!(recover < observer && observer < loop_start);
|
assert!(recover < observer_preparation && observer_preparation < reclaim);
|
||||||
|
assert!(reclaim < observer_start && observer_start < loop_start);
|
||||||
assert!(
|
assert!(
|
||||||
startup[materialize_gate..materialize]
|
startup[materialize_gate..materialize]
|
||||||
.contains("needs_initial_session_head_materialization")
|
.contains("needs_initial_session_head_materialization")
|
||||||
|
|||||||
@@ -702,31 +702,46 @@ impl WorkspaceHttpWorkdirBackend {
|
|||||||
ATTACH_TOOL,
|
ATTACH_TOOL,
|
||||||
"working_directory_id",
|
"working_directory_id",
|
||||||
)?;
|
)?;
|
||||||
let alias_key = workdir::WorkdirAttachmentAlias::new(alias)
|
|
||||||
.map_err(|error| ToolError::InvalidArgument(error.to_string()))?;
|
|
||||||
if self.session_router.session(&alias_key).is_some() {
|
|
||||||
return Err(ToolError::InvalidArgument(format!(
|
|
||||||
"attachment alias `{alias}` already exists"
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
let response = self.attach_response(alias, working_directory_id)?;
|
let response = self.attach_response(alias, working_directory_id)?;
|
||||||
if let Err(error) = self.session_router.attach(
|
if !response.attachment.attached
|
||||||
alias_key.clone(),
|
|| response.attachment.working_directory_id != working_directory_id
|
||||||
|
{
|
||||||
|
return Err(ToolError::ExecutionFailed(
|
||||||
|
"Workdir attachment response does not match the target".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
// Backend ownership decides the name. A requested alias cannot rename
|
||||||
|
// an existing attachment or create a second route to the same Workdir.
|
||||||
|
let attached_alias = &response.attachment.alias;
|
||||||
|
let alias_key = workdir::WorkdirAttachmentAlias::new(attached_alias.clone())
|
||||||
|
.map_err(|error| ToolError::ExecutionFailed(error.to_string()))?;
|
||||||
|
if let Some(session) = self.session_router.session(&alias_key) {
|
||||||
|
if session.workdir().id().as_str() != working_directory_id {
|
||||||
|
return Err(ToolError::ExecutionFailed("Local Workdir attachment differs from Backend; refresh attachments before retrying".into()));
|
||||||
|
}
|
||||||
|
} else if let Err(error) = self.session_router.attach(
|
||||||
|
alias_key,
|
||||||
WorkspaceAttachedWorkdirSession::handle_for_workdir_with_environment(
|
WorkspaceAttachedWorkdirSession::handle_for_workdir_with_environment(
|
||||||
self.client.clone(),
|
self.client.clone(),
|
||||||
alias,
|
attached_alias,
|
||||||
working_directory_id,
|
working_directory_id,
|
||||||
response.attachment.capabilities,
|
response.attachment.capabilities,
|
||||||
response.environment.clone(),
|
response.environment.clone(),
|
||||||
),
|
),
|
||||||
) {
|
) {
|
||||||
let _ = self.detach(alias);
|
if !response.attachment.already_attached {
|
||||||
|
let _ = self.detach(attached_alias);
|
||||||
|
}
|
||||||
return Err(ToolError::ExecutionFailed(format!(
|
return Err(ToolError::ExecutionFailed(format!(
|
||||||
"attach Workdir session route: {error}"
|
"attach Workdir session route: {error}"
|
||||||
)));
|
)));
|
||||||
}
|
}
|
||||||
workdir_output(
|
workdir_output(
|
||||||
format!("Attached Workdir {working_directory_id} as {alias}"),
|
if response.attachment.already_attached {
|
||||||
|
format!("Workdir {working_directory_id} is already attached as {attached_alias}")
|
||||||
|
} else {
|
||||||
|
format!("Attached Workdir {working_directory_id} as {attached_alias}")
|
||||||
|
},
|
||||||
&response,
|
&response,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
@@ -737,15 +752,31 @@ impl WorkspaceHttpWorkdirBackend {
|
|||||||
working_directory_id: &str,
|
working_directory_id: &str,
|
||||||
) -> Result<WorkdirSessionAttachmentResponse, ToolError> {
|
) -> Result<WorkdirSessionAttachmentResponse, ToolError> {
|
||||||
let workspace_id = encode_path_segment(self.workspace_id()?);
|
let workspace_id = encode_path_segment(self.workspace_id()?);
|
||||||
self.execute_json::<WorkdirSessionAttachmentResponse>(WorkspaceRequest::json(
|
let response = self
|
||||||
WorkspaceRequestMethod::Post,
|
.client
|
||||||
format!("/api/w/{workspace_id}/workers/self/workdir-attachments"),
|
.execute(WorkspaceRequest::json(
|
||||||
serde_json::to_string(&WorkdirAttachRequest {
|
WorkspaceRequestMethod::Post,
|
||||||
alias: alias.to_string(),
|
format!("/api/w/{workspace_id}/workers/self/workdir-attachments"),
|
||||||
working_directory_id: working_directory_id.to_string(),
|
serde_json::to_string(&WorkdirAttachRequest {
|
||||||
})
|
alias: alias.to_string(),
|
||||||
.map_err(decode_error)?,
|
working_directory_id: working_directory_id.to_string(),
|
||||||
))
|
})
|
||||||
|
.map_err(decode_error)?,
|
||||||
|
))
|
||||||
|
.map_err(|error| {
|
||||||
|
ToolError::ExecutionFailed(format!("Workspace Workdir request failed: {error}"))
|
||||||
|
})?;
|
||||||
|
if response.status == 409
|
||||||
|
&& let Ok(error) =
|
||||||
|
serde_json::from_str::<server_api::RepositoryApiError>(&response.body)
|
||||||
|
&& error.error == "workdir_occupied"
|
||||||
|
{
|
||||||
|
return Err(ToolError::StructuredConflict {
|
||||||
|
code: "workdir_occupied".into(),
|
||||||
|
message: "Workdir is occupied by another Worker".into(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
decode_response(response)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn detach(&self, alias: &str) -> Result<ToolOutput, ToolError> {
|
fn detach(&self, alias: &str) -> Result<ToolOutput, ToolError> {
|
||||||
@@ -778,7 +809,11 @@ impl WorkspaceHttpWorkdirBackend {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn delete(&self, input: WorkdirDeleteInput) -> Result<ToolOutput, ToolError> {
|
fn delete(
|
||||||
|
&self,
|
||||||
|
input: WorkdirDeleteInput,
|
||||||
|
operation_id: String,
|
||||||
|
) -> Result<ToolOutput, ToolError> {
|
||||||
let workdir_id = validate_identity(
|
let workdir_id = validate_identity(
|
||||||
&input.working_directory_id,
|
&input.working_directory_id,
|
||||||
DELETE_TOOL,
|
DELETE_TOOL,
|
||||||
@@ -790,6 +825,7 @@ impl WorkspaceHttpWorkdirBackend {
|
|||||||
WorkspaceRequestMethod::Delete,
|
WorkspaceRequestMethod::Delete,
|
||||||
format!("/api/w/{workspace_id}/working-directories/{workdir_path}"),
|
format!("/api/w/{workspace_id}/working-directories/{workdir_path}"),
|
||||||
serde_json::to_string(&WorkdirRemovalRequest {
|
serde_json::to_string(&WorkdirRemovalRequest {
|
||||||
|
operation_id,
|
||||||
reason: validate_delete_reason(&input.reason)?.to_string(),
|
reason: validate_delete_reason(&input.reason)?.to_string(),
|
||||||
})
|
})
|
||||||
.map_err(decode_error)?,
|
.map_err(decode_error)?,
|
||||||
@@ -800,7 +836,7 @@ impl WorkspaceHttpWorkdirBackend {
|
|||||||
response.disposition,
|
response.disposition,
|
||||||
removal_guidance(response.failure_category.as_deref()),
|
removal_guidance(response.failure_category.as_deref()),
|
||||||
if response.retryable {
|
if response.retryable {
|
||||||
" After resolving the cause, repeat this WorkdirDelete with the same id and reason; current safety conditions will be rechecked."
|
" After resolving the cause, a new explicit WorkdirDelete request will recheck current safety conditions. This failed request will not be automatically re-executed."
|
||||||
} else {
|
} else {
|
||||||
""
|
""
|
||||||
}
|
}
|
||||||
@@ -954,9 +990,10 @@ impl Tool for WorkspaceHttpWorkdirTool {
|
|||||||
self.backend.detach_managed(&input.alias, None).await
|
self.backend.detach_managed(&input.alias, None).await
|
||||||
}
|
}
|
||||||
|
|
||||||
WorkdirOperation::Delete => self
|
WorkdirOperation::Delete => self.backend.delete(
|
||||||
.backend
|
parse_input::<WorkdirDeleteInput>(input_json)?,
|
||||||
.delete(parse_input::<WorkdirDeleteInput>(input_json)?),
|
ctx.call_id.to_string(),
|
||||||
|
),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1018,8 +1055,14 @@ fn removal_guidance(category: Option<&str>) -> &'static str {
|
|||||||
Some("runtime_unavailable" | "provider_unavailable") => {
|
Some("runtime_unavailable" | "provider_unavailable") => {
|
||||||
"Restore the Runtime connection; the registry remains until deletion is confirmed."
|
"Restore the Runtime connection; the registry remains until deletion is confirmed."
|
||||||
}
|
}
|
||||||
Some("provider_observation_unknown" | "provider_cleanup_outcome_unknown") => {
|
Some("provider_cleanup_outcome_unknown") => {
|
||||||
"Deletion is not confirmed. Re-observe with normal removal after restoring provider access; the registry is retained."
|
"Deletion outcome is unknown. Restore provider access and reconcile the existing request; do not submit a fresh deletion intent. Recovery only observes the target and never repeats deletion."
|
||||||
|
}
|
||||||
|
Some("provider_observation_unknown") => {
|
||||||
|
"The target could not be observed before deletion. Restore provider access before a new request."
|
||||||
|
}
|
||||||
|
Some("request_interrupted") => {
|
||||||
|
"The request was interrupted before execution. A new explicit request is required."
|
||||||
}
|
}
|
||||||
Some("unsupported_target") => {
|
Some("unsupported_target") => {
|
||||||
"Ask the owner for the provider's supported cleanup operation."
|
"Ask the owner for the provider's supported cleanup operation."
|
||||||
@@ -1219,6 +1262,8 @@ struct WorkdirAttachmentResponse {
|
|||||||
working_directory_id: String,
|
working_directory_id: String,
|
||||||
capabilities: WorkdirSessionCapabilities,
|
capabilities: WorkdirSessionCapabilities,
|
||||||
attached: bool,
|
attached: bool,
|
||||||
|
#[serde(default)]
|
||||||
|
already_attached: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
|
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
@@ -1337,8 +1382,44 @@ mod tests {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn delete_call_identity_distinguishes_new_requests_from_exact_resends() {
|
||||||
|
let client = Arc::new(RecordingWorkspaceClient::new(
|
||||||
|
(0..3)
|
||||||
|
.map(|_| {
|
||||||
|
response(json!({
|
||||||
|
"working_directory_id": "wd-request",
|
||||||
|
"disposition": "attention_required",
|
||||||
|
"retryable": true,
|
||||||
|
"failure_category": "resource_busy"
|
||||||
|
}))
|
||||||
|
})
|
||||||
|
.collect(),
|
||||||
|
));
|
||||||
|
let tool = WorkspaceHttpWorkdirTool {
|
||||||
|
backend: WorkspaceHttpWorkdirBackend::new(client.clone()),
|
||||||
|
operation: WorkdirOperation::Delete,
|
||||||
|
};
|
||||||
|
let input = r#"{"working_directory_id":"wd-request","reason":"remove unused Workdir"}"#;
|
||||||
|
for id in ["delete-1", "delete-1", "delete-2"] {
|
||||||
|
tool.execute(input, ToolExecutionContext::new(id, "batch", 0))
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
let bodies: Vec<serde_json::Value> = client
|
||||||
|
.requests()
|
||||||
|
.iter()
|
||||||
|
.map(|r| serde_json::from_str(r.body.as_deref().unwrap()).unwrap())
|
||||||
|
.collect();
|
||||||
|
assert_eq!(bodies[0], bodies[1]);
|
||||||
|
assert_eq!(bodies[0]["operation_id"], "delete-1");
|
||||||
|
assert_eq!(bodies[2]["operation_id"], "delete-2");
|
||||||
|
assert_eq!(bodies[0]["reason"], bodies[2]["reason"]);
|
||||||
|
assert!(delete_schema()["properties"].get("operation_id").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn ordinary_workdir_delete_reports_safe_cause_and_exact_retry_guidance() {
|
fn ordinary_workdir_delete_reports_safe_cause_and_new_request_guidance() {
|
||||||
let client = Arc::new(RecordingWorkspaceClient::new(vec![response(json!({
|
let client = Arc::new(RecordingWorkspaceClient::new(vec![response(json!({
|
||||||
"working_directory_id": "wd-retry",
|
"working_directory_id": "wd-retry",
|
||||||
"disposition": "attention_required",
|
"disposition": "attention_required",
|
||||||
@@ -1347,10 +1428,13 @@ mod tests {
|
|||||||
}))]));
|
}))]));
|
||||||
let backend = WorkspaceHttpWorkdirBackend::new(client.clone());
|
let backend = WorkspaceHttpWorkdirBackend::new(client.clone());
|
||||||
let output = backend
|
let output = backend
|
||||||
.delete(WorkdirDeleteInput {
|
.delete(
|
||||||
working_directory_id: "wd-retry".into(),
|
WorkdirDeleteInput {
|
||||||
reason: "remove unused Workdir".into(),
|
working_directory_id: "wd-retry".into(),
|
||||||
})
|
reason: "remove unused Workdir".into(),
|
||||||
|
},
|
||||||
|
"delete-call-1".to_string(),
|
||||||
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert!(output.summary.contains("Confirm its owner"));
|
assert!(output.summary.contains("Confirm its owner"));
|
||||||
assert!(
|
assert!(
|
||||||
@@ -1358,7 +1442,11 @@ mod tests {
|
|||||||
.summary
|
.summary
|
||||||
.contains("Do not recursively delete mounted data")
|
.contains("Do not recursively delete mounted data")
|
||||||
);
|
);
|
||||||
assert!(output.summary.contains("same id and reason"));
|
assert!(
|
||||||
|
output
|
||||||
|
.summary
|
||||||
|
.contains("new explicit WorkdirDelete request")
|
||||||
|
);
|
||||||
assert!(output.summary.len() <= 600);
|
assert!(output.summary.len() <= 600);
|
||||||
let content: serde_json::Value =
|
let content: serde_json::Value =
|
||||||
serde_json::from_str(output.content.as_deref().unwrap()).unwrap();
|
serde_json::from_str(output.content.as_deref().unwrap()).unwrap();
|
||||||
@@ -1596,10 +1684,13 @@ mod tests {
|
|||||||
.unwrap();
|
.unwrap();
|
||||||
backend.detach("checkout").unwrap();
|
backend.detach("checkout").unwrap();
|
||||||
backend
|
backend
|
||||||
.delete(WorkdirDeleteInput {
|
.delete(
|
||||||
working_directory_id: "wd-created".to_string(),
|
WorkdirDeleteInput {
|
||||||
reason: "remove stale Workdir".to_string(),
|
working_directory_id: "wd-created".to_string(),
|
||||||
})
|
reason: "remove stale Workdir".to_string(),
|
||||||
|
},
|
||||||
|
"delete-call-1".to_string(),
|
||||||
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
|
||||||
let requests = client.requests();
|
let requests = client.requests();
|
||||||
@@ -1637,7 +1728,102 @@ mod tests {
|
|||||||
assert_eq!(requests[4].method, WorkspaceRequestMethod::Delete);
|
assert_eq!(requests[4].method, WorkspaceRequestMethod::Delete);
|
||||||
let body: serde_json::Value =
|
let body: serde_json::Value =
|
||||||
serde_json::from_str(requests[4].body.as_deref().unwrap()).unwrap();
|
serde_json::from_str(requests[4].body.as_deref().unwrap()).unwrap();
|
||||||
assert_eq!(body, json!({"reason": "remove stale Workdir"}));
|
assert_eq!(
|
||||||
|
body,
|
||||||
|
json!({"reason": "remove stale Workdir", "operation_id": "delete-call-1"})
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn attach_reports_other_worker_occupancy_without_exposing_backend_details() {
|
||||||
|
let client = Arc::new(RecordingWorkspaceClient::new(vec![WorkspaceResponse {
|
||||||
|
status: 409,
|
||||||
|
body: serde_json::to_string(&server_api::RepositoryApiError::new(
|
||||||
|
409,
|
||||||
|
"workdir_occupied",
|
||||||
|
"private backend detail",
|
||||||
|
Vec::new(),
|
||||||
|
))
|
||||||
|
.unwrap(),
|
||||||
|
}]));
|
||||||
|
let backend = WorkspaceHttpWorkdirBackend::new(client.clone());
|
||||||
|
let error = backend
|
||||||
|
.attach(WorkdirAttachInput {
|
||||||
|
alias: "work".into(),
|
||||||
|
working_directory_id: "occupied".into(),
|
||||||
|
})
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(
|
||||||
|
matches!(error, ToolError::StructuredConflict { code, message } if code == "workdir_occupied" && message == "Workdir is occupied by another Worker")
|
||||||
|
);
|
||||||
|
assert_eq!(client.requests().len(), 1);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn attach_reuses_backend_alias_without_duplicate_routes_or_detach() {
|
||||||
|
let client = Arc::new(RecordingWorkspaceClient::new((0..3).map(|_| response(json!({
|
||||||
|
"workspace_id": "workspace/test", "alias": "existing", "working_directory_id": "wd-existing",
|
||||||
|
"capabilities": {"bits": 63}, "attached": true, "already_attached": true,
|
||||||
|
"environment": workdir::WorkdirSessionEnvironment::root(WorkdirSessionCapabilities::ALL)
|
||||||
|
}))).collect()));
|
||||||
|
let backend = WorkspaceHttpWorkdirBackend::new(client.clone());
|
||||||
|
let occupied_alias = workdir::WorkdirAttachmentAlias::new("other").unwrap();
|
||||||
|
backend
|
||||||
|
.session_router
|
||||||
|
.attach(
|
||||||
|
occupied_alias.clone(),
|
||||||
|
WorkspaceAttachedWorkdirSession::handle_for_workdir(
|
||||||
|
client.clone(),
|
||||||
|
"other",
|
||||||
|
"wd-other",
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
for alias in ["new-name", "existing", "other"] {
|
||||||
|
let output = backend
|
||||||
|
.attach(WorkdirAttachInput {
|
||||||
|
alias: alias.into(),
|
||||||
|
working_directory_id: "wd-existing".into(),
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
output.summary,
|
||||||
|
"Workdir wd-existing is already attached as existing"
|
||||||
|
);
|
||||||
|
let body: serde_json::Value =
|
||||||
|
serde_json::from_str(output.content.as_deref().unwrap()).unwrap();
|
||||||
|
assert_eq!(body["alias"], "existing");
|
||||||
|
assert_eq!(body["already_attached"], true);
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
backend
|
||||||
|
.session_router
|
||||||
|
.session(&workdir::WorkdirAttachmentAlias::new("new-name").unwrap())
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
backend
|
||||||
|
.session_router
|
||||||
|
.session(&occupied_alias)
|
||||||
|
.unwrap()
|
||||||
|
.workdir()
|
||||||
|
.id()
|
||||||
|
.as_str(),
|
||||||
|
"wd-other"
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
backend
|
||||||
|
.session_router
|
||||||
|
.session(&workdir::WorkdirAttachmentAlias::new("existing").unwrap())
|
||||||
|
.is_some()
|
||||||
|
);
|
||||||
|
assert_eq!(client.requests().len(), 3);
|
||||||
|
assert!(
|
||||||
|
client
|
||||||
|
.requests()
|
||||||
|
.iter()
|
||||||
|
.all(|request| request.method == WorkspaceRequestMethod::Post)
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
|
|||||||
@@ -8,6 +8,7 @@
|
|||||||
//! the regular registry decides Tool-name collisions.
|
//! the regular registry decides Tool-name collisions.
|
||||||
|
|
||||||
use std::collections::BTreeMap;
|
use std::collections::BTreeMap;
|
||||||
|
use std::io;
|
||||||
use std::path::Path;
|
use std::path::Path;
|
||||||
use std::sync::Arc;
|
use std::sync::Arc;
|
||||||
|
|
||||||
@@ -25,6 +26,7 @@ use mcp::stdio::{
|
|||||||
};
|
};
|
||||||
use serde::Serialize;
|
use serde::Serialize;
|
||||||
use serde_json::{Map, Value};
|
use serde_json::{Map, Value};
|
||||||
|
use tokio::sync::Mutex;
|
||||||
|
|
||||||
use super::{
|
use super::{
|
||||||
FeatureDescriptor, FeatureDiagnostic, FeatureInstallContext, FeatureInstallError,
|
FeatureDescriptor, FeatureDiagnostic, FeatureInstallContext, FeatureInstallError,
|
||||||
@@ -52,8 +54,88 @@ const MAX_LIST_ITEMS: usize = 128;
|
|||||||
const MAX_RESOURCE_CONTENTS: usize = 16;
|
const MAX_RESOURCE_CONTENTS: usize = 16;
|
||||||
const MAX_PROMPT_MESSAGES: usize = 32;
|
const MAX_PROMPT_MESSAGES: usize = 32;
|
||||||
|
|
||||||
|
/// Owns discovery clients until process reaping is confirmed. Retain a clone in
|
||||||
|
/// startup cleanup *before* awaiting discovery. Stop/cancel discovery before
|
||||||
|
/// calling `shutdown`; the shared lock serializes discovery and cleanup, not
|
||||||
|
/// cancellation of an in-flight discovery task.
|
||||||
|
///
|
||||||
|
/// Error or cancellation leaves the actual clients here for another shutdown
|
||||||
|
/// attempt. Dropping the final owner only provides best-effort process killing.
|
||||||
|
#[derive(Clone, Default)]
|
||||||
|
pub struct McpDiscoveryCleanup {
|
||||||
|
clients: Arc<Mutex<Vec<McpStdioClient>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl McpDiscoveryCleanup {
|
||||||
|
pub fn new() -> Self {
|
||||||
|
Self::default()
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Try every retained client once, removing only confirmed shutdowns. An
|
||||||
|
/// error is not a cleanup receipt; retry this same owner, not discovery.
|
||||||
|
pub async fn shutdown(&self) -> io::Result<()> {
|
||||||
|
let mut clients = self.clients.lock().await;
|
||||||
|
let mut first_error = None;
|
||||||
|
let mut index = 0;
|
||||||
|
while index < clients.len() {
|
||||||
|
match clients[index].shutdown().await {
|
||||||
|
Ok(_) => {
|
||||||
|
clients.remove(index);
|
||||||
|
}
|
||||||
|
Err(err) => {
|
||||||
|
first_error.get_or_insert_with(|| discovery_shutdown_error(&err));
|
||||||
|
index += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
match first_error {
|
||||||
|
Some(err) => Err(err),
|
||||||
|
None => Ok(()),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn discovery_shutdown_error(err: &McpClientError) -> io::Error {
|
||||||
|
io::Error::other(bounded_diagnostic(format!(
|
||||||
|
"MCP discovery shutdown unconfirmed; client retained for cleanup retry: {err}"
|
||||||
|
)))
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Restore preparation must fail if discovery cannot confirm process shutdown.
|
||||||
|
/// The caller retains `cleanup` before this future is polled, including during
|
||||||
|
/// initialization. Ordinary discovery failures remain isolated contributions
|
||||||
|
/// only after their client has been reaped.
|
||||||
|
pub async fn discover_stdio_tool_feature_with_cleanup(
|
||||||
|
config: &McpConfig,
|
||||||
|
workspace_root: &Path,
|
||||||
|
cleanup: &McpDiscoveryCleanup,
|
||||||
|
) -> Result<Option<McpStdioToolFeature>, io::Error> {
|
||||||
|
if config.stdio_servers.is_empty() {
|
||||||
|
return Ok(None);
|
||||||
|
}
|
||||||
|
let mut feature = McpStdioToolFeature::new();
|
||||||
|
for server in &config.stdio_servers {
|
||||||
|
match resolve_stdio_server(server, workspace_root, None) {
|
||||||
|
Ok(spec) => {
|
||||||
|
feature.add_contribution(
|
||||||
|
discover_server_tools_with_cleanup(spec, cleanup, McpStdioLimits::default())
|
||||||
|
.await?,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
Err(err) => {
|
||||||
|
feature.add_diagnostic(FeatureDiagnostic::error(bounded_diagnostic(format!(
|
||||||
|
"failed to resolve MCP stdio server `{}`: {err}",
|
||||||
|
server.name
|
||||||
|
))))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(Some(feature))
|
||||||
|
}
|
||||||
|
|
||||||
/// Discover enabled MCP stdio server tools and return a single feature module
|
/// Discover enabled MCP stdio server tools and return a single feature module
|
||||||
/// containing startup contributions for normal ToolRegistry installation.
|
/// containing startup contributions for normal ToolRegistry installation.
|
||||||
|
/// Callers needing retryable startup cleanup must use the cleanup-aware API.
|
||||||
pub async fn discover_stdio_tool_feature(
|
pub async fn discover_stdio_tool_feature(
|
||||||
config: &McpConfig,
|
config: &McpConfig,
|
||||||
workspace_root: &Path,
|
workspace_root: &Path,
|
||||||
@@ -81,21 +163,66 @@ pub async fn discover_stdio_tool_feature(
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn discover_server_tools(spec: McpStdioServerSpec) -> ProtocolProviderContribution {
|
async fn discover_server_tools(spec: McpStdioServerSpec) -> ProtocolProviderContribution {
|
||||||
|
let declaration = provider_declaration(&spec.name, None);
|
||||||
|
let cleanup = McpDiscoveryCleanup::new();
|
||||||
|
match discover_server_tools_with_cleanup(spec, &cleanup, McpStdioLimits::default()).await {
|
||||||
|
Ok(contribution) => contribution,
|
||||||
|
Err(err) => {
|
||||||
|
ProtocolProviderContribution::failed(declaration, bounded_diagnostic(err.to_string()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn discover_server_tools_with_cleanup(
|
||||||
|
spec: McpStdioServerSpec,
|
||||||
|
cleanup: &McpDiscoveryCleanup,
|
||||||
|
limits: McpStdioLimits,
|
||||||
|
) -> io::Result<ProtocolProviderContribution> {
|
||||||
|
let declaration = provider_declaration(&spec.name, None);
|
||||||
|
let mut clients = cleanup.clients.lock().await;
|
||||||
|
if !clients.is_empty() {
|
||||||
|
return Err(io::Error::other(
|
||||||
|
"MCP discovery cleanup is pending; shut down the retained owner before retrying discovery",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let client = match McpStdioClient::spawn_uninitialized(spec.clone(), limits) {
|
||||||
|
Ok(client) => client,
|
||||||
|
Err(err) => {
|
||||||
|
// The synchronous spawn API only errors before a process exists;
|
||||||
|
// pipe/handshake failures retain a client and use the shutdown path.
|
||||||
|
return Ok(ProtocolProviderContribution::failed(
|
||||||
|
declaration,
|
||||||
|
bounded_diagnostic(err.to_string()),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
// Spawn is synchronous: the owner holds the client before the first await
|
||||||
|
// that can leave a running process on initialization failure/cancellation.
|
||||||
|
clients.push(client);
|
||||||
|
let client = clients.last_mut().expect("retained discovery client");
|
||||||
|
let contribution = match client.initialize_with_timeout().await {
|
||||||
|
Ok(()) => discover_connected_server_tools(spec, client).await,
|
||||||
|
Err(err) => {
|
||||||
|
ProtocolProviderContribution::failed(declaration, bounded_diagnostic(err.to_string()))
|
||||||
|
}
|
||||||
|
};
|
||||||
|
client
|
||||||
|
.shutdown()
|
||||||
|
.await
|
||||||
|
.map_err(|err| discovery_shutdown_error(&err))?;
|
||||||
|
drop(clients.pop());
|
||||||
|
Ok(contribution)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn discover_connected_server_tools(
|
||||||
|
spec: McpStdioServerSpec,
|
||||||
|
client: &mut McpStdioClient,
|
||||||
|
) -> ProtocolProviderContribution {
|
||||||
let declaration = provider_declaration(&spec.name, None);
|
let declaration = provider_declaration(&spec.name, None);
|
||||||
let mut contribution = ProtocolProviderContribution::ready(declaration.clone());
|
let mut contribution = ProtocolProviderContribution::ready(declaration.clone());
|
||||||
let server_namespace = sanitize_segment(&spec.name);
|
let server_namespace = sanitize_segment(&spec.name);
|
||||||
let execution_spec = spec.clone();
|
let execution_spec = spec.clone();
|
||||||
|
|
||||||
let mut client = match McpStdioClient::connect(spec, McpStdioLimits::default()).await {
|
|
||||||
Ok(client) => client,
|
|
||||||
Err(err) => {
|
|
||||||
return ProtocolProviderContribution::failed(
|
|
||||||
declaration,
|
|
||||||
bounded_diagnostic(err.to_string()),
|
|
||||||
);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
let (server_version, has_tools, has_resources, has_prompts) = client
|
let (server_version, has_tools, has_resources, has_prompts) = client
|
||||||
.initialize_result()
|
.initialize_result()
|
||||||
.map(|result| {
|
.map(|result| {
|
||||||
@@ -133,18 +260,10 @@ async fn discover_server_tools(spec: McpStdioServerSpec) -> ProtocolProviderCont
|
|||||||
{
|
{
|
||||||
Ok(list) => list,
|
Ok(list) => list,
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
let mut failed = ProtocolProviderContribution::failed(
|
return ProtocolProviderContribution::failed(
|
||||||
declaration,
|
declaration,
|
||||||
bounded_diagnostic(err.to_string()),
|
bounded_diagnostic(err.to_string()),
|
||||||
);
|
);
|
||||||
if let Err(shutdown_err) = client.shutdown().await {
|
|
||||||
failed = failed.with_diagnostic(FeatureDiagnostic::warning(
|
|
||||||
bounded_diagnostic(format!(
|
|
||||||
"MCP server shutdown after discovery failure failed: {shutdown_err}"
|
|
||||||
)),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
return failed;
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let list_changes = client.snapshot_list_changes().await;
|
let list_changes = client.snapshot_list_changes().await;
|
||||||
@@ -166,20 +285,12 @@ async fn discover_server_tools(spec: McpStdioServerSpec) -> ProtocolProviderCont
|
|||||||
{
|
{
|
||||||
Ok(list) => list,
|
Ok(list) => list,
|
||||||
Err(err) => {
|
Err(err) => {
|
||||||
let mut failed = ProtocolProviderContribution::failed(
|
return ProtocolProviderContribution::failed(
|
||||||
declaration,
|
declaration,
|
||||||
bounded_diagnostic(format!(
|
bounded_diagnostic(format!(
|
||||||
"MCP server sent notifications/tools/list_changed during tool discovery and refresh failed: {err}"
|
"MCP server sent notifications/tools/list_changed during tool discovery and refresh failed: {err}"
|
||||||
)),
|
)),
|
||||||
);
|
);
|
||||||
if let Err(shutdown_err) = client.shutdown().await {
|
|
||||||
failed = failed.with_diagnostic(FeatureDiagnostic::warning(
|
|
||||||
bounded_diagnostic(format!(
|
|
||||||
"MCP server shutdown after discovery refresh failure failed: {shutdown_err}"
|
|
||||||
)),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
return failed;
|
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
let refresh_changes = client.snapshot_list_changes().await;
|
let refresh_changes = client.snapshot_list_changes().await;
|
||||||
@@ -197,13 +308,6 @@ async fn discover_server_tools(spec: McpStdioServerSpec) -> ProtocolProviderCont
|
|||||||
} else {
|
} else {
|
||||||
None
|
None
|
||||||
};
|
};
|
||||||
let shutdown_result = client.shutdown().await;
|
|
||||||
if let Err(err) = shutdown_result {
|
|
||||||
contribution =
|
|
||||||
contribution.with_diagnostic(FeatureDiagnostic::warning(bounded_diagnostic(format!(
|
|
||||||
"MCP server shutdown after capability discovery failed: {err}"
|
|
||||||
))));
|
|
||||||
}
|
|
||||||
|
|
||||||
if let Some(list) = list {
|
if let Some(list) = list {
|
||||||
contribution = normalize_listed_tools(
|
contribution = normalize_listed_tools(
|
||||||
@@ -1819,6 +1923,140 @@ read exit_notification || true
|
|||||||
McpStdioServerSpec::new("shell-mock", "/bin/sh").args(["-c".to_string(), script])
|
McpStdioServerSpec::new("shell-mock", "/bin/sh").args(["-c".to_string(), script])
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
fn discovery_config(spec: McpStdioServerSpec) -> McpConfig {
|
||||||
|
McpConfig {
|
||||||
|
stdio_servers: vec![manifest::McpStdioServerConfig {
|
||||||
|
name: spec.name,
|
||||||
|
command: spec.command,
|
||||||
|
args: spec.args,
|
||||||
|
cwd: None,
|
||||||
|
env: Default::default(),
|
||||||
|
}],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
#[tokio::test]
|
||||||
|
async fn cancelled_discovery_retains_initializing_client_in_cleanup_clone() {
|
||||||
|
use std::future::{Future, poll_fn};
|
||||||
|
use std::task::Poll;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
let config = discovery_config(
|
||||||
|
McpStdioServerSpec::new("initializing", "/bin/sh")
|
||||||
|
.args(["-c", "while read line; do :; done"]),
|
||||||
|
);
|
||||||
|
let cleanup = McpDiscoveryCleanup::new();
|
||||||
|
let retained = cleanup.clone();
|
||||||
|
let mut discovery = Box::pin(discover_stdio_tool_feature_with_cleanup(
|
||||||
|
&config,
|
||||||
|
Path::new("."),
|
||||||
|
&cleanup,
|
||||||
|
));
|
||||||
|
poll_fn(|cx| {
|
||||||
|
assert!(discovery.as_mut().poll(cx).is_pending());
|
||||||
|
Poll::Ready(())
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
drop(discovery);
|
||||||
|
drop(cleanup);
|
||||||
|
assert_eq!(retained.clients.lock().await.len(), 1);
|
||||||
|
let retry_error =
|
||||||
|
discover_stdio_tool_feature_with_cleanup(&config, Path::new("."), &retained)
|
||||||
|
.await
|
||||||
|
.err()
|
||||||
|
.expect("discovery must not replace an unreaped client");
|
||||||
|
assert!(retry_error.to_string().contains("cleanup is pending"));
|
||||||
|
assert_eq!(retained.clients.lock().await.len(), 1);
|
||||||
|
tokio::time::timeout(Duration::from_secs(3), retained.shutdown())
|
||||||
|
.await
|
||||||
|
.expect("cleanup bounded")
|
||||||
|
.expect("initializing process reaped");
|
||||||
|
assert!(retained.clients.lock().await.is_empty());
|
||||||
|
retained
|
||||||
|
.shutdown()
|
||||||
|
.await
|
||||||
|
.expect("confirmed cleanup repeatable");
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
#[tokio::test]
|
||||||
|
async fn cancelled_owner_shutdown_keeps_client_for_retry() {
|
||||||
|
use std::future::{Future, poll_fn};
|
||||||
|
use std::task::Poll;
|
||||||
|
use std::time::Duration;
|
||||||
|
|
||||||
|
let cleanup = McpDiscoveryCleanup::new();
|
||||||
|
cleanup.clients.lock().await.push(
|
||||||
|
McpStdioClient::spawn_uninitialized(
|
||||||
|
McpStdioServerSpec::new("shutdown-pending", "/bin/sh")
|
||||||
|
.args(["-c", "while :; do :; done"]),
|
||||||
|
McpStdioLimits {
|
||||||
|
shutdown_timeout: Duration::from_millis(50),
|
||||||
|
kill_timeout: Duration::from_millis(100),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.expect("local process spawned"),
|
||||||
|
);
|
||||||
|
let mut shutdown = Box::pin(cleanup.shutdown());
|
||||||
|
poll_fn(|cx| {
|
||||||
|
assert!(shutdown.as_mut().poll(cx).is_pending());
|
||||||
|
Poll::Ready(())
|
||||||
|
})
|
||||||
|
.await;
|
||||||
|
drop(shutdown);
|
||||||
|
assert_eq!(cleanup.clients.lock().await.len(), 1);
|
||||||
|
tokio::time::timeout(Duration::from_secs(2), cleanup.shutdown())
|
||||||
|
.await
|
||||||
|
.expect("retry bounded")
|
||||||
|
.expect("retained client shutdown");
|
||||||
|
assert!(cleanup.clients.lock().await.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
#[tokio::test]
|
||||||
|
async fn cleanup_aware_discovery_reaps_success_and_initialize_failure() {
|
||||||
|
let cleanup = McpDiscoveryCleanup::new();
|
||||||
|
let config = discovery_config(shell_capability_server(r#"{"resources":{}}"#));
|
||||||
|
let feature = discover_stdio_tool_feature_with_cleanup(&config, Path::new("."), &cleanup)
|
||||||
|
.await
|
||||||
|
.expect("discovery cleanup confirmed")
|
||||||
|
.expect("configured feature");
|
||||||
|
assert!(
|
||||||
|
feature.contributions[0]
|
||||||
|
.tools
|
||||||
|
.iter()
|
||||||
|
.any(|tool| { tool.name() == "Mcp_shell_mock_resources_read" })
|
||||||
|
);
|
||||||
|
assert!(cleanup.clients.lock().await.is_empty());
|
||||||
|
|
||||||
|
let spec = McpStdioServerSpec::new("init-rejected", "/bin/sh").args([
|
||||||
|
"-c",
|
||||||
|
r#"read init || exit 1
|
||||||
|
printf '%s\n' '{"jsonrpc":"2.0","id":1,"error":{"code":-32000,"message":"initialization rejected"}}'
|
||||||
|
while read line; do :; done
|
||||||
|
"#,
|
||||||
|
]);
|
||||||
|
let config = discovery_config(spec);
|
||||||
|
let feature = discover_stdio_tool_feature_with_cleanup(&config, Path::new("."), &cleanup)
|
||||||
|
.await
|
||||||
|
.expect("failure isolated only after cleanup")
|
||||||
|
.expect("failed contribution preserved");
|
||||||
|
assert!(feature.contributions[0].tools.is_empty());
|
||||||
|
assert!(
|
||||||
|
feature.contributions[0]
|
||||||
|
.diagnostics
|
||||||
|
.iter()
|
||||||
|
.any(|diagnostic| {
|
||||||
|
diagnostic.severity == FeatureDiagnosticSeverity::Error
|
||||||
|
&& diagnostic.message.contains("initialization rejected")
|
||||||
|
})
|
||||||
|
);
|
||||||
|
assert!(cleanup.clients.lock().await.is_empty());
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn mcp_resource_prompt_operation_tools_are_namespaced_and_untrusted() {
|
fn mcp_resource_prompt_operation_tools_are_namespaced_and_untrusted() {
|
||||||
let declaration = provider_declaration("demo", Some("1.0.0"));
|
let declaration = provider_declaration("demo", Some("1.0.0"));
|
||||||
|
|||||||
@@ -19,6 +19,9 @@ pub mod model_client;
|
|||||||
mod paste_artifact_tool;
|
mod paste_artifact_tool;
|
||||||
pub mod profile_manifest;
|
pub mod profile_manifest;
|
||||||
pub mod prompt;
|
pub mod prompt;
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "../../../tests/support/restore_environment.rs"]
|
||||||
|
mod restore_test_environment;
|
||||||
pub mod runtime;
|
pub mod runtime;
|
||||||
pub mod segment_log_sink;
|
pub mod segment_log_sink;
|
||||||
mod session_capture;
|
mod session_capture;
|
||||||
@@ -69,8 +72,9 @@ pub use session_history::{
|
|||||||
};
|
};
|
||||||
pub use shared_state::WorkerSharedState;
|
pub use shared_state::WorkerSharedState;
|
||||||
pub use worker::{
|
pub use worker::{
|
||||||
LocalWorkingDirectory, SubjektivSessionAttributionLifecycle, Worker, WorkerError,
|
LocalWorkingDirectory, ReviewerChildWorkspaceClient, ReviewerContext,
|
||||||
WorkerFilesystemAuthority, WorkerRunResult, WorkerWorkspaceContext, WorkspaceBinaryRequest,
|
SubjektivSessionAttributionLifecycle, Worker, WorkerError, WorkerFilesystemAuthority,
|
||||||
|
WorkerRunResult, WorkerStartupFeatureCleanup, WorkerWorkspaceContext, WorkspaceBinaryRequest,
|
||||||
WorkspaceBinaryResponse, WorkspaceClient, WorkspaceClientError, WorkspaceId, WorkspaceIdError,
|
WorkspaceBinaryResponse, WorkspaceClient, WorkspaceClientError, WorkspaceId, WorkspaceIdError,
|
||||||
WorkspacePromptCatalogResolution, WorkspaceRequest, WorkspaceRequestMethod, WorkspaceResponse,
|
WorkspacePromptCatalogResolution, WorkspaceRequest, WorkspaceRequestMethod, WorkspaceResponse,
|
||||||
WorkspaceServerOperation, WorkspaceWorkerDiscoveryRequest, apply_worker_manifest,
|
WorkspaceServerOperation, WorkspaceWorkerDiscoveryRequest, apply_worker_manifest,
|
||||||
|
|||||||
@@ -834,7 +834,10 @@ mod tests {
|
|||||||
assert!(common.contains("target-only movement preserves approval"));
|
assert!(common.contains("target-only movement preserves approval"));
|
||||||
assert!(reviewer.contains("complete linked MR/result snapshot"));
|
assert!(reviewer.contains("complete linked MR/result snapshot"));
|
||||||
assert!(reviewer.contains("Submit `ReviewMergeRequest` once"));
|
assert!(reviewer.contains("Submit `ReviewMergeRequest` once"));
|
||||||
assert!(reviewer.contains("without modifying implementation files"));
|
assert!(reviewer.contains("Bash"));
|
||||||
|
assert!(reviewer.contains("Git"));
|
||||||
|
assert!(!reviewer.contains("without modifying"));
|
||||||
|
assert!(!reviewer.contains("read-only"));
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
|
|||||||
@@ -21,8 +21,9 @@ pub use error::ScopeLockError;
|
|||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
pub(crate) use lifecycle::install_top_level_at_for_test;
|
pub(crate) use lifecycle::install_top_level_at_for_test;
|
||||||
pub use lifecycle::{
|
pub use lifecycle::{
|
||||||
ScopeAllocationGuard, SegmentActivationCommitError, SegmentActivationGuard, SegmentLockInfo,
|
ScopeAllocationCleanup, ScopeAllocationGuard, SegmentActivationCommitError,
|
||||||
adopt_allocation, install_top_level, install_top_level_with_deny, lookup_segment,
|
SegmentActivationGuard, SegmentLockInfo, adopt_allocation, install_top_level,
|
||||||
|
install_top_level_with_deny, lookup_segment,
|
||||||
};
|
};
|
||||||
pub use mutate::{
|
pub use mutate::{
|
||||||
delegate_scope, reclaim_delegated_scope, reclaim_stale, reclaim_stale_with, register_worker,
|
delegate_scope, reclaim_delegated_scope, reclaim_stale, reclaim_stale_with, register_worker,
|
||||||
|
|||||||
@@ -11,6 +11,14 @@ use session_store::SegmentId;
|
|||||||
pub enum ScopeLockError {
|
pub enum ScopeLockError {
|
||||||
#[error("I/O error on workers.json: {0}")]
|
#[error("I/O error on workers.json: {0}")]
|
||||||
Io(#[from] io::Error),
|
Io(#[from] io::Error),
|
||||||
|
#[error("worker `{worker_name}` allocation still has a live guard")]
|
||||||
|
AllocationGuardAttached { worker_name: String },
|
||||||
|
#[error("allocation acquisition was published but its save failed: {source}")]
|
||||||
|
AcquisitionSaveFailed {
|
||||||
|
#[source]
|
||||||
|
source: io::Error,
|
||||||
|
cleanup: super::lifecycle::ScopeAllocationCleanup,
|
||||||
|
},
|
||||||
#[error("worker `{0}` is already allocated")]
|
#[error("worker `{0}` is already allocated")]
|
||||||
DuplicateWorkerName(String),
|
DuplicateWorkerName(String),
|
||||||
#[error("requested scope `{}` conflicts with worker allocation `{competitor}` rule `{}`", .rule.target.display(), .competitor_rule.target.display())]
|
#[error("requested scope `{}` conflicts with worker allocation `{competitor}` rule `{}`", .rule.target.display(), .competitor_rule.target.display())]
|
||||||
@@ -46,3 +54,15 @@ pub enum ScopeLockError {
|
|||||||
actual: Option<SegmentId>,
|
actual: Option<SegmentId>,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl ScopeLockError {
|
||||||
|
/// Retain the release authority for an owned acquisition whose table rename
|
||||||
|
/// succeeded before save returned an error. Refusals and unpublished saves
|
||||||
|
/// do not confer ownership of any preexisting allocation.
|
||||||
|
pub fn allocation_cleanup_handle(&self) -> Option<super::lifecycle::ScopeAllocationCleanup> {
|
||||||
|
match self {
|
||||||
|
Self::AcquisitionSaveFailed { cleanup, .. } => Some(cleanup.clone()),
|
||||||
|
_ => None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -3,6 +3,7 @@
|
|||||||
//! up on drop.
|
//! up on drop.
|
||||||
|
|
||||||
use std::path::{Path, PathBuf};
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::sync::{Arc, Mutex};
|
||||||
|
|
||||||
use manifest::ScopeRule;
|
use manifest::ScopeRule;
|
||||||
use session_store::SegmentId;
|
use session_store::SegmentId;
|
||||||
@@ -11,13 +12,82 @@ use super::error::ScopeLockError;
|
|||||||
use super::mutate::release_worker;
|
use super::mutate::release_worker;
|
||||||
use super::table::{LockFileGuard, default_allocation_path};
|
use super::table::{LockFileGuard, default_allocation_path};
|
||||||
|
|
||||||
/// Owned allocation: on drop, opens the lock file and releases this
|
/// Owned allocation shared with any retained cleanup handles. Only the final
|
||||||
/// Worker's entry. The guard keeps only the name + lock-file path; it
|
/// owner's drop attempts best-effort release; no `flock` is held for its lifetime.
|
||||||
/// does not hold the `flock` for the Worker's lifetime.
|
|
||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub struct ScopeAllocationGuard {
|
pub struct ScopeAllocationGuard {
|
||||||
|
owner: Arc<ScopeAllocationOwner>,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Retains an allocation's release authority even after its Worker is consumed
|
||||||
|
/// or dropped. A successful controller transfer should drop this handle without
|
||||||
|
/// explicitly releasing the live Worker's allocation.
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct ScopeAllocationCleanup {
|
||||||
|
owner: Arc<ScopeAllocationOwner>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
struct ScopeAllocationOwner {
|
||||||
worker_name: String,
|
worker_name: String,
|
||||||
lock_path: PathBuf,
|
lock_path: PathBuf,
|
||||||
|
state: Mutex<ScopeAllocationOwnerState>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Debug)]
|
||||||
|
struct ScopeAllocationOwnerState {
|
||||||
|
guard_attached: bool,
|
||||||
|
released: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ScopeAllocationOwner {
|
||||||
|
fn release(&self) -> Result<(), ScopeLockError> {
|
||||||
|
let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner());
|
||||||
|
if state.guard_attached {
|
||||||
|
return Err(ScopeLockError::AllocationGuardAttached {
|
||||||
|
worker_name: self.worker_name.clone(),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if state.released {
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
|
let mut guard = LockFileGuard::open(&self.lock_path)?;
|
||||||
|
if guard.data().find(&self.worker_name).is_some() {
|
||||||
|
release_worker(&mut guard, &self.worker_name)?;
|
||||||
|
} else {
|
||||||
|
// A previous save may have removed the entry but failed after rename.
|
||||||
|
// Complete the durable save before reporting release as confirmed.
|
||||||
|
guard.save()?;
|
||||||
|
}
|
||||||
|
state.released = true;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for ScopeAllocationOwner {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
let _ = self.release();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Drop for ScopeAllocationGuard {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
self.owner
|
||||||
|
.state
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|error| error.into_inner())
|
||||||
|
.guard_attached = false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
impl ScopeAllocationCleanup {
|
||||||
|
/// Release only after the factory has returned an error and no live
|
||||||
|
/// Controller can still use this allocation. An attached guard rejects
|
||||||
|
/// release. Success is idempotent across all clones; failure retains the
|
||||||
|
/// authority for an explicit retry.
|
||||||
|
pub fn release(&self) -> Result<(), ScopeLockError> {
|
||||||
|
self.owner.release()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Exclusive authority to activate a replacement Segment for one allocated Worker.
|
/// Exclusive authority to activate a replacement Segment for one allocated Worker.
|
||||||
@@ -56,12 +126,58 @@ impl SegmentActivationCommitError {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl ScopeAllocationGuard {
|
impl ScopeAllocationGuard {
|
||||||
|
fn new(worker_name: String, lock_path: PathBuf) -> Self {
|
||||||
|
Self {
|
||||||
|
owner: Arc::new(ScopeAllocationOwner {
|
||||||
|
worker_name,
|
||||||
|
lock_path,
|
||||||
|
state: Mutex::new(ScopeAllocationOwnerState {
|
||||||
|
guard_attached: true,
|
||||||
|
released: false,
|
||||||
|
}),
|
||||||
|
}),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn commit_acquisition(self, table: &mut LockFileGuard) -> Result<Self, ScopeLockError> {
|
||||||
|
let mut published = false;
|
||||||
|
match table.save_with_publication(&mut published) {
|
||||||
|
Ok(()) => Ok(self),
|
||||||
|
Err(source) => {
|
||||||
|
let cleanup = if published {
|
||||||
|
Some(self.cleanup_handle())
|
||||||
|
} else {
|
||||||
|
// A staged entry is not ours until rename. Disarm before
|
||||||
|
// detaching so Drop cannot remove a preexisting allocation
|
||||||
|
// (or reopen the table lock still held by this caller).
|
||||||
|
self.owner
|
||||||
|
.state
|
||||||
|
.lock()
|
||||||
|
.unwrap_or_else(|error| error.into_inner())
|
||||||
|
.released = true;
|
||||||
|
None
|
||||||
|
};
|
||||||
|
drop(self);
|
||||||
|
match cleanup {
|
||||||
|
Some(cleanup) => Err(ScopeLockError::AcquisitionSaveFailed { source, cleanup }),
|
||||||
|
None => Err(ScopeLockError::Io(source)),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
pub fn worker_name(&self) -> &str {
|
pub fn worker_name(&self) -> &str {
|
||||||
&self.worker_name
|
&self.owner.worker_name
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn lock_path(&self) -> &Path {
|
pub fn lock_path(&self) -> &Path {
|
||||||
&self.lock_path
|
&self.owner.lock_path
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn cleanup_handle(&self) -> ScopeAllocationCleanup {
|
||||||
|
ScopeAllocationCleanup {
|
||||||
|
owner: Arc::clone(&self.owner),
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Lock the machine-wide table and validate a Segment replacement before
|
/// Lock the machine-wide table and validate a Segment replacement before
|
||||||
@@ -71,24 +187,25 @@ impl ScopeAllocationGuard {
|
|||||||
expected: SegmentId,
|
expected: SegmentId,
|
||||||
replacement: SegmentId,
|
replacement: SegmentId,
|
||||||
) -> Result<SegmentActivationGuard, ScopeLockError> {
|
) -> Result<SegmentActivationGuard, ScopeLockError> {
|
||||||
let guard = LockFileGuard::open(&self.lock_path)?;
|
let guard = LockFileGuard::open(self.lock_path())?;
|
||||||
let actual = guard
|
let actual = guard
|
||||||
.data()
|
.data()
|
||||||
.allocations
|
.allocations
|
||||||
.iter()
|
.iter()
|
||||||
.find(|allocation| allocation.worker_name == self.worker_name)
|
.find(|allocation| allocation.worker_name == self.worker_name())
|
||||||
.and_then(|allocation| allocation.segment_id);
|
.and_then(|allocation| allocation.segment_id);
|
||||||
|
|
||||||
if actual != Some(expected) {
|
if actual != Some(expected) {
|
||||||
return Err(ScopeLockError::SegmentChanged {
|
return Err(ScopeLockError::SegmentChanged {
|
||||||
worker_name: self.worker_name.clone(),
|
worker_name: self.worker_name().to_owned(),
|
||||||
expected,
|
expected,
|
||||||
actual,
|
actual,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
if let Some(existing) = guard.data().allocations.iter().find(|allocation| {
|
if let Some(existing) = guard.data().allocations.iter().find(|allocation| {
|
||||||
allocation.segment_id == Some(replacement) && allocation.worker_name != self.worker_name
|
allocation.segment_id == Some(replacement)
|
||||||
|
&& allocation.worker_name != self.worker_name()
|
||||||
}) {
|
}) {
|
||||||
return Err(ScopeLockError::SegmentConflict {
|
return Err(ScopeLockError::SegmentConflict {
|
||||||
segment_id: replacement,
|
segment_id: replacement,
|
||||||
@@ -99,7 +216,7 @@ impl ScopeAllocationGuard {
|
|||||||
|
|
||||||
Ok(SegmentActivationGuard {
|
Ok(SegmentActivationGuard {
|
||||||
guard,
|
guard,
|
||||||
worker_name: self.worker_name.clone(),
|
worker_name: self.worker_name().to_owned(),
|
||||||
expected,
|
expected,
|
||||||
replacement,
|
replacement,
|
||||||
})
|
})
|
||||||
@@ -145,14 +262,6 @@ impl SegmentActivationGuard {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl Drop for ScopeAllocationGuard {
|
|
||||||
fn drop(&mut self) {
|
|
||||||
if let Ok(mut guard) = LockFileGuard::open(&self.lock_path) {
|
|
||||||
let _ = release_worker(&mut guard, &self.worker_name);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/// Open the default lock file, register a top-level Worker, and return a
|
/// Open the default lock file, register a top-level Worker, and return a
|
||||||
/// guard that will release the allocation on drop.
|
/// guard that will release the allocation on drop.
|
||||||
pub fn install_top_level(
|
pub fn install_top_level(
|
||||||
@@ -185,7 +294,7 @@ pub fn install_top_level_with_deny(
|
|||||||
) -> Result<ScopeAllocationGuard, ScopeLockError> {
|
) -> Result<ScopeAllocationGuard, ScopeLockError> {
|
||||||
let lock_path = default_allocation_path()?;
|
let lock_path = default_allocation_path()?;
|
||||||
let mut guard = LockFileGuard::open(&lock_path)?;
|
let mut guard = LockFileGuard::open(&lock_path)?;
|
||||||
super::mutate::register_worker_with_deny(
|
super::mutate::prepare_worker_with_deny(
|
||||||
&mut guard,
|
&mut guard,
|
||||||
worker_name.clone(),
|
worker_name.clone(),
|
||||||
pid,
|
pid,
|
||||||
@@ -194,10 +303,7 @@ pub fn install_top_level_with_deny(
|
|||||||
scope_deny,
|
scope_deny,
|
||||||
segment_id,
|
segment_id,
|
||||||
)?;
|
)?;
|
||||||
Ok(ScopeAllocationGuard {
|
ScopeAllocationGuard::new(worker_name, lock_path).commit_acquisition(&mut guard)
|
||||||
worker_name,
|
|
||||||
lock_path,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
@@ -210,7 +316,7 @@ pub(crate) fn install_top_level_at_for_test(
|
|||||||
segment_id: SegmentId,
|
segment_id: SegmentId,
|
||||||
) -> Result<ScopeAllocationGuard, ScopeLockError> {
|
) -> Result<ScopeAllocationGuard, ScopeLockError> {
|
||||||
let mut guard = LockFileGuard::open(&lock_path)?;
|
let mut guard = LockFileGuard::open(&lock_path)?;
|
||||||
super::mutate::register_worker_with_deny(
|
super::mutate::prepare_worker_with_deny(
|
||||||
&mut guard,
|
&mut guard,
|
||||||
worker_name.clone(),
|
worker_name.clone(),
|
||||||
pid,
|
pid,
|
||||||
@@ -219,10 +325,7 @@ pub(crate) fn install_top_level_at_for_test(
|
|||||||
Vec::new(),
|
Vec::new(),
|
||||||
segment_id,
|
segment_id,
|
||||||
)?;
|
)?;
|
||||||
Ok(ScopeAllocationGuard {
|
ScopeAllocationGuard::new(worker_name, lock_path).commit_acquisition(&mut guard)
|
||||||
worker_name,
|
|
||||||
lock_path,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Take ownership of an existing allocation that was pre-registered by
|
/// Take ownership of an existing allocation that was pre-registered by
|
||||||
@@ -240,6 +343,15 @@ pub fn adopt_allocation(
|
|||||||
segment_id: SegmentId,
|
segment_id: SegmentId,
|
||||||
) -> Result<ScopeAllocationGuard, ScopeLockError> {
|
) -> Result<ScopeAllocationGuard, ScopeLockError> {
|
||||||
let lock_path = default_allocation_path()?;
|
let lock_path = default_allocation_path()?;
|
||||||
|
adopt_allocation_at(lock_path, worker_name, new_pid, segment_id)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn adopt_allocation_at(
|
||||||
|
lock_path: PathBuf,
|
||||||
|
worker_name: String,
|
||||||
|
new_pid: u32,
|
||||||
|
segment_id: SegmentId,
|
||||||
|
) -> Result<ScopeAllocationGuard, ScopeLockError> {
|
||||||
let mut guard = LockFileGuard::open(&lock_path)?;
|
let mut guard = LockFileGuard::open(&lock_path)?;
|
||||||
let alloc = guard
|
let alloc = guard
|
||||||
.data_mut()
|
.data_mut()
|
||||||
@@ -247,11 +359,7 @@ pub fn adopt_allocation(
|
|||||||
.ok_or_else(|| ScopeLockError::UnknownWorker(worker_name.clone()))?;
|
.ok_or_else(|| ScopeLockError::UnknownWorker(worker_name.clone()))?;
|
||||||
alloc.pid = new_pid;
|
alloc.pid = new_pid;
|
||||||
alloc.segment_id = Some(segment_id);
|
alloc.segment_id = Some(segment_id);
|
||||||
guard.save()?;
|
ScopeAllocationGuard::new(worker_name, lock_path).commit_acquisition(&mut guard)
|
||||||
Ok(ScopeAllocationGuard {
|
|
||||||
worker_name,
|
|
||||||
lock_path,
|
|
||||||
})
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Information about a Worker that currently holds an allocation for a
|
/// Information about a Worker that currently holds an allocation for a
|
||||||
@@ -329,6 +437,388 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn retained_cleanup_handle_defers_release_until_final_owner_drop() {
|
||||||
|
let dir = TempDir::new().unwrap();
|
||||||
|
let lock_path = dir.path().join("workers.json");
|
||||||
|
let guard = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"retained".into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("retained"),
|
||||||
|
Vec::new(),
|
||||||
|
sid(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let cleanup = guard.cleanup_handle();
|
||||||
|
let retained = cleanup.clone();
|
||||||
|
|
||||||
|
drop(guard);
|
||||||
|
drop(cleanup);
|
||||||
|
{
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(table.data().find("retained").is_some());
|
||||||
|
}
|
||||||
|
drop(retained);
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(table.data().find("retained").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn dropping_cleanup_handle_after_transfer_preserves_live_guard() {
|
||||||
|
let dir = TempDir::new().unwrap();
|
||||||
|
let lock_path = dir.path().join("workers.json");
|
||||||
|
let segment = sid();
|
||||||
|
let guard = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"transferred".into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("transferred"),
|
||||||
|
Vec::new(),
|
||||||
|
segment,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
|
||||||
|
drop(guard.cleanup_handle());
|
||||||
|
// A successful controller transfer leaves the guard usable, including
|
||||||
|
// the existing activation protocol, without retaining a cleanup handle.
|
||||||
|
let replacement = sid();
|
||||||
|
guard
|
||||||
|
.begin_segment_activation(segment, replacement)
|
||||||
|
.unwrap()
|
||||||
|
.commit()
|
||||||
|
.unwrap();
|
||||||
|
{
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
table.data().find("transferred").unwrap().segment_id,
|
||||||
|
Some(replacement)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
drop(guard);
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(table.data().find("transferred").is_none());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn cleanup_rejects_live_guard_then_releases_only_detached_allocation() {
|
||||||
|
let dir = TempDir::new().unwrap();
|
||||||
|
let lock_path = dir.path().join("workers.json");
|
||||||
|
let guard = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"live".into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("live"),
|
||||||
|
Vec::new(),
|
||||||
|
sid(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let _competitor = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"competitor".into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("competitor"),
|
||||||
|
Vec::new(),
|
||||||
|
sid(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let cleanup = guard.cleanup_handle();
|
||||||
|
assert!(matches!(
|
||||||
|
cleanup.release(),
|
||||||
|
Err(ScopeLockError::AllocationGuardAttached { worker_name }) if worker_name == "live"
|
||||||
|
));
|
||||||
|
drop(cleanup.clone());
|
||||||
|
{
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(table.data().find("live").is_some());
|
||||||
|
assert!(table.data().find("competitor").is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
drop(guard);
|
||||||
|
cleanup.release().unwrap();
|
||||||
|
cleanup.clone().release().unwrap();
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(table.data().find("live").is_none());
|
||||||
|
assert!(table.data().find("competitor").is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn published_install_failure_retains_cleanup_across_error_drop_and_retry() {
|
||||||
|
use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test};
|
||||||
|
|
||||||
|
let dir = TempDir::new().unwrap();
|
||||||
|
let lock_path = dir.path().join("workers.json");
|
||||||
|
let _competitor = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"competitor".into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("competitor"),
|
||||||
|
Vec::new(),
|
||||||
|
sid(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let segment = sid();
|
||||||
|
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::AfterRename);
|
||||||
|
let error = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"failed".into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("failed"),
|
||||||
|
Vec::new(),
|
||||||
|
segment,
|
||||||
|
)
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(matches!(
|
||||||
|
&error,
|
||||||
|
ScopeLockError::AcquisitionSaveFailed { .. }
|
||||||
|
));
|
||||||
|
let cleanup = error.allocation_cleanup_handle().unwrap();
|
||||||
|
let retry = error.allocation_cleanup_handle().unwrap();
|
||||||
|
drop(error);
|
||||||
|
{
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
table.data().find("failed").unwrap().segment_id,
|
||||||
|
Some(segment)
|
||||||
|
);
|
||||||
|
assert!(table.data().find("competitor").is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::BeforeRename);
|
||||||
|
assert!(matches!(cleanup.release(), Err(ScopeLockError::Io(_))));
|
||||||
|
drop(cleanup);
|
||||||
|
{
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(table.data().find("failed").is_some());
|
||||||
|
}
|
||||||
|
retry.release().unwrap();
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(table.data().find("failed").is_none());
|
||||||
|
assert!(table.data().find("competitor").is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn refused_install_has_no_cleanup_authority_over_existing_allocation() {
|
||||||
|
let dir = TempDir::new().unwrap();
|
||||||
|
let lock_path = dir.path().join("workers.json");
|
||||||
|
let segment = sid();
|
||||||
|
let _competitor = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"competitor".into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("competitor"),
|
||||||
|
vec![write_rule("/shared", true)],
|
||||||
|
segment,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let cases = [
|
||||||
|
("competitor", sid(), Vec::new()),
|
||||||
|
("other", segment, Vec::new()),
|
||||||
|
("other", sid(), vec![write_rule("/shared", true)]),
|
||||||
|
];
|
||||||
|
for (name, requested_segment, scope) in cases {
|
||||||
|
let error = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
name.into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("refused"),
|
||||||
|
scope,
|
||||||
|
requested_segment,
|
||||||
|
)
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(matches!(
|
||||||
|
&error,
|
||||||
|
ScopeLockError::DuplicateWorkerName(_)
|
||||||
|
| ScopeLockError::SegmentConflict { .. }
|
||||||
|
| ScopeLockError::WriteConflict { .. }
|
||||||
|
));
|
||||||
|
assert!(error.allocation_cleanup_handle().is_none());
|
||||||
|
drop(error);
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert_eq!(table.data().allocations.len(), 1);
|
||||||
|
assert_eq!(
|
||||||
|
table.data().find("competitor").unwrap().socket,
|
||||||
|
sock("competitor")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
table.data().find("competitor").unwrap().segment_id,
|
||||||
|
Some(segment)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unpublished_install_failure_has_no_cleanup_authority() {
|
||||||
|
use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test};
|
||||||
|
|
||||||
|
let dir = TempDir::new().unwrap();
|
||||||
|
let lock_path = dir.path().join("workers.json");
|
||||||
|
let _competitor = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"competitor".into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("competitor"),
|
||||||
|
Vec::new(),
|
||||||
|
sid(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::BeforeRename);
|
||||||
|
let error = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"failed".into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("failed"),
|
||||||
|
Vec::new(),
|
||||||
|
sid(),
|
||||||
|
)
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(matches!(&error, ScopeLockError::Io(_)));
|
||||||
|
assert!(error.allocation_cleanup_handle().is_none());
|
||||||
|
drop(error);
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(table.data().find("failed").is_none());
|
||||||
|
assert!(table.data().find("competitor").is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn published_adoption_failure_retains_cleanup_for_retry() {
|
||||||
|
use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test};
|
||||||
|
|
||||||
|
let dir = TempDir::new().unwrap();
|
||||||
|
let lock_path = dir.path().join("workers.json");
|
||||||
|
{
|
||||||
|
let mut table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
delegate_placeholder(&mut table, "child", std::process::id());
|
||||||
|
delegate_placeholder(&mut table, "competitor", std::process::id());
|
||||||
|
}
|
||||||
|
let segment = sid();
|
||||||
|
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::AfterRename);
|
||||||
|
let error = adopt_allocation_at(
|
||||||
|
lock_path.clone(),
|
||||||
|
"child".into(),
|
||||||
|
std::process::id(),
|
||||||
|
segment,
|
||||||
|
)
|
||||||
|
.unwrap_err();
|
||||||
|
let cleanup = error.allocation_cleanup_handle().unwrap();
|
||||||
|
drop(error);
|
||||||
|
{
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
table.data().find("child").unwrap().segment_id,
|
||||||
|
Some(segment)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::AfterRename);
|
||||||
|
assert!(matches!(cleanup.release(), Err(ScopeLockError::Io(_))));
|
||||||
|
{
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(table.data().find("child").is_none());
|
||||||
|
assert!(table.data().find("competitor").is_some());
|
||||||
|
}
|
||||||
|
cleanup.release().unwrap();
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(table.data().find("child").is_none());
|
||||||
|
assert!(table.data().find("competitor").is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unpublished_adoption_failure_preserves_placeholder_without_cleanup_authority() {
|
||||||
|
use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test};
|
||||||
|
|
||||||
|
let dir = TempDir::new().unwrap();
|
||||||
|
let lock_path = dir.path().join("workers.json");
|
||||||
|
let placeholder_pid = std::process::id();
|
||||||
|
{
|
||||||
|
let mut table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
delegate_placeholder(&mut table, "child", placeholder_pid);
|
||||||
|
}
|
||||||
|
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::BeforeRename);
|
||||||
|
let error = adopt_allocation_at(
|
||||||
|
lock_path.clone(),
|
||||||
|
"child".into(),
|
||||||
|
placeholder_pid.wrapping_add(1),
|
||||||
|
sid(),
|
||||||
|
)
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(matches!(&error, ScopeLockError::Io(_)));
|
||||||
|
assert!(error.allocation_cleanup_handle().is_none());
|
||||||
|
drop(error);
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
let allocation = table.data().find("child").unwrap();
|
||||||
|
assert_eq!(allocation.pid, placeholder_pid);
|
||||||
|
assert_eq!(allocation.segment_id, None);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn failed_cleanup_release_retains_authority_for_retry() {
|
||||||
|
use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test};
|
||||||
|
|
||||||
|
for point in [SaveFaultPoint::BeforeRename, SaveFaultPoint::AfterRename] {
|
||||||
|
let dir = TempDir::new().unwrap();
|
||||||
|
let lock_path = dir.path().join("workers.json");
|
||||||
|
let guard = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"failed".into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("failed"),
|
||||||
|
vec![read_rule("/shared", true)],
|
||||||
|
sid(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let _other = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"other".into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("other"),
|
||||||
|
vec![read_rule("/shared", true)],
|
||||||
|
sid(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let cleanup = guard.cleanup_handle();
|
||||||
|
let retry = cleanup.clone();
|
||||||
|
drop(guard);
|
||||||
|
fail_next_save_at_for_test(&lock_path, point);
|
||||||
|
|
||||||
|
assert!(matches!(cleanup.release(), Err(ScopeLockError::Io(_))));
|
||||||
|
drop(cleanup);
|
||||||
|
{
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
table.data().find("failed").is_some(),
|
||||||
|
point == SaveFaultPoint::BeforeRename
|
||||||
|
);
|
||||||
|
assert!(table.data().find("other").is_some());
|
||||||
|
}
|
||||||
|
retry.release().unwrap();
|
||||||
|
{
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert!(table.data().find("failed").is_none());
|
||||||
|
assert!(table.data().find("other").is_some());
|
||||||
|
}
|
||||||
|
|
||||||
|
// Success disarms every clone and final Drop, even if the name is
|
||||||
|
// subsequently allocated to a different Worker.
|
||||||
|
let _replacement = install_top_level_at_for_test(
|
||||||
|
lock_path.clone(),
|
||||||
|
"failed".into(),
|
||||||
|
std::process::id(),
|
||||||
|
sock("replacement"),
|
||||||
|
vec![read_rule("/shared", true)],
|
||||||
|
sid(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
retry.clone().release().unwrap();
|
||||||
|
drop(retry);
|
||||||
|
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
table.data().find("failed").unwrap().socket,
|
||||||
|
sock("replacement")
|
||||||
|
);
|
||||||
|
assert!(table.data().find("other").is_some());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn adopt_allocation_rewrites_pid_and_releases_on_drop() {
|
fn adopt_allocation_rewrites_pid_and_releases_on_drop() {
|
||||||
let dir = TempDir::new().unwrap();
|
let dir = TempDir::new().unwrap();
|
||||||
@@ -358,6 +848,7 @@ mod tests {
|
|||||||
let dir = TempDir::new().unwrap();
|
let dir = TempDir::new().unwrap();
|
||||||
let _sandbox = RuntimeDirSandbox::new(dir.path());
|
let _sandbox = RuntimeDirSandbox::new(dir.path());
|
||||||
let err = adopt_allocation("ghost".into(), 42, sid()).unwrap_err();
|
let err = adopt_allocation("ghost".into(), 42, sid()).unwrap_err();
|
||||||
|
assert!(err.allocation_cleanup_handle().is_none());
|
||||||
assert!(matches!(err, ScopeLockError::UnknownWorker(ref n) if n == "ghost"));
|
assert!(matches!(err, ScopeLockError::UnknownWorker(ref n) if n == "ghost"));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -57,6 +57,30 @@ pub fn register_worker_with_deny(
|
|||||||
scope_allow: Vec<ScopeRule>,
|
scope_allow: Vec<ScopeRule>,
|
||||||
scope_deny: Vec<ScopeRule>,
|
scope_deny: Vec<ScopeRule>,
|
||||||
segment_id: SegmentId,
|
segment_id: SegmentId,
|
||||||
|
) -> Result<(), ScopeLockError> {
|
||||||
|
prepare_worker_with_deny(
|
||||||
|
guard,
|
||||||
|
worker_name,
|
||||||
|
pid,
|
||||||
|
socket,
|
||||||
|
scope_allow,
|
||||||
|
scope_deny,
|
||||||
|
segment_id,
|
||||||
|
)?;
|
||||||
|
guard.save()?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Admit and stage a registration without publishing it. Owned installation
|
||||||
|
/// constructs its cleanup authority after admission and before committing.
|
||||||
|
pub(super) fn prepare_worker_with_deny(
|
||||||
|
guard: &mut LockFileGuard,
|
||||||
|
worker_name: String,
|
||||||
|
pid: u32,
|
||||||
|
socket: PathBuf,
|
||||||
|
scope_allow: Vec<ScopeRule>,
|
||||||
|
scope_deny: Vec<ScopeRule>,
|
||||||
|
segment_id: SegmentId,
|
||||||
) -> Result<(), ScopeLockError> {
|
) -> Result<(), ScopeLockError> {
|
||||||
reclaim_stale(guard);
|
reclaim_stale(guard);
|
||||||
if guard.data().find(&worker_name).is_some() {
|
if guard.data().find(&worker_name).is_some() {
|
||||||
@@ -101,7 +125,6 @@ pub fn register_worker_with_deny(
|
|||||||
delegated_from: None,
|
delegated_from: None,
|
||||||
segment_id: Some(segment_id),
|
segment_id: Some(segment_id),
|
||||||
});
|
});
|
||||||
guard.save()?;
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -246,6 +246,14 @@ impl LockFileGuard {
|
|||||||
/// Persist with atomic replacement while the separate allocation lock stays
|
/// Persist with atomic replacement while the separate allocation lock stays
|
||||||
/// held. A crash exposes either the old complete table or the new one.
|
/// held. A crash exposes either the old complete table or the new one.
|
||||||
pub fn save(&mut self) -> io::Result<()> {
|
pub fn save(&mut self) -> io::Result<()> {
|
||||||
|
let mut published = false;
|
||||||
|
self.save_with_publication(&mut published)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Report this save's rename while the caller still holds the table lock.
|
||||||
|
/// In-memory mutation alone is not evidence that an acquisition took place.
|
||||||
|
pub(super) fn save_with_publication(&mut self, published: &mut bool) -> io::Result<()> {
|
||||||
|
*published = false;
|
||||||
let json = serde_json::to_vec_pretty(&self.data).map_err(io::Error::other)?;
|
let json = serde_json::to_vec_pretty(&self.data).map_err(io::Error::other)?;
|
||||||
let mut temp_os = self.data_path.as_os_str().to_owned();
|
let mut temp_os = self.data_path.as_os_str().to_owned();
|
||||||
temp_os.push(".tmp");
|
temp_os.push(".tmp");
|
||||||
@@ -265,6 +273,7 @@ impl LockFileGuard {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
fs::rename(&temp_path, &self.data_path)?;
|
fs::rename(&temp_path, &self.data_path)?;
|
||||||
|
*published = true;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
if take_save_fault(&self.data_path, SaveFaultPoint::AfterRename) {
|
if take_save_fault(&self.data_path, SaveFaultPoint::AfterRename) {
|
||||||
return Err(io::Error::other(
|
return Err(io::Error::other(
|
||||||
|
|||||||
@@ -340,6 +340,84 @@ pub struct SpawnedWorkerRegistryLoad {
|
|||||||
pub reclaimed_unreachable: bool,
|
pub reclaimed_unreachable: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Read-only restore preparation using the registry that will be published by the Host.
|
||||||
|
/// The Host must retain its lifecycle exclusion from preparation through application;
|
||||||
|
/// the metadata reread detects source drift, but is not an atomic store transaction.
|
||||||
|
/// Dropping an unapplied plan does not reclaim legacy child state.
|
||||||
|
pub(crate) struct PreparedSpawnedWorkerRegistryLoad<St> {
|
||||||
|
pub(crate) registry: Arc<SpawnedWorkerRegistry>,
|
||||||
|
/// Projected outcome, not evidence that reclamation has already happened.
|
||||||
|
pub(crate) reclaimed_unreachable: bool,
|
||||||
|
runtime_dir: Arc<RuntimeDir>,
|
||||||
|
store: St,
|
||||||
|
worker_name: String,
|
||||||
|
parent_scope: Option<SharedScope>,
|
||||||
|
persisted_children: Vec<WorkerSpawnedChild>,
|
||||||
|
valid_records: Vec<SpawnedWorkerRecord>,
|
||||||
|
legacy_projection_exists: bool,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl<St> PreparedSpawnedWorkerRegistryLoad<St>
|
||||||
|
where
|
||||||
|
St: WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||||
|
{
|
||||||
|
/// Apply only after all other fallible tool initialization, before socket publication.
|
||||||
|
/// Effects retain their original order: projection, canonical reclaim history, then
|
||||||
|
/// allocation/scope. An error may follow committed effects; no rollback is attempted.
|
||||||
|
pub(crate) async fn apply(&mut self) -> io::Result<SpawnedWorkerRegistryLoad> {
|
||||||
|
let metadata = self
|
||||||
|
.store
|
||||||
|
.read_by_name(&self.worker_name)
|
||||||
|
.map_err(store_error_to_io)?;
|
||||||
|
let reclaimed: Vec<_> = self
|
||||||
|
.persisted_children
|
||||||
|
.iter()
|
||||||
|
.map(reclaimed_child_from_metadata)
|
||||||
|
.collect();
|
||||||
|
// A prior apply may have committed the canonical reclaim and then failed
|
||||||
|
// in allocation/scope cleanup. Under the host's same lifecycle lock,
|
||||||
|
// confirm its exact retained facts rather than appending them again.
|
||||||
|
let already_reclaimed = !reclaimed.is_empty()
|
||||||
|
&& metadata.as_ref().is_some_and(|metadata| {
|
||||||
|
metadata.spawned_children.is_empty()
|
||||||
|
&& metadata.reclaimed_children.ends_with(&reclaimed)
|
||||||
|
});
|
||||||
|
if !already_reclaimed
|
||||||
|
&& metadata
|
||||||
|
.as_ref()
|
||||||
|
.map(|metadata| metadata.spawned_children.as_slice())
|
||||||
|
.unwrap_or_default()
|
||||||
|
!= self.persisted_children.as_slice()
|
||||||
|
{
|
||||||
|
return Err(io::Error::new(
|
||||||
|
io::ErrorKind::InvalidData,
|
||||||
|
"spawned-child metadata changed after registry preparation",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
|
||||||
|
// Runtime projection is migration input only; the normal Internal registry is never
|
||||||
|
// materialized into spawned_workers.json. Only existence was captured, not its contents.
|
||||||
|
if !self.persisted_children.is_empty() || self.legacy_projection_exists {
|
||||||
|
self.runtime_dir.write_spawned_workers(&[]).await?;
|
||||||
|
}
|
||||||
|
if !reclaimed.is_empty() && !already_reclaimed {
|
||||||
|
self.store
|
||||||
|
.reclaim_spawned_children(&self.worker_name, reclaimed)
|
||||||
|
.map_err(store_error_to_io)?;
|
||||||
|
}
|
||||||
|
for record in &self.valid_records {
|
||||||
|
reclaim_record(&self.worker_name, self.parent_scope.as_ref(), record)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
let reclaimed_unreachable = self.reclaimed_unreachable;
|
||||||
|
self.reclaimed_unreachable = false;
|
||||||
|
Ok(SpawnedWorkerRegistryLoad {
|
||||||
|
registry: self.registry.clone(),
|
||||||
|
reclaimed_unreachable,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
impl SpawnedWorkerRegistry {
|
impl SpawnedWorkerRegistry {
|
||||||
pub(crate) fn bash_output(&self) -> tools::bash_output::BashOutputView {
|
pub(crate) fn bash_output(&self) -> tools::bash_output::BashOutputView {
|
||||||
self.bash_output.lock().unwrap().clone()
|
self.bash_output.lock().unwrap().clone()
|
||||||
@@ -426,6 +504,23 @@ impl SpawnedWorkerRegistry {
|
|||||||
worker_name: String,
|
worker_name: String,
|
||||||
parent_scope: Option<SharedScope>,
|
parent_scope: Option<SharedScope>,
|
||||||
) -> io::Result<SpawnedWorkerRegistryLoad>
|
) -> io::Result<SpawnedWorkerRegistryLoad>
|
||||||
|
where
|
||||||
|
St: WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||||
|
{
|
||||||
|
Self::prepare_from_worker_state_with_reclaim(runtime_dir, store, worker_name, parent_scope)?
|
||||||
|
.apply()
|
||||||
|
.await
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Capture legacy child facts without starting children or changing metadata,
|
||||||
|
/// projection, allocation, or scope. Tool initialization must clone this registry,
|
||||||
|
/// not construct a replacement, and defer reclamation to the owned plan's `apply`.
|
||||||
|
pub(crate) fn prepare_from_worker_state_with_reclaim<St>(
|
||||||
|
runtime_dir: Arc<RuntimeDir>,
|
||||||
|
store: St,
|
||||||
|
worker_name: String,
|
||||||
|
parent_scope: Option<SharedScope>,
|
||||||
|
) -> io::Result<PreparedSpawnedWorkerRegistryLoad<St>>
|
||||||
where
|
where
|
||||||
St: WorkerMetadataStore + Clone + Send + Sync + 'static,
|
St: WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||||
{
|
{
|
||||||
@@ -442,54 +537,33 @@ impl SpawnedWorkerRegistry {
|
|||||||
Ok(record) => {
|
Ok(record) => {
|
||||||
warn!(
|
warn!(
|
||||||
worker = %record.worker_name,
|
worker = %record.worker_name,
|
||||||
"reclaiming legacy persisted process SubWorker during Internal session restore"
|
"captured legacy persisted process SubWorker for reclamation during Internal session restore"
|
||||||
);
|
);
|
||||||
valid_records.push(record);
|
valid_records.push(record);
|
||||||
}
|
}
|
||||||
Err(error) => warn!(
|
Err(error) => warn!(
|
||||||
error = %error,
|
error = %error,
|
||||||
worker = %child.worker_name,
|
worker = %child.worker_name,
|
||||||
"clearing corrupt legacy persisted process SubWorker record"
|
"captured corrupt legacy persisted process SubWorker record for clearing"
|
||||||
),
|
),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Runtime projection is migration input only; the normal Internal registry is never
|
|
||||||
// materialized into spawned_workers.json.
|
|
||||||
let legacy_projection_exists = runtime_dir.path().join("spawned_workers.json").exists();
|
let legacy_projection_exists = runtime_dir.path().join("spawned_workers.json").exists();
|
||||||
if !persisted_children.is_empty() || legacy_projection_exists {
|
let registry = match &parent_scope {
|
||||||
runtime_dir.write_spawned_workers(&[]).await?;
|
Some(scope) => Self::new_internal(worker_name.clone(), scope.clone()),
|
||||||
}
|
None => Self::new(runtime_dir.clone()),
|
||||||
if !persisted_children.is_empty() {
|
};
|
||||||
let reclaimed = persisted_children
|
Ok(PreparedSpawnedWorkerRegistryLoad {
|
||||||
.iter()
|
registry,
|
||||||
.map(reclaimed_child_from_metadata)
|
|
||||||
.collect();
|
|
||||||
store
|
|
||||||
.reclaim_spawned_children(&worker_name, reclaimed)
|
|
||||||
.map_err(store_error_to_io)?;
|
|
||||||
}
|
|
||||||
for record in &valid_records {
|
|
||||||
reclaim_record(&worker_name, parent_scope.as_ref(), record)?;
|
|
||||||
}
|
|
||||||
|
|
||||||
Ok(SpawnedWorkerRegistryLoad {
|
|
||||||
registry: Arc::new(Self {
|
|
||||||
internal_records: std::sync::Mutex::new(Vec::new()),
|
|
||||||
service_records: std::sync::Mutex::new(Vec::new()),
|
|
||||||
#[cfg(test)]
|
|
||||||
fail_service_stops: std::sync::Mutex::new(HashMap::new()),
|
|
||||||
internal_names: std::sync::Mutex::new(HashSet::new()),
|
|
||||||
internal_shutting_down: AtomicBool::new(false),
|
|
||||||
pending_internal_spawns: AtomicUsize::new(0),
|
|
||||||
pending_internal_notify: Notify::new(),
|
|
||||||
internal_spawn_cleanup_failed: AtomicBool::new(false),
|
|
||||||
parent_scope,
|
|
||||||
bash_output: Mutex::new(tools::bash_output::BashOutputView::new()),
|
|
||||||
parent_protocol: Mutex::new(None),
|
|
||||||
protocol_publish_lock: Arc::new(Mutex::new(())),
|
|
||||||
}),
|
|
||||||
reclaimed_unreachable: !persisted_children.is_empty(),
|
reclaimed_unreachable: !persisted_children.is_empty(),
|
||||||
|
runtime_dir,
|
||||||
|
store,
|
||||||
|
worker_name,
|
||||||
|
parent_scope,
|
||||||
|
persisted_children,
|
||||||
|
valid_records,
|
||||||
|
legacy_projection_exists,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1140,6 +1214,10 @@ impl Drop for SpawnedWorkerRegistry {
|
|||||||
.filter(|record| !record.scope_reclaimed.load(Ordering::Acquire))
|
.filter(|record| !record.scope_reclaimed.load(Ordering::Acquire))
|
||||||
.flat_map(delegated_write_rules)
|
.flat_map(delegated_write_rules)
|
||||||
.collect::<Vec<_>>();
|
.collect::<Vec<_>>();
|
||||||
|
// Empty prepared registries own no delegations, including when abandoned or rejected.
|
||||||
|
if write_rules.is_empty() {
|
||||||
|
return;
|
||||||
|
}
|
||||||
let _ = parent_scope.update(|current| current.with_removed_deny_rules(write_rules));
|
let _ = parent_scope.update(|current| current.with_removed_deny_rules(write_rules));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1165,19 +1243,18 @@ fn reclaim_record(
|
|||||||
parent_scope: Option<&SharedScope>,
|
parent_scope: Option<&SharedScope>,
|
||||||
record: &SpawnedWorkerRecord,
|
record: &SpawnedWorkerRecord,
|
||||||
) -> io::Result<()> {
|
) -> io::Result<()> {
|
||||||
if let Ok(path) = worker_allocation::default_allocation_path() {
|
let path = worker_allocation::default_allocation_path().map_err(io::Error::other)?;
|
||||||
if let Ok(mut guard) = worker_allocation::LockFileGuard::open(&path) {
|
let mut guard = worker_allocation::LockFileGuard::open(&path).map_err(io::Error::other)?;
|
||||||
match worker_allocation::reclaim_delegated_scope(
|
match worker_allocation::reclaim_delegated_scope(
|
||||||
&mut guard,
|
&mut guard,
|
||||||
parent_name,
|
parent_name,
|
||||||
&record.worker_name,
|
&record.worker_name,
|
||||||
&record.scope_delegated,
|
&record.scope_delegated,
|
||||||
) {
|
) {
|
||||||
Ok(()) | Err(worker_allocation::ScopeLockError::UnknownWorker(_)) => {}
|
Ok(()) | Err(worker_allocation::ScopeLockError::UnknownWorker(_)) => {}
|
||||||
Err(error) => return Err(io::Error::other(error)),
|
Err(error) => return Err(io::Error::other(error)),
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
drop(guard);
|
||||||
if let Some(parent_scope) = parent_scope {
|
if let Some(parent_scope) = parent_scope {
|
||||||
let write_rules = record
|
let write_rules = record
|
||||||
.scope_delegated
|
.scope_delegated
|
||||||
@@ -1240,10 +1317,18 @@ fn store_error_to_io(error: WorkerStoreError) -> io::Error {
|
|||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
|
//! Legacy restore cases connect the registry to canonical aggregate storage,
|
||||||
|
//! runtime projections, and temporary SharedScope rules. Successful apply uses
|
||||||
|
//! a corrupt legacy record to exercise durable clearing without consulting the
|
||||||
|
//! process-global legacy allocation file; valid-record scope reclaim is also
|
||||||
|
//! covered by the existing external restore test.
|
||||||
|
|
||||||
|
use std::path::PathBuf;
|
||||||
use std::time::Duration;
|
use std::time::Duration;
|
||||||
|
|
||||||
use manifest::{Scope, ScopeConfig};
|
use manifest::{Scope, ScopeConfig};
|
||||||
use session_store::LogEntry;
|
use session_store::{LogEntry, WorkerAggregateStore, WorkerMetadata, WorkerSpawnedScopeRule};
|
||||||
|
use tempfile::TempDir;
|
||||||
|
|
||||||
use super::*;
|
use super::*;
|
||||||
use crate::internal_worker::{
|
use crate::internal_worker::{
|
||||||
@@ -1251,6 +1336,249 @@ mod tests {
|
|||||||
test_internal_worker_session_with_greeting,
|
test_internal_worker_session_with_greeting,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
struct LegacyRestoreFixture {
|
||||||
|
runtime_dir: Arc<RuntimeDir>,
|
||||||
|
store: WorkerAggregateStore,
|
||||||
|
parent_scope: SharedScope,
|
||||||
|
scope_root: PathBuf,
|
||||||
|
metadata_path: PathBuf,
|
||||||
|
// Keep all storage and scope paths alive until the other fixture fields are dropped.
|
||||||
|
_root: TempDir,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl LegacyRestoreFixture {
|
||||||
|
async fn new(permission: &str) -> Self {
|
||||||
|
let root = TempDir::new().unwrap();
|
||||||
|
let scope_root = root.path().join("scope");
|
||||||
|
std::fs::create_dir(&scope_root).unwrap();
|
||||||
|
let aggregate = root.path().join("aggregate");
|
||||||
|
let store = WorkerAggregateStore::new(&aggregate, "parent").unwrap();
|
||||||
|
let runtime_dir = Arc::new(
|
||||||
|
RuntimeDir::create(&root.path().join("runtime"), "parent")
|
||||||
|
.await
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
let mut metadata =
|
||||||
|
WorkerMetadata::new("parent", None).with_workspace_id("restore-test-workspace");
|
||||||
|
metadata.spawned_children.push(WorkerSpawnedChild {
|
||||||
|
worker_name: "legacy-child".into(),
|
||||||
|
socket_path: root.path().join("legacy.sock"),
|
||||||
|
callback_address: runtime_dir.path().join("sock"),
|
||||||
|
scope_delegated: vec![WorkerSpawnedScopeRule {
|
||||||
|
target: scope_root.clone(),
|
||||||
|
permission: permission.into(),
|
||||||
|
recursive: true,
|
||||||
|
symlink_policy: Default::default(),
|
||||||
|
}],
|
||||||
|
});
|
||||||
|
store.write(&metadata).unwrap();
|
||||||
|
let write_rule = ScopeRule {
|
||||||
|
target: scope_root.clone(),
|
||||||
|
permission: Permission::Write,
|
||||||
|
recursive: true,
|
||||||
|
symlink_policy: Default::default(),
|
||||||
|
};
|
||||||
|
let parent_scope = SharedScope::new(
|
||||||
|
Scope::from_config(&ScopeConfig {
|
||||||
|
allow: vec![write_rule.clone()],
|
||||||
|
deny: vec![write_rule],
|
||||||
|
})
|
||||||
|
.unwrap(),
|
||||||
|
);
|
||||||
|
Self {
|
||||||
|
runtime_dir,
|
||||||
|
store,
|
||||||
|
parent_scope,
|
||||||
|
scope_root,
|
||||||
|
metadata_path: aggregate.join("metadata.json"),
|
||||||
|
_root: root,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn prepare(&self) -> PreparedSpawnedWorkerRegistryLoad<WorkerAggregateStore> {
|
||||||
|
SpawnedWorkerRegistry::prepare_from_worker_state_with_reclaim(
|
||||||
|
self.runtime_dir.clone(),
|
||||||
|
self.store.clone(),
|
||||||
|
"parent".into(),
|
||||||
|
Some(self.parent_scope.clone()),
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn projection_path(&self) -> PathBuf {
|
||||||
|
self.runtime_dir.path().join("spawned_workers.json")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn registry_preparation_and_abandonment_leave_legacy_state_unchanged() {
|
||||||
|
for projection_exists in [false, true] {
|
||||||
|
let fixture = LegacyRestoreFixture::new("write").await;
|
||||||
|
if projection_exists {
|
||||||
|
// Projection contents are deliberately not decoded during preparation.
|
||||||
|
std::fs::write(fixture.projection_path(), b"obsolete projection").unwrap();
|
||||||
|
}
|
||||||
|
let metadata_before = std::fs::read(&fixture.metadata_path).unwrap();
|
||||||
|
let projection_before = std::fs::read(fixture.projection_path()).ok();
|
||||||
|
let scope_before = fixture.parent_scope.snapshot();
|
||||||
|
assert!(!scope_before.is_writable(&fixture.scope_root));
|
||||||
|
|
||||||
|
let prepared = fixture.prepare();
|
||||||
|
|
||||||
|
assert!(prepared.reclaimed_unreachable);
|
||||||
|
assert!(prepared.registry.list_internal().is_empty());
|
||||||
|
assert!(prepared.registry.internal_worker_snapshots().is_empty());
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read(&fixture.metadata_path).unwrap(),
|
||||||
|
metadata_before
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read(fixture.projection_path()).ok(),
|
||||||
|
projection_before
|
||||||
|
);
|
||||||
|
assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot()));
|
||||||
|
assert!(!fixture.runtime_dir.path().join("sock").exists());
|
||||||
|
drop(prepared);
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read(&fixture.metadata_path).unwrap(),
|
||||||
|
metadata_before
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read(fixture.projection_path()).ok(),
|
||||||
|
projection_before
|
||||||
|
);
|
||||||
|
assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot()));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn prepared_apply_clears_canonical_children_and_projection_on_same_registry() {
|
||||||
|
let fixture = LegacyRestoreFixture::new("unsupported-legacy-permission").await;
|
||||||
|
std::fs::write(fixture.projection_path(), b"obsolete projection").unwrap();
|
||||||
|
let scope_before = fixture.parent_scope.snapshot();
|
||||||
|
let mut prepared = fixture.prepare();
|
||||||
|
let shared_registry = prepared.registry.clone();
|
||||||
|
// Unrelated metadata may advance while tools are initialized.
|
||||||
|
fixture.store.add_peer("parent", "new-peer").unwrap();
|
||||||
|
let mut expected = fixture.store.read_by_name("parent").unwrap().unwrap();
|
||||||
|
expected.reclaimed_children.extend(
|
||||||
|
expected
|
||||||
|
.spawned_children
|
||||||
|
.iter()
|
||||||
|
.map(reclaimed_child_from_metadata),
|
||||||
|
);
|
||||||
|
expected.spawned_children.clear();
|
||||||
|
|
||||||
|
let loaded = prepared.apply().await.unwrap();
|
||||||
|
|
||||||
|
assert!(loaded.reclaimed_unreachable);
|
||||||
|
assert!(Arc::ptr_eq(&shared_registry, &loaded.registry));
|
||||||
|
assert!(shared_registry.list_internal().is_empty());
|
||||||
|
assert_eq!(
|
||||||
|
fixture.store.read_by_name("parent").unwrap(),
|
||||||
|
Some(expected)
|
||||||
|
);
|
||||||
|
let projection: serde_json::Value =
|
||||||
|
serde_json::from_slice(&std::fs::read(fixture.projection_path()).unwrap()).unwrap();
|
||||||
|
assert_eq!(projection, serde_json::json!([]));
|
||||||
|
// Corrupt records are audited and cleared, not interpreted as scope grants.
|
||||||
|
assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot()));
|
||||||
|
assert!(
|
||||||
|
!fixture
|
||||||
|
.parent_scope
|
||||||
|
.snapshot()
|
||||||
|
.is_writable(&fixture.scope_root)
|
||||||
|
);
|
||||||
|
assert!(!fixture.runtime_dir.path().join("sock").exists());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn prepared_apply_rejects_child_source_drift_before_any_effects() {
|
||||||
|
for drift in ["scope", "socket", "removal", "metadata-deletion"] {
|
||||||
|
let fixture = LegacyRestoreFixture::new("write").await;
|
||||||
|
std::fs::write(fixture.projection_path(), b"obsolete projection").unwrap();
|
||||||
|
let scope_before = fixture.parent_scope.snapshot();
|
||||||
|
let mut prepared = fixture.prepare();
|
||||||
|
if drift == "metadata-deletion" {
|
||||||
|
fixture.store.delete_by_name("parent").unwrap();
|
||||||
|
} else {
|
||||||
|
fixture
|
||||||
|
.store
|
||||||
|
.update_by_name("parent", |metadata| match drift {
|
||||||
|
"scope" => {
|
||||||
|
metadata.spawned_children[0].scope_delegated[0].recursive = false
|
||||||
|
}
|
||||||
|
"socket" => {
|
||||||
|
metadata.spawned_children[0].socket_path =
|
||||||
|
fixture.runtime_dir.path().join("replacement.sock");
|
||||||
|
}
|
||||||
|
"removal" => metadata.spawned_children.clear(),
|
||||||
|
_ => unreachable!(),
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
let metadata_before = std::fs::read(&fixture.metadata_path).ok();
|
||||||
|
let projection_before = std::fs::read(fixture.projection_path()).unwrap();
|
||||||
|
|
||||||
|
let error = prepared
|
||||||
|
.apply()
|
||||||
|
.await
|
||||||
|
.err()
|
||||||
|
.expect("source drift must fail");
|
||||||
|
|
||||||
|
assert_eq!(error.kind(), io::ErrorKind::InvalidData);
|
||||||
|
assert_eq!(std::fs::read(&fixture.metadata_path).ok(), metadata_before);
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read(fixture.projection_path()).unwrap(),
|
||||||
|
projection_before
|
||||||
|
);
|
||||||
|
assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot()));
|
||||||
|
assert!(!fixture.runtime_dir.path().join("sock").exists());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn prepared_apply_clears_projection_only_without_reporting_child_reclaim() {
|
||||||
|
let fixture = LegacyRestoreFixture::new("write").await;
|
||||||
|
fixture
|
||||||
|
.store
|
||||||
|
.set_spawned_children("parent", Vec::new())
|
||||||
|
.unwrap();
|
||||||
|
std::fs::write(fixture.projection_path(), b"obsolete projection").unwrap();
|
||||||
|
let metadata_before = std::fs::read(&fixture.metadata_path).unwrap();
|
||||||
|
let mut prepared = fixture.prepare();
|
||||||
|
assert!(!prepared.reclaimed_unreachable);
|
||||||
|
|
||||||
|
let loaded = prepared.apply().await.unwrap();
|
||||||
|
|
||||||
|
assert!(!loaded.reclaimed_unreachable);
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read(&fixture.metadata_path).unwrap(),
|
||||||
|
metadata_before
|
||||||
|
);
|
||||||
|
let projection: serde_json::Value =
|
||||||
|
serde_json::from_slice(&std::fs::read(fixture.projection_path()).unwrap()).unwrap();
|
||||||
|
assert_eq!(projection, serde_json::json!([]));
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn prepared_apply_projection_failure_keeps_canonical_children_and_scope() {
|
||||||
|
let fixture = LegacyRestoreFixture::new("write").await;
|
||||||
|
std::fs::create_dir(fixture.projection_path()).unwrap();
|
||||||
|
let metadata_before = std::fs::read(&fixture.metadata_path).unwrap();
|
||||||
|
let scope_before = fixture.parent_scope.snapshot();
|
||||||
|
let mut prepared = fixture.prepare();
|
||||||
|
|
||||||
|
assert!(prepared.apply().await.is_err());
|
||||||
|
|
||||||
|
assert_eq!(
|
||||||
|
std::fs::read(&fixture.metadata_path).unwrap(),
|
||||||
|
metadata_before
|
||||||
|
);
|
||||||
|
assert!(fixture.projection_path().is_dir());
|
||||||
|
assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot()));
|
||||||
|
}
|
||||||
|
|
||||||
fn registry() -> Arc<SpawnedWorkerRegistry> {
|
fn registry() -> Arc<SpawnedWorkerRegistry> {
|
||||||
let scope = Scope::from_config(&ScopeConfig {
|
let scope = Scope::from_config(&ScopeConfig {
|
||||||
allow: vec![ScopeRule {
|
allow: vec![ScopeRule {
|
||||||
|
|||||||
@@ -70,7 +70,8 @@ struct SubWorkerSpawnInput {
|
|||||||
/// Explicitly grant command execution through the parent-owned Workdir tool broker.
|
/// Explicitly grant command execution through the parent-owned Workdir tool broker.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
command: bool,
|
command: bool,
|
||||||
/// Binds an actual read-only builtin Reviewer child to the current Merge Request candidate.
|
/// Binds an actual builtin Reviewer child to the current Merge Request candidate.
|
||||||
|
/// Bash may be enabled for Git and other commands.
|
||||||
/// Review capability material is generated by the trusted spawn layer.
|
/// Review capability material is generated by the trusted spawn layer.
|
||||||
#[serde(default)]
|
#[serde(default)]
|
||||||
review: Option<ReviewerHandoffInput>,
|
review: Option<ReviewerHandoffInput>,
|
||||||
@@ -1176,6 +1177,17 @@ mod tests {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn reviewer_handoff_schema_allows_bash_for_git_commands() {
|
||||||
|
let schema = serde_json::to_value(schemars::schema_for!(SubWorkerSpawnInput)).unwrap();
|
||||||
|
let description = schema["properties"]["review"]["description"]
|
||||||
|
.as_str()
|
||||||
|
.expect("review handoff description");
|
||||||
|
assert!(description.contains("Bash"));
|
||||||
|
assert!(description.contains("Git"));
|
||||||
|
assert!(!description.contains("read-only"));
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn reviewer_handoff_requires_explicit_builtin_profile_and_writable_scope() {
|
fn reviewer_handoff_requires_explicit_builtin_profile_and_writable_scope() {
|
||||||
let valid: SubWorkerSpawnInput = serde_json::from_value(serde_json::json!({
|
let valid: SubWorkerSpawnInput = serde_json::from_value(serde_json::json!({
|
||||||
|
|||||||
+1928
-60
File diff suppressed because it is too large
Load Diff
@@ -1,3 +1,4 @@
|
|||||||
|
use std::path::{Path, PathBuf};
|
||||||
use std::pin::Pin;
|
use std::pin::Pin;
|
||||||
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
|
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
|
||||||
use std::sync::{Arc, Mutex};
|
use std::sync::{Arc, Mutex};
|
||||||
@@ -150,6 +151,19 @@ impl LlmClient for MockClient {
|
|||||||
// Helpers
|
// Helpers
|
||||||
// ---------------------------------------------------------------------------
|
// ---------------------------------------------------------------------------
|
||||||
|
|
||||||
|
#[path = "../../../tests/support/restore_environment.rs"]
|
||||||
|
mod restore_test_environment;
|
||||||
|
use restore_test_environment::require_restore_test_environment;
|
||||||
|
|
||||||
|
fn owned_relative_runtime_base() -> (tempfile::TempDir, PathBuf) {
|
||||||
|
require_restore_test_environment();
|
||||||
|
let temp = tempfile::tempdir_in(".").unwrap();
|
||||||
|
// The relative reference and the TempDir name the same owned tree, without
|
||||||
|
// putting the checkout prefix into sockaddr_un or changing process cwd.
|
||||||
|
let runtime_base = Path::new(temp.path().file_name().unwrap()).join("runtime");
|
||||||
|
(temp, runtime_base)
|
||||||
|
}
|
||||||
|
|
||||||
fn simple_text_events() -> Vec<LlmEvent> {
|
fn simple_text_events() -> Vec<LlmEvent> {
|
||||||
vec![
|
vec![
|
||||||
LlmEvent::text_block_start(0),
|
LlmEvent::text_block_start(0),
|
||||||
@@ -295,6 +309,7 @@ async fn make_worker_with_manifest_and_workspace_context(
|
|||||||
manifest: WorkerManifest,
|
manifest: WorkerManifest,
|
||||||
workspace_context: WorkerWorkspaceContext,
|
workspace_context: WorkerWorkspaceContext,
|
||||||
) -> (Worker<MockClient, TestStore>, std::path::PathBuf) {
|
) -> (Worker<MockClient, TestStore>, std::path::PathBuf) {
|
||||||
|
require_restore_test_environment();
|
||||||
let store_tmp = tempfile::tempdir().unwrap();
|
let store_tmp = tempfile::tempdir().unwrap();
|
||||||
let store = CombinedStore::new(
|
let store = CombinedStore::new(
|
||||||
FsStore::new(store_tmp.path()).unwrap(),
|
FsStore::new(store_tmp.path()).unwrap(),
|
||||||
@@ -320,18 +335,21 @@ async fn make_worker_with_manifest_and_workspace_context(
|
|||||||
}
|
}
|
||||||
|
|
||||||
async fn spawn_controller(worker: Worker<MockClient, TestStore>) -> WorkerHandle {
|
async fn spawn_controller(worker: Worker<MockClient, TestStore>) -> WorkerHandle {
|
||||||
let tmp = tempfile::tempdir().unwrap();
|
let (tmp, runtime_base) = owned_relative_runtime_base();
|
||||||
let runtime_base = tmp.path().to_owned();
|
let (handle, shutdown_rx) = WorkerController::spawn(worker, &runtime_base)
|
||||||
std::mem::forget(tmp);
|
|
||||||
let (handle, _shutdown_rx) = WorkerController::spawn(worker, &runtime_base)
|
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
// Retain ownership until controller shutdown (or test-runtime teardown).
|
||||||
|
tokio::spawn(async move {
|
||||||
|
let _ = shutdown_rx.await;
|
||||||
|
drop(tmp);
|
||||||
|
});
|
||||||
handle
|
handle
|
||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn canonical_worker_handle_uploads_and_submits_attachment() {
|
async fn canonical_worker_handle_uploads_and_submits_attachment() {
|
||||||
let root = tempfile::tempdir().unwrap();
|
let (root, runtime_base) = owned_relative_runtime_base();
|
||||||
let manifest = WorkerManifest::from_toml(MANIFEST_TOML).unwrap();
|
let manifest = WorkerManifest::from_toml(MANIFEST_TOML).unwrap();
|
||||||
let session = session_store::WorkerSessionStore::new(root.path().join("session")).unwrap();
|
let session = session_store::WorkerSessionStore::new(root.path().join("session")).unwrap();
|
||||||
let store = CombinedStore::new(
|
let store = CombinedStore::new(
|
||||||
@@ -353,7 +371,7 @@ async fn canonical_worker_handle_uploads_and_submits_attachment() {
|
|||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let (handle, shutdown_rx) = WorkerController::spawn(worker, &root.path().join("runtime"))
|
let (handle, shutdown_rx) = WorkerController::spawn(worker, &runtime_base)
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let mut rx = handle.subscribe();
|
let mut rx = handle.subscribe();
|
||||||
@@ -762,6 +780,227 @@ async fn controller_startup_failure_closes_bound_workdir_session() {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
mod registry_startup_failure {
|
||||||
|
//! Connect real controller preparation, registry source fences and session cleanup.
|
||||||
|
//! The Workdir telemetry boundary substitutes a concurrent metadata change at subscription;
|
||||||
|
//! the real registry must reject drift before starting its prepared observer.
|
||||||
|
use super::*;
|
||||||
|
use session_store::WorkerMetadataStore;
|
||||||
|
use tokio::sync::broadcast;
|
||||||
|
use workdir::WorkdirSession;
|
||||||
|
|
||||||
|
struct ObservedSession {
|
||||||
|
local: LocalWorkdirSession,
|
||||||
|
metadata_store: TestStore,
|
||||||
|
worker_name: String,
|
||||||
|
events: broadcast::Sender<workdir::CommandEvent>,
|
||||||
|
subscriptions: AtomicUsize,
|
||||||
|
receivers_at_close: Mutex<Vec<usize>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl std::fmt::Debug for ObservedSession {
|
||||||
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
|
f.debug_struct("ObservedSession")
|
||||||
|
.field("worker_name", &self.worker_name)
|
||||||
|
.finish_non_exhaustive()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl WorkdirSession for ObservedSession {
|
||||||
|
fn workdir(&self) -> &Workdir {
|
||||||
|
self.local.workdir()
|
||||||
|
}
|
||||||
|
|
||||||
|
fn capabilities(&self) -> WorkdirSessionCapabilities {
|
||||||
|
self.local.capabilities()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn stat(
|
||||||
|
&self,
|
||||||
|
request: workdir::StatRequest,
|
||||||
|
) -> Result<workdir::StatResult, WorkdirError> {
|
||||||
|
self.local.stat(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn read(
|
||||||
|
&self,
|
||||||
|
request: workdir::ReadRequest,
|
||||||
|
) -> Result<workdir::ReadResult, WorkdirError> {
|
||||||
|
self.local.read(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn write(
|
||||||
|
&self,
|
||||||
|
request: workdir::WriteRequest,
|
||||||
|
) -> Result<workdir::WriteResult, WorkdirError> {
|
||||||
|
self.local.write(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn edit(
|
||||||
|
&self,
|
||||||
|
request: workdir::EditRequest,
|
||||||
|
) -> Result<workdir::EditResult, WorkdirError> {
|
||||||
|
self.local.edit(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn list(
|
||||||
|
&self,
|
||||||
|
request: workdir::ListRequest,
|
||||||
|
) -> Result<workdir::ListResult, WorkdirError> {
|
||||||
|
self.local.list(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn glob(
|
||||||
|
&self,
|
||||||
|
request: workdir::GlobRequest,
|
||||||
|
) -> Result<workdir::GlobResult, WorkdirError> {
|
||||||
|
self.local.glob(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn grep(
|
||||||
|
&self,
|
||||||
|
request: workdir::GrepRequest,
|
||||||
|
) -> Result<workdir::GrepResult, WorkdirError> {
|
||||||
|
self.local.grep(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn start_command(
|
||||||
|
&self,
|
||||||
|
request: CommandRequest,
|
||||||
|
) -> Result<workdir::CommandHandle, WorkdirError> {
|
||||||
|
self.local.start_command(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn command_status(
|
||||||
|
&self,
|
||||||
|
handle: workdir::CommandHandle,
|
||||||
|
) -> Result<workdir::CommandStatus, WorkdirError> {
|
||||||
|
self.local.command_status(handle).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn command_output(
|
||||||
|
&self,
|
||||||
|
request: CommandOutputRequest,
|
||||||
|
) -> Result<workdir::CommandOutput, WorkdirError> {
|
||||||
|
self.local.command_output(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn cancel_command(&self, handle: workdir::CommandHandle) -> Result<(), WorkdirError> {
|
||||||
|
self.local.cancel_command(handle).await
|
||||||
|
}
|
||||||
|
|
||||||
|
fn subscribe_command_events(&self) -> Option<broadcast::Receiver<workdir::CommandEvent>> {
|
||||||
|
// Interleave a canonical change after read-only registry preparation.
|
||||||
|
let mut metadata = self
|
||||||
|
.metadata_store
|
||||||
|
.read_by_name(&self.worker_name)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap_or_else(|| session_store::WorkerMetadata::new(&self.worker_name, None));
|
||||||
|
metadata
|
||||||
|
.spawned_children
|
||||||
|
.push(session_store::WorkerSpawnedChild {
|
||||||
|
worker_name: "legacy-drift".into(),
|
||||||
|
socket_path: PathBuf::new(),
|
||||||
|
callback_address: PathBuf::new(),
|
||||||
|
scope_delegated: vec![],
|
||||||
|
});
|
||||||
|
self.metadata_store.write(&metadata).unwrap();
|
||||||
|
self.subscriptions.fetch_add(1, Ordering::SeqCst);
|
||||||
|
Some(self.events.subscribe())
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn close(&self) -> Result<(), WorkdirError> {
|
||||||
|
// Record before any await: aborting without joining must not pass by getting a
|
||||||
|
// scheduler turn during the provider's asynchronous close. The sender stays alive
|
||||||
|
// even after close, so channel closure cannot accidentally dispose of the observer.
|
||||||
|
self.receivers_at_close
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.push(self.events.receiver_count());
|
||||||
|
self.local.close().await
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "current_thread")]
|
||||||
|
async fn registry_drift_drops_prepared_observer_before_closing_session() {
|
||||||
|
let root = tempfile::tempdir().unwrap();
|
||||||
|
let run_dir = root.path().join("runs/1");
|
||||||
|
let client = MockClient::new(simple_text_events());
|
||||||
|
let manifest = WorkerManifest::from_toml(MANIFEST_TOML).unwrap();
|
||||||
|
let store = CombinedStore::new(
|
||||||
|
FsStore::new(root.path().join("sessions")).unwrap(),
|
||||||
|
FsWorkerStore::new(root.path().join("workers")).unwrap(),
|
||||||
|
);
|
||||||
|
let engine =
|
||||||
|
Engine::<_, agen::state::Mutable, worker::SessionHistoryMetadata>::new_annotated(
|
||||||
|
client.clone(),
|
||||||
|
);
|
||||||
|
let mut worker = Worker::new(
|
||||||
|
manifest,
|
||||||
|
engine,
|
||||||
|
store.clone(),
|
||||||
|
WorkerWorkspaceContext::local_filesystem(None),
|
||||||
|
WorkerFilesystemAuthority::local(root.path().into(), root.path().into()),
|
||||||
|
manifest::Scope::writable(root.path()).unwrap(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let (events, unused_receiver) = broadcast::channel(8);
|
||||||
|
drop(unused_receiver);
|
||||||
|
let session = Arc::new(ObservedSession {
|
||||||
|
local: LocalWorkdirSession::materialized_bound(
|
||||||
|
Workdir::new("registry-startup-failure"),
|
||||||
|
root.path().into(),
|
||||||
|
root.path().into(),
|
||||||
|
worker.scope().clone(),
|
||||||
|
WorkdirSessionCapabilities::ALL,
|
||||||
|
),
|
||||||
|
metadata_store: store,
|
||||||
|
worker_name: worker.manifest().worker.name.clone(),
|
||||||
|
events,
|
||||||
|
subscriptions: AtomicUsize::new(0),
|
||||||
|
receivers_at_close: Mutex::new(Vec::new()),
|
||||||
|
});
|
||||||
|
worker.bind_single_workdir_session(Some(session.clone()));
|
||||||
|
let sessions = worker.workdir_sessions();
|
||||||
|
|
||||||
|
let result = tokio::time::timeout(
|
||||||
|
std::time::Duration::from_secs(5),
|
||||||
|
WorkerController::spawn_runtime_managed_run(worker, &run_dir),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.expect("registry rejection and cleanup must settle");
|
||||||
|
let error = match result {
|
||||||
|
Err(error) => error,
|
||||||
|
Ok(_) => panic!("registry source drift must reject controller startup"),
|
||||||
|
};
|
||||||
|
assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
|
||||||
|
assert_eq!(session.subscriptions.load(Ordering::SeqCst), 1);
|
||||||
|
assert_eq!(*session.receivers_at_close.lock().unwrap(), vec![0]);
|
||||||
|
assert_eq!(session.events.receiver_count(), 0);
|
||||||
|
assert!(sessions.aliases().is_empty());
|
||||||
|
// No controller retains the Worker/Engine after the failed startup.
|
||||||
|
assert_eq!(Arc::strong_count(&client.call_count), 1);
|
||||||
|
assert_eq!(client.call_count.load(Ordering::SeqCst), 0);
|
||||||
|
assert!(!run_dir.join("worker.sock").exists());
|
||||||
|
assert!(!run_dir.join("status.json").exists());
|
||||||
|
assert!(!run_dir.join("manifest.toml").exists());
|
||||||
|
assert!(matches!(
|
||||||
|
session
|
||||||
|
.start_command(CommandRequest {
|
||||||
|
command: "printf unreachable".to_owned(),
|
||||||
|
timeout_secs: 5,
|
||||||
|
transport_byte_budget: workdir::MAX_COMMAND_TRANSPORT_BYTES,
|
||||||
|
cwd: workdir::WorkdirPath::root(),
|
||||||
|
tool_call_id: None,
|
||||||
|
})
|
||||||
|
.await,
|
||||||
|
Err(ref error) if matches!(error.classification_source(), WorkdirError::Unavailable(_))
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async fn wait_for_status(handle: &WorkerHandle, status: WorkerStatus) {
|
async fn wait_for_status(handle: &WorkerHandle, status: WorkerStatus) {
|
||||||
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
|
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
|
||||||
loop {
|
loop {
|
||||||
@@ -1561,11 +1800,12 @@ permission = "write"
|
|||||||
"#;
|
"#;
|
||||||
let client = MockClient::new(simple_text_events());
|
let client = MockClient::new(simple_text_events());
|
||||||
let worker = make_worker_with_pwd_and_manifest(client, manifest).await.0;
|
let worker = make_worker_with_pwd_and_manifest(client, manifest).await.0;
|
||||||
let tmp = tempfile::tempdir().unwrap();
|
let (_tmp, runtime_base) = owned_relative_runtime_base();
|
||||||
let result = WorkerController::spawn(worker, tmp.path()).await;
|
let result = WorkerController::spawn(worker, &runtime_base).await;
|
||||||
assert!(
|
assert!(
|
||||||
result.is_ok(),
|
result.is_ok(),
|
||||||
"feature exposure must not imply delegation authority"
|
"feature exposure must not imply delegation authority: {:?}",
|
||||||
|
result.as_ref().err()
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -10,6 +10,10 @@ autobins = false
|
|||||||
name = "yoi-server"
|
name = "yoi-server"
|
||||||
path = "src/main.rs"
|
path = "src/main.rs"
|
||||||
|
|
||||||
|
[features]
|
||||||
|
# Offline fixture preparation only; build product binaries without this feature.
|
||||||
|
test-support = []
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
api-macros = { workspace = true, features = ["axum"] }
|
api-macros = { workspace = true, features = ["axum"] }
|
||||||
async-trait.workspace = true
|
async-trait.workspace = true
|
||||||
|
|||||||
@@ -477,7 +477,8 @@ fn tracking_store_real_schema91_to92_retains_queue_cases_principals_audit_receip
|
|||||||
(100, "Provisional users and scoped passkey activation".into()),
|
(100, "Provisional users and scoped passkey activation".into()),
|
||||||
(101, "Durable explicit Subject body release".into()),
|
(101, "Durable explicit Subject body release".into()),
|
||||||
(102, "Durable failed Subject creation recovery".into()),
|
(102, "Durable failed Subject creation recovery".into()),
|
||||||
(103, "Job input retirement independent of immutable receipts".into())
|
(103, "Passkey metadata and atomic addition results".into()),
|
||||||
|
(104, "Converge Passkey results and Job input retention".into())
|
||||||
]
|
]
|
||||||
);
|
);
|
||||||
for (table, rows) in retained.iter().zip(original.iter()) {
|
for (table, rows) in retained.iter().zip(original.iter()) {
|
||||||
|
|||||||
@@ -228,13 +228,21 @@ fn resolve_whoami_sync(
|
|||||||
use server_api::{BrowserSessionState, WhoamiResponse};
|
use server_api::{BrowserSessionState, WhoamiResponse};
|
||||||
let anonymous = || WhoamiResponse {
|
let anonymous = || WhoamiResponse {
|
||||||
actor: None,
|
actor: None,
|
||||||
|
authentication_reference: None,
|
||||||
session: BrowserSessionState::Anonymous {},
|
session: BrowserSessionState::Anonymous {},
|
||||||
};
|
};
|
||||||
// Preserve the existing API-token precedence and fallback rules, including expired tokens.
|
// Preserve the existing API-token precedence and fallback rules, including expired tokens.
|
||||||
if let Some((actor, expires_at)) =
|
if let Some((actor, expires_at)) =
|
||||||
resolve_credentials_sync(store, bearer, session_token.clone())?
|
resolve_credentials_sync(store, bearer.clone(), session_token.clone())?
|
||||||
{
|
{
|
||||||
|
let credential = match actor.auth_method {
|
||||||
|
ActorAuthMethod::ApiToken => bearer.as_deref(),
|
||||||
|
ActorAuthMethod::BrowserSession => session_token.as_deref(),
|
||||||
|
}
|
||||||
|
.ok_or_else(|| auth_error("auth_required", "authentication binding missing"))?;
|
||||||
|
let authentication_reference = Some(authentication_reference(&token_hash(credential)));
|
||||||
return Ok(WhoamiResponse {
|
return Ok(WhoamiResponse {
|
||||||
|
authentication_reference,
|
||||||
actor: Some(actor),
|
actor: Some(actor),
|
||||||
session: BrowserSessionState::Authenticated { expires_at },
|
session: BrowserSessionState::Authenticated { expires_at },
|
||||||
});
|
});
|
||||||
@@ -255,6 +263,7 @@ fn resolve_whoami_sync(
|
|||||||
if within_session_expiry_grace(&session.expires_at, Utc::now()) {
|
if within_session_expiry_grace(&session.expires_at, Utc::now()) {
|
||||||
return Ok(WhoamiResponse {
|
return Ok(WhoamiResponse {
|
||||||
actor: None,
|
actor: None,
|
||||||
|
authentication_reference: None,
|
||||||
session: BrowserSessionState::Expired {},
|
session: BrowserSessionState::Expired {},
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -262,6 +271,13 @@ fn resolve_whoami_sync(
|
|||||||
Ok(anonymous())
|
Ok(anonymous())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Domain-separated public comparison value; never accepted as a credential.
|
||||||
|
pub(crate) fn authentication_reference(credential_hash: &str) -> String {
|
||||||
|
token_hash(&format!(
|
||||||
|
"passkey-authentication-reference:{credential_hash}"
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
fn within_session_expiry_grace(expires_at: &str, now: DateTime<Utc>) -> bool {
|
fn within_session_expiry_grace(expires_at: &str, now: DateTime<Utc>) -> bool {
|
||||||
DateTime::parse_from_rfc3339(expires_at).is_ok_and(|expires_at| {
|
DateTime::parse_from_rfc3339(expires_at).is_ok_and(|expires_at| {
|
||||||
expires_at <= now && now < expires_at + Duration::seconds(SESSION_EXPIRY_GRACE_SECONDS)
|
expires_at <= now && now < expires_at + Duration::seconds(SESSION_EXPIRY_GRACE_SECONDS)
|
||||||
|
|||||||
@@ -5113,6 +5113,9 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime {
|
|||||||
runtime_api::WorkerRestoreState::RolledBack => {
|
runtime_api::WorkerRestoreState::RolledBack => {
|
||||||
server_api::WorkerRestoreState::RolledBack
|
server_api::WorkerRestoreState::RolledBack
|
||||||
}
|
}
|
||||||
|
runtime_api::WorkerRestoreState::Superseded => {
|
||||||
|
server_api::WorkerRestoreState::Superseded
|
||||||
|
}
|
||||||
runtime_api::WorkerRestoreState::ReconciliationRequired => {
|
runtime_api::WorkerRestoreState::ReconciliationRequired => {
|
||||||
server_api::WorkerRestoreState::ReconciliationRequired
|
server_api::WorkerRestoreState::ReconciliationRequired
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -381,10 +381,13 @@ async fn implementation_result_v97_to_v98_preserves_legacy_bytes_receipts_and_no
|
|||||||
DROP TABLE ticket_implementation_results;
|
DROP TABLE ticket_implementation_results;
|
||||||
DROP TABLE ticket_implementation_delegations;
|
DROP TABLE ticket_implementation_delegations;
|
||||||
DROP TABLE merge_request_source_observations;
|
DROP TABLE merge_request_source_observations;
|
||||||
|
DROP TRIGGER passkey_metadata_insert;
|
||||||
|
DROP TABLE passkey_addition_receipts;
|
||||||
|
DROP TABLE passkey_metadata;
|
||||||
DROP TRIGGER backend_job_input_retirement;
|
DROP TRIGGER backend_job_input_retirement;
|
||||||
DROP TRIGGER backend_job_identity_immutable;
|
DROP TRIGGER backend_job_identity_immutable;
|
||||||
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
|
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
|
||||||
UPDATE __yoi_schema_migrations SET version=102 WHERE version=103;
|
UPDATE __yoi_schema_migrations SET version=102 WHERE version=104;
|
||||||
DROP TABLE subject_create_recoveries;
|
DROP TABLE subject_create_recoveries;
|
||||||
UPDATE __yoi_schema_migrations SET version=101 WHERE version=102;
|
UPDATE __yoi_schema_migrations SET version=101 WHERE version=102;
|
||||||
DROP TABLE subject_worker_releases;
|
DROP TABLE subject_worker_releases;
|
||||||
@@ -521,10 +524,13 @@ async fn mr_notification_upgrade_recovers_events_not_comments_and_retains_unknow
|
|||||||
// Reconstruct the prior composition before its new business capture existed.
|
// Reconstruct the prior composition before its new business capture existed.
|
||||||
conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture;
|
conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture;
|
||||||
DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations;
|
DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations;
|
||||||
|
DROP TRIGGER passkey_metadata_insert;
|
||||||
|
DROP TABLE passkey_addition_receipts;
|
||||||
|
DROP TABLE passkey_metadata;
|
||||||
DROP TRIGGER backend_job_input_retirement;
|
DROP TRIGGER backend_job_input_retirement;
|
||||||
DROP TRIGGER backend_job_identity_immutable;
|
DROP TRIGGER backend_job_identity_immutable;
|
||||||
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
|
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
|
||||||
UPDATE __yoi_schema_migrations SET version=102 WHERE version=103;
|
UPDATE __yoi_schema_migrations SET version=102 WHERE version=104;
|
||||||
DROP TABLE subject_create_recoveries;
|
DROP TABLE subject_create_recoveries;
|
||||||
UPDATE __yoi_schema_migrations SET version=101 WHERE version=102;
|
UPDATE __yoi_schema_migrations SET version=101 WHERE version=102;
|
||||||
DROP TABLE subject_worker_releases;
|
DROP TABLE subject_worker_releases;
|
||||||
@@ -687,10 +693,13 @@ async fn mr_event_actors_preserve_self_suppression_in_capture_and_backfill() {
|
|||||||
if historical {
|
if historical {
|
||||||
conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture;
|
conn.execute_batch("DROP TRIGGER notification_mr_event; DROP TRIGGER notification_mr_source; DROP TRIGGER notification_implementation_result; DROP TRIGGER IF EXISTS implementation_delegation_capture;
|
||||||
DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations;
|
DROP TABLE ticket_implementation_results; DROP TABLE ticket_implementation_delegations; DROP TABLE merge_request_source_observations;
|
||||||
DROP TRIGGER backend_job_input_retirement;
|
DROP TRIGGER passkey_metadata_insert;
|
||||||
|
DROP TABLE passkey_addition_receipts;
|
||||||
|
DROP TABLE passkey_metadata;
|
||||||
|
DROP TRIGGER backend_job_input_retirement;
|
||||||
DROP TRIGGER backend_job_identity_immutable;
|
DROP TRIGGER backend_job_identity_immutable;
|
||||||
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
|
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
|
||||||
UPDATE __yoi_schema_migrations SET version=102 WHERE version=103;
|
UPDATE __yoi_schema_migrations SET version=102 WHERE version=104;
|
||||||
DROP TABLE subject_create_recoveries;
|
DROP TABLE subject_create_recoveries;
|
||||||
UPDATE __yoi_schema_migrations SET version=101 WHERE version=102;
|
UPDATE __yoi_schema_migrations SET version=101 WHERE version=102;
|
||||||
DROP TABLE subject_worker_releases;
|
DROP TABLE subject_worker_releases;
|
||||||
|
|||||||
@@ -19,12 +19,17 @@ pub mod repositories;
|
|||||||
pub mod repository_access;
|
pub mod repository_access;
|
||||||
pub mod repository_source;
|
pub mod repository_source;
|
||||||
pub mod resource_broker;
|
pub mod resource_broker;
|
||||||
|
#[cfg(test)]
|
||||||
|
#[path = "../../../tests/support/restore_environment.rs"]
|
||||||
|
mod restore_test_environment;
|
||||||
pub mod retention;
|
pub mod retention;
|
||||||
pub mod runtime_settings;
|
pub mod runtime_settings;
|
||||||
pub mod runtime_subscription;
|
pub mod runtime_subscription;
|
||||||
pub mod server;
|
pub mod server;
|
||||||
pub mod skills;
|
pub mod skills;
|
||||||
pub mod store;
|
pub mod store;
|
||||||
|
#[cfg(feature = "test-support")]
|
||||||
|
pub use store::fixture_support as test_support;
|
||||||
pub mod subjektiv;
|
pub mod subjektiv;
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod ticket_browse_performance_tests;
|
mod ticket_browse_performance_tests;
|
||||||
@@ -129,6 +134,12 @@ pub enum Error {
|
|||||||
TicketAssignmentConflict(String),
|
TicketAssignmentConflict(String),
|
||||||
#[error("Workdir attachment conflict: {0}")]
|
#[error("Workdir attachment conflict: {0}")]
|
||||||
WorkdirAttachmentConflict(String),
|
WorkdirAttachmentConflict(String),
|
||||||
|
#[error("Workdir {workdir_id} is already attached to Worker {runtime_id}:{worker_id}")]
|
||||||
|
WorkdirOccupied {
|
||||||
|
workdir_id: String,
|
||||||
|
runtime_id: String,
|
||||||
|
worker_id: String,
|
||||||
|
},
|
||||||
#[error("Workspace permission denied: {0}")]
|
#[error("Workspace permission denied: {0}")]
|
||||||
WorkspacePermissionDenied(String),
|
WorkspacePermissionDenied(String),
|
||||||
#[error("Workspace config update conflict: {0}")]
|
#[error("Workspace config update conflict: {0}")]
|
||||||
|
|||||||
@@ -0,0 +1,18 @@
|
|||||||
|
CREATE TABLE passkey_metadata (
|
||||||
|
credential_id TEXT PRIMARY KEY REFERENCES passkey_credentials(credential_id) ON DELETE CASCADE,
|
||||||
|
key_reference TEXT NOT NULL UNIQUE,
|
||||||
|
normal_login_verified_at TEXT
|
||||||
|
);
|
||||||
|
INSERT INTO passkey_metadata(credential_id, key_reference)
|
||||||
|
SELECT credential_id, lower(hex(randomblob(16))) FROM passkey_credentials;
|
||||||
|
CREATE TRIGGER passkey_metadata_insert AFTER INSERT ON passkey_credentials
|
||||||
|
BEGIN
|
||||||
|
INSERT INTO passkey_metadata(credential_id, key_reference)
|
||||||
|
VALUES (NEW.credential_id, lower(hex(randomblob(16))));
|
||||||
|
END;
|
||||||
|
CREATE TABLE passkey_addition_receipts (
|
||||||
|
challenge_id TEXT PRIMARY KEY,
|
||||||
|
credential_id TEXT NOT NULL UNIQUE REFERENCES passkey_credentials(credential_id) ON DELETE CASCADE,
|
||||||
|
user_id TEXT NOT NULL REFERENCES users(user_id) ON DELETE CASCADE,
|
||||||
|
authentication_reference TEXT NOT NULL
|
||||||
|
);
|
||||||
@@ -325,7 +325,7 @@ impl SqliteWorkspaceStore {
|
|||||||
let now = Utc::now().to_rfc3339();
|
let now = Utc::now().to_rfc3339();
|
||||||
self.with_conn_mut(|conn| {
|
self.with_conn_mut(|conn| {
|
||||||
let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
let tx=conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||||
let policy=load_removal_policy(&tx,&req.workspace_id,&req.worker,erased_journal)?.ok_or_else(|| StoreError::InvalidInput(format!("policy-missing:{}",req.workspace_id)))?;
|
let policy=load_removal_policy(&tx,&req.workspace_id,&req.worker,erased_journal,true)?.ok_or_else(|| StoreError::InvalidInput(format!("policy-missing:{}",req.workspace_id)))?;
|
||||||
let worker=match load_worker(&tx,&req.workspace_id,&req.worker)? {
|
let worker=match load_worker(&tx,&req.workspace_id,&req.worker)? {
|
||||||
Some(v)=>v,
|
Some(v)=>v,
|
||||||
None=>{
|
None=>{
|
||||||
@@ -371,7 +371,7 @@ impl SqliteWorkspaceStore {
|
|||||||
return Err(StoreError::InvalidInput(format!("stale:{plan_id}:plan state {} is not executable", state_s(plan.state))));
|
return Err(StoreError::InvalidInput(format!("stale:{plan_id}:plan state {} is not executable", state_s(plan.state))));
|
||||||
}
|
}
|
||||||
if !plan.blockers.is_empty(){return Err(StoreError::InvalidInput(format!("blocked:{}",serde_json::to_string(&plan.blockers).unwrap())));}
|
if !plan.blockers.is_empty(){return Err(StoreError::InvalidInput(format!("blocked:{}",serde_json::to_string(&plan.blockers).unwrap())));}
|
||||||
let policy=load_removal_policy(&tx,workspace_id,&plan.worker,plan.policy_id==ERASED_JOURNAL_POLICY_ID)?.ok_or_else(||StoreError::InvalidInput(format!("policy-missing:{workspace_id}")))?;
|
let policy=load_removal_policy(&tx,workspace_id,&plan.worker,plan.policy_id==ERASED_JOURNAL_POLICY_ID,plan.policy_id=="backend-subject-removal-archive-v1")?.ok_or_else(||StoreError::InvalidInput(format!("policy-missing:{workspace_id}")))?;
|
||||||
if policy.policy_id!=plan.policy_id||policy.policy_digest!=plan.policy_digest{
|
if policy.policy_id!=plan.policy_id||policy.policy_digest!=plan.policy_digest{
|
||||||
mark_stale(&tx,&plan,"policy content changed")?;tx.commit()?;
|
mark_stale(&tx,&plan,"policy content changed")?;tx.commit()?;
|
||||||
return Err(stale_error(&plan,"policy content changed"));
|
return Err(stale_error(&plan,"policy content changed"));
|
||||||
@@ -1020,6 +1020,7 @@ fn load_removal_policy(
|
|||||||
w: &str,
|
w: &str,
|
||||||
worker: &RuntimeWorkerRef,
|
worker: &RuntimeWorkerRef,
|
||||||
erased_journal: bool,
|
erased_journal: bool,
|
||||||
|
preserve_subject_history: bool,
|
||||||
) -> crate::Result<Option<WorkerRetentionPolicy>> {
|
) -> crate::Result<Option<WorkerRetentionPolicy>> {
|
||||||
let Some(mut policy) = load_policy(c, w)? else {
|
let Some(mut policy) = load_policy(c, w)? else {
|
||||||
return Ok(None);
|
return Ok(None);
|
||||||
@@ -1044,6 +1045,25 @@ fn load_removal_policy(
|
|||||||
policy.archive_retention = update.archive_retention;
|
policy.archive_retention = update.archive_retention;
|
||||||
policy.diagnostics_disposition = update.diagnostics_disposition;
|
policy.diagnostics_disposition = update.diagnostics_disposition;
|
||||||
policy.diagnostics_retention_seconds = None;
|
policy.diagnostics_retention_seconds = None;
|
||||||
|
} else if preserve_subject_history && policy.session_disposition == SessionDisposition::Purge && c.query_row(
|
||||||
|
"SELECT EXISTS(SELECT 1 FROM worker_create_reservations WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND substr(singleton_key,1,10)='subjektiv:')",
|
||||||
|
params![w, worker.runtime_id, worker.worker_id],
|
||||||
|
|r| r.get::<_, bool>(0),
|
||||||
|
)? {
|
||||||
|
// Removing a Subject body releases occupancy, not its original history.
|
||||||
|
// A configured Purge must not silently erase an unfinished Journal. Explicit
|
||||||
|
// Session erasure uses the separate, proof-bound branch above.
|
||||||
|
let update = WorkerRetentionPolicyUpdate {
|
||||||
|
policy_id: "backend-subject-removal-archive-v1".into(),
|
||||||
|
session_disposition: SessionDisposition::Archive,
|
||||||
|
metadata_disposition: policy.metadata_disposition,
|
||||||
|
archive_retention: policy.archive_retention,
|
||||||
|
diagnostics_disposition: policy.diagnostics_disposition,
|
||||||
|
diagnostics_retention_seconds: policy.diagnostics_retention_seconds,
|
||||||
|
};
|
||||||
|
policy.policy_id = update.policy_id.clone();
|
||||||
|
policy.policy_digest = worker_retention_policy_digest(&update);
|
||||||
|
policy.session_disposition = update.session_disposition;
|
||||||
}
|
}
|
||||||
Ok(Some(policy))
|
Ok(Some(policy))
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,240 @@
|
|||||||
|
// Included in store::tests. Exercise the production migration entrypoints on
|
||||||
|
// isolated SQLite files; both published branches are built from retained DDL.
|
||||||
|
mod schema_convergence {
|
||||||
|
use super::*;
|
||||||
|
use std::path::PathBuf;
|
||||||
|
|
||||||
|
fn variant_fixture(job_variant: bool, baseline_only: bool) -> (tempfile::TempDir, PathBuf) {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let path = dir.path().join("schema-103.sqlite");
|
||||||
|
prepare_retained_schema(&path, 102);
|
||||||
|
let conn = Connection::open(&path).unwrap();
|
||||||
|
configure_sqlite(&conn).unwrap();
|
||||||
|
if job_variant {
|
||||||
|
(JOB_INPUT_RETENTION_SCHEMA_103.apply)(&conn).unwrap();
|
||||||
|
} else {
|
||||||
|
migrate_passkey_results_v102_to_v103(&conn).unwrap();
|
||||||
|
}
|
||||||
|
if baseline_only {
|
||||||
|
conn.execute_batch("DELETE FROM __yoi_schema_migrations;
|
||||||
|
INSERT INTO __yoi_schema_migrations(version,name) VALUES(103,'workspace schema baseline');").unwrap();
|
||||||
|
}
|
||||||
|
conn.execute_batch(r#"
|
||||||
|
INSERT INTO accounts(account_id,kind,handle,display_name,created_at,updated_at)
|
||||||
|
VALUES('owner','user','owner','Owner','1','2');
|
||||||
|
INSERT INTO users(user_id,account_id,handle,display_name,created_at,updated_at)
|
||||||
|
VALUES('user','owner','owner','Owner','1','2');
|
||||||
|
INSERT INTO workspaces(workspace_id,owner_account_id,display_name,state,created_at,updated_at)
|
||||||
|
VALUES('workspace','owner','Workspace','active','1','2');
|
||||||
|
INSERT INTO passkey_credentials(credential_id,user_id,public_key_cose,transports_json,sign_count,created_at,last_used_at)
|
||||||
|
VALUES('key','user','preserved-public-key','["internal"]',9,'1','2');
|
||||||
|
"#).unwrap();
|
||||||
|
if !job_variant {
|
||||||
|
conn.execute_batch("UPDATE passkey_metadata SET key_reference='stable-reference',normal_login_verified_at='verified';
|
||||||
|
INSERT INTO passkey_addition_receipts(challenge_id,credential_id,user_id,authentication_reference)
|
||||||
|
VALUES('challenge','key','user','saved-authentication-reference');").unwrap();
|
||||||
|
}
|
||||||
|
for disposition in ["available", "reclaimed", "erased", "invalidated"] {
|
||||||
|
let request = resource_job_request(disposition, disposition, "candidate");
|
||||||
|
conn.execute("INSERT INTO backend_jobs(workspace_id,job_id,purpose,input_digest,input_ref,request_json,intent_fingerprint,state,current_attempt,result_json,result_digest,created_at,updated_at,completed_at)
|
||||||
|
VALUES('workspace',?1,?2,?3,?4,?5,?6,'completed',1,'{\"saved\":true}','result-digest','1','2','2')",
|
||||||
|
params![request.job_id, request.purpose, request.input_digest().unwrap(), request.input_ref, serde_json::to_string(&request).unwrap(), request.fingerprint().unwrap()]).unwrap();
|
||||||
|
conn.execute("INSERT INTO backend_job_attempts(workspace_id,job_id,attempt_id,attempt,input_digest,state,deadline_at,result_json,result_digest,created_at,updated_at,completed_at)
|
||||||
|
VALUES('workspace',?1,?1,1,?2,'completed','deadline','{\"saved\":true}','result-digest','1','2','2')",
|
||||||
|
params![request.job_id, request.input_digest().unwrap()]).unwrap();
|
||||||
|
if job_variant && disposition != "available" {
|
||||||
|
conn.execute("UPDATE backend_jobs SET request_json=json_set(request_json,'$.input',json('null')),input_disposition=?1 WHERE job_id=?1", [disposition]).unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
verify_schema_history(&conn, 103).unwrap();
|
||||||
|
(dir, path)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn feature_rows(conn: &Connection, job_variant: bool) -> Vec<Vec<Vec<rusqlite::types::Value>>> {
|
||||||
|
let mut tables = vec!["passkey_credentials", "backend_job_attempts"];
|
||||||
|
if !job_variant {
|
||||||
|
tables.extend(["passkey_metadata", "passkey_addition_receipts"]);
|
||||||
|
}
|
||||||
|
let mut rows: Vec<_> = tables
|
||||||
|
.into_iter()
|
||||||
|
.map(|table| stored_fixture_rows(conn, table))
|
||||||
|
.collect();
|
||||||
|
// A Passkey-branch Job gains only input_disposition; all old columns must
|
||||||
|
// remain byte-for-byte equal, including body, digest and result receipts.
|
||||||
|
rows.push(fixture_query_rows(conn, "SELECT workspace_id,job_id,purpose,input_digest,input_ref,resource_key,request_json,intent_fingerprint,state,current_attempt,result_json,result_digest,failure_category,failure_detail,created_at,updated_at,completed_at FROM backend_jobs ORDER BY job_id"));
|
||||||
|
if job_variant {
|
||||||
|
rows.push(fixture_query_rows(
|
||||||
|
conn,
|
||||||
|
"SELECT job_id,input_disposition FROM backend_jobs ORDER BY job_id",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
rows
|
||||||
|
}
|
||||||
|
|
||||||
|
fn schema_rows(conn: &Connection) -> Vec<Vec<rusqlite::types::Value>> {
|
||||||
|
fixture_query_rows(
|
||||||
|
conn,
|
||||||
|
"SELECT type,name,tbl_name,sql FROM sqlite_schema ORDER BY type,name",
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn schema_103_variants_converge_preserving_history_values_and_constraints() {
|
||||||
|
for job_variant in [false, true] {
|
||||||
|
for baseline_only in [false, true] {
|
||||||
|
let (_dir, path) = variant_fixture(job_variant, baseline_only);
|
||||||
|
let conn = Connection::open(&path).unwrap();
|
||||||
|
let history = workspace_schema_migration_history(&conn).unwrap();
|
||||||
|
let rows = feature_rows(&conn, job_variant);
|
||||||
|
let schema = schema_rows(&conn);
|
||||||
|
let plan = SqliteWorkspaceStore::migration_plan(&path).unwrap();
|
||||||
|
assert_eq!(plan.current_schema_version, 103);
|
||||||
|
assert_eq!(
|
||||||
|
plan.migrations,
|
||||||
|
vec![WorkspaceSchemaMigrationStep {
|
||||||
|
version: 104,
|
||||||
|
name: "Converge Passkey results and Job input retention".into(),
|
||||||
|
}]
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
schema_rows(&conn),
|
||||||
|
schema,
|
||||||
|
"preflight must not mutate the source"
|
||||||
|
);
|
||||||
|
assert_eq!(feature_rows(&conn, job_variant), rows);
|
||||||
|
drop(conn);
|
||||||
|
let upgraded = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||||
|
let fresh = SqliteWorkspaceStore::in_memory_for_test().unwrap();
|
||||||
|
upgraded.with_conn(|conn| {
|
||||||
|
assert_eq!(feature_rows(conn, job_variant), rows);
|
||||||
|
let mut expected = history.clone();
|
||||||
|
expected.push((104, "Converge Passkey results and Job input retention".into()));
|
||||||
|
assert_eq!(workspace_schema_migration_history(conn)?, expected);
|
||||||
|
for table in ["passkey_credentials", "passkey_metadata", "passkey_addition_receipts", "backend_jobs", "backend_job_attempts"] {
|
||||||
|
assert_eq!(state_contract_schema_signature(conn, table), fresh.with_conn(|c| Ok(state_contract_schema_signature(c, table)))?, "{table}");
|
||||||
|
}
|
||||||
|
assert_eq!(conn.query_row("SELECT count(*) FROM pragma_foreign_key_check", [], |r| r.get::<_, i64>(0))?, 0);
|
||||||
|
assert!(conn.execute("UPDATE backend_jobs SET input_digest='changed'", []).is_err());
|
||||||
|
assert!(conn.execute("UPDATE backend_jobs SET input_disposition='unsupported'", []).is_err());
|
||||||
|
assert!(conn.execute("UPDATE backend_jobs SET request_json=json_set(request_json,'$.input','replacement')", []).is_err());
|
||||||
|
assert!(conn.execute("INSERT INTO passkey_metadata(credential_id,key_reference) VALUES('missing','new')", []).is_err());
|
||||||
|
assert!(conn.execute("INSERT INTO passkey_addition_receipts VALUES('other','missing','user','reference')", []).is_err());
|
||||||
|
let reference: (String, Option<String>) = conn.query_row("SELECT key_reference,normal_login_verified_at FROM passkey_metadata WHERE credential_id='key'", [], |r| Ok((r.get(0)?,r.get(1)?)))?;
|
||||||
|
if job_variant {
|
||||||
|
assert_eq!(reference.0.len(), 32);
|
||||||
|
assert_eq!(reference.1, None);
|
||||||
|
} else {
|
||||||
|
assert_eq!(reference, ("stable-reference".into(), Some("verified".into())));
|
||||||
|
assert_eq!(conn.query_row("SELECT count(*) FROM backend_jobs WHERE input_disposition='available'", [], |r| r.get::<_, i64>(0))?, 4);
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}).unwrap();
|
||||||
|
let summaries = upgraded.list_passkey_summaries("user").unwrap();
|
||||||
|
drop(upgraded);
|
||||||
|
let reopened = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||||
|
assert_eq!(reopened.list_passkey_summaries("user").unwrap(), summaries);
|
||||||
|
reopened
|
||||||
|
.with_conn(|conn| {
|
||||||
|
assert_eq!(feature_rows(conn, job_variant), rows);
|
||||||
|
assert_eq!(
|
||||||
|
workspace_schema_migration_history(conn)?.len(),
|
||||||
|
history.len() + 1
|
||||||
|
);
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn schema_103_convergence_failure_rolls_back_both_variants_and_retries() {
|
||||||
|
for job_variant in [false, true] {
|
||||||
|
for baseline_only in [false, true] {
|
||||||
|
let (_dir, path) = variant_fixture(job_variant, baseline_only);
|
||||||
|
let conn = Connection::open(&path).unwrap();
|
||||||
|
conn.execute_batch("CREATE TRIGGER interrupt_convergence BEFORE INSERT ON __yoi_schema_migrations WHEN NEW.version=104 BEGIN SELECT RAISE(ABORT,'injected convergence failure'); END;").unwrap();
|
||||||
|
let schema = schema_rows(&conn);
|
||||||
|
let history = workspace_schema_migration_history(&conn).unwrap();
|
||||||
|
let rows = feature_rows(&conn, job_variant);
|
||||||
|
drop(conn);
|
||||||
|
let error = SqliteWorkspaceStore::open_for_test(&path)
|
||||||
|
.err()
|
||||||
|
.expect("migration must fail");
|
||||||
|
assert!(
|
||||||
|
error.to_string().contains("injected convergence failure"),
|
||||||
|
"{error}"
|
||||||
|
);
|
||||||
|
let conn = Connection::open(&path).unwrap();
|
||||||
|
assert_eq!(schema_rows(&conn), schema);
|
||||||
|
assert_eq!(workspace_schema_migration_history(&conn).unwrap(), history);
|
||||||
|
assert_eq!(feature_rows(&conn, job_variant), rows);
|
||||||
|
conn.execute_batch("DROP TRIGGER interrupt_convergence")
|
||||||
|
.unwrap();
|
||||||
|
drop(conn);
|
||||||
|
let upgraded = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||||
|
upgraded
|
||||||
|
.with_conn(|conn| {
|
||||||
|
assert_eq!(feature_rows(conn, job_variant), rows);
|
||||||
|
assert_eq!(current_schema_version(conn)?, 104);
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn schema_103_unknown_migration_identity_is_rejected_without_changes() {
|
||||||
|
for job_variant in [false, true] {
|
||||||
|
let (_dir, path) = variant_fixture(job_variant, false);
|
||||||
|
let conn = Connection::open(&path).unwrap();
|
||||||
|
conn.execute("UPDATE __yoi_schema_migrations SET name='unrecognized schema 103' WHERE version=103", []).unwrap();
|
||||||
|
let schema = schema_rows(&conn);
|
||||||
|
let history = workspace_schema_migration_history(&conn).unwrap();
|
||||||
|
let rows = feature_rows(&conn, job_variant);
|
||||||
|
assert!(SqliteWorkspaceStore::migration_plan(&path).is_err());
|
||||||
|
assert!(SqliteWorkspaceStore::open_for_test(&path).is_err());
|
||||||
|
assert_eq!(schema_rows(&conn), schema);
|
||||||
|
assert_eq!(workspace_schema_migration_history(&conn).unwrap(), history);
|
||||||
|
assert_eq!(feature_rows(&conn, job_variant), rows);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn schema_103_mismatched_identity_or_incomplete_features_are_rejected_atomically() {
|
||||||
|
for job_variant in [false, true] {
|
||||||
|
for mismatched_identity in [false, true] {
|
||||||
|
let (_dir, path) = variant_fixture(job_variant, false);
|
||||||
|
let conn = Connection::open(&path).unwrap();
|
||||||
|
if mismatched_identity {
|
||||||
|
let other = if job_variant {
|
||||||
|
"Passkey metadata and atomic addition results"
|
||||||
|
} else {
|
||||||
|
"Job input retirement independent of immutable receipts"
|
||||||
|
};
|
||||||
|
conn.execute(
|
||||||
|
"UPDATE __yoi_schema_migrations SET name=?1 WHERE version=103",
|
||||||
|
[other],
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
} else {
|
||||||
|
let trigger = if job_variant {
|
||||||
|
"backend_job_identity_immutable"
|
||||||
|
} else {
|
||||||
|
"passkey_metadata_insert"
|
||||||
|
};
|
||||||
|
conn.execute_batch(&format!("DROP TRIGGER {trigger}"))
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
let schema = schema_rows(&conn);
|
||||||
|
let history = workspace_schema_migration_history(&conn).unwrap();
|
||||||
|
let rows = feature_rows(&conn, job_variant);
|
||||||
|
assert!(SqliteWorkspaceStore::open_for_test(&path).is_err());
|
||||||
|
assert_eq!(schema_rows(&conn), schema);
|
||||||
|
assert_eq!(workspace_schema_migration_history(&conn).unwrap(), history);
|
||||||
|
assert_eq!(feature_rows(&conn, job_variant), rows);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,40 +1,41 @@
|
|||||||
// Included in store::tests so both paths use the real retained migration chain.
|
// Included in store::tests so both paths use the real retained migration chain.
|
||||||
|
fn state_contract_schema_signature(conn: &Connection, table: &str) -> Vec<Vec<String>> {
|
||||||
|
let queries = [
|
||||||
|
"SELECT name, type, CAST(\"notnull\" AS TEXT), COALESCE(dflt_value, '<NULL>'), CAST(pk AS TEXT) FROM pragma_table_info(?1) ORDER BY name",
|
||||||
|
"SELECT \"table\", \"from\", \"to\", on_update, on_delete, match, CAST(seq AS TEXT) FROM pragma_foreign_key_list(?1) ORDER BY \"table\", \"from\", \"to\", seq",
|
||||||
|
"SELECT CAST(i.\"unique\" AS TEXT), CAST(i.partial AS TEXT), (SELECT group_concat(name, ',') FROM (SELECT name FROM pragma_index_info(i.name) ORDER BY seqno)) FROM pragma_index_list(?1) AS i ORDER BY 1, 2, 3",
|
||||||
|
"SELECT name, sql FROM sqlite_schema WHERE type='trigger' AND tbl_name=?1 ORDER BY name",
|
||||||
|
];
|
||||||
|
queries
|
||||||
|
.into_iter()
|
||||||
|
.flat_map(|sql| {
|
||||||
|
let mut statement = conn.prepare(sql).unwrap();
|
||||||
|
let count = statement.column_count();
|
||||||
|
statement
|
||||||
|
.query_map([table], |row| {
|
||||||
|
(0..count)
|
||||||
|
.map(|i| row.get::<_, String>(i))
|
||||||
|
.collect::<rusqlite::Result<Vec<_>>>()
|
||||||
|
})
|
||||||
|
.unwrap()
|
||||||
|
.map(|row| {
|
||||||
|
row.unwrap()
|
||||||
|
.into_iter()
|
||||||
|
.map(|value| {
|
||||||
|
value
|
||||||
|
.chars()
|
||||||
|
.filter(|c| !c.is_whitespace() && *c != '"')
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>()
|
||||||
|
})
|
||||||
|
.collect()
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn fresh_and_upgraded_state_contract_schemas_enforce_the_same_constraints() {
|
fn fresh_and_upgraded_state_contract_schemas_enforce_the_same_constraints() {
|
||||||
fn signature(conn: &Connection, table: &str) -> Vec<Vec<String>> {
|
|
||||||
let queries = [
|
|
||||||
"SELECT name, type, CAST(\"notnull\" AS TEXT), COALESCE(dflt_value, '<NULL>'), CAST(pk AS TEXT) FROM pragma_table_info(?1) ORDER BY name",
|
|
||||||
"SELECT \"table\", \"from\", \"to\", on_update, on_delete, match, CAST(seq AS TEXT) FROM pragma_foreign_key_list(?1) ORDER BY \"table\", \"from\", \"to\", seq",
|
|
||||||
"SELECT CAST(i.\"unique\" AS TEXT), CAST(i.partial AS TEXT), (SELECT group_concat(name, ',') FROM (SELECT name FROM pragma_index_info(i.name) ORDER BY seqno)) FROM pragma_index_list(?1) AS i ORDER BY 1, 2, 3",
|
|
||||||
"SELECT name, sql FROM sqlite_schema WHERE type='trigger' AND tbl_name=?1 ORDER BY name",
|
|
||||||
];
|
|
||||||
queries
|
|
||||||
.into_iter()
|
|
||||||
.flat_map(|sql| {
|
|
||||||
let mut statement = conn.prepare(sql).unwrap();
|
|
||||||
let count = statement.column_count();
|
|
||||||
statement
|
|
||||||
.query_map([table], |row| {
|
|
||||||
(0..count)
|
|
||||||
.map(|i| row.get::<_, String>(i))
|
|
||||||
.collect::<rusqlite::Result<Vec<_>>>()
|
|
||||||
})
|
|
||||||
.unwrap()
|
|
||||||
.map(|row| {
|
|
||||||
row.unwrap()
|
|
||||||
.into_iter()
|
|
||||||
.map(|value| {
|
|
||||||
value
|
|
||||||
.chars()
|
|
||||||
.filter(|c| !c.is_whitespace() && *c != '"')
|
|
||||||
.collect()
|
|
||||||
})
|
|
||||||
.collect()
|
|
||||||
})
|
|
||||||
.collect::<Vec<_>>()
|
|
||||||
})
|
|
||||||
.collect()
|
|
||||||
}
|
|
||||||
let temp = tempfile::tempdir().unwrap();
|
let temp = tempfile::tempdir().unwrap();
|
||||||
let path = temp.path().join("upgraded.db");
|
let path = temp.path().join("upgraded.db");
|
||||||
prepare_retained_schema(&path, 85);
|
prepare_retained_schema(&path, 85);
|
||||||
@@ -58,6 +59,9 @@ fn fresh_and_upgraded_state_contract_schemas_enforce_the_same_constraints() {
|
|||||||
"workspace_memory_settings",
|
"workspace_memory_settings",
|
||||||
"worker_create_reservations",
|
"worker_create_reservations",
|
||||||
"subject_worker_releases",
|
"subject_worker_releases",
|
||||||
|
"passkey_credentials",
|
||||||
|
"passkey_metadata",
|
||||||
|
"passkey_addition_receipts",
|
||||||
"backend_jobs",
|
"backend_jobs",
|
||||||
"backend_job_attempts",
|
"backend_job_attempts",
|
||||||
"flow_sources",
|
"flow_sources",
|
||||||
@@ -92,9 +96,11 @@ fn fresh_and_upgraded_state_contract_schemas_enforce_the_same_constraints() {
|
|||||||
"workdir_create_legacy_ssh_archives",
|
"workdir_create_legacy_ssh_archives",
|
||||||
"repository_secret_legacy_receipts",
|
"repository_secret_legacy_receipts",
|
||||||
] {
|
] {
|
||||||
let expected = fresh.with_conn(|conn| Ok(signature(conn, table))).unwrap();
|
let expected = fresh
|
||||||
|
.with_conn(|conn| Ok(state_contract_schema_signature(conn, table)))
|
||||||
|
.unwrap();
|
||||||
let actual = upgraded
|
let actual = upgraded
|
||||||
.with_conn(|conn| Ok(signature(conn, table)))
|
.with_conn(|conn| Ok(state_contract_schema_signature(conn, table)))
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert!(!expected.is_empty(), "missing canonical table {table}");
|
assert!(!expected.is_empty(), "missing canonical table {table}");
|
||||||
if actual != expected {
|
if actual != expected {
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@@ -92,6 +92,20 @@ pub(super) fn prepare(api: &WorkspaceApi, reservation: &BackendJobReservation) -
|
|||||||
};
|
};
|
||||||
let store = store(api)?;
|
let store = store(api)?;
|
||||||
let host = host(api, &store, grant)?;
|
let host = host(api, &store, grant)?;
|
||||||
|
let attr = store
|
||||||
|
.session_attribution(&grant.session_id)
|
||||||
|
.map_err(|e| Error::Store(e.to_string()))?
|
||||||
|
.ok_or_else(|| {
|
||||||
|
Error::WorkspacePermissionDenied("Journal Session attribution missing".into())
|
||||||
|
})?;
|
||||||
|
if store
|
||||||
|
.journal_worker_was_released(&attr.subject_id, &attr.runtime_id, &attr.worker_id)
|
||||||
|
.map_err(domain)?
|
||||||
|
{
|
||||||
|
return Err(Error::WorkspacePermissionDenied(
|
||||||
|
"released Journal body cannot dispatch another attempt".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
let input = saved_input(&reservation.job.request, grant)?;
|
let input = saved_input(&reservation.job.request, grant)?;
|
||||||
let g = store
|
let g = store
|
||||||
.prepare_journal_input(
|
.prepare_journal_input(
|
||||||
@@ -584,6 +598,18 @@ fn dispatch(
|
|||||||
.lock()
|
.lock()
|
||||||
.map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))?;
|
.map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))?;
|
||||||
let store = store(api)?;
|
let store = store(api)?;
|
||||||
|
if store
|
||||||
|
.journal_worker_was_released(
|
||||||
|
&attribution.subject_id,
|
||||||
|
&attribution.runtime_id,
|
||||||
|
&attribution.worker_id,
|
||||||
|
)
|
||||||
|
.map_err(domain)?
|
||||||
|
{
|
||||||
|
// Deletion can retain an unfinished prefix. It must not mint attempts or
|
||||||
|
// transfer the removed body's write capability to its successor.
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
let host =
|
let host =
|
||||||
JournalHostContext::validated_session(&store, attribution.clone()).map_err(domain)?;
|
JournalHostContext::validated_session(&store, attribution.clone()).map_err(domain)?;
|
||||||
if let Some(g) = store
|
if let Some(g) = store
|
||||||
@@ -1740,8 +1766,8 @@ fn finish_erased_release(
|
|||||||
.release_subject_singleton(api.workspace_id(), &lease, binding_id, &operation)
|
.release_subject_singleton(api.workspace_id(), &lease, binding_id, &operation)
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Normal deletion cannot substitute for Subject release, even for a stopped or disabled body.
|
/// Ordinary removal owns writer revocation and occupancy release. Before execution
|
||||||
/// Explicit erasure remains a separate monotonic source/diagnostics destruction path.
|
/// only preserve/erase source requirements are checked; an active writer is not a blocker.
|
||||||
pub(super) fn prepare_worker_removal(
|
pub(super) fn prepare_worker_removal(
|
||||||
service: &WorkerRemovalService,
|
service: &WorkerRemovalService,
|
||||||
runtime: &RuntimeRegistry,
|
runtime: &RuntimeRegistry,
|
||||||
@@ -1762,7 +1788,6 @@ pub(super) fn prepare_worker_removal(
|
|||||||
&service.journal_registration,
|
&service.journal_registration,
|
||||||
)
|
)
|
||||||
.map_err(|e| Error::Store(e.to_string()))?;
|
.map_err(|e| Error::Store(e.to_string()))?;
|
||||||
let host = JournalHostContext::validated_session(&store, attr.clone()).map_err(domain)?;
|
|
||||||
let source = runtime_api::SessionPublicSource::Retained {
|
let source = runtime_api::SessionPublicSource::Retained {
|
||||||
worker_id: worker_runtime::identity::WorkerId::parse(&target.worker_id)
|
worker_id: worker_runtime::identity::WorkerId::parse(&target.worker_id)
|
||||||
.ok_or_else(|| Error::Store("Worker identity invalid".into()))?,
|
.ok_or_else(|| Error::Store("Worker identity invalid".into()))?,
|
||||||
@@ -1779,25 +1804,9 @@ pub(super) fn prepare_worker_removal(
|
|||||||
finish_erased_release(&api, &attr)?;
|
finish_erased_release(&api, &attr)?;
|
||||||
return Ok(Some(proof));
|
return Ok(Some(proof));
|
||||||
}
|
}
|
||||||
let Some(binding_id) = host.writer_binding_id() else {
|
// Do not require a live Controller, complete Journal, or prior release here.
|
||||||
return Err(Error::RepositoryConflict(
|
// Runtime removal is the physical fence; finish_worker_removal revokes the
|
||||||
"Subject writer evidence is missing; ordinary deletion cannot bypass release".into(),
|
// exact old body only after its successful, matching retention receipt.
|
||||||
));
|
|
||||||
};
|
|
||||||
let writer = JournalWriterBinding {
|
|
||||||
subject_id: attr.subject_id.clone(),
|
|
||||||
runtime_id: attr.runtime_id.clone(),
|
|
||||||
worker_id: attr.worker_id.clone(),
|
|
||||||
binding_id: binding_id.into(),
|
|
||||||
};
|
|
||||||
if store.journal_writer_active(&writer).map_err(domain)? {
|
|
||||||
return Err(Error::RepositoryConflict("Subject remains occupied: restore to actual Idle, finish Journal recording, then explicitly release before ordinary deletion".into()));
|
|
||||||
}
|
|
||||||
if store.journal_retention_required(&host).map_err(domain)? {
|
|
||||||
return Err(Error::RepositoryConflict(
|
|
||||||
"Released Subject has unresolved Journal material; retain occupancy and source".into(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
Ok(None)
|
Ok(None)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1886,20 +1895,6 @@ pub(super) fn validate_removal_retention(
|
|||||||
// its retained permanent fence, including recovery after the aggregate is gone.
|
// its retained permanent fence, including recovery after the aggregate is gone.
|
||||||
return Ok(());
|
return Ok(());
|
||||||
}
|
}
|
||||||
let binding_id = host.writer_binding_id().ok_or_else(|| {
|
|
||||||
Error::RepositoryConflict("Subject writer evidence missing during removal recovery".into())
|
|
||||||
})?;
|
|
||||||
let binding = JournalWriterBinding {
|
|
||||||
subject_id: attr.subject_id.clone(),
|
|
||||||
runtime_id: attr.runtime_id.clone(),
|
|
||||||
worker_id: attr.worker_id.clone(),
|
|
||||||
binding_id: binding_id.into(),
|
|
||||||
};
|
|
||||||
if store.journal_writer_active(&binding).map_err(domain)? {
|
|
||||||
return Err(Error::RepositoryConflict(
|
|
||||||
"Prepared removal cannot bypass explicit Subject release".into(),
|
|
||||||
));
|
|
||||||
}
|
|
||||||
let source = runtime_api::SessionPublicSource::Retained {
|
let source = runtime_api::SessionPublicSource::Retained {
|
||||||
worker_id: worker_runtime::identity::WorkerId::parse(&target.worker_id)
|
worker_id: worker_runtime::identity::WorkerId::parse(&target.worker_id)
|
||||||
.ok_or_else(|| Error::Store("Worker identity invalid".into()))?,
|
.ok_or_else(|| Error::Store("Worker identity invalid".into()))?,
|
||||||
@@ -1907,37 +1902,140 @@ pub(super) fn validate_removal_retention(
|
|||||||
let runtime = service.runtime.upgrade().ok_or_else(|| {
|
let runtime = service.runtime.upgrade().ok_or_else(|| {
|
||||||
Error::Store("Runtime unavailable for source retention confirmation".into())
|
Error::Store("Runtime unavailable for source retention confirmation".into())
|
||||||
})?;
|
})?;
|
||||||
if store.journal_retention_required(&host).map_err(domain)? {
|
let retention_required = store.journal_retention_required(&host).map_err(domain)?;
|
||||||
if prepared.runtime_request.session_disposition
|
if prepared.runtime_request.session_disposition
|
||||||
!= worker_runtime::retention::SessionDisposition::Archive
|
== worker_runtime::retention::SessionDisposition::Purge
|
||||||
|| prepared.runtime_request.archive_id.is_none()
|
&& prepared.runtime_request.archive_id.is_none()
|
||||||
{
|
&& !retention_required
|
||||||
return Err(Error::RepositoryConflict(
|
&& store
|
||||||
"unreflected Journal requires a retained original Session archive".into(),
|
.journal_worker_was_released(&attr.subject_id, &attr.runtime_id, &attr.worker_id)
|
||||||
));
|
.map_err(domain)?
|
||||||
|
{
|
||||||
|
// Compatibility for an already issued, history-complete ordinary Purge
|
||||||
|
// plan. New Subject plans always Archive. Keep the old fingerprint and
|
||||||
|
// disposition; never turn this into an erasure or allow an active writer.
|
||||||
|
// A missing Runtime Worker is NOT success: execute_worker_retention still
|
||||||
|
// has to return the exact persisted receipt before anything can commit.
|
||||||
|
// A pending receipt may outlive the physical aggregate while Runtime's
|
||||||
|
// in-memory catalog still contains the Worker. Pin release is best effort
|
||||||
|
// in this historical branch; exact Runtime retention replay still enforces
|
||||||
|
// pins if the source exists, and requires a receipt if it is already gone.
|
||||||
|
if let Err(error) = retain_source(&runtime, &service.workspace_id, &attr, source, true) {
|
||||||
|
tracing::debug!(%error, "Historical Purge source unavailable; reconcile exact Runtime receipt");
|
||||||
}
|
}
|
||||||
if let Err(original) = retain_source(&runtime, &service.workspace_id, &attr, source, false)
|
return Ok(());
|
||||||
{
|
}
|
||||||
// Recovery may already have transferred the physical source before its Backend
|
if prepared.runtime_request.session_disposition
|
||||||
// catalog commit; authenticate the exact reserved archive, never another Session.
|
!= worker_runtime::retention::SessionDisposition::Archive
|
||||||
let archive = runtime_api::SessionPublicSource::Archived {
|
|| prepared.runtime_request.archive_id.is_none()
|
||||||
archive_id: prepared
|
{
|
||||||
.runtime_request
|
return Err(Error::RepositoryConflict(
|
||||||
.archive_id
|
"Subject removal retains its original Session; explicit erasure is required for Purge"
|
||||||
.clone()
|
.into(),
|
||||||
.expect("checked"),
|
));
|
||||||
};
|
}
|
||||||
retain_source(&runtime, &service.workspace_id, &attr, archive, false)
|
let release = !retention_required;
|
||||||
.map_err(|_| original)?;
|
if let Err(original) = retain_source(&runtime, &service.workspace_id, &attr, source, release) {
|
||||||
}
|
// A lost reply may follow the physical transfer. Reconcile only the exact
|
||||||
} else {
|
// reserved archive and Session, whether or not an admitted remainder exists.
|
||||||
// This stopped/removal-fenced Session has no admitted remainder; purge need not
|
let archive = runtime_api::SessionPublicSource::Archived {
|
||||||
// force a generation. A failed release conservatively blocks ordinary deletion.
|
archive_id: prepared
|
||||||
retain_source(&runtime, &service.workspace_id, &attr, source, true)?;
|
.runtime_request
|
||||||
|
.archive_id
|
||||||
|
.clone()
|
||||||
|
.expect("checked"),
|
||||||
|
};
|
||||||
|
retain_source(&runtime, &service.workspace_id, &attr, archive, release)
|
||||||
|
.map_err(|_| original)?;
|
||||||
}
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// A successful Runtime receipt, not catalog liveness, authorizes ending the old
|
||||||
|
/// writer. The ordinary removal commit releases occupancy atomically with catalog
|
||||||
|
/// deletion. Until then a crash leaves the singleton occupied and the same receipt
|
||||||
|
/// can safely repeat this step; no Controller restore or Journal generation is needed.
|
||||||
|
pub(super) fn finish_worker_removal<T>(
|
||||||
|
service: &WorkerRemovalService,
|
||||||
|
target: &RuntimeWorkerRef,
|
||||||
|
prepared: &crate::retention::PreparedWorkerRemoval,
|
||||||
|
result: &worker_runtime::retention::WorkerRetentionExecutionResult,
|
||||||
|
commit: impl FnOnce() -> T,
|
||||||
|
) -> Result<T> {
|
||||||
|
let Some(attr) = removal_attribution(service, target)? else {
|
||||||
|
return Ok(commit());
|
||||||
|
};
|
||||||
|
let request = &prepared.runtime_request;
|
||||||
|
if prepared.plan.worker != *target
|
||||||
|
|| request.workspace_id != service.workspace_id
|
||||||
|
|| request.source_runtime_id != target.runtime_id
|
||||||
|
|| request.worker_id.to_string() != target.worker_id
|
||||||
|
|| !result.source_removed
|
||||||
|
|| result.worker_id != request.worker_id
|
||||||
|
|| result.operation_id != request.operation_id
|
||||||
|
|| result.input_fingerprint != request.input_fingerprint
|
||||||
|
|| result.session_disposition != request.session_disposition
|
||||||
|
|| result.diagnostics_disposition != request.diagnostics_disposition
|
||||||
|
|| result.diagnostics_retained
|
||||||
|
!= (request.diagnostics_disposition
|
||||||
|
== worker_runtime::retention::DiagnosticsDisposition::Retain)
|
||||||
|
{
|
||||||
|
return Err(Error::RepositoryConflict(
|
||||||
|
"Subject removal receipt does not match its plan".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if request.session_disposition == worker_runtime::retention::SessionDisposition::Archive {
|
||||||
|
let archive = result.archive.as_ref().ok_or_else(|| {
|
||||||
|
Error::RepositoryConflict("Subject removal archive receipt is missing".into())
|
||||||
|
})?;
|
||||||
|
if Some(&archive.archive_id) != request.archive_id.as_ref()
|
||||||
|
|| archive.workspace_id != request.workspace_id
|
||||||
|
|| archive.source_runtime_id != request.source_runtime_id
|
||||||
|
|| archive.source_worker_id != request.worker_id
|
||||||
|
|| archive.source_session_id != attr.session_id
|
||||||
|
|| archive.operation_id != request.operation_id
|
||||||
|
|| archive.input_fingerprint != request.input_fingerprint
|
||||||
|
|| archive.policy_id != request.policy_id
|
||||||
|
{
|
||||||
|
return Err(Error::RepositoryConflict(
|
||||||
|
"Subject removal archive identity mismatch".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let api = service.journal_dispatch_api.upgrade().ok_or_else(|| {
|
||||||
|
Error::Store("Workspace API unavailable for Subject removal completion".into())
|
||||||
|
})?;
|
||||||
|
let gate = lifecycle_lock(&api, &attr.subject_id)?;
|
||||||
|
let _guard = gate
|
||||||
|
.lock()
|
||||||
|
.map_err(|_| Error::Store("Journal lifecycle lock poisoned".into()))?;
|
||||||
|
let store = store(&api)?;
|
||||||
|
let host = JournalHostContext::validated_session(&store, attr.clone()).map_err(domain)?;
|
||||||
|
if store.journal_session_erased(&host).map_err(domain)? {
|
||||||
|
if request.session_disposition == worker_runtime::retention::SessionDisposition::Archive {
|
||||||
|
// Erasure may win after Runtime archive transfer. Do not publish that
|
||||||
|
// archive or skip writer revocation: the same removal retry must take
|
||||||
|
// erased_archive_removal_proof and its physical-erasure commit path.
|
||||||
|
return Err(Error::RepositoryConflict(
|
||||||
|
"Session erased during removal; reconcile the same removal operation".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
return Ok(commit());
|
||||||
|
}
|
||||||
|
store.release_removed_journal_body(&host).map_err(domain)?;
|
||||||
|
for session in store
|
||||||
|
.subject_session_attributions(&attr.subject_id)
|
||||||
|
.map_err(|e| Error::Store(e.to_string()))?
|
||||||
|
.into_iter()
|
||||||
|
.filter(|a| a.runtime_id == target.runtime_id && a.worker_id == target.worker_id)
|
||||||
|
{
|
||||||
|
reconcile_session_jobs(&api, &session, false)?;
|
||||||
|
}
|
||||||
|
// Keep erasure excluded through the catalog commit: otherwise a tombstone
|
||||||
|
// could win between revocation and publication of the retained archive.
|
||||||
|
Ok(commit())
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Deserialize)]
|
#[derive(Deserialize)]
|
||||||
#[serde(deny_unknown_fields)]
|
#[serde(deny_unknown_fields)]
|
||||||
pub(super) struct Empty {}
|
pub(super) struct Empty {}
|
||||||
@@ -2089,7 +2187,28 @@ pub(super) fn reconcile_unknown(
|
|||||||
.as_ref()
|
.as_ref()
|
||||||
.ok_or_else(|| Error::InvalidInput("Journal recovery requires a grant".into()))?;
|
.ok_or_else(|| Error::InvalidInput("Journal recovery requires a grant".into()))?;
|
||||||
let store = store(api)?;
|
let store = store(api)?;
|
||||||
let host = host(api, &store, grant)?;
|
let attr = store
|
||||||
|
.session_attribution(&grant.session_id)
|
||||||
|
.map_err(|e| Error::Store(e.to_string()))?
|
||||||
|
.filter(|a| a.subject_id == grant.subject_id)
|
||||||
|
.ok_or_else(|| Error::WorkspacePermissionDenied("Journal attribution mismatch".into()))?;
|
||||||
|
let released = store
|
||||||
|
.journal_worker_was_released(&attr.subject_id, &attr.runtime_id, &attr.worker_id)
|
||||||
|
.map_err(domain)?;
|
||||||
|
let host = if released {
|
||||||
|
// Recovery reads immutable evidence, never renews a legacy writer grant.
|
||||||
|
let host = JournalHostContext::validated_session(&store, attr.clone()).map_err(domain)?;
|
||||||
|
if !grant.binding_id.is_empty()
|
||||||
|
&& host.writer_binding_id() != Some(grant.binding_id.as_str())
|
||||||
|
{
|
||||||
|
return Err(Error::WorkspacePermissionDenied(
|
||||||
|
"Journal recovery binding mismatch".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
host
|
||||||
|
} else {
|
||||||
|
host(api, &store, grant)?
|
||||||
|
};
|
||||||
let erased = store.journal_session_erased(&host).map_err(domain)?;
|
let erased = store.journal_session_erased(&host).map_err(domain)?;
|
||||||
if !erased {
|
if !erased {
|
||||||
job.load_executable_request()?;
|
job.load_executable_request()?;
|
||||||
@@ -2097,70 +2216,93 @@ pub(super) fn reconcile_unknown(
|
|||||||
job_id: job.request.job_id.clone(),
|
job_id: job.request.job_id.clone(),
|
||||||
attempt_id: attempt.attempt_id.clone(),
|
attempt_id: attempt.attempt_id.clone(),
|
||||||
};
|
};
|
||||||
let g = if legacy_descriptor(&job.request) {
|
let generation = if released {
|
||||||
// Only reconcile already-saved legacy receipts; never redispatch descriptors.
|
|
||||||
store
|
store
|
||||||
.prepare_journal_generation(
|
.retained_journal_generation(&host, &grant.journal_id, &binding)
|
||||||
&host,
|
|
||||||
&grant.journal_id,
|
|
||||||
binding,
|
|
||||||
grant.selected_ranges.clone(),
|
|
||||||
)
|
|
||||||
.map_err(domain)?
|
.map_err(domain)?
|
||||||
|
} else if legacy_descriptor(&job.request) {
|
||||||
|
Some(
|
||||||
|
store
|
||||||
|
.prepare_journal_generation(
|
||||||
|
&host,
|
||||||
|
&grant.journal_id,
|
||||||
|
binding,
|
||||||
|
grant.selected_ranges.clone(),
|
||||||
|
)
|
||||||
|
.map_err(domain)?,
|
||||||
|
)
|
||||||
} else {
|
} else {
|
||||||
let input = saved_input(&job.request, grant)?;
|
// Preserve active-body recovery: its saved intent can still establish
|
||||||
let generation = store
|
// the original generation fence. A released body must only read it.
|
||||||
.prepare_journal_input(&host, &input, binding)
|
Some(
|
||||||
.map_err(domain)?;
|
store
|
||||||
input.validate_generation(&generation).map_err(domain)?;
|
.prepare_journal_input(&host, &saved_input(&job.request, grant)?, binding)
|
||||||
generation
|
.map_err(domain)?,
|
||||||
|
)
|
||||||
};
|
};
|
||||||
let ctx = JournalJobContext::validated_recovery(&store, &host, &g.id, g.job.clone())
|
if let Some(g) = generation {
|
||||||
.map_err(domain)?;
|
if g.selected_ranges != grant.selected_ranges {
|
||||||
match journal::service::finish(&store, &ctx) {
|
return Err(Error::WorkspacePermissionDenied(
|
||||||
Ok(completion) => {
|
"Journal recovery selection mismatch".into(),
|
||||||
let submission = BackendJobResultSubmission {
|
));
|
||||||
job_id: job.request.job_id.clone(),
|
}
|
||||||
attempt_id: attempt.attempt_id.clone(),
|
if !legacy_descriptor(&job.request) {
|
||||||
input_digest: attempt.input_digest.clone(),
|
saved_input(&job.request, grant)?
|
||||||
result: serde_json::to_value(completion)
|
.validate_generation(&g)
|
||||||
.map_err(|e| Error::Store(e.to_string()))?,
|
.map_err(domain)?;
|
||||||
};
|
}
|
||||||
let current = api.store.reconcile_backend_journal_attempt(
|
let ctx = JournalJobContext::validated_recovery(&store, &host, &g.id, g.job.clone())
|
||||||
api.workspace_id(),
|
.map_err(domain)?;
|
||||||
attempt,
|
match journal::service::finish(&store, &ctx) {
|
||||||
Some(&submission),
|
Ok(completion) => {
|
||||||
&Utc::now().to_rfc3339(),
|
let submission = BackendJobResultSubmission {
|
||||||
)?;
|
job_id: job.request.job_id.clone(),
|
||||||
let saved = api
|
attempt_id: attempt.attempt_id.clone(),
|
||||||
.store
|
input_digest: attempt.input_digest.clone(),
|
||||||
.get_backend_job(api.workspace_id(), &job.request.job_id)?
|
result: serde_json::to_value(completion)
|
||||||
.ok_or_else(|| Error::Store("reconciled Job missing".into()))?;
|
.map_err(|e| Error::Store(e.to_string()))?,
|
||||||
notification_delivery::deliver_job_result(
|
};
|
||||||
api,
|
let current = api.store.reconcile_backend_journal_attempt(
|
||||||
&BackendJobResultAcceptance {
|
api.workspace_id(),
|
||||||
job: saved,
|
attempt,
|
||||||
attempt: current.clone(),
|
Some(&submission),
|
||||||
replayed: false,
|
&Utc::now().to_rfc3339(),
|
||||||
},
|
)?;
|
||||||
);
|
let saved = api
|
||||||
if current.worker_cleanup_state == Some(BackendJobWorkerCleanupState::Completed) {
|
.store
|
||||||
drain_after_completion(api, &job.request);
|
.get_backend_job(api.workspace_id(), &job.request.job_id)?
|
||||||
if job.input_disposition == crate::backend_job::JobInputDisposition::Available {
|
.ok_or_else(|| Error::Store("reconciled Job missing".into()))?;
|
||||||
api.store.reclaim_backend_job_input(
|
notification_delivery::deliver_job_result(
|
||||||
api.workspace_id(),
|
api,
|
||||||
&job.request.job_id,
|
&BackendJobResultAcceptance {
|
||||||
&job.input_digest,
|
job: saved,
|
||||||
crate::backend_job::JobInputRetirementReason::Completed,
|
attempt: current.clone(),
|
||||||
)?;
|
replayed: false,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
if current.worker_cleanup_state == Some(BackendJobWorkerCleanupState::Completed)
|
||||||
|
{
|
||||||
|
if !released {
|
||||||
|
drain_after_completion(api, &job.request);
|
||||||
|
}
|
||||||
|
if job.input_disposition
|
||||||
|
== crate::backend_job::JobInputDisposition::Available
|
||||||
|
{
|
||||||
|
api.store.reclaim_backend_job_input(
|
||||||
|
api.workspace_id(),
|
||||||
|
&job.request.job_id,
|
||||||
|
&job.input_digest,
|
||||||
|
crate::backend_job::JobInputRetirementReason::Completed,
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
api.schedule_backend_job_worker_cleanup(¤t);
|
||||||
}
|
}
|
||||||
} else {
|
return Ok(());
|
||||||
api.schedule_backend_job_worker_cleanup(¤t);
|
|
||||||
}
|
}
|
||||||
return Ok(());
|
Err(JournalError::Conflict(_) | JournalError::PermissionDenied(_)) => {}
|
||||||
|
Err(error) => return Err(domain(error)),
|
||||||
}
|
}
|
||||||
Err(JournalError::Conflict(_) | JournalError::PermissionDenied(_)) => {}
|
|
||||||
Err(error) => return Err(domain(error)),
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
if attempt.worker_cleanup_state != Some(BackendJobWorkerCleanupState::Completed) {
|
if attempt.worker_cleanup_state != Some(BackendJobWorkerCleanupState::Completed) {
|
||||||
@@ -2179,11 +2321,17 @@ pub(super) fn reconcile_unknown(
|
|||||||
false
|
false
|
||||||
};
|
};
|
||||||
if !confirmed_purge {
|
if !confirmed_purge {
|
||||||
|
if released {
|
||||||
|
// Source deletion already fenced this generation. An unbound Unknown
|
||||||
|
// still lacks execution-cleanup proof: preserve Unknown, but it no
|
||||||
|
// longer owns the Subject or blocks removal of the original body.
|
||||||
|
return Ok(());
|
||||||
|
}
|
||||||
return Err(Error::RepositoryConflict(
|
return Err(Error::RepositoryConflict(
|
||||||
"Unknown Journal attempt has no committed stop/Purge fence; not retried".into(),
|
"Unknown Journal attempt has no committed stop/Purge fence; not retried".into(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
if !erased {
|
if !erased && !released {
|
||||||
close(api, &job.request, attempt)?;
|
close(api, &job.request, attempt)?;
|
||||||
}
|
}
|
||||||
api.store.reconcile_backend_journal_attempt(
|
api.store.reconcile_backend_journal_attempt(
|
||||||
@@ -2192,7 +2340,7 @@ pub(super) fn reconcile_unknown(
|
|||||||
None,
|
None,
|
||||||
&Utc::now().to_rfc3339(),
|
&Utc::now().to_rfc3339(),
|
||||||
)?;
|
)?;
|
||||||
if !erased {
|
if !erased && !released {
|
||||||
// Resume the same saved body input only after canonical execution/domain fences.
|
// Resume the same saved body input only after canonical execution/domain fences.
|
||||||
// dispatch selects its retained intent before it can capture any newer source.
|
// dispatch selects its retained intent before it can capture any newer source.
|
||||||
let attr = store
|
let attr = store
|
||||||
@@ -2209,6 +2357,10 @@ fn reconcile_session_jobs(
|
|||||||
attr: &crate::subjektiv::SubjectSessionAttribution,
|
attr: &crate::subjektiv::SubjectSessionAttribution,
|
||||||
erased: bool,
|
erased: bool,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
|
let released = !erased
|
||||||
|
&& store(api)?
|
||||||
|
.journal_worker_was_released(&attr.subject_id, &attr.runtime_id, &attr.worker_id)
|
||||||
|
.map_err(domain)?;
|
||||||
let mut after: Option<String> = None;
|
let mut after: Option<String> = None;
|
||||||
loop {
|
loop {
|
||||||
let jobs = api.store.list_backend_journal_jobs(
|
let jobs = api.store.list_backend_journal_jobs(
|
||||||
@@ -2223,6 +2375,29 @@ fn reconcile_session_jobs(
|
|||||||
.store
|
.store
|
||||||
.get_backend_job_attempt(api.workspace_id(), &job.request.job_id, &id)?
|
.get_backend_job_attempt(api.workspace_id(), &job.request.job_id, &id)?
|
||||||
.ok_or_else(|| Error::Store("Journal current attempt missing".into()))?;
|
.ok_or_else(|| Error::Store("Journal current attempt missing".into()))?;
|
||||||
|
if released {
|
||||||
|
// Writer revocation has already fenced all Journal mutations. End
|
||||||
|
// pending/Unknown attempts without claiming their input was reflected;
|
||||||
|
// saved input, operations and original source remain retained.
|
||||||
|
if !attempt.state.terminal() {
|
||||||
|
attempt = api.store.finish_backend_job_attempt(
|
||||||
|
api.workspace_id(),
|
||||||
|
&job.request.job_id,
|
||||||
|
&id,
|
||||||
|
BackendJobAttemptState::Failed,
|
||||||
|
"journal_body_released",
|
||||||
|
"Source Worker was removed; unfinished Journal history retained",
|
||||||
|
&Utc::now().to_rfc3339(),
|
||||||
|
)?;
|
||||||
|
}
|
||||||
|
api.schedule_backend_job_worker_cleanup(&attempt);
|
||||||
|
if attempt.state == BackendJobAttemptState::Unknown {
|
||||||
|
// A saved checkpoint may settle success; otherwise retain Unknown
|
||||||
|
// until canonical Job-worker cleanup has supplied its Purge fence.
|
||||||
|
reconcile_unknown(api, job, &attempt)?;
|
||||||
|
}
|
||||||
|
continue;
|
||||||
|
}
|
||||||
if !erased
|
if !erased
|
||||||
&& legacy_descriptor(&job.request)
|
&& legacy_descriptor(&job.request)
|
||||||
&& job.input_disposition == crate::backend_job::JobInputDisposition::Available
|
&& job.input_disposition == crate::backend_job::JobInputDisposition::Available
|
||||||
|
|||||||
@@ -11,6 +11,20 @@ use worker_runtime::worker_backend::{
|
|||||||
WorkerRuntimeExecutionBackend,
|
WorkerRuntimeExecutionBackend,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
pub(super) fn connected_restore_tempdir() -> tempfile::TempDir {
|
||||||
|
crate::restore_test_environment::require_restore_test_environment();
|
||||||
|
let checkout = Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||||
|
.parent()
|
||||||
|
.unwrap()
|
||||||
|
.parent()
|
||||||
|
.unwrap();
|
||||||
|
tempfile::Builder::new()
|
||||||
|
.prefix("c")
|
||||||
|
.rand_bytes(3)
|
||||||
|
.tempdir_in(checkout.join("target"))
|
||||||
|
.unwrap()
|
||||||
|
}
|
||||||
|
|
||||||
const WATCHDOG: Duration = Duration::from_secs(45);
|
const WATCHDOG: Duration = Duration::from_secs(45);
|
||||||
|
|
||||||
#[derive(Clone)]
|
#[derive(Clone)]
|
||||||
@@ -82,24 +96,91 @@ impl RuntimeWorkerFactory for CapturingFactory {
|
|||||||
) -> std::result::Result<(), String> {
|
) -> std::result::Result<(), String> {
|
||||||
self.inner.preflight_restore(request).await
|
self.inner.preflight_restore(request).await
|
||||||
}
|
}
|
||||||
async fn restore_controller(
|
async fn recover_restore_source(
|
||||||
|
&self,
|
||||||
|
request: &worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||||
|
) -> std::result::Result<bool, worker_runtime::worker_backend::RuntimeRestoreFailure> {
|
||||||
|
self.inner.recover_restore_source(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn prepare_restore(
|
||||||
&self,
|
&self,
|
||||||
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||||
) -> std::result::Result<RuntimeWorkerController, String> {
|
) -> std::result::Result<
|
||||||
|
Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||||
|
> {
|
||||||
let worker_ref = request.worker_ref.clone();
|
let worker_ref = request.worker_ref.clone();
|
||||||
let controller = self.inner.restore_controller(request).await?;
|
let inner = self.inner.prepare_restore(request).await?;
|
||||||
|
Ok(Box::new(CapturingCandidate {
|
||||||
|
inner,
|
||||||
|
worker_ref,
|
||||||
|
captures: self.captures.clone(),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn cleanup_failed_restore(
|
||||||
|
&self,
|
||||||
|
request: &worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||||
|
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
|
||||||
|
self.inner.cleanup_failed_restore(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn reconcile_stopped_worker(
|
||||||
|
&self,
|
||||||
|
worker_ref: &worker_runtime::identity::WorkerRef,
|
||||||
|
) -> std::result::Result<(), String> {
|
||||||
|
self.inner.reconcile_stopped_worker(worker_ref).await
|
||||||
|
}
|
||||||
|
|
||||||
|
fn activate_restored_controller(
|
||||||
|
&self,
|
||||||
|
worker_ref: &worker_runtime::identity::WorkerRef,
|
||||||
|
workspace_id: Option<&str>,
|
||||||
|
handle: &worker::WorkerHandle,
|
||||||
|
) -> std::result::Result<(), String> {
|
||||||
|
self.inner
|
||||||
|
.activate_restored_controller(worker_ref, workspace_id, handle)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct CapturingCandidate {
|
||||||
|
inner: Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
|
||||||
|
worker_ref: worker_runtime::identity::WorkerRef,
|
||||||
|
captures: Arc<Captures>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait::async_trait]
|
||||||
|
impl worker_runtime::worker_backend::RuntimeRestoreCandidate for CapturingCandidate {
|
||||||
|
fn initial_state(&self) -> protocol::WorkerStateSnapshot {
|
||||||
|
self.inner.initial_state()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn activate(
|
||||||
|
&mut self,
|
||||||
|
) -> std::result::Result<
|
||||||
|
RuntimeWorkerController,
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||||
|
> {
|
||||||
|
let controller = self.inner.activate().await?;
|
||||||
self.captures
|
self.captures
|
||||||
.handles
|
.handles
|
||||||
.lock()
|
.lock()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.push((worker_ref.clone(), controller.handle.clone()));
|
.push((self.worker_ref.clone(), controller.handle.clone()));
|
||||||
self.captures
|
self.captures
|
||||||
.restored_clients
|
.restored_clients
|
||||||
.lock()
|
.lock()
|
||||||
.unwrap()
|
.unwrap()
|
||||||
.push((worker_ref, controller.workspace_client.clone()));
|
.push((self.worker_ref.clone(), controller.workspace_client.clone()));
|
||||||
Ok(controller)
|
Ok(controller)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async fn discard(
|
||||||
|
&mut self,
|
||||||
|
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
|
||||||
|
self.inner.discard().await
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
struct Resources {
|
struct Resources {
|
||||||
@@ -259,7 +340,7 @@ async fn call(tools: &BTreeMap<String, Arc<dyn Tool>>, name: &str, args: Value)
|
|||||||
|
|
||||||
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
#[tokio::test(flavor = "multi_thread", worker_threads = 4)]
|
||||||
async fn restored_runtime_parent_lists_and_inputs_existing_child() {
|
async fn restored_runtime_parent_lists_and_inputs_existing_child() {
|
||||||
let root = tempfile::tempdir().unwrap();
|
let root = connected_restore_tempdir();
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
let base = format!("http://{}", listener.local_addr().unwrap());
|
let base = format!("http://{}", listener.local_addr().unwrap());
|
||||||
let mut config = test_server_config(root.path());
|
let mut config = test_server_config(root.path());
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -18,6 +18,57 @@ const EVIDENCE_MR: &str = "ticket-evidence-mr";
|
|||||||
const EVIDENCE_SOURCE: &str = "immutable-approved-source";
|
const EVIDENCE_SOURCE: &str = "immutable-approved-source";
|
||||||
const EVIDENCE_TARGET: &str = "immutable-target-result";
|
const EVIDENCE_TARGET: &str = "immutable-target-result";
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn integration_preserves_legacy_empty_post_ref_and_exposes_new_ref_with_server_schema() {
|
||||||
|
let fixture = EvidenceApiFixture::new().await;
|
||||||
|
let approval = fixture.approve("integration-post-ref");
|
||||||
|
let connection = Connection::open(&fixture.api.config.database_path).unwrap();
|
||||||
|
let legacy_index: i64 = connection.query_row(
|
||||||
|
"SELECT COALESCE(MAX(event_index),-1)+1 FROM typed_ticket_events WHERE workspace_id=?1 AND ticket_id=?2",
|
||||||
|
params![TEST_WORKSPACE_ID, fixture.ticket_id], |row| row.get(0),
|
||||||
|
).unwrap();
|
||||||
|
connection.execute(
|
||||||
|
"INSERT INTO typed_ticket_events(workspace_id,ticket_id,event_index,event_ref,kind,body)
|
||||||
|
VALUES(?1,?2,?3,'','comment','Legacy integration')",
|
||||||
|
params![TEST_WORKSPACE_ID, fixture.ticket_id, legacy_index],
|
||||||
|
).unwrap();
|
||||||
|
|
||||||
|
let merged = fixture.integrate(&approval);
|
||||||
|
assert_eq!(fixture.integrate(&approval), merged);
|
||||||
|
let detail = fixture
|
||||||
|
.backend
|
||||||
|
.show(fixture.ticket_id.clone().into())
|
||||||
|
.unwrap();
|
||||||
|
let legacy = detail
|
||||||
|
.events
|
||||||
|
.iter()
|
||||||
|
.find(|event| event.body.as_str() == "Legacy integration")
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(legacy.event_ref, "");
|
||||||
|
let integrations = detail
|
||||||
|
.events
|
||||||
|
.iter()
|
||||||
|
.filter(|event| {
|
||||||
|
event.attributes.get("merge_request_id").map(String::as_str) == Some(EVIDENCE_MR)
|
||||||
|
})
|
||||||
|
.collect::<Vec<_>>();
|
||||||
|
assert_eq!(
|
||||||
|
integrations.len(),
|
||||||
|
1,
|
||||||
|
"completion replay must not duplicate the Ticket post"
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
integrations[0].event_ref,
|
||||||
|
format!("{}:{}", fixture.ticket_id, legacy_index + 1)
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
integrations[0].attributes["operation_id"],
|
||||||
|
"evidence-integration"
|
||||||
|
);
|
||||||
|
let shown = fixture.show().await;
|
||||||
|
assert_eq!(shown.merge_requests[0].state, "merged");
|
||||||
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn ticket_browse_does_not_observe_sources_but_authoritative_query_and_show_still_do() {
|
async fn ticket_browse_does_not_observe_sources_but_authoritative_query_and_show_still_do() {
|
||||||
struct CountingObserver(Arc<std::sync::atomic::AtomicUsize>);
|
struct CountingObserver(Arc<std::sync::atomic::AtomicUsize>);
|
||||||
|
|||||||
@@ -652,6 +652,7 @@ async fn source_workdir_removal_keeps_published_drive_bytes_and_receipt() {
|
|||||||
&removal_id,
|
&removal_id,
|
||||||
"account:drive-test",
|
"account:drive-test",
|
||||||
"source no longer needed",
|
"source no longer needed",
|
||||||
|
"drive-source-removal",
|
||||||
)
|
)
|
||||||
})
|
})
|
||||||
.await
|
.await
|
||||||
|
|||||||
@@ -476,6 +476,7 @@ impl AttentionWorkerFactory {
|
|||||||
shutdown: Arc::new(tokio::sync::Mutex::new(Some(started.shutdown))),
|
shutdown: Arc::new(tokio::sync::Mutex::new(Some(started.shutdown))),
|
||||||
controller_task: started.controller_task,
|
controller_task: started.controller_task,
|
||||||
workspace_client,
|
workspace_client,
|
||||||
|
restore_streams: None,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -489,12 +490,61 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for AttentionWorkerFac
|
|||||||
self.controller(request.worker_ref.worker_id.to_string(), false)
|
self.controller(request.worker_ref.worker_id.to_string(), false)
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
async fn restore_controller(
|
async fn prepare_restore(
|
||||||
&self,
|
&self,
|
||||||
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||||
) -> std::result::Result<worker_runtime::worker_backend::RuntimeWorkerController, String> {
|
) -> std::result::Result<
|
||||||
self.controller(request.worker_ref.worker_id.to_string(), true)
|
Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
|
||||||
.await
|
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||||
|
> {
|
||||||
|
Ok(Box::new(AttentionRestoreCandidate {
|
||||||
|
inputs: Some((
|
||||||
|
AttentionWorkerFactory {
|
||||||
|
root: self.root.clone(),
|
||||||
|
},
|
||||||
|
request.worker_ref.worker_id.to_string(),
|
||||||
|
)),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Preparation owns only fixture inputs; reconstruction and Controller startup
|
||||||
|
// remain lazy so discard never needs to shut down a started Worker.
|
||||||
|
struct AttentionRestoreCandidate {
|
||||||
|
inputs: Option<(AttentionWorkerFactory, String)>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait::async_trait]
|
||||||
|
impl worker_runtime::worker_backend::RuntimeRestoreCandidate for AttentionRestoreCandidate {
|
||||||
|
fn initial_state(&self) -> protocol::WorkerStateSnapshot {
|
||||||
|
protocol::WorkerStateSnapshot::initial()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn activate(
|
||||||
|
&mut self,
|
||||||
|
) -> std::result::Result<
|
||||||
|
worker_runtime::worker_backend::RuntimeWorkerController,
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||||
|
> {
|
||||||
|
let (factory, worker_id) = self.inputs.take().ok_or_else(|| {
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreFailure::refused(
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreStage::Controller,
|
||||||
|
"attention fixture candidate already consumed",
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
factory.controller(worker_id, true).await.map_err(|_| {
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreFailure::unproven(
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreStage::Reconstruction,
|
||||||
|
"attention fixture reconstruction failed",
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn discard(
|
||||||
|
&mut self,
|
||||||
|
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
|
||||||
|
self.inputs.take();
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -601,7 +651,7 @@ async fn orchestrator_attention_real_worker_lost_response_retry_commits_once() {
|
|||||||
|
|
||||||
async fn real_attention_worker_start_restore(lose_response: bool) {
|
async fn real_attention_worker_start_restore(lose_response: bool) {
|
||||||
use session_store::{LogEntry, Store, WorkerSessionStore};
|
use session_store::{LogEntry, Store, WorkerSessionStore};
|
||||||
let dir = tempfile::tempdir().unwrap();
|
let dir = restored_parent_control_tests::connected_restore_tempdir();
|
||||||
init_clean_git_workspace(dir.path());
|
init_clean_git_workspace(dir.path());
|
||||||
let config = test_server_config(dir.path());
|
let config = test_server_config(dir.path());
|
||||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap());
|
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap());
|
||||||
|
|||||||
@@ -14,17 +14,37 @@ async fn notification_diagnostics_http_auth_scope_and_readonly_evidence() {
|
|||||||
seed_test_api_token(api.store.as_ref(), TEST_WORKSPACE_ID);
|
seed_test_api_token(api.store.as_ref(), TEST_WORKSPACE_ID);
|
||||||
let ticket = delivery_ticket(&api, "Notification diagnostics auth fixture");
|
let ticket = delivery_ticket(&api, "Notification diagnostics auth fixture");
|
||||||
delivery_assign(&api, ticket.id.as_str(), "diagnostics-assignment").await;
|
delivery_assign(&api, ticket.id.as_str(), "diagnostics-assignment").await;
|
||||||
let path = format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?ticket={}&limit=1",ticket.id.as_str());
|
let path = format!(
|
||||||
let page = delivery_http(&api,"GET",&path,None,StatusCode::OK).await;
|
"/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?ticket={}&limit=1",
|
||||||
assert_eq!(page["items"].as_array().unwrap().len(),1);
|
ticket.id.as_str()
|
||||||
assert_eq!(page["items"][0]["model_processing_confirmed"],false);
|
);
|
||||||
|
let page = delivery_http(&api, "GET", &path, None, StatusCode::OK).await;
|
||||||
|
assert_eq!(page["items"].as_array().unwrap().len(), 1);
|
||||||
|
assert_eq!(page["items"][0]["model_processing_confirmed"], false);
|
||||||
assert!(page["items"][0].get("content").is_none());
|
assert!(page["items"][0].get("content").is_none());
|
||||||
let again=delivery_http(&api,"GET",&path,None,StatusCode::OK).await;
|
let again = delivery_http(&api, "GET", &path, None, StatusCode::OK).await;
|
||||||
assert_eq!(again,page);
|
assert_eq!(again, page);
|
||||||
let response=build_router(api.clone()).oneshot(Request::builder().uri(&path).body(Body::empty()).unwrap()).await.unwrap();
|
let response = build_router(api.clone())
|
||||||
assert_eq!(response.status(),StatusCode::UNAUTHORIZED);
|
.oneshot(Request::builder().uri(&path).body(Body::empty()).unwrap())
|
||||||
delivery_http(&api,"GET","/api/w/other-workspace/notification-diagnostics",None,StatusCode::NOT_FOUND).await;
|
.await
|
||||||
delivery_http(&api,"GET",&format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?limit=101"),None,StatusCode::BAD_REQUEST).await;
|
.unwrap();
|
||||||
|
assert_eq!(response.status(), StatusCode::UNAUTHORIZED);
|
||||||
|
delivery_http(
|
||||||
|
&api,
|
||||||
|
"GET",
|
||||||
|
"/api/w/other-workspace/notification-diagnostics",
|
||||||
|
None,
|
||||||
|
StatusCode::NOT_FOUND,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
delivery_http(
|
||||||
|
&api,
|
||||||
|
"GET",
|
||||||
|
&format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?limit=101"),
|
||||||
|
None,
|
||||||
|
StatusCode::BAD_REQUEST,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
}
|
}
|
||||||
|
|
||||||
fn delivery_assignment_path(ticket: &str) -> String {
|
fn delivery_assignment_path(ticket: &str) -> String {
|
||||||
@@ -814,6 +834,7 @@ impl DeliveryWorkerFactory {
|
|||||||
shutdown: Arc::new(tokio::sync::Mutex::new(Some(started.shutdown))),
|
shutdown: Arc::new(tokio::sync::Mutex::new(Some(started.shutdown))),
|
||||||
controller_task: started.controller_task,
|
controller_task: started.controller_task,
|
||||||
workspace_client,
|
workspace_client,
|
||||||
|
restore_streams: None,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -827,12 +848,62 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for DeliveryWorkerFact
|
|||||||
self.controller(request.worker_ref.worker_id.to_string(), false)
|
self.controller(request.worker_ref.worker_id.to_string(), false)
|
||||||
.await
|
.await
|
||||||
}
|
}
|
||||||
async fn restore_controller(
|
async fn prepare_restore(
|
||||||
&self,
|
&self,
|
||||||
request: delivery_execution::WorkerExecutionRestoreRequest,
|
request: delivery_execution::WorkerExecutionRestoreRequest,
|
||||||
) -> std::result::Result<worker_runtime::worker_backend::RuntimeWorkerController, String> {
|
) -> std::result::Result<
|
||||||
self.controller(request.worker_ref.worker_id.to_string(), true)
|
Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
|
||||||
.await
|
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||||
|
> {
|
||||||
|
Ok(Box::new(DeliveryRestoreCandidate {
|
||||||
|
inputs: Some((
|
||||||
|
DeliveryWorkerFactory {
|
||||||
|
root: self.root.clone(),
|
||||||
|
client: self.client.clone(),
|
||||||
|
},
|
||||||
|
request.worker_ref.worker_id.to_string(),
|
||||||
|
)),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Retain the same scripted client/receipt store without opening either store or
|
||||||
|
// starting a Controller until activation. Discard drops only these owned inputs.
|
||||||
|
struct DeliveryRestoreCandidate {
|
||||||
|
inputs: Option<(DeliveryWorkerFactory, String)>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait::async_trait]
|
||||||
|
impl worker_runtime::worker_backend::RuntimeRestoreCandidate for DeliveryRestoreCandidate {
|
||||||
|
fn initial_state(&self) -> protocol::WorkerStateSnapshot {
|
||||||
|
protocol::WorkerStateSnapshot::initial()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn activate(
|
||||||
|
&mut self,
|
||||||
|
) -> std::result::Result<
|
||||||
|
worker_runtime::worker_backend::RuntimeWorkerController,
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||||
|
> {
|
||||||
|
let (factory, worker_id) = self.inputs.take().ok_or_else(|| {
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreFailure::refused(
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreStage::Controller,
|
||||||
|
"delivery fixture candidate already consumed",
|
||||||
|
)
|
||||||
|
})?;
|
||||||
|
factory.controller(worker_id, true).await.map_err(|_| {
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreFailure::unproven(
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreStage::Reconstruction,
|
||||||
|
"delivery fixture reconstruction failed",
|
||||||
|
)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn discard(
|
||||||
|
&mut self,
|
||||||
|
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
|
||||||
|
self.inputs.take();
|
||||||
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -931,6 +1002,7 @@ async fn delivery_real_fixture(
|
|||||||
Arc<DeliveryResponseLossBackend>,
|
Arc<DeliveryResponseLossBackend>,
|
||||||
DeliveryScriptedClient,
|
DeliveryScriptedClient,
|
||||||
) {
|
) {
|
||||||
|
let _environment = restored_parent_control_tests::connected_restore_tempdir();
|
||||||
let config = test_server_config(root);
|
let config = test_server_config(root);
|
||||||
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap());
|
let store = Arc::new(SqliteWorkspaceStore::open_for_test(&config.database_path).unwrap());
|
||||||
seed_test_registered_workspace(store.as_ref(), &config)
|
seed_test_registered_workspace(store.as_ref(), &config)
|
||||||
@@ -1136,9 +1208,22 @@ async fn assignment_real_worker_response_loss_restore_and_concurrent_scans_commi
|
|||||||
"uncertain delivery changed request id or payload: {attempts:?}"
|
"uncertain delivery changed request id or payload: {attempts:?}"
|
||||||
);
|
);
|
||||||
assert_eq!(delivery_receipts(&api, &worker).len(), 1);
|
assert_eq!(delivery_receipts(&api, &worker).len(), 1);
|
||||||
let unknown = api.store.notification_diagnostics(TEST_WORKSPACE_ID, &server_api::NotificationDiagnosticsQuery {
|
let unknown = api
|
||||||
assignment_id: Some(assignment.into()), ..Default::default()
|
.store
|
||||||
}).unwrap().items.into_iter().find(|item| item.stage == server_api::NotificationDiagnosticStage::AcceptanceUnknown).expect("saved unknown acceptance must remain visible while Runtime already has its receipt");
|
.notification_diagnostics(
|
||||||
|
TEST_WORKSPACE_ID,
|
||||||
|
&server_api::NotificationDiagnosticsQuery {
|
||||||
|
assignment_id: Some(assignment.into()),
|
||||||
|
..Default::default()
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap()
|
||||||
|
.items
|
||||||
|
.into_iter()
|
||||||
|
.find(|item| item.stage == server_api::NotificationDiagnosticStage::AcceptanceUnknown)
|
||||||
|
.expect(
|
||||||
|
"saved unknown acceptance must remain visible while Runtime already has its receipt",
|
||||||
|
);
|
||||||
assert!(!unknown.model_processing_confirmed);
|
assert!(!unknown.model_processing_confirmed);
|
||||||
// Change mutable rendering context while the acknowledgement is unknown.
|
// Change mutable rendering context while the acknowledgement is unknown.
|
||||||
// A new self-report is not a new input and cannot change the frozen retry.
|
// A new self-report is not a new input and cannot change the frozen retry.
|
||||||
@@ -1199,11 +1284,24 @@ async fn assignment_real_worker_response_loss_restore_and_concurrent_scans_commi
|
|||||||
}
|
}
|
||||||
delivery_wait_idle(&reopened, &worker, 2).await;
|
delivery_wait_idle(&reopened, &worker, 2).await;
|
||||||
assert_eq!(delivery_receipts(&reopened, &worker).len(), 2);
|
assert_eq!(delivery_receipts(&reopened, &worker).len(), 2);
|
||||||
let evidence = delivery_http(&reopened,"GET",&format!("/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?source_id={}",unknown.source_id),None,StatusCode::OK).await;
|
let evidence = delivery_http(
|
||||||
assert_eq!(evidence["items"][0]["stage"],"runtime_accepted");
|
&reopened,
|
||||||
assert_eq!(evidence["items"][0]["delivery_id"],json!(unknown.delivery_id));
|
"GET",
|
||||||
assert_eq!(evidence["items"][0]["model_processing_confirmed"],false);
|
&format!(
|
||||||
assert_eq!(evidence["items"][0]["retry_candidate"],false);
|
"/api/w/{TEST_WORKSPACE_ID}/notification-diagnostics?source_id={}",
|
||||||
|
unknown.source_id
|
||||||
|
),
|
||||||
|
None,
|
||||||
|
StatusCode::OK,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(evidence["items"][0]["stage"], "runtime_accepted");
|
||||||
|
assert_eq!(
|
||||||
|
evidence["items"][0]["delivery_id"],
|
||||||
|
json!(unknown.delivery_id)
|
||||||
|
);
|
||||||
|
assert_eq!(evidence["items"][0]["model_processing_confirmed"], false);
|
||||||
|
assert_eq!(evidence["items"][0]["retry_candidate"], false);
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
delivery_commits(&reopened, &worker)
|
delivery_commits(&reopened, &worker)
|
||||||
.iter()
|
.iter()
|
||||||
|
|||||||
@@ -0,0 +1,399 @@
|
|||||||
|
// Integration boundary: generated HTTP admission -> real WebAuthn verifier ->
|
||||||
|
// SQLite credential/receipt transaction -> real session issuance. Only the
|
||||||
|
// authenticator is synthetic; failures are SQLite triggers, not success fakes.
|
||||||
|
|
||||||
|
impl Fixture {
|
||||||
|
async fn addition_result(&self, cookie: &str, challenge_id: &str) -> (StatusCode, Value) {
|
||||||
|
let mut request = self.expected_owner(Some(cookie), None).await;
|
||||||
|
request["challenge_id"] = challenge_id.into();
|
||||||
|
let (status, _, body) = self
|
||||||
|
.post(
|
||||||
|
"/api/auth/passkeys/addition/result",
|
||||||
|
Some(cookie),
|
||||||
|
Some(ORIGIN_URL),
|
||||||
|
request,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
(status, body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn addition_receipt_is_atomic_and_recovers_credential_commit_not_session_success() {
|
||||||
|
for failure in [
|
||||||
|
"before_credential",
|
||||||
|
"receipt_insert",
|
||||||
|
"session_insert",
|
||||||
|
"lost_response",
|
||||||
|
"lost_body_accepted_cookie",
|
||||||
|
] {
|
||||||
|
let f = Fixture::new().await;
|
||||||
|
let mut original = authenticator();
|
||||||
|
let (cookie, owner) = f.enroll("registration", None, "owner", &mut original).await;
|
||||||
|
let original_expectation = f.expected_owner(Some(&cookie), None).await;
|
||||||
|
let options = f.options("addition", Some(&cookie), "").await;
|
||||||
|
let challenge_id = options.challenge_id.clone();
|
||||||
|
let mut added = authenticator();
|
||||||
|
let request = Fixture::credential(options, &mut added);
|
||||||
|
let (status, result) = f.addition_result(&cookie, &challenge_id).await;
|
||||||
|
assert_eq!(status, StatusCode::OK);
|
||||||
|
assert_eq!(result["status"], "unknown");
|
||||||
|
let table = match failure {
|
||||||
|
"before_credential" => Some("passkey_credentials"),
|
||||||
|
"receipt_insert" => Some("passkey_addition_receipts"),
|
||||||
|
"session_insert" => Some("browser_sessions"),
|
||||||
|
_ => None,
|
||||||
|
};
|
||||||
|
if let Some(table) = table {
|
||||||
|
f.api.config_store.with_conn(|conn| {
|
||||||
|
conn.execute_batch(&format!("CREATE TRIGGER inject_passkey_failure BEFORE INSERT ON {table} BEGIN SELECT RAISE(ABORT, 'synthetic failure'); END;"))?;
|
||||||
|
Ok(())
|
||||||
|
}).unwrap();
|
||||||
|
}
|
||||||
|
// Discard the completion body; one variant still accepts Set-Cookie.
|
||||||
|
let (status, headers, _) = f
|
||||||
|
.post(
|
||||||
|
"/api/auth/passkeys/addition/complete",
|
||||||
|
Some(&cookie),
|
||||||
|
Some(ORIGIN_URL),
|
||||||
|
request.clone(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(
|
||||||
|
status,
|
||||||
|
if matches!(failure, "lost_response" | "lost_body_accepted_cookie") {
|
||||||
|
StatusCode::OK
|
||||||
|
} else {
|
||||||
|
StatusCode::INTERNAL_SERVER_ERROR
|
||||||
|
}
|
||||||
|
);
|
||||||
|
if !matches!(failure, "lost_response" | "lost_body_accepted_cookie") {
|
||||||
|
assert!(!headers.contains_key(axum::http::header::SET_COOKIE));
|
||||||
|
}
|
||||||
|
let saved = matches!(
|
||||||
|
failure,
|
||||||
|
"session_insert" | "lost_response" | "lost_body_accepted_cookie"
|
||||||
|
);
|
||||||
|
if failure == "lost_body_accepted_cookie" {
|
||||||
|
let accepted_cookie = headers[axum::http::header::SET_COOKIE]
|
||||||
|
.to_str()
|
||||||
|
.unwrap()
|
||||||
|
.split(';')
|
||||||
|
.next()
|
||||||
|
.unwrap();
|
||||||
|
let current = f.expected_owner(Some(accepted_cookie), None).await;
|
||||||
|
assert!(
|
||||||
|
current["expected_authentication_reference"]
|
||||||
|
!= original_expectation["expected_authentication_reference"],
|
||||||
|
"addition issued a distinct session"
|
||||||
|
);
|
||||||
|
let mut recovery = original_expectation.clone();
|
||||||
|
recovery["challenge_id"] = challenge_id.clone().into();
|
||||||
|
let (status, _, result) = f
|
||||||
|
.post(
|
||||||
|
"/api/auth/passkeys/addition/result",
|
||||||
|
Some(accepted_cookie),
|
||||||
|
Some(ORIGIN_URL),
|
||||||
|
recovery,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(status, StatusCode::OK);
|
||||||
|
assert_eq!(result["status"], "saved");
|
||||||
|
assert_eq!(result["user"]["user_id"], owner.user_id);
|
||||||
|
}
|
||||||
|
let (status, result) = f.addition_result(&cookie, &challenge_id).await;
|
||||||
|
assert_eq!(status, StatusCode::OK);
|
||||||
|
assert_eq!(result["status"], if saved { "saved" } else { "unknown" });
|
||||||
|
assert_eq!(f.credentials(&owner).len(), if saved { 2 } else { 1 });
|
||||||
|
let receipt_count = f
|
||||||
|
.api
|
||||||
|
.config_store
|
||||||
|
.with_conn(|conn| {
|
||||||
|
Ok(conn.query_row(
|
||||||
|
"SELECT COUNT(*) FROM passkey_addition_receipts",
|
||||||
|
[],
|
||||||
|
|row| row.get::<_, u64>(0),
|
||||||
|
)?)
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(receipt_count, u64::from(saved));
|
||||||
|
if saved {
|
||||||
|
assert_eq!(result["user"]["user_id"], owner.user_id);
|
||||||
|
let (_, listed) = f
|
||||||
|
.get_authenticated("/api/auth/passkeys", Some(&cookie), None)
|
||||||
|
.await;
|
||||||
|
let reference = result["key"]["key_reference"].as_str().unwrap();
|
||||||
|
assert!(
|
||||||
|
listed["keys"]
|
||||||
|
.as_array()
|
||||||
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.any(|key| key["key_reference"] == reference)
|
||||||
|
);
|
||||||
|
assert!(result["key"]["normal_login_verified_at"].is_null());
|
||||||
|
}
|
||||||
|
if table.is_some() {
|
||||||
|
f.api
|
||||||
|
.config_store
|
||||||
|
.with_conn(|conn| {
|
||||||
|
conn.execute_batch("DROP TRIGGER inject_passkey_failure")?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
// Consumed completion is not retry authority, even after a transaction failure.
|
||||||
|
let (status, headers, _) = f
|
||||||
|
.post(
|
||||||
|
"/api/auth/passkeys/addition/complete",
|
||||||
|
Some(&cookie),
|
||||||
|
Some(ORIGIN_URL),
|
||||||
|
request,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(status, StatusCode::BAD_REQUEST);
|
||||||
|
assert!(!headers.contains_key(axum::http::header::SET_COOKIE));
|
||||||
|
assert_eq!(f.credentials(&owner).len(), if saved { 2 } else { 1 });
|
||||||
|
if saved {
|
||||||
|
assert_eq!(f.discoverable_login(&mut added).await, owner);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn stale_displayed_owner_or_session_cannot_start_addition_or_read_another_binding_result() {
|
||||||
|
let f = Fixture::new().await;
|
||||||
|
let mut alice_key = authenticator();
|
||||||
|
let (alice_cookie, alice) = f
|
||||||
|
.enroll("registration", None, "alice", &mut alice_key)
|
||||||
|
.await;
|
||||||
|
let (bob_cookie, _) = f
|
||||||
|
.enroll("registration", None, "bob", &mut authenticator())
|
||||||
|
.await;
|
||||||
|
let expected = f.expected_owner(Some(&alice_cookie), None).await;
|
||||||
|
let (replacement_cookie, _) = f.login_session("alice", &mut alice_key).await;
|
||||||
|
let count_challenges = || {
|
||||||
|
f.api
|
||||||
|
.config_store
|
||||||
|
.with_conn(|conn| {
|
||||||
|
Ok(
|
||||||
|
conn.query_row("SELECT COUNT(*) FROM auth_challenges", [], |row| {
|
||||||
|
row.get::<_, u64>(0)
|
||||||
|
})?,
|
||||||
|
)
|
||||||
|
})
|
||||||
|
.unwrap()
|
||||||
|
};
|
||||||
|
let before = count_challenges();
|
||||||
|
for cookie in [&bob_cookie, &replacement_cookie] {
|
||||||
|
let (status, _, _) = f
|
||||||
|
.post(
|
||||||
|
"/api/auth/passkeys/addition/options",
|
||||||
|
Some(cookie),
|
||||||
|
Some(ORIGIN_URL),
|
||||||
|
expected.clone(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(status, StatusCode::BAD_REQUEST);
|
||||||
|
assert_eq!(
|
||||||
|
count_challenges(),
|
||||||
|
before,
|
||||||
|
"stale options must not persist a ceremony"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let options = f.options("addition", Some(&alice_cookie), "").await;
|
||||||
|
let challenge_id = options.challenge_id.clone();
|
||||||
|
let request = Fixture::credential(options, &mut authenticator());
|
||||||
|
assert_eq!(
|
||||||
|
f.post(
|
||||||
|
"/api/auth/passkeys/addition/complete",
|
||||||
|
Some(&alice_cookie),
|
||||||
|
Some(ORIGIN_URL),
|
||||||
|
request
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.0,
|
||||||
|
StatusCode::OK
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
f.addition_result(&alice_cookie, &challenge_id).await.1["status"],
|
||||||
|
"saved"
|
||||||
|
);
|
||||||
|
// Current owner authorization is separate from original operation binding.
|
||||||
|
let mut recovery = expected.clone();
|
||||||
|
recovery["challenge_id"] = challenge_id.clone().into();
|
||||||
|
let (status, _, result) = f
|
||||||
|
.post(
|
||||||
|
"/api/auth/passkeys/addition/result",
|
||||||
|
Some(&replacement_cookie),
|
||||||
|
Some(ORIGIN_URL),
|
||||||
|
recovery.clone(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(status, StatusCode::OK);
|
||||||
|
assert_eq!(result["status"], "saved");
|
||||||
|
assert_eq!(
|
||||||
|
f.post(
|
||||||
|
"/api/auth/passkeys/addition/result",
|
||||||
|
Some(&bob_cookie),
|
||||||
|
Some(ORIGIN_URL),
|
||||||
|
recovery.clone()
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.0,
|
||||||
|
StatusCode::BAD_REQUEST
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
f.post(
|
||||||
|
"/api/auth/passkeys/addition/result",
|
||||||
|
None,
|
||||||
|
Some(ORIGIN_URL),
|
||||||
|
recovery.clone()
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.0,
|
||||||
|
StatusCode::FORBIDDEN
|
||||||
|
);
|
||||||
|
recovery["expected_authentication_reference"] = "changed-original-reference".into();
|
||||||
|
assert_eq!(
|
||||||
|
f.post(
|
||||||
|
"/api/auth/passkeys/addition/result",
|
||||||
|
Some(&replacement_cookie),
|
||||||
|
Some(ORIGIN_URL),
|
||||||
|
recovery
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.0,
|
||||||
|
StatusCode::BAD_REQUEST
|
||||||
|
);
|
||||||
|
// Bob's own expected fields cannot discover Alice's operation either.
|
||||||
|
let (status, result) = f.addition_result(&bob_cookie, &challenge_id).await;
|
||||||
|
assert_eq!(status, StatusCode::OK);
|
||||||
|
assert_eq!(result, serde_json::json!({"status":"unknown"}));
|
||||||
|
assert_eq!(f.credentials(&alice).len(), 2);
|
||||||
|
assert_eq!(
|
||||||
|
f.post(
|
||||||
|
"/api/auth/logout",
|
||||||
|
Some(&alice_cookie),
|
||||||
|
Some(ORIGIN_URL),
|
||||||
|
serde_json::json!({})
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.0,
|
||||||
|
StatusCode::OK
|
||||||
|
);
|
||||||
|
let mut request = expected;
|
||||||
|
request["challenge_id"] = challenge_id.into();
|
||||||
|
assert_eq!(
|
||||||
|
f.post(
|
||||||
|
"/api/auth/passkeys/addition/result",
|
||||||
|
Some(&alice_cookie),
|
||||||
|
Some(ORIGIN_URL),
|
||||||
|
request
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.0,
|
||||||
|
StatusCode::FORBIDDEN
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn own_key_list_is_safe_and_only_verified_normal_login_marks_confirmation() {
|
||||||
|
let f = Fixture::new().await;
|
||||||
|
let mut key = authenticator();
|
||||||
|
let (cookie, owner) = f.enroll("registration", None, "owner", &mut key).await;
|
||||||
|
let (other_cookie, _) = f
|
||||||
|
.enroll("registration", None, "other", &mut authenticator())
|
||||||
|
.await;
|
||||||
|
assert_eq!(
|
||||||
|
f.get_authenticated("/api/auth/passkeys", None, None)
|
||||||
|
.await
|
||||||
|
.0,
|
||||||
|
StatusCode::FORBIDDEN
|
||||||
|
);
|
||||||
|
let (status, list) = f
|
||||||
|
.get_authenticated("/api/auth/passkeys", Some(&cookie), None)
|
||||||
|
.await;
|
||||||
|
assert_eq!(status, StatusCode::OK);
|
||||||
|
assert_eq!(list["user"]["user_id"], owner.user_id);
|
||||||
|
assert_eq!(list["keys"].as_array().unwrap().len(), 1);
|
||||||
|
let summary = &list["keys"][0];
|
||||||
|
assert!(summary["normal_login_verified_at"].is_null());
|
||||||
|
assert!(summary["last_used_at"].is_null());
|
||||||
|
let reference = summary["key_reference"].as_str().unwrap();
|
||||||
|
assert_eq!(reference.len(), 32);
|
||||||
|
assert_ne!(reference, f.credentials(&owner)[0].credential_id);
|
||||||
|
assert_eq!(
|
||||||
|
summary
|
||||||
|
.as_object()
|
||||||
|
.unwrap()
|
||||||
|
.keys()
|
||||||
|
.map(String::as_str)
|
||||||
|
.collect::<std::collections::BTreeSet<_>>(),
|
||||||
|
[
|
||||||
|
"key_reference",
|
||||||
|
"created_at",
|
||||||
|
"last_used_at",
|
||||||
|
"normal_login_verified_at"
|
||||||
|
]
|
||||||
|
.into_iter()
|
||||||
|
.collect()
|
||||||
|
);
|
||||||
|
let (_, other) = f
|
||||||
|
.get_authenticated("/api/auth/passkeys", Some(&other_cookie), None)
|
||||||
|
.await;
|
||||||
|
assert_ne!(other["keys"][0]["key_reference"], reference);
|
||||||
|
assert_eq!(f.login("owner", &mut key).await, owner);
|
||||||
|
let (_, legacy) = f
|
||||||
|
.get_authenticated("/api/auth/passkeys", Some(&cookie), None)
|
||||||
|
.await;
|
||||||
|
assert!(legacy["keys"][0]["last_used_at"].is_string());
|
||||||
|
assert!(legacy["keys"][0]["normal_login_verified_at"].is_null());
|
||||||
|
// Rejected normal assertion does not produce confirmation metadata.
|
||||||
|
let mut invalid = f.discoverable_request(&mut key).await;
|
||||||
|
invalid.credential.response.signature = vec![0u8; 64].into();
|
||||||
|
assert_eq!(
|
||||||
|
f.post(
|
||||||
|
"/api/auth/passkeys/login/complete",
|
||||||
|
None,
|
||||||
|
Some(ORIGIN_URL),
|
||||||
|
serde_json::to_value(invalid).unwrap()
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.0,
|
||||||
|
StatusCode::BAD_REQUEST
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
f.api.store.list_passkey_summaries(&owner.user_id).unwrap()[0]
|
||||||
|
.normal_login_verified_at
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert_eq!(f.discoverable_login(&mut key).await, owner);
|
||||||
|
let (_, confirmed) = f
|
||||||
|
.get_authenticated("/api/auth/passkeys", Some(&cookie), None)
|
||||||
|
.await;
|
||||||
|
assert_eq!(confirmed["keys"][0]["key_reference"], reference);
|
||||||
|
assert!(confirmed["keys"][0]["normal_login_verified_at"].is_string());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test]
|
||||||
|
async fn unknown_transport_metadata_does_not_discard_verified_registration() {
|
||||||
|
let f = Fixture::new().await;
|
||||||
|
let options = f.options("registration", None, "future-transport").await;
|
||||||
|
let mut key = authenticator();
|
||||||
|
let mut request = Fixture::credential(options, &mut key);
|
||||||
|
request["credential"]["response"]["transports"] =
|
||||||
|
serde_json::json!(["internal", "future-transport", "test"]);
|
||||||
|
let (status, _, body) = f
|
||||||
|
.post(
|
||||||
|
"/api/auth/passkeys/registration/complete",
|
||||||
|
None,
|
||||||
|
Some(ORIGIN_URL),
|
||||||
|
request,
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
assert_eq!(status, StatusCode::OK);
|
||||||
|
let user = serde_json::from_value::<AuthUserResponse>(body)
|
||||||
|
.unwrap()
|
||||||
|
.user;
|
||||||
|
assert_eq!(f.discoverable_login(&mut key).await, user);
|
||||||
|
}
|
||||||
@@ -120,6 +120,14 @@ struct TicketSurface {
|
|||||||
}
|
}
|
||||||
impl TicketSurface {
|
impl TicketSurface {
|
||||||
async fn new(root: &Path, client: Arc<TicketRouterClient>, wip: bool) -> Self {
|
async fn new(root: &Path, client: Arc<TicketRouterClient>, wip: bool) -> Self {
|
||||||
|
Self::with_access(root, client, wip, TicketFeatureAccess::workflow()).await
|
||||||
|
}
|
||||||
|
async fn with_access(
|
||||||
|
root: &Path,
|
||||||
|
client: Arc<dyn WorkspaceClient>,
|
||||||
|
wip: bool,
|
||||||
|
access: TicketFeatureAccess,
|
||||||
|
) -> Self {
|
||||||
let manifest = worker::WorkerManifest::from_toml("[worker]\nname=\"ticket-tools\"\n[model]\nscheme=\"anthropic\"\nmodel_id=\"unused\"\n[engine]\n[scope]\nallow=[]\n").unwrap();
|
let manifest = worker::WorkerManifest::from_toml("[worker]\nname=\"ticket-tools\"\n[model]\nscheme=\"anthropic\"\nmodel_id=\"unused\"\n[engine]\n[scope]\nallow=[]\n").unwrap();
|
||||||
let engine =
|
let engine =
|
||||||
agen::Engine::<_, agen::state::Mutable, worker::SessionHistoryMetadata>::new_annotated(
|
agen::Engine::<_, agen::state::Mutable, worker::SessionHistoryMetadata>::new_annotated(
|
||||||
@@ -138,22 +146,18 @@ impl TicketSurface {
|
|||||||
)
|
)
|
||||||
.await
|
.await
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let report =
|
let report = host.install_features(
|
||||||
host.install_features(worker::feature::FeatureRegistryBuilder::new().with_module(
|
worker::feature::FeatureRegistryBuilder::new()
|
||||||
TicketFeature::new(client.clone(), TicketFeatureAccess::workflow()),
|
.with_module(TicketFeature::new(client.clone(), access)),
|
||||||
));
|
);
|
||||||
assert!(!report.has_errors());
|
assert!(!report.has_errors());
|
||||||
let handle = host.engine().tool_server_handle();
|
let handle = host.engine().tool_server_handle();
|
||||||
handle.flush_pending();
|
handle.flush_pending();
|
||||||
let tools = [
|
let tools = access
|
||||||
"TicketAssignmentTrackingQuery",
|
.tool_names()
|
||||||
"TicketAssignmentAnswer",
|
.into_iter()
|
||||||
"TicketImplementationResult",
|
.map(|name| (name.into(), handle.get_tool(name).unwrap().1))
|
||||||
"TicketImplementationResultQuery",
|
.collect();
|
||||||
]
|
|
||||||
.into_iter()
|
|
||||||
.map(|name| (name.into(), handle.get_tool(name).unwrap().1))
|
|
||||||
.collect();
|
|
||||||
if !wip {
|
if !wip {
|
||||||
return Self { tools, wip };
|
return Self { tools, wip };
|
||||||
}
|
}
|
||||||
@@ -162,7 +166,7 @@ impl TicketSurface {
|
|||||||
worker::feature::builtin::ticket::mount_workspace_http_ticket_wip(
|
worker::feature::builtin::ticket::mount_workspace_http_ticket_wip(
|
||||||
&mut mounts,
|
&mut mounts,
|
||||||
client,
|
client,
|
||||||
TicketFeatureAccess::workflow(),
|
access,
|
||||||
None,
|
None,
|
||||||
&route,
|
&route,
|
||||||
)
|
)
|
||||||
@@ -189,8 +193,9 @@ impl TicketSurface {
|
|||||||
if !self.wip {
|
if !self.wip {
|
||||||
return execute_tool(self.tools[name].clone(), input, context).await;
|
return execute_tool(self.tools[name].clone(), input, context).await;
|
||||||
}
|
}
|
||||||
let path = format!("/tickets/{}", input["ticket"].as_str().unwrap());
|
let reference_field = if name == "ShowTicket" { "id" } else { "ticket" };
|
||||||
input.as_object_mut().unwrap().remove("ticket");
|
let path = format!("/tickets/{}", input[reference_field].as_str().unwrap());
|
||||||
|
input.as_object_mut().unwrap().remove(reference_field);
|
||||||
// Observe the real item representation before invoking a mutation. Tracking
|
// Observe the real item representation before invoking a mutation. Tracking
|
||||||
// reads alone do not establish the Ticket WIP object validator. Explicit
|
// reads alone do not establish the Ticket WIP object validator. Explicit
|
||||||
// refresh also reconciles stale observations after an unknown result;
|
// refresh also reconciles stale observations after an unknown result;
|
||||||
@@ -201,6 +206,9 @@ impl TicketSurface {
|
|||||||
ToolExecutionContext::direct(),
|
ToolExecutionContext::direct(),
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
if name == "ShowTicket" {
|
||||||
|
return execute_tool(self.tools["Invoke"].clone(), serde_json::json!({"path":path,"interface":inspected["interfaces"][0]["reference"],"operation":"read","arguments":input}), context).await;
|
||||||
|
}
|
||||||
execute_tool(self.tools["Invoke"].clone(), serde_json::json!({"path":path,"interface":inspected["interfaces"][0]["reference"],"operation":"read","arguments":{}}), ToolExecutionContext::direct()).await?;
|
execute_tool(self.tools["Invoke"].clone(), serde_json::json!({"path":path,"interface":inspected["interfaces"][0]["reference"],"operation":"read","arguments":{}}), ToolExecutionContext::direct()).await?;
|
||||||
let inspected = execute_tool(
|
let inspected = execute_tool(
|
||||||
self.tools["Inspect"].clone(),
|
self.tools["Inspect"].clone(),
|
||||||
@@ -364,6 +372,210 @@ async fn ticket_public_roundtrip_observed_answer_replays_without_changing_waitin
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Reviewer policy, tool projection, reference resolution, authentication and
|
||||||
|
// persistence remain production code. Only the synchronous HTTP transport is
|
||||||
|
// replaced by signed in-process router dispatch. Child spawning and MR attestation
|
||||||
|
// have their own suites; this fixture does not claim to exercise those boundaries.
|
||||||
|
fn reviewer_client(inner: Arc<TicketRouterClient>, bound_ticket: &str) -> Arc<dyn WorkspaceClient> {
|
||||||
|
Arc::new(worker::ReviewerChildWorkspaceClient::new(
|
||||||
|
inner,
|
||||||
|
worker::ReviewerContext {
|
||||||
|
ticket_id: bound_ticket.into(),
|
||||||
|
merge_request_id: "unsubmitted-review-candidate".into(),
|
||||||
|
},
|
||||||
|
"unused-review-capability".into(),
|
||||||
|
))
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
async fn reviewer_reads_saved_related_tickets_by_key_and_id_and_pages_their_threads() {
|
||||||
|
for wip in [false, true] {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let (api, _runtime, identity, source, bound_id) = tracking_fixture(dir.path()).await;
|
||||||
|
let backend = browser_ticket_backend(&api).unwrap();
|
||||||
|
let mut related = ticket::NewTicket::new("Related requirements");
|
||||||
|
related.body = ticket::MarkdownText::new("Requirements from a different Ticket");
|
||||||
|
let related = backend.create(related).unwrap();
|
||||||
|
backend
|
||||||
|
.add_ticket_relation(
|
||||||
|
bound_id.clone().into(),
|
||||||
|
ticket::NewTicketRelation {
|
||||||
|
kind: ticket::TicketRelationKind::DependsOn,
|
||||||
|
target: related.id.clone(),
|
||||||
|
note: None,
|
||||||
|
author: None,
|
||||||
|
},
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
for body in ["Earlier decision", "Latest decision"] {
|
||||||
|
backend
|
||||||
|
.add_event(
|
||||||
|
related.id.clone().into(),
|
||||||
|
ticket::NewTicketEvent::new(ticket::TicketEventKind::Comment, body),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
let bound = backend.show(bound_id.clone().into()).unwrap();
|
||||||
|
let related_before = backend.show(related.id.clone().into()).unwrap();
|
||||||
|
let bound_key = bound.meta.resource_key.as_deref().unwrap();
|
||||||
|
let related_key = related.resource_key.as_deref().unwrap();
|
||||||
|
let transport = ticket_client(&api, identity, &source);
|
||||||
|
// One client must retain its review binding while following related Tickets.
|
||||||
|
let client = reviewer_client(transport.clone(), &bound_id);
|
||||||
|
let surface =
|
||||||
|
TicketSurface::with_access(dir.path(), client, wip, TicketFeatureAccess::review())
|
||||||
|
.await;
|
||||||
|
for (reference, expected_key, expected_title, expected_body) in [
|
||||||
|
(
|
||||||
|
bound_key,
|
||||||
|
bound_key,
|
||||||
|
"Tracking authority",
|
||||||
|
bound.document.body.as_str(),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
related_key,
|
||||||
|
related_key,
|
||||||
|
"Related requirements",
|
||||||
|
related_before.document.body.as_str(),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
related.id.as_str(),
|
||||||
|
related_key,
|
||||||
|
"Related requirements",
|
||||||
|
related_before.document.body.as_str(),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
bound_id.as_str(),
|
||||||
|
bound_key,
|
||||||
|
"Tracking authority",
|
||||||
|
bound.document.body.as_str(),
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
let shown = surface
|
||||||
|
.call(
|
||||||
|
"ShowTicket",
|
||||||
|
serde_json::json!({"id":reference}),
|
||||||
|
ToolExecutionContext::direct(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(shown["ticket"], expected_key);
|
||||||
|
assert_eq!(shown["title"], expected_title);
|
||||||
|
assert_eq!(shown["body"], expected_body);
|
||||||
|
if expected_key == bound_key {
|
||||||
|
assert!(
|
||||||
|
shown["relations"]["outgoing"]
|
||||||
|
.as_array()
|
||||||
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.any(|relation| relation["ticket"] == related_key
|
||||||
|
&& relation["kind"] == "depends_on")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let first = surface
|
||||||
|
.call(
|
||||||
|
"ShowTicket",
|
||||||
|
serde_json::json!({"id":related_key,"event_limit":1}),
|
||||||
|
ToolExecutionContext::direct(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(first["thread"].as_array().unwrap().len(), 1);
|
||||||
|
assert_eq!(first["thread"][0]["body"], "Latest decision");
|
||||||
|
assert_eq!(first["event_page"]["has_more"], true);
|
||||||
|
let second = surface.call("ShowTicket", serde_json::json!({"id":related_key,"event_limit":1,"event_cursor":first["event_page"]["next_cursor"]}), ToolExecutionContext::direct()).await.unwrap();
|
||||||
|
assert_eq!(second["thread"].as_array().unwrap().len(), 1);
|
||||||
|
assert_eq!(second["thread"][0]["body"], "Earlier decision");
|
||||||
|
assert_eq!(backend.show(bound_id.into()).unwrap(), bound);
|
||||||
|
assert_eq!(backend.show(related.id.into()).unwrap(), related_before);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||||
|
async fn reviewer_read_uses_live_backend_membership_and_cannot_mutate_saved_ticket() {
|
||||||
|
for wip in [false, true] {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let (api, _runtime, identity, source, ticket) = tracking_fixture(dir.path()).await;
|
||||||
|
let backend = browser_ticket_backend(&api).unwrap();
|
||||||
|
let before = backend.show(ticket.clone().into()).unwrap();
|
||||||
|
let transport = ticket_client(&api, identity, &source);
|
||||||
|
let client = reviewer_client(transport.clone(), &ticket);
|
||||||
|
let surface = TicketSurface::with_access(
|
||||||
|
dir.path(),
|
||||||
|
client.clone(),
|
||||||
|
wip,
|
||||||
|
TicketFeatureAccess::review(),
|
||||||
|
)
|
||||||
|
.await;
|
||||||
|
surface
|
||||||
|
.call(
|
||||||
|
"ShowTicket",
|
||||||
|
serde_json::json!({"id":ticket}),
|
||||||
|
ToolExecutionContext::direct(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap();
|
||||||
|
let request_count = transport.requests.lock().unwrap().len();
|
||||||
|
for (method, path, body) in [
|
||||||
|
(
|
||||||
|
WorkspaceRequestMethod::Post,
|
||||||
|
format!("/api/w/{TEST_WORKSPACE_ID}/tickets/{ticket}/comments"),
|
||||||
|
serde_json::json!({"body":"unauthorized comment"}),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
WorkspaceRequestMethod::Patch,
|
||||||
|
format!("/api/w/{TEST_WORKSPACE_ID}/tickets/{ticket}"),
|
||||||
|
serde_json::json!({"title":"unauthorized title"}),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
WorkspaceRequestMethod::Post,
|
||||||
|
format!("/api/w/another-workspace/tickets/{ticket}/show"),
|
||||||
|
serde_json::json!({}),
|
||||||
|
),
|
||||||
|
(
|
||||||
|
WorkspaceRequestMethod::Post,
|
||||||
|
format!("/api/w/{TEST_WORKSPACE_ID}/merge-requests/other-candidate/reviews"),
|
||||||
|
serde_json::json!({"verdict":"approved"}),
|
||||||
|
),
|
||||||
|
] {
|
||||||
|
let client = client.clone();
|
||||||
|
let error = tokio::task::spawn_blocking(move || {
|
||||||
|
client.execute(WorkspaceRequest::json(method, path, body.to_string()))
|
||||||
|
})
|
||||||
|
.await
|
||||||
|
.unwrap()
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(error.to_string().contains("read-only"), "{error}");
|
||||||
|
}
|
||||||
|
assert_eq!(transport.requests.lock().unwrap().len(), request_count);
|
||||||
|
assert_eq!(backend.show(ticket.clone().into()).unwrap(), before);
|
||||||
|
// Keep a valid Runtime signature but select an unregistered Worker.
|
||||||
|
// This must fail at the real Backend, not the Reviewer route filter.
|
||||||
|
*transport.source.lock().unwrap() =
|
||||||
|
RuntimeWorkerRef::new(source.runtime_id, uuid::Uuid::now_v7().to_string());
|
||||||
|
let error = surface
|
||||||
|
.call(
|
||||||
|
"ShowTicket",
|
||||||
|
serde_json::json!({"id":ticket}),
|
||||||
|
ToolExecutionContext::direct(),
|
||||||
|
)
|
||||||
|
.await
|
||||||
|
.unwrap_err();
|
||||||
|
assert!(error.to_string().contains("401"), "{error}");
|
||||||
|
assert!(
|
||||||
|
transport
|
||||||
|
.requests
|
||||||
|
.lock()
|
||||||
|
.unwrap()
|
||||||
|
.iter()
|
||||||
|
.skip(request_count)
|
||||||
|
.any(|(_, status, _)| *status == 401)
|
||||||
|
);
|
||||||
|
assert_eq!(backend.show(ticket.into()).unwrap(), before);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
fn report_counts(api: &WorkspaceApi) -> (i64, i64) {
|
fn report_counts(api: &WorkspaceApi) -> (i64, i64) {
|
||||||
let conn = rusqlite::Connection::open(&api.config.database_path).unwrap();
|
let conn = rusqlite::Connection::open(&api.config.database_path).unwrap();
|
||||||
(
|
(
|
||||||
|
|||||||
@@ -819,6 +819,7 @@ mod workspace_config_integration {
|
|||||||
},
|
},
|
||||||
"owner".into(),
|
"owner".into(),
|
||||||
WorkingDirectoryRemovalRequest {
|
WorkingDirectoryRemovalRequest {
|
||||||
|
operation_id: "unsupported-logical-cleanup".into(),
|
||||||
reason: "unsupported logical cleanup".into()
|
reason: "unsupported logical cleanup".into()
|
||||||
}
|
}
|
||||||
)
|
)
|
||||||
|
|||||||
@@ -187,7 +187,14 @@ async fn config_call(
|
|||||||
runtime
|
runtime
|
||||||
.invoke(
|
.invoke(
|
||||||
path.into(),
|
path.into(),
|
||||||
if matches!(operation,"edit"|"write") {wip_protocol::InterfaceReference {scope:"/".into(),name:format!("yoi.file/{operation}-text/v1")}} else {config_interface(path)},
|
if matches!(operation, "edit" | "write") {
|
||||||
|
wip_protocol::InterfaceReference {
|
||||||
|
scope: "/".into(),
|
||||||
|
name: format!("yoi.file/{operation}-text/v1"),
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
config_interface(path)
|
||||||
|
},
|
||||||
operation.into(),
|
operation.into(),
|
||||||
args,
|
args,
|
||||||
agen::tool::ToolExecutionContext::direct(),
|
agen::tool::ToolExecutionContext::direct(),
|
||||||
@@ -581,11 +588,88 @@ impl worker_runtime::worker_backend::RuntimeWorkerFactory for ConfigCaptureFacto
|
|||||||
*self.controller_abort.lock().unwrap() = Some(controller.controller_task.abort_handle());
|
*self.controller_abort.lock().unwrap() = Some(controller.controller_task.abort_handle());
|
||||||
Ok(controller)
|
Ok(controller)
|
||||||
}
|
}
|
||||||
async fn restore_controller(
|
async fn preflight_restore(
|
||||||
|
&self,
|
||||||
|
request: &worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||||
|
) -> std::result::Result<(), String> {
|
||||||
|
self.inner.preflight_restore(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn recover_restore_source(
|
||||||
|
&self,
|
||||||
|
request: &worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||||
|
) -> std::result::Result<bool, worker_runtime::worker_backend::RuntimeRestoreFailure> {
|
||||||
|
self.inner.recover_restore_source(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn prepare_restore(
|
||||||
&self,
|
&self,
|
||||||
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
request: worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||||
) -> std::result::Result<worker_runtime::worker_backend::RuntimeWorkerController, String> {
|
) -> std::result::Result<
|
||||||
self.inner.restore_controller(request).await
|
Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||||
|
> {
|
||||||
|
let inner = self.inner.prepare_restore(request).await?;
|
||||||
|
Ok(Box::new(ConfigCaptureCandidate {
|
||||||
|
inner,
|
||||||
|
client: self.client.clone(),
|
||||||
|
controller_abort: self.controller_abort.clone(),
|
||||||
|
}))
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn cleanup_failed_restore(
|
||||||
|
&self,
|
||||||
|
request: &worker_runtime::execution::WorkerExecutionRestoreRequest,
|
||||||
|
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
|
||||||
|
self.inner.cleanup_failed_restore(request).await
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn reconcile_stopped_worker(
|
||||||
|
&self,
|
||||||
|
worker_ref: &worker_runtime::identity::WorkerRef,
|
||||||
|
) -> std::result::Result<(), String> {
|
||||||
|
self.inner.reconcile_stopped_worker(worker_ref).await
|
||||||
|
}
|
||||||
|
|
||||||
|
fn activate_restored_controller(
|
||||||
|
&self,
|
||||||
|
worker_ref: &worker_runtime::identity::WorkerRef,
|
||||||
|
workspace_id: Option<&str>,
|
||||||
|
handle: &worker::WorkerHandle,
|
||||||
|
) -> std::result::Result<(), String> {
|
||||||
|
self.inner
|
||||||
|
.activate_restored_controller(worker_ref, workspace_id, handle)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
struct ConfigCaptureCandidate {
|
||||||
|
inner: Box<dyn worker_runtime::worker_backend::RuntimeRestoreCandidate>,
|
||||||
|
client: Arc<Mutex<Option<Arc<dyn worker::WorkspaceClient>>>>,
|
||||||
|
controller_abort: Arc<Mutex<Option<tokio::task::AbortHandle>>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[async_trait]
|
||||||
|
impl worker_runtime::worker_backend::RuntimeRestoreCandidate for ConfigCaptureCandidate {
|
||||||
|
fn initial_state(&self) -> protocol::WorkerStateSnapshot {
|
||||||
|
self.inner.initial_state()
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn activate(
|
||||||
|
&mut self,
|
||||||
|
) -> std::result::Result<
|
||||||
|
worker_runtime::worker_backend::RuntimeWorkerController,
|
||||||
|
worker_runtime::worker_backend::RuntimeRestoreFailure,
|
||||||
|
> {
|
||||||
|
let controller = self.inner.activate().await?;
|
||||||
|
*self.client.lock().unwrap() = Some(controller.workspace_client.clone());
|
||||||
|
*self.controller_abort.lock().unwrap() = Some(controller.controller_task.abort_handle());
|
||||||
|
Ok(controller)
|
||||||
|
}
|
||||||
|
|
||||||
|
async fn discard(
|
||||||
|
&mut self,
|
||||||
|
) -> std::result::Result<(), worker_runtime::worker_backend::RuntimeRestoreCleanupFailure> {
|
||||||
|
self.inner.discard().await
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -714,6 +798,7 @@ async fn workspace_config_actual_remote_and_embedded_clients_use_authenticated_w
|
|||||||
RUNTIME_REQUEST_SOURCE_PROOF_HEADER, WORKSPACE_REQUEST_PERMISSION,
|
RUNTIME_REQUEST_SOURCE_PROOF_HEADER, WORKSPACE_REQUEST_PERMISSION,
|
||||||
decode_runtime_request_source_claims,
|
decode_runtime_request_source_claims,
|
||||||
};
|
};
|
||||||
|
let root = restored_parent_control_tests::connected_restore_tempdir();
|
||||||
// Remote Runtime-owned HTTP adapter, including its real request signer.
|
// Remote Runtime-owned HTTP adapter, including its real request signer.
|
||||||
let mut fixture = manual_worker_assignment_fixture().await;
|
let mut fixture = manual_worker_assignment_fixture().await;
|
||||||
let identity = RuntimeIdentityMaterial::generate(&fixture.worker.runtime_id).unwrap();
|
let identity = RuntimeIdentityMaterial::generate(&fixture.worker.runtime_id).unwrap();
|
||||||
@@ -793,7 +878,6 @@ async fn workspace_config_actual_remote_and_embedded_clients_use_authenticated_w
|
|||||||
|
|
||||||
// Actual embedded Runtime factory -> real Worker controller -> captured
|
// Actual embedded Runtime factory -> real Worker controller -> captured
|
||||||
// production WorkspaceClient -> signed HTTP -> production Backend middleware.
|
// production WorkspaceClient -> signed HTTP -> production Backend middleware.
|
||||||
let root = tempfile::tempdir().unwrap();
|
|
||||||
init_clean_git_workspace(root.path());
|
init_clean_git_workspace(root.path());
|
||||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||||
let addr = listener.local_addr().unwrap();
|
let addr = listener.local_addr().unwrap();
|
||||||
|
|||||||
@@ -119,7 +119,7 @@ pub use ticket::{
|
|||||||
};
|
};
|
||||||
|
|
||||||
const OLDEST_SCHEMA_VERSION: i64 = 50;
|
const OLDEST_SCHEMA_VERSION: i64 = 50;
|
||||||
const LATEST_SCHEMA_VERSION: i64 = 103;
|
const LATEST_SCHEMA_VERSION: i64 = 104;
|
||||||
const WORKSPACE_DRIVE_GRANTS_MIGRATION_NAME: &str = "durable Workspace Drive grants";
|
const WORKSPACE_DRIVE_GRANTS_MIGRATION_NAME: &str = "durable Workspace Drive grants";
|
||||||
const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs";
|
const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs";
|
||||||
const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline";
|
const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline";
|
||||||
@@ -439,11 +439,24 @@ const MIGRATIONS: &[Migration] = &[
|
|||||||
},
|
},
|
||||||
Migration {
|
Migration {
|
||||||
version: 103,
|
version: 103,
|
||||||
name: "Job input retirement independent of immutable receipts",
|
name: "Passkey metadata and atomic addition results",
|
||||||
apply: migrate_job_input_retention_v102_to_v103,
|
apply: migrate_passkey_results_v102_to_v103,
|
||||||
|
},
|
||||||
|
Migration {
|
||||||
|
version: 104,
|
||||||
|
name: "Converge Passkey results and Job input retention",
|
||||||
|
apply: migrate_schema_103_variants_to_v104,
|
||||||
},
|
},
|
||||||
];
|
];
|
||||||
|
|
||||||
|
// Both schema-103 identities shipped. Keep the alternate implementation and accept
|
||||||
|
// its exact receipt without renaming or replacing any applied history.
|
||||||
|
const JOB_INPUT_RETENTION_SCHEMA_103: Migration = Migration {
|
||||||
|
version: 103,
|
||||||
|
name: "Job input retirement independent of immutable receipts",
|
||||||
|
apply: migrate_job_input_retention_v102_to_v103,
|
||||||
|
};
|
||||||
|
|
||||||
#[derive(Clone, Copy)]
|
#[derive(Clone, Copy)]
|
||||||
struct Migration {
|
struct Migration {
|
||||||
version: i64,
|
version: i64,
|
||||||
@@ -754,6 +767,15 @@ pub enum PasskeyAuthenticationBinding {
|
|||||||
ApiToken(String),
|
ApiToken(String),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
impl PasskeyAuthenticationBinding {
|
||||||
|
pub(crate) fn reference(&self) -> String {
|
||||||
|
let hash = match self {
|
||||||
|
Self::BrowserSession(hash) | Self::ApiToken(hash) => hash,
|
||||||
|
};
|
||||||
|
crate::auth::authentication_reference(hash)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||||
pub struct PasskeyCredentialRecord {
|
pub struct PasskeyCredentialRecord {
|
||||||
pub credential_id: String,
|
pub credential_id: String,
|
||||||
@@ -1718,6 +1740,27 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore {
|
|||||||
&self,
|
&self,
|
||||||
target: &PasskeyRegistrationTarget,
|
target: &PasskeyRegistrationTarget,
|
||||||
credential: &PasskeyCredentialRecord,
|
credential: &PasskeyCredentialRecord,
|
||||||
|
) -> Result<()> {
|
||||||
|
self.register_verified_passkey_with_result(target, credential, None)
|
||||||
|
}
|
||||||
|
/// Receipt and credential commit together; no receipt claims session issuance.
|
||||||
|
fn register_verified_passkey_with_result(
|
||||||
|
&self,
|
||||||
|
target: &PasskeyRegistrationTarget,
|
||||||
|
credential: &PasskeyCredentialRecord,
|
||||||
|
addition_challenge_id: Option<&str>,
|
||||||
|
) -> Result<()>;
|
||||||
|
fn list_passkey_summaries(&self, user_id: &str) -> Result<Vec<server_api::PasskeySummary>>;
|
||||||
|
fn read_passkey_addition_result(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
original_authentication_reference: &str,
|
||||||
|
challenge_id: &str,
|
||||||
|
) -> Result<Option<server_api::PasskeySummary>>;
|
||||||
|
fn mark_passkey_normal_login_verified(
|
||||||
|
&self,
|
||||||
|
credential_id: &str,
|
||||||
|
verified_at: &str,
|
||||||
) -> Result<()>;
|
) -> Result<()>;
|
||||||
fn upsert_passkey_credential(&self, record: &PasskeyCredentialRecord) -> Result<()>;
|
fn upsert_passkey_credential(&self, record: &PasskeyCredentialRecord) -> Result<()>;
|
||||||
fn get_passkey_credential(
|
fn get_passkey_credential(
|
||||||
@@ -2603,6 +2646,10 @@ pub struct SqliteWorkspaceStore {
|
|||||||
db_executor: crate::db_executor::DbExecutor,
|
db_executor: crate::db_executor::DbExecutor,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[cfg(feature = "test-support")]
|
||||||
|
#[path = "test_support.rs"]
|
||||||
|
pub mod fixture_support;
|
||||||
|
|
||||||
impl SqliteWorkspaceStore {
|
impl SqliteWorkspaceStore {
|
||||||
pub fn open(path: impl AsRef<Path>) -> Result<Self> {
|
pub fn open(path: impl AsRef<Path>) -> Result<Self> {
|
||||||
let conn = Connection::open(path)?;
|
let conn = Connection::open(path)?;
|
||||||
@@ -2914,6 +2961,33 @@ impl SqliteWorkspaceStore {
|
|||||||
request_fingerprint: &str,
|
request_fingerprint: &str,
|
||||||
singleton_key: Option<&str>,
|
singleton_key: Option<&str>,
|
||||||
current_memory_settings: &WorkspaceMemorySettingsRecord,
|
current_memory_settings: &WorkspaceMemorySettingsRecord,
|
||||||
|
) -> Result<WorkerCreateReservation> {
|
||||||
|
self.reserve_worker_create_with(
|
||||||
|
workspace_id,
|
||||||
|
runtime_id,
|
||||||
|
allocation_key,
|
||||||
|
request_fingerprint,
|
||||||
|
singleton_key,
|
||||||
|
current_memory_settings,
|
||||||
|
|_| Ok(None),
|
||||||
|
|_, _| Ok(()),
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Keep reservation validation, Memory binding and writes in one transaction. The
|
||||||
|
// nondefault test-support module supplies fixed identity / offline catalog hooks;
|
||||||
|
// ordinary creation always supplies a newly generated ID and no completion hook.
|
||||||
|
#[allow(clippy::too_many_arguments)]
|
||||||
|
fn reserve_worker_create_with(
|
||||||
|
&self,
|
||||||
|
workspace_id: &str,
|
||||||
|
runtime_id: &str,
|
||||||
|
allocation_key: &str,
|
||||||
|
request_fingerprint: &str,
|
||||||
|
singleton_key: Option<&str>,
|
||||||
|
current_memory_settings: &WorkspaceMemorySettingsRecord,
|
||||||
|
prepare: impl FnOnce(&rusqlite::Transaction<'_>) -> Result<Option<WorkerId>>,
|
||||||
|
complete: impl FnOnce(&rusqlite::Transaction<'_>, &WorkerCreateReservation) -> Result<()>,
|
||||||
) -> Result<WorkerCreateReservation> {
|
) -> Result<WorkerCreateReservation> {
|
||||||
if allocation_key.trim().is_empty() || request_fingerprint.trim().is_empty() {
|
if allocation_key.trim().is_empty() || request_fingerprint.trim().is_empty() {
|
||||||
return Err(Error::InvalidInput(
|
return Err(Error::InvalidInput(
|
||||||
@@ -2926,6 +3000,7 @@ impl SqliteWorkspaceStore {
|
|||||||
validate_workspace_memory_settings_record(current_memory_settings, workspace_id)?;
|
validate_workspace_memory_settings_record(current_memory_settings, workspace_id)?;
|
||||||
self.with_conn_mut(|conn| {
|
self.with_conn_mut(|conn| {
|
||||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||||
|
let fixed_worker_id = prepare(&tx)?;
|
||||||
let existing = tx
|
let existing = tx
|
||||||
.query_row(
|
.query_row(
|
||||||
"SELECT worker_id, runtime_id, request_fingerprint, create_fingerprint, \
|
"SELECT worker_id, runtime_id, request_fingerprint, create_fingerprint, \
|
||||||
@@ -3037,8 +3112,8 @@ impl SqliteWorkspaceStore {
|
|||||||
},
|
},
|
||||||
)
|
)
|
||||||
.optional()?;
|
.optional()?;
|
||||||
// Subject occupancy survives removal until an explicit release
|
// An unconfirmed legacy removal cannot release Subject occupancy.
|
||||||
// proof. It must not resurrect the removed creation reservation:
|
// Completed removal or explicit release does; never resurrect a reservation:
|
||||||
// Runtime callbacks require a live reservation or catalog member.
|
// Runtime callbacks require a live reservation or catalog member.
|
||||||
if stored.as_ref().is_some_and(|row| row.6 == "removed") {
|
if stored.as_ref().is_some_and(|row| row.6 == "removed") {
|
||||||
return Err(Error::RepositoryConflict(format!(
|
return Err(Error::RepositoryConflict(format!(
|
||||||
@@ -3108,7 +3183,7 @@ impl SqliteWorkspaceStore {
|
|||||||
validate_workspace_memory_settings_snapshot(&snapshot, workspace_id)?;
|
validate_workspace_memory_settings_snapshot(&snapshot, workspace_id)?;
|
||||||
let create_fingerprint =
|
let create_fingerprint =
|
||||||
bound_worker_create_fingerprint(request_fingerprint, &snapshot);
|
bound_worker_create_fingerprint(request_fingerprint, &snapshot);
|
||||||
let worker_id = WorkerId::now_v7();
|
let worker_id = fixed_worker_id.unwrap_or_else(WorkerId::now_v7);
|
||||||
let worker_id_text = worker_id.to_string();
|
let worker_id_text = worker_id.to_string();
|
||||||
let now = chrono::Utc::now().to_rfc3339();
|
let now = chrono::Utc::now().to_rfc3339();
|
||||||
let singleton_generation = match singleton_owner.as_ref() {
|
let singleton_generation = match singleton_owner.as_ref() {
|
||||||
@@ -3197,13 +3272,15 @@ impl SqliteWorkspaceStore {
|
|||||||
));
|
));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
tx.commit()?;
|
let reservation = WorkerCreateReservation {
|
||||||
Ok(WorkerCreateReservation {
|
|
||||||
worker_id,
|
worker_id,
|
||||||
create_fingerprint,
|
create_fingerprint,
|
||||||
memory_settings: snapshot,
|
memory_settings: snapshot,
|
||||||
singleton,
|
singleton,
|
||||||
})
|
};
|
||||||
|
complete(&tx, &reservation)?;
|
||||||
|
tx.commit()?;
|
||||||
|
Ok(reservation)
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -6952,10 +7029,11 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
fn register_verified_passkey(
|
fn register_verified_passkey_with_result(
|
||||||
&self,
|
&self,
|
||||||
target: &PasskeyRegistrationTarget,
|
target: &PasskeyRegistrationTarget,
|
||||||
credential: &PasskeyCredentialRecord,
|
credential: &PasskeyCredentialRecord,
|
||||||
|
addition_challenge_id: Option<&str>,
|
||||||
) -> Result<()> {
|
) -> Result<()> {
|
||||||
self.with_immediate_transaction(|tx| {
|
self.with_immediate_transaction(|tx| {
|
||||||
let user = match target {
|
let user = match target {
|
||||||
@@ -7022,6 +7100,15 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
|||||||
"INSERT INTO passkey_credentials (credential_id, user_id, public_key_cose, transports_json, sign_count, created_at, last_used_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)",
|
"INSERT INTO passkey_credentials (credential_id, user_id, public_key_cose, transports_json, sign_count, created_at, last_used_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)",
|
||||||
params![credential.credential_id, credential.user_id, credential.public_key_cose, credential.transports_json, credential.sign_count, credential.created_at, credential.last_used_at],
|
params![credential.credential_id, credential.user_id, credential.public_key_cose, credential.transports_json, credential.sign_count, credential.created_at, credential.last_used_at],
|
||||||
)?;
|
)?;
|
||||||
|
if let Some(challenge_id) = addition_challenge_id {
|
||||||
|
let PasskeyRegistrationTarget::Addition { user, authentication } = target else {
|
||||||
|
return Err(crate::auth::auth_error("passkey_target_mismatch", "receipt requires an addition"));
|
||||||
|
};
|
||||||
|
tx.execute(
|
||||||
|
"INSERT INTO passkey_addition_receipts(challenge_id, credential_id, user_id, authentication_reference) VALUES (?1, ?2, ?3, ?4)",
|
||||||
|
params![challenge_id, credential.credential_id, user.user_id, authentication.reference()],
|
||||||
|
)?;
|
||||||
|
}
|
||||||
if let PasskeyRegistrationTarget::Activation { user, issuance_id } = target {
|
if let PasskeyRegistrationTarget::Activation { user, issuance_id } = target {
|
||||||
if tx.execute("DELETE FROM user_activations WHERE user_id=?1 AND issuance_id=?2", params![user.user_id, issuance_id])? != 1 {
|
if tx.execute("DELETE FROM user_activations WHERE user_id=?1 AND issuance_id=?2", params![user.user_id, issuance_id])? != 1 {
|
||||||
return Err(crate::auth::auth_error("activation_code_invalid", "activation issuance no longer exists"));
|
return Err(crate::auth::auth_error("activation_code_invalid", "activation issuance no longer exists"));
|
||||||
@@ -7031,6 +7118,57 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn list_passkey_summaries(&self, user_id: &str) -> Result<Vec<server_api::PasskeySummary>> {
|
||||||
|
self.with_conn(|conn| {
|
||||||
|
let mut stmt = conn.prepare(
|
||||||
|
"SELECT m.key_reference, c.created_at, c.last_used_at, m.normal_login_verified_at
|
||||||
|
FROM passkey_credentials c JOIN passkey_metadata m USING(credential_id)
|
||||||
|
WHERE c.user_id = ?1 ORDER BY c.created_at, m.key_reference",
|
||||||
|
)?;
|
||||||
|
let rows = stmt.query_map([user_id], read_passkey_summary)?;
|
||||||
|
rows.collect::<std::result::Result<Vec<_>, _>>()
|
||||||
|
.map_err(Error::from)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_passkey_addition_result(
|
||||||
|
&self,
|
||||||
|
user_id: &str,
|
||||||
|
original_authentication_reference: &str,
|
||||||
|
challenge_id: &str,
|
||||||
|
) -> Result<Option<server_api::PasskeySummary>> {
|
||||||
|
self.with_conn(|conn| {
|
||||||
|
// Current authenticated ownership authorizes reading. The original
|
||||||
|
// binding reference identifies the saved operation, not the current
|
||||||
|
// session: Set-Cookie may have arrived even if its body was lost.
|
||||||
|
let result = conn.query_row(
|
||||||
|
"SELECT m.key_reference, c.created_at, c.last_used_at, m.normal_login_verified_at, r.authentication_reference
|
||||||
|
FROM passkey_addition_receipts r
|
||||||
|
JOIN passkey_credentials c USING(credential_id)
|
||||||
|
JOIN passkey_metadata m USING(credential_id)
|
||||||
|
WHERE r.challenge_id = ?1 AND r.user_id = ?2 AND c.user_id = ?2",
|
||||||
|
params![challenge_id, user_id],
|
||||||
|
|row| Ok((read_passkey_summary(row)?, row.get::<_, String>(4)?)),
|
||||||
|
).optional()?;
|
||||||
|
match result {
|
||||||
|
Some((key, reference)) if reference == original_authentication_reference => Ok(Some(key)),
|
||||||
|
Some(_) => Err(crate::auth::auth_error("passkey_result_binding_mismatch", "original operation authentication reference does not match")),
|
||||||
|
None => Ok(None),
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
fn mark_passkey_normal_login_verified(
|
||||||
|
&self,
|
||||||
|
credential_id: &str,
|
||||||
|
verified_at: &str,
|
||||||
|
) -> Result<()> {
|
||||||
|
self.with_conn(|conn| {
|
||||||
|
conn.execute("UPDATE passkey_metadata SET normal_login_verified_at = ?2 WHERE credential_id = ?1", params![credential_id, verified_at])?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
fn upsert_passkey_credential(&self, record: &PasskeyCredentialRecord) -> Result<()> {
|
fn upsert_passkey_credential(&self, record: &PasskeyCredentialRecord) -> Result<()> {
|
||||||
self.with_immediate_transaction(|conn| {
|
self.with_immediate_transaction(|conn| {
|
||||||
activation::require_active(conn, &record.user_id)?;
|
activation::require_active(conn, &record.user_id)?;
|
||||||
@@ -10981,49 +11119,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
|||||||
record.workdir_id
|
record.workdir_id
|
||||||
)));
|
)));
|
||||||
}
|
}
|
||||||
let active_for_alias = tx
|
// Ownership is keyed by Workdir, not by the requested local name.
|
||||||
.query_row(
|
|
||||||
r#"SELECT workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, linked_at, unlinked_at, connection_id
|
|
||||||
FROM worker_workdir_links
|
|
||||||
WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3
|
|
||||||
AND alias = ?4 AND unlinked_at IS NULL"#,
|
|
||||||
params![
|
|
||||||
record.workspace_id,
|
|
||||||
record.worker.runtime_id,
|
|
||||||
record.worker.worker_id,
|
|
||||||
record.alias,
|
|
||||||
],
|
|
||||||
read_worker_workdir_link_record,
|
|
||||||
)
|
|
||||||
.optional()?;
|
|
||||||
if let Some(mut active) = active_for_alias {
|
|
||||||
if active.workdir_id == record.workdir_id {
|
|
||||||
let capabilities = active.capabilities.intersection(record.capabilities);
|
|
||||||
if capabilities != active.capabilities {
|
|
||||||
tx.execute(
|
|
||||||
r#"UPDATE worker_workdir_links
|
|
||||||
SET capabilities = ?1
|
|
||||||
WHERE workspace_id = ?2 AND runtime_id = ?3 AND worker_id = ?4
|
|
||||||
AND workdir_id = ?5 AND alias = ?6 AND unlinked_at IS NULL"#,
|
|
||||||
params![
|
|
||||||
encode_workdir_link_capabilities(capabilities)?,
|
|
||||||
record.workspace_id,
|
|
||||||
record.worker.runtime_id,
|
|
||||||
record.worker.worker_id,
|
|
||||||
record.workdir_id,
|
|
||||||
record.alias,
|
|
||||||
],
|
|
||||||
)?;
|
|
||||||
active.capabilities = capabilities;
|
|
||||||
}
|
|
||||||
tx.commit()?;
|
|
||||||
return Ok(active);
|
|
||||||
}
|
|
||||||
return Err(Error::WorkdirAttachmentConflict(format!(
|
|
||||||
"Worker {}:{} attachment alias `{}` is already bound to Workdir {}",
|
|
||||||
record.worker.runtime_id, record.worker.worker_id, record.alias, active.workdir_id
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
let active_for_workdir = tx
|
let active_for_workdir = tx
|
||||||
.query_row(
|
.query_row(
|
||||||
r#"SELECT workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, linked_at, unlinked_at, connection_id
|
r#"SELECT workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, linked_at, unlinked_at, connection_id
|
||||||
@@ -11033,10 +11129,45 @@ impl ControlPlaneStore for SqliteWorkspaceStore {
|
|||||||
read_worker_workdir_link_record,
|
read_worker_workdir_link_record,
|
||||||
)
|
)
|
||||||
.optional()?;
|
.optional()?;
|
||||||
if let Some(active) = active_for_workdir {
|
if let Some(mut active) = active_for_workdir {
|
||||||
|
if active.worker != record.worker {
|
||||||
|
return Err(Error::WorkdirOccupied {
|
||||||
|
workdir_id: record.workdir_id.clone(),
|
||||||
|
runtime_id: active.worker.runtime_id,
|
||||||
|
worker_id: active.worker.worker_id,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
let capabilities = active.capabilities.intersection(record.capabilities);
|
||||||
|
if capabilities != active.capabilities {
|
||||||
|
tx.execute(
|
||||||
|
r#"UPDATE worker_workdir_links SET capabilities = ?1
|
||||||
|
WHERE workspace_id = ?2 AND runtime_id = ?3 AND worker_id = ?4
|
||||||
|
AND workdir_id = ?5 AND alias = ?6 AND unlinked_at IS NULL"#,
|
||||||
|
params![
|
||||||
|
encode_workdir_link_capabilities(capabilities)?,
|
||||||
|
active.workspace_id, active.worker.runtime_id, active.worker.worker_id,
|
||||||
|
active.workdir_id, active.alias,
|
||||||
|
],
|
||||||
|
)?;
|
||||||
|
active.capabilities = capabilities;
|
||||||
|
}
|
||||||
|
tx.commit()?;
|
||||||
|
return Ok(active);
|
||||||
|
}
|
||||||
|
let active_for_alias = tx
|
||||||
|
.query_row(
|
||||||
|
r#"SELECT workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, linked_at, unlinked_at, connection_id
|
||||||
|
FROM worker_workdir_links
|
||||||
|
WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3
|
||||||
|
AND alias = ?4 AND unlinked_at IS NULL"#,
|
||||||
|
params![record.workspace_id, record.worker.runtime_id, record.worker.worker_id, record.alias],
|
||||||
|
read_worker_workdir_link_record,
|
||||||
|
)
|
||||||
|
.optional()?;
|
||||||
|
if let Some(active) = active_for_alias {
|
||||||
return Err(Error::WorkdirAttachmentConflict(format!(
|
return Err(Error::WorkdirAttachmentConflict(format!(
|
||||||
"Workdir {} is already attached to Worker {}:{}",
|
"Worker {}:{} attachment alias `{}` is already bound to Workdir {}",
|
||||||
record.workdir_id, active.worker.runtime_id, active.worker.worker_id
|
record.worker.runtime_id, record.worker.worker_id, record.alias, active.workdir_id
|
||||||
)));
|
)));
|
||||||
}
|
}
|
||||||
// Direct attachments (including ordinary recovery/reconnection)
|
// Direct attachments (including ordinary recovery/reconnection)
|
||||||
@@ -11939,6 +12070,15 @@ fn passkey_select_sql(where_clause: &str) -> String {
|
|||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn read_passkey_summary(row: &rusqlite::Row<'_>) -> rusqlite::Result<server_api::PasskeySummary> {
|
||||||
|
Ok(server_api::PasskeySummary {
|
||||||
|
key_reference: row.get(0)?,
|
||||||
|
created_at: row.get(1)?,
|
||||||
|
last_used_at: row.get(2)?,
|
||||||
|
normal_login_verified_at: row.get(3)?,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
fn read_passkey_credential_record(
|
fn read_passkey_credential_record(
|
||||||
row: &rusqlite::Row<'_>,
|
row: &rusqlite::Row<'_>,
|
||||||
) -> rusqlite::Result<PasskeyCredentialRecord> {
|
) -> rusqlite::Result<PasskeyCredentialRecord> {
|
||||||
@@ -15218,6 +15358,22 @@ fn migrate_result_reports_v97_to_v98(conn: &Connection) -> Result<()> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn migrate_passkey_results_v102_to_v103(conn: &Connection) -> Result<()> {
|
||||||
|
if current_schema_version(conn)? != 102 {
|
||||||
|
return Err(Error::Store(
|
||||||
|
"Passkey results migration requires schema 102".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let tx = conn.unchecked_transaction()?;
|
||||||
|
tx.execute_batch(include_str!("passkey_results_schema.sql"))?;
|
||||||
|
tx.execute(
|
||||||
|
"INSERT INTO __yoi_schema_migrations(version,name) VALUES(103,?1)",
|
||||||
|
["Passkey metadata and atomic addition results"],
|
||||||
|
)?;
|
||||||
|
tx.commit()?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
fn migrate_job_input_retention_v102_to_v103(conn: &Connection) -> Result<()> {
|
fn migrate_job_input_retention_v102_to_v103(conn: &Connection) -> Result<()> {
|
||||||
if current_schema_version(conn)? != 102 {
|
if current_schema_version(conn)? != 102 {
|
||||||
return Err(Error::Store(
|
return Err(Error::Store(
|
||||||
@@ -15236,6 +15392,73 @@ fn migrate_job_input_retention_v102_to_v103(conn: &Connection) -> Result<()> {
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
fn migrate_schema_103_variants_to_v104(conn: &Connection) -> Result<()> {
|
||||||
|
if current_schema_version(conn)? != 103 {
|
||||||
|
return Err(Error::Store(
|
||||||
|
"Schema convergence migration requires schema 103".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
verify_schema_history(conn, 103)?;
|
||||||
|
let tx = conn.unchecked_transaction()?;
|
||||||
|
// Fresh databases on either branch recorded only a schema-103 baseline, so
|
||||||
|
// the receipt alone cannot identify which feature needs to be installed.
|
||||||
|
// Never rebuild existing metadata or receipts: key references and retired
|
||||||
|
// Job input dispositions are already externally observable saved values.
|
||||||
|
let passkey_results = table_exists(&tx, "passkey_metadata")?;
|
||||||
|
let job_retention = column_exists(&tx, "backend_jobs", "input_disposition")?;
|
||||||
|
let identity: String = tx.query_row(
|
||||||
|
"SELECT name FROM __yoi_schema_migrations WHERE version=103",
|
||||||
|
[],
|
||||||
|
|row| row.get(0),
|
||||||
|
)?;
|
||||||
|
let matches_identity = match identity.as_str() {
|
||||||
|
SCHEMA_BASELINE_NAME => passkey_results != job_retention,
|
||||||
|
"Passkey metadata and atomic addition results" => passkey_results && !job_retention,
|
||||||
|
"Job input retirement independent of immutable receipts" => {
|
||||||
|
job_retention && !passkey_results
|
||||||
|
}
|
||||||
|
_ => false,
|
||||||
|
};
|
||||||
|
if !matches_identity {
|
||||||
|
return Err(Error::Store(
|
||||||
|
"Schema-103 features do not match the applied migration identity".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if !passkey_results {
|
||||||
|
tx.execute_batch(include_str!("passkey_results_schema.sql"))?;
|
||||||
|
}
|
||||||
|
if !job_retention {
|
||||||
|
tx.execute_batch("ALTER TABLE backend_jobs ADD COLUMN input_disposition TEXT NOT NULL DEFAULT 'available' CHECK (input_disposition IN ('available','reclaimed','erased','invalidated'));")?;
|
||||||
|
tx.execute_batch(crate::backend_job::INPUT_RETENTION_TRIGGERS)?;
|
||||||
|
}
|
||||||
|
let objects: i64 = tx.query_row(
|
||||||
|
"SELECT count(*) FROM sqlite_schema WHERE
|
||||||
|
(type='table' AND name IN ('passkey_metadata','passkey_addition_receipts')) OR
|
||||||
|
(type='trigger' AND name IN ('passkey_metadata_insert','backend_job_identity_immutable','backend_job_input_retirement'))",
|
||||||
|
[],
|
||||||
|
|row| row.get(0),
|
||||||
|
)?;
|
||||||
|
if objects != 5 {
|
||||||
|
return Err(Error::Store("Incomplete schema-103 feature schema".into()));
|
||||||
|
}
|
||||||
|
verify_workspace_resource_constraints(&tx)?;
|
||||||
|
let violations: i64 =
|
||||||
|
tx.query_row("SELECT count(*) FROM pragma_foreign_key_check", [], |row| {
|
||||||
|
row.get(0)
|
||||||
|
})?;
|
||||||
|
if violations != 0 {
|
||||||
|
return Err(Error::Store(
|
||||||
|
"Schema convergence foreign-key violation".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
tx.execute(
|
||||||
|
"INSERT INTO __yoi_schema_migrations(version,name) VALUES(104,?1)",
|
||||||
|
["Converge Passkey results and Job input retention"],
|
||||||
|
)?;
|
||||||
|
tx.commit()?;
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
fn migrate_subject_create_recovery_v101_to_v102(conn: &Connection) -> Result<()> {
|
fn migrate_subject_create_recovery_v101_to_v102(conn: &Connection) -> Result<()> {
|
||||||
if current_schema_version(conn)? != 101 {
|
if current_schema_version(conn)? != 101 {
|
||||||
return Err(Error::Store(
|
return Err(Error::Store(
|
||||||
@@ -15327,6 +15550,7 @@ fn create_latest_workspace_schema(conn: &Connection) -> Result<()> {
|
|||||||
fn create_workspace_schema_structure(conn: &Connection) -> Result<()> {
|
fn create_workspace_schema_structure(conn: &Connection) -> Result<()> {
|
||||||
create_workspace_schema_structure_from_sql(conn, include_str!("latest_schema.sql"), true)?;
|
create_workspace_schema_structure_from_sql(conn, include_str!("latest_schema.sql"), true)?;
|
||||||
conn.execute_batch(include_str!("activation_schema.sql"))?;
|
conn.execute_batch(include_str!("activation_schema.sql"))?;
|
||||||
|
conn.execute_batch(include_str!("passkey_results_schema.sql"))?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -15992,7 +16216,7 @@ fn current_worker_singleton_owner(
|
|||||||
CASE WHEN substr(owner.singleton_key,1,10)='subjektiv:' AND reservation.state='removed' \
|
CASE WHEN substr(owner.singleton_key,1,10)='subjektiv:' AND reservation.state='removed' \
|
||||||
AND NOT EXISTS(SELECT 1 FROM subject_worker_releases r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.completed=1) \
|
AND NOT EXISTS(SELECT 1 FROM subject_worker_releases r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.completed=1) \
|
||||||
AND NOT EXISTS(SELECT 1 FROM subject_create_recoveries r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.create_fingerprint=reservation.create_fingerprint AND r.completed=1) \
|
AND NOT EXISTS(SELECT 1 FROM subject_create_recoveries r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.singleton_key=owner.singleton_key AND r.singleton_generation=owner.generation AND r.create_fingerprint=reservation.create_fingerprint AND r.completed=1) \
|
||||||
THEN 'created' ELSE reservation.state END \
|
AND NOT EXISTS(SELECT 1 FROM worker_removal_operations r WHERE r.workspace_id=owner.workspace_id AND r.worker_id=owner.worker_id AND r.runtime_id=owner.runtime_id AND r.state='succeeded') THEN 'created' ELSE reservation.state END \
|
||||||
FROM worker_singleton_owners owner \
|
FROM worker_singleton_owners owner \
|
||||||
LEFT JOIN worker_create_reservations reservation \
|
LEFT JOIN worker_create_reservations reservation \
|
||||||
ON reservation.workspace_id = owner.workspace_id \
|
ON reservation.workspace_id = owner.workspace_id \
|
||||||
@@ -16198,6 +16422,17 @@ fn verify_schema_history(conn: &Connection, current: i64) -> Result<()> {
|
|||||||
.iter()
|
.iter()
|
||||||
.filter(|migration| migration.version > *baseline_version && migration.version <= current)
|
.filter(|migration| migration.version > *baseline_version && migration.version <= current)
|
||||||
{
|
{
|
||||||
|
// The two published schema-103 migrations are alternatives, not aliases.
|
||||||
|
// Match the exact saved identity while retaining every other history check.
|
||||||
|
let migration = if migration.version == JOB_INPUT_RETENTION_SCHEMA_103.version
|
||||||
|
&& rows.iter().any(|(version, name)| {
|
||||||
|
*version == JOB_INPUT_RETENTION_SCHEMA_103.version
|
||||||
|
&& name == JOB_INPUT_RETENTION_SCHEMA_103.name
|
||||||
|
}) {
|
||||||
|
&JOB_INPUT_RETENTION_SCHEMA_103
|
||||||
|
} else {
|
||||||
|
migration
|
||||||
|
};
|
||||||
expected.push((migration.version, migration.name.to_string()));
|
expected.push((migration.version, migration.name.to_string()));
|
||||||
}
|
}
|
||||||
if rows != expected || expected.last().map(|row| row.0) != Some(current) {
|
if rows != expected || expected.last().map(|row| row.0) != Some(current) {
|
||||||
@@ -18608,6 +18843,7 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
|||||||
|
|
||||||
include!("activation_tests.rs");
|
include!("activation_tests.rs");
|
||||||
include!("schema_equivalence_tests.rs");
|
include!("schema_equivalence_tests.rs");
|
||||||
|
include!("schema_convergence_tests.rs");
|
||||||
include!("ticket_tag_upgrade_tests.rs");
|
include!("ticket_tag_upgrade_tests.rs");
|
||||||
include!("ticket_post_upgrade_tests.rs");
|
include!("ticket_post_upgrade_tests.rs");
|
||||||
include!("migration_entrypoint_tests.rs");
|
include!("migration_entrypoint_tests.rs");
|
||||||
@@ -19020,7 +19256,8 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
|||||||
WorkspaceSchemaMigrationStep {version:100, name:"Provisional users and scoped passkey activation".into()},
|
WorkspaceSchemaMigrationStep {version:100, name:"Provisional users and scoped passkey activation".into()},
|
||||||
WorkspaceSchemaMigrationStep {version:101, name:"Durable explicit Subject body release".into()},
|
WorkspaceSchemaMigrationStep {version:101, name:"Durable explicit Subject body release".into()},
|
||||||
WorkspaceSchemaMigrationStep {version:102, name:"Durable failed Subject creation recovery".into()},
|
WorkspaceSchemaMigrationStep {version:102, name:"Durable failed Subject creation recovery".into()},
|
||||||
WorkspaceSchemaMigrationStep {version:103, name:"Job input retirement independent of immutable receipts".into()},
|
WorkspaceSchemaMigrationStep {version:103, name:"Passkey metadata and atomic addition results".into()},
|
||||||
|
WorkspaceSchemaMigrationStep {version:104, name:"Converge Passkey results and Job input retention".into()},
|
||||||
]
|
]
|
||||||
);
|
);
|
||||||
|
|
||||||
@@ -19133,7 +19370,8 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
|||||||
(100, "Provisional users and scoped passkey activation".to_string()),
|
(100, "Provisional users and scoped passkey activation".to_string()),
|
||||||
(101, "Durable explicit Subject body release".to_string()),
|
(101, "Durable explicit Subject body release".to_string()),
|
||||||
(102, "Durable failed Subject creation recovery".to_string()),
|
(102, "Durable failed Subject creation recovery".to_string()),
|
||||||
(103, "Job input retirement independent of immutable receipts".to_string()),
|
(103, "Passkey metadata and atomic addition results".to_string()),
|
||||||
|
(104, "Converge Passkey results and Job input retention".to_string()),
|
||||||
]
|
]
|
||||||
);
|
);
|
||||||
assert!(!table_exists(conn, "trusted_runtime_records")?);
|
assert!(!table_exists(conn, "trusted_runtime_records")?);
|
||||||
@@ -21362,6 +21600,83 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
|||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn passkey_results_migration_preserves_old_keys_unknown_confirmation_and_rolls_back() {
|
||||||
|
for baseline in [50, 102] {
|
||||||
|
let dir = tempfile::tempdir().unwrap();
|
||||||
|
let database = dir.path().join("server.db");
|
||||||
|
if baseline == 50 {
|
||||||
|
prepare_schema_v50(&database, None);
|
||||||
|
} else {
|
||||||
|
prepare_retained_schema(&database, baseline);
|
||||||
|
}
|
||||||
|
let conn = Connection::open(&database).unwrap();
|
||||||
|
configure_sqlite(&conn).unwrap();
|
||||||
|
conn.execute_batch(r#"INSERT INTO accounts(account_id, kind, handle, display_name, created_at, updated_at)
|
||||||
|
VALUES('account-old', 'user', 'old', 'Old', 'created', 'updated');
|
||||||
|
INSERT INTO users(user_id, account_id, handle, display_name, created_at, updated_at)
|
||||||
|
VALUES('user-old', 'account-old', 'old', 'Old', 'created', 'updated');
|
||||||
|
INSERT INTO passkey_credentials(credential_id, user_id, public_key_cose, transports_json, sign_count, created_at, last_used_at)
|
||||||
|
VALUES('old-key', 'user-old', 'unchanged-key-material', '["internal"]', 7, 'created-key', 'used-key');"#)
|
||||||
|
.unwrap();
|
||||||
|
if baseline == 102 {
|
||||||
|
conn.execute_batch("CREATE TRIGGER interrupt_passkey_migration BEFORE INSERT ON __yoi_schema_migrations
|
||||||
|
WHEN NEW.version=103 BEGIN SELECT RAISE(ABORT, 'synthetic migration interruption'); END;").unwrap();
|
||||||
|
assert!(migrate_passkey_results_v102_to_v103(&conn).is_err());
|
||||||
|
assert_eq!(current_schema_version(&conn).unwrap(), 102);
|
||||||
|
assert!(!table_exists(&conn, "passkey_metadata").unwrap());
|
||||||
|
assert!(!table_exists(&conn, "passkey_addition_receipts").unwrap());
|
||||||
|
conn.execute_batch("DROP TRIGGER interrupt_passkey_migration")
|
||||||
|
.unwrap();
|
||||||
|
}
|
||||||
|
drop(conn);
|
||||||
|
let preflight = SqliteWorkspaceStore::migration_plan(&database).unwrap();
|
||||||
|
assert!(preflight.migration_required());
|
||||||
|
// Preflight must not install metadata in the source database.
|
||||||
|
let conn = Connection::open(&database).unwrap();
|
||||||
|
assert!(!table_exists(&conn, "passkey_metadata").unwrap());
|
||||||
|
drop(conn);
|
||||||
|
let migrated = SqliteWorkspaceStore::open_for_test(&database).unwrap();
|
||||||
|
let record = migrated.get_passkey_credential("old-key").unwrap().unwrap();
|
||||||
|
assert_eq!(record.public_key_cose, "unchanged-key-material");
|
||||||
|
assert_eq!(record.sign_count, 7);
|
||||||
|
assert_eq!(record.last_used_at.as_deref(), Some("used-key"));
|
||||||
|
let summary = migrated
|
||||||
|
.list_passkey_summaries("user-old")
|
||||||
|
.unwrap()
|
||||||
|
.remove(0);
|
||||||
|
assert_eq!(summary.created_at, "created-key");
|
||||||
|
assert_eq!(summary.last_used_at.as_deref(), Some("used-key"));
|
||||||
|
assert!(summary.normal_login_verified_at.is_none());
|
||||||
|
assert_eq!(summary.key_reference.len(), 32);
|
||||||
|
let fresh = SqliteWorkspaceStore::in_memory_for_test().unwrap();
|
||||||
|
let schema = |conn: &Connection| -> Result<Vec<(String, String)>> {
|
||||||
|
let mut stmt = conn.prepare("SELECT name, sql FROM sqlite_schema WHERE name IN
|
||||||
|
('passkey_metadata', 'passkey_metadata_insert', 'passkey_addition_receipts') ORDER BY name")?;
|
||||||
|
Ok(stmt
|
||||||
|
.query_map([], |row| Ok((row.get(0)?, row.get(1)?)))?
|
||||||
|
.collect::<std::result::Result<Vec<_>, _>>()?)
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
migrated.with_conn(schema).unwrap(),
|
||||||
|
fresh.with_conn(schema).unwrap()
|
||||||
|
);
|
||||||
|
migrated.with_conn(|conn| {
|
||||||
|
assert!(conn.execute("INSERT INTO passkey_metadata(credential_id,key_reference) VALUES('missing-key','ref')", []).is_err());
|
||||||
|
assert!(conn.execute("INSERT INTO passkey_addition_receipts(challenge_id,credential_id,user_id,authentication_reference)
|
||||||
|
VALUES('operation','missing-key','user-old','{}')", []).is_err());
|
||||||
|
assert_eq!(conn.query_row("SELECT count(*) FROM pragma_foreign_key_check", [], |row| row.get::<_, u64>(0))?, 0);
|
||||||
|
Ok(())
|
||||||
|
}).unwrap();
|
||||||
|
drop(migrated);
|
||||||
|
let reopened = SqliteWorkspaceStore::open_for_test(&database).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
reopened.list_passkey_summaries("user-old").unwrap(),
|
||||||
|
vec![summary]
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn subject_create_recovery_migration_is_atomic_and_matches_fresh_schema() {
|
fn subject_create_recovery_migration_is_atomic_and_matches_fresh_schema() {
|
||||||
let dir = tempfile::tempdir().unwrap();
|
let dir = tempfile::tempdir().unwrap();
|
||||||
@@ -22341,7 +22656,8 @@ CREATE UNIQUE INDEX ticket_current_worker_role_idx
|
|||||||
(100, "Provisional users and scoped passkey activation".into()),
|
(100, "Provisional users and scoped passkey activation".into()),
|
||||||
(101, "Durable explicit Subject body release".into()),
|
(101, "Durable explicit Subject body release".into()),
|
||||||
(102, "Durable failed Subject creation recovery".into()),
|
(102, "Durable failed Subject creation recovery".into()),
|
||||||
(103, "Job input retirement independent of immutable receipts".into())]);
|
(103, "Passkey metadata and atomic addition results".into()),
|
||||||
|
(104, "Converge Passkey results and Job input retention".into())]);
|
||||||
for (table, rows) in unchanged.iter().zip(before.iter()) {
|
for (table, rows) in unchanged.iter().zip(before.iter()) {
|
||||||
assert_eq!(&stored_fixture_rows(conn, table), rows, "preserve {table}");
|
assert_eq!(&stored_fixture_rows(conn, table), rows, "preserve {table}");
|
||||||
}
|
}
|
||||||
@@ -25173,6 +25489,24 @@ INSERT INTO worker_registry (
|
|||||||
2
|
2
|
||||||
);
|
);
|
||||||
|
|
||||||
|
for alias in ["another-name", "docs"] {
|
||||||
|
let requested = WorkerWorkdirLinkRecord {
|
||||||
|
alias: alias.into(),
|
||||||
|
..link.clone()
|
||||||
|
};
|
||||||
|
assert_eq!(
|
||||||
|
store.attach_worker_workdir(&requested).unwrap(),
|
||||||
|
downgraded_link
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
store
|
||||||
|
.list_worker_workdir_links("local-dev", &worker.worker)
|
||||||
|
.unwrap()
|
||||||
|
.len(),
|
||||||
|
2
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
let second_worker = WorkerRegistryRecord {
|
let second_worker = WorkerRegistryRecord {
|
||||||
worker: RuntimeWorkerRef::new("embedded", "2"),
|
worker: RuntimeWorkerRef::new("embedded", "2"),
|
||||||
display_name: "Browser 2".to_string(),
|
display_name: "Browser 2".to_string(),
|
||||||
@@ -25188,7 +25522,7 @@ INSERT INTO worker_registry (
|
|||||||
};
|
};
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
store.attach_worker_workdir(&workdir_conflict),
|
store.attach_worker_workdir(&workdir_conflict),
|
||||||
Err(Error::WorkdirAttachmentConflict(_))
|
Err(Error::WorkdirOccupied { .. })
|
||||||
));
|
));
|
||||||
assert!(matches!(
|
assert!(matches!(
|
||||||
store.detach_worker_workdir("local-dev", &worker.worker, Some("wrong-workdir"), "6",),
|
store.detach_worker_workdir("local-dev", &worker.worker, Some("wrong-workdir"), "6",),
|
||||||
@@ -26536,16 +26870,21 @@ INSERT INTO worker_registry (
|
|||||||
store.with_conn(|conn| {
|
store.with_conn(|conn| {
|
||||||
conn.execute_batch("DROP TRIGGER backend_job_input_retirement; DROP TRIGGER backend_job_identity_immutable;
|
conn.execute_batch("DROP TRIGGER backend_job_input_retirement; DROP TRIGGER backend_job_identity_immutable;
|
||||||
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
|
ALTER TABLE backend_jobs DROP COLUMN input_disposition;
|
||||||
UPDATE __yoi_schema_migrations SET version=102 WHERE version=103;
|
DROP TRIGGER passkey_metadata_insert;
|
||||||
|
DROP TABLE passkey_addition_receipts;
|
||||||
|
DROP TABLE passkey_metadata;
|
||||||
|
UPDATE __yoi_schema_migrations SET version=102 WHERE version=104;
|
||||||
CREATE TRIGGER backend_job_input_retirement BEFORE UPDATE ON backend_job_attempts BEGIN SELECT RAISE(ABORT,'collision'); END;")?; Ok(())
|
CREATE TRIGGER backend_job_input_retirement BEFORE UPDATE ON backend_job_attempts BEGIN SELECT RAISE(ABORT,'collision'); END;")?; Ok(())
|
||||||
}).unwrap();
|
}).unwrap();
|
||||||
drop(store);
|
drop(store);
|
||||||
assert!(SqliteWorkspaceStore::open_for_test(&path).is_err());
|
|
||||||
let conn = Connection::open(&path).unwrap();
|
let conn = Connection::open(&path).unwrap();
|
||||||
|
configure_sqlite(&conn).unwrap();
|
||||||
|
assert!(migrate_job_input_retention_v102_to_v103(&conn).is_err());
|
||||||
assert_eq!(current_schema_version(&conn).unwrap(), 102);
|
assert_eq!(current_schema_version(&conn).unwrap(), 102);
|
||||||
assert!(!column_exists(&conn, "backend_jobs", "input_disposition").unwrap());
|
assert!(!column_exists(&conn, "backend_jobs", "input_disposition").unwrap());
|
||||||
conn.execute_batch("DROP TRIGGER backend_job_input_retirement")
|
conn.execute_batch("DROP TRIGGER backend_job_input_retirement")
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
migrate_job_input_retention_v102_to_v103(&conn).unwrap();
|
||||||
drop(conn);
|
drop(conn);
|
||||||
let reopened = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
let reopened = SqliteWorkspaceStore::open_for_test(&path).unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
|
|||||||
@@ -0,0 +1,688 @@
|
|||||||
|
//! Offline Backend fixture preparation, available only with `test-support`.
|
||||||
|
//!
|
||||||
|
//! Use a new isolated database and the existing account, Workspace signing-identity /
|
||||||
|
//! `ControlPlaneStore::create_workspace_bootstrap`, and Runtime binding APIs first.
|
||||||
|
//! Generate fresh account credentials and signing keys for each environment; no fixture
|
||||||
|
//! credentials are accepted here. Install Profile configuration through the existing
|
||||||
|
//! config-source mechanisms before starting the product processes.
|
||||||
|
//!
|
||||||
|
//! This module seeds Backend ownership only. It neither writes Runtime storage nor
|
||||||
|
//! launches a Worker, fabricates Runtime observations, changes authorization, or sends
|
||||||
|
//! initial input. The caller must separately install the matching stopped Runtime
|
||||||
|
//! fixture. Build product binaries in a separate Cargo invocation **without** this
|
||||||
|
//! feature (Cargo features unify within an invocation).
|
||||||
|
|
||||||
|
use super::*;
|
||||||
|
use crate::hosts::{WorkerSpawnIntent, WorkerSpawnRequest, worker_spawn_create_fingerprint};
|
||||||
|
use worker_runtime::catalog::ProfileSelector;
|
||||||
|
|
||||||
|
/// One ordinary, attachment-free Worker from an externally validated fixture.
|
||||||
|
///
|
||||||
|
/// IDs must match the Runtime fixture. `request` is the unresolved canonical create
|
||||||
|
/// intent, not a Runtime launch request containing credentials or execution grants.
|
||||||
|
/// The Profile selector must already be installed by the harness. `initial_submit`
|
||||||
|
/// must be empty: saved Session contents belong to the Runtime fixture, not new input.
|
||||||
|
/// Singleton/service, Ticket-assigned and attachment-bearing Workers are deliberately
|
||||||
|
/// unsupported.
|
||||||
|
pub struct FixedWorkerSeed<'a> {
|
||||||
|
pub workspace_id: &'a str,
|
||||||
|
pub runtime_id: &'a str,
|
||||||
|
pub worker_id: WorkerId,
|
||||||
|
pub allocation_key: &'a str,
|
||||||
|
pub request: &'a WorkerSpawnRequest,
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Insert a fixed Worker catalog entry and completed creation reservation atomically.
|
||||||
|
///
|
||||||
|
/// Uses the normal reservation validators, authoritative Memory snapshot, fingerprint
|
||||||
|
/// binding, resource-key allocator and immediate transaction. Even an exact repeat is
|
||||||
|
/// rejected: this is not an upsert/import/recovery API. Existing allocations, Worker
|
||||||
|
/// identities and residual lifecycle/attachment records are never adopted or repaired.
|
||||||
|
/// Returns the creation binding needed to check the corresponding Runtime fixture.
|
||||||
|
pub fn seed_fixed_worker(
|
||||||
|
store: &SqliteWorkspaceStore,
|
||||||
|
seed: FixedWorkerSeed<'_>,
|
||||||
|
) -> Result<WorkerCreateReservation> {
|
||||||
|
validate_identifier("workspace_id", seed.workspace_id)?;
|
||||||
|
validate_identifier("runtime_id", seed.runtime_id)?;
|
||||||
|
let request = seed.request;
|
||||||
|
if !request.initial_submit.is_empty() {
|
||||||
|
return Err(Error::InvalidInput(
|
||||||
|
"fixture Worker cannot contain initial_submit; offline seeding accepts no new input"
|
||||||
|
.into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if !matches!(request.intent, WorkerSpawnIntent::WorkspaceCoding)
|
||||||
|
|| request.singleton_key.is_some()
|
||||||
|
|| request.ticket_assignment.is_some()
|
||||||
|
|| request.resolved_subjektiv_attached
|
||||||
|
|| !request.workdir_attachment_requests.is_empty()
|
||||||
|
|| !request.resolved_workdir_attachment_requests.is_empty()
|
||||||
|
|| !request.resolved_workdir_attachments.is_empty()
|
||||||
|
|| request.resolved_config_bundle.is_some()
|
||||||
|
|| request.resolved_workspace_api.is_some()
|
||||||
|
|| request.resolved_memory_settings.is_some()
|
||||||
|
|| request.resolved_worker_observation_enabled
|
||||||
|
|| !request.resolved_worker_observation_grants.is_empty()
|
||||||
|
|| request.resolved_control_operation.is_some()
|
||||||
|
|| request.resolved_execution_grant.is_some()
|
||||||
|
{
|
||||||
|
return Err(Error::InvalidInput(
|
||||||
|
"fixture Worker must be ordinary unresolved create intent without singleton, assignments, attachments or grants".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
// Builtin selectors already carry their canonical `builtin:` prefix.
|
||||||
|
let profile = match &request.profile {
|
||||||
|
ProfileSelector::Builtin(name) | ProfileSelector::Named(name) => name,
|
||||||
|
};
|
||||||
|
validate_non_empty("fixture Profile selector", profile)?;
|
||||||
|
let worker_id = seed.worker_id.to_string();
|
||||||
|
let display_name = request
|
||||||
|
.requested_worker_name
|
||||||
|
.as_deref()
|
||||||
|
.unwrap_or(&worker_id);
|
||||||
|
validate_non_empty("fixture Worker display name", display_name)?;
|
||||||
|
let fingerprint = worker_spawn_create_fingerprint(request).map_err(Error::InvalidInput)?;
|
||||||
|
let memory = store.get_workspace_memory_settings(seed.workspace_id)?;
|
||||||
|
store.reserve_worker_create_with(
|
||||||
|
seed.workspace_id,
|
||||||
|
seed.runtime_id,
|
||||||
|
seed.allocation_key,
|
||||||
|
&fingerprint,
|
||||||
|
None,
|
||||||
|
&memory,
|
||||||
|
|tx| {
|
||||||
|
require_seed_target(tx, &seed)?;
|
||||||
|
Ok(Some(seed.worker_id))
|
||||||
|
},
|
||||||
|
|tx, reservation| {
|
||||||
|
let now = chrono::Utc::now().to_rfc3339();
|
||||||
|
tx.execute(
|
||||||
|
"INSERT INTO worker_registry (
|
||||||
|
workspace_id, runtime_id, worker_id, display_name, profile,
|
||||||
|
retention_state, created_at, updated_at
|
||||||
|
) VALUES (?1, ?2, ?3, ?4, ?5, 'normal', ?6, ?6)",
|
||||||
|
params![seed.workspace_id, seed.runtime_id, worker_id, display_name, profile, now],
|
||||||
|
)?;
|
||||||
|
let changed = tx.execute(
|
||||||
|
"UPDATE worker_create_reservations SET state = 'created', updated_at = ?4
|
||||||
|
WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3 AND state = 'reserved'",
|
||||||
|
params![seed.workspace_id, seed.runtime_id, reservation.worker_id.to_string(), now],
|
||||||
|
)?;
|
||||||
|
if changed != 1 {
|
||||||
|
return Err(Error::Store("fixture creation reservation disappeared".into()));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
},
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn require_seed_target(tx: &rusqlite::Transaction<'_>, seed: &FixedWorkerSeed<'_>) -> Result<()> {
|
||||||
|
let active: bool = tx.query_row(
|
||||||
|
"SELECT EXISTS(SELECT 1 FROM workspaces w
|
||||||
|
JOIN workspace_runtime_bindings r ON r.workspace_id = w.workspace_id
|
||||||
|
JOIN workspace_signing_identities s ON s.workspace_id = w.workspace_id
|
||||||
|
WHERE w.workspace_id = ?1 AND w.state = 'active'
|
||||||
|
AND r.runtime_id = ?2 AND r.revoked_at IS NULL
|
||||||
|
AND r.authentication_mode = 'workspace_identity' AND s.state = 'active')",
|
||||||
|
params![seed.workspace_id, seed.runtime_id],
|
||||||
|
|row| row.get(0),
|
||||||
|
)?;
|
||||||
|
if !active {
|
||||||
|
return Err(Error::InvalidInput(
|
||||||
|
"fixture requires an active bootstrapped Workspace and authenticated Runtime binding"
|
||||||
|
.into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let worker_id = seed.worker_id.to_string();
|
||||||
|
let allocation_exists: bool = tx.query_row(
|
||||||
|
"SELECT EXISTS(SELECT 1 FROM worker_create_reservations
|
||||||
|
WHERE workspace_id = ?1 AND allocation_key = ?2)",
|
||||||
|
params![seed.workspace_id, seed.allocation_key],
|
||||||
|
|row| row.get(0),
|
||||||
|
)?;
|
||||||
|
if allocation_exists {
|
||||||
|
return Err(Error::InvalidInput(
|
||||||
|
"fixture Worker allocation already exists".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
// Check by WorkerId, not just the proposed Runtime tuple: orphaned and terminal
|
||||||
|
// ownership is still a collision. These tables are owned by this store schema.
|
||||||
|
for table in [
|
||||||
|
"worker_create_reservations",
|
||||||
|
"worker_registry",
|
||||||
|
"worker_registry_observations",
|
||||||
|
"worker_registry_projection_removals",
|
||||||
|
"worker_removal_operations",
|
||||||
|
"worker_singleton_owners",
|
||||||
|
"worker_workdir_links",
|
||||||
|
"worker_restore_intents",
|
||||||
|
"worker_tombstones",
|
||||||
|
"worker_orphan_diagnostics",
|
||||||
|
"worker_registry_projection_diagnostics",
|
||||||
|
"worker_session_archives",
|
||||||
|
"worker_diagnostics_archives",
|
||||||
|
"ticket_assignment_worker_tombstones",
|
||||||
|
] {
|
||||||
|
let exists: bool = tx.query_row(
|
||||||
|
&format!("SELECT EXISTS(SELECT 1 FROM {table} WHERE worker_id = ?1)"),
|
||||||
|
[&worker_id],
|
||||||
|
|row| row.get(0),
|
||||||
|
)?;
|
||||||
|
if exists {
|
||||||
|
return Err(Error::InvalidInput(format!(
|
||||||
|
"fixture Worker identity already exists in {table}"
|
||||||
|
)));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A resource key without a catalog/reservation must not be silently reused.
|
||||||
|
let mut statement = tx.prepare(
|
||||||
|
"SELECT resource_id FROM workspace_resource_keys WHERE resource_kind = 'worker'",
|
||||||
|
)?;
|
||||||
|
let resources = statement.query_map([], |row| row.get::<_, String>(0))?;
|
||||||
|
for resource in resources {
|
||||||
|
let resource = resource?;
|
||||||
|
let existing = resource
|
||||||
|
.is_ascii()
|
||||||
|
.then(|| parse_worker_resource_id(&resource))
|
||||||
|
.flatten()
|
||||||
|
.ok_or_else(|| {
|
||||||
|
Error::InvalidInput("unknown fixture Worker resource identity collision".into())
|
||||||
|
})?;
|
||||||
|
if existing.worker_id == worker_id {
|
||||||
|
return Err(Error::InvalidInput(
|
||||||
|
"fixture Worker resource identity already exists".into(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use crate::hosts::{
|
||||||
|
WorkerSpawnAcceptanceRequirement, WorkerSpawnWorkingDirectoryAttachmentRequest,
|
||||||
|
WorkerSpawnWorkingDirectoryRequest,
|
||||||
|
};
|
||||||
|
use crate::workspace_signing_identity::{
|
||||||
|
InMemoryWorkspaceSigningMaterialStore, WorkspaceSigningIdentityService,
|
||||||
|
};
|
||||||
|
|
||||||
|
const WORKSPACE: &str = "0197a949-4b6b-7f2a-9d9a-1f87e3a4c501";
|
||||||
|
const RUNTIME: &str = "fixture-runtime";
|
||||||
|
const WORKER: &str = "0197a949-4b6b-7f2a-9d9a-1f87e3a4c502";
|
||||||
|
|
||||||
|
// Exercises the same public bootstrap APIs the offline harness uses. All account
|
||||||
|
// and key identities are newly generated, even across stores with fixed fixture IDs.
|
||||||
|
fn bootstrap() -> SqliteWorkspaceStore {
|
||||||
|
let store = SqliteWorkspaceStore::in_memory_for_test().unwrap();
|
||||||
|
let account = AccountRecord {
|
||||||
|
account_id: format!("fixture-account-{}", Uuid::now_v7()),
|
||||||
|
kind: "user".into(),
|
||||||
|
handle: format!("fixture-{}", Uuid::now_v7()),
|
||||||
|
display_name: "Fixture owner".into(),
|
||||||
|
created_at: "2026-10-24T00:00:00Z".into(),
|
||||||
|
updated_at: "2026-10-24T00:00:00Z".into(),
|
||||||
|
};
|
||||||
|
store.upsert_account(&account).unwrap();
|
||||||
|
let identities = WorkspaceSigningIdentityService::new(
|
||||||
|
Arc::new(store.clone()),
|
||||||
|
Arc::new(InMemoryWorkspaceSigningMaterialStore::default()),
|
||||||
|
);
|
||||||
|
let (identity, operation) = identities
|
||||||
|
.prepare_workspace_creation(
|
||||||
|
"fixture-bootstrap",
|
||||||
|
"fixture-bootstrap",
|
||||||
|
WORKSPACE,
|
||||||
|
&account.account_id,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
store
|
||||||
|
.create_workspace_bootstrap(
|
||||||
|
&WorkspaceBootstrapRecord {
|
||||||
|
operation_key: "fixture-bootstrap".into(),
|
||||||
|
request_fingerprint: "fixture-bootstrap".into(),
|
||||||
|
workspace: WorkspaceRecord {
|
||||||
|
workspace_id: WORKSPACE.into(),
|
||||||
|
owner_account_id: account.account_id,
|
||||||
|
display_name: "Fixture".into(),
|
||||||
|
state: "active".into(),
|
||||||
|
created_at: account.created_at,
|
||||||
|
updated_at: account.updated_at,
|
||||||
|
},
|
||||||
|
repository: None,
|
||||||
|
},
|
||||||
|
&identity,
|
||||||
|
&operation,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let runtime_key = worker_runtime::auth::RuntimeIdentityMaterial::generate(RUNTIME).unwrap();
|
||||||
|
store
|
||||||
|
.upsert_workspace_runtime_binding(
|
||||||
|
WorkspaceRuntimeBinding {
|
||||||
|
workspace_id: WORKSPACE.into(),
|
||||||
|
runtime_id: RUNTIME.into(),
|
||||||
|
display_name: "Fixture Runtime".into(),
|
||||||
|
base_url: "http://127.0.0.1:12345".into(),
|
||||||
|
public_key: runtime_key.public_key,
|
||||||
|
public_key_fingerprint: String::new(),
|
||||||
|
binding_id: String::new(),
|
||||||
|
authentication_mode: WorkspaceRuntimeAuthenticationMode::WorkspaceIdentity,
|
||||||
|
created_at: "2026-10-24T00:00:00Z".into(),
|
||||||
|
updated_at: "2026-10-24T00:00:00Z".into(),
|
||||||
|
revoked_at: None,
|
||||||
|
},
|
||||||
|
false,
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
store
|
||||||
|
}
|
||||||
|
|
||||||
|
fn request() -> WorkerSpawnRequest {
|
||||||
|
WorkerSpawnRequest {
|
||||||
|
intent: WorkerSpawnIntent::WorkspaceCoding,
|
||||||
|
requested_worker_name: Some("Fixture Worker".into()),
|
||||||
|
singleton_key: None,
|
||||||
|
acceptance: WorkerSpawnAcceptanceRequirement::SocketReady,
|
||||||
|
profile: ProfileSelector::Builtin("builtin:coder".into()),
|
||||||
|
ticket_assignment: None,
|
||||||
|
initial_submit: Vec::new(),
|
||||||
|
workdir_attachment_requests: Vec::new(),
|
||||||
|
resolved_workdir_attachment_requests: Vec::new(),
|
||||||
|
resolved_workdir_attachments: Vec::new(),
|
||||||
|
resolved_config_bundle: None,
|
||||||
|
resolved_workspace_api: None,
|
||||||
|
resolved_memory_settings: None,
|
||||||
|
resolved_subjektiv_attached: false,
|
||||||
|
resolved_worker_observation_enabled: false,
|
||||||
|
resolved_worker_observation_grants: Vec::new(),
|
||||||
|
resolved_control_operation: None,
|
||||||
|
resolved_execution_grant: None,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn seed(request: &WorkerSpawnRequest) -> FixedWorkerSeed<'_> {
|
||||||
|
FixedWorkerSeed {
|
||||||
|
workspace_id: WORKSPACE,
|
||||||
|
runtime_id: RUNTIME,
|
||||||
|
worker_id: WORKER.parse().unwrap(),
|
||||||
|
allocation_key: "fixture-allocation",
|
||||||
|
request,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn worker() -> RuntimeWorkerRef {
|
||||||
|
RuntimeWorkerRef::new(RUNTIME, WORKER)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn fixed_ids_are_isolated_and_seed_has_no_runtime_observation() {
|
||||||
|
let a = bootstrap();
|
||||||
|
let b = bootstrap();
|
||||||
|
let request = request();
|
||||||
|
let first = seed_fixed_worker(&a, seed(&request)).unwrap();
|
||||||
|
let second = seed_fixed_worker(&b, seed(&request)).unwrap();
|
||||||
|
assert_eq!(first, second);
|
||||||
|
assert_eq!(first.worker_id.to_string(), WORKER);
|
||||||
|
assert_eq!(first.memory_settings.workspace_id, WORKSPACE);
|
||||||
|
assert_eq!(first.memory_settings.language, "English");
|
||||||
|
assert!(first.singleton.is_none());
|
||||||
|
assert_ne!(
|
||||||
|
a.get_workspace_signing_identity(WORKSPACE)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
.public_key,
|
||||||
|
b.get_workspace_signing_identity(WORKSPACE)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
.public_key
|
||||||
|
);
|
||||||
|
assert_ne!(
|
||||||
|
a.get_workspace_runtime_binding(WORKSPACE, RUNTIME)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
.public_key,
|
||||||
|
b.get_workspace_runtime_binding(WORKSPACE, RUNTIME)
|
||||||
|
.unwrap()
|
||||||
|
.unwrap()
|
||||||
|
.public_key
|
||||||
|
);
|
||||||
|
for store in [&a, &b] {
|
||||||
|
assert_eq!(
|
||||||
|
store
|
||||||
|
.worker_resource_key(WORKSPACE, &worker())
|
||||||
|
.unwrap()
|
||||||
|
.as_deref(),
|
||||||
|
Some("W-1")
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
store
|
||||||
|
.resolve_worker_resource_reference(WORKSPACE, "W-1")
|
||||||
|
.unwrap(),
|
||||||
|
Some(worker())
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!store
|
||||||
|
.has_active_worker_create_reservation(WORKSPACE, &worker())
|
||||||
|
.unwrap()
|
||||||
|
);
|
||||||
|
let registry = store
|
||||||
|
.get_worker_registry(WORKSPACE, &worker())
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert_eq!(registry.profile.as_deref(), Some("builtin:coder"));
|
||||||
|
assert_eq!(registry.display_name, "Fixture Worker");
|
||||||
|
assert_eq!(registry.retention_state, "normal");
|
||||||
|
let projection = store
|
||||||
|
.with_conn(|conn| read_worker_registry_projection(conn, WORKSPACE, &worker()))
|
||||||
|
.unwrap()
|
||||||
|
.unwrap();
|
||||||
|
assert!(projection.observation.is_none());
|
||||||
|
assert!(projection.job.is_none());
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn initial_submit_is_refused_without_registration_or_allocation() {
|
||||||
|
let store = bootstrap();
|
||||||
|
let mut request = request();
|
||||||
|
// Even an empty text segment is new input; the vector itself must be empty.
|
||||||
|
for text in ["new input must not be seeded", ""] {
|
||||||
|
request.initial_submit = vec![protocol::Segment::text(text)];
|
||||||
|
let error = seed_fixed_worker(&store, seed(&request)).unwrap_err();
|
||||||
|
assert!(
|
||||||
|
matches!(error, Error::InvalidInput(ref message) if message.contains("initial_submit"))
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.get_worker_registry(WORKSPACE, &worker())
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.worker_resource_key(WORKSPACE, &worker())
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
!store
|
||||||
|
.has_active_worker_create_reservation(WORKSPACE, &worker())
|
||||||
|
.unwrap()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
// The rejected attempts must not consume the allocation or resource counter.
|
||||||
|
request.initial_submit.clear();
|
||||||
|
seed_fixed_worker(&store, seed(&request)).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
store
|
||||||
|
.worker_resource_key(WORKSPACE, &worker())
|
||||||
|
.unwrap()
|
||||||
|
.as_deref(),
|
||||||
|
Some("W-1")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn exact_replay_changed_input_and_cross_runtime_identity_are_refused() {
|
||||||
|
let store = bootstrap();
|
||||||
|
let mut request = request();
|
||||||
|
seed_fixed_worker(&store, seed(&request)).unwrap();
|
||||||
|
let before = store.get_worker_registry(WORKSPACE, &worker()).unwrap();
|
||||||
|
assert!(seed_fixed_worker(&store, seed(&request)).is_err());
|
||||||
|
request.requested_worker_name = Some("Must not overwrite".into());
|
||||||
|
assert!(seed_fixed_worker(&store, seed(&request)).is_err());
|
||||||
|
let mut new_allocation = seed(&request);
|
||||||
|
new_allocation.allocation_key = "another-allocation";
|
||||||
|
assert!(seed_fixed_worker(&store, new_allocation).is_err());
|
||||||
|
let mut same_allocation = seed(&request);
|
||||||
|
same_allocation.worker_id = WorkerId::now_v7();
|
||||||
|
assert!(seed_fixed_worker(&store, same_allocation).is_err());
|
||||||
|
assert_eq!(
|
||||||
|
store.get_worker_registry(WORKSPACE, &worker()).unwrap(),
|
||||||
|
before
|
||||||
|
);
|
||||||
|
|
||||||
|
let other = bootstrap();
|
||||||
|
let mut record = before.unwrap();
|
||||||
|
record.worker.runtime_id = "different-runtime".into();
|
||||||
|
other.upsert_worker_registry(&record).unwrap();
|
||||||
|
assert!(seed_fixed_worker(&other, seed(&request)).is_err());
|
||||||
|
assert!(
|
||||||
|
other
|
||||||
|
.get_worker_registry(WORKSPACE, &worker())
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn reserved_and_orphaned_resource_identities_are_not_adopted() {
|
||||||
|
let store = bootstrap();
|
||||||
|
let request = request();
|
||||||
|
let memory = store.get_workspace_memory_settings(WORKSPACE).unwrap();
|
||||||
|
let ordinary = store
|
||||||
|
.reserve_worker_create(WORKSPACE, RUNTIME, "ordinary", "fingerprint", None, &memory)
|
||||||
|
.unwrap();
|
||||||
|
let mut collision = seed(&request);
|
||||||
|
collision.worker_id = ordinary.worker_id;
|
||||||
|
assert!(seed_fixed_worker(&store, collision).is_err());
|
||||||
|
// Store-level fault setup only: the external harness never writes SQL.
|
||||||
|
store
|
||||||
|
.with_conn(|conn| {
|
||||||
|
allocate_resource_key(
|
||||||
|
conn,
|
||||||
|
WORKSPACE,
|
||||||
|
WorkspaceResourceKind::Worker,
|
||||||
|
&worker_resource_id(&worker()),
|
||||||
|
"now",
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert!(seed_fixed_worker(&store, seed(&request)).is_err());
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.get_worker_registry(WORKSPACE, &worker())
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unknown_resource_identity_and_retained_removal_fence_are_refused() {
|
||||||
|
let request = request();
|
||||||
|
let malformed = bootstrap();
|
||||||
|
malformed
|
||||||
|
.with_conn(|conn| {
|
||||||
|
allocate_resource_key(
|
||||||
|
conn,
|
||||||
|
WORKSPACE,
|
||||||
|
WorkspaceResourceKind::Worker,
|
||||||
|
"unknown-identity",
|
||||||
|
"now",
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert!(matches!(
|
||||||
|
seed_fixed_worker(&malformed, seed(&request)),
|
||||||
|
Err(Error::InvalidInput(_))
|
||||||
|
));
|
||||||
|
assert!(
|
||||||
|
malformed
|
||||||
|
.get_worker_registry(WORKSPACE, &worker())
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
|
||||||
|
let removed = bootstrap();
|
||||||
|
removed.with_conn(|conn| {
|
||||||
|
conn.execute(
|
||||||
|
"INSERT INTO worker_registry_projection_removals(workspace_id, runtime_id, worker_id) VALUES (?1, ?2, ?3)",
|
||||||
|
params![WORKSPACE, RUNTIME, WORKER],
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
}).unwrap();
|
||||||
|
assert!(matches!(
|
||||||
|
seed_fixed_worker(&removed, seed(&request)),
|
||||||
|
Err(Error::InvalidInput(_))
|
||||||
|
));
|
||||||
|
assert!(
|
||||||
|
removed
|
||||||
|
.worker_resource_key(WORKSPACE, &worker())
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unknown_workspace_runtime_and_revoked_binding_are_refused() {
|
||||||
|
let store = bootstrap();
|
||||||
|
let request = request();
|
||||||
|
let mut unknown = seed(&request);
|
||||||
|
unknown.workspace_id = "unknown";
|
||||||
|
assert!(seed_fixed_worker(&store, unknown).is_err());
|
||||||
|
let mut unknown = seed(&request);
|
||||||
|
unknown.runtime_id = "unknown";
|
||||||
|
assert!(seed_fixed_worker(&store, unknown).is_err());
|
||||||
|
store
|
||||||
|
.revoke_workspace_runtime_binding(WORKSPACE, RUNTIME, "2026-10-24T01:00:00Z")
|
||||||
|
.unwrap();
|
||||||
|
assert!(seed_fixed_worker(&store, seed(&request)).is_err());
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.worker_resource_key(WORKSPACE, &worker())
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn unsupported_intent_is_refused_without_partial_registration() {
|
||||||
|
let store = bootstrap();
|
||||||
|
let base = request();
|
||||||
|
let mut requests = Vec::new();
|
||||||
|
let mut request = base.clone();
|
||||||
|
request.singleton_key = Some("fixture-singleton".into());
|
||||||
|
requests.push(request);
|
||||||
|
let mut request = base.clone();
|
||||||
|
request.intent = WorkerSpawnIntent::WorkspaceOrchestrator;
|
||||||
|
requests.push(request);
|
||||||
|
let mut request = base.clone();
|
||||||
|
request.resolved_subjektiv_attached = true;
|
||||||
|
requests.push(request);
|
||||||
|
let mut request = base.clone();
|
||||||
|
request
|
||||||
|
.workdir_attachment_requests
|
||||||
|
.push(WorkerSpawnWorkingDirectoryAttachmentRequest {
|
||||||
|
alias: workdir::WorkdirAttachmentAlias::new("repo").unwrap(),
|
||||||
|
working_directory: WorkerSpawnWorkingDirectoryRequest {
|
||||||
|
repository_key: "repo".into(),
|
||||||
|
selector: None,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
requests.push(request);
|
||||||
|
for request in requests {
|
||||||
|
assert!(seed_fixed_worker(&store, seed(&request)).is_err());
|
||||||
|
}
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.get_worker_registry(WORKSPACE, &worker())
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.worker_resource_key(WORKSPACE, &worker())
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert_eq!(
|
||||||
|
seed_fixed_worker(&store, seed(&base))
|
||||||
|
.unwrap()
|
||||||
|
.worker_id
|
||||||
|
.to_string(),
|
||||||
|
WORKER
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn late_catalog_failure_rolls_back_reservation_and_resource_allocation() {
|
||||||
|
let store = bootstrap();
|
||||||
|
let request = request();
|
||||||
|
store
|
||||||
|
.with_conn(|conn| {
|
||||||
|
conn.execute_batch(
|
||||||
|
"CREATE TEMP TRIGGER fail_fixture_catalog BEFORE INSERT ON worker_registry
|
||||||
|
BEGIN SELECT RAISE(ABORT, 'injected catalog failure'); END",
|
||||||
|
)?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
assert!(seed_fixed_worker(&store, seed(&request)).is_err());
|
||||||
|
assert!(
|
||||||
|
!store
|
||||||
|
.has_active_worker_create_reservation(WORKSPACE, &worker())
|
||||||
|
.unwrap()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.worker_resource_key(WORKSPACE, &worker())
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
store
|
||||||
|
.with_conn(|conn| {
|
||||||
|
conn.execute_batch("DROP TRIGGER fail_fixture_catalog")?;
|
||||||
|
Ok(())
|
||||||
|
})
|
||||||
|
.unwrap();
|
||||||
|
seed_fixed_worker(&store, seed(&request)).unwrap();
|
||||||
|
assert_eq!(
|
||||||
|
store
|
||||||
|
.worker_resource_key(WORKSPACE, &worker())
|
||||||
|
.unwrap()
|
||||||
|
.as_deref(),
|
||||||
|
Some("W-1")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn ordinary_creation_still_allocates_and_replays_without_catalog_insertion() {
|
||||||
|
let store = bootstrap();
|
||||||
|
let memory = store.get_workspace_memory_settings(WORKSPACE).unwrap();
|
||||||
|
let reserve = |key| {
|
||||||
|
store
|
||||||
|
.reserve_worker_create(WORKSPACE, RUNTIME, key, "fingerprint", None, &memory)
|
||||||
|
.unwrap()
|
||||||
|
};
|
||||||
|
let first = reserve("ordinary-one");
|
||||||
|
assert_eq!(first, reserve("ordinary-one"));
|
||||||
|
let second = reserve("ordinary-two");
|
||||||
|
assert_ne!(first.worker_id, second.worker_id);
|
||||||
|
assert_ne!(first.worker_id.to_string(), WORKER);
|
||||||
|
assert_eq!(
|
||||||
|
first.worker_id.as_uuid().get_version(),
|
||||||
|
Some(uuid::Version::SortRand)
|
||||||
|
);
|
||||||
|
let worker = RuntimeWorkerRef::new(RUNTIME, first.worker_id.to_string());
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.has_active_worker_create_reservation(WORKSPACE, &worker)
|
||||||
|
.unwrap()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.get_worker_registry(WORKSPACE, &worker)
|
||||||
|
.unwrap()
|
||||||
|
.is_none()
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.reserve_worker_create(WORKSPACE, RUNTIME, "ordinary-one", "changed", None, &memory)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,24 +1,15 @@
|
|||||||
//! Persists durable deletion intent and attempts for Repository-backed Workdirs.
|
//! Records one Repository-backed Workdir removal request and its result.
|
||||||
//!
|
//!
|
||||||
//! Deletion intent fixes Workspace, Workdir, owning Runtime, Repository/materialization, actor,
|
//! Caller-generated request identities distinguish an exact resend from a new explicit request;
|
||||||
//! and reason. Endpoints, host paths, and provider handles do not grant caller deletion authority.
|
//! reason text is audit input, never a replay key. Failed and completed records are terminal,
|
||||||
|
//! including legacy failed records marked retryable. Retryability only advises a new explicit
|
||||||
|
//! request after resolving its cause; it is not continuing authority to delete.
|
||||||
|
//!
|
||||||
|
//! Only one unresolved request per Workdir is allowed. Attempts are claimed transactionally and
|
||||||
|
//! fenced by process/start marker and attempt identity. Recovery observes uncertain external
|
||||||
|
//! effects but never reissues deletion. Confirmed absence and the matching materialization permit
|
||||||
|
//! registry removal and the saved result to commit together. Unconfirmed effects remain pending.
|
||||||
//! External Workdirs follow grant revocation, not this operation.
|
//! External Workdirs follow grant revocation, not this operation.
|
||||||
//!
|
|
||||||
//! Only one pending operation per Workdir is allowed; attempts are claimed transactionally.
|
|
||||||
//! Process IDs and startup markers are retained. Recovery reclaims the exact owner/attempt by
|
|
||||||
//! compare-and-set only after confirming the previous owner is gone; live or unobservable owners
|
|
||||||
//! are not displaced. In-process locks alone are insufficient.
|
|
||||||
//!
|
|
||||||
//! Server execution revalidates stored materialization and current attachments, reservations,
|
|
||||||
//! assignments, retention, and creation operations, then observes occupancy and cleanliness via
|
|
||||||
//! Runtime. Dirty resources remain; unknown observations or provider failures retain the registry
|
|
||||||
//! and record attention/retryability. Only confirmed deletion or exact working_directory_not_found
|
|
||||||
//! is deletion evidence; registry removal and completed/removed commit in the same transaction.
|
|
||||||
//!
|
|
||||||
//! Normal deletion, Workspace deletion, cleanup, and startup recovery use the same reserved Server
|
|
||||||
//! operation execution. Failed DB commits after provider deletion are reobserved through that
|
|
||||||
//! operation. Recovery relies on provider identity and results, not an exactly-once guarantee
|
|
||||||
//! across all interruption points.
|
|
||||||
|
|
||||||
use chrono::Utc;
|
use chrono::Utc;
|
||||||
use rusqlite::{Connection, OptionalExtension, TransactionBehavior, params};
|
use rusqlite::{Connection, OptionalExtension, TransactionBehavior, params};
|
||||||
@@ -118,10 +109,22 @@ pub fn workdir_materialization_fingerprint(record: &WorkdirRegistryRecord) -> St
|
|||||||
hex_sha256(&bytes)
|
hex_sha256(&bytes)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
pub fn workdir_removal_request_id(
|
||||||
|
workspace_id: &str,
|
||||||
|
source_actor: &str,
|
||||||
|
request_id: &str,
|
||||||
|
) -> Result<String> {
|
||||||
|
validate_bounded("operation id", request_id, 256)?;
|
||||||
|
let bytes = serde_json::to_vec(&(workspace_id, source_actor, request_id))
|
||||||
|
.map_err(|error| Error::Store(error.to_string()))?;
|
||||||
|
Ok(format!("wdr_{}", &hex_sha256(&bytes)[..32]))
|
||||||
|
}
|
||||||
|
|
||||||
pub fn workdir_removal_intent(
|
pub fn workdir_removal_intent(
|
||||||
record: &WorkdirRegistryRecord,
|
record: &WorkdirRegistryRecord,
|
||||||
source_actor: &str,
|
source_actor: &str,
|
||||||
reason: &str,
|
reason: &str,
|
||||||
|
request_id: &str,
|
||||||
) -> Result<WorkdirRemovalIntent> {
|
) -> Result<WorkdirRemovalIntent> {
|
||||||
validate_bounded("source actor", source_actor, MAX_ACTOR_BYTES)?;
|
validate_bounded("source actor", source_actor, MAX_ACTOR_BYTES)?;
|
||||||
validate_bounded("reason", reason, MAX_REASON_BYTES)?;
|
validate_bounded("reason", reason, MAX_REASON_BYTES)?;
|
||||||
@@ -146,7 +149,7 @@ pub fn workdir_removal_intent(
|
|||||||
));
|
));
|
||||||
};
|
};
|
||||||
Ok(WorkdirRemovalIntent {
|
Ok(WorkdirRemovalIntent {
|
||||||
operation_id: format!("wdr_{}", &request_fingerprint[..32]),
|
operation_id: workdir_removal_request_id(&record.workspace_id, source_actor, request_id)?,
|
||||||
request_fingerprint,
|
request_fingerprint,
|
||||||
workspace_id: record.workspace_id.clone(),
|
workspace_id: record.workspace_id.clone(),
|
||||||
working_directory_id: record.workdir_id.clone(),
|
working_directory_id: record.workdir_id.clone(),
|
||||||
@@ -285,17 +288,10 @@ impl SqliteWorkspaceStore {
|
|||||||
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||||
let operation =
|
let operation =
|
||||||
require_operation(&tx, workspace_id, operation_id, request_fingerprint)?;
|
require_operation(&tx, workspace_id, operation_id, request_fingerprint)?;
|
||||||
if operation.state == WorkdirRemovalOperationState::Completed
|
if operation.state != WorkdirRemovalOperationState::Pending {
|
||||||
&& operation.disposition == Some(WorkdirRemovalDisposition::Removed)
|
|
||||||
{
|
|
||||||
tx.commit()?;
|
tx.commit()?;
|
||||||
return Ok(operation);
|
return Ok(operation);
|
||||||
}
|
}
|
||||||
if operation.state == WorkdirRemovalOperationState::Failed && !operation.retryable {
|
|
||||||
return Err(Error::InvalidInput(format!(
|
|
||||||
"Workdir removal operation `{operation_id}` is not retryable"
|
|
||||||
)));
|
|
||||||
}
|
|
||||||
if let Some((expected_owner, expected_attempt_id)) = recovery_expected {
|
if let Some((expected_owner, expected_attempt_id)) = recovery_expected {
|
||||||
if operation.state != WorkdirRemovalOperationState::Pending
|
if operation.state != WorkdirRemovalOperationState::Pending
|
||||||
|| operation.attempt_owner != expected_owner
|
|| operation.attempt_owner != expected_owner
|
||||||
@@ -473,7 +469,7 @@ impl SqliteWorkspaceStore {
|
|||||||
&operation.operation_id,
|
&operation.operation_id,
|
||||||
&operation.request_fingerprint,
|
&operation.request_fingerprint,
|
||||||
)?;
|
)?;
|
||||||
if current.state == WorkdirRemovalOperationState::Completed {
|
if current.state != WorkdirRemovalOperationState::Pending {
|
||||||
tx.commit()?;
|
tx.commit()?;
|
||||||
return Ok(current);
|
return Ok(current);
|
||||||
}
|
}
|
||||||
@@ -502,6 +498,39 @@ impl SqliteWorkspaceStore {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Keep an uncertain external outcome for observation, not automatic re-execution.
|
||||||
|
pub fn record_workdir_removal_unknown(
|
||||||
|
&self,
|
||||||
|
operation: &WorkdirRemovalOperation,
|
||||||
|
) -> Result<WorkdirRemovalOperation> {
|
||||||
|
self.with_conn_mut(|conn| {
|
||||||
|
let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?;
|
||||||
|
let current = require_operation(
|
||||||
|
&tx,
|
||||||
|
&operation.workspace_id,
|
||||||
|
&operation.operation_id,
|
||||||
|
&operation.request_fingerprint,
|
||||||
|
)?;
|
||||||
|
if current.state != WorkdirRemovalOperationState::Pending {
|
||||||
|
tx.commit()?;
|
||||||
|
return Ok(current);
|
||||||
|
}
|
||||||
|
require_matching_attempt(operation, ¤t)?;
|
||||||
|
tx.execute(
|
||||||
|
"UPDATE workdir_removal_operations SET retryable=0, disposition='attention_required', failure_category='provider_cleanup_outcome_unknown', attempt_owner_pid=NULL, attempt_owner_start_marker=NULL, updated_at=?1 WHERE workspace_id=?2 AND operation_id=?3",
|
||||||
|
params![Utc::now().to_rfc3339(), operation.workspace_id, operation.operation_id],
|
||||||
|
)?;
|
||||||
|
let updated = require_operation(
|
||||||
|
&tx,
|
||||||
|
&operation.workspace_id,
|
||||||
|
&operation.operation_id,
|
||||||
|
&operation.request_fingerprint,
|
||||||
|
)?;
|
||||||
|
tx.commit()?;
|
||||||
|
Ok(updated)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
pub fn commit_workdir_removal_removed(
|
pub fn commit_workdir_removal_removed(
|
||||||
&self,
|
&self,
|
||||||
operation: &WorkdirRemovalOperation,
|
operation: &WorkdirRemovalOperation,
|
||||||
@@ -531,7 +560,7 @@ impl SqliteWorkspaceStore {
|
|||||||
&operation.operation_id,
|
&operation.operation_id,
|
||||||
&operation.request_fingerprint,
|
&operation.request_fingerprint,
|
||||||
)?;
|
)?;
|
||||||
if current.state == WorkdirRemovalOperationState::Completed {
|
if current.state != WorkdirRemovalOperationState::Pending {
|
||||||
tx.commit()?;
|
tx.commit()?;
|
||||||
return Ok(current);
|
return Ok(current);
|
||||||
}
|
}
|
||||||
@@ -541,24 +570,29 @@ impl SqliteWorkspaceStore {
|
|||||||
&tx,
|
&tx,
|
||||||
&operation.workspace_id,
|
&operation.workspace_id,
|
||||||
&operation.working_directory_id,
|
&operation.working_directory_id,
|
||||||
)?
|
|
||||||
.ok_or_else(|| Error::RegistryInconsistency(
|
|
||||||
"Workdir registry row disappeared before durable removal commit".to_string(),
|
|
||||||
))?;
|
|
||||||
require_operation_materialization(operation, ®istry)?;
|
|
||||||
require_no_removal_blockers(
|
|
||||||
&tx,
|
|
||||||
&operation.workspace_id,
|
|
||||||
&operation.working_directory_id,
|
|
||||||
)?;
|
)?;
|
||||||
let deleted = tx.execute(
|
if let Some(registry) = registry {
|
||||||
"DELETE FROM workdir_registry WHERE workspace_id=?1 AND workdir_id=?2",
|
require_operation_materialization(operation, ®istry)?;
|
||||||
params![operation.workspace_id, operation.working_directory_id],
|
require_no_removal_blockers(
|
||||||
)?;
|
&tx,
|
||||||
if deleted != 1 {
|
&operation.workspace_id,
|
||||||
return Err(Error::RegistryInconsistency(
|
&operation.working_directory_id,
|
||||||
"Workdir registry deletion did not remove exactly one row".to_string(),
|
)?;
|
||||||
));
|
tx.execute(
|
||||||
|
"DELETE FROM workdir_registry WHERE workspace_id=?1 AND workdir_id=?2",
|
||||||
|
params![operation.workspace_id, operation.working_directory_id],
|
||||||
|
)?;
|
||||||
|
} else {
|
||||||
|
let removed: bool = tx.query_row(
|
||||||
|
"SELECT EXISTS(SELECT 1 FROM workdir_removal_operations WHERE workspace_id=?1 AND workdir_id=?2 AND materialization_fingerprint=?3 AND state='completed' AND disposition='removed')",
|
||||||
|
params![operation.workspace_id, operation.working_directory_id, operation.materialization_fingerprint],
|
||||||
|
|row| row.get(0),
|
||||||
|
)?;
|
||||||
|
if !removed {
|
||||||
|
return Err(Error::RegistryInconsistency(
|
||||||
|
"Workdir registry disappeared without confirmed removal".to_string(),
|
||||||
|
));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
let now = Utc::now().to_rfc3339();
|
let now = Utc::now().to_rfc3339();
|
||||||
@@ -585,20 +619,18 @@ impl SqliteWorkspaceStore {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn find_workdir_removal_operation_by_intent(
|
pub fn completed_workdir_removal(
|
||||||
&self,
|
&self,
|
||||||
workspace_id: &str,
|
workspace_id: &str,
|
||||||
working_directory_id: &str,
|
working_directory_id: &str,
|
||||||
source_actor: &str,
|
|
||||||
reason: &str,
|
|
||||||
) -> Result<Option<WorkdirRemovalOperation>> {
|
) -> Result<Option<WorkdirRemovalOperation>> {
|
||||||
self.with_conn(|conn| {
|
self.with_conn(|conn| {
|
||||||
conn.query_row(
|
conn.query_row(
|
||||||
&format!(
|
&format!(
|
||||||
"{} WHERE workspace_id=?1 AND workdir_id=?2 AND source_actor=?3 AND reason=?4 ORDER BY created_at DESC, operation_id DESC LIMIT 1",
|
"{} WHERE workspace_id=?1 AND workdir_id=?2 AND state='completed' AND disposition='removed' ORDER BY completed_at DESC, operation_id DESC LIMIT 1",
|
||||||
operation_select_sql()
|
operation_select_sql()
|
||||||
),
|
),
|
||||||
params![workspace_id, working_directory_id, source_actor, reason],
|
params![workspace_id, working_directory_id],
|
||||||
read_operation,
|
read_operation,
|
||||||
)
|
)
|
||||||
.optional()
|
.optional()
|
||||||
@@ -614,7 +646,7 @@ impl SqliteWorkspaceStore {
|
|||||||
self.with_conn(|conn| {
|
self.with_conn(|conn| {
|
||||||
let mut statement = conn.prepare(
|
let mut statement = conn.prepare(
|
||||||
&format!(
|
&format!(
|
||||||
"{} WHERE workspace_id=?1 AND (state='pending' OR (state='failed' AND retryable=1)) ORDER BY updated_at ASC, operation_id ASC LIMIT ?2",
|
"{} WHERE workspace_id=?1 AND state='pending' ORDER BY updated_at ASC, operation_id ASC LIMIT ?2",
|
||||||
operation_select_sql()
|
operation_select_sql()
|
||||||
),
|
),
|
||||||
)?;
|
)?;
|
||||||
@@ -1007,8 +1039,13 @@ mod tests {
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn exact_replay_reuses_operation_and_conflicting_intent_fails() {
|
async fn exact_replay_reuses_operation_and_conflicting_intent_fails() {
|
||||||
let (store, workdir) = seeded_store().await;
|
let (store, workdir) = seeded_store().await;
|
||||||
let intent =
|
let intent = workdir_removal_intent(
|
||||||
workdir_removal_intent(&workdir, "worker:W-1", "remove stale Workdir").unwrap();
|
&workdir,
|
||||||
|
"worker:W-1",
|
||||||
|
"remove stale Workdir",
|
||||||
|
"test-removal-request",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
let first = store.reserve_workdir_removal_operation(&intent).unwrap();
|
let first = store.reserve_workdir_removal_operation(&intent).unwrap();
|
||||||
let replay = store.reserve_workdir_removal_operation(&intent).unwrap();
|
let replay = store.reserve_workdir_removal_operation(&intent).unwrap();
|
||||||
assert_eq!(replay, first);
|
assert_eq!(replay, first);
|
||||||
@@ -1023,10 +1060,15 @@ mod tests {
|
|||||||
}
|
}
|
||||||
|
|
||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn failed_attempt_is_retryable_and_completed_retry_replays() {
|
async fn failed_request_is_terminal_and_new_request_has_independent_result() {
|
||||||
let (store, workdir) = seeded_store().await;
|
let (store, workdir) = seeded_store().await;
|
||||||
let intent =
|
let intent = workdir_removal_intent(
|
||||||
workdir_removal_intent(&workdir, "workspace-api", "remove clean Workdir").unwrap();
|
&workdir,
|
||||||
|
"workspace-api",
|
||||||
|
"remove clean Workdir",
|
||||||
|
"test-removal-request",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
let reserved = store.reserve_workdir_removal_operation(&intent).unwrap();
|
let reserved = store.reserve_workdir_removal_operation(&intent).unwrap();
|
||||||
let first = store
|
let first = store
|
||||||
.begin_workdir_removal_attempt(
|
.begin_workdir_removal_attempt(
|
||||||
@@ -1049,7 +1091,32 @@ mod tests {
|
|||||||
attempt_owner(),
|
attempt_owner(),
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_ne!(retry.attempt_id, first.attempt_id);
|
assert_eq!(retry, failed);
|
||||||
|
assert!(
|
||||||
|
store
|
||||||
|
.recoverable_workdir_removal_operations("workspace-a", 100)
|
||||||
|
.unwrap()
|
||||||
|
.is_empty()
|
||||||
|
);
|
||||||
|
let next_intent = workdir_removal_intent(
|
||||||
|
&workdir,
|
||||||
|
"workspace-api",
|
||||||
|
"remove clean Workdir",
|
||||||
|
"new-request",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let next = store
|
||||||
|
.reserve_workdir_removal_operation(&next_intent)
|
||||||
|
.unwrap();
|
||||||
|
let retry = store
|
||||||
|
.begin_workdir_removal_attempt(
|
||||||
|
&next.workspace_id,
|
||||||
|
&next.operation_id,
|
||||||
|
&next.request_fingerprint,
|
||||||
|
attempt_owner(),
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
assert_ne!(retry.operation_id, failed.operation_id);
|
||||||
let completed = store.commit_workdir_removal_removed(&retry).unwrap();
|
let completed = store.commit_workdir_removal_removed(&retry).unwrap();
|
||||||
assert_eq!(
|
assert_eq!(
|
||||||
completed.disposition,
|
completed.disposition,
|
||||||
@@ -1075,7 +1142,13 @@ mod tests {
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn recovery_and_completion_are_fenced_by_attempt_identity() {
|
async fn recovery_and_completion_are_fenced_by_attempt_identity() {
|
||||||
let (store, workdir) = seeded_store().await;
|
let (store, workdir) = seeded_store().await;
|
||||||
let intent = workdir_removal_intent(&workdir, "workspace-api", "remove Workdir").unwrap();
|
let intent = workdir_removal_intent(
|
||||||
|
&workdir,
|
||||||
|
"workspace-api",
|
||||||
|
"remove Workdir",
|
||||||
|
"test-removal-request",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
let reserved = store.reserve_workdir_removal_operation(&intent).unwrap();
|
let reserved = store.reserve_workdir_removal_operation(&intent).unwrap();
|
||||||
let first = store
|
let first = store
|
||||||
.begin_workdir_removal_attempt(
|
.begin_workdir_removal_attempt(
|
||||||
@@ -1130,8 +1203,13 @@ mod tests {
|
|||||||
use std::sync::{Arc, Barrier};
|
use std::sync::{Arc, Barrier};
|
||||||
|
|
||||||
let (store, workdir) = seeded_store().await;
|
let (store, workdir) = seeded_store().await;
|
||||||
let intent =
|
let intent = workdir_removal_intent(
|
||||||
workdir_removal_intent(&workdir, "workspace-api", "remove clean Workdir").unwrap();
|
&workdir,
|
||||||
|
"workspace-api",
|
||||||
|
"remove clean Workdir",
|
||||||
|
"test-removal-request",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
let operation = store.reserve_workdir_removal_operation(&intent).unwrap();
|
let operation = store.reserve_workdir_removal_operation(&intent).unwrap();
|
||||||
let barrier = Arc::new(Barrier::new(3));
|
let barrier = Arc::new(Barrier::new(3));
|
||||||
let provider_calls = Arc::new(AtomicUsize::new(0));
|
let provider_calls = Arc::new(AtomicUsize::new(0));
|
||||||
@@ -1218,8 +1296,13 @@ mod tests {
|
|||||||
"2",
|
"2",
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let intent =
|
let intent = workdir_removal_intent(
|
||||||
workdir_removal_intent(&workdir, "workspace-api", "remove clean Workdir").unwrap();
|
&workdir,
|
||||||
|
"workspace-api",
|
||||||
|
"remove clean Workdir",
|
||||||
|
"test-removal-request",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
store.reserve_workdir_removal_operation(&intent).unwrap();
|
store.reserve_workdir_removal_operation(&intent).unwrap();
|
||||||
|
|
||||||
let error = store
|
let error = store
|
||||||
@@ -1236,8 +1319,13 @@ mod tests {
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn pending_removal_fences_new_attachment_and_retry_rereads_live_reservation() {
|
async fn pending_removal_fences_new_attachment_and_retry_rereads_live_reservation() {
|
||||||
let (store, workdir) = seeded_store().await;
|
let (store, workdir) = seeded_store().await;
|
||||||
let intent =
|
let intent = workdir_removal_intent(
|
||||||
workdir_removal_intent(&workdir, "workspace-api", "remove clean Workdir").unwrap();
|
&workdir,
|
||||||
|
"workspace-api",
|
||||||
|
"remove clean Workdir",
|
||||||
|
"test-removal-request",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
let pending = store.reserve_workdir_removal_operation(&intent).unwrap();
|
let pending = store.reserve_workdir_removal_operation(&intent).unwrap();
|
||||||
|
|
||||||
let error = store
|
let error = store
|
||||||
@@ -1251,11 +1339,22 @@ mod tests {
|
|||||||
store
|
store
|
||||||
.reserve_worker_workdir_attachment("workspace-a", "workdir-a", "reservation-b", "3")
|
.reserve_worker_workdir_attachment("workspace-a", "workdir-a", "reservation-b", "3")
|
||||||
.unwrap();
|
.unwrap();
|
||||||
|
let next_intent = workdir_removal_intent(
|
||||||
|
&workdir,
|
||||||
|
"workspace-api",
|
||||||
|
"remove clean Workdir",
|
||||||
|
"new-request",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
|
let next = store
|
||||||
|
.reserve_workdir_removal_operation(&next_intent)
|
||||||
|
.unwrap();
|
||||||
|
assert_ne!(next.operation_id, failed.operation_id);
|
||||||
let retry = store
|
let retry = store
|
||||||
.begin_workdir_removal_attempt(
|
.begin_workdir_removal_attempt(
|
||||||
&failed.workspace_id,
|
&next.workspace_id,
|
||||||
&failed.operation_id,
|
&next.operation_id,
|
||||||
&failed.request_fingerprint,
|
&next.request_fingerprint,
|
||||||
attempt_owner(),
|
attempt_owner(),
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
@@ -1324,7 +1423,13 @@ mod tests {
|
|||||||
Ok(())
|
Ok(())
|
||||||
})
|
})
|
||||||
.unwrap();
|
.unwrap();
|
||||||
let intent = workdir_removal_intent(&workdir, "workspace-api", "remove Workdir").unwrap();
|
let intent = workdir_removal_intent(
|
||||||
|
&workdir,
|
||||||
|
"workspace-api",
|
||||||
|
"remove Workdir",
|
||||||
|
"test-removal-request",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
let operation = store.reserve_workdir_removal_operation(&intent).unwrap();
|
let operation = store.reserve_workdir_removal_operation(&intent).unwrap();
|
||||||
let categories = |guards: Vec<WorkdirRemovalGuard>| {
|
let categories = |guards: Vec<WorkdirRemovalGuard>| {
|
||||||
guards
|
guards
|
||||||
@@ -1447,8 +1552,13 @@ mod tests {
|
|||||||
#[tokio::test]
|
#[tokio::test]
|
||||||
async fn retained_completion_keeps_registry_and_is_auditable() {
|
async fn retained_completion_keeps_registry_and_is_auditable() {
|
||||||
let (store, workdir) = seeded_store().await;
|
let (store, workdir) = seeded_store().await;
|
||||||
let intent =
|
let intent = workdir_removal_intent(
|
||||||
workdir_removal_intent(&workdir, "workspace-api", "inspect dirty Workdir").unwrap();
|
&workdir,
|
||||||
|
"workspace-api",
|
||||||
|
"inspect dirty Workdir",
|
||||||
|
"test-removal-request",
|
||||||
|
)
|
||||||
|
.unwrap();
|
||||||
let operation = store.reserve_workdir_removal_operation(&intent).unwrap();
|
let operation = store.reserve_workdir_removal_operation(&intent).unwrap();
|
||||||
let retained = store
|
let retained = store
|
||||||
.complete_workdir_removal_retained(
|
.complete_workdir_removal_retained(
|
||||||
@@ -1477,10 +1587,12 @@ mod tests {
|
|||||||
attempt_owner(),
|
attempt_owner(),
|
||||||
)
|
)
|
||||||
.unwrap();
|
.unwrap();
|
||||||
assert_eq!(retry.state, WorkdirRemovalOperationState::Pending);
|
assert_eq!(retry, retained);
|
||||||
assert!(retry.attempt_id.is_some());
|
assert!(
|
||||||
assert_eq!(retry.disposition, None);
|
store
|
||||||
assert_eq!(retry.failure_category, None);
|
.recoverable_workdir_removal_operations("workspace-a", 100)
|
||||||
assert!(retry.retryable);
|
.unwrap()
|
||||||
|
.is_empty()
|
||||||
|
);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,8 +1,8 @@
|
|||||||
{
|
{
|
||||||
"tasks": {
|
"tasks": {
|
||||||
"test": "deno run --allow-read --allow-write=target --allow-run --allow-env tools/test.ts",
|
"test": "deno run --allow-read --allow-write=target --allow-run --allow-env --allow-sys=uid tools/test.ts",
|
||||||
"test:ticket:real-api": "cargo test -p yoi-workspace-server --lib ticket_web_roundtrip_tests -- --ignored",
|
"test:ticket:real-api": "cargo test -p yoi-workspace-server --lib ticket_web_roundtrip_tests -- --ignored",
|
||||||
"test:runner": "deno test --allow-read --allow-write --allow-run --allow-env tools/test.test.ts"
|
"test:runner": "deno test --allow-read --allow-write --allow-run --allow-env --allow-sys=uid tools/test.test.ts"
|
||||||
},
|
},
|
||||||
"fmt": {
|
"fmt": {
|
||||||
"proseWrap": "never"
|
"proseWrap": "never"
|
||||||
|
|||||||
+283
-11
@@ -298,12 +298,17 @@
|
|||||||
"AuthUserResponse": {
|
"AuthUserResponse": {
|
||||||
"additionalProperties": false,
|
"additionalProperties": false,
|
||||||
"properties": {
|
"properties": {
|
||||||
|
"authentication_reference": {
|
||||||
|
"description": "Comparison evidence for the exact newly issued session, not a credential.",
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
"user": {
|
"user": {
|
||||||
"$ref": "#/components/schemas/AuthenticatedUser"
|
"$ref": "#/components/schemas/AuthenticatedUser"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"required": [
|
"required": [
|
||||||
"user"
|
"user",
|
||||||
|
"authentication_reference"
|
||||||
],
|
],
|
||||||
"type": "object"
|
"type": "object"
|
||||||
},
|
},
|
||||||
@@ -6709,15 +6714,112 @@
|
|||||||
},
|
},
|
||||||
"PasskeyAdditionOptionsRequest": {
|
"PasskeyAdditionOptionsRequest": {
|
||||||
"additionalProperties": false,
|
"additionalProperties": false,
|
||||||
"description": "Adds a credential to the current authenticated actor; target identity is never public input.",
|
"description": "Adds to the current actor. Expected identity/binding fields only fence stale UI;\nthey never select an owner or authorize an addition.",
|
||||||
"properties": {
|
"properties": {
|
||||||
"browser_origin": {
|
"browser_origin": {
|
||||||
"type": [
|
"type": [
|
||||||
"string",
|
"string",
|
||||||
"null"
|
"null"
|
||||||
]
|
]
|
||||||
|
},
|
||||||
|
"expected_account_id": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"expected_authentication_reference": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"expected_user_id": {
|
||||||
|
"type": "string"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"required": [
|
||||||
|
"expected_user_id",
|
||||||
|
"expected_account_id",
|
||||||
|
"expected_authentication_reference"
|
||||||
|
],
|
||||||
|
"type": "object"
|
||||||
|
},
|
||||||
|
"PasskeyAdditionResultRequest": {
|
||||||
|
"additionalProperties": false,
|
||||||
|
"properties": {
|
||||||
|
"challenge_id": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"expected_account_id": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"expected_authentication_reference": {
|
||||||
|
"description": "Original options authentication reference; the current same-owner session may differ.",
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"expected_user_id": {
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"challenge_id",
|
||||||
|
"expected_user_id",
|
||||||
|
"expected_account_id",
|
||||||
|
"expected_authentication_reference"
|
||||||
|
],
|
||||||
|
"type": "object"
|
||||||
|
},
|
||||||
|
"PasskeyAdditionResultResponse": {
|
||||||
|
"description": "Unknown is not proof that registration failed: completion may still be in flight.\nSaved proves the credential commit, not session issuance or current session health.",
|
||||||
|
"oneOf": [
|
||||||
|
{
|
||||||
|
"additionalProperties": false,
|
||||||
|
"properties": {
|
||||||
|
"status": {
|
||||||
|
"const": "unknown",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"status"
|
||||||
|
],
|
||||||
|
"type": "object"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"additionalProperties": false,
|
||||||
|
"properties": {
|
||||||
|
"key": {
|
||||||
|
"$ref": "#/components/schemas/PasskeySummary"
|
||||||
|
},
|
||||||
|
"status": {
|
||||||
|
"const": "saved",
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"user": {
|
||||||
|
"$ref": "#/components/schemas/AuthenticatedUser"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"status",
|
||||||
|
"user",
|
||||||
|
"key"
|
||||||
|
],
|
||||||
|
"type": "object"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"PasskeyListResponse": {
|
||||||
|
"additionalProperties": false,
|
||||||
|
"properties": {
|
||||||
|
"keys": {
|
||||||
|
"items": {
|
||||||
|
"$ref": "#/components/schemas/PasskeySummary"
|
||||||
|
},
|
||||||
|
"type": "array"
|
||||||
|
},
|
||||||
|
"user": {
|
||||||
|
"$ref": "#/components/schemas/AuthenticatedUser"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"user",
|
||||||
|
"keys"
|
||||||
|
],
|
||||||
"type": "object"
|
"type": "object"
|
||||||
},
|
},
|
||||||
"PasskeyLoginCompleteRequest": {
|
"PasskeyLoginCompleteRequest": {
|
||||||
@@ -6820,6 +6922,35 @@
|
|||||||
],
|
],
|
||||||
"type": "object"
|
"type": "object"
|
||||||
},
|
},
|
||||||
|
"PasskeySummary": {
|
||||||
|
"additionalProperties": false,
|
||||||
|
"description": "Safe account metadata. Neither the WebAuthn credential ID nor key material is exposed.\nNormal-login verification is observed verifier success, not inferred discoverability.",
|
||||||
|
"properties": {
|
||||||
|
"created_at": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"key_reference": {
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
"last_used_at": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"normal_login_verified_at": {
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": [
|
||||||
|
"key_reference",
|
||||||
|
"created_at"
|
||||||
|
],
|
||||||
|
"type": "object"
|
||||||
|
},
|
||||||
"PasteArtifactAvailability": {
|
"PasteArtifactAvailability": {
|
||||||
"enum": [
|
"enum": [
|
||||||
"available",
|
"available",
|
||||||
@@ -15143,6 +15274,13 @@
|
|||||||
}
|
}
|
||||||
]
|
]
|
||||||
},
|
},
|
||||||
|
"authentication_reference": {
|
||||||
|
"description": "Non-secret comparison evidence, never an authentication credential.",
|
||||||
|
"type": [
|
||||||
|
"string",
|
||||||
|
"null"
|
||||||
|
]
|
||||||
|
},
|
||||||
"session": {
|
"session": {
|
||||||
"$ref": "#/components/schemas/BrowserSessionState"
|
"$ref": "#/components/schemas/BrowserSessionState"
|
||||||
}
|
}
|
||||||
@@ -15697,14 +15835,23 @@
|
|||||||
"type": "object"
|
"type": "object"
|
||||||
},
|
},
|
||||||
"WorkerRestoreState": {
|
"WorkerRestoreState": {
|
||||||
"description": "Cross-layer outcome of an explicit Worker restore operation.\n\n`Rejected` is reserved for read-only preflight failures. Once live restore\nwork starts, failure is either a confirmed `RolledBack` operation or a\n`ReconciliationRequired` result whose commit state must be reread/retried.",
|
"description": "Cross-layer outcome of an explicit Worker restore operation.\n\n`Rejected` is reserved for read-only preflight failures. Once live restore\nwork starts, distinguish confirmed `RolledBack`, unresolved\n`ReconciliationRequired`, and a later Stop's `Superseded` outcome.",
|
||||||
"enum": [
|
"oneOf": [
|
||||||
"accepted",
|
{
|
||||||
"rejected",
|
"enum": [
|
||||||
"rolled_back",
|
"accepted",
|
||||||
"reconciliation_required"
|
"rejected",
|
||||||
],
|
"rolled_back",
|
||||||
"type": "string"
|
"reconciliation_required"
|
||||||
|
],
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"const": "superseded",
|
||||||
|
"description": "A later Stop superseded this restore; neither rollback nor prior Accepted is asserted.",
|
||||||
|
"type": "string"
|
||||||
|
}
|
||||||
|
]
|
||||||
},
|
},
|
||||||
"WorkerRetentionResponse": {
|
"WorkerRetentionResponse": {
|
||||||
"additionalProperties": false,
|
"additionalProperties": false,
|
||||||
@@ -16143,11 +16290,16 @@
|
|||||||
"WorkingDirectoryRemovalRequest": {
|
"WorkingDirectoryRemovalRequest": {
|
||||||
"additionalProperties": false,
|
"additionalProperties": false,
|
||||||
"properties": {
|
"properties": {
|
||||||
|
"operation_id": {
|
||||||
|
"description": "Stable for an exact resend; a new explicit request uses a new identity.",
|
||||||
|
"type": "string"
|
||||||
|
},
|
||||||
"reason": {
|
"reason": {
|
||||||
"type": "string"
|
"type": "string"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"required": [
|
"required": [
|
||||||
|
"operation_id",
|
||||||
"reason"
|
"reason"
|
||||||
],
|
],
|
||||||
"type": "object"
|
"type": "object"
|
||||||
@@ -18155,7 +18307,7 @@
|
|||||||
"info": {
|
"info": {
|
||||||
"title": "Yoi Server API",
|
"title": "Yoi Server API",
|
||||||
"version": "0.1.0",
|
"version": "0.1.0",
|
||||||
"x-yoi-source-digest": "sha256:5de3f3b716555717d3274b8c6a385fb66624585941628a53b8b3050cb58f8ab7"
|
"x-yoi-source-digest": "sha256:3436eada52ceea48f4c50f40195cba02bec37fbbff4ea8f5a332133e6fcde962"
|
||||||
},
|
},
|
||||||
"openapi": "3.1.0",
|
"openapi": "3.1.0",
|
||||||
"paths": {
|
"paths": {
|
||||||
@@ -18589,6 +18741,61 @@
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"/api/auth/passkeys": {
|
||||||
|
"get": {
|
||||||
|
"operationId": "auth_passkey_list",
|
||||||
|
"responses": {
|
||||||
|
"200": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/PasskeyListResponse"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "Successful response"
|
||||||
|
},
|
||||||
|
"400": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/RepositoryApiError"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "Error response"
|
||||||
|
},
|
||||||
|
"403": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/RepositoryApiError"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "Error response"
|
||||||
|
},
|
||||||
|
"500": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/RepositoryApiError"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "Error response"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"security": [
|
||||||
|
{
|
||||||
|
"bearerAuth": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"browserSession": []
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
"/api/auth/passkeys/addition/complete": {
|
"/api/auth/passkeys/addition/complete": {
|
||||||
"post": {
|
"post": {
|
||||||
"operationId": "auth_passkey_addition_complete",
|
"operationId": "auth_passkey_addition_complete",
|
||||||
@@ -18736,6 +18943,71 @@
|
|||||||
]
|
]
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"/api/auth/passkeys/addition/result": {
|
||||||
|
"post": {
|
||||||
|
"operationId": "auth_passkey_addition_result",
|
||||||
|
"requestBody": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/PasskeyAdditionResultRequest"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"required": true
|
||||||
|
},
|
||||||
|
"responses": {
|
||||||
|
"200": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/PasskeyAdditionResultResponse"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "Successful response"
|
||||||
|
},
|
||||||
|
"400": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/RepositoryApiError"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "Error response"
|
||||||
|
},
|
||||||
|
"403": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/RepositoryApiError"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "Error response"
|
||||||
|
},
|
||||||
|
"500": {
|
||||||
|
"content": {
|
||||||
|
"application/json": {
|
||||||
|
"schema": {
|
||||||
|
"$ref": "#/components/schemas/RepositoryApiError"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"description": "Error response"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"security": [
|
||||||
|
{
|
||||||
|
"bearerAuth": []
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"browserSession": []
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
"/api/auth/passkeys/login/complete": {
|
"/api/auth/passkeys/login/complete": {
|
||||||
"post": {
|
"post": {
|
||||||
"operationId": "auth_passkey_login_complete",
|
"operationId": "auth_passkey_login_complete",
|
||||||
|
|||||||
@@ -0,0 +1,344 @@
|
|||||||
|
# Workspace Plugin APIの設計契約
|
||||||
|
|
||||||
|
本書はWorkspace管理のPluginに採用する設計を定める。実行形式はWebAssembly Component Model、engineはWasmtime、guest/Host境界はWITとする。実装済みAPIの利用ガイドではない。既存の[v1 Tool world](wit/yoi-plugin-tool-v1.wit)、[v1 instance world](wit/yoi-plugin-instance-v1.wit)、[v1 Host imports](wit/deps/yoi-host/yoi-host-v1.wit)、PDK、templateは旧v1のoffline authoring資産であり、本書のv2実行契約をまだ提供していない。
|
||||||
|
|
||||||
|
管理、guest実行、Server上の拡張データを横断する判断をWIT資産の近くで一か所に保持する。実装時には各wire型と境界testを契約の機械的な根拠にし、ここには所有境界・失敗保証・採用理由を残す。作業順と進捗の正本はO-4、T-469、T-465、T-533とする。
|
||||||
|
|
||||||
|
## 1. 三つのAPIと所有境界
|
||||||
|
|
||||||
|
Pluginを管理する権限とPluginコードに渡す権限を分離する。Serverに保存するからServer operatorしか導入できない、とはしない。
|
||||||
|
|
||||||
|
| 境界 | 呼び出し方向 | 責任 |
|
||||||
|
| --- | --- | --- |
|
||||||
|
| Workspace管理API | 認可された管理client → Server | 導入・更新・失効・削除、設定、検証結果、診断 |
|
||||||
|
| Component exports | Runtime Host → guest | Tool呼び出し、Service/event処理 |
|
||||||
|
| Host imports | guest → Runtime Host → 必要なdomain owner | 外部通信、拡張データ、診断等の限定された作用 |
|
||||||
|
|
||||||
|
ServerはWorkspace所属、設定、installation、artifact、永続操作の正本を持つ。Runtimeはplanを適用し、guestと接続・一時handleを所有する。公開HTTP DTOは`server-api`、domain認可と永続化は`workspace-server`、guest ABIは`resources/plugin/wit`とPDKの責任とする。Rust traitやServer RESTを汎用proxyとしてguestへ渡さない。
|
||||||
|
|
||||||
|
Workspace管理者は自Workspaceに導入・更新できる。読取りcallerにはWorkspace読取り権限、mutationにはWorkspace管理権限を要求し、既存の認証済みactorから判断する。operator権限は通常導入の前提にしない。Worker/model/repository/packageの自己申告は管理権限にならない。
|
||||||
|
|
||||||
|
Hostが実行する全操作の上限は、packageの要求、Workspaceの付与、Profileの縮小、呼出元から委譲可能な範囲、現在の対象資源・運用制約の積集合とする。Serviceはlaunch時に明示されたService用grantを使い、管理者のsession credentialを借用しない。SubWorkerへのPlugin継承も再解決された委譲範囲内のみとし、親のhandleを移送しない。
|
||||||
|
|
||||||
|
## 2. identityと保存するもの
|
||||||
|
|
||||||
|
installationはWorkspaceに属し、artifactはimmutable bytesで識別する。同じpackage名やversionは権限でも実体の一意性の証拠でもない。
|
||||||
|
|
||||||
|
- `installation_id`はServerが発行するWorkspace内の導入identity。package作者の`id`は名前であり、このidentityを置換しない。
|
||||||
|
- 一つのinstallationに複数のimmutable artifactを登録できる。updateは新しいdigestの登録であり既存bytesを置換しない。同じinstallation内のpackage id/versionに異なる内容を登録する場合はversion衝突として拒否する。他Workspaceの同名versionは検査・存在開示の対象にしない。
|
||||||
|
- 同一package idの別installationは許すが、設定aliasを一意にする。同じpackageを別credentialで使う用途をversion名の改変で代替しない。
|
||||||
|
- Tool公開名はinstallationの不変ID由来namespaceと宣言Tool名からHostが構成する。表示aliasの変更で既存planのTool名を変えない。衝突・長さ超過はplan検証時に拒否し、登録時に黙ってrenameしない。
|
||||||
|
- artifact blobの共有は物理dedupeに限定する。他Workspaceのcatalog、設定、導入履歴、data、grantは共有されない。hashを知っているだけでは取得できない。
|
||||||
|
|
||||||
|
### package digestと配送digest
|
||||||
|
|
||||||
|
既存`manifest::plugin`のdigestは、path順のfile集合をhashするpackage identityであり、ZIP全体のSHA-256ではない。両者を同じfieldとして検証しない。
|
||||||
|
|
||||||
|
- `package_digest`は既存のfile集合digestを維持する。schema/manifest/componentを含む論理内容の同一性を検証する。
|
||||||
|
- `artifact_digest`はServerが保存・配送するcanonical archive bytesのSHA-256とする。`artifact_size`と共に配送・cacheを検証する。
|
||||||
|
- Serverは検証済みfile集合から既存canonical pack規則でarchiveを生成し、そのbytesを一度保存する。取得時に再packしない。upload元ZIPのbytesと保存bytesが異なり得るため、両digestと保存sizeを応答する。
|
||||||
|
- planは両digest、package schema、exact Component world/API versionを固定する。Runtimeは受信bytesのhashに加え、package validatorで論理digestを検証する。
|
||||||
|
|
||||||
|
## 3. Workspace管理API
|
||||||
|
|
||||||
|
管理操作は既存のtyped `ServerApi`へ追加する。以下は採用するoperationの意味であり、別のPlugin専用HTTP frameworkは作らない。HTTP prefixは既存の`/api/w/{workspace}`とし、route/DTO/OpenAPI/clientは同じ宣言から生成する。
|
||||||
|
|
||||||
|
### 操作と競合
|
||||||
|
|
||||||
|
| 操作 | 契約 |
|
||||||
|
| --- | --- |
|
||||||
|
| `POST /plugin-uploads` | bounded binary `.yoi-plugin`を受け、caller/Workspaceに束縛した一時upload identityと検証結果を返す。install・enableはしない |
|
||||||
|
| `POST /plugins` | upload identity、Workspace内alias、operation keyでinstallationを作成。結果はinstallation/artifact identityと確定operationの識別子 |
|
||||||
|
| `GET /plugins` / `GET /plugins/{installation}` | Workspace-scoped bounded一覧・詳細。opaque cursor、page sizeの上限あり。guest codeは実行しない |
|
||||||
|
| `POST /plugins/{installation}/artifacts` | 新しい検証済みuploadからartifact追加。観測した最終catalog変更operationと新しいoperation keyが必要。既存plan/configは変更しない |
|
||||||
|
| `POST /plugins/{installation}/revoke` | 指定artifact digestの導入許可を失効。観測した最終catalog変更operation、理由、新しいoperation key。新規実行と既存実行の操作権限を撤回 |
|
||||||
|
| `POST /plugins/{installation}/uninstall` | 観測した最終catalog変更operationと新しいoperation keyで除去を要求。参照があれば拒否し、安全な阻害情報を返す |
|
||||||
|
| `GET /plugin-operations/{operation}` | 認可された同Workspace callerが保存結果を照合。結果不明のmutationを新しいkeyでやり直さない |
|
||||||
|
|
||||||
|
uploadはmanagement operationと分ける。巨大bytesをJSON/idempotency記録に埋めず、検証に失敗したuploadからcatalogが生じないようにするためである。upload identityは24時間で失効する一時資源で、同じcontentの再uploadは可能。失効後も既にcommitしたinstallationを消さない。usage/quotaは未確定uploadも含む。
|
||||||
|
|
||||||
|
catalog mutationのoperation keyはWorkspace・認証主体・操作kindに束縛し、canonical request fingerprintと保存結果をdomain更新と同じtransactionで確定する。fingerprintは対象、観測した最終catalog変更operation、内容digest、alias、理由など意味を変える入力を含み、生成された結果IDや秘密値を含まない。同じkey/入力には元の結果、異なる入力にはconflictを返す。replay時も現在の読取り権限を確認し、権限失効後に以前の結果を漏らさない。保存済みreplayは最新stateの観測ではない。
|
||||||
|
|
||||||
|
catalogの追加・失効・除去では、そのinstallationの最終変更を確定したoperation identityをtransaction内で比較してから更新する。時刻やstateだけでは、同じstateのままartifactが追加された後の古い削除判断を拒否できないためである。設定内容はconfig treeの実内容digestで比較し、別の汎用更新カウンタは導入しない。
|
||||||
|
|
||||||
|
configのenable/disable、artifact選択、接続binding、grant変更は既存Workspace config Preview/Commit/CASへ統合する。別のenable REST endpointを作って設定正本を二重化しない。Preview後のinstallation/schema変更はcommit時にも照合する。
|
||||||
|
|
||||||
|
### install・disable・revoke・uninstall
|
||||||
|
|
||||||
|
- installは導入可能なartifactを登録するだけで、Workerの開始やTool登録を行わない。
|
||||||
|
- updateとconfig disableは新規/replacementのplanに反映する。既存Workerのpinをhot reloadしない。即時停止が必要なら明示revokeを使う。
|
||||||
|
- revokeはinstallationとartifact digestの組に対して不可逆とする。同じ組を再許可して古い実行を復活させない。管理者が同じbytesを再利用する場合は新しいinstallationを明示作成し、新しいconfig/planを生成する。古いinstallationの失効tombstoneとpinは保持する。
|
||||||
|
- revokeは新規Host操作を拒否し、稼働中guestの取消・接続closeを要求する。既に外部へ送った効果の取り消しは保証しない。Remote Runtimeへ失効通知を送るだけでは認可が撤回された証拠にならない。保護操作のadmissionはownerでlive確認し、確認不能なら拒否する。stream接続はHostの失効監視で閉じ、監視を維持できない場合も継続しない。
|
||||||
|
- uninstallはconfig、active/pending/restorable Worker pinが参照する間は拒否する。revokeしてもpinを勝手に消さない。Worker retention解除・config変更・uninstallを別操作として扱う。
|
||||||
|
- installationの拡張データが残る場合もuninstallは拒否する。後述の明示purgeで消すか、データを保持したdisabled installationとして残す。
|
||||||
|
- artifact GCは全Workspaceの参照を確認する。DB参照より先に永続blobをpublishし、crash時の未参照blobは後で回収する。未publish blobをcatalogから参照させない。
|
||||||
|
|
||||||
|
参照検査・新規pin/config参照の作成・削除fenceは同じdomain transaction境界で競合を解決する。削除直前の単なるlist結果では判断しない。
|
||||||
|
|
||||||
|
HTTP errorは既存共通error envelopeへ合わせる。invalid=400、未認証=401、権限不足=403、不可視対象=404、最終catalog変更operation不一致/operation-key/参照競合=409、size=413、quota=429、owner unavailable=503を使う。他Workspace対象の存在をerror詳細から漏らさない。errorの`retryable`だけでmutationの新規再送を許可しない。
|
||||||
|
|
||||||
|
## 4. package宣言とABI version
|
||||||
|
|
||||||
|
新しい実行契約は`yoi:plugin@2.0.0`と`yoi:host@2.0.0`にする。JSON文字列だけのHost呼出し、数値handle、guest側WebSocket recvを変えるため、v1の意味を黙って差し替えない。
|
||||||
|
|
||||||
|
- 新しいmanifest schemaはv2とし、required worlds/interfaces、Tool schema、接続slot、拡張data schema、event subscriptionを静的に宣言する。schema contributionを得るためにguestを実行しない。
|
||||||
|
- Workspace設定はDCDLのclosed typed schemaで検証する。Toolのmodel-facing入力はJSON Schemaで定義する。JSON SchemaとDCDL schemaを同じものと仮定しない。
|
||||||
|
- Componentの実imports/exportsを静的に検査してmanifestと照合する。manifestの申告だけで未公開importをlinkしない。
|
||||||
|
- 初期実装はexact world/interface versionだけを受け入れる。semver rangeでHost APIを黙って置換しない。互換な追加は別versionとして明示サポートし、既存pinには元の契約を提供するかunsupportedで止める。
|
||||||
|
- 旧v1 packageはofflineで読めても新Runtimeでは実行対象にしない。v2向けrebuild/explicit installを要求し、汎用JSON adapterで旧権限経路を復活させない。
|
||||||
|
- PDK/template/validator/WITを一緒にv2へ更新する。今回の設計文書だけを根拠に既存v1 templateが新Runtimeで動くとは扱わない。
|
||||||
|
|
||||||
|
## 5. Component exports
|
||||||
|
|
||||||
|
Tool payloadだけはPluginごとにschemaが異なるため、schema検証済みUTF-8 JSONを運ぶ。それ以外の共通context、error、event envelope、Host request/resultはWIT record/variant/resourceで定義する。
|
||||||
|
|
||||||
|
以下は採用するinterfaceの形を説明する最小例で、現行WITへ追加済みという意味ではない。
|
||||||
|
|
||||||
|
```wit
|
||||||
|
package yoi:plugin@2.0.0;
|
||||||
|
|
||||||
|
interface tool {
|
||||||
|
record invocation {
|
||||||
|
invocation-id: string,
|
||||||
|
tool-name: string,
|
||||||
|
input-json: string,
|
||||||
|
config-json: string,
|
||||||
|
remaining-ms: u32,
|
||||||
|
}
|
||||||
|
record output {
|
||||||
|
summary: string,
|
||||||
|
content: option<string>,
|
||||||
|
structured-json: option<string>,
|
||||||
|
}
|
||||||
|
record tool-error {
|
||||||
|
code: string,
|
||||||
|
message: string,
|
||||||
|
}
|
||||||
|
invoke: func(input: invocation) -> result<output, tool-error>;
|
||||||
|
}
|
||||||
|
|
||||||
|
world tool-plugin {
|
||||||
|
export tool;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`invocation-id`はHost生成の追跡identityであり権限ではない。contextにServer token、host path、任意Workspace selectorを含めない。configはplanのnon-secret snapshotで、secret値は含めない。structured outputには宣言されたoutput schemaを要求し、Hostが検証する。
|
||||||
|
|
||||||
|
Tool worldは呼出しごとにfresh Store/instanceとし、compiled Componentは共有できる。Toolのglobal/static memoryを永続化や他呼出しとの通信に使えない。v2初期Toolは応答一回のbounded callとし、guestがmodelへ直接streamを注入するAPIは設けない。
|
||||||
|
|
||||||
|
Service/event worldには`on-start(config) -> result`、`on-event(envelope) -> event-result`、`on-stop(reason) -> result`を設ける。いずれもfresh Storeで呼び、HostがWorker/installation/planに束縛したService resourceを保持する。ToolとServiceで共有する状態はHost経由の明示dataとし、guest memoryの残存を契約にしない。`status()`を呼んで診断のためにguestを実行せず、Hostが実行・接続・失敗の観測を返す。
|
||||||
|
|
||||||
|
`on-start`は設定検証とローカル初期化だけを行い、logging以外のHost作用を認めない。GETも副作用を持ち得るため、汎用HTTPや付随data書込みを初期化中に許可しない。外部event sourceの接続は全required Pluginの検証・導入成功後にHostが開始する。initialization失敗時に外部効果のrollbackを装わない。`on-stop`もloggingだけのbest effort通知で、resource解放の前提にしない。
|
||||||
|
|
||||||
|
## 6. Host importsとcapability lifetime
|
||||||
|
|
||||||
|
公開するinterfaceは機能ごとに分ける。汎用`invoke(method, json)`、汎用Server HTTP proxy、DB connection、host path、任意process実行APIは提供しない。
|
||||||
|
|
||||||
|
| interface | 操作の意味 |
|
||||||
|
| --- | --- |
|
||||||
|
| `connections` | 宣言slot名から、その呼出しに許可されたconnection resourceを取得 |
|
||||||
|
| `http` | connectionに束縛されたbounded request/response |
|
||||||
|
| `extension-data` | Plugin所有schemaのdataを対象object capability経由でread/CAS update |
|
||||||
|
| `logging` | bounded structured診断。model contextへの注入ではない |
|
||||||
|
| `workdir` | 後続で既存attachment-scoped operationを型付きにadapt。ambient filesystemは不可 |
|
||||||
|
| `websocket` | 後続のHost-owned driverへのsend/close。recvはevent配送で行う |
|
||||||
|
|
||||||
|
`connections.open(slot)`はWIT resourceを返す。slotはmanifestの宣言名で、Workspace設定が具体的な接続先とsecret参照にbindする。guestは任意credential IDや別Workspace IDを指定しない。resourceはStore/呼出し内だけで有効で、serialize・persist・別Workerへtransferできない。Serviceの長寿命接続自体はHost所有とし、各event処理時にfresh handleを発行する。
|
||||||
|
|
||||||
|
Hostは各importでcurrent invocation、installation、plan、付与範囲、対象資源、deadline/cancellationを照合する。lookup済みhandleを後続操作の無条件許可にしない。guestのComponent start sectionからの外部I/Oも禁止し、export呼出し用contextのないimportを拒否する。
|
||||||
|
|
||||||
|
planはconfig treeの内容digest、Profile projection digest、installation/artifact identities、接続binding identityとpolicyの内容digest、宣言schemaと実効grant、実行主体と委譲identityを固定する。管理操作を行った人の資格情報はguestの実行主体ではない。
|
||||||
|
|
||||||
|
| 変更・状態 | 既存plan/new export/restoreへの作用 |
|
||||||
|
| --- | --- |
|
||||||
|
| artifact追加、config enable/disable、Profileの選択・grant設定変更 | 新規/replacement planだけに適用。既存pinの集合や付与snapshotは変更しない |
|
||||||
|
| Workspace/対象資源のlive grant上限の縮小、明示的grant revoke、実行主体の委譲失効 | 既存snapshotとの積集合を直ちに縮小。次のadmission/Host操作を拒否し、進行中処理へ取消要求 |
|
||||||
|
| live grant拡大 | 古いsnapshotを広げない。追加利用には新規planが必要 |
|
||||||
|
| 接続binding失効、installation+artifactのrevoke | 新しいexport・接続・restoreを拒否。既存guest/driverを取消対象にする |
|
||||||
|
| 管理者の管理権限喪失 | 以後の管理操作を拒否。独立したWorker実行権限まで導入者の権限と同一視しない |
|
||||||
|
| ownerへ確認できない | fresh export、restore、保護操作・保存結果読取りを拒否。cached planを認可にしない |
|
||||||
|
|
||||||
|
各fresh exportの開始とrestoreにはServer ownerでinstallation/plan/実行主体のlive admissionを行う。pure Toolも例外にしない。既にadmit済みのローカル計算は取消・deadlineまで進み得るが、失効後の結果公開と追加Host作用は拒否する。初期Toolにoffline実行leaseは設けない。設定でのgrant変更とlive security grantの失効を混同しないよう、管理画面/APIの結果も適用対象を明示する。
|
||||||
|
|
||||||
|
WASIを一括linkしない。v2 worldに宣言したimportだけをallowlistし、ambient filesystem/network/environment/stdin/stdoutを与えない。clock/random等を追加する場合も個別のversioned interfaceとして明示する。PDKはこのimport集合だけでcompileできるようにする。
|
||||||
|
|
||||||
|
Wasmtimeのasync Host実装を使って待機中のexecutor threadを占有しない。ただしguestにexperimental async Component ABIを要求せず、初期WITは通常のcall/resultとする。Storeを並行利用せず、同一Service consumerへのevent処理を直列化する。
|
||||||
|
|
||||||
|
## 7. 外部HTTP接続
|
||||||
|
|
||||||
|
HTTPはURLを自由に受けるAPIではなく、connection resourceに対する相対pathのrequestとする。Server/Runtimeの資格情報を外部requestへ転用しない。
|
||||||
|
|
||||||
|
requestはmethod enum、absolute-path形式の相対path、query pair、許可されたheader pair、body bytes、要求timeoutを持つ。responseはstatus、許可されたheader、body bytesを持つ。4xx/5xxもHTTP応答として返し、transport errorと区別する。binaryはbyte listで運び、base64やJSON stringで二重に包まない。
|
||||||
|
|
||||||
|
初期Host interfaceの型は次の形にする。Tool worldにはこれらの必要なinterfaceを明示importし、未使用の機能を一括importするworldにはしない。
|
||||||
|
|
||||||
|
```wit
|
||||||
|
package yoi:host@2.0.0;
|
||||||
|
|
||||||
|
interface errors {
|
||||||
|
enum code {
|
||||||
|
invalid-input, denied, not-found, conflict, unsupported,
|
||||||
|
quota-exceeded, deadline-exceeded, cancelled, unavailable,
|
||||||
|
indeterminate, receipt-expired,
|
||||||
|
}
|
||||||
|
enum effect-observation { not-dispatched, response-observed, indeterminate }
|
||||||
|
record host-error {
|
||||||
|
code: code,
|
||||||
|
message: string,
|
||||||
|
diagnostic-id: string,
|
||||||
|
effect: effect-observation,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
interface connections {
|
||||||
|
use errors.{host-error};
|
||||||
|
resource connection;
|
||||||
|
open: func(slot: string) -> result<connection, host-error>;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface http {
|
||||||
|
use connections.{connection};
|
||||||
|
use errors.{host-error};
|
||||||
|
enum method { get, head, post, put, patch, delete, options }
|
||||||
|
record pair { name: string, value: string }
|
||||||
|
record request {
|
||||||
|
method: method,
|
||||||
|
path: string,
|
||||||
|
query: list<pair>,
|
||||||
|
headers: list<pair>,
|
||||||
|
body: list<u8>,
|
||||||
|
timeout-ms: u32,
|
||||||
|
effect-key: string,
|
||||||
|
}
|
||||||
|
record response {
|
||||||
|
status: u16,
|
||||||
|
headers: list<pair>,
|
||||||
|
body: list<u8>,
|
||||||
|
}
|
||||||
|
send: func(target: borrow<connection>, input: request) -> result<response, host-error>;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
queryはHostがpercent-encodeして送る。path中への`?`埋込みを拒否し、query/headerの重複・順序は許可されたprotocol semanticsを保持する。guest指定の接続/転送制御headerは拒否し、Hostが必要なものを生成する。effect-keyは最大128 UTF-8 bytesの非秘密のopaque文字列とし、Hostのinvocationに束縛する。
|
||||||
|
|
||||||
|
- Workspace bindingはexact origin(scheme/host/port)、許可method、segment境界のpath範囲、secret参照、資源上限を持つ。packageはslotと要求範囲を宣言できるが、bindingを自己発行できない。
|
||||||
|
- 初期APIはHTTPSのみ。TLS検証を無効にできない。raw socket、任意proxy、Unix socket、CONNECT、upgrade、redirect自動追従は提供しない。
|
||||||
|
- pathはHostが一度canonicalizeし検査・送信に同じ値を使う。scheme/authority変更、userinfo、fragment、backslash、encoded separator/dot traversalを拒否する。prefixの文字列一致で`/api`に`/apix`を含めない。
|
||||||
|
- HostがDNS解決した全候補を検査し、許可したaddressへ接続を固定する。接続時の再解決で検査を迂回しない。TLSの名前検証とSNIは許可hostnameを使う。
|
||||||
|
- loopback、link-local、metadata endpoint、Server/Runtime管理endpointを拒否する。private LANはServer運用policyが許す範囲内でWorkspaceが明示bindingできる。Workspace管理権限だけで共有Hostの内部network全体を開かない。
|
||||||
|
- `Authorization`、Cookie、Host、proxy/hop-by-hop header等はguestが指定できない。接続policyで許可したnon-secret headerだけを受け付ける。credentialはHostが指定origin/pathへだけ注入し、responseのSet-Cookie等を返さない。
|
||||||
|
- credentialの値をguestへ返すAPIはない。ただし相手serverがsecretをechoする可能性まで完全に除去できるとは主張しない。credentialの配送先はWorkspaceが管理する信頼境界として扱う。
|
||||||
|
- planはsecret参照と接続policyの内容digestを固定し、secret値は保存しない。通常のsecret rotationは同じbindingのHost解決で反映できるが、宛先/権限変更は新規planにする。live失効・policy縮小は古いplanにも効く。
|
||||||
|
- request/responseはbuffered boundedとし、上限到達時は切断してlimit errorを返す。切断されたresponseを成功に見せない。圧縮展開後の上限も検査する。
|
||||||
|
|
||||||
|
### 外部副作用と再試行
|
||||||
|
|
||||||
|
HostはHTTP requestを自動再送しない。GETであってもprovider固有semanticsがあるため、transport層が勝手に再試行しない。redirectもguestにresponseとして返すだけで、別originへの追跡は新しい認可済みbindingが必要になる。
|
||||||
|
|
||||||
|
全HTTP requestに`(Workspace, installation, Worker, consumer-or-tool-call, logical-invocation, effect-key)`のscopeを使い、request fingerprint、送信admission、保存結果を記録する。consumerはServiceの場合の固定identity、Toolの場合は永続Tool-call identityであり、別consumerで同じevent-idを処理してもreceiptを共有しない。GETも例外にせず、methodだけから副作用なしを推定しない。同じkey/内容は保存結果を返し、異なる内容はconflict。guestがkeyを変えれば新しい作用なので、package内の再試行も自動的に安全になるわけではない。
|
||||||
|
|
||||||
|
fingerprintはinstallation/plan、接続binding identityとslotとpolicyの内容digest、method、正規化path、順序を保つquery/header、body hashを含む。secretの値、残りdeadline、attemptは含めない。saved responseはboundedでWorkspace保護データとして保持し、diagnostic logとは分離する。request body/secret headerの生値をreceiptへ保存しない。同一requestの保存結果を返す際にも現在のinstallation/接続/読取り権限を検証する。
|
||||||
|
|
||||||
|
invocation identityはHostの永続Tool-call/event記録に結び付け、同じTool-callの回復や同じconsumer/eventの再配送では固定する。guestがidentityを指定・変更できない。新しいユーザーの呼出しは新identityとする。返却済みTool結果は再実行せず既存結果を使い、未完了の回復を行う場合だけ同じeffect-key領域を使う。
|
||||||
|
|
||||||
|
送信前にdurable admissionを記録し、記録不能なら送らない。送信した可能性があるままtimeout/切断/crashした場合は`indeterminate`とし、同じkeyでも汎用Hostは再送しない。provider固有の照合・idempotency APIがある場合だけ、そのadapterの契約で回復できる。Tool結果が失敗でも外部効果が取り消されたとは表示しない。
|
||||||
|
|
||||||
|
保持期限を過ぎたeffect keyは未使用keyとして扱わず`receipt-expired`で拒否する。古いinvocation/eventの再開も同様で、再実行は明示的な新しい依頼にする。invocationの受理時にHostがreceipt照合期限を固定し、期限の過ぎたidentityには新しいeffect-keyを追加できない。期限情報/tombstoneは関連Worker/eventの保持期間中残し、bodyを削除しても未実行扱いに戻さない。receipt保持とevent再配送可能期間をHostで連動させ、exactly-onceを約束しない。
|
||||||
|
|
||||||
|
## 8. 共通error、出力、診断
|
||||||
|
|
||||||
|
Host errorは閉じたWIT variantで返し、少なくともinvalid-input、denied、not-found、conflict、unsupported、quota-exceeded、deadline-exceeded、cancelled、unavailable、indeterminate、receipt-expiredを区別する。guest business errorは宣言されたcodeとbounded messageで返す。未知guest codeはplugin-failedへ正規化し、Host errorの権限をguestが偽装できないようにする。
|
||||||
|
|
||||||
|
Host errorには安全なmessage、診断ID、外部効果の観測(not-dispatched / response-observed / indeterminate)を付ける。`response-observed`はbusiness成功や外部transaction commitの証明ではない。再試行の可否はerror文字列でなくoperation契約で判断する。trap、不正output、resource超過はHostが実行失敗として記録する。
|
||||||
|
|
||||||
|
Tool outputは通常のWorker Tool結果・履歴経路へ流す。Pluginがhidden prompt、System instruction、任意Session entryを直接差し込むAPIはない。structured JSONはschema検証し、不正・超過時に切り詰めたJSONを返さない。textの表示短縮はHost projectionで明示する。
|
||||||
|
|
||||||
|
loggingはlevel、event code、安全なmessageとscalar fieldsに限定する。raw request/response、secret、prompt、host pathを標準診断に保存しない。Pluginが受け取った外部データ自体はuntrusted contentであり、命令や権限として扱わない。
|
||||||
|
|
||||||
|
## 9. 付随データの契約
|
||||||
|
|
||||||
|
Server拡張はdomain objectに関連付けるPlugin所有のtyped dataとし、既存tableへの任意column追加では実現しない。物理保存schemaはHostが管理する。`feature-storage`のnative migration/SQLite handleは信頼済みHost用でありguestへ公開しない。
|
||||||
|
|
||||||
|
addressはHost contextのWorkspace/installationと、認可済みobject resource、宣言collection、schema id/version、keyから決まる。schema id/versionも物理・論理namespaceに含め、新旧artifactが同じcellを別schemaで読み書きしない。guestにWorkspace selectorやraw table名を与えない。最初のobject kindはTicketとし、Hostが存在・Workspace・callerのread/write権限を検証したTicket resourceだけを使う。一般objectへの拡張はadapterと権限を追加して行う。
|
||||||
|
|
||||||
|
後続の`objects.resolve-ticket(ticket-ref, access)`はreadまたはextension-writeを要求し、現在のWorker主体・呼出元からの委譲とPlugin grantを検証して呼出し内だけ有効なWIT `ticket` resourceを返す。Tool入力のTicket refもevent envelopeのrefも単なる選択子であり、どちらも同じ解決を通す。ServiceにはService用grantを適用し、eventを受けたことだけでwriteを許可しない。削除済み・不可視・別Workspaceの対象はnot-foundとする。
|
||||||
|
|
||||||
|
`ticket-read.read(borrow<ticket>)`はread権限を再検査し、title、case state、content digest、bounded bodyとtruncated flagだけを返す専用read APIとする。thread、添付、認証・管理情報、将来増えるfieldを汎用serializeで公開しない。上限は64 KiB、完全本文が必要なら本APIではunsupportedとする。eventの削除通知自体は安全なmetadataとして配送できるが、削除済みTicket resourceをresolve/readできる意味ではない。
|
||||||
|
|
||||||
|
- packageはcollectionごとにschema id/versionとclosed typed schemaを宣言する。data valueのJSONはそのschemaで検証する。
|
||||||
|
- APIは`get`、`put(expected_last_write | never_written, operation_key, value)`、`delete(expected_last_write, operation_key)`、bounded `list(cursor, limit)`。任意query、SQL、cross-object transactionは公開しない。
|
||||||
|
- 同じcellへのput/deleteを確定した最終operation identityを、書込みtransaction内で観測値と比較する。CAS成功とoperation receiptを同じtransactionで保存し、自動読直しによるlast-write-winsへ変えない。同じ内容の再書込みや削除・再作成を時刻/内容hashだけで区別できないため、このoperationを比較する。
|
||||||
|
- deleteは最終delete operationのtombstoneを保持する。再作成はそのoperationを期待値にし、`never_written`を通さない。これによりABAを防ぐ。`get`は対象objectの認可成功後に限り、present(value, last-write)、never-written、deleted(last-delete)を区別する。不可視objectと不存在は同じnot-foundにする。
|
||||||
|
- data mutationのreceiptはHTTPと同じ実行scope(Workspace/installation/Worker/consumer-or-tool-call/logical-invocation)とguest operation-keyに束縛する。fingerprintに完全なcell address、put/delete種別、期待last-write、schemaとvalueのdigestを含める。異入力でkeyを再利用すればconflictとする。現在の対象認可確認後にreceipt照合をCAS検査より先に行い、成功済み同一要求を最新cellとの差で失敗にしない。照合期限・expired・tombstone保持はHTTPと同じinvocation規則を使う。
|
||||||
|
- `lookup-mutation(operation-key)`で同じinvocationの保存結果(committed: mutation identityと結果digest / not-recorded / expired)を現在の対象認可の下で照合できる。event再配送では先にreceiptを照合し、成功済みなら新しい値を読み直して同じkeyで別のputを作らない。ownerがnot-recordedを返した場合だけ現在のcellから要求を作れる。照合通信失敗はnot-recordedではない。外部HTTP・data・event ackを跨ぐ原子性は保証せず、handlerは段階ごとの保存結果で回復する。
|
||||||
|
- listはstable key順のbounded探索でありsnapshot全体の一貫性は保証しない。cursorはWorkspace/installation/object/queryに束縛し、不正・失効cursorを最初のpageに黙って戻さない。
|
||||||
|
- Ticketの通常fieldやworkflow判断をこのdata APIで変更できない。必要ならそのdomainの専用Host APIを別に設計する。
|
||||||
|
- 親object削除とPlugin data書込みはowner側の削除fenceで競合を解決する。削除後に新しい子dataを作らず、retention後にHostが回収する。
|
||||||
|
- schemaの破壊的変更は新しいschema version/namespaceとする。既存dataをguest起動時にmigrationしない。移行は新旧schemaを明示した認可付き操作として後続で設計し、初期APIでは自動変換しない。
|
||||||
|
- rollbackで旧artifactを使う場合も元schemaだけを読む。最新schemaへ読み替えない。
|
||||||
|
- revoke/disableではdataを消さない。Workspace管理者の明示purgeをoperation keyと期待data-generationで行い、先に実行をrevoke/fenceする。purge receiptを残し、uninstallとの一操作への暗黙統合をしない。
|
||||||
|
|
||||||
|
## 10. commit済みeventとService
|
||||||
|
|
||||||
|
最初のhookはcommit済みeventの観察とする。PluginをServer transaction内で実行せず、元操作の成功をPluginの応答待ちにしない。domain ownerがtransactional outboxへeventを記録し、Hostが認可・投影して配送する。共有の無制限event busは提供しない。
|
||||||
|
|
||||||
|
最初のeventはTicket変更通知とし、envelopeはevent-id、event-type/version、occurred-at、対象resource reference、変更を確定したdomain eventのidentityと対象内容digest、correlation/causation identity、bounded typed payloadを持つ。全文や秘密fieldは載せず、必要な詳細は認可付きAPIで取得する。削除eventは削除済みreferenceと安全なmetadataのみとし、resource accessを復活させない。
|
||||||
|
|
||||||
|
subscriptionはmanifest宣言・Workspace grant・Profile選択から確定し、guestが任意event名を動的subscribeしない。consumerはWorker/installation/plan内のservice単位であり、別Workerに同じPluginを入れると別consumerになる。Workspace-wide singletonや分散consumer groupは暗黙に作らない。
|
||||||
|
|
||||||
|
- event-idは再配送でも固定、attemptはHost生成の観測値。consumer内はHost保存順に一件ずつ処理する。Workspace全体や別consumer間の全順序は約束しない。
|
||||||
|
- `event-result`はack、retry-after、rejectのvariant。ackはHostのcursor commitで確定し、guestのreturnだけでは確定しない。
|
||||||
|
- deliveryは保持期間内のat-least-once。crash/timeoutで再配送され得るため、外部作用はevent-id由来invocationとstable effect-keyを使う。payload/keyを変えた再送を同一作用として扱わない。
|
||||||
|
- retry-afterはeffectが結果不明でない場合のみ再配送を予約できる。indeterminateな外部効果を含むeventは要確認でconsumerを止め、汎用retryで再送しない。
|
||||||
|
- reject、繰返し失敗、queue上限、retention gapではconsumerをpausedとして可視化する。silent drop、無限retry、詰まったeventの無断skipはしない。復旧・明示skipはWorkspace管理操作として監査する。
|
||||||
|
- cursorが保持範囲から外れたらgapを返し、最新位置へ自動復旧しない。payload取得時にも対象への現在のread権限を再検査する。
|
||||||
|
- disableは新規planに作用し、既存consumerの即時停止はrevoke/Worker stopで行う。restoreは同じconsumer identity/pin/cursorを使い、replacementは新しいconsumerとして開始する。開始時点以前のeventを自動backfillしない。
|
||||||
|
- 同じconsumerの旧・新Runtimeが同時に処理しないようHost実行leaseでfenceする。receipt/cursor mutationもleaseを検証する。
|
||||||
|
|
||||||
|
WebSocketは別のHost-owned source driverがeventへ変換する。guestの`recv(timeout)` loopは復活させない。network frame自体にdurable delivery保証はないため、domain outbox eventと同じ保証を表示しない。必要なら外部protocolのsequence/resumeをbridge側で使う。
|
||||||
|
|
||||||
|
処理前の検証・拒否hookは初期APIに含めない。将来導入時は対象操作ごとに入力/返却値/timeout/failure policy/再入・副作用を契約化し、汎用interceptorとしてServer内部を公開しない。
|
||||||
|
|
||||||
|
## 11. 資源上限と終了
|
||||||
|
|
||||||
|
以下は新APIの初期defaultとして採用する。既存実装値の転載ではない。Hostは実効limitを起動前に固定して診断へ公開し、運用上限は縮小できる。guestから無制限や上限引上げを要求できない。
|
||||||
|
|
||||||
|
| 資源 | 初期default |
|
||||||
|
| --- | --- |
|
||||||
|
| Tool input / output、event payload | 各64 KiB(UTF-8/serialized bytes) |
|
||||||
|
| config、付随dataの1 cell | 各64 KiB |
|
||||||
|
| guest linear memory | 256 MiB / Store |
|
||||||
|
| Wasmtime fuel | 10,000,000 / export call |
|
||||||
|
| Tool/event export wall deadline | 30秒、上位caller deadlineが短ければそちら |
|
||||||
|
| start / stop hook | 5秒 / 2秒 |
|
||||||
|
| HTTP request body / response body | 各1 MiB |
|
||||||
|
| HTTP header総量 | 16 KiB、64 pairまで |
|
||||||
|
| HTTP request deadline | 15秒、残りcall deadline以下 |
|
||||||
|
| guest発行Host操作 | 同時4、累計32 / call |
|
||||||
|
| log総量 | 16 KiB / call、超過はdrop件数のみ記録 |
|
||||||
|
| Service待機queue | 256 event、満杯時は配送をpause |
|
||||||
|
| event再試行 | 最大5 attempt、待機上限5分、その後pause |
|
||||||
|
| event自動再配送・effect receipt照合 | 最低7日、期限後は明示expired/gap |
|
||||||
|
| 管理list | default 50、上限100件 |
|
||||||
|
|
||||||
|
fuelはCPU時間の精密尺度ではない。wall deadline、memory/table/instance/resource数のWasmtime limiterと併用する。HTTPとguest実行で別々に30秒ずつ延長せず、同じcall deadlineから残り時間を計算する。package最大size/file数は既存validator上限を再利用し、値の第二正本を作らない。Workspace全体の保存/queue/実行同時数はHostの有限quotaでadmissionし、個別上限だけで総量を無制限にしない。
|
||||||
|
|
||||||
|
終了順は新規admission停止、invocation取消、接続/driver停止、期限付きon-stop、登録lease解除、Store/一時resource解放とする。取消中に届いた結果で終了済みinvocationへ追加作用を発行しない。失効後のon-stopは外部write禁止。guestがstopに応答しなくてもHostはresourceを回収する。既に送信された外部効果はrollbackしない。
|
||||||
|
|
||||||
|
## 12. 初期範囲と契約検証
|
||||||
|
|
||||||
|
最初の実行APIはTool、connection-bound HTTPS、loggingに絞る。付随データとcommit済みeventは本書の設計を採用する後続interfaceであり、初期worldへ動かないstubを登録しない。WebSocket、Workdir、MCP transport、判断hookも未対応importとして明示拒否する。MCP専用のprocess/credential APIを先行して作らず、必要なHost capabilityを個別に追加する。
|
||||||
|
|
||||||
|
最初のvertical contract testは、Server operatorでないWorkspace管理者によるupload/install/config、plan生成、Remote/Embedded双方のWasmtime Tool実行、Host HTTPS、通常Tool履歴への結果、同一pin restoreを通す。mock guestだけでHost境界の証明を代替しない。
|
||||||
|
|
||||||
|
境界testでは少なくとも次を固定する。
|
||||||
|
|
||||||
|
- 同じblobでも別Workspaceのcatalog/plan/secret/dataを読めず、digestだけのfetchができない。
|
||||||
|
- v1/未対応world、manifestと実importの不一致、未宣言Tool、invalid input/outputを実行前または結果公開前に拒否する。
|
||||||
|
- path canonicalization、DNS rebinding、redirect、credential header、private/control-plane宛先を実際のHost adapterで検証する。
|
||||||
|
- upload/publish/DB commitの部分失敗、CAS競合、同一key再送、異入力key reuse、uninstallとpin作成の競合を検証する。
|
||||||
|
- effectの送信前失敗、送信後応答喪失、receipt期限、cancel/revokeを区別し、indeterminateから自動再送しない。
|
||||||
|
- fuel/memory/wall deadline、Host待機中の取消、trap、不応答on-stopでもresourceが回収される。
|
||||||
|
- 後続data/eventではCAS/ABA、親削除fence、schema rollback、再配送、retention gap、lease競合、外部効果不明時pauseを検証する。
|
||||||
|
|
||||||
|
採用する契約と実装手段を区別する。DBのtable名、内部trait名、物理blob配置、adapterのmodule分割は実装裁量だが、Workspace分離、pin、APIの型・失敗保証・再試行・拡張境界はこの契約を満たす必要がある。
|
||||||
@@ -1,5 +1,5 @@
|
|||||||
You are the independent MR Reviewer, a direct SubWorker of the review requester.
|
You are the independent MR Reviewer, a direct SubWorker of the review requester.
|
||||||
|
|
||||||
Evaluate the captured MR subject, Ticket `content_digest`, and complete linked MR/result snapshot against the Ticket requirements. Approval attests that snapshot, not just the selected diff. Inspect and test without modifying implementation files, repository refs, or Ticket state.
|
Evaluate the captured MR subject, Ticket `content_digest`, and complete linked MR/result snapshot against the Ticket requirements. Approval attests that snapshot, not just the selected diff. You may use Bash for Git and other commands.
|
||||||
|
|
||||||
Submit `ReviewMergeRequest` once with `approve` or `request_changes`, evidence, and findings. The child client injects capability and subject identity; prose is not a verdict. Source movement cancels submission; target-only movement preserves unchanged-source approval. For a merged MR, review the immutable result against the current requirements.
|
Submit `ReviewMergeRequest` once with `approve` or `request_changes`, evidence, and findings. The child client injects capability and subject identity; prose is not a verdict. Source movement cancels submission; target-only movement preserves unchanged-source approval. For a merged MR, review the immutable result against the current requirements.
|
||||||
|
|||||||
@@ -9,6 +9,8 @@ autotests = false
|
|||||||
[features]
|
[features]
|
||||||
default = []
|
default = []
|
||||||
e2e = []
|
e2e = []
|
||||||
|
fixture-support = ["yoi-workspace-server/test-support", "worker-runtime/test-support"]
|
||||||
|
restore-checkpoints = ["fixture-support", "worker-runtime/restore-checkpoints"]
|
||||||
|
|
||||||
[dependencies]
|
[dependencies]
|
||||||
libc.workspace = true
|
libc.workspace = true
|
||||||
@@ -17,11 +19,17 @@ serde = { workspace = true, features = ["derive"] }
|
|||||||
serde_json.workspace = true
|
serde_json.workspace = true
|
||||||
sha2.workspace = true
|
sha2.workspace = true
|
||||||
tempfile.workspace = true
|
tempfile.workspace = true
|
||||||
|
protocol.workspace = true
|
||||||
|
tungstenite = { version = "0.29", default-features = false, features = ["handshake"] }
|
||||||
|
|
||||||
[dev-dependencies]
|
[dev-dependencies]
|
||||||
chrono = { version = "0.4", default-features = false, features = ["clock"] }
|
chrono = { version = "0.4", default-features = false, features = ["clock"] }
|
||||||
yoi-workspace-server.workspace = true
|
yoi-workspace-server.workspace = true
|
||||||
server-api.workspace = true
|
server-api.workspace = true
|
||||||
|
worker-runtime.workspace = true
|
||||||
|
session-store.workspace = true
|
||||||
|
config-source.workspace = true
|
||||||
|
manifest.workspace = true
|
||||||
webauthn-rs-proto = "0.5.5"
|
webauthn-rs-proto = "0.5.5"
|
||||||
webauthn-authenticator-rs = { version = "0.5.5", default-features = false, features = ["softpasskey"] }
|
webauthn-authenticator-rs = { version = "0.5.5", default-features = false, features = ["softpasskey"] }
|
||||||
|
|
||||||
@@ -44,3 +52,8 @@ required-features = ["e2e"]
|
|||||||
name = "worker_environment"
|
name = "worker_environment"
|
||||||
path = "tests/worker_environment.rs"
|
path = "tests/worker_environment.rs"
|
||||||
required-features = ["e2e"]
|
required-features = ["e2e"]
|
||||||
|
|
||||||
|
[[test]]
|
||||||
|
name = "worker_restore"
|
||||||
|
path = "tests/worker_restore.rs"
|
||||||
|
required-features = ["e2e", "fixture-support"]
|
||||||
|
|||||||
@@ -7,3 +7,68 @@
|
|||||||
- [`PanelHarness`]: PTY経由の入力、観測イベントによる同期、失敗時のartifact保存。
|
- [`PanelHarness`]: PTY経由の入力、観測イベントによる同期、失敗時のartifact保存。
|
||||||
- [`FixtureWorkspace`]: TUI用WorkspaceとHOME/XDG領域の隔離。
|
- [`FixtureWorkspace`]: TUI用WorkspaceとHOME/XDG領域の隔離。
|
||||||
- [`WorkerE2eEnvironment`]: Backend/Runtime用プロセスとfixtureの管理。
|
- [`WorkerE2eEnvironment`]: Backend/Runtime用プロセスとfixtureの管理。
|
||||||
|
- [`FixtureSource`]: 提供資産の全ファイルSHA-256記録、独立copy、原本不変の検証。
|
||||||
|
- [`WorkerSessionSubscription`]: 公開Worker WebSocketの初期full Snapshot照合と停止確認。モデル呼出しは行わない。
|
||||||
|
- `tests/auth_passkeys/native_migration.rs`・`browser/native_passkey_migration.ts`: 実Backend/本番shell/Chromium native CTAP2の旧鍵移行。fixture境界・未確認範囲・明示選択コマンドはmoduleに置く。通常suiteやsoftware authenticatorの成功で代替しない。
|
||||||
|
|
||||||
|
## 固定Workerの段階起動
|
||||||
|
|
||||||
|
1. 提供fixtureを所有crateの型で検証し、`FixtureSource`で原本hashを記録する。各環境の一時領域へcopyする。fixtureが未提供・不適合なら失敗とし、代替Workerを生成しない。
|
||||||
|
2. `initialize_runtime_identity`で固定Runtime IDに新しい鍵を発行する。Backendの型付きstore APIで新しいAccount認証・Workspace署名identity・Runtime binding・固定Worker登録を準備する。
|
||||||
|
3. `start_seeded_backend`へ固定Workspace/Runtime IDとその環境の新しいbearerを渡す。Server起動後の公開APIからWorkspaceの公開identityを取得する。
|
||||||
|
4. `start_runtime_with_seed`はtrust設定後、Runtime processの起動前にseed closureを呼ぶ。ここで実Server URLを使いRuntime所有crateの型付きinstallを呼ぶ。Runtime store rootは`runtime_data_dir.join("runtime")`。
|
||||||
|
5. Worker一覧とdetailから停止状態・観測tokenを確認し、一つの固定request IDで`restore_worker`を一度だけ呼ぶ。結果不明・拒否は成功扱いせず、tokenを自動更新しない。
|
||||||
|
6. `WorkerSessionSubscription::connect`で期待`protocol::SessionSnapshot`全体(entry ID、timestamp、順序、本文を含む)を照合する。二環境を同時に保持し、一方のstop/cleanup後も他方の購読・一覧・内容を再確認する。
|
||||||
|
7. `stop_worker`、購読の`expect_stopped`、環境cleanup、原本`assert_unchanged`を確認する。
|
||||||
|
|
||||||
|
所有crateの`test-support`依存はテスト側の非default `fixture-support` featureだけで有効にする。製品binary providerは別Cargo invocationと専用`target/e2e-products`へ`yoi`・`yoi-server`・`yoi-runtime`を追加featureなしでビルドする。support test自身をこのtargetへビルドしてはならない。`YOI_E2E_BIN`等の明示binary overrideを使用する場合も、呼出し側が同じ分離を保証する。既存TUI用`e2e-test` featureは別の`target/e2e-tui-products`へビルドし、restore製品binaryへ混入しない。
|
||||||
|
|
||||||
|
固定資産の正常restore・並行隔離・応答喪失回復E2E:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
cargo test -p yoi-e2e --features e2e,fixture-support --test worker_restore
|
||||||
|
# 親から提供された候補の検証(読取専用、原本の自動採用や生成はしない)
|
||||||
|
YOI_E2E_RESTORE_FIXTURE=target/fixture-candidate-v1 \
|
||||||
|
cargo test -p yoi-e2e --features e2e,fixture-support --test worker_restore
|
||||||
|
```
|
||||||
|
|
||||||
|
標準資産は`tests/e2e/fixtures/worker-codex-v1`の`manifest.json`・`worker-template.json`・`expected.json`・`workspace-config.dcdl`。manifestの各asset hash/size、schemaと固定ID、生成元commit、modelを検証する。Workspace configのimport参照を黙って最新へ置換せず、Gitで生成元commitに対する`resources`と`crates/builtin-source`の一致を確認する(Git履歴がない場合も失敗)。移植したSessionの正規公開projectionを供給された会話行と照合し、その全snapshotを実executionのWebSocketと比較する。欠落fixture・hash不一致・未実行は成功扱いしない。
|
||||||
|
|
||||||
|
応答喪失ケースは同じ`WorkerE2eEnvironment`内の単発loopback relayで公開restore要求をそのまま転送し、完全なupstream応答を受け取った後に呼出元へ0byteで切断する。呼出元のtransport失敗とrelay側のAccepted観測は別の事実として記録する。回収は元のrequest ID・観測tokenをそのまま使い、fresh intentへの古いtokenはHTTP409を要求する。
|
||||||
|
|
||||||
|
process終了ケースはそのAccepted receipt確認後に環境が所有するRuntime childだけをkill/reapし、同じroot・Runtime identity・trustから再起動する。seedの再実行・tokenの差替えはしない。再起動前後のcanonical owner全体のhash・operation ID・owner数が一致し、公開APIの保存結果とfull Session snapshotを回収できることを確認する。公開Worker投影の観測tokenは再起動で変わるため、そのfieldだけはcurrent detailとの一致を別に確認する。canonical診断はboundedな読取りのみで、要求・receiptの内容や認証情報はartifactへ保存しない。
|
||||||
|
|
||||||
|
これは**commit済みかつterminal receipt保存後**のprocess終了であり、未commit候補やcommit直後・receipt前の終了証明ではない。固定Codex資産へSubmit/Notifyを送らないため、通常入力・Paused admissionの製品E2Eの証明でもない。検証jobはこの明示targetを選択する必要があり、通常suiteだけでは代替しない。
|
||||||
|
|
||||||
|
## Opt-in Runtime checkpoint process試験
|
||||||
|
|
||||||
|
上のfeatureless製品4case(target内の既存seed projection試験1件とは別)に加え、`restore-checkpoints`を明示した実Runtimeでdecision前後とactivation後・terminal前のlive windowを試験する。このfeatureはoffline fixture-supportと独立しており、通常RuntimeではCLI入口も環境変数による入口も無い。
|
||||||
|
|
||||||
|
```sh
|
||||||
|
# 同期を有効にしたRuntimeだけを専用targetへ。通常製品binaryを上書きしない。
|
||||||
|
cargo build --locked -p worker-runtime --bin yoi-runtime \
|
||||||
|
--features restore-checkpoints --target-dir target/e2e-checkpoint-products
|
||||||
|
cargo test --locked -p yoi-e2e --features e2e,restore-checkpoints \
|
||||||
|
--test worker_restore checkpoint:: -- --test-threads=1
|
||||||
|
```
|
||||||
|
|
||||||
|
`WorkerE2eEnvironment::use_checkpoint_runtime`はその環境のRuntime binary/hashと、隔離root内の明示configだけを選択する。global provider・環境変数・通常起動への暗黙設定は変更しない。configは一度だけCLIへ渡し、再起動時は渡さない。test側のlink featureをfeatureless製品binaryへ混入させない。
|
||||||
|
|
||||||
|
共有harnessはloopback ackのstage/Worker/operation/PID/nonceを確認し、直ちに所有childをkill/reapする。**offline正本を確認してから**同roots/identity/trustを再起動するため、ack後のdeadlineとの競合を成功扱いしない。NotCommittedの実私有候補は同operationのcleanup-only→RolledBack/Stopped、Committed/preterminalは同じdecision→実protocol/full Session snapshot→Acceptedへ回収する。元のrequest/token、owner数1、同operation、stale fresh intent拒否、保存source hash、restore run path数、最終cleanupを検証する。JSONでpending状態を作らず、seedは初回のみ。
|
||||||
|
|
||||||
|
orphan upload pinの代表repairは、隔離copyへ正規Session storeのupload/pin APIで作った未結付けpinを、実Profileのcommit後activationが解放する経路を使う。Prepared窓は未修復のsource保持、Activated窓は実修復bytes/hashの保持を区別し、kill/reap後に正本を読んでから再起動する。materialized headの固定資産を巻き戻さないため、この試験は先行repair済みNotCommittedのprocess証明ではない。Activated後のstale allocationは通常登録と同じallocation authorityで回収し、live ownerが残る場合はrun artifactsを回収しない。
|
||||||
|
|
||||||
|
明示解除・peer EOF・observer scopeの意図的panicも試験する。gateの8秒deadlineとscoped HTTP callの15秒deadlineが待機を制限し、通常operationの継続後に環境のRAIIがprocess/rootを回収する。これはtest scope失敗の回収証明であり、OSによるharness強制終了や電源断の一般的な保証ではない。原fixtureへのSubmit/Notifyや実モデル呼出しは行わない。checkpoint ack/at-death/recovery、kill/reap、binary/hashとcleanup artifactはfeaturelessログと分けて報告する。
|
||||||
|
|
||||||
|
通常のharness回帰試験(製品processは起動しない):
|
||||||
|
|
||||||
|
```sh
|
||||||
|
cargo test -p yoi-e2e --lib --no-default-features
|
||||||
|
```
|
||||||
|
|
||||||
|
既存の製品process E2Eは別結果として実行・報告する:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
cargo test -p yoi-e2e --features e2e --test worker_environment
|
||||||
|
cargo test -p yoi-e2e --features e2e --test panel --test rewind
|
||||||
|
```
|
||||||
|
|||||||
@@ -4,7 +4,7 @@
|
|||||||
// 固定expiry/短命cookieを作る明示fixtureだけが発行境界を代替し、実分類はBackendへ委ねる。
|
// 固定expiry/短命cookieを作る明示fixtureだけが発行境界を代替し、実分類はBackendへ委ねる。
|
||||||
// navigator.credentialsのみRustのResidentAuthenticatorへ接続する(物理認証器ではない)。
|
// navigator.credentialsのみRustのResidentAuthenticatorへ接続する(物理認証器ではない)。
|
||||||
import { assert, assertEquals } from "@std/assert";
|
import { assert, assertEquals } from "@std/assert";
|
||||||
import { extname, join, normalize } from "@std/path";
|
import { serveProductionShell } from "./static_shell.ts";
|
||||||
import { type Browser, type BrowserContext, chromium } from "playwright";
|
import { type Browser, type BrowserContext, chromium } from "playwright";
|
||||||
|
|
||||||
const [backend, origin, build, handle] = Deno.args;
|
const [backend, origin, build, handle] = Deno.args;
|
||||||
@@ -13,82 +13,43 @@ if (!backend || !origin || !build || !handle) {
|
|||||||
"Expected isolated Backend URL, browser origin, production build, handle",
|
"Expected isolated Backend URL, browser origin, production build, handle",
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
await Deno.stat(join(build, "index.html")); // 暗黙buildや別asset rootへのfallbackはしない。
|
const server = await serveProductionShell(
|
||||||
const mime: Record<string, string> = {
|
backend,
|
||||||
".html": "text/html; charset=utf-8",
|
origin,
|
||||||
".js": "text/javascript",
|
build,
|
||||||
".css": "text/css",
|
async (request) => {
|
||||||
".json": "application/json",
|
const url = new URL(request.url);
|
||||||
".svg": "image/svg+xml",
|
if (
|
||||||
".png": "image/png",
|
url.pathname === "/__e2e/expired-session" && request.method === "POST"
|
||||||
".woff2": "font/woff2",
|
) {
|
||||||
".ico": "image/x-icon",
|
if (request.headers.get("origin") !== origin) {
|
||||||
};
|
return new Response(null, { status: 403 });
|
||||||
const server = Deno.serve({
|
}
|
||||||
hostname: "127.0.0.1",
|
// Rustは環境専用DBの実登録userへ公開store APIで保存する。secretをbody/ログへ出さない。
|
||||||
port: Number(new URL(origin).port),
|
const fixture = await bridge("expired-session", {}) as {
|
||||||
onListen() {},
|
set_cookies: string[];
|
||||||
}, async (request) => {
|
};
|
||||||
const url = new URL(request.url);
|
const headers = new Headers({ "cache-control": "no-store" });
|
||||||
if (url.pathname === "/__e2e/expired-session" && request.method === "POST") {
|
for (const cookie of fixture.set_cookies) {
|
||||||
if (request.headers.get("origin") !== origin) {
|
headers.append("set-cookie", cookie);
|
||||||
return new Response(null, { status: 403 });
|
}
|
||||||
|
return new Response(null, { status: 204, headers });
|
||||||
}
|
}
|
||||||
// Rustは環境専用DBの実登録userへ公開store APIで保存する。secretをbody/ログへ出さない。
|
if (
|
||||||
const fixture = await bridge("expired-session", {}) as {
|
url.pathname === "/__e2e/primary-cookie-present" &&
|
||||||
set_cookies: string[];
|
request.method === "GET"
|
||||||
};
|
) {
|
||||||
const headers = new Headers({ "cache-control": "no-store" });
|
// HTTPで受け取ったprimaryの有無だけを返す。値、marker、DB状態は公開しない。
|
||||||
for (const cookie of fixture.set_cookies) {
|
const present = request.headers.get("cookie")?.split(";").some((pair) =>
|
||||||
headers.append("set-cookie", cookie);
|
pair.trim().split("=", 1)[0] === "yoi_workspace_session"
|
||||||
|
) ?? false;
|
||||||
|
return Response.json(present, {
|
||||||
|
headers: { "cache-control": "no-store" },
|
||||||
|
});
|
||||||
}
|
}
|
||||||
return new Response(null, { status: 204, headers });
|
return undefined;
|
||||||
}
|
},
|
||||||
if (
|
);
|
||||||
url.pathname === "/__e2e/primary-cookie-present" && request.method === "GET"
|
|
||||||
) {
|
|
||||||
// HTTPで受け取ったprimaryの有無だけを返す。値、marker、DB状態は公開しない。
|
|
||||||
const present = request.headers.get("cookie")?.split(";").some((pair) =>
|
|
||||||
pair.trim().split("=", 1)[0] === "yoi_workspace_session"
|
|
||||||
) ?? false;
|
|
||||||
return Response.json(present, { headers: { "cache-control": "no-store" } });
|
|
||||||
}
|
|
||||||
if (url.pathname.startsWith("/api/")) {
|
|
||||||
const headers = new Headers(request.headers);
|
|
||||||
headers.delete("host");
|
|
||||||
headers.delete("connection");
|
|
||||||
// OriginとCookieを保持し、Set-Cookieを含む実応答をそのまま返す。
|
|
||||||
return await fetch(new URL(url.pathname + url.search, backend), {
|
|
||||||
method: request.method,
|
|
||||||
headers,
|
|
||||||
body: ["GET", "HEAD"].includes(request.method)
|
|
||||||
? undefined
|
|
||||||
: await request.arrayBuffer(),
|
|
||||||
redirect: "manual",
|
|
||||||
});
|
|
||||||
}
|
|
||||||
const relative = normalize(url.pathname.replace(/^\/+/, "") || "index.html");
|
|
||||||
if (relative.startsWith("..")) {
|
|
||||||
return new Response("not found", { status: 404 });
|
|
||||||
}
|
|
||||||
let file = join(build, relative);
|
|
||||||
try {
|
|
||||||
if ((await Deno.stat(file)).isDirectory) file = join(file, "index.html");
|
|
||||||
return new Response(await Deno.readFile(file), {
|
|
||||||
headers: {
|
|
||||||
"content-type": mime[extname(file)] ?? "application/octet-stream",
|
|
||||||
},
|
|
||||||
});
|
|
||||||
} catch (error) {
|
|
||||||
if (!(error instanceof Deno.errors.NotFound)) throw error;
|
|
||||||
if (url.pathname.startsWith("/_app/")) {
|
|
||||||
return new Response("not found", { status: 404 });
|
|
||||||
}
|
|
||||||
return new Response(await Deno.readFile(join(build, "index.html")), {
|
|
||||||
headers: { "content-type": mime[".html"] },
|
|
||||||
});
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
// 逐次ceremony/fixtureだけを扱う短いNDJSON bridge。署名/cookieをartifactへ記録しない。
|
// 逐次ceremony/fixtureだけを扱う短いNDJSON bridge。署名/cookieをartifactへ記録しない。
|
||||||
const input = Deno.stdin.readable.pipeThrough(new TextDecoderStream())
|
const input = Deno.stdin.readable.pipeThrough(new TextDecoderStream())
|
||||||
@@ -313,7 +274,8 @@ try {
|
|||||||
|
|
||||||
// 14日実時間の前進ではない。固定expiry実store recordとMax-Age=2のHTTP cookieで、
|
// 14日実時間の前進ではない。固定expiry実store recordとMax-Age=2のHTTP cookieで、
|
||||||
// 自然消失後のmarker→実whoami→製品loader/router→実署名loginの接続を検証する。
|
// 自然消失後のmarker→実whoami→製品loader/router→実署名loginの接続を検証する。
|
||||||
const expiredTarget = "/dashboard?view=workspaces&filter=a%2Fb&ratio=100%25#selected%25";
|
const expiredTarget =
|
||||||
|
"/dashboard?view=workspaces&filter=a%2Fb&ratio=100%25#selected%25";
|
||||||
const catalogResponse = () =>
|
const catalogResponse = () =>
|
||||||
page.waitForResponse((response) =>
|
page.waitForResponse((response) =>
|
||||||
new URL(response.url()).pathname === "/api/workspaces" &&
|
new URL(response.url()).pathname === "/api/workspaces" &&
|
||||||
@@ -360,6 +322,7 @@ try {
|
|||||||
assert(expiredCookieNames.includes(marker.name));
|
assert(expiredCookieNames.includes(marker.name));
|
||||||
assertEquals(await whoami(), {
|
assertEquals(await whoami(), {
|
||||||
actor: null,
|
actor: null,
|
||||||
|
authentication_reference: null,
|
||||||
session: { kind: "expired" },
|
session: { kind: "expired" },
|
||||||
});
|
});
|
||||||
privateRequests.length = 0;
|
privateRequests.length = 0;
|
||||||
@@ -383,7 +346,11 @@ try {
|
|||||||
[],
|
[],
|
||||||
"expired reload must gate the real private catalog loader",
|
"expired reload must gate the real private catalog loader",
|
||||||
);
|
);
|
||||||
assertEquals(await whoami(), { actor: null, session: { kind: "expired" } });
|
assertEquals(await whoami(), {
|
||||||
|
actor: null,
|
||||||
|
authentication_reference: null,
|
||||||
|
session: { kind: "expired" },
|
||||||
|
});
|
||||||
await page.reload();
|
await page.reload();
|
||||||
await at(expiredLogin);
|
await at(expiredLogin);
|
||||||
await page.getByRole("button", { name: "Log in with passkey", exact: true })
|
await page.getByRole("button", { name: "Log in with passkey", exact: true })
|
||||||
@@ -416,7 +383,11 @@ try {
|
|||||||
// authenticatedにもexpiredにもならない。これを14日経過の成功とは数えない。
|
// authenticatedにもexpiredにもならない。これを14日経過の成功とは数えない。
|
||||||
await context.clearCookies({ name: session.name });
|
await context.clearCookies({ name: session.name });
|
||||||
const partial = await whoami();
|
const partial = await whoami();
|
||||||
assertEquals(partial, { actor: null, session: { kind: "anonymous" } });
|
assertEquals(partial, {
|
||||||
|
actor: null,
|
||||||
|
authentication_reference: null,
|
||||||
|
session: { kind: "anonymous" },
|
||||||
|
});
|
||||||
await page.reload();
|
await page.reload();
|
||||||
await at("/");
|
await at("/");
|
||||||
await context.clearCookies();
|
await context.clearCookies();
|
||||||
|
|||||||
@@ -0,0 +1,611 @@
|
|||||||
|
/// <reference lib="dom" />
|
||||||
|
// 実Backend + 本番static shell + Chromium native WebAuthn/CTAP2の旧鍵移行matrix。
|
||||||
|
// navigator.credentials、HTTP応答、暗号検証を置換しない。CDPは合成認証器の
|
||||||
|
// 能力/本人確認/タッチだけを制御し、credentialのexport/import/deleteはしない。
|
||||||
|
// 旧鍵fixtureだけは実optionsのuser.idとresident指定を変更してnative createする。
|
||||||
|
// 登録attestationはuser handleを署名しないため、本番verifier/storeで保存できる。
|
||||||
|
// 現行製品の登録動作の再現ではなく、旧random handle保存状態の明示的な合成である。
|
||||||
|
// 実スマホ/OSのprovider選択/物理鍵は対象外。mobileはChromiumのviewportのみ。
|
||||||
|
import { join } from "@std/path";
|
||||||
|
import { type Browser, type CDPSession, chromium, type Page } from "playwright";
|
||||||
|
import { serveProductionShell } from "./static_shell.ts";
|
||||||
|
|
||||||
|
const [backend, origin, build, handlePrefix, artifacts] = Deno.args;
|
||||||
|
if (!backend || !origin || !build || !handlePrefix || !artifacts) {
|
||||||
|
throw new Error(
|
||||||
|
"Expected owned Backend, origin, production build, synthetic handle prefix, artifacts",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const started = performance.now();
|
||||||
|
const server = await serveProductionShell(backend, origin, build);
|
||||||
|
let browser: Browser | undefined;
|
||||||
|
let phase = "launch";
|
||||||
|
const rows: {
|
||||||
|
resident: boolean;
|
||||||
|
authenticator: string;
|
||||||
|
outcome: string;
|
||||||
|
elapsed_ms: number;
|
||||||
|
}[] = [];
|
||||||
|
class NativeCheckError extends Error {}
|
||||||
|
function check(condition: unknown, diagnostic: string): asserts condition {
|
||||||
|
// 値をdumpするassertEqualsは使わない。例外/HTTP本文/credential/cookieをログに残さない。
|
||||||
|
if (!condition) throw new NativeCheckError(diagnostic);
|
||||||
|
}
|
||||||
|
type User = { user_id: string; account_id: string; handle: string };
|
||||||
|
type Key = {
|
||||||
|
key_reference: string;
|
||||||
|
normal_login_verified_at: string | null;
|
||||||
|
last_used_at: string | null;
|
||||||
|
};
|
||||||
|
async function whoami(page: Page) {
|
||||||
|
const response = await page.request.get(`${origin}/api/auth/whoami`);
|
||||||
|
check(response.status() === 200, "whoami status");
|
||||||
|
return await response.json() as {
|
||||||
|
actor: User | null;
|
||||||
|
authentication_reference: string | null;
|
||||||
|
session: { kind: string };
|
||||||
|
};
|
||||||
|
}
|
||||||
|
async function owner(page: Page, user: User, reference?: string) {
|
||||||
|
const current = await whoami(page);
|
||||||
|
check(
|
||||||
|
current.session.kind === "authenticated" &&
|
||||||
|
current.actor?.user_id === user.user_id &&
|
||||||
|
current.actor.account_id === user.account_id &&
|
||||||
|
current.actor.handle === user.handle,
|
||||||
|
"authenticated owner mismatch",
|
||||||
|
);
|
||||||
|
check(
|
||||||
|
typeof current.authentication_reference === "string",
|
||||||
|
"missing authentication reference",
|
||||||
|
);
|
||||||
|
if (reference) {
|
||||||
|
check(
|
||||||
|
current.authentication_reference === reference,
|
||||||
|
"issued authentication mismatch",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return current;
|
||||||
|
}
|
||||||
|
async function keys(page: Page, user: User): Promise<Key[]> {
|
||||||
|
const response = await page.request.get(`${origin}/api/auth/passkeys`);
|
||||||
|
check(response.status() === 200, "key list status");
|
||||||
|
const result = await response.json();
|
||||||
|
check(
|
||||||
|
result.user.user_id === user.user_id &&
|
||||||
|
result.user.account_id === user.account_id,
|
||||||
|
"key list owner",
|
||||||
|
);
|
||||||
|
return result.keys;
|
||||||
|
}
|
||||||
|
async function presence(
|
||||||
|
cdp: CDPSession,
|
||||||
|
authenticatorId: string,
|
||||||
|
enabled: boolean,
|
||||||
|
) {
|
||||||
|
await cdp.send("WebAuthn.setAutomaticPresenceSimulation", {
|
||||||
|
authenticatorId,
|
||||||
|
enabled,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
async function authenticator(
|
||||||
|
cdp: CDPSession,
|
||||||
|
resident: boolean,
|
||||||
|
automaticPresenceSimulation = true,
|
||||||
|
) {
|
||||||
|
return (await cdp.send("WebAuthn.addVirtualAuthenticator", {
|
||||||
|
options: {
|
||||||
|
protocol: "ctap2",
|
||||||
|
transport: "usb",
|
||||||
|
hasResidentKey: resident,
|
||||||
|
hasUserVerification: true,
|
||||||
|
isUserVerified: true,
|
||||||
|
automaticPresenceSimulation,
|
||||||
|
},
|
||||||
|
})).authenticatorId;
|
||||||
|
}
|
||||||
|
async function account(page: Page) {
|
||||||
|
await page.goto(origin + "/account");
|
||||||
|
await page.getByRole("heading", { name: "Account", exact: true }).waitFor();
|
||||||
|
await page.getByRole("button", { name: "Log out", exact: true }).waitFor();
|
||||||
|
}
|
||||||
|
async function logout(page: Page) {
|
||||||
|
await account(page);
|
||||||
|
await page.getByRole("button", { name: "Log out", exact: true }).click();
|
||||||
|
await page.waitForURL((url) => url.origin === origin && url.pathname === "/");
|
||||||
|
check(
|
||||||
|
(await whoami(page)).session.kind === "anonymous",
|
||||||
|
"logout must clear authentication",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
async function login(page: Page, user: User, legacy: boolean) {
|
||||||
|
await page.goto(origin + "/login?returnTo=%2Faccount");
|
||||||
|
const button = page.getByRole("button", {
|
||||||
|
name: legacy ? "Log in with older key" : "Log in with passkey",
|
||||||
|
exact: true,
|
||||||
|
});
|
||||||
|
if (legacy) {
|
||||||
|
await page.getByText("Use an older key with account ID", { exact: true })
|
||||||
|
.click();
|
||||||
|
await page.getByLabel("Account ID (handle)", { exact: true }).fill(
|
||||||
|
user.handle,
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
// 互換入口は閉じたまま。通常loginのrequestにhandle/user/accountが入らないことも検査する。
|
||||||
|
check(
|
||||||
|
!await page.getByLabel("Account ID (handle)", { exact: true })
|
||||||
|
.isVisible(),
|
||||||
|
"normal login asks for ID",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const options = page.waitForResponse((r) =>
|
||||||
|
new URL(r.url()).pathname === "/api/auth/passkeys/login/options"
|
||||||
|
);
|
||||||
|
const complete = page.waitForResponse((r) =>
|
||||||
|
new URL(r.url()).pathname === "/api/auth/passkeys/login/complete"
|
||||||
|
);
|
||||||
|
await button.click();
|
||||||
|
const optionResponse = await options;
|
||||||
|
check(optionResponse.status() === 200, "login options status");
|
||||||
|
const input = optionResponse.request().postDataJSON();
|
||||||
|
if (!legacy) {
|
||||||
|
check(
|
||||||
|
Object.keys(input).every((key) => key === "browser_origin"),
|
||||||
|
"normal login selected an account",
|
||||||
|
);
|
||||||
|
const wire = await optionResponse.json();
|
||||||
|
check(
|
||||||
|
!wire.public_key.publicKey.allowCredentials?.length,
|
||||||
|
"normal login sends an allow-list",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const response = await complete;
|
||||||
|
check(response.status() === 200, "native login complete status");
|
||||||
|
const issued = await response.json();
|
||||||
|
check(
|
||||||
|
issued.user.user_id === user.user_id &&
|
||||||
|
issued.user.account_id === user.account_id,
|
||||||
|
"login returned wrong owner",
|
||||||
|
);
|
||||||
|
await page.waitForURL((url) => url.pathname === "/account");
|
||||||
|
await owner(page, user, issued.authentication_reference);
|
||||||
|
}
|
||||||
|
|
||||||
|
const terminate = () => {
|
||||||
|
void (async () => {
|
||||||
|
await browser?.close();
|
||||||
|
await server.shutdown();
|
||||||
|
Deno.exit(143);
|
||||||
|
})();
|
||||||
|
};
|
||||||
|
Deno.addSignalListener("SIGTERM", terminate);
|
||||||
|
try {
|
||||||
|
browser = await chromium.launch({ headless: true, timeout: 15_000 });
|
||||||
|
for (const resident of [false, true]) {
|
||||||
|
const context = await browser.newContext({
|
||||||
|
viewport: { width: 1280, height: 900 },
|
||||||
|
serviceWorkers: "block",
|
||||||
|
});
|
||||||
|
context.setDefaultTimeout(12_000);
|
||||||
|
// 製品もfixtureも外部通信しない。loopback proxy以外へのbrowser requestは拒否する。
|
||||||
|
await context.route(
|
||||||
|
"**/*",
|
||||||
|
(route) =>
|
||||||
|
new URL(route.request().url()).origin === origin
|
||||||
|
? route.continue()
|
||||||
|
: route.abort(),
|
||||||
|
);
|
||||||
|
let page = await context.newPage();
|
||||||
|
let pageErrors = 0;
|
||||||
|
page.on("pageerror", () => ++pageErrors);
|
||||||
|
let cdp = await context.newCDPSession(page);
|
||||||
|
await cdp.send("WebAuthn.enable", { enableUI: false });
|
||||||
|
// 両行とも認証器はresident key対応にする。residentは旧credentialの作成指定だけを変え、
|
||||||
|
// resident必須の追加が能力不足で拒否されるケースとduplicate exclusionを混同しない。
|
||||||
|
const old = await authenticator(cdp, true);
|
||||||
|
let completionPosts = 0;
|
||||||
|
const completions = () => completionPosts;
|
||||||
|
page.on("request", (r) => {
|
||||||
|
if (
|
||||||
|
new URL(r.url()).pathname === "/api/auth/passkeys/addition/complete"
|
||||||
|
) ++completionPosts;
|
||||||
|
});
|
||||||
|
try {
|
||||||
|
phase = `legacy-fixture-resident-${resident}`;
|
||||||
|
await page.goto(origin + "/register");
|
||||||
|
// 生options/credentialはpage内にとどめる。user.id/residency以外はServerの要求を維持する。
|
||||||
|
const fixture = await page.evaluate(async ({ handle, resident }) => {
|
||||||
|
const post = async (path: string, body: unknown) => {
|
||||||
|
const response = await fetch(path, {
|
||||||
|
method: "POST",
|
||||||
|
headers: { "content-type": "application/json" },
|
||||||
|
body: JSON.stringify(body),
|
||||||
|
});
|
||||||
|
if (!response.ok) {
|
||||||
|
throw new Error("synthetic legacy fixture HTTP rejected");
|
||||||
|
}
|
||||||
|
return await response.json();
|
||||||
|
};
|
||||||
|
const wire = await post("/api/auth/passkeys/registration/options", {
|
||||||
|
handle,
|
||||||
|
display_name: "Synthetic native migration",
|
||||||
|
browser_origin: location.origin,
|
||||||
|
});
|
||||||
|
const bytes = (value: string) =>
|
||||||
|
Uint8Array.from(
|
||||||
|
atob(value.replaceAll("-", "+").replaceAll("_", "/")),
|
||||||
|
(c) => c.charCodeAt(0),
|
||||||
|
);
|
||||||
|
const options = wire.public_key.publicKey;
|
||||||
|
const random = crypto.getRandomValues(new Uint8Array(32));
|
||||||
|
const stable = bytes(options.user.id);
|
||||||
|
const differs = stable.length !== random.length ||
|
||||||
|
stable.some((byte: number, index: number) => byte !== random[index]);
|
||||||
|
options.challenge = bytes(options.challenge);
|
||||||
|
options.user.id = random;
|
||||||
|
options.excludeCredentials = options.excludeCredentials?.map((
|
||||||
|
item: { id: string },
|
||||||
|
) => ({ ...item, id: bytes(item.id) }));
|
||||||
|
options.authenticatorSelection = {
|
||||||
|
...options.authenticatorSelection,
|
||||||
|
residentKey: resident ? "required" : "discouraged",
|
||||||
|
requireResidentKey: resident,
|
||||||
|
};
|
||||||
|
const credential = await navigator.credentials.create({
|
||||||
|
publicKey: options,
|
||||||
|
}) as PublicKeyCredential | null;
|
||||||
|
if (
|
||||||
|
!credential ||
|
||||||
|
!(credential.response instanceof AuthenticatorAttestationResponse)
|
||||||
|
) throw new Error("native attestation missing");
|
||||||
|
const encode = (value: ArrayBuffer) =>
|
||||||
|
btoa(String.fromCharCode(...new Uint8Array(value))).replaceAll(
|
||||||
|
"+",
|
||||||
|
"-",
|
||||||
|
).replaceAll("/", "_").replaceAll("=", "");
|
||||||
|
const issued = await post("/api/auth/passkeys/registration/complete", {
|
||||||
|
challenge_id: wire.challenge_id,
|
||||||
|
credential: {
|
||||||
|
id: credential.id,
|
||||||
|
rawId: encode(credential.rawId),
|
||||||
|
type: credential.type,
|
||||||
|
response: {
|
||||||
|
attestationObject: encode(credential.response.attestationObject),
|
||||||
|
clientDataJSON: encode(credential.response.clientDataJSON),
|
||||||
|
transports: credential.response.getTransports(),
|
||||||
|
},
|
||||||
|
extensions: credential.getClientExtensionResults(),
|
||||||
|
},
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
user: issued.user,
|
||||||
|
authentication_reference: issued.authentication_reference,
|
||||||
|
randomDiffersFromStable: differs,
|
||||||
|
};
|
||||||
|
}, {
|
||||||
|
handle: `${handlePrefix}-${resident ? "resident" : "nonresident"}`,
|
||||||
|
resident,
|
||||||
|
});
|
||||||
|
check(
|
||||||
|
fixture.randomDiffersFromStable,
|
||||||
|
"fixture must use a legacy random handle",
|
||||||
|
);
|
||||||
|
const user: User = fixture.user;
|
||||||
|
await owner(page, user, fixture.authentication_reference);
|
||||||
|
const initial = await keys(page, user);
|
||||||
|
check(
|
||||||
|
initial.length === 1 && initial[0].normal_login_verified_at === null,
|
||||||
|
"legacy key initially unverified",
|
||||||
|
);
|
||||||
|
const oldReference = initial[0].key_reference;
|
||||||
|
await logout(page);
|
||||||
|
phase = `older-key-login-before-resident-${resident}`;
|
||||||
|
await login(page, user, true);
|
||||||
|
|
||||||
|
phase = `same-authenticator-excluded-resident-${resident}`;
|
||||||
|
const sameStarted = performance.now();
|
||||||
|
const add = () =>
|
||||||
|
page.getByRole("button", {
|
||||||
|
name: `Add passkey to ${user.handle}`,
|
||||||
|
exact: true,
|
||||||
|
});
|
||||||
|
const optionsResponse = page.waitForResponse((r) =>
|
||||||
|
new URL(r.url()).pathname === "/api/auth/passkeys/addition/options"
|
||||||
|
);
|
||||||
|
await add().click();
|
||||||
|
check(
|
||||||
|
(await optionsResponse).status() === 200,
|
||||||
|
"addition options status",
|
||||||
|
);
|
||||||
|
const excluded = page.getByRole("alert").filter({
|
||||||
|
hasText: "[authenticator/excluded]",
|
||||||
|
});
|
||||||
|
await excluded.waitFor({ state: "visible" });
|
||||||
|
check(
|
||||||
|
(await excluded.innerText()).includes(
|
||||||
|
"different device, security key, or passkey provider",
|
||||||
|
),
|
||||||
|
"exclusion needs a concrete alternative",
|
||||||
|
);
|
||||||
|
check(
|
||||||
|
(await excluded.innerText()).includes("do not delete"),
|
||||||
|
"exclusion must preserve working access",
|
||||||
|
);
|
||||||
|
check(
|
||||||
|
completions() === 0,
|
||||||
|
"native exclusion must not submit completion",
|
||||||
|
);
|
||||||
|
const afterExcluded = await keys(page, user);
|
||||||
|
check(
|
||||||
|
afterExcluded.length === 1 &&
|
||||||
|
afterExcluded[0].key_reference === oldReference,
|
||||||
|
"exclusion changed old key",
|
||||||
|
);
|
||||||
|
await owner(page, user);
|
||||||
|
rows.push({
|
||||||
|
resident,
|
||||||
|
authenticator: "same",
|
||||||
|
outcome: "native-exclusion-no-completion-old-key-preserved",
|
||||||
|
elapsed_ms: Math.round(performance.now() - sameStarted),
|
||||||
|
});
|
||||||
|
|
||||||
|
phase = `other-authenticator-pending-resident-${resident}`;
|
||||||
|
const otherStarted = performance.now();
|
||||||
|
// CDPの仮想認証器選択は物理providerの選択UIを再現しない。別targetに別認証器を置き、
|
||||||
|
// 旧targetと旧鍵は生存させる。共有contextの既存sessionを使い、cookie/鍵の移送はしない。
|
||||||
|
const legacyPage = page;
|
||||||
|
const legacyCdp = cdp;
|
||||||
|
page = await context.newPage();
|
||||||
|
page.on("pageerror", () => ++pageErrors);
|
||||||
|
page.on("request", (r) => {
|
||||||
|
if (
|
||||||
|
new URL(r.url()).pathname === "/api/auth/passkeys/addition/complete"
|
||||||
|
) ++completionPosts;
|
||||||
|
});
|
||||||
|
cdp = await context.newCDPSession(page);
|
||||||
|
await cdp.send("WebAuthn.enable", { enableUI: false });
|
||||||
|
const newer = await authenticator(cdp, true, false);
|
||||||
|
await account(page);
|
||||||
|
// options応答後、native操作はCTAPのpresence=falseで止める。timeout/sleepでpromptを推測しない。
|
||||||
|
const beforeAddition = await owner(page, user);
|
||||||
|
const additionOptions = page.waitForResponse((r) =>
|
||||||
|
new URL(r.url()).pathname === "/api/auth/passkeys/addition/options"
|
||||||
|
);
|
||||||
|
const additionComplete = page.waitForResponse((r) =>
|
||||||
|
new URL(r.url()).pathname === "/api/auth/passkeys/addition/complete"
|
||||||
|
);
|
||||||
|
await add().click();
|
||||||
|
const optionResponse = await additionOptions;
|
||||||
|
check(
|
||||||
|
optionResponse.status() === 200,
|
||||||
|
"other authenticator options status",
|
||||||
|
);
|
||||||
|
let optionsInput = optionResponse.request().postDataJSON();
|
||||||
|
check(
|
||||||
|
optionsInput.expected_user_id === user.user_id &&
|
||||||
|
optionsInput.expected_account_id === user.account_id &&
|
||||||
|
optionsInput.expected_authentication_reference ===
|
||||||
|
beforeAddition.authentication_reference,
|
||||||
|
"addition must fence displayed owner and session",
|
||||||
|
);
|
||||||
|
let wire = await optionResponse.json();
|
||||||
|
check(
|
||||||
|
wire.public_key.publicKey.excludeCredentials.length === 1,
|
||||||
|
"old credential must remain excluded",
|
||||||
|
);
|
||||||
|
check(
|
||||||
|
wire.public_key.publicKey.authenticatorSelection.userVerification ===
|
||||||
|
"required",
|
||||||
|
"addition must require UV",
|
||||||
|
);
|
||||||
|
check(
|
||||||
|
wire.public_key.publicKey.authenticatorSelection.residentKey ===
|
||||||
|
"required",
|
||||||
|
"product addition must require residency",
|
||||||
|
);
|
||||||
|
const pending = page.getByRole("status").filter({
|
||||||
|
hasText: "Waiting for passkey addition",
|
||||||
|
});
|
||||||
|
await pending.waitFor({ state: "visible" });
|
||||||
|
check(
|
||||||
|
await page.getByRole("heading", { name: "Account", exact: true })
|
||||||
|
.isVisible(),
|
||||||
|
"account hidden while native pending",
|
||||||
|
);
|
||||||
|
check(
|
||||||
|
await add().isDisabled() && completions() === 0,
|
||||||
|
"pending must prevent duplicate/early completion",
|
||||||
|
);
|
||||||
|
for (
|
||||||
|
const viewport of [{ width: 1280, height: 900 }, {
|
||||||
|
width: 390,
|
||||||
|
height: 844,
|
||||||
|
}]
|
||||||
|
) {
|
||||||
|
await page.setViewportSize(viewport);
|
||||||
|
await pending.scrollIntoViewIfNeeded();
|
||||||
|
check(
|
||||||
|
await pending.isVisible(),
|
||||||
|
"pending must remain visible at viewport",
|
||||||
|
);
|
||||||
|
await page.screenshot({
|
||||||
|
path: join(
|
||||||
|
artifacts,
|
||||||
|
`native-addition-pending-${resident}-${viewport.width}.png`,
|
||||||
|
),
|
||||||
|
fullPage: false,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
await page.setViewportSize({ width: 1280, height: 900 });
|
||||||
|
// presence=falseで既に待機中のCTAP要求を後からtrueにして解放できるとは仮定しない。
|
||||||
|
// pending/取消と成功を分ける。送信前の停止を確認し、別ceremonyを明示開始する。
|
||||||
|
await page.getByRole("button", { name: "Stop waiting", exact: true })
|
||||||
|
.click();
|
||||||
|
await page.getByRole("alert").filter({
|
||||||
|
hasText: "Completion was not sent.",
|
||||||
|
}).waitFor();
|
||||||
|
check(
|
||||||
|
completions() === 0 && (await keys(page, user)).length === 1,
|
||||||
|
"cancel before touch must not save or complete",
|
||||||
|
);
|
||||||
|
await presence(cdp, newer, true);
|
||||||
|
const freshOptions = page.waitForResponse((r) =>
|
||||||
|
new URL(r.url()).pathname === "/api/auth/passkeys/addition/options"
|
||||||
|
);
|
||||||
|
await add().click();
|
||||||
|
const freshResponse = await freshOptions;
|
||||||
|
check(freshResponse.status() === 200, "fresh addition options status");
|
||||||
|
optionsInput = freshResponse.request().postDataJSON();
|
||||||
|
wire = await freshResponse.json();
|
||||||
|
phase = `other-authenticator-completion-resident-${resident}`;
|
||||||
|
const complete = await additionComplete;
|
||||||
|
check(complete.status() === 200, "native addition complete status");
|
||||||
|
const issued = await complete.json();
|
||||||
|
check(
|
||||||
|
issued.user.user_id === user.user_id &&
|
||||||
|
issued.user.account_id === user.account_id &&
|
||||||
|
typeof issued.authentication_reference === "string",
|
||||||
|
"addition complete owner/reference",
|
||||||
|
);
|
||||||
|
await page.getByRole("status").filter({
|
||||||
|
hasText: "Passkey added to your account",
|
||||||
|
}).waitFor({ state: "visible" });
|
||||||
|
await owner(page, user, issued.authentication_reference);
|
||||||
|
check(completions() === 1, "addition must submit exactly once");
|
||||||
|
// 元のbinding referenceはreceipt照合用。Cookie発行後も同じ現在ownerから確認する。
|
||||||
|
const receipt = await page.request.post(
|
||||||
|
`${origin}/api/auth/passkeys/addition/result`,
|
||||||
|
{
|
||||||
|
headers: { Origin: origin },
|
||||||
|
data: {
|
||||||
|
challenge_id: wire.challenge_id,
|
||||||
|
expected_user_id: user.user_id,
|
||||||
|
expected_account_id: user.account_id,
|
||||||
|
expected_authentication_reference:
|
||||||
|
optionsInput.expected_authentication_reference,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
check(
|
||||||
|
receipt.status() === 200,
|
||||||
|
"saved addition receipt after Cookie issuance",
|
||||||
|
);
|
||||||
|
const saved = await receipt.json();
|
||||||
|
check(
|
||||||
|
saved.status === "saved" && saved.user.user_id === user.user_id &&
|
||||||
|
saved.user.account_id === user.account_id,
|
||||||
|
"receipt must identify saved owner",
|
||||||
|
);
|
||||||
|
const added = await keys(page, user);
|
||||||
|
check(
|
||||||
|
added.length === 2 &&
|
||||||
|
added.some((key) => key.key_reference === oldReference),
|
||||||
|
"addition must retain older key",
|
||||||
|
);
|
||||||
|
const newKey = added.find((key) => key.key_reference !== oldReference);
|
||||||
|
check(
|
||||||
|
newKey && saved.key.key_reference === newKey.key_reference &&
|
||||||
|
newKey.normal_login_verified_at === null,
|
||||||
|
"storage must not assert normal login",
|
||||||
|
);
|
||||||
|
await logout(page);
|
||||||
|
phase = `handle-free-login-resident-${resident}`;
|
||||||
|
await login(page, user, false);
|
||||||
|
const verified = await keys(page, user);
|
||||||
|
check(
|
||||||
|
verified.length === 2 &&
|
||||||
|
verified.find((key) => key.key_reference === newKey.key_reference)
|
||||||
|
?.normal_login_verified_at != null,
|
||||||
|
"actual handle-free verifier success must mark the new key",
|
||||||
|
);
|
||||||
|
check(
|
||||||
|
verified.find((key) => key.key_reference === oldReference)
|
||||||
|
?.normal_login_verified_at === null,
|
||||||
|
"normal login must not mark the unused legacy key",
|
||||||
|
);
|
||||||
|
await logout(page);
|
||||||
|
phase = `older-key-login-after-resident-${resident}`;
|
||||||
|
await presence(cdp, newer, false);
|
||||||
|
page = legacyPage;
|
||||||
|
cdp = legacyCdp;
|
||||||
|
await presence(cdp, old, true);
|
||||||
|
await login(page, user, true);
|
||||||
|
const retained = await keys(page, user);
|
||||||
|
check(
|
||||||
|
retained.length === 2 &&
|
||||||
|
retained.find((key) => key.key_reference === oldReference)
|
||||||
|
?.last_used_at != null,
|
||||||
|
"older key must remain usable after migration",
|
||||||
|
);
|
||||||
|
check(
|
||||||
|
retained.find((key) => key.key_reference === newKey.key_reference)
|
||||||
|
?.normal_login_verified_at != null,
|
||||||
|
"older login must preserve new key verification",
|
||||||
|
);
|
||||||
|
check(pageErrors === 0, "product page emitted an uncaught error");
|
||||||
|
rows.push({
|
||||||
|
resident,
|
||||||
|
authenticator: "other",
|
||||||
|
outcome:
|
||||||
|
"saved-correct-owner-handle-free-verified-older-login-preserved",
|
||||||
|
elapsed_ms: Math.round(performance.now() - otherStarted),
|
||||||
|
});
|
||||||
|
} finally {
|
||||||
|
await context.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await Deno.writeTextFile(
|
||||||
|
join(artifacts, "native-migration-result.json"),
|
||||||
|
JSON.stringify(
|
||||||
|
{
|
||||||
|
status: "passed",
|
||||||
|
boundary:
|
||||||
|
"real Backend/static shell/native Chromium CTAP2; synthetic legacy creation override only",
|
||||||
|
chromium: browser.version(),
|
||||||
|
rows,
|
||||||
|
elapsed_ms: Math.round(performance.now() - started),
|
||||||
|
viewports: ["1280x900", "390x844"],
|
||||||
|
physical_smartphone: "unconfirmed",
|
||||||
|
physical_authenticator: "unconfirmed",
|
||||||
|
},
|
||||||
|
null,
|
||||||
|
2,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
console.error(
|
||||||
|
`native-migration: PASS ${rows.length} matrix rows; ${
|
||||||
|
Math.round(performance.now() - started)
|
||||||
|
}ms`,
|
||||||
|
);
|
||||||
|
} catch (error) {
|
||||||
|
const diagnostic = error instanceof NativeCheckError
|
||||||
|
? error.message
|
||||||
|
: "browser-or-tool-operation-failed";
|
||||||
|
await Deno.writeTextFile(
|
||||||
|
join(artifacts, "native-migration-result.json"),
|
||||||
|
JSON.stringify(
|
||||||
|
{
|
||||||
|
status: "failed",
|
||||||
|
phase,
|
||||||
|
diagnostic,
|
||||||
|
rows,
|
||||||
|
elapsed_ms: Math.round(performance.now() - started),
|
||||||
|
physical_smartphone: "unconfirmed",
|
||||||
|
physical_authenticator: "unconfirmed",
|
||||||
|
},
|
||||||
|
null,
|
||||||
|
2,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
// Playwrightの例外にはrequest/引数を含み得るため本文やstackを保存しない。
|
||||||
|
console.error(
|
||||||
|
`native-migration: FAIL phase=${phase}; completed_rows=${rows.length}; elapsed_ms=${
|
||||||
|
Math.round(performance.now() - started)
|
||||||
|
}`,
|
||||||
|
);
|
||||||
|
Deno.exitCode = 1;
|
||||||
|
} finally {
|
||||||
|
Deno.removeSignalListener("SIGTERM", terminate);
|
||||||
|
await browser?.close();
|
||||||
|
await server.shutdown();
|
||||||
|
}
|
||||||
@@ -0,0 +1,76 @@
|
|||||||
|
// 実Backendへの透過proxyと本番SPA配信をbrowser suite間で共有する。
|
||||||
|
// Cookie/Origin/署名検証/認可は置換しない。fixture入口は呼出側が明示的に渡す。
|
||||||
|
import { extname, join, normalize } from "@std/path";
|
||||||
|
|
||||||
|
export async function serveProductionShell(
|
||||||
|
backend: string,
|
||||||
|
origin: string,
|
||||||
|
build: string,
|
||||||
|
fixture?: (request: Request) => Promise<Response | undefined>,
|
||||||
|
) {
|
||||||
|
for (const address of [backend, origin]) {
|
||||||
|
const url = new URL(address);
|
||||||
|
if (
|
||||||
|
url.protocol !== "http:" ||
|
||||||
|
!["localhost", "127.0.0.1"].includes(url.hostname) || !url.port
|
||||||
|
) {
|
||||||
|
throw new Error("Browser E2E requires explicit owned loopback endpoints");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await Deno.stat(join(build, "index.html")); // 暗黙buildや別asset rootへのfallbackはしない。
|
||||||
|
const mime: Record<string, string> = {
|
||||||
|
".html": "text/html; charset=utf-8",
|
||||||
|
".js": "text/javascript",
|
||||||
|
".css": "text/css",
|
||||||
|
".json": "application/json",
|
||||||
|
".svg": "image/svg+xml",
|
||||||
|
".png": "image/png",
|
||||||
|
".woff2": "font/woff2",
|
||||||
|
".ico": "image/x-icon",
|
||||||
|
};
|
||||||
|
return Deno.serve({
|
||||||
|
hostname: "127.0.0.1",
|
||||||
|
port: Number(new URL(origin).port),
|
||||||
|
onListen() {},
|
||||||
|
}, async (request) => {
|
||||||
|
const url = new URL(request.url);
|
||||||
|
const special = await fixture?.(request);
|
||||||
|
if (special) return special;
|
||||||
|
if (url.pathname.startsWith("/api/")) {
|
||||||
|
const headers = new Headers(request.headers);
|
||||||
|
headers.delete("host");
|
||||||
|
headers.delete("connection");
|
||||||
|
return await fetch(new URL(url.pathname + url.search, backend), {
|
||||||
|
method: request.method,
|
||||||
|
headers,
|
||||||
|
body: ["GET", "HEAD"].includes(request.method)
|
||||||
|
? undefined
|
||||||
|
: await request.arrayBuffer(),
|
||||||
|
redirect: "manual",
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const relative = normalize(
|
||||||
|
url.pathname.replace(/^\/+/, "") || "index.html",
|
||||||
|
);
|
||||||
|
if (relative.startsWith("..")) {
|
||||||
|
return new Response("not found", { status: 404 });
|
||||||
|
}
|
||||||
|
let file = join(build, relative);
|
||||||
|
try {
|
||||||
|
if ((await Deno.stat(file)).isDirectory) file = join(file, "index.html");
|
||||||
|
return new Response(await Deno.readFile(file), {
|
||||||
|
headers: {
|
||||||
|
"content-type": mime[extname(file)] ?? "application/octet-stream",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
} catch (error) {
|
||||||
|
if (!(error instanceof Deno.errors.NotFound)) throw error;
|
||||||
|
if (url.pathname.startsWith("/_app/")) {
|
||||||
|
return new Response("not found", { status: 404 });
|
||||||
|
}
|
||||||
|
return new Response(await Deno.readFile(join(build, "index.html")), {
|
||||||
|
headers: { "content-type": mime[".html"] },
|
||||||
|
});
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}
|
||||||
@@ -0,0 +1,52 @@
|
|||||||
|
{
|
||||||
|
"source_revision": "1639be98411921308238b3a6a6e75c784c045156",
|
||||||
|
"model": "codex-oauth/gpt-6.1-sol",
|
||||||
|
"worker_id": "01a1657d-b762-7791-87ae-a49f28cadbce",
|
||||||
|
"session_id": "01a1657d-b818-77c1-bd73-175a27562f63",
|
||||||
|
"rows": [
|
||||||
|
{
|
||||||
|
"entry_id": "01a1657f-18bf-7a03-b8b8-953d27041fe0",
|
||||||
|
"timestamp": 1792704190655,
|
||||||
|
"role": "user",
|
||||||
|
"parts": [
|
||||||
|
{
|
||||||
|
"kind": "text",
|
||||||
|
"content": "これは保存と復元を確認するための人工的な会話です。合言葉は「青い灯台」です。覚えたことを一文で返してください。Toolは使わないでください。"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entry_id": "01a1657f-1f55-71d3-b418-8199b863f59a",
|
||||||
|
"timestamp": 1792704192341,
|
||||||
|
"role": "assistant",
|
||||||
|
"parts": [
|
||||||
|
{
|
||||||
|
"kind": "text",
|
||||||
|
"text": "合言葉は「青い灯台」と覚えました。"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entry_id": "01a1657f-3053-71f3-bf4f-91290d82488b",
|
||||||
|
"timestamp": 1792704196691,
|
||||||
|
"role": "user",
|
||||||
|
"parts": [
|
||||||
|
{
|
||||||
|
"kind": "text",
|
||||||
|
"content": "先ほどの合言葉は何でしたか?合言葉だけを答えてください。"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"entry_id": "01a1657f-36ef-78b1-b0c8-b141f13b0c7b",
|
||||||
|
"timestamp": 1792704198383,
|
||||||
|
"role": "assistant",
|
||||||
|
"parts": [
|
||||||
|
{
|
||||||
|
"kind": "text",
|
||||||
|
"text": "青い灯台"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
@@ -0,0 +1,34 @@
|
|||||||
|
{
|
||||||
|
"format_version": 1,
|
||||||
|
"source_revision": "1639be98411921308238b3a6a6e75c784c045156",
|
||||||
|
"runtime_schema_version": 12,
|
||||||
|
"session_schema_version": 6,
|
||||||
|
"workspace_id": "01a16553-b901-7191-8ceb-8e2306fe3da9",
|
||||||
|
"runtime_id": "fixture-951707ee-93ca-4205-9c12-45f08a104a74",
|
||||||
|
"worker_id": "01a1657d-b762-7791-87ae-a49f28cadbce",
|
||||||
|
"session_id": "01a1657d-b818-77c1-bd73-175a27562f63",
|
||||||
|
"segment_id": "01a1657d-b818-77c1-bd73-176172a4bd82",
|
||||||
|
"model": "codex-oauth/gpt-6.1-sol",
|
||||||
|
"state": "stopped",
|
||||||
|
"acquisition": "Real isolated Codex OAuth two-turn artificial conversation; typed offline StoppedWorkerTemplate extraction. Session bytes unchanged; environment credentials and execution receipts excluded.",
|
||||||
|
"source_file_sha256": {
|
||||||
|
"worker.json": "912e620013a17f05c1aaf900da48c9ee0ecb1f54121efc3721e03f8755eb5eca",
|
||||||
|
"metadata.json": "657928a89ec621bd092b86924201ed7b562e9683261789ce18bdb1d248bdc450",
|
||||||
|
"session/session.json": "22b01daab68a28b9777a1f7abb9ef9f7f6cbae113089446602f5d15421e7c7f3",
|
||||||
|
"session/segments/01a1657d-b818-77c1-bd73-176172a4bd82.jsonl": "7c5663bb06d9406d5bd0a5086159b31c0d7799458fcb1c6f81e18b63c49f4df5"
|
||||||
|
},
|
||||||
|
"files": {
|
||||||
|
"worker-template.json": {
|
||||||
|
"bytes": 48343,
|
||||||
|
"sha256": "8bf6ce3c71c4f05c82314de93c1e8e51b7502b00915b6adb8b33046944e463b2"
|
||||||
|
},
|
||||||
|
"expected.json": {
|
||||||
|
"bytes": 1474,
|
||||||
|
"sha256": "b9846db47ecb8f3f53b2abd9d5f0c766122f86015a7a7df7addf8beab3e9c55a"
|
||||||
|
},
|
||||||
|
"workspace-config.dcdl": {
|
||||||
|
"bytes": 1141,
|
||||||
|
"sha256": "9b07507b71733081f077e2e9968a7f1f676c916a243bb355d1813f067a55c415"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,22 @@
|
|||||||
|
{
|
||||||
|
profile = {
|
||||||
|
default_profile = "project:fixture-chat";
|
||||||
|
entries = [{
|
||||||
|
selector = "project:fixture-chat";
|
||||||
|
label = "Isolated Codex fixture conversation";
|
||||||
|
description = "Artificial conversation only; no Workdir, Tools, Subject or background work.";
|
||||||
|
profile = (import "$builtin/profiles/job.dcdl") // {
|
||||||
|
slug = "fixture-chat";
|
||||||
|
worker = { mode = "tools"; };
|
||||||
|
model = { ref = "codex-oauth/gpt-6.1-sol"; };
|
||||||
|
scope = "workspace_read";
|
||||||
|
delegation_scope = "workspace_read";
|
||||||
|
permissions = { default_action = "deny"; rule = []; };
|
||||||
|
engine = { instruction = "default"; reasoning = "low"; max_tokens = 2048; max_turns = 4; language = "Japanese"; };
|
||||||
|
session = { record_event_trace = false; };
|
||||||
|
feature = { workdir_catalog = { enabled = false; }; };
|
||||||
|
};
|
||||||
|
}];
|
||||||
|
};
|
||||||
|
prompts = { default_prompt = "これは隔離環境の人工的な会話テストです。日本語で簡潔に応答してください。Toolや外部操作は行わず、ユーザーが会話中に提示した情報だけを使ってください。"; };
|
||||||
|
} as WorkspaceConfigSchema
|
||||||
@@ -0,0 +1,336 @@
|
|||||||
|
//! Bounded byte-preserving transport, independent of the owning store's schema validation.
|
||||||
|
use std::collections::{BTreeMap, BTreeSet};
|
||||||
|
use std::fs;
|
||||||
|
use std::io::{Read, Write};
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
|
||||||
|
use sha2::{Digest, Sha256};
|
||||||
|
|
||||||
|
use crate::{HarnessError, Result};
|
||||||
|
|
||||||
|
#[derive(Debug, Clone, Copy)]
|
||||||
|
struct Limits {
|
||||||
|
file_bytes: u64,
|
||||||
|
total_bytes: u64,
|
||||||
|
entries: usize,
|
||||||
|
depth: usize,
|
||||||
|
}
|
||||||
|
|
||||||
|
const LIMITS: Limits = Limits {
|
||||||
|
file_bytes: 8 * 1024 * 1024,
|
||||||
|
total_bytes: 32 * 1024 * 1024,
|
||||||
|
entries: 256,
|
||||||
|
depth: 8,
|
||||||
|
};
|
||||||
|
|
||||||
|
#[derive(Debug, Clone)]
|
||||||
|
pub struct FixtureSource {
|
||||||
|
root: PathBuf,
|
||||||
|
hashes: BTreeMap<PathBuf, String>,
|
||||||
|
flat_files: Option<BTreeSet<PathBuf>>,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn reject(message: &'static str) -> HarnessError {
|
||||||
|
HarnessError::Protocol(message.into())
|
||||||
|
}
|
||||||
|
|
||||||
|
impl FixtureSource {
|
||||||
|
/// Capture a bounded tree without generating or repairing store data.
|
||||||
|
pub fn capture(root: &Path) -> Result<Self> {
|
||||||
|
Self::capture_with(root, None)
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Validate the complete flat file set before reading any content. The same
|
||||||
|
/// allowlist is retained for copy and later source-integrity checks.
|
||||||
|
pub fn capture_flat(root: &Path, names: &[&str]) -> Result<Self> {
|
||||||
|
let files: BTreeSet<_> = names.iter().map(PathBuf::from).collect();
|
||||||
|
if files.len() != names.len()
|
||||||
|
|| files.iter().any(|name| {
|
||||||
|
name.components().count() != 1
|
||||||
|
|| !matches!(
|
||||||
|
name.components().next(),
|
||||||
|
Some(std::path::Component::Normal(_))
|
||||||
|
)
|
||||||
|
})
|
||||||
|
{
|
||||||
|
return Err(reject("invalid fixture file allowlist"));
|
||||||
|
}
|
||||||
|
Self::capture_with(root, Some(files))
|
||||||
|
}
|
||||||
|
|
||||||
|
fn capture_with(root: &Path, flat_files: Option<BTreeSet<PathBuf>>) -> Result<Self> {
|
||||||
|
let hashes = inventory(root, flat_files.as_ref(), LIMITS)?;
|
||||||
|
if hashes.is_empty() {
|
||||||
|
return Err(reject("provided fixture is empty"));
|
||||||
|
}
|
||||||
|
Ok(Self {
|
||||||
|
root: root.to_owned(),
|
||||||
|
hashes,
|
||||||
|
flat_files,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Read a known asset through a descriptor with a real byte limit, including
|
||||||
|
/// when its size grows after metadata inspection. Errors never include paths.
|
||||||
|
pub fn read_bounded(path: &Path) -> Result<Vec<u8>> {
|
||||||
|
read_regular(path, LIMITS.file_bytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn hashes(&self) -> &BTreeMap<PathBuf, String> {
|
||||||
|
&self.hashes
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn assert_unchanged(&self) -> Result<()> {
|
||||||
|
if inventory(&self.root, self.flat_files.as_ref(), LIMITS)? != self.hashes {
|
||||||
|
return Err(reject("provided fixture changed during execution"));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
/// The destination must not exist. A copy is never a link to the source.
|
||||||
|
pub fn copy_to(&self, destination: &Path) -> Result<()> {
|
||||||
|
self.assert_unchanged()?;
|
||||||
|
let parent = destination
|
||||||
|
.parent()
|
||||||
|
.filter(|path| !path.as_os_str().is_empty())
|
||||||
|
.unwrap_or_else(|| Path::new("."));
|
||||||
|
if parent
|
||||||
|
.canonicalize()?
|
||||||
|
.starts_with(self.root.canonicalize()?)
|
||||||
|
{
|
||||||
|
return Err(reject("fixture copy destination is inside the original"));
|
||||||
|
}
|
||||||
|
fs::create_dir(destination)?;
|
||||||
|
let mut remaining = LIMITS.total_bytes;
|
||||||
|
for (relative, expected) in &self.hashes {
|
||||||
|
let bytes = read_regular(&self.root.join(relative), LIMITS.file_bytes.min(remaining))?;
|
||||||
|
remaining -= bytes.len() as u64;
|
||||||
|
if hash(&bytes) != *expected {
|
||||||
|
return Err(reject("fixture changed while copying"));
|
||||||
|
}
|
||||||
|
let target = destination.join(relative);
|
||||||
|
fs::create_dir_all(target.parent().unwrap())?;
|
||||||
|
let mut file = fs::OpenOptions::new()
|
||||||
|
.write(true)
|
||||||
|
.create_new(true)
|
||||||
|
.open(target)?;
|
||||||
|
file.write_all(&bytes)?;
|
||||||
|
}
|
||||||
|
if inventory(destination, self.flat_files.as_ref(), LIMITS)? != self.hashes {
|
||||||
|
return Err(reject("fixture copy failed hash verification"));
|
||||||
|
}
|
||||||
|
self.assert_unchanged()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn hash(bytes: &[u8]) -> String {
|
||||||
|
format!(
|
||||||
|
"sha256:{}",
|
||||||
|
Sha256::digest(bytes)
|
||||||
|
.iter()
|
||||||
|
.map(|byte| format!("{byte:02x}"))
|
||||||
|
.collect::<String>()
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_limited(reader: impl Read, limit: u64) -> Result<Vec<u8>> {
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
reader
|
||||||
|
.take(limit + 1)
|
||||||
|
.read_to_end(&mut bytes)
|
||||||
|
.map_err(|_| reject("fixture content read failed"))?;
|
||||||
|
if bytes.len() as u64 > limit {
|
||||||
|
return Err(reject("fixture byte budget exceeded"));
|
||||||
|
}
|
||||||
|
Ok(bytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_regular(path: &Path, limit: u64) -> Result<Vec<u8>> {
|
||||||
|
let mut options = fs::OpenOptions::new();
|
||||||
|
options.read(true);
|
||||||
|
#[cfg(unix)]
|
||||||
|
{
|
||||||
|
use std::os::unix::fs::OpenOptionsExt;
|
||||||
|
// Do not follow a replaced final symlink or block on a replaced FIFO.
|
||||||
|
options.custom_flags(libc::O_NOFOLLOW | libc::O_NONBLOCK);
|
||||||
|
}
|
||||||
|
let file = options
|
||||||
|
.open(path)
|
||||||
|
.map_err(|_| reject("fixture asset open failed"))?;
|
||||||
|
let metadata = file
|
||||||
|
.metadata()
|
||||||
|
.map_err(|_| reject("fixture asset metadata failed"))?;
|
||||||
|
if !metadata.is_file() || metadata.len() > limit {
|
||||||
|
return Err(reject("fixture asset is not a bounded regular file"));
|
||||||
|
}
|
||||||
|
read_limited(file, limit)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn inventory(
|
||||||
|
root: &Path,
|
||||||
|
flat: Option<&BTreeSet<PathBuf>>,
|
||||||
|
limits: Limits,
|
||||||
|
) -> Result<BTreeMap<PathBuf, String>> {
|
||||||
|
struct Budget {
|
||||||
|
entries: usize,
|
||||||
|
bytes: u64,
|
||||||
|
}
|
||||||
|
fn visit(
|
||||||
|
root: &Path,
|
||||||
|
path: &Path,
|
||||||
|
depth: usize,
|
||||||
|
limits: Limits,
|
||||||
|
flat: Option<&BTreeSet<PathBuf>>,
|
||||||
|
budget: &mut Budget,
|
||||||
|
files: &mut BTreeSet<PathBuf>,
|
||||||
|
) -> Result<()> {
|
||||||
|
if depth > limits.depth || budget.entries == 0 {
|
||||||
|
return Err(reject("fixture tree budget exceeded"));
|
||||||
|
}
|
||||||
|
budget.entries -= 1;
|
||||||
|
let metadata = fs::symlink_metadata(path).map_err(|_| reject("fixture metadata failed"))?;
|
||||||
|
let relative = path.strip_prefix(root).unwrap();
|
||||||
|
if depth != 0 && flat.is_some_and(|names| !names.contains(relative)) {
|
||||||
|
return Err(reject("unexpected fixture file set"));
|
||||||
|
}
|
||||||
|
if metadata.is_dir() && (depth == 0 || flat.is_none()) {
|
||||||
|
for entry in fs::read_dir(path).map_err(|_| reject("fixture directory read failed"))? {
|
||||||
|
let entry = entry.map_err(|_| reject("fixture directory entry failed"))?;
|
||||||
|
visit(root, &entry.path(), depth + 1, limits, flat, budget, files)?;
|
||||||
|
}
|
||||||
|
} else if metadata.is_file() && depth != 0 {
|
||||||
|
if metadata.len() > limits.file_bytes || metadata.len() > budget.bytes {
|
||||||
|
return Err(reject("fixture byte budget exceeded"));
|
||||||
|
}
|
||||||
|
budget.bytes -= metadata.len();
|
||||||
|
files.insert(relative.to_owned());
|
||||||
|
} else {
|
||||||
|
return Err(reject("fixture contains a symlink or special file"));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
// Preflight the entire shape and metadata budget before reading any body.
|
||||||
|
let mut files = BTreeSet::new();
|
||||||
|
visit(
|
||||||
|
root,
|
||||||
|
root,
|
||||||
|
0,
|
||||||
|
limits,
|
||||||
|
flat,
|
||||||
|
&mut Budget {
|
||||||
|
entries: limits.entries,
|
||||||
|
bytes: limits.total_bytes,
|
||||||
|
},
|
||||||
|
&mut files,
|
||||||
|
)?;
|
||||||
|
if flat.is_some_and(|expected| expected != &files) {
|
||||||
|
return Err(reject("unexpected fixture file set"));
|
||||||
|
}
|
||||||
|
let mut remaining = limits.total_bytes;
|
||||||
|
let mut hashes = BTreeMap::new();
|
||||||
|
for relative in files {
|
||||||
|
let bytes = read_regular(&root.join(&relative), limits.file_bytes.min(remaining))?;
|
||||||
|
remaining -= bytes.len() as u64;
|
||||||
|
hashes.insert(relative, hash(&bytes));
|
||||||
|
}
|
||||||
|
Ok(hashes)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
// Transport-only synthetic bytes, never a substitute saved Worker fixture.
|
||||||
|
#[test]
|
||||||
|
fn copies_are_independent_and_source_mutation_is_detected() -> Result<()> {
|
||||||
|
let temp = tempfile::tempdir()?;
|
||||||
|
let source = temp.path().join("source");
|
||||||
|
fs::create_dir(&source)?;
|
||||||
|
fs::write(source.join("bytes"), b"original")?;
|
||||||
|
let fixture = FixtureSource::capture_flat(&source, &["bytes"])?;
|
||||||
|
assert!(fixture.copy_to(&source.join("nested-copy")).is_err());
|
||||||
|
assert!(!source.join("nested-copy").exists());
|
||||||
|
fixture.assert_unchanged()?;
|
||||||
|
let first = temp.path().join("first");
|
||||||
|
let second = temp.path().join("second");
|
||||||
|
fixture.copy_to(&first)?;
|
||||||
|
fixture.copy_to(&second)?;
|
||||||
|
fs::write(first.join("bytes"), b"changed")?;
|
||||||
|
fs::remove_dir_all(first)?;
|
||||||
|
assert_eq!(fs::read(second.join("bytes"))?, b"original");
|
||||||
|
fixture.assert_unchanged()?;
|
||||||
|
fs::write(source.join("bytes"), b"changed")?;
|
||||||
|
assert!(fixture.assert_unchanged().is_err());
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn rejects_missing_empty_existing_destination_and_extra_file() -> Result<()> {
|
||||||
|
let temp = tempfile::tempdir()?;
|
||||||
|
assert!(FixtureSource::capture(&temp.path().join("missing")).is_err());
|
||||||
|
assert!(FixtureSource::capture(temp.path()).is_err());
|
||||||
|
fs::write(temp.path().join("bytes"), b"original")?;
|
||||||
|
let source = FixtureSource::capture(temp.path())?;
|
||||||
|
assert!(source.copy_to(temp.path()).is_err());
|
||||||
|
fs::write(temp.path().join("extra"), b"new")?;
|
||||||
|
assert!(source.assert_unchanged().is_err());
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn flat_allowlist_rejects_extra_entries_before_reading_content() -> Result<()> {
|
||||||
|
let temp = tempfile::tempdir()?;
|
||||||
|
fs::write(temp.path().join("asset"), b"ok")?;
|
||||||
|
// Sparse oversized input is never read. Even an allowed file that cannot
|
||||||
|
// be decoded does not take precedence over an unexpected file set.
|
||||||
|
fs::File::create(temp.path().join("extra"))?.set_len(LIMITS.file_bytes + 1)?;
|
||||||
|
let error = FixtureSource::capture_flat(temp.path(), &["asset"]).unwrap_err();
|
||||||
|
assert!(error.to_string().contains("unexpected fixture file set"));
|
||||||
|
fs::remove_file(temp.path().join("extra"))?;
|
||||||
|
assert!(FixtureSource::capture_flat(temp.path(), &["missing"]).is_err());
|
||||||
|
fs::create_dir(temp.path().join("dir"))?;
|
||||||
|
assert!(FixtureSource::capture_flat(temp.path(), &["asset", "dir"]).is_err());
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn tree_count_depth_total_and_actual_read_budgets_are_enforced() -> Result<()> {
|
||||||
|
let temp = tempfile::tempdir()?;
|
||||||
|
fs::write(temp.path().join("a"), b"1234")?;
|
||||||
|
fs::write(temp.path().join("b"), b"1234")?;
|
||||||
|
for limits in [
|
||||||
|
Limits {
|
||||||
|
entries: 2,
|
||||||
|
..LIMITS
|
||||||
|
},
|
||||||
|
Limits {
|
||||||
|
total_bytes: 7,
|
||||||
|
..LIMITS
|
||||||
|
},
|
||||||
|
Limits {
|
||||||
|
file_bytes: 3,
|
||||||
|
..LIMITS
|
||||||
|
},
|
||||||
|
] {
|
||||||
|
assert!(inventory(temp.path(), None, limits).is_err());
|
||||||
|
}
|
||||||
|
fs::create_dir_all(temp.path().join("nested/deep"))?;
|
||||||
|
assert!(inventory(temp.path(), None, Limits { depth: 1, ..LIMITS }).is_err());
|
||||||
|
// A reader can return more bytes than earlier metadata reported. Bound
|
||||||
|
// actual IO rather than trusting that observation or an EOF at stat size.
|
||||||
|
assert!(read_limited(std::io::Cursor::new(b"12345"), 4).is_err());
|
||||||
|
assert_eq!(read_limited(std::io::Cursor::new(b"1234"), 4)?, b"1234");
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(unix)]
|
||||||
|
#[test]
|
||||||
|
fn rejects_symbolic_links() -> Result<()> {
|
||||||
|
let temp = tempfile::tempdir()?;
|
||||||
|
fs::write(temp.path().join("bytes"), b"original")?;
|
||||||
|
std::os::unix::fs::symlink("bytes", temp.path().join("link"))?;
|
||||||
|
assert!(FixtureSource::capture(temp.path()).is_err());
|
||||||
|
assert!(FixtureSource::read_bounded(&temp.path().join("link")).is_err());
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
}
|
||||||
+35
-5
@@ -137,7 +137,9 @@ fn tested_yoi_env_policy_overview() -> TestedYoiEnvPolicy {
|
|||||||
#[derive(Debug)]
|
#[derive(Debug)]
|
||||||
pub enum HarnessError {
|
pub enum HarnessError {
|
||||||
Io(io::Error),
|
Io(io::Error),
|
||||||
Json(serde_json::Error),
|
// Never retain decode errors: serde messages can contain rejected field names
|
||||||
|
// or values from private fixture/protocol payloads, including through Debug.
|
||||||
|
Json,
|
||||||
CommandFailed {
|
CommandFailed {
|
||||||
program: PathBuf,
|
program: PathBuf,
|
||||||
args: Vec<String>,
|
args: Vec<String>,
|
||||||
@@ -163,7 +165,7 @@ impl std::fmt::Display for HarnessError {
|
|||||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||||
match self {
|
match self {
|
||||||
Self::Io(err) => write!(f, "io error: {err}"),
|
Self::Io(err) => write!(f, "io error: {err}"),
|
||||||
Self::Json(err) => write!(f, "json error: {err}"),
|
Self::Json => write!(f, "invalid JSON document"),
|
||||||
Self::CommandFailed {
|
Self::CommandFailed {
|
||||||
program,
|
program,
|
||||||
args,
|
args,
|
||||||
@@ -209,8 +211,8 @@ impl From<io::Error> for HarnessError {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl From<serde_json::Error> for HarnessError {
|
impl From<serde_json::Error> for HarnessError {
|
||||||
fn from(value: serde_json::Error) -> Self {
|
fn from(_: serde_json::Error) -> Self {
|
||||||
Self::Json(value)
|
Self::Json
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1268,7 +1270,11 @@ impl Drop for PanelHarness {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
mod fixture_source;
|
||||||
|
pub use fixture_source::FixtureSource;
|
||||||
mod worker_environment;
|
mod worker_environment;
|
||||||
|
mod worker_session;
|
||||||
|
pub use worker_session::{WorkerSessionSubscription, validate_full_snapshot};
|
||||||
|
|
||||||
pub use worker_environment::{
|
pub use worker_environment::{
|
||||||
WorkerBinaryProviderInfo, WorkerE2eEnvironment, WorkerEnvironmentCleanupReport,
|
WorkerBinaryProviderInfo, WorkerE2eEnvironment, WorkerEnvironmentCleanupReport,
|
||||||
@@ -1627,6 +1633,8 @@ fn resolve_yoi_binary() -> Result<BinaryProviderInfo> {
|
|||||||
"e2e-test".to_owned(),
|
"e2e-test".to_owned(),
|
||||||
"--bin".to_owned(),
|
"--bin".to_owned(),
|
||||||
"yoi".to_owned(),
|
"yoi".to_owned(),
|
||||||
|
"--target-dir".to_owned(),
|
||||||
|
tui_product_target_dir()?.to_string_lossy().into_owned(),
|
||||||
];
|
];
|
||||||
if test_profile() == "release" {
|
if test_profile() == "release" {
|
||||||
args.push("--release".to_owned());
|
args.push("--release".to_owned());
|
||||||
@@ -1648,7 +1656,9 @@ fn resolve_yoi_binary() -> Result<BinaryProviderInfo> {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
let binary = current_target_profile_dir()?.join(binary_name());
|
let binary = tui_product_target_dir()?
|
||||||
|
.join(test_profile())
|
||||||
|
.join(binary_name());
|
||||||
let info = BinaryProviderInfo {
|
let info = BinaryProviderInfo {
|
||||||
provider: "cargo-build".to_owned(),
|
provider: "cargo-build".to_owned(),
|
||||||
binary,
|
binary,
|
||||||
@@ -1675,6 +1685,26 @@ fn workspace_root() -> Result<PathBuf> {
|
|||||||
.ok_or_else(|| HarnessError::Protocol("could not resolve workspace root".to_owned()))
|
.ok_or_else(|| HarnessError::Protocol("could not resolve workspace root".to_owned()))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Product processes must never reuse the target used by test-support builds.
|
||||||
|
fn product_target_dir() -> Result<PathBuf> {
|
||||||
|
dedicated_product_target("e2e-products")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Existing PTY tests need their observer feature, never on restore binaries.
|
||||||
|
fn tui_product_target_dir() -> Result<PathBuf> {
|
||||||
|
dedicated_product_target("e2e-tui-products")
|
||||||
|
}
|
||||||
|
|
||||||
|
fn dedicated_product_target(name: &str) -> Result<PathBuf> {
|
||||||
|
let target = workspace_root()?.join("target").join(name);
|
||||||
|
if current_target_profile_dir()?.parent() == Some(target.as_path()) {
|
||||||
|
return Err(HarnessError::Protocol(
|
||||||
|
"run support tests outside reserved product binary targets".to_owned(),
|
||||||
|
));
|
||||||
|
}
|
||||||
|
Ok(target)
|
||||||
|
}
|
||||||
|
|
||||||
fn current_target_profile_dir() -> Result<PathBuf> {
|
fn current_target_profile_dir() -> Result<PathBuf> {
|
||||||
let mut path = std::env::current_exe()?;
|
let mut path = std::env::current_exe()?;
|
||||||
while let Some(name) = path.file_name().and_then(|name| name.to_str()) {
|
while let Some(name) = path.file_name().and_then(|name| name.to_str()) {
|
||||||
|
|||||||
+1224
-140
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,198 @@
|
|||||||
|
//! A single bounded response-loss injection in the shared product harness.
|
||||||
|
use std::io::{Read, Write};
|
||||||
|
use std::net::{IpAddr, TcpListener, TcpStream};
|
||||||
|
use std::time::Instant;
|
||||||
|
|
||||||
|
use reqwest::blocking::{Client, Request};
|
||||||
|
use serde_json::Value;
|
||||||
|
|
||||||
|
use super::{EXIT_WAIT, HTTP_WAIT_STEP, STARTUP_WAIT};
|
||||||
|
use crate::{HarnessError, Result};
|
||||||
|
|
||||||
|
const MAX_HEADER: usize = 16 * 1024;
|
||||||
|
const MAX_MESSAGE: usize = 2 * 1024 * 1024;
|
||||||
|
|
||||||
|
fn failure(message: &str) -> HarnessError {
|
||||||
|
HarnessError::Protocol(message.into())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn drop_response(client: &Client, mut request: Request) -> Result<Value> {
|
||||||
|
let host: IpAddr = request
|
||||||
|
.url()
|
||||||
|
.host_str()
|
||||||
|
.ok_or_else(|| failure("missing relay host"))?
|
||||||
|
.parse()
|
||||||
|
.map_err(|_| failure("relay target must be a literal loopback address"))?;
|
||||||
|
if !host.is_loopback() || request.url().scheme() != "http" {
|
||||||
|
return Err(failure(
|
||||||
|
"response-loss relay only permits owned loopback HTTP",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let port = request
|
||||||
|
.url()
|
||||||
|
.port()
|
||||||
|
.ok_or_else(|| failure("missing relay port"))?;
|
||||||
|
let listener = TcpListener::bind((std::net::Ipv4Addr::LOCALHOST, 0))?;
|
||||||
|
listener.set_nonblocking(true)?;
|
||||||
|
let relay_address = listener.local_addr()?;
|
||||||
|
request
|
||||||
|
.url_mut()
|
||||||
|
.set_port(Some(relay_address.port()))
|
||||||
|
.map_err(|_| failure("set relay port"))?;
|
||||||
|
request
|
||||||
|
.url_mut()
|
||||||
|
.set_host(Some("127.0.0.1"))
|
||||||
|
.map_err(|_| failure("set relay host"))?;
|
||||||
|
std::thread::scope(|scope| {
|
||||||
|
let relay = scope.spawn(move || -> Result<Value> {
|
||||||
|
let deadline = Instant::now() + STARTUP_WAIT;
|
||||||
|
let mut downstream = loop {
|
||||||
|
match listener.accept() {
|
||||||
|
Ok((stream, peer)) if peer.ip().is_loopback() => break stream,
|
||||||
|
Ok(_) => return Err(failure("non-loopback relay peer")),
|
||||||
|
Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => {
|
||||||
|
if Instant::now() >= deadline {
|
||||||
|
return Err(failure("response-loss relay accept deadline exceeded"));
|
||||||
|
}
|
||||||
|
std::thread::sleep(HTTP_WAIT_STEP);
|
||||||
|
}
|
||||||
|
Err(error) => return Err(error.into()),
|
||||||
|
}
|
||||||
|
};
|
||||||
|
let request_bytes = read_message(&mut DeadlineStream {
|
||||||
|
stream: &mut downstream,
|
||||||
|
deadline,
|
||||||
|
})?;
|
||||||
|
let mut upstream = TcpStream::connect_timeout(&(host, port).into(), EXIT_WAIT)?;
|
||||||
|
upstream.set_write_timeout(Some(EXIT_WAIT))?;
|
||||||
|
// The body, request identity and authorization bytes are forwarded
|
||||||
|
// unchanged. Neither request nor response headers enter diagnostics.
|
||||||
|
upstream.write_all(&request_bytes)?;
|
||||||
|
let response = read_message(&mut DeadlineStream {
|
||||||
|
stream: &mut upstream,
|
||||||
|
deadline,
|
||||||
|
})?;
|
||||||
|
let boundary =
|
||||||
|
header_end(&response).ok_or_else(|| failure("missing upstream header"))?;
|
||||||
|
let status = std::str::from_utf8(&response[..boundary])
|
||||||
|
.ok()
|
||||||
|
.and_then(|header| header.lines().next())
|
||||||
|
.and_then(|line| line.split_whitespace().nth(1));
|
||||||
|
if status != Some("200") {
|
||||||
|
return Err(failure("restore relay upstream did not return HTTP 200"));
|
||||||
|
}
|
||||||
|
let value = serde_json::from_slice(&response[boundary..])?;
|
||||||
|
// Intentionally send zero bytes back; the caller must observe loss.
|
||||||
|
drop(downstream);
|
||||||
|
Ok(value)
|
||||||
|
});
|
||||||
|
let caller_failed = client.execute(request).is_err();
|
||||||
|
let evidence = relay
|
||||||
|
.join()
|
||||||
|
.map_err(|_| failure("response-loss relay panicked"))??;
|
||||||
|
if !caller_failed {
|
||||||
|
return Err(failure("caller unexpectedly received a response"));
|
||||||
|
}
|
||||||
|
Ok(evidence)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
struct DeadlineStream<'a> {
|
||||||
|
stream: &'a mut TcpStream,
|
||||||
|
deadline: Instant,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl Read for DeadlineStream<'_> {
|
||||||
|
fn read(&mut self, buffer: &mut [u8]) -> std::io::Result<usize> {
|
||||||
|
let remaining = self.deadline.saturating_duration_since(Instant::now());
|
||||||
|
if remaining.is_zero() {
|
||||||
|
return Err(std::io::Error::new(
|
||||||
|
std::io::ErrorKind::TimedOut,
|
||||||
|
"relay deadline exceeded",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
self.stream.set_read_timeout(Some(remaining))?;
|
||||||
|
self.stream.read(buffer)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn header_end(bytes: &[u8]) -> Option<usize> {
|
||||||
|
bytes
|
||||||
|
.windows(4)
|
||||||
|
.position(|window| window == b"\r\n\r\n")
|
||||||
|
.map(|i| i + 4)
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read_message(stream: &mut impl Read) -> Result<Vec<u8>> {
|
||||||
|
let mut bytes = Vec::new();
|
||||||
|
let mut buffer = [0; 4096];
|
||||||
|
let mut expected = None;
|
||||||
|
loop {
|
||||||
|
let count = stream.read(&mut buffer)?;
|
||||||
|
if count == 0 {
|
||||||
|
return Err(failure("relay peer closed before complete HTTP message"));
|
||||||
|
}
|
||||||
|
bytes.extend_from_slice(&buffer[..count]);
|
||||||
|
if bytes.len() > MAX_MESSAGE {
|
||||||
|
return Err(failure("relay HTTP byte budget exceeded"));
|
||||||
|
}
|
||||||
|
if expected.is_none() {
|
||||||
|
if let Some(boundary) = header_end(&bytes) {
|
||||||
|
if boundary > MAX_HEADER {
|
||||||
|
return Err(failure("relay HTTP header budget exceeded"));
|
||||||
|
}
|
||||||
|
let header = std::str::from_utf8(&bytes[..boundary])
|
||||||
|
.map_err(|_| failure("invalid relay HTTP header"))?;
|
||||||
|
let mut lengths = header
|
||||||
|
.lines()
|
||||||
|
.filter_map(|line| line.split_once(':'))
|
||||||
|
.filter(|(name, _)| name.eq_ignore_ascii_case("content-length"))
|
||||||
|
.map(|(_, value)| value.trim().parse::<usize>());
|
||||||
|
let length = lengths
|
||||||
|
.next()
|
||||||
|
.ok_or_else(|| failure("relay requires Content-Length"))?
|
||||||
|
.map_err(|_| failure("invalid relay Content-Length"))?;
|
||||||
|
if lengths.next().is_some()
|
||||||
|
|| length > MAX_MESSAGE - boundary
|
||||||
|
|| header
|
||||||
|
.lines()
|
||||||
|
.filter_map(|line| line.split_once(':'))
|
||||||
|
.any(|(name, _)| name.eq_ignore_ascii_case("transfer-encoding"))
|
||||||
|
{
|
||||||
|
return Err(failure("unsupported relay HTTP framing"));
|
||||||
|
}
|
||||||
|
expected = Some(boundary + length);
|
||||||
|
} else if bytes.len() > MAX_HEADER {
|
||||||
|
return Err(failure("relay HTTP header budget exceeded"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Some(expected) = expected {
|
||||||
|
if bytes.len() == expected {
|
||||||
|
return Ok(bytes);
|
||||||
|
}
|
||||||
|
if bytes.len() > expected {
|
||||||
|
return Err(failure("unexpected relay trailing bytes"));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn framing_rejects_incomplete_ambiguous_or_unbounded_messages() {
|
||||||
|
for bytes in [
|
||||||
|
b"HTTP/1.1 200 OK\r\n\r\n".as_slice(),
|
||||||
|
b"HTTP/1.1 200 OK\r\nContent-Length: 8\r\n\r\nshort",
|
||||||
|
b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nContent-Length: 0\r\n\r\n",
|
||||||
|
b"HTTP/1.1 200 OK\r\nContent-Length: 0\r\nTransfer-Encoding: chunked\r\n\r\n",
|
||||||
|
b"HTTP/1.1 200 OK\r\nContent-Length: 999999999\r\n\r\n",
|
||||||
|
] {
|
||||||
|
assert!(read_message(&mut &bytes[..]).is_err());
|
||||||
|
}
|
||||||
|
let valid = b"HTTP/1.1 200 OK\r\nContent-Length: 2\r\n\r\n{}";
|
||||||
|
assert_eq!(read_message(&mut &valid[..]).unwrap(), valid);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,284 @@
|
|||||||
|
//! Live public Worker protocol assertions. No Submit/Resume/model request is sent.
|
||||||
|
use std::net::TcpStream;
|
||||||
|
use std::time::{Duration, Instant};
|
||||||
|
|
||||||
|
use protocol::{Event, SessionSnapshot, WorkerState};
|
||||||
|
use tungstenite::{Message, WebSocket, client::IntoClientRequest};
|
||||||
|
|
||||||
|
use crate::{HarnessError, Result, WorkerE2eEnvironment};
|
||||||
|
|
||||||
|
const WS_WAIT: Duration = Duration::from_secs(10);
|
||||||
|
|
||||||
|
pub struct WorkerSessionSubscription {
|
||||||
|
socket: WebSocket<TcpStream>,
|
||||||
|
}
|
||||||
|
|
||||||
|
impl WorkerSessionSubscription {
|
||||||
|
pub fn connect(
|
||||||
|
environment: &WorkerE2eEnvironment,
|
||||||
|
worker_id: &str,
|
||||||
|
expected: &SessionSnapshot,
|
||||||
|
) -> Result<Self> {
|
||||||
|
let authorized = environment.worker_protocol_request(worker_id)?;
|
||||||
|
let mut url = authorized.url().clone();
|
||||||
|
url.set_scheme("ws")
|
||||||
|
.map_err(|_| error("invalid protocol URL"))?;
|
||||||
|
let mut request = url
|
||||||
|
.as_str()
|
||||||
|
.into_client_request()
|
||||||
|
.map_err(|_| error("build protocol handshake"))?;
|
||||||
|
request.headers_mut().extend(authorized.headers().clone());
|
||||||
|
// Only this environment's loopback endpoint is permitted, never a proxy.
|
||||||
|
let address = url
|
||||||
|
.socket_addrs(|| None)
|
||||||
|
.map_err(|_| error("resolve protocol address"))?
|
||||||
|
.into_iter()
|
||||||
|
.find(|address| address.ip().is_loopback())
|
||||||
|
.ok_or_else(|| error("protocol address is not loopback"))?;
|
||||||
|
let stream = TcpStream::connect_timeout(&address, WS_WAIT)?;
|
||||||
|
stream.set_read_timeout(Some(WS_WAIT))?;
|
||||||
|
stream.set_write_timeout(Some(WS_WAIT))?;
|
||||||
|
let (socket, _) =
|
||||||
|
tungstenite::client(request, stream).map_err(|_| error("protocol handshake failed"))?;
|
||||||
|
let mut subscription = Self { socket };
|
||||||
|
let deadline = Instant::now() + WS_WAIT;
|
||||||
|
loop {
|
||||||
|
match subscription.read(deadline)? {
|
||||||
|
Message::Text(text) => {
|
||||||
|
let raw: serde_json::Value = serde_json::from_str(&text)?;
|
||||||
|
validate_full_snapshot(&raw, expected)?;
|
||||||
|
// Only validated public content is retained, never request headers.
|
||||||
|
std::fs::write(
|
||||||
|
environment.artifacts_dir.join("restored-session.json"),
|
||||||
|
serde_json::to_vec_pretty(&raw)?,
|
||||||
|
)?;
|
||||||
|
return Ok(subscription);
|
||||||
|
}
|
||||||
|
Message::Ping(_) => subscription
|
||||||
|
.socket
|
||||||
|
.flush()
|
||||||
|
.map_err(|_| error("flush protocol Pong"))?,
|
||||||
|
Message::Pong(_) => {}
|
||||||
|
_ => return Err(error("protocol ended before its initial full Snapshot")),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Prove an already-open subscription remains usable after the other
|
||||||
|
/// environment is stopped and cleaned up. This does not invoke the model.
|
||||||
|
pub fn assert_alive(&mut self) -> Result<()> {
|
||||||
|
let payload = b"restore-e2e-isolation".to_vec();
|
||||||
|
self.socket
|
||||||
|
.send(Message::Ping(payload.clone().into()))
|
||||||
|
.map_err(|_| error("send protocol Ping"))?;
|
||||||
|
let deadline = Instant::now() + WS_WAIT;
|
||||||
|
loop {
|
||||||
|
match self.read(deadline)? {
|
||||||
|
Message::Pong(bytes) if bytes.as_ref() == payload.as_slice() => return Ok(()),
|
||||||
|
Message::Ping(_) => self
|
||||||
|
.socket
|
||||||
|
.flush()
|
||||||
|
.map_err(|_| error("flush protocol Pong"))?,
|
||||||
|
Message::Pong(_) => {}
|
||||||
|
_ => {
|
||||||
|
return Err(error(
|
||||||
|
"idle subscription changed or ended during isolation check",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// A public stop must terminate the connected execution, not merely update
|
||||||
|
/// a retained catalog row. Require the shutdown event or websocket close.
|
||||||
|
pub fn expect_stopped(mut self) -> Result<()> {
|
||||||
|
let deadline = Instant::now() + WS_WAIT;
|
||||||
|
loop {
|
||||||
|
match self.read(deadline)? {
|
||||||
|
Message::Close(_) => return Ok(()),
|
||||||
|
Message::Text(text) => match serde_json::from_str::<Event>(&text)? {
|
||||||
|
Event::Shutdown => return Ok(()),
|
||||||
|
Event::WorkerState { .. } => {}
|
||||||
|
_ => return Err(error("unexpected protocol event after Worker stop")),
|
||||||
|
},
|
||||||
|
Message::Ping(_) => self
|
||||||
|
.socket
|
||||||
|
.flush()
|
||||||
|
.map_err(|_| error("flush protocol Pong"))?,
|
||||||
|
Message::Pong(_) => {}
|
||||||
|
_ => return Err(error("unexpected protocol frame after Worker stop")),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn read(&mut self, deadline: Instant) -> Result<Message> {
|
||||||
|
let remaining = deadline
|
||||||
|
.checked_duration_since(Instant::now())
|
||||||
|
.filter(|duration| !duration.is_zero())
|
||||||
|
.ok_or_else(|| error("protocol deadline exceeded"))?;
|
||||||
|
self.socket.get_mut().set_read_timeout(Some(remaining))?;
|
||||||
|
self.socket
|
||||||
|
.read()
|
||||||
|
.map_err(|_| error("protocol read failed or timed out"))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fn error(message: &str) -> HarnessError {
|
||||||
|
HarnessError::Protocol(message.into())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn validate_full_snapshot(raw: &serde_json::Value, expected: &SessionSnapshot) -> Result<()> {
|
||||||
|
let event: Event = serde_json::from_value(raw.clone())?;
|
||||||
|
// Product DTOs intentionally tolerate unknown fields for compatibility. This
|
||||||
|
// fixture boundary does not: anything discarded by typed decoding must never
|
||||||
|
// be persisted to an artifact, including fields outside the Session payload.
|
||||||
|
if serde_json::to_value(&event)? != *raw {
|
||||||
|
return Err(error(
|
||||||
|
"noncanonical or unknown fields in full protocol Snapshot envelope",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
let Event::Snapshot {
|
||||||
|
session,
|
||||||
|
state,
|
||||||
|
in_flight,
|
||||||
|
internal_workers,
|
||||||
|
..
|
||||||
|
} = event
|
||||||
|
else {
|
||||||
|
return Err(error("first protocol event is not a full Snapshot"));
|
||||||
|
};
|
||||||
|
if state.state != WorkerState::Idle || !in_flight.is_empty() || !internal_workers.is_empty() {
|
||||||
|
return Err(error(
|
||||||
|
"restored Worker is not an idle, completed standalone Session",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
// Typed equality covers entry order/ID/timestamp/provenance/content; raw
|
||||||
|
// equality also refuses unknown fields that could leak private data.
|
||||||
|
if session != *expected
|
||||||
|
|| raw.pointer("/data/session") != Some(&serde_json::to_value(expected)?)
|
||||||
|
{
|
||||||
|
return Err(error(
|
||||||
|
"full Session snapshot differs from supplied expected rows",
|
||||||
|
));
|
||||||
|
}
|
||||||
|
if !session.pending_submissions.submissions.is_empty() {
|
||||||
|
return Err(error("fixture has pending submissions"));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
#[cfg(test)]
|
||||||
|
mod tests {
|
||||||
|
use super::*;
|
||||||
|
use protocol::{
|
||||||
|
Greeting, InFlightSnapshot, Segment, SessionEntryProvenance, SessionSnapshotEntry,
|
||||||
|
SessionSnapshotEntryData, WorkerStateSnapshot, WorkerStatus,
|
||||||
|
};
|
||||||
|
|
||||||
|
// Protocol-validation samples only, not a substitute saved Worker fixture.
|
||||||
|
fn sample() -> (serde_json::Value, SessionSnapshot) {
|
||||||
|
let session = SessionSnapshot {
|
||||||
|
pending_submissions: Default::default(),
|
||||||
|
entries: vec![SessionSnapshotEntry {
|
||||||
|
entry_id: "E-unit-1".into(),
|
||||||
|
timestamp: 123,
|
||||||
|
provenance: SessionEntryProvenance::HumanInput,
|
||||||
|
derived_from: vec![],
|
||||||
|
use_policy: None,
|
||||||
|
data: SessionSnapshotEntryData::UserInput {
|
||||||
|
segments: vec![Segment::text("unit input")],
|
||||||
|
},
|
||||||
|
}],
|
||||||
|
};
|
||||||
|
let event = Event::Snapshot {
|
||||||
|
session: session.clone(),
|
||||||
|
state: WorkerStateSnapshot::from(WorkerStatus::Idle),
|
||||||
|
in_flight: InFlightSnapshot::default(),
|
||||||
|
internal_workers: vec![],
|
||||||
|
greeting: Greeting {
|
||||||
|
worker_name: "unit".into(),
|
||||||
|
cwd: ".".into(),
|
||||||
|
provider: "unit".into(),
|
||||||
|
model: "unit".into(),
|
||||||
|
reasoning: None,
|
||||||
|
scope_summary: "none".into(),
|
||||||
|
tools: vec![],
|
||||||
|
context_window: 0,
|
||||||
|
context_tokens: 0,
|
||||||
|
context_usage: None,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
(serde_json::to_value(event).unwrap(), session)
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn full_snapshot_requires_exact_ids_timestamps_contents_and_no_unknown_fields() {
|
||||||
|
let (raw, expected) = sample();
|
||||||
|
validate_full_snapshot(&raw, &expected).unwrap();
|
||||||
|
for (key, value) in [
|
||||||
|
("entry_id", serde_json::json!("different")),
|
||||||
|
("timestamp", serde_json::json!(124)),
|
||||||
|
(
|
||||||
|
"hidden_reasoning",
|
||||||
|
serde_json::json!("must never be exposed"),
|
||||||
|
),
|
||||||
|
("segments", serde_json::json!([])),
|
||||||
|
] {
|
||||||
|
let mut changed = raw.clone();
|
||||||
|
changed["data"]["session"]["entries"][0][key] = value;
|
||||||
|
assert!(
|
||||||
|
validate_full_snapshot(&changed, &expected).is_err(),
|
||||||
|
"{key}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let mut missing = raw.clone();
|
||||||
|
missing["data"]["session"]["entries"] = serde_json::json!([]);
|
||||||
|
assert!(validate_full_snapshot(&missing, &expected).is_err());
|
||||||
|
let mut extra = raw;
|
||||||
|
let entry = extra["data"]["session"]["entries"][0].clone();
|
||||||
|
extra["data"]["session"]["entries"]
|
||||||
|
.as_array_mut()
|
||||||
|
.unwrap()
|
||||||
|
.push(entry);
|
||||||
|
assert!(validate_full_snapshot(&extra, &expected).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn full_snapshot_rejects_unknown_fields_outside_session_before_artifact_write() {
|
||||||
|
let (raw, expected) = sample();
|
||||||
|
for pointer in [
|
||||||
|
"",
|
||||||
|
"/data",
|
||||||
|
"/data/greeting",
|
||||||
|
"/data/state",
|
||||||
|
"/data/state/state",
|
||||||
|
] {
|
||||||
|
let mut changed = raw.clone();
|
||||||
|
changed
|
||||||
|
.pointer_mut(pointer)
|
||||||
|
.unwrap()
|
||||||
|
.as_object_mut()
|
||||||
|
.unwrap()
|
||||||
|
.insert(
|
||||||
|
"hidden_reasoning".into(),
|
||||||
|
serde_json::json!("must not reach artifact"),
|
||||||
|
);
|
||||||
|
assert!(
|
||||||
|
validate_full_snapshot(&changed, &expected).is_err(),
|
||||||
|
"{pointer}"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
let mut changed = raw;
|
||||||
|
changed["data"]["greeting"]["private_key"] = serde_json::json!("must not reach artifact");
|
||||||
|
assert!(validate_full_snapshot(&changed, &expected).is_err());
|
||||||
|
}
|
||||||
|
|
||||||
|
#[test]
|
||||||
|
fn non_snapshot_event_cannot_stand_in_for_live_snapshot() {
|
||||||
|
let (_, expected) = sample();
|
||||||
|
assert!(
|
||||||
|
validate_full_snapshot(&serde_json::to_value(Event::Shutdown).unwrap(), &expected)
|
||||||
|
.is_err()
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -4,8 +4,9 @@
|
|||||||
//! `cargo test -p yoi-e2e --features e2e --test auth_passkeys`で明示選択する。
|
//! `cargo test -p yoi-e2e --features e2e --test auth_passkeys`で明示選択する。
|
||||||
//! binary供給、HOME/XDG/DB/動的port、readiness、ログ、RAII回収は共有harnessが所有する。
|
//! binary供給、HOME/XDG/DB/動的port、readiness、ログ、RAII回収は共有harnessが所有する。
|
||||||
//! 通常ケースのHTTPはproxy/redirect/cookie storeなし、各要求にtimeoutを設ける。
|
//! 通常ケースのHTTPはproxy/redirect/cookie storeなし、各要求にtimeoutを設ける。
|
||||||
//! 実Server CLIにはstatic assets設定経路がない。追加の明示選択browser_sessionsだけが
|
//! 実Server CLIにはstatic assets設定経路がない。明示選択browser_sessionsは本番static shellを
|
||||||
//! 本番static shellをloopback proxyで実Backendへ接続し、Cookieの自然消失と復帰を検証する。
|
//! loopback proxyで実Backendへ接続し、Cookieの自然消失と復帰を検証する。
|
||||||
|
//! native_migrationは同じshellと実BackendをChromium native WebAuthn/CTAP2へ接続する。
|
||||||
//! 固定expiry sessionを公開store APIで作るfixture境界と、14日実時間経過の未検証は同moduleに記す。
|
//! 固定expiry sessionを公開store APIで作るfixture境界と、14日実時間経過の未検証は同moduleに記す。
|
||||||
|
|
||||||
use std::error::Error;
|
use std::error::Error;
|
||||||
@@ -21,8 +22,8 @@ use serde::Deserialize;
|
|||||||
use serde::de::DeserializeOwned;
|
use serde::de::DeserializeOwned;
|
||||||
use serde_json::{Value, json};
|
use serde_json::{Value, json};
|
||||||
use server_api::{
|
use server_api::{
|
||||||
ActorAuthMethod, AuthUserResponse, AuthenticatedUser, PasskeyLoginCompleteRequest,
|
ActorAuthMethod, AuthUserResponse, AuthenticatedUser, PasskeyAdditionOptionsRequest,
|
||||||
PasskeyLoginOptionsResponse, PasskeyRegistrationCompleteRequest,
|
PasskeyLoginCompleteRequest, PasskeyLoginOptionsResponse, PasskeyRegistrationCompleteRequest,
|
||||||
PasskeyRegistrationOptionsResponse, WhoamiResponse,
|
PasskeyRegistrationOptionsResponse, WhoamiResponse,
|
||||||
};
|
};
|
||||||
#[path = "../src/resident_authenticator.rs"]
|
#[path = "../src/resident_authenticator.rs"]
|
||||||
@@ -30,8 +31,12 @@ mod resident_model;
|
|||||||
use resident_model::ResidentAuthenticator;
|
use resident_model::ResidentAuthenticator;
|
||||||
use yoi_e2e::WorkerE2eEnvironment;
|
use yoi_e2e::WorkerE2eEnvironment;
|
||||||
|
|
||||||
|
#[path = "auth_passkeys/browser_process.rs"]
|
||||||
|
mod browser_process;
|
||||||
#[path = "auth_passkeys/browser_sessions.rs"]
|
#[path = "auth_passkeys/browser_sessions.rs"]
|
||||||
mod browser_sessions;
|
mod browser_sessions;
|
||||||
|
#[path = "auth_passkeys/native_migration.rs"]
|
||||||
|
mod native_migration;
|
||||||
|
|
||||||
type Result<T = ()> = std::result::Result<T, Box<dyn Error>>;
|
type Result<T = ()> = std::result::Result<T, Box<dyn Error>>;
|
||||||
const BROWSER_ORIGIN: &str = "http://localhost:5173";
|
const BROWSER_ORIGIN: &str = "http://localhost:5173";
|
||||||
@@ -270,6 +275,29 @@ fn logout(client: &Client, base: &str, artifacts: &Path, cookie: &str) -> Result
|
|||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// fixtureも現在の本人/認証bindingをwhoamiから取得する。期待値は対象を選ぶ権限ではない。
|
||||||
|
fn addition_expectation(
|
||||||
|
request: RequestBuilder,
|
||||||
|
artifacts: &Path,
|
||||||
|
) -> Result<PasskeyAdditionOptionsRequest> {
|
||||||
|
let identity: WhoamiResponse = decode(send(
|
||||||
|
request,
|
||||||
|
"read addition owner and authentication reference",
|
||||||
|
artifacts,
|
||||||
|
)?)?;
|
||||||
|
let actor = identity
|
||||||
|
.actor
|
||||||
|
.ok_or("addition fixture requires an authenticated actor")?;
|
||||||
|
Ok(PasskeyAdditionOptionsRequest {
|
||||||
|
expected_user_id: actor.user_id,
|
||||||
|
expected_account_id: actor.account_id,
|
||||||
|
expected_authentication_reference: identity
|
||||||
|
.authentication_reference
|
||||||
|
.ok_or("missing addition authentication reference")?,
|
||||||
|
browser_origin: Some(BROWSER_ORIGIN.to_owned()),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
fn host_admin_login(
|
fn host_admin_login(
|
||||||
environment: &WorkerE2eEnvironment,
|
environment: &WorkerE2eEnvironment,
|
||||||
client: &Client,
|
client: &Client,
|
||||||
@@ -291,11 +319,17 @@ fn host_admin_login(
|
|||||||
let bearer = setup["access_token"]
|
let bearer = setup["access_token"]
|
||||||
.as_str()
|
.as_str()
|
||||||
.ok_or("host setup omitted initial bearer")?;
|
.ok_or("host setup omitted initial bearer")?;
|
||||||
|
let addition_body = addition_expectation(
|
||||||
|
client
|
||||||
|
.get(format!("{base}/api/auth/whoami"))
|
||||||
|
.bearer_auth(bearer),
|
||||||
|
artifacts,
|
||||||
|
)?;
|
||||||
let options: PasskeyRegistrationOptionsResponse = decode(send(
|
let options: PasskeyRegistrationOptionsResponse = decode(send(
|
||||||
client
|
client
|
||||||
.post(format!("{base}/api/auth/passkeys/addition/options"))
|
.post(format!("{base}/api/auth/passkeys/addition/options"))
|
||||||
.bearer_auth(bearer)
|
.bearer_auth(bearer)
|
||||||
.json(&json!({"browser_origin": BROWSER_ORIGIN})),
|
.json(&addition_body),
|
||||||
"host-created administrator bearer starts actor-derived passkey addition",
|
"host-created administrator bearer starts actor-derived passkey addition",
|
||||||
artifacts,
|
artifacts,
|
||||||
)?)?;
|
)?)?;
|
||||||
@@ -348,6 +382,12 @@ fn disabled_registration_preserves_existing_login_self_addition_and_workspace_ow
|
|||||||
policy(&client, base, artifacts, false)?;
|
policy(&client, base, artifacts, false)?;
|
||||||
let mut admin_key = ResidentAuthenticator::new(true);
|
let mut admin_key = ResidentAuthenticator::new(true);
|
||||||
let (admin, admin_cookie) = host_admin_login(&environment, &client, &mut admin_key)?;
|
let (admin, admin_cookie) = host_admin_login(&environment, &client, &mut admin_key)?;
|
||||||
|
let admin_addition = addition_expectation(
|
||||||
|
client
|
||||||
|
.get(format!("{base}/api/auth/whoami"))
|
||||||
|
.header(COOKIE, &admin_cookie),
|
||||||
|
artifacts,
|
||||||
|
)?;
|
||||||
policy(&client, base, artifacts, false)?;
|
policy(&client, base, artifacts, false)?;
|
||||||
set_policy(&client, base, artifacts, &admin_cookie, true)?;
|
set_policy(&client, base, artifacts, &admin_cookie, true)?;
|
||||||
let whoami: Value = decode(send(
|
let whoami: Value = decode(send(
|
||||||
@@ -379,7 +419,7 @@ fn disabled_registration_preserves_existing_login_self_addition_and_workspace_ow
|
|||||||
client
|
client
|
||||||
.post(format!("{base}/api/auth/passkeys/addition/options"))
|
.post(format!("{base}/api/auth/passkeys/addition/options"))
|
||||||
.header(ORIGIN, BROWSER_ORIGIN)
|
.header(ORIGIN, BROWSER_ORIGIN)
|
||||||
.json(&json!({"browser_origin": BROWSER_ORIGIN})),
|
.json(&admin_addition),
|
||||||
"unauthenticated addition options",
|
"unauthenticated addition options",
|
||||||
artifacts,
|
artifacts,
|
||||||
)?,
|
)?,
|
||||||
@@ -445,12 +485,18 @@ fn disabled_registration_preserves_existing_login_self_addition_and_workspace_ow
|
|||||||
)?)?;
|
)?)?;
|
||||||
assert!(options.public_key.public_key.allow_credentials.is_empty());
|
assert!(options.public_key.public_key.allow_credentials.is_empty());
|
||||||
|
|
||||||
|
let addition_body = addition_expectation(
|
||||||
|
client
|
||||||
|
.get(format!("{base}/api/auth/whoami"))
|
||||||
|
.header(COOKIE, &cookie),
|
||||||
|
artifacts,
|
||||||
|
)?;
|
||||||
for (field, value) in [
|
for (field, value) in [
|
||||||
("handle", &other.user.handle),
|
("handle", &other.user.handle),
|
||||||
("user_id", &other.user.user_id),
|
("user_id", &other.user.user_id),
|
||||||
("account_id", &other.user.account_id),
|
("account_id", &other.user.account_id),
|
||||||
] {
|
] {
|
||||||
let mut body = json!({"browser_origin": BROWSER_ORIGIN});
|
let mut body = serde_json::to_value(&addition_body)?;
|
||||||
body[field] = json!(value);
|
body[field] = json!(value);
|
||||||
rejected(
|
rejected(
|
||||||
send(
|
send(
|
||||||
@@ -470,7 +516,7 @@ fn disabled_registration_preserves_existing_login_self_addition_and_workspace_ow
|
|||||||
let mut request = client
|
let mut request = client
|
||||||
.post(format!("{base}/api/auth/passkeys/addition/options"))
|
.post(format!("{base}/api/auth/passkeys/addition/options"))
|
||||||
.header(COOKIE, &cookie)
|
.header(COOKIE, &cookie)
|
||||||
.json(&json!({"browser_origin": BROWSER_ORIGIN}));
|
.json(&addition_body);
|
||||||
if let Some(origin) = origin {
|
if let Some(origin) = origin {
|
||||||
request = request.header(ORIGIN, origin);
|
request = request.header(ORIGIN, origin);
|
||||||
}
|
}
|
||||||
@@ -490,7 +536,7 @@ fn disabled_registration_preserves_existing_login_self_addition_and_workspace_ow
|
|||||||
.post(format!("{base}/api/auth/passkeys/addition/options"))
|
.post(format!("{base}/api/auth/passkeys/addition/options"))
|
||||||
.header(COOKIE, &cookie)
|
.header(COOKIE, &cookie)
|
||||||
.header(ORIGIN, BROWSER_ORIGIN)
|
.header(ORIGIN, BROWSER_ORIGIN)
|
||||||
.json(&json!({"browser_origin": BROWSER_ORIGIN})),
|
.json(&addition_body),
|
||||||
"authenticated self addition options",
|
"authenticated self addition options",
|
||||||
artifacts,
|
artifacts,
|
||||||
)?)?;
|
)?)?;
|
||||||
@@ -591,7 +637,7 @@ fn disabled_registration_preserves_existing_login_self_addition_and_workspace_ow
|
|||||||
.post(format!("{base}/api/auth/passkeys/addition/options"))
|
.post(format!("{base}/api/auth/passkeys/addition/options"))
|
||||||
.header(COOKIE, revoked_cookie)
|
.header(COOKIE, revoked_cookie)
|
||||||
.header(ORIGIN, BROWSER_ORIGIN)
|
.header(ORIGIN, BROWSER_ORIGIN)
|
||||||
.json(&json!({"browser_origin": BROWSER_ORIGIN})),
|
.json(&addition_body),
|
||||||
"logout prevents further self addition with the old cookie",
|
"logout prevents further self addition with the old cookie",
|
||||||
artifacts,
|
artifacts,
|
||||||
)?,
|
)?,
|
||||||
@@ -1797,7 +1843,8 @@ fn activation_codes_challenges_and_unverified_state_cannot_cross_account_or_purp
|
|||||||
client.post(format!("{base}/api/auth/device-login/approve")).header(ORIGIN, BROWSER_ORIGIN)
|
client.post(format!("{base}/api/auth/device-login/approve")).header(ORIGIN, BROWSER_ORIGIN)
|
||||||
.json(&json!({"user_code": flow["user_code"]})),
|
.json(&json!({"user_code": flow["user_code"]})),
|
||||||
client.post(format!("{base}/api/auth/passkeys/addition/options")).header(ORIGIN, BROWSER_ORIGIN)
|
client.post(format!("{base}/api/auth/passkeys/addition/options")).header(ORIGIN, BROWSER_ORIGIN)
|
||||||
.json(&json!({"browser_origin": BROWSER_ORIGIN})),
|
.json(&json!({"browser_origin": BROWSER_ORIGIN, "expected_user_id": issue.user.user_id,
|
||||||
|
"expected_account_id": issue.user.account_id, "expected_authentication_reference": "unauthenticated-fixture-not-a-credential"})),
|
||||||
];
|
];
|
||||||
for mut request in requests {
|
for mut request in requests {
|
||||||
if let Some(bearer) = bearer {
|
if let Some(bearer) = bearer {
|
||||||
@@ -1901,7 +1948,12 @@ fn activation_codes_challenges_and_unverified_state_cannot_cross_account_or_purp
|
|||||||
.post(format!("{base}/api/auth/passkeys/addition/options"))
|
.post(format!("{base}/api/auth/passkeys/addition/options"))
|
||||||
.header(COOKIE, &cookie)
|
.header(COOKIE, &cookie)
|
||||||
.header(ORIGIN, BROWSER_ORIGIN)
|
.header(ORIGIN, BROWSER_ORIGIN)
|
||||||
.json(&json!({"browser_origin": BROWSER_ORIGIN})),
|
.json(&addition_expectation(
|
||||||
|
client
|
||||||
|
.get(format!("{base}/api/auth/whoami"))
|
||||||
|
.header(COOKIE, &cookie),
|
||||||
|
artifacts,
|
||||||
|
)?),
|
||||||
"start distinct authenticated self-addition ceremony",
|
"start distinct authenticated self-addition ceremony",
|
||||||
artifacts,
|
artifacts,
|
||||||
)?)?;
|
)?)?;
|
||||||
|
|||||||
@@ -0,0 +1,58 @@
|
|||||||
|
//! browser suiteの子process回収と実行期限を共有する。診断にraw IPCを複製しない。
|
||||||
|
use super::*;
|
||||||
|
use std::process::{Child, ExitStatus};
|
||||||
|
use std::sync::mpsc;
|
||||||
|
|
||||||
|
pub(super) struct BrowserProcess(pub(super) Option<Child>);
|
||||||
|
impl BrowserProcess {
|
||||||
|
pub(super) fn child(&mut self) -> &mut Child {
|
||||||
|
self.0.as_mut().expect("browser process retained")
|
||||||
|
}
|
||||||
|
|
||||||
|
pub(super) fn wait_bounded(mut self, timeout: Duration) -> Result<ExitStatus> {
|
||||||
|
let mut child = self.0.take().expect("browser process retained");
|
||||||
|
let pid = child.id() as libc::pid_t;
|
||||||
|
let (send, receive) = mpsc::channel();
|
||||||
|
let waiter = std::thread::spawn(move || {
|
||||||
|
let _ = send.send(child.wait());
|
||||||
|
});
|
||||||
|
let result = receive.recv_timeout(timeout);
|
||||||
|
if result.is_err() {
|
||||||
|
unsafe {
|
||||||
|
libc::kill(pid, libc::SIGTERM);
|
||||||
|
}
|
||||||
|
if receive.recv_timeout(Duration::from_secs(3)).is_err() {
|
||||||
|
unsafe {
|
||||||
|
libc::kill(pid, libc::SIGKILL);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let _ = waiter.join();
|
||||||
|
Ok(result.map_err(|_| "browser E2E timed out; see sanitized browser diagnostic")??)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
impl Drop for BrowserProcess {
|
||||||
|
fn drop(&mut self) {
|
||||||
|
let Some(mut child) = self.0.take() else {
|
||||||
|
return;
|
||||||
|
};
|
||||||
|
if matches!(child.try_wait(), Ok(Some(_))) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let pid = child.id() as libc::pid_t;
|
||||||
|
// Denoのsignal handlerがPlaywright/HTTPを閉じる猶予を与える。
|
||||||
|
unsafe {
|
||||||
|
libc::kill(pid, libc::SIGTERM);
|
||||||
|
}
|
||||||
|
let (send, receive) = mpsc::channel();
|
||||||
|
let waiter = std::thread::spawn(move || {
|
||||||
|
let _ = send.send(child.wait());
|
||||||
|
});
|
||||||
|
if receive.recv_timeout(Duration::from_secs(3)).is_err() {
|
||||||
|
unsafe {
|
||||||
|
libc::kill(pid, libc::SIGKILL);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
let _ = waiter.join();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -8,10 +8,11 @@
|
|||||||
//! Deno/Playwright Chromiumと最新production buildが必要な明示選択suite:
|
//! Deno/Playwright Chromiumと最新production buildが必要な明示選択suite:
|
||||||
//! `cargo test -p yoi-e2e --features e2e --test auth_passkeys real_backend_browser -- --ignored`
|
//! `cargo test -p yoi-e2e --features e2e --test auth_passkeys real_backend_browser -- --ignored`
|
||||||
//! 通常auth_passkeysの成功/ignoreを、このbrowser接続の実行成功とは数えない。
|
//! 通常auth_passkeysの成功/ignoreを、このbrowser接続の実行成功とは数えない。
|
||||||
|
use super::browser_process::BrowserProcess;
|
||||||
use super::*;
|
use super::*;
|
||||||
use std::io::{BufRead, BufReader, Read};
|
use std::io::{BufRead, BufReader, Read};
|
||||||
use std::net::TcpListener;
|
use std::net::TcpListener;
|
||||||
use std::process::{Child, Command, Stdio};
|
use std::process::Stdio;
|
||||||
use std::sync::mpsc;
|
use std::sync::mpsc;
|
||||||
use std::time::Instant;
|
use std::time::Instant;
|
||||||
use yoi_workspace_server::auth::{
|
use yoi_workspace_server::auth::{
|
||||||
@@ -62,38 +63,6 @@ fn expired_session_fixture(environment: &WorkerE2eEnvironment) -> Result<Value>
|
|||||||
]}))
|
]}))
|
||||||
}
|
}
|
||||||
|
|
||||||
struct BrowserProcess(Option<Child>);
|
|
||||||
impl BrowserProcess {
|
|
||||||
fn child(&mut self) -> &mut Child {
|
|
||||||
self.0.as_mut().expect("browser process retained")
|
|
||||||
}
|
|
||||||
}
|
|
||||||
impl Drop for BrowserProcess {
|
|
||||||
fn drop(&mut self) {
|
|
||||||
let Some(mut child) = self.0.take() else {
|
|
||||||
return;
|
|
||||||
};
|
|
||||||
if matches!(child.try_wait(), Ok(Some(_))) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
let pid = child.id() as libc::pid_t;
|
|
||||||
// Denoのsignal handlerがPlaywright/HTTPを閉じる猶予を与える。
|
|
||||||
unsafe {
|
|
||||||
libc::kill(pid, libc::SIGTERM);
|
|
||||||
}
|
|
||||||
let (send, receive) = mpsc::channel();
|
|
||||||
let waiter = std::thread::spawn(move || {
|
|
||||||
let _ = send.send(child.wait());
|
|
||||||
});
|
|
||||||
if receive.recv_timeout(Duration::from_secs(3)).is_err() {
|
|
||||||
unsafe {
|
|
||||||
libc::kill(pid, libc::SIGKILL);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
let _ = waiter.join();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
#[ignore = "requires production Web assets, Deno and Playwright Chromium"]
|
#[ignore = "requires production Web assets, Deno and Playwright Chromium"]
|
||||||
fn real_backend_browser_cookies_preserve_identity_expiry_return_url_and_explicit_logout() -> Result
|
fn real_backend_browser_cookies_preserve_identity_expiry_return_url_and_explicit_logout() -> Result
|
||||||
@@ -132,7 +101,8 @@ fn real_backend_browser_cookies_preserve_identity_expiry_return_url_and_explicit
|
|||||||
drop(reservation);
|
drop(reservation);
|
||||||
let stderr = std::fs::File::create(environment.artifacts_dir.join("browser.stderr.log"))?;
|
let stderr = std::fs::File::create(environment.artifacts_dir.join("browser.stderr.log"))?;
|
||||||
let mut process = BrowserProcess(Some(
|
let mut process = BrowserProcess(Some(
|
||||||
Command::new("deno")
|
environment
|
||||||
|
.browser_command(Path::new("deno"))?
|
||||||
.current_dir(&root)
|
.current_dir(&root)
|
||||||
.args([
|
.args([
|
||||||
"run",
|
"run",
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user