diff --git a/Cargo.lock b/Cargo.lock index e7470de00..c0879f5e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3490,6 +3490,7 @@ dependencies = [ "serde", "serde_json", "sha2 0.11.0", + "thiserror 2.0.18", "tokio", "ts-rs", "uuid", @@ -5374,6 +5375,7 @@ dependencies = [ "toml", "unicode-width", "uuid", + "worker-runtime", ] [[package]] @@ -6416,6 +6418,7 @@ dependencies = [ "agen", "arc-swap", "async-trait", + "axum", "base64 0.22.1", "chrono", "clap", diff --git a/crates/client/src/backend_runtime.rs b/crates/client/src/backend_runtime.rs index 7b9fbd42a..00feb7b74 100644 --- a/crates/client/src/backend_runtime.rs +++ b/crates/client/src/backend_runtime.rs @@ -10,6 +10,7 @@ pub use server_api::{ WorkerLaunchProfileCandidate as BackendWorkerLaunchProfileCandidate, WorkerLaunchRuntimeOption as BackendWorkerLaunchRuntimeOption, WorkerOperationState as BackendWorkerOperationState, + WorkerRestoreRequest as BackendWorkerRestoreRequest, WorkerRestoreResponse as BackendWorkerRestoreResponse, WorkerRestoreResult as BackendWorkerRestoreResult, WorkerRestoreState as BackendWorkerRestoreState, WorkerSummary as BackendWorkerSummary, @@ -259,6 +260,15 @@ impl fmt::Display for BackendRuntimeClientError { } } +impl BackendRuntimeClientError { + /// A stale intent is not a rejected Restore operation and must not be retried + /// with a silently refreshed token. + pub fn is_restore_observation_conflict(&self) -> bool { + matches!(self, Self::ContractApi(server_api::client_support::ClientError::Public { status, error }) + if *status == reqwest::StatusCode::CONFLICT && error.error == "restore_observation_conflict") + } +} + impl std::error::Error for BackendRuntimeClientError {} impl From for BackendRuntimeClientError { @@ -502,6 +512,7 @@ pub async fn list_backend_stopped_workers( pub async fn restore_backend_worker( target: &BackendRuntimeTarget, + request: BackendWorkerRestoreRequest, ) -> Result { validate_target(target)?; let api = BackendApiClient::from_stored_token(&target.base_url)?; @@ -514,6 +525,7 @@ pub async fn restore_backend_worker( ticket_id: None, assignment_operation_id: None, }, + request, ) .await .map_err(Into::into) diff --git a/crates/client/src/lib.rs b/crates/client/src/lib.rs index 32e1c1a10..53e0392dc 100644 --- a/crates/client/src/lib.rs +++ b/crates/client/src/lib.rs @@ -26,12 +26,12 @@ pub use backend_runtime::{ BackendRuntimeListTarget, BackendRuntimeSummary, BackendRuntimeTarget, BackendWorkerImplementationSummary, BackendWorkerLaunchOptions, BackendWorkerLaunchProfileCandidate, BackendWorkerLaunchRuntimeOption, - BackendWorkerLaunchTarget, BackendWorkerOperationState, BackendWorkerRestoreResponse, - BackendWorkerRestoreResult, BackendWorkerRestoreState, BackendWorkerSummary, - BackendWorkerWorkspaceSummary, BackendWorkingDirectorySource, BackendWorkingDirectorySummary, - connect_backend_runtime, create_backend_worker, get_backend_worker_launch_options, - list_backend_stopped_workers, list_backend_workers, observe_backend_worker_session, - restore_backend_worker, + BackendWorkerLaunchTarget, BackendWorkerOperationState, BackendWorkerRestoreRequest, + BackendWorkerRestoreResponse, BackendWorkerRestoreResult, BackendWorkerRestoreState, + BackendWorkerSummary, BackendWorkerWorkspaceSummary, BackendWorkingDirectorySource, + BackendWorkingDirectorySummary, connect_backend_runtime, create_backend_worker, + get_backend_worker_launch_options, list_backend_stopped_workers, list_backend_workers, + observe_backend_worker_session, restore_backend_worker, }; pub use backend_workspace::{ BackendWorkspace, BackendWorkspaceCatalogTarget, BackendWorkspaceClientError, diff --git a/crates/fs-operation/Cargo.toml b/crates/fs-operation/Cargo.toml index 6d360b536..65f4a02ae 100644 --- a/crates/fs-operation/Cargo.toml +++ b/crates/fs-operation/Cargo.toml @@ -13,9 +13,7 @@ ignore = "0.4.25" libc = "0.2" schemars.workspace = true serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true sha2 = "0.10.9" tempfile.workspace = true thiserror.workspace = true - -[dev-dependencies] -serde_json.workspace = true diff --git a/crates/fs-operation/src/checked.rs b/crates/fs-operation/src/checked.rs new file mode 100644 index 000000000..f27b251bb --- /dev/null +++ b/crates/fs-operation/src/checked.rs @@ -0,0 +1,518 @@ +//! Identity-fenced access adapter for the existing read/edit/write engine. +//! +//! Descriptor-rooted resolution is Linux-only and symlinks/mount crossings fail +//! closed. The final compare and atomic rename are NOT a kernel CAS against an +//! arbitrary external editor: external editors must cooperate with provider +//! serialization. A hostile rename in that last syscall gap is not excluded. +use crate::{AtomicWriteMode, FsAccessPolicy}; +use std::fs::{File, Metadata}; +use std::io::{Seek, SeekFrom}; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicBool, Ordering}; + +/// Identity and state, deliberately not just a content digest. atime is omitted +/// so reading does not invalidate an observation. No host paths are encoded. +pub fn identity_validator(metadata: &Metadata) -> std::io::Result> { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let mut bytes = b"yoi-checkout-v1".to_vec(); + for value in [ + metadata.dev(), + metadata.ino(), + metadata.len(), + metadata.mode() as u64, + metadata.mtime() as u64, + metadata.mtime_nsec() as u64, + metadata.ctime() as u64, + metadata.ctime_nsec() as u64, + ] { + bytes.extend_from_slice(&value.to_le_bytes()); + } + Ok(bytes) + } + #[cfg(not(unix))] + { + let _ = metadata; + Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "identity validators unavailable", + )) + } +} + +/// One pinned target and its save parent. Retained through the shared operation. +pub struct CheckedTarget<'a> { + policy: &'a dyn FsAccessPolicy, + root: File, + relative: PathBuf, + logical: PathBuf, + file: File, + parent: Option, + validator: Vec, + effects_possible: AtomicBool, +} +impl<'a> CheckedTarget<'a> { + pub fn pin( + root: &File, + relative: &Path, + logical: &Path, + policy: &'a dyn FsAccessPolicy, + write: bool, + ) -> std::io::Result { + policy.check_cancelled()?; + if !(if write { + policy.is_writable_paths(logical, logical) + } else { + policy.is_readable_paths(logical, logical) + }) { + return Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "checkout target denied", + )); + } + let parent = if relative.as_os_str().is_empty() { + None + } else { + Some(crate::open_beneath_no_symlinks_at( + root, + relative.parent().unwrap_or(Path::new("")), + )?) + }; + let file = if let Some(parent) = &parent { + crate::open_beneath_no_symlinks_at(parent, Path::new(relative.file_name().unwrap()))? + } else { + crate::open_beneath_no_symlinks_at(root, Path::new(""))? + }; + let metadata = file.metadata()?; + if !metadata.is_file() && !metadata.is_dir() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "checkout requires regular file or directory", + )); + } + let result = Self { + policy, + root: root.try_clone()?, + relative: relative.to_path_buf(), + logical: logical.to_path_buf(), + file, + parent, + validator: identity_validator(&metadata)?, + effects_possible: AtomicBool::new(false), + }; + result.confirm()?; + Ok(result) + } + pub fn validator(&self) -> &[u8] { + &self.validator + } + pub fn metadata(&self) -> std::io::Result { + self.file.metadata() + } + pub fn effects_possible(&self) -> bool { + self.effects_possible.load(Ordering::Acquire) + } + pub fn confirm(&self) -> std::io::Result<()> { + self.policy.check_cancelled()?; + if let Some(parent) = &self.parent { + let named_parent = crate::open_beneath_no_symlinks_at( + &self.root, + self.relative.parent().unwrap_or(Path::new("")), + )?; + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let pinned_metadata = parent.metadata()?; + let named_metadata = named_parent.metadata()?; + if (pinned_metadata.dev(), pinned_metadata.ino()) + != (named_metadata.dev(), named_metadata.ino()) + { + return Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "checkout parent identity changed", + )); + } + } + } + let named = crate::open_beneath_no_symlinks_at(&self.root, &self.relative)?; + if identity_validator(&named.metadata()?)? != self.validator + || identity_validator(&self.file.metadata()?)? != self.validator + { + return Err(std::io::Error::new( + std::io::ErrorKind::WouldBlock, + "checkout observation changed", + )); + } + Ok(()) + } + /// Refresh a directory observation after Create, retaining the original + /// pinned target/root/parent handles. A replacement named directory cannot + /// supply the validator for effects performed in the original directory. + pub fn refresh_directory_observation(&mut self) -> std::io::Result<()> { + let metadata = self.file.metadata()?; + if !metadata.is_dir() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "Create target must be directory", + )); + } + self.validator = identity_validator(&metadata)?; + self.confirm() + } + + /// Adapter for no-clobber creation beneath a pinned directory. Both target + /// and destination authorization remain in the provider's normal policy. + pub fn create_access<'b>(&'b self, suffix: PathBuf, logical: PathBuf) -> CheckedCreate<'b, 'a> { + CheckedCreate { + target: self, + suffix, + logical, + } + } +} +impl FsAccessPolicy for CheckedTarget<'_> { + fn is_readable(&self, p: &Path) -> bool { + self.policy.is_readable(p) + } + fn is_writable(&self, p: &Path) -> bool { + self.policy.is_writable(p) + } + fn is_readable_paths(&self, p: &Path, r: &Path) -> bool { + p == self.logical && r == p && self.policy.is_readable_paths(p, r) + } + fn is_writable_paths(&self, p: &Path, r: &Path) -> bool { + p == self.logical && r == p && self.policy.is_writable_paths(p, r) + } + fn check_cancelled(&self) -> std::io::Result<()> { + self.policy.check_cancelled() + } + fn resolve_access_path(&self, p: &Path) -> std::io::Result { + Ok(p.to_path_buf()) + } + fn read_metadata(&self, _: &Path, _: &Path) -> std::io::Result { + self.file.metadata() + } + fn open_read_file(&self, _: &Path, _: &Path) -> std::io::Result { + self.confirm()?; + let mut file = self.file.try_clone()?; + file.seek(SeekFrom::Start(0))?; + Ok(file) + } + fn max_write_bytes(&self) -> Option { + Some( + self.policy + .max_write_bytes() + .unwrap_or(8 * 1024 * 1024) + .min(8 * 1024 * 1024), + ) + } + fn max_edit_replacements(&self) -> Option { + Some( + self.policy + .max_edit_replacements() + .unwrap_or(10_000) + .min(10_000), + ) + } + fn atomic_write_file( + &self, + _: &Path, + _: &Path, + content: &[u8], + mode: AtomicWriteMode, + ) -> std::io::Result<()> { + self.confirm()?; + let parent = self.parent.as_ref().ok_or_else(|| { + std::io::Error::new(std::io::ErrorKind::InvalidInput, "cannot save root") + })?; + crate::atomic_write_beneath_no_symlinks_at( + parent, + Path::new(self.relative.file_name().unwrap()), + content, + mode, + || { + self.confirm()?; + // From here rename/encoding/post-observation failures have an unknown outcome. + self.effects_possible.store(true, Ordering::Release); + Ok(()) + }, + ) + } +} + +pub struct CheckedCreate<'b, 'a> { + target: &'b CheckedTarget<'a>, + suffix: PathBuf, + logical: PathBuf, +} +impl FsAccessPolicy for CheckedCreate<'_, '_> { + fn is_readable(&self, p: &Path) -> bool { + self.target.policy.is_readable(p) + } + fn is_writable(&self, p: &Path) -> bool { + p == self.logical && self.target.policy.is_writable_paths(p, p) + } + fn resolve_access_path(&self, p: &Path) -> std::io::Result { + Ok(p.to_path_buf()) + } + fn check_cancelled(&self) -> std::io::Result<()> { + self.target.policy.check_cancelled() + } + fn max_write_bytes(&self) -> Option { + self.target.max_write_bytes() + } + fn read_metadata(&self, _: &Path, _: &Path) -> std::io::Result { + match crate::open_beneath_no_symlinks_at(&self.target.file, &self.suffix) { + Ok(_) => Err(std::io::Error::new( + std::io::ErrorKind::AlreadyExists, + "Create destination exists", + )), + Err(e) => Err(e), + } + } + fn atomic_write_file( + &self, + _: &Path, + _: &Path, + content: &[u8], + mode: AtomicWriteMode, + ) -> std::io::Result<()> { + self.target.confirm()?; + if mode != AtomicWriteMode::CreateNew || !self.target.file.metadata()?.is_dir() { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + "Create requires directory", + )); + } + // Creating parents must not turn a narrow destination grant into + // permission to populate ungranted intermediate directories. + let mut parent = self.target.logical.clone(); + let mut components = self.suffix.components().peekable(); + while let Some(component) = components.next() { + parent.push(component.as_os_str()); + if components.peek().is_some() + && !self.target.policy.is_writable_paths(&parent, &parent) + { + return Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "Create parent denied", + )); + } + } + // Nested mkdir may have effects before failure; never auto-retry. + self.target.effects_possible.store(true, Ordering::Release); + crate::atomic_write_beneath_no_symlinks_at( + &self.target.file, + &self.suffix, + content, + mode, + || self.target.policy.check_cancelled(), + ) + } +} + +#[cfg(all(test, target_os = "linux"))] +mod tests { + use super::*; + use crate::{FsError, FsPath, ReadRequest, WriteRequest}; + use sha2::{Digest, Sha256}; + use std::sync::atomic::AtomicUsize; + struct Access { + path: PathBuf, + counter: AtomicUsize, + trigger: usize, + replacement: bool, + } + impl FsAccessPolicy for Access { + fn is_readable(&self, _: &Path) -> bool { + true + } + fn is_writable(&self, _: &Path) -> bool { + true + } + fn check_cancelled(&self) -> std::io::Result<()> { + if self.counter.fetch_add(1, Ordering::SeqCst) + 1 == self.trigger { + if self.replacement { + std::fs::rename(&self.path, self.path.with_extension("old"))?; + std::fs::write(&self.path, "replacement")?; + } else { + std::fs::write(&self.path, "updated")?; + } + } + Ok(()) + } + } + #[test] + fn checked_save_boundary_rejects_name_substitution_without_mutating_replacement() { + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("a"); + std::fs::write(&path, "original").unwrap(); + let policy = Access { + path: path.clone(), + counter: AtomicUsize::new(0), + trigger: 4, + replacement: true, + }; + let handle = crate::open_root_no_symlinks(root.path()).unwrap(); + let checked = CheckedTarget::pin(&handle, Path::new("a"), &path, &policy, true).unwrap(); + policy.counter.store(0, Ordering::SeqCst); + let error = crate::run_write( + root.path(), + WriteRequest { + path: FsPath::new("a").unwrap(), + content: b"worker".to_vec(), + expected_hash: Some(Sha256::digest(b"original").into()), + }, + &checked, + ) + .unwrap_err(); + assert!( + matches!(error, FsError::Io { source, .. } if source.kind() == std::io::ErrorKind::WouldBlock) + ); + assert!(!checked.effects_possible()); + assert_eq!(std::fs::read_to_string(&path).unwrap(), "replacement"); + assert_eq!( + std::fs::read_to_string(path.with_extension("old")).unwrap(), + "original" + ); + } + #[test] + fn checked_read_midread_name_replacement_never_pairs_old_bytes_with_new_validator() { + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("a"); + std::fs::write(&path, "original").unwrap(); + let policy = Access { + path: path.clone(), + counter: AtomicUsize::new(0), + trigger: 3, + replacement: true, + }; + let handle = crate::open_root_no_symlinks(root.path()).unwrap(); + let checked = CheckedTarget::pin(&handle, Path::new("a"), &path, &policy, false).unwrap(); + let validator = checked.validator().to_vec(); + policy.counter.store(0, Ordering::SeqCst); + let result = crate::run_read( + root.path(), + ReadRequest { + path: FsPath::new("a").unwrap(), + offset: 0, + limit: 20, + max_bytes: 1024, + }, + &checked, + ); + match result { + Err(FsError::Conflict(_)) => {} + Ok(read) => { + // Same pinned inode supplies bytes and hash. Execute's final + // named-target check must reject, not refresh to replacement. + assert_eq!(read.bytes, b"original"); + let expected_hash: crate::ContentHash = Sha256::digest(b"original").into(); + assert_eq!(read.content_hash, expected_hash); + assert_eq!( + checked.confirm().unwrap_err().kind(), + std::io::ErrorKind::WouldBlock + ); + } + other => panic!("unexpected Read result: {other:?}"), + } + assert_eq!(checked.validator(), validator); + assert_ne!( + identity_validator(&std::fs::metadata(&path).unwrap()).unwrap(), + validator + ); + assert_eq!(std::fs::read_to_string(&path).unwrap(), "replacement"); + } + + #[test] + fn create_post_parent_observation_retains_bound_directory_and_rejects_substitution() { + let root = tempfile::tempdir().unwrap(); + let directory = root.path().join("d"); + std::fs::create_dir(&directory).unwrap(); + let policy = Access { + path: directory.clone(), + counter: AtomicUsize::new(0), + trigger: usize::MAX, + replacement: false, + }; + let handle = crate::open_root_no_symlinks(root.path()).unwrap(); + let mut checked = + CheckedTarget::pin(&handle, Path::new("d"), &directory, &policy, true).unwrap(); + let initial = checked.validator().to_vec(); + let create = checked.create_access(PathBuf::from("new"), directory.join("new")); + crate::run_write( + root.path(), + WriteRequest { + path: FsPath::new("d/new").unwrap(), + content: b"new".to_vec(), + expected_hash: None, + }, + &create, + ) + .unwrap(); + checked.refresh_directory_observation().unwrap(); + assert_ne!(initial, checked.validator()); + assert_eq!( + checked.validator(), + identity_validator(&std::fs::metadata(&directory).unwrap()).unwrap() + ); + std::fs::rename(&directory, root.path().join("old")).unwrap(); + std::fs::create_dir(&directory).unwrap(); + assert_eq!( + checked.refresh_directory_observation().unwrap_err().kind(), + std::io::ErrorKind::WouldBlock + ); + assert!(checked.effects_possible()); + } + + #[test] + fn checked_parent_swap_is_rejected_even_with_a_hardlinked_same_inode_target() { + let root = tempfile::tempdir().unwrap(); + let directory = root.path().join("d"); + std::fs::create_dir(&directory).unwrap(); + let path = directory.join("a"); + std::fs::write(&path, "original").unwrap(); + let policy = Access { + path: path.clone(), + counter: AtomicUsize::new(0), + trigger: usize::MAX, + replacement: false, + }; + let handle = crate::open_root_no_symlinks(root.path()).unwrap(); + let checked = CheckedTarget::pin(&handle, Path::new("d/a"), &path, &policy, true).unwrap(); + std::fs::rename(&directory, root.path().join("old")).unwrap(); + std::fs::create_dir(&directory).unwrap(); + std::fs::hard_link(root.path().join("old/a"), &path).unwrap(); + assert_eq!( + checked.confirm().unwrap_err().kind(), + std::io::ErrorKind::WouldBlock + ); + assert!(!checked.effects_possible()); + } + + #[test] + fn shared_text_read_rejects_mid_read_metadata_change() { + let root = tempfile::tempdir().unwrap(); + let path = root.path().join("a"); + std::fs::write(&path, "original").unwrap(); + let policy = Access { + path, + counter: AtomicUsize::new(0), + trigger: 2, + replacement: false, + }; + let error = crate::run_read( + root.path(), + ReadRequest { + path: FsPath::new("a").unwrap(), + offset: 0, + limit: 20, + max_bytes: 1024, + }, + &policy, + ) + .unwrap_err(); + assert!(matches!(error, FsError::Conflict(_))); + } +} diff --git a/crates/fs-operation/src/descriptor_ignore.rs b/crates/fs-operation/src/descriptor_ignore.rs new file mode 100644 index 000000000..f8089049e --- /dev/null +++ b/crates/fs-operation/src/descriptor_ignore.rs @@ -0,0 +1,294 @@ +//! Ignore rules loaded only through the provider's authorized descriptor opens. +//! Never use GitignoreBuilder::add/build_global: those reopen host pathnames. +use crate::{FsAccessPolicy, FsError}; +use ignore::gitignore::{Gitignore, GitignoreBuilder}; +use std::io::{BufRead, BufReader, Read}; +use std::path::{Path, PathBuf}; + +pub(crate) const MAX_IGNORE_SOURCE_BYTES: u64 = 8 * 1024 * 1024; +const MAX_IGNORE_FILE_BYTES: u64 = 1024 * 1024; +const MAX_IGNORE_LINE_BYTES: u64 = 16 * 1024; + +pub(crate) struct SourceBoundedReader<'a, R> { + pub inner: R, + pub remaining: &'a mut u64, + pub access: &'a dyn FsAccessPolicy, +} +impl Read for SourceBoundedReader<'_, R> { + fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { + self.access.check_cancelled()?; + if buffer.is_empty() { + return Ok(0); + } + if *self.remaining == 0 { + let mut probe = [0_u8; 1]; + return match self.inner.read(&mut probe)? { + 0 => Ok(0), + _ => Err(std::io::Error::new( + std::io::ErrorKind::InvalidData, + "search source exceeded provider byte limit", + )), + }; + } + let limit = usize::try_from(*self.remaining) + .unwrap_or(usize::MAX) + .min(buffer.len()); + let read = self.inner.read(&mut buffer[..limit])?; + *self.remaining = (*self.remaining).saturating_sub(read as u64); + Ok(read) + } +} + +/// Shared Glob/Grep grammar and provider authorization. Glob, like WalkBuilder, +/// ignores unavailable configuration files; Grep retains its existing explicit +/// error on a rejected ignore-file open. Cancellation/bounds always fail closed. +pub(crate) fn add_checked_ignore_file( + builder: &mut GitignoreBuilder, + path: &Path, + remaining: &mut u64, + access: &dyn FsAccessPolicy, + skip_unavailable: bool, +) -> Result { + access.check_cancelled().map_err(|e| FsError::io(path, e))?; + let resolved = match access.resolve_access_path(path) { + Ok(path) => path, + Err(error) if skip_unavailable && error.kind() != std::io::ErrorKind::Interrupted => { + return Ok(false); + } + Err(error) => return Err(FsError::io(path, error)), + }; + if !access.is_readable_paths(path, &resolved) { + return Ok(false); + } + let file = match access.open_read_file(path, &resolved) { + Ok(file) => file, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(false), + Err(error) if skip_unavailable && error.kind() != std::io::ErrorKind::Interrupted => { + return Ok(false); + } + Err(error) => return Err(FsError::io(path, error)), + }; + let metadata = file.metadata().map_err(|e| FsError::io(path, e))?; + if !metadata.is_file() { + return Ok(false); + } + if metadata.len() > MAX_IGNORE_FILE_BYTES { + return Err(FsError::InvalidArgument( + "ignore file exceeds provider limit 1 MiB".into(), + )); + } + let bounded = SourceBoundedReader { + inner: file, + remaining, + access, + }; + // Take also fences files that grow after metadata. Each line is bounded + // before allocation, and every underlying read checks cancellation. + let mut reader = BufReader::new(bounded.take(MAX_IGNORE_FILE_BYTES + 1)); + let mut line = Vec::new(); + let mut bytes = 0_u64; + let mut added = false; + loop { + line.clear(); + // BufRead::read_line/read_until retry Interrupted internally. Explicit + // fill_buf prevents a cooperative cancellation from becoming a retry + // loop, and caps allocation before copying even a growing long line. + loop { + access.check_cancelled().map_err(|e| FsError::io(path, e))?; + let buffer = reader.fill_buf().map_err(|e| FsError::io(path, e))?; + if buffer.is_empty() { + break; + } + let count = buffer + .iter() + .position(|byte| *byte == b'\n') + .map_or(buffer.len(), |index| index + 1); + if line.len().saturating_add(count) as u64 > MAX_IGNORE_LINE_BYTES { + return Err(FsError::InvalidArgument( + "ignore line exceeds provider limit 16 KiB".into(), + )); + } + let finished = buffer[count - 1] == b'\n'; + line.extend_from_slice(&buffer[..count]); + reader.consume(count); + if finished { + break; + } + } + if line.is_empty() { + break; + } + bytes = bytes.saturating_add(line.len() as u64); + if bytes > MAX_IGNORE_FILE_BYTES { + return Err(FsError::InvalidArgument( + "ignore file exceeds provider limit 1 MiB".into(), + )); + } + let line = std::str::from_utf8(&line) + .map_err(|_| FsError::InvalidArgument("ignore file must be UTF-8".into()))? + .trim_end_matches(['\r', '\n']); + // One invalid glob must not discard other valid rules. + if builder.add_line(Some(path.to_path_buf()), line).is_ok() { + added = true; + } + } + Ok(added) +} + +struct DirectoryRules { + path: PathBuf, + ignore: Gitignore, + git_ignore: Gitignore, + exclude: Gitignore, + has_git: bool, +} + +/// Request-local ancestor stack, not a file-tree cache. Only the current +/// directory's ancestry is retained; source bytes and depth bound its matchers. +pub(crate) struct DescriptorIgnores { + directories: Vec, + remaining: u64, + require_git: bool, + skip_unavailable: bool, +} +impl DescriptorIgnores { + pub fn new(root: &Path, base: &Path, access: &dyn FsAccessPolicy) -> Result { + Self::with_options(root, base, access, true, true) + } + pub fn grep(root: &Path, base: &Path, access: &dyn FsAccessPolicy) -> Result { + // Preserve the existing descriptor Grep provider contract: gitignore + // sources apply without a .git marker and rejected opens fail closed. + Self::with_options(root, base, access, false, false) + } + fn with_options( + root: &Path, + base: &Path, + access: &dyn FsAccessPolicy, + require_git: bool, + skip_unavailable: bool, + ) -> Result { + let relative = base.strip_prefix(root).map_err(|_| { + FsError::InvalidArgument("glob base is outside its provider root".into()) + })?; + if relative.components().count() >= 128 { + return Err(FsError::InvalidArgument( + "ignore ancestry exceeds provider depth limit 128".into(), + )); + } + let mut this = Self { + directories: Vec::new(), + remaining: MAX_IGNORE_SOURCE_BYTES, + require_git, + skip_unavailable, + }; + let mut directory = root.to_path_buf(); + for component in relative.components() { + this.load_directory(&directory, access)?; + directory.push(component); + } + Ok(this) + } + + pub fn load_directory( + &mut self, + directory: &Path, + access: &dyn FsAccessPolicy, + ) -> Result<(), FsError> { + self.directories + .retain(|rules| directory.starts_with(&rules.path)); + if self + .directories + .last() + .is_some_and(|rules| rules.path == directory) + { + return Ok(()); + } + if self.directories.len() >= 128 { + return Err(FsError::InvalidArgument( + "ignore ancestry exceeds provider depth limit 128".into(), + )); + } + access + .check_cancelled() + .map_err(|e| FsError::io(directory, e))?; + let marker = directory.join(".git"); + let git_kind = access + .resolve_access_path(&marker) + .ok() + .and_then(|resolved| { + if !access.is_readable_paths(&marker, &resolved) { + return None; + } + access + .read_metadata(&marker, &resolved) + .ok() + .map(|metadata| metadata.file_type()) + }); + let has_git = git_kind.is_some_and(|kind| kind.is_dir() || kind.is_file()); + let mut load = |name: &str| -> Result { + let mut builder = GitignoreBuilder::new(directory); + add_checked_ignore_file( + &mut builder, + &directory.join(name), + &mut self.remaining, + access, + self.skip_unavailable, + )?; + builder.build().map_err(|_| { + FsError::InvalidArgument("ignore patterns could not be compiled".into()) + }) + }; + let ignore = load(".ignore")?; + let git_ignore = load(".gitignore")?; + // A linked-worktree .git file activates repository rules but is not a + // directory. Never chase its arbitrary host gitdir/commondir pointers. + let exclude = if git_kind.is_some_and(|kind| kind.is_dir()) { + load(".git/info/exclude")? + } else { + Gitignore::empty() + }; + self.directories.push(DirectoryRules { + path: directory.to_path_buf(), + ignore, + git_ignore, + exclude, + has_git, + }); + Ok(()) + } + + pub fn is_ignored(&mut self, path: &Path, is_dir: bool) -> bool { + self.directories + .retain(|rules| path.starts_with(&rules.path)); + // .ignore has priority over all .gitignore/exclude rules. Within each + // class the nearest directory's last matching rule wins. Match entries, + // not any parents: ignored directories are pruned by the walker, while + // an explicitly selected ignored search root remains traversable. + for rules in self.directories.iter().rev() { + let matched = rules.ignore.matched(path, is_dir); + if !matched.is_none() { + return matched.is_ignore(); + } + } + if self.require_git && !self.directories.iter().any(|rules| rules.has_git) { + return false; + } + let mut exclude = None; + for rules in self.directories.iter().rev() { + let matched = rules.git_ignore.matched(path, is_dir); + if !matched.is_none() { + return matched.is_ignore(); + } + if exclude.is_none() { + let matched = rules.exclude.matched(path, is_dir); + if !matched.is_none() { + exclude = Some(matched.is_ignore()); + } + } + if rules.has_git { + break; + } + } + exclude.unwrap_or(false) + } +} diff --git a/crates/fs-operation/src/glob.rs b/crates/fs-operation/src/glob.rs index 2ade239b6..739c779b8 100644 --- a/crates/fs-operation/src/glob.rs +++ b/crates/fs-operation/src/glob.rs @@ -44,34 +44,29 @@ pub fn run_glob( .as_ref() .map(|traversal| traversal.path()) .unwrap_or(base); - let mut walker = WalkBuilder::new(walker_root); - walker.hidden(false).follow_links(false); - if traversal.is_some() { - // Descriptor providers must never let WalkBuilder open pathname-based - // ignore files, which may be symlinked outside the pinned root. Glob's - // explicit pattern remains the complete selection contract. - walker - .git_ignore(false) - .git_global(false) - .git_exclude(false) - .ignore(false) - .parents(false); - let search_relative = base.strip_prefix(root).map_err(|_| { - FsError::InvalidArgument("glob base is outside its provider root".to_string()) - })?; - let search_relative = search_relative.to_path_buf(); - let filter_root = walker_root.to_path_buf(); - walker.filter_entry(move |entry| { - entry - .path() - .strip_prefix(&filter_root) - .is_ok_and(|relative| { - relative.starts_with(&search_relative) || search_relative.starts_with(relative) - }) - }); - } + let logical_walk_root = traversal + .as_ref() + .and_then(|t| t.logical_root()) + .unwrap_or(root); + let mut descriptor_ignores = traversal + .as_ref() + .map(|_| crate::descriptor_ignore::DescriptorIgnores::new(root, logical_walk_root, access)) + .transpose()?; let mut visited = 0_usize; - for entry in walker.build().flatten() { + let mut walk = if traversal.is_some() { + // Configuration is read via authorized provider opens, never by + // WalkBuilder's host-path ignore discovery. + crate::walk::Walk::descriptor(logical_walk_root, access) + } else { + crate::walk::Walk::Plain( + WalkBuilder::new(walker_root) + .hidden(false) + .follow_links(false) + .build(), + ) + }; + while let Some(entry) = walk.next() { + let entry = entry?; access.check_cancelled().map_err(|source| FsError::Io { path: PathBuf::from(request.path.as_str()), source, @@ -83,21 +78,27 @@ pub fn run_glob( crate::MAX_TRAVERSAL_ENTRIES ))); } - let walked_path = entry.path(); - let path = if traversal.is_some() { - let relative = walked_path.strip_prefix(walker_root).map_err(|_| { - FsError::InvalidArgument( - "descriptor traversal returned a path outside its root".to_string(), - ) - })?; - root.join(relative) - } else { - walked_path.to_path_buf() - }; + let path = entry.path; + if path.strip_prefix(root).unwrap_or(&path).to_str().is_none() { + continue; + } let resolved = match access.resolve_access_path(&path) { Ok(resolved) if access.is_readable_paths(&path, &resolved) => resolved, _ => continue, }; + if let Some(ignores) = descriptor_ignores.as_mut() { + let is_dir = entry.kind.is_some_and(|kind| kind.is_dir()); + if path != logical_walk_root && ignores.is_ignored(&path, is_dir) { + if is_dir { + walk.skip_current_dir(); + } + continue; + } + if is_dir { + ignores.load_directory(&path, access)?; + continue; + } + } let Ok(file) = access.open_read_file(&path, &resolved) else { continue; }; @@ -111,7 +112,12 @@ pub fn run_glob( let logical = path.strip_prefix(root).map_err(|_| { FsError::InvalidArgument("provider returned a path outside its root".to_string()) })?; - let result_path = FsPath::new(logical.to_string_lossy())?; + let Some(logical) = logical.to_str() else { + continue; + }; + let Ok(result_path) = FsPath::new(logical) else { + continue; + }; let retained = result_path.as_str().len().saturating_add(16); if retained_path_bytes.saturating_add(retained) > crate::MAX_RESULT_PATH_BYTES { provider_truncated = true; diff --git a/crates/fs-operation/src/ignore_tests.rs b/crates/fs-operation/src/ignore_tests.rs new file mode 100644 index 000000000..b55f8caf7 --- /dev/null +++ b/crates/fs-operation/src/ignore_tests.rs @@ -0,0 +1,326 @@ +#![cfg(target_os = "linux")] +use crate::*; +use std::path::{Path, PathBuf}; +use std::sync::{ + Mutex, + atomic::{AtomicUsize, Ordering}, +}; + +struct Access { + root: PathBuf, + denied: Option, + opened: Mutex>, + enumerated: Mutex>, + checks: AtomicUsize, + cancel_at: usize, +} +impl Access { + fn new(root: &Path) -> Self { + Self { + root: root.into(), + denied: None, + opened: Mutex::new(Vec::new()), + enumerated: Mutex::new(Vec::new()), + checks: AtomicUsize::new(0), + cancel_at: usize::MAX, + } + } +} +impl FsAccessPolicy for Access { + fn is_readable(&self, path: &Path) -> bool { + path.starts_with(&self.root) && self.denied.as_ref().is_none_or(|denied| path != denied) + } + fn is_writable(&self, _: &Path) -> bool { + false + } + fn resolve_access_path(&self, path: &Path) -> std::io::Result { + Ok(path.into()) + } + fn check_cancelled(&self) -> std::io::Result<()> { + if self.checks.fetch_add(1, Ordering::Relaxed) >= self.cancel_at { + Err(std::io::ErrorKind::Interrupted.into()) + } else { + Ok(()) + } + } + fn open_read_file(&self, logical: &Path, resolved: &Path) -> std::io::Result { + assert!( + self.is_readable(logical), + "denied ignore/source file opened" + ); + self.opened.lock().unwrap().push(logical.into()); + open_beneath_no_symlinks(&self.root, resolved) + } + fn read_metadata(&self, _: &Path, resolved: &Path) -> std::io::Result { + open_beneath_no_symlinks(&self.root, resolved)?.metadata() + } + fn open_traversal_root( + &self, + _: &Path, + resolved: &Path, + ) -> std::io::Result> { + use std::os::fd::AsRawFd; + let directory = open_beneath_no_symlinks(&self.root, resolved)?; + let walk = PathBuf::from(format!("/proc/self/fd/{}", directory.as_raw_fd())); + Ok(Some(FsTraversalRoot::new_at( + walk, + directory, + resolved.into(), + ))) + } + fn open_read_dir(&self, logical: &Path, resolved: &Path) -> std::io::Result { + use std::os::fd::AsRawFd; + self.enumerated.lock().unwrap().push(logical.into()); + let directory = open_beneath_no_symlinks(&self.root, resolved)?; + std::fs::read_dir(format!("/proc/self/fd/{}", directory.as_raw_fd())) + } +} +fn glob(root: &Path, access: &dyn FsAccessPolicy) -> Result { + run_glob( + root, + root, + GlobRequest { + path: FsPath::root(), + pattern: "**/*.txt".into(), + limit: 100, + }, + access, + ) +} +#[test] +fn descriptor_glob_prunes_ignored_directories_before_enumeration_and_loading_rules() { + let root = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(root.path().join("ignored/deep")).unwrap(); + std::fs::write(root.path().join(".ignore"), "ignored/\n").unwrap(); + // This rejected directory must never even attempt to load ignore files. + std::fs::File::create(root.path().join("ignored/.ignore")) + .unwrap() + .set_len(2 * 1024 * 1024) + .unwrap(); + std::fs::write(root.path().join("ignored/deep/secret.txt"), "secret").unwrap(); + std::fs::write(root.path().join("visible.txt"), "visible").unwrap(); + let access = Access::new(root.path()); + assert_eq!( + glob(root.path(), &access).unwrap().paths, + vec![FsPath::new("visible.txt").unwrap()] + ); + assert_eq!( + *access.enumerated.lock().unwrap(), + vec![root.path().to_path_buf()] + ); + assert!( + !access + .opened + .lock() + .unwrap() + .iter() + .any(|path| path.starts_with(root.path().join("ignored"))) + ); +} +#[test] +fn descriptor_glob_denied_and_symlinked_ignore_files_cannot_change_selection() { + let root = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + std::fs::write(outside.path().join("rules"), "visible.txt\n").unwrap(); + std::fs::create_dir(root.path().join(".git")).unwrap(); + std::os::unix::fs::symlink(outside.path().join("rules"), root.path().join(".gitignore")) + .unwrap(); + std::fs::write(root.path().join(".ignore"), "visible.txt\n").unwrap(); + std::fs::write(root.path().join("visible.txt"), "visible").unwrap(); + let mut access = Access::new(root.path()); + access.denied = Some(root.path().join(".ignore")); + assert_eq!( + glob(root.path(), &access).unwrap().paths, + vec![FsPath::new("visible.txt").unwrap()] + ); + assert!( + !access + .opened + .lock() + .unwrap() + .contains(&root.path().join(".ignore")) + ); +} +#[test] +fn descriptor_ignore_loading_checks_cancellation_without_retrying_interrupted() { + let root = tempfile::tempdir().unwrap(); + std::fs::write(root.path().join(".ignore"), "ignored.txt\n".repeat(1000)).unwrap(); + let mut access = Access::new(root.path()); + // First check is before resolution, second at the first fill_buf. + access.cancel_at = 1; + let error = crate::descriptor_ignore::add_checked_ignore_file( + &mut ignore::gitignore::GitignoreBuilder::new(root.path()), + &root.path().join(".ignore"), + &mut 1024_000, + &access, + true, + ) + .unwrap_err(); + assert!( + matches!(error, FsError::Io { source, .. } if source.kind() == std::io::ErrorKind::Interrupted) + ); + assert_eq!(access.checks.load(Ordering::Relaxed), 2); +} +#[test] +fn descriptor_ignore_loading_bounds_file_line_and_aggregate_bytes() { + let root = tempfile::tempdir().unwrap(); + let path = root.path().join(".ignore"); + let access = Access::new(root.path()); + std::fs::File::create(&path) + .unwrap() + .set_len(1024 * 1024 + 1) + .unwrap(); + assert!(matches!( + glob(root.path(), &access), + Err(FsError::InvalidArgument(_)) + )); + std::fs::write(&path, "x".repeat(16 * 1024 + 1)).unwrap(); + assert!(matches!( + glob(root.path(), &access), + Err(FsError::InvalidArgument(_)) + )); + std::fs::write(&path, "ignored.txt\n").unwrap(); + let error = crate::descriptor_ignore::add_checked_ignore_file( + &mut ignore::gitignore::GitignoreBuilder::new(root.path()), + &path, + &mut 3, + &access, + true, + ) + .unwrap_err(); + assert!( + matches!(error, FsError::Io { source, .. } if source.kind() == std::io::ErrorKind::InvalidData) + ); +} +#[test] +fn descriptor_ignore_ancestry_is_bounded_without_enumerating_a_tree() { + let root = tempfile::tempdir().unwrap(); + let access = Access::new(root.path()); + let base = (0..128).fold(root.path().to_path_buf(), |path, _| path.join("d")); + assert!(matches!( + crate::descriptor_ignore::DescriptorIgnores::new(root.path(), &base, &access), + Err(FsError::InvalidArgument(_)) + )); + assert!(access.opened.lock().unwrap().is_empty()); +} + +struct Plain(PathBuf); +impl FsAccessPolicy for Plain { + fn is_readable(&self, path: &Path) -> bool { + path.starts_with(&self.0) + } + fn is_writable(&self, _: &Path) -> bool { + false + } +} +#[test] +fn descriptor_glob_matches_plain_git_markers_exclude_and_nested_repository_boundaries() { + let root = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(root.path().join("nested/.git/info")).unwrap(); + std::fs::create_dir_all(root.path().join(".git/info")).unwrap(); + std::fs::write( + root.path().join(".gitignore"), + "*.outer.txt\n!include.txt\n", + ) + .unwrap(); + std::fs::write(root.path().join(".ignore"), "always.txt\n").unwrap(); + std::fs::write( + root.path().join(".git/info/exclude"), + "excluded.txt\ninclude.txt\n", + ) + .unwrap(); + std::fs::write( + root.path().join("nested/.git/info/exclude"), + "/excluded.txt\n", + ) + .unwrap(); + for name in [ + "a.outer.txt", + "excluded.txt", + "include.txt", + "always.txt", + "nested/a.outer.txt", + "nested/excluded.txt", + "nested/always.txt", + ] { + std::fs::write(root.path().join(name), "data").unwrap(); + } + let native = glob(root.path(), &Access::new(root.path())).unwrap(); + let ordinary = glob(root.path(), &Plain(root.path().into())).unwrap(); + assert_eq!(native, ordinary); + assert_eq!( + native.paths, + vec![ + FsPath::new("include.txt").unwrap(), + FsPath::new("nested/a.outer.txt").unwrap() + ] + ); +} +#[test] +fn descriptor_glob_requires_authorized_git_marker_for_gitignore_not_for_ignore() { + let root = tempfile::tempdir().unwrap(); + std::fs::write(root.path().join(".gitignore"), "gitignored.txt\n").unwrap(); + std::fs::write(root.path().join(".ignore"), "ignored.txt\n").unwrap(); + for name in ["gitignored.txt", "ignored.txt", "visible.txt"] { + std::fs::write(root.path().join(name), "data").unwrap(); + } + let native = glob(root.path(), &Access::new(root.path())).unwrap(); + // Ambient ancestors may themselves be repositories (e.g. the harness's + // TMPDIR). Their markers/ignore files are outside this provider root and + // cannot activate Git rules or be inspected by descriptor search. + assert_eq!(native.paths.len(), 2); + std::fs::create_dir(root.path().join(".git")).unwrap(); + let native = glob(root.path(), &Access::new(root.path())).unwrap(); + assert_eq!( + native, + glob(root.path(), &Plain(root.path().into())).unwrap() + ); + assert_eq!(native.paths, vec![FsPath::new("visible.txt").unwrap()]); + let mut denied = Access::new(root.path()); + denied.denied = Some(root.path().join(".git")); + assert_eq!(glob(root.path(), &denied).unwrap().paths.len(), 2); + std::fs::remove_dir(root.path().join(".git")).unwrap(); + let outside = tempfile::tempdir().unwrap(); + std::fs::write(outside.path().join("exclude"), "visible.txt\n").unwrap(); + std::fs::write( + root.path().join(".git"), + format!("gitdir: {}\n", outside.path().display()), + ) + .unwrap(); + // Real managed Workdirs use a .git file. Activate root .gitignore without + // failing on .git/info/exclude or following external administration paths. + let access = Access::new(root.path()); + assert_eq!( + glob(root.path(), &access).unwrap().paths, + vec![FsPath::new("visible.txt").unwrap()] + ); + std::fs::write(root.path().join("visible.txt"), "needle\n").unwrap(); + let grep = run_grep( + root.path(), + root.path().into(), + GrepRequest { + path: FsPath::root(), + pattern: "needle".into(), + glob: None, + file_type: None, + case_insensitive: false, + before_context: 0, + after_context: 0, + multiline: false, + output_mode: GrepOutputMode::FilesWithMatches, + offset: 0, + limit: 100, + }, + &access, + ) + .unwrap(); + assert_eq!(grep.paths, vec![FsPath::new("visible.txt").unwrap()]); + assert!( + !access + .opened + .lock() + .unwrap() + .iter() + .any(|path| path.starts_with(outside.path())) + ); +} diff --git a/crates/fs-operation/src/lib.rs b/crates/fs-operation/src/lib.rs index 7fa897df8..63436ed05 100644 --- a/crates/fs-operation/src/lib.rs +++ b/crates/fs-operation/src/lib.rs @@ -4,10 +4,19 @@ //! transport, or LLM Tool implementations. Paths are logical and root-relative; //! providers supply host roots and access policy. +mod checked; +pub use checked::{CheckedTarget, identity_validator}; +mod descriptor_ignore; mod glob; +#[cfg(test)] +mod ignore_tests; mod local; mod operation; mod search; +pub mod text; +#[cfg(test)] +mod traversal_tests; +mod walk; use std::path::{Path, PathBuf}; @@ -272,6 +281,7 @@ pub fn atomic_write_beneath_no_symlinks_at( #[derive(Debug)] pub struct FsTraversalRoot { path: PathBuf, + logical_root: Option, _directory: std::fs::File, } @@ -279,20 +289,44 @@ impl FsTraversalRoot { pub fn new(path: PathBuf, directory: std::fs::File) -> Self { Self { path, + logical_root: None, _directory: directory, } } + /// Map walked paths to this logical anchor rather than the provider root. + pub fn new_at(path: PathBuf, directory: std::fs::File, logical_root: PathBuf) -> Self { + Self { + path, + logical_root: Some(logical_root), + _directory: directory, + } + } + pub fn logical_root(&self) -> Option<&Path> { + self.logical_root.as_deref() + } pub fn path(&self) -> &Path { &self.path } } +/// Owned predicate used before walker descent. Paths are provider-logical; +/// true directories require enumeration authority, other entries require read. +pub type FsTraversalFilter = std::sync::Arc bool + Send + Sync>; + /// Provider-owned access policy used by local filesystem operations. pub trait FsAccessPolicy: Send + Sync { fn is_readable(&self, path: &Path) -> bool; fn is_writable(&self, path: &Path) -> bool; + fn can_enumerate_directory(&self, logical: &Path, resolved: &Path) -> bool { + self.is_readable_paths(logical, resolved) + } + /// Owned because the shared walker requires a 'static entry predicate. + fn traversal_filter(&self) -> Option { + None + } + /// Cooperatively stop provider-owned filesystem work. Implementations may /// use this for per-operation cancellation and deadlines. fn check_cancelled(&self) -> std::io::Result<()> { diff --git a/crates/fs-operation/src/local.rs b/crates/fs-operation/src/local.rs index 34f60b92d..887a09248 100644 --- a/crates/fs-operation/src/local.rs +++ b/crates/fs-operation/src/local.rs @@ -162,6 +162,19 @@ fn run_read_with_limits( } } + #[cfg(unix)] + if crate::identity_validator(&metadata).map_err(|e| map_io(&logical, e))? + != crate::identity_validator( + &reader + .get_ref() + .metadata() + .map_err(|e| map_io(&logical, e))?, + ) + .map_err(|e| map_io(&logical, e))? + { + return Err(FsError::Conflict(logical.as_str().to_string())); + } + let total_lines = current_line.saturating_add(usize::from(last_byte.is_some_and(|byte| byte != b'\n'))); if request.offset > total_lines && request.offset != 0 { @@ -176,15 +189,7 @@ fn run_read_with_limits( .min(total_lines); let byte_truncated = selected_bytes_seen > response_limit; if byte_truncated { - let mut byte_end = response_limit.min(selected.len()); - while byte_end > 0 { - match std::str::from_utf8(&selected[..byte_end]) { - Ok(_) => break, - Err(error) if error.error_len().is_none() => byte_end -= 1, - Err(_) => break, - } - } - selected.truncate(byte_end); + selected.truncate(crate::text::bounded_utf8_len(&selected, response_limit)); } Ok(ReadResult { path: logical, @@ -402,55 +407,33 @@ pub fn run_edit( let content = String::from_utf8(bytes).map_err(|_| { FsError::InvalidArgument(format!("{} is not valid UTF-8", logical.as_str())) })?; - let occurrences = content.matches(&request.old_string).count(); - if occurrences == 0 { - return Err(FsError::InvalidArgument( - "old_string was not found".to_string(), - )); - } - if !request.replace_all && occurrences != 1 { - return Err(FsError::InvalidArgument(format!( - "old_string matched {occurrences} times; set replace_all=true or provide a unique string" - ))); - } - let replacement_count = if request.replace_all { occurrences } else { 1 }; - if let Some(limit) = access.max_edit_replacements() - && replacement_count > limit - { - return Err(FsError::InvalidArgument(format!( - "{} exceeds provider Edit replacement limit {limit}", - logical.as_str() - ))); - } - let removed_bytes = request - .old_string - .len() - .checked_mul(replacement_count) - .ok_or_else(|| FsError::InvalidArgument("edited content size overflowed".to_string()))?; - let added_bytes = request - .new_string - .len() - .checked_mul(replacement_count) - .ok_or_else(|| FsError::InvalidArgument("edited content size overflowed".to_string()))?; - let edited_len = content - .len() - .checked_sub(removed_bytes) - .and_then(|size| size.checked_add(added_bytes)) - .ok_or_else(|| FsError::InvalidArgument("edited content size overflowed".to_string()))?; - enforce_write_bound(&logical, edited_len, access)?; - let edited = if request.replace_all { - content.replace(&request.old_string, &request.new_string) - } else { - content.replacen(&request.old_string, &request.new_string, 1) + let args = crate::text::EditArgs { + old_string: request.old_string, + new_string: request.new_string, + replace_all: request.replace_all, }; - debug_assert_eq!(edited.len(), edited_len); + let edited = crate::text::edit( + &content, + &args, + crate::text::TextLimits { + max_input_bytes: access.max_write_bytes(), + max_output_bytes: access.max_write_bytes(), + max_replacements: access.max_edit_replacements(), + }, + ) + .map_err(|error| map_text_error(&logical, error))?; access - .atomic_write_file(&path, &target, edited.as_bytes(), AtomicWriteMode::Replace) + .atomic_write_file( + &path, + &target, + edited.content.as_bytes(), + AtomicWriteMode::Replace, + ) .map_err(|error| map_io(&logical, error))?; Ok(EditResult { - replacements: if request.replace_all { occurrences } else { 1 }, - bytes_written: edited.len(), - content_hash: hash_bytes(edited.as_bytes()), + replacements: edited.replacements, + bytes_written: edited.bytes_written, + content_hash: hash_bytes(edited.content.as_bytes()), }) } @@ -473,6 +456,14 @@ pub fn run_list( if !metadata.is_dir() { return Err(FsError::NotDirectory(PathBuf::from(logical.as_str()))); } + if !access.can_enumerate_directory(&logical_base, &path) { + return Ok(ListResult { + entries: Vec::new(), + total_entries: 0, + total_bytes: 0, + truncated: false, + }); + } let mut entries = Vec::new(); let mut retained_path_bytes = 0_usize; let mut provider_truncated = false; @@ -490,6 +481,9 @@ pub fn run_list( ))); } let entry = entry.map_err(|error| map_io(&logical, error))?; + if entry.file_name().to_str().is_none() { + continue; + } let logical_absolute = logical_base.join(entry.file_name()); let resolved = match access.resolve_access_path(&logical_absolute) { Ok(resolved) => resolved, @@ -513,7 +507,12 @@ pub fn run_list( let relative = logical_absolute.strip_prefix(root).map_err(|_| { FsError::InvalidArgument("provider returned a path outside its root".to_string()) })?; - let result_path = FsPath::new(relative.to_string_lossy())?; + let Some(relative) = relative.to_str() else { + continue; + }; + let Ok(result_path) = FsPath::new(relative) else { + continue; + }; let retained = result_path.as_str().len().saturating_add(64); if retained_path_bytes.saturating_add(retained) > crate::MAX_RESULT_PATH_BYTES { provider_truncated = true; @@ -550,15 +549,37 @@ fn enforce_write_bound( bytes: usize, access: &dyn FsAccessPolicy, ) -> Result<(), FsError> { - if let Some(limit) = access.max_write_bytes() - && bytes > limit - { - return Err(FsError::InvalidArgument(format!( - "{} exceeds provider write limit {limit}", + crate::text::check_output_bytes( + bytes, + crate::text::TextLimits { + max_output_bytes: access.max_write_bytes(), + ..Default::default() + }, + ) + .map_err(|error| map_text_error(logical, error)) +} + +fn map_text_error(logical: &FsPath, error: crate::text::TextError) -> FsError { + use crate::text::TextError; + + let message = match error { + TextError::InputTooLarge { limit, .. } | TextError::OutputTooLarge { limit, .. } => { + format!("{} exceeds provider write limit {limit}", logical.as_str()) + } + TextError::ReplacementLimitExceeded { limit, .. } => format!( + "{} exceeds provider Edit replacement limit {limit}", logical.as_str() - ))); - } - Ok(()) + ), + TextError::Decode(error) => format!("invalid arguments: {error}"), + TextError::EmptyOldString => "old_string must not be empty".to_string(), + TextError::IdenticalStrings => "old_string and new_string are identical".to_string(), + TextError::NotFound => "old_string was not found".to_string(), + TextError::MultipleMatches { occurrences } => format!( + "old_string matched {occurrences} times; set replace_all=true or provide a unique string" + ), + TextError::Overflow => "edited content size overflowed".to_string(), + }; + FsError::InvalidArgument(message) } fn read_mutation_preimage( @@ -610,9 +631,21 @@ fn require_access( write: bool, allow_symlink_directory: bool, ) -> Result { - let resolved = access - .resolve_access_path(path) - .map_err(|error| map_io(logical, error))?; + let resolved = access.resolve_access_path(path).map_err(|error| { + // A dangling direct link has no resolved identity, but still needs the + // existing sanitized repair diagnostic rather than an ordinary missing-file error. + if error.kind() == std::io::ErrorKind::NotFound + && direct_symlink(path).is_some_and(|info| !info.target_exists) + { + FsError::BrokenSymlink { + path: PathBuf::from(logical.as_str()), + link: PathBuf::from(logical.as_str()), + target: PathBuf::from(""), + } + } else { + map_io(logical, error) + } + })?; let symlink = (resolved != path).then(|| direct_symlink(path)).flatten(); if let Some(info) = symlink.as_ref() && !info.target_exists @@ -773,3 +806,189 @@ mod bounded_read_tests { assert!(result.truncated); } } + +#[cfg(test)] +mod text_policy_tests { + use super::*; + + struct Access { + root: PathBuf, + bytes: Option, + replacements: Option, + } + + impl FsAccessPolicy for Access { + fn is_readable(&self, path: &Path) -> bool { + path.starts_with(&self.root) + } + + fn is_writable(&self, path: &Path) -> bool { + path.starts_with(&self.root) + } + + fn max_write_bytes(&self) -> Option { + self.bytes + } + + fn max_edit_replacements(&self) -> Option { + self.replacements + } + } + + #[test] + fn edit_policy_errors_preserve_the_preimage_and_provider_messages() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let source = "a a"; + fs::write(root.join("text.txt"), source).unwrap(); + let access = Access { + root: root.clone(), + bytes: Some(4), + replacements: Some(1), + }; + for (old, new, all, expected_message) in [ + ("", "x", false, "old_string must not be empty"), + ("a", "a", false, "old_string and new_string are identical"), + ( + "missing", + "x", + false, + "text.txt exceeds provider write limit 4", + ), + ("b", "x", false, "old_string was not found"), + ( + "a", + "x", + false, + "old_string matched 2 times; set replace_all=true or provide a unique string", + ), + ( + "a", + "x", + true, + "text.txt exceeds provider Edit replacement limit 1", + ), + ( + "a a", + "😀x", + false, + "text.txt exceeds provider write limit 4", + ), + ] { + let error = run_edit( + &root, + EditRequest { + path: FsPath::new("text.txt").unwrap(), + old_string: old.into(), + new_string: new.into(), + replace_all: all, + expected_hash: hash_bytes(source.as_bytes()), + }, + &access, + ) + .unwrap_err(); + assert!( + matches!(error, FsError::InvalidArgument(message) if message == expected_message) + ); + assert_eq!(fs::read_to_string(root.join("text.txt")).unwrap(), source); + } + let access = Access { + replacements: None, + ..access + }; + let error = run_edit( + &root, + EditRequest { + path: FsPath::new("text.txt").unwrap(), + old_string: "a".into(), + new_string: "xxx".into(), + replace_all: true, + expected_hash: hash_bytes(source.as_bytes()), + }, + &access, + ) + .unwrap_err(); + assert!( + matches!(error, FsError::InvalidArgument(message) if message == "text.txt exceeds provider write limit 4") + ); + assert_eq!(fs::read_to_string(root.join("text.txt")).unwrap(), source); + } + + #[test] + fn edit_keeps_cas_before_content_policy_and_hashes_the_core_result() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + fs::write(root.join("text.txt"), "é😀").unwrap(); + let access = Access { + root: root.clone(), + bytes: None, + replacements: None, + }; + let error = run_edit( + &root, + EditRequest { + path: FsPath::new("text.txt").unwrap(), + old_string: String::new(), + new_string: "x".into(), + replace_all: false, + expected_hash: [0; 32], + }, + &access, + ) + .unwrap_err(); + assert!(matches!(error, FsError::Conflict(_))); + let result = run_edit( + &root, + EditRequest { + path: FsPath::new("text.txt").unwrap(), + old_string: "é".into(), + new_string: "界".into(), + replace_all: false, + expected_hash: hash_bytes("é😀".as_bytes()), + }, + &access, + ) + .unwrap(); + assert_eq!(result.replacements, 1); + assert_eq!(result.bytes_written, 7); + assert_eq!(result.content_hash, hash_bytes("界😀".as_bytes())); + assert_eq!(fs::read_to_string(root.join("text.txt")).unwrap(), "界😀"); + } + + #[test] + fn local_write_remains_binary_safe_and_maps_shared_bounds() { + let temp = tempfile::tempdir().unwrap(); + let root = temp.path().canonicalize().unwrap(); + let access = Access { + root: root.clone(), + bytes: Some(2), + replacements: None, + }; + let result = run_write( + &root, + WriteRequest { + path: FsPath::new("binary").unwrap(), + content: vec![0xff, 0], + expected_hash: None, + }, + &access, + ) + .unwrap(); + assert_eq!(result.bytes_written, 2); + assert_eq!(fs::read(root.join("binary")).unwrap(), vec![0xff, 0]); + let error = run_write( + &root, + WriteRequest { + path: FsPath::new("large").unwrap(), + content: vec![0xff, 0, 1], + expected_hash: None, + }, + &access, + ) + .unwrap_err(); + assert!( + matches!(error, FsError::InvalidArgument(message) if message == "large exceeds provider write limit 2") + ); + assert!(!root.join("large").exists()); + } +} diff --git a/crates/fs-operation/src/operation.rs b/crates/fs-operation/src/operation.rs index ddb309a8d..3d6e612f3 100644 --- a/crates/fs-operation/src/operation.rs +++ b/crates/fs-operation/src/operation.rs @@ -353,6 +353,10 @@ pub struct GrepRequest { #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] pub struct GrepResult { + /// Exact typed provider paths represented in this returned report, not all + /// candidate/matched source files. Older providers may omit this field. + #[serde(default)] + pub paths: Vec, /// Provider-rendered bounded grep report. Keeping rendering here avoids /// transferring candidate files across a remote provider boundary. pub output: String, diff --git a/crates/fs-operation/src/search.rs b/crates/fs-operation/src/search.rs index eb1129ea9..8506b611a 100644 --- a/crates/fs-operation/src/search.rs +++ b/crates/fs-operation/src/search.rs @@ -1,50 +1,20 @@ -use std::collections::{BTreeMap, HashSet}; +use std::collections::BTreeMap; use std::fmt::Write as _; -use std::io::{BufRead, BufReader, Read}; +use std::io::Read; use std::path::{Path, PathBuf}; +use crate::descriptor_ignore::{DescriptorIgnores, SourceBoundedReader}; + use crate::FsAccessPolicy; use grep_regex::RegexMatcherBuilder; use grep_searcher::sinks::UTF8 as UTF8Sink; use grep_searcher::{BinaryDetection, Searcher, SearcherBuilder, Sink, SinkContext, SinkMatch}; use ignore::WalkBuilder; -use ignore::gitignore::{Gitignore, GitignoreBuilder}; use ignore::overrides::{Override, OverrideBuilder}; use ignore::types::{Types, TypesBuilder}; use crate::{FsError, GrepOutputMode, GrepRequest, GrepResult, direct_symlink}; -struct SourceBoundedReader<'a, R> { - inner: R, - remaining: &'a mut u64, - access: &'a dyn FsAccessPolicy, -} - -impl Read for SourceBoundedReader<'_, R> { - fn read(&mut self, buffer: &mut [u8]) -> std::io::Result { - self.access.check_cancelled()?; - if buffer.is_empty() { - return Ok(0); - } - if *self.remaining == 0 { - let mut probe = [0_u8; 1]; - return match self.inner.read(&mut probe)? { - 0 => Ok(0), - _ => Err(std::io::Error::new( - std::io::ErrorKind::InvalidData, - "grep source exceeded provider byte limit", - )), - }; - } - let limit = usize::try_from(*self.remaining) - .unwrap_or(usize::MAX) - .min(buffer.len()); - let read = self.inner.read(&mut buffer[..limit])?; - *self.remaining = (*self.remaining).saturating_sub(read as u64); - Ok(read) - } -} - struct ContentLine { path: PathBuf, line_number: Option, @@ -79,6 +49,28 @@ fn reserve_report_bytes(report: &mut GrepReport, bytes: usize) -> bool { impl GrepReport { fn into_result(self, root: &Path) -> GrepResult { + // Build Object paths from retained provider identities, never by parsing + // rendered text. Offset, limit and truncation already selected this set. + let paths = match self.mode { + GrepOutputMode::FilesWithMatches => self + .files + .iter() + .filter_map(|path| logical_path(root, path)) + .collect(), + GrepOutputMode::Count => self + .counts + .iter() + .filter_map(|(path, _)| logical_path(root, path)) + .collect(), + GrepOutputMode::Content => self + .lines + .iter() + .map(|line| line.path.as_path()) + .collect::>() + .into_iter() + .filter_map(|path| logical_path(root, path)) + .collect(), + }; let (match_count, matched_files) = match self.mode { GrepOutputMode::FilesWithMatches => (self.files.len(), self.files.len()), GrepOutputMode::Count => ( @@ -116,6 +108,7 @@ impl GrepReport { } } GrepResult { + paths, output, match_count, matched_files, @@ -166,11 +159,15 @@ fn render_content_lines(root: &Path, lines: &[ContentLine], show_line_numbers: b output } +fn logical_path(root: &Path, path: &Path) -> Option { + crate::FsPath::new_scoped(path.strip_prefix(root).unwrap_or(path).to_str()?).ok() +} + fn logical_display(root: &Path, path: &Path) -> String { - path.strip_prefix(root) - .unwrap_or(path) - .to_string_lossy() - .replace('\\', "/") + logical_path(root, path) + .expect("only representable provider paths enter a report") + .as_str() + .to_string() } const DEFAULT_HEAD_LIMIT: usize = 250; @@ -207,119 +204,6 @@ fn direct_file_selected(path: &Path, overrides: Option<&Override>, types: Option && !types.is_some_and(|filter| filter.matched(path, false).is_ignore()) } -#[derive(Default)] -struct DescriptorIgnoreMatchers { - loaded_directories: HashSet, - by_directory: BTreeMap, -} - -impl DescriptorIgnoreMatchers { - fn load_directory( - &mut self, - directory: &Path, - explicit_base: &Path, - source_bytes_remaining: &mut u64, - access: &dyn FsAccessPolicy, - ) -> Result<(), FsError> { - if !self.loaded_directories.insert(directory.to_path_buf()) { - return Ok(()); - } - let mut builder = GitignoreBuilder::new(directory); - let mut has_patterns = false; - for name in [".gitignore", ".ignore"] { - let path = directory.join(name); - let resolved = access - .resolve_access_path(&path) - .map_err(|error| FsError::io(&path, error))?; - if !access.is_readable_paths(&path, &resolved) { - continue; - } - let file = match access.open_read_file(&path, &resolved) { - Ok(file) => file, - Err(error) if error.kind() == std::io::ErrorKind::NotFound => continue, - Err(error) => return Err(FsError::io(&path, error)), - }; - if !file - .metadata() - .map_err(|error| FsError::io(&path, error))? - .is_file() - { - continue; - } - let bounded = SourceBoundedReader { - inner: file, - remaining: source_bytes_remaining, - access, - }; - let mut reader = BufReader::new(bounded); - let mut line = String::new(); - let mut line_number = 0_u64; - loop { - line.clear(); - let read = reader - .read_line(&mut line) - .map_err(|error| FsError::io(&path, error))?; - if read == 0 { - break; - } - line_number = line_number.saturating_add(1); - if line.len() > 1024 * 1024 { - return Err(FsError::InvalidArgument(format!( - "ignore pattern line {line_number} exceeds provider limit" - ))); - } - let line = line.trim_end_matches(['\r', '\n']); - // Match the ignore crate's partial-error behavior: one invalid - // pattern does not discard the remaining valid lines. - if builder.add_line(Some(path.clone()), line).is_ok() { - has_patterns = true; - } - } - } - if !has_patterns { - return Ok(()); - } - let matcher = builder.build().map_err(|_| { - FsError::InvalidArgument("ignore patterns could not be compiled".to_string()) - })?; - // An explicitly selected search root is traversed even when a strict - // ancestor ignores that directory, matching path-backed Grep behavior. - if directory != explicit_base - && explicit_base.starts_with(directory) - && matcher - .matched_path_or_any_parents(explicit_base, true) - .is_ignore() - { - return Ok(()); - } - self.by_directory.insert(directory.to_path_buf(), matcher); - Ok(()) - } - - fn is_ignored(&self, root: &Path, path: &Path) -> bool { - let mut directories = path - .parent() - .into_iter() - .flat_map(Path::ancestors) - .take_while(|directory| directory.starts_with(root)) - .collect::>(); - directories.reverse(); - let mut ignored = false; - for directory in directories { - let Some(matcher) = self.by_directory.get(directory) else { - continue; - }; - let matched = matcher.matched_path_or_any_parents(path, false); - if matched.is_ignore() { - ignored = true; - } else if matched.is_whitelist() { - ignored = false; - } - } - ignored - } -} - struct GrepParams { pattern: String, path: Option, @@ -464,6 +348,9 @@ pub fn run_grep( retained_bytes: 0, truncated: false, }; + if head_limit == 0 { + return Ok(report.into_result(root)); + } let mut matching_files_seen = 0; let mut matches_seen = 0; @@ -474,6 +361,7 @@ pub fn run_grep( &mut searcher, &matcher, &base, + root, mode, &mut report, &mut matching_files_seen, @@ -494,6 +382,10 @@ pub fn run_grep( .as_ref() .map(|traversal| traversal.path()) .unwrap_or(&base); + let logical_walk_root = traversal + .as_ref() + .and_then(|t| t.logical_root()) + .unwrap_or(root); let mut walker = WalkBuilder::new(walker_root); if traversal.is_some() { walker @@ -506,19 +398,6 @@ pub fn run_grep( .ignore(false) .parents(false) .follow_links(false); - let search_relative = base.strip_prefix(root).map_err(|_| { - FsError::InvalidArgument("grep base is outside its provider root".to_string()) - })?; - let search_relative = search_relative.to_path_buf(); - let filter_root = walker_root.to_path_buf(); - walker.filter_entry(move |entry| { - entry - .path() - .strip_prefix(&filter_root) - .is_ok_and(|relative| { - relative.starts_with(&search_relative) || search_relative.starts_with(relative) - }) - }); } else { walker .hidden(true) @@ -538,32 +417,17 @@ pub fn run_grep( let mut visited = 0_usize; let mut source_bytes_remaining = crate::MAX_GREP_SOURCE_BYTES; - let mut descriptor_ignores = DescriptorIgnoreMatchers::default(); - if traversal.is_some() { - let mut directory = root.to_path_buf(); - descriptor_ignores.load_directory( - &directory, - &base, - &mut source_bytes_remaining, - access, - )?; - for component in base - .strip_prefix(root) - .map_err(|_| { - FsError::InvalidArgument("grep base is outside its provider root".to_string()) - })? - .components() - { - directory.push(component); - descriptor_ignores.load_directory( - &directory, - &base, - &mut source_bytes_remaining, - access, - )?; - } - } - for entry in walker.build().flatten() { + let mut descriptor_ignores = traversal + .as_ref() + .map(|_| DescriptorIgnores::grep(root, logical_walk_root, access)) + .transpose()?; + let mut walk = if traversal.is_some() { + crate::walk::Walk::descriptor(logical_walk_root, access) + } else { + crate::walk::Walk::Plain(walker.build()) + }; + while let Some(entry) = walk.next() { + let entry = entry?; access .check_cancelled() .map_err(|error| FsError::io(&base, error))?; @@ -574,39 +438,41 @@ pub fn run_grep( crate::MAX_TRAVERSAL_ENTRIES ))); } - let walked_path = entry.path(); - let path = if traversal.is_some() { - let relative = walked_path.strip_prefix(walker_root).map_err(|_| { - FsError::InvalidArgument( - "descriptor traversal returned a path outside its root".to_string(), - ) - })?; - root.join(relative) - } else { - walked_path.to_path_buf() - }; - let file_type = entry.file_type(); - if traversal.is_some() && file_type.as_ref().is_some_and(|kind| kind.is_dir()) { - descriptor_ignores.load_directory(&path, &base, &mut source_bytes_remaining, access)?; + let path = entry.path; + if logical_path(root, &path).is_none() { continue; } - if !file_type.map(|kind| kind.is_file()).unwrap_or(false) { - continue; - } - if traversal.is_some() { + let file_type = entry.kind; + if let Some(ignores) = descriptor_ignores.as_mut() { let relative = path.strip_prefix(&base).map_err(|_| { FsError::InvalidArgument( "descriptor traversal returned a path outside its search base".to_string(), ) })?; - if relative - .components() - .any(|component| component.as_os_str().to_string_lossy().starts_with('.')) - || descriptor_ignores.is_ignored(root, &path) - || !direct_file_selected(&path, overrides.as_ref(), types.as_ref()) + let is_dir = file_type.is_some_and(|kind| kind.is_dir()); + if path != logical_walk_root + && (relative.components().any(|component| { + component + .as_os_str() + .to_str() + .is_some_and(|name| name.starts_with('.')) + }) || ignores.is_ignored(&path, is_dir)) { + if is_dir { + walk.skip_current_dir(); + } continue; } + if is_dir { + ignores.load_directory(&path, access)?; + continue; + } + if !direct_file_selected(&path, overrides.as_ref(), types.as_ref()) { + continue; + } + } + if !file_type.map(|kind| kind.is_file()).unwrap_or(false) { + continue; } let readable = access .resolve_access_path(&path) @@ -618,6 +484,7 @@ pub fn run_grep( &mut searcher, &matcher, &path, + root, mode, &mut report, &mut matching_files_seen, @@ -639,6 +506,7 @@ fn scan_path( searcher: &mut Searcher, matcher: &grep_regex::RegexMatcher, path: &Path, + root: &Path, mode: GrepOutputMode, report: &mut GrepReport, matching_files_seen: &mut usize, @@ -648,6 +516,9 @@ fn scan_path( source_bytes_remaining: &mut u64, access: &dyn FsAccessPolicy, ) -> Result { + if logical_path(root, path).is_none() { + return Ok(false); + } let resolved = access .resolve_access_path(path) .map_err(|error| FsError::io(path, error))?; @@ -678,7 +549,7 @@ fn scan_path( return Ok(false); } if *matching_files_seen >= offset { - if !reserve_report_bytes(report, path.to_string_lossy().len().saturating_add(1)) { + if !reserve_report_bytes(report, path.as_os_str().len().saturating_add(1)) { return Ok(true); } report.files.push(path.to_path_buf()); @@ -695,7 +566,7 @@ fn scan_path( return Ok(false); } if *matching_files_seen >= offset { - if !reserve_report_bytes(report, path.to_string_lossy().len().saturating_add(32)) { + if !reserve_report_bytes(report, path.as_os_str().len().saturating_add(32)) { return Ok(true); } report.counts.push((path.to_path_buf(), count)); @@ -780,7 +651,7 @@ impl ContentSink<'_> { fn reserve(&mut self, content_bytes: usize) -> bool { let retained = self .path - .to_string_lossy() + .as_os_str() .len() .saturating_add(content_bytes) .saturating_add(64); diff --git a/crates/fs-operation/src/text.rs b/crates/fs-operation/src/text.rs new file mode 100644 index 000000000..c541ef5b5 --- /dev/null +++ b/crates/fs-operation/src/text.rs @@ -0,0 +1,900 @@ +//! Pure text-operation policy shared by filesystem providers and operation frontends. +//! +//! This module owns no paths, descriptors, version fences, or I/O. Limits count +//! UTF-8 bytes, not characters, and default to unlimited. + +use schemars::JsonSchema; +use serde::{Deserialize, de::DeserializeOwned}; +use thiserror::Error; + +/// The shared text Operation definitions, independent of paths and frontends. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum TextOperation { + Read, + ReadLines, + Edit, + Write, +} + +impl TextOperation { + /// Resolve a lower-case Operation name. A frontend chooses whether its + /// `read` Operation accepts whole-text or line-range arguments. + pub fn from_name(name: &str, line_read: bool) -> Option { + match name { + "read" if line_read => Some(Self::ReadLines), + "read" => Some(Self::Read), + "edit" => Some(Self::Edit), + "write" => Some(Self::Write), + _ => None, + } + } + + pub fn argument_schema(self) -> schemars::Schema { + match self { + Self::Read => schemars::schema_for!(ReadArgs), + Self::ReadLines => schemars::schema_for!(LineReadArgs), + Self::Edit => schemars::schema_for!(EditArgs), + Self::Write => schemars::schema_for!(WriteArgs), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, JsonSchema)] +pub struct EditArgs { + /// String to replace. Must be nonempty and unique unless replace_all is true. + pub old_string: String, + /// Replacement text. Must differ from old_string; may be empty to delete it. + pub new_string: String, + /// Replace every non-overlapping occurrence. Defaults to false. + #[serde(default)] + pub replace_all: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Deserialize, JsonSchema)] +pub struct WriteArgs { + /// Full text to write. + pub content: String, +} + +/// Whole-text reads accept no operation-specific arguments. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct ReadArgs {} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Deserialize, JsonSchema)] +pub struct LineReadArgs { + /// Zero-based line offset. Omitted or null means zero. + #[serde(default)] + pub offset: Option, + /// Maximum lines to return. Omitted or null uses the frontend's default. + #[serde(default)] + pub limit: Option, +} + +impl LineReadArgs { + pub fn range(&self, default_limit: usize) -> (usize, usize) { + ( + self.offset.unwrap_or(0), + self.limit.unwrap_or(default_limit).max(1), + ) + } +} + +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)] +pub struct TextLimits { + pub max_input_bytes: Option, + pub max_output_bytes: Option, + pub max_replacements: Option, +} + +#[derive(Debug, Error)] +pub enum TextError { + #[error("invalid arguments: {0}")] + Decode(#[source] serde_json::Error), + #[error("old_string must not be empty")] + EmptyOldString, + #[error("old_string and new_string are identical")] + IdenticalStrings, + #[error("old_string was not found")] + NotFound, + #[error( + "old_string matched {occurrences} times; set replace_all=true or provide a unique string" + )] + MultipleMatches { occurrences: usize }, + #[error("input size {bytes} exceeds limit {limit}")] + InputTooLarge { bytes: usize, limit: usize }, + #[error("output size {bytes} exceeds limit {limit}")] + OutputTooLarge { bytes: usize, limit: usize }, + #[error("replacement count {replacements} exceeds limit {limit}")] + ReplacementLimitExceeded { replacements: usize, limit: usize }, + #[error("edited content size overflowed")] + Overflow, +} + +pub fn decode(value: serde_json::Value) -> Result { + serde_json::from_value(value).map_err(TextError::Decode) +} + +/// Decode raw Tool JSON directly, retaining duplicate-field rejection rather +/// than losing that information through an intermediate JSON object. +pub fn decode_json(input: &str) -> Result { + serde_json::from_str(input).map_err(TextError::Decode) +} + +impl EditArgs { + /// Validate argument-only policy before a caller acquires a preimage. + pub fn validate(&self, limits: TextLimits) -> Result<(), TextError> { + if self.old_string.is_empty() { + return Err(TextError::EmptyOldString); + } + if self.old_string == self.new_string { + return Err(TextError::IdenticalStrings); + } + check_input_bytes(self.old_string.len(), limits)?; + check_input_bytes(self.new_string.len(), limits)?; + Ok(()) + } +} + +/// A numbered, bounded text response with provider line metadata. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Rendered { + pub body: String, + pub line_count: usize, + pub total_lines: usize, + pub truncated: bool, +} + +const RENDERED_TRUNCATION_MARKER: &str = "\n[truncated at rendered output byte limit]\n"; + +/// Render selected text with one-based line numbers and a hard UTF-8 byte bound. +/// +/// As in the Tool renderer, room for the truncation marker is always reserved, +/// and a partially emitted numbered line counts as a line. Tiny bounds emit +/// only the marker prefix that fits. Line numbers saturate rather than wrap. +pub fn render_numbered( + text: &str, + start_line: usize, + total_lines: usize, + truncated: bool, + max_bytes: usize, +) -> Rendered { + let budget = max_bytes.saturating_sub(RENDERED_TRUNCATION_MARKER.len()); + let mut body = String::with_capacity(text.len().min(max_bytes)); + let mut line_count = 0usize; + let mut output_truncated = false; + for (index, line) in text.lines().enumerate() { + // Build only the small numeric prefix, not an unbounded copy of a line. + let number = start_line.saturating_add(index).saturating_add(1); + let prefix = format!("{number:>6}\t"); + let numbered_len = prefix.len().saturating_add(line.len()).saturating_add(1); + let mut remaining = budget.saturating_sub(body.len()); + if remaining > 0 { + line_count = line_count.saturating_add(1); + } + let line_truncated = numbered_len > remaining; + for fragment in [prefix.as_str(), line, "\n"] { + let mut end = remaining.min(fragment.len()); + while !fragment.is_char_boundary(end) { + end -= 1; + } + body.push_str(&fragment[..end]); + remaining -= end; + // Do not skip a character that failed to fit and then emit later + // fragments; the response must be a prefix of the numbered line. + if end < fragment.len() { + break; + } + } + if line_truncated { + let marker_len = max_bytes + .saturating_sub(body.len()) + .min(RENDERED_TRUNCATION_MARKER.len()); + body.push_str(&RENDERED_TRUNCATION_MARKER[..marker_len]); + output_truncated = true; + break; + } + } + Rendered { + body, + line_count, + total_lines, + truncated: start_line > 0 || truncated || output_truncated, + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EditOutcome { + pub content: String, + pub replacements: usize, + pub bytes_written: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TextOutcome { + pub content: String, + pub bytes: usize, +} + +pub fn edit(original: &str, args: &EditArgs, limits: TextLimits) -> Result { + args.validate(limits)?; + check_input_bytes(original.len(), limits)?; + let occurrences = original.matches(&args.old_string).count(); + if occurrences == 0 { + return Err(TextError::NotFound); + } + if !args.replace_all && occurrences != 1 { + return Err(TextError::MultipleMatches { occurrences }); + } + let replacements = occurrences; + if let Some(limit) = limits.max_replacements + && replacements > limit + { + return Err(TextError::ReplacementLimitExceeded { + replacements, + limit, + }); + } + // All arithmetic and output policy are checked before allocating the result. + let bytes_written = edited_len( + original.len(), + args.old_string.len(), + args.new_string.len(), + replacements, + )?; + check_output_bytes(bytes_written, limits)?; + let mut content = String::with_capacity(bytes_written); + let mut cursor = 0; + for (start, matched) in original.match_indices(&args.old_string) { + content.push_str(&original[cursor..start]); + content.push_str(&args.new_string); + cursor = start + matched.len(); + } + content.push_str(&original[cursor..]); + debug_assert_eq!(content.len(), bytes_written); + Ok(EditOutcome { + content, + replacements, + bytes_written, + }) +} + +fn edited_len( + original: usize, + old: usize, + new: usize, + replacements: usize, +) -> Result { + let removed = old.checked_mul(replacements).ok_or(TextError::Overflow)?; + let added = new.checked_mul(replacements).ok_or(TextError::Overflow)?; + let bytes = original + .checked_sub(removed) + .and_then(|bytes| bytes.checked_add(added)) + .ok_or(TextError::Overflow)?; + // String/Vec capacity is bounded by isize::MAX even on platforms where usize + // arithmetic still fits. Reject that case before String::with_capacity. + if bytes > isize::MAX as usize { + return Err(TextError::Overflow); + } + Ok(bytes) +} + +pub fn write(content: String, limits: TextLimits) -> Result { + text_outcome(content, limits) +} + +pub fn read(content: String, limits: TextLimits) -> Result { + text_outcome(content, limits) +} + +fn text_outcome(content: String, limits: TextLimits) -> Result { + let bytes = content.len(); + check_input_bytes(bytes, limits)?; + check_output_bytes(bytes, limits)?; + Ok(TextOutcome { content, bytes }) +} + +pub(crate) fn check_input_bytes(bytes: usize, limits: TextLimits) -> Result<(), TextError> { + if let Some(limit) = limits.max_input_bytes + && bytes > limit + { + return Err(TextError::InputTooLarge { bytes, limit }); + } + Ok(()) +} + +pub(crate) fn check_output_bytes(bytes: usize, limits: TextLimits) -> Result<(), TextError> { + if let Some(limit) = limits.max_output_bytes + && bytes > limit + { + return Err(TextError::OutputTooLarge { bytes, limit }); + } + Ok(()) +} + +/// Keep valid UTF-8 prefixes intact without changing byte-read behavior on +/// genuinely invalid UTF-8. Providers must remain able to return binary data. +pub(crate) fn bounded_utf8_len(bytes: &[u8], max_bytes: usize) -> usize { + let mut end = max_bytes.min(bytes.len()); + while end > 0 { + match std::str::from_utf8(&bytes[..end]) { + Ok(_) => break, + Err(error) if error.error_len().is_none() => end -= 1, + Err(_) => break, + } + } + end +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + fn args(old: &str, new: &str, replace_all: bool) -> EditArgs { + EditArgs { + old_string: old.into(), + new_string: new.into(), + replace_all, + } + } + + #[test] + fn decode_uses_shared_argument_defaults_and_types() { + let edit = decode::(json!({"old_string": "a", "new_string": "b"})).unwrap(); + assert!(!edit.replace_all); + assert!( + decode::(json!({"old_string": "a", "new_string": "b", "replace_all": true})) + .unwrap() + .replace_all + ); + assert_eq!( + decode::(json!({"content": "😀"})) + .unwrap() + .content, + "😀" + ); + assert_eq!(decode::(json!({})).unwrap(), ReadArgs {}); + for value in [ + json!({"old_string": "a"}), + json!({"old_string": 2, "new_string": "b"}), + json!({"old_string": "a", "new_string": "b", "replace_all": null}), + ] { + assert!(matches!( + decode::(value), + Err(TextError::Decode(_)) + )); + } + assert!(matches!( + decode::(json!({})), + Err(TextError::Decode(_)) + )); + assert!(matches!( + decode::(json!({"offset": 0})), + Err(TextError::Decode(_)) + )); + assert!(matches!( + decode::(json!(null)), + Err(TextError::Decode(_)) + )); + } + + #[test] + fn line_range_preserves_null_defaults_and_minimum_one() { + for value in [json!({}), json!({"offset": null, "limit": null})] { + assert_eq!( + decode::(value).unwrap().range(2000), + (0, 2000) + ); + } + assert_eq!( + decode::(json!({"offset": 7, "limit": 0})) + .unwrap() + .range(99), + (7, 1) + ); + assert_eq!(LineReadArgs::default().range(0), (0, 1)); + assert_eq!( + LineReadArgs { + offset: Some(usize::MAX), + limit: Some(usize::MAX) + } + .range(1), + (usize::MAX, usize::MAX) + ); + for value in [ + json!({"offset": -1}), + json!({"limit": "2"}), + json!({"limit": 1.5}), + ] { + assert!(matches!( + decode::(value), + Err(TextError::Decode(_)) + )); + } + } + + #[test] + fn schema_is_the_argument_contract() { + let edit = serde_json::to_value(schemars::schema_for!(EditArgs)).unwrap(); + assert_eq!(edit["required"], json!(["old_string", "new_string"])); + assert_eq!(edit["properties"]["replace_all"]["default"], json!(false)); + let read = serde_json::to_value(schemars::schema_for!(ReadArgs)).unwrap(); + assert_eq!(read["additionalProperties"], json!(false)); + assert!( + read.get("properties") + .is_none_or(|properties| properties == &json!({})) + ); + let lines = serde_json::to_value(schemars::schema_for!(LineReadArgs)).unwrap(); + assert_eq!( + lines["properties"]["offset"]["type"], + json!(["integer", "null"]) + ); + assert_eq!(lines["properties"]["limit"]["default"], json!(null)); + } + + #[test] + fn edit_rejects_empty_identical_missing_and_ambiguous_matches() { + assert!(matches!( + args("", "x", true).validate(TextLimits::default()), + Err(TextError::EmptyOldString) + )); + assert!(matches!( + edit("a", &args("a", "a", false), TextLimits::default()), + Err(TextError::IdenticalStrings) + )); + assert!(matches!( + edit("a", &args("b", "c", false), TextLimits::default()), + Err(TextError::NotFound) + )); + assert!(matches!( + edit("aa", &args("a", "b", false), TextLimits::default()), + Err(TextError::MultipleMatches { occurrences: 2 }) + )); + assert!(matches!( + edit("", &args("a", "", false), TextLimits::default()), + Err(TextError::NotFound) + )); + } + + #[test] + fn edit_handles_unique_all_deletion_literal_and_nonoverlapping_matches() { + for (original, old, new, all, expected, replacements) in [ + ( + "before old after", + "old", + "new", + false, + "before new after", + 1, + ), + ("aaa", "aa", "x", true, "xa", 1), + ("a-a-a", "a", "", true, "--", 3), + ("[a].* [a].*", "[a].*", "$1", true, "$1 $1", 2), + ("x", "x", "", false, "", 1), + ] { + let result = edit(original, &args(old, new, all), TextLimits::default()).unwrap(); + assert_eq!(result.content, expected); + assert_eq!(result.replacements, replacements); + assert_eq!(result.bytes_written, expected.len()); + } + } + + #[test] + fn edit_preserves_unicode_and_counts_bytes() { + let result = edit("é😀\né😀", &args("é😀", "界", true), TextLimits::default()).unwrap(); + assert_eq!(result.content, "界\n界"); + assert_eq!(result.replacements, 2); + assert_eq!(result.bytes_written, 7); + // Matching is literal, without Unicode normalization. + assert!(matches!( + edit("e\u{301}", &args("é", "x", false), TextLimits::default()), + Err(TextError::NotFound) + )); + } + + #[test] + fn edit_input_limits_cover_arguments_and_preimage_with_inclusive_boundaries() { + let limits = TextLimits { + max_input_bytes: Some(4), + ..Default::default() + }; + args("😀", "a", false).validate(limits).unwrap(); + assert!(matches!( + args("😀x", "a", false).validate(limits), + Err(TextError::InputTooLarge { bytes: 5, limit: 4 }) + )); + assert!(matches!( + args("a", "😀x", false).validate(limits), + Err(TextError::InputTooLarge { bytes: 5, limit: 4 }) + )); + assert!(matches!( + edit("aaaaa", &args("a", "b", true), limits), + Err(TextError::InputTooLarge { bytes: 5, limit: 4 }) + )); + assert_eq!( + edit("aaaa", &args("a", "b", true), limits).unwrap().content, + "bbbb" + ); + } + + #[test] + fn edit_output_and_replacement_limits_apply_before_constructing_result() { + let args = args("a", "😀", true); + let limits = TextLimits { + max_output_bytes: Some(7), + ..Default::default() + }; + assert!(matches!( + edit("aa", &args, limits), + Err(TextError::OutputTooLarge { bytes: 8, limit: 7 }) + )); + assert_eq!( + edit( + "aa", + &args, + TextLimits { + max_output_bytes: Some(8), + ..limits + } + ) + .unwrap() + .bytes_written, + 8 + ); + assert!(matches!( + edit( + "aa", + &args, + TextLimits { + max_replacements: Some(1), + ..Default::default() + } + ), + Err(TextError::ReplacementLimitExceeded { + replacements: 2, + limit: 1 + }) + )); + assert!(matches!( + edit( + "a", + &args, + TextLimits { + max_replacements: Some(0), + ..Default::default() + } + ), + Err(TextError::ReplacementLimitExceeded { + replacements: 1, + limit: 0 + }) + )); + assert_eq!( + edit( + "aa", + &args, + TextLimits { + max_replacements: Some(2), + ..Default::default() + } + ) + .unwrap() + .replacements, + 2 + ); + assert_eq!( + edit( + "a", + &EditArgs { + old_string: "a".into(), + new_string: String::new(), + replace_all: false + }, + TextLimits { + max_output_bytes: Some(0), + ..Default::default() + } + ) + .unwrap() + .content, + "" + ); + } + + #[test] + fn raw_json_decode_preserves_duplicate_argument_rejection() { + for input in [ + r#"{"old_string":"a","old_string":"b","new_string":"c"}"#, + r#"{"old_string":"a","new_string":"b","replace_all":false,"replace_all":true}"#, + ] { + assert!(matches!( + decode_json::(input), + Err(TextError::Decode(_)) + )); + } + assert!(matches!( + decode_json::(r#"{"content":"a","content":"b"}"#), + Err(TextError::Decode(_)) + )); + } + + #[test] + fn size_arithmetic_rejects_every_overflow_before_preallocation() { + // Exercise impossible-to-allocate sizes through the exact helper edit + // invokes before allocating; no giant strings need to be constructed. + for (original, old, new, count) in [ + (usize::MAX, usize::MAX, 0, 2), // removed multiplication + (2, 1, usize::MAX, 2), // added multiplication + (1, 2, 0, 1), // subtraction + (usize::MAX, 1, 2, 1), // addition + (1, 1, isize::MAX as usize + 1, 1), // allocation capacity + ] { + assert!(matches!( + edited_len(original, old, new, count), + Err(TextError::Overflow) + )); + } + assert_eq!( + edited_len(1, 1, isize::MAX as usize, 1).unwrap(), + isize::MAX as usize + ); + assert_eq!(edited_len(8, 2, 1, 3).unwrap(), 5); + } + + #[test] + fn read_and_write_preserve_whole_text_and_enforce_both_byte_limits() { + for operation in [read, write] { + assert_eq!( + operation(String::new(), TextLimits::default()).unwrap(), + TextOutcome { + content: String::new(), + bytes: 0 + } + ); + let content = "é\r\n😀\n"; + let limits = TextLimits { + max_input_bytes: Some(content.len()), + max_output_bytes: Some(content.len()), + max_replacements: Some(0), + }; + let result = operation(content.into(), limits).unwrap(); + assert_eq!(result.content, content); + assert_eq!(result.bytes, content.len()); + assert!(matches!( + operation( + "😀".into(), + TextLimits { + max_input_bytes: Some(3), + ..Default::default() + } + ), + Err(TextError::InputTooLarge { bytes: 4, limit: 3 }) + )); + assert!(matches!( + operation( + "😀".into(), + TextLimits { + max_output_bytes: Some(3), + ..Default::default() + } + ), + Err(TextError::OutputTooLarge { bytes: 4, limit: 3 }) + )); + assert!(matches!( + operation( + "a".into(), + TextLimits { + max_output_bytes: Some(0), + ..Default::default() + } + ), + Err(TextError::OutputTooLarge { bytes: 1, limit: 0 }) + )); + } + } + + #[test] + fn utf8_byte_bounds_preserve_binary_compatibility() { + for (limit, expected) in [ + (0, 0), + (1, 1), + (2, 1), + (3, 1), + (4, 1), + (5, 5), + (usize::MAX, 6), + ] { + assert_eq!(bounded_utf8_len("a😀z".as_bytes(), limit), expected); + } + assert_eq!(bounded_utf8_len(&[0xff, 0xfe, 0x00], 2), 2); + } + + #[test] + fn operation_names_and_schemas_are_shared_authority() { + assert_eq!( + TextOperation::from_name("read", false), + Some(TextOperation::Read) + ); + assert_eq!( + TextOperation::from_name("read", true), + Some(TextOperation::ReadLines) + ); + for line_read in [false, true] { + assert_eq!( + TextOperation::from_name("edit", line_read), + Some(TextOperation::Edit) + ); + assert_eq!( + TextOperation::from_name("write", line_read), + Some(TextOperation::Write) + ); + for name in ["Read", "WRITE", "read_lines", "stat", "", " read"] { + assert_eq!(TextOperation::from_name(name, line_read), None); + } + } + for (operation, expected) in [ + (TextOperation::Read, schemars::schema_for!(ReadArgs)), + ( + TextOperation::ReadLines, + schemars::schema_for!(LineReadArgs), + ), + (TextOperation::Edit, schemars::schema_for!(EditArgs)), + (TextOperation::Write, schemars::schema_for!(WriteArgs)), + ] { + assert_eq!(operation.argument_schema(), expected); + } + let schema = serde_json::to_value(TextOperation::Edit.argument_schema()).unwrap(); + assert!( + schema["properties"]["new_string"]["description"] + .as_str() + .unwrap() + .contains("Must differ") + ); + } + + #[test] + fn renderer_preserves_numbering_lines_and_provider_metadata() { + let result = render_numbered("alpha\r\n\nβeta\n", 0, 3, false, 1024); + assert_eq!(result.body, " 1\talpha\n 2\t\n 3\tβeta\n"); + assert_eq!(result.line_count, 3); + assert_eq!(result.total_lines, 3); + assert!(!result.truncated); + let result = render_numbered("last", 9, 10, false, 1024); + assert_eq!(result.body, " 10\tlast\n"); + assert!(result.truncated); + assert!(render_numbered("last", 0, 10, true, 1024).truncated); + assert_eq!( + render_numbered("", 0, 0, false, 0), + Rendered { + body: String::new(), + line_count: 0, + total_lines: 0, + truncated: false, + } + ); + assert!(render_numbered("", 1, 1, false, 0).truncated); + } + + #[test] + fn renderer_reserves_marker_and_counts_partial_lines_as_before() { + let marker = RENDERED_TRUNCATION_MARKER; + let result = render_numbered("a\nb", 0, 2, false, marker.len() + 9); + assert_eq!(result.body, format!(" 1\ta\n{marker}")); + assert_eq!(result.line_count, 1); + assert_eq!(result.total_lines, 2); + assert!(result.truncated); + let result = render_numbered("a", 0, 1, false, marker.len() + 1); + assert_eq!(result.body, format!(" {marker}")); + assert_eq!(result.line_count, 1); + assert!(result.truncated); + let result = render_numbered("a", 0, 1, false, marker.len() + 9); + assert_eq!(result.body, " 1\ta\n"); + assert_eq!(result.line_count, 1); + assert!(!result.truncated); + } + + #[test] + fn renderer_tiny_bounds_unicode_and_large_offsets_never_overflow() { + let marker = RENDERED_TRUNCATION_MARKER; + for max_bytes in 0..=marker.len() { + let result = render_numbered("😀", 0, 1, false, max_bytes); + assert_eq!(result.body, marker[..max_bytes]); + assert_eq!(result.line_count, 0); + assert!(result.truncated); + } + for extra in 1..=4 { + let result = render_numbered("😀😀", 0, 1, false, marker.len() + 7 + extra); + let expected = if extra < 4 { + " 1\t" + } else { + " 1\t😀" + }; + assert_eq!(result.body, format!("{expected}{marker}")); + assert_eq!(result.line_count, 1); + assert!(result.body.len() <= marker.len() + 7 + extra); + assert!(result.truncated); + } + let result = render_numbered("a\nb", usize::MAX, usize::MAX, false, usize::MAX); + assert_eq!( + result.body, + format!("{}\ta\n{}\tb\n", usize::MAX, usize::MAX) + ); + assert_eq!(result.line_count, 2); + assert_eq!(result.total_lines, usize::MAX); + assert!(result.truncated); + } + + #[test] + fn renderer_matches_original_behavior_for_ordinary_bounds() { + fn original( + text: &str, + start_line: usize, + total_lines: usize, + truncated: bool, + max_bytes: usize, + ) -> Rendered { + let marker = RENDERED_TRUNCATION_MARKER; + let budget = max_bytes - marker.len(); + let mut body = String::with_capacity(text.len().min(max_bytes)); + let mut line_count = 0; + let mut output_truncated = false; + for (index, line) in text.lines().enumerate() { + let numbered = format!("{:>6}\t{}\n", start_line + index + 1, line); + let remaining = budget.saturating_sub(body.len()); + if remaining > 0 { + line_count += 1; + } + if numbered.len() > remaining { + let mut end = remaining; + while !numbered.is_char_boundary(end) { + end -= 1; + } + body.push_str(&numbered[..end]); + body.push_str(marker); + output_truncated = true; + break; + } + body.push_str(&numbered); + } + Rendered { + body, + line_count, + total_lines, + truncated: start_line > 0 || truncated || output_truncated, + } + } + for text in ["", "a", "\n", "😀é界\nsecond\n", "first\r\n\r\nlast"] { + for start in [0, 1, 999_999] { + for bound in RENDERED_TRUNCATION_MARKER.len()..=160 { + let result = render_numbered(text, start, 99, false, bound); + assert_eq!(result, original(text, start, 99, false, bound)); + assert!(result.body.len() <= bound); + } + } + } + } + + #[test] + fn errors_keep_actionable_messages_and_decode_source() { + assert_eq!( + TextError::EmptyOldString.to_string(), + "old_string must not be empty" + ); + assert_eq!( + TextError::IdenticalStrings.to_string(), + "old_string and new_string are identical" + ); + assert_eq!(TextError::NotFound.to_string(), "old_string was not found"); + assert_eq!( + TextError::MultipleMatches { occurrences: 3 }.to_string(), + "old_string matched 3 times; set replace_all=true or provide a unique string" + ); + assert_eq!( + TextError::Overflow.to_string(), + "edited content size overflowed" + ); + let error = decode::(json!({"unknown": true})).unwrap_err(); + assert!(std::error::Error::source(&error).is_some()); + } +} diff --git a/crates/fs-operation/src/traversal_tests.rs b/crates/fs-operation/src/traversal_tests.rs new file mode 100644 index 000000000..41cc8cddf --- /dev/null +++ b/crates/fs-operation/src/traversal_tests.rs @@ -0,0 +1,108 @@ +#![cfg(target_os = "linux")] +use crate::*; +use std::path::{Path, PathBuf}; +use std::sync::{Arc, Mutex}; +struct Filtered { + root: PathBuf, + seen: Arc>>, +} +impl FsAccessPolicy for Filtered { + fn is_readable(&self, path: &Path) -> bool { + path.starts_with(&self.root) && !path.starts_with(self.root.join("search/denied")) + } + fn is_writable(&self, _: &Path) -> bool { + false + } + fn open_traversal_root( + &self, + _: &Path, + resolved: &Path, + ) -> std::io::Result> { + use std::os::fd::AsRawFd; + let file = std::fs::File::open(resolved)?; + let walk = PathBuf::from(format!("/proc/self/fd/{}", file.as_raw_fd())); + Ok(Some(FsTraversalRoot::new_at( + walk, + file, + resolved.to_path_buf(), + ))) + } + fn traversal_filter(&self) -> Option { + let denied = self.root.join("search/denied"); + let seen = self.seen.clone(); + Some(Arc::new(move |path, _| { + seen.lock().unwrap().push(path.to_path_buf()); + !path.starts_with(&denied) + })) + } + fn open_read_file(&self, _: &Path, resolved: &Path) -> std::io::Result { + assert!( + !resolved.starts_with(self.root.join("search/denied")), + "denied source opened" + ); + std::fs::File::open(resolved) + } +} +#[test] +fn descriptor_search_anchor_and_predicate_prune_before_denied_directory_descent() { + let root = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(root.path().join("search/denied/deep")).unwrap(); + std::fs::write(root.path().join("search/allowed.txt"), "needle allowed").unwrap(); + std::fs::write( + root.path().join("search/denied/deep/secret.txt"), + "needle secret", + ) + .unwrap(); + std::fs::write(root.path().join("outside.txt"), "needle outside").unwrap(); + let policy = Filtered { + root: root.path().to_path_buf(), + seen: Arc::new(Mutex::new(Vec::new())), + }; + let glob = run_glob( + root.path(), + &root.path().join("search"), + GlobRequest { + path: FsPath::new("search").unwrap(), + pattern: "**/*.txt".into(), + limit: 100, + }, + &policy, + ) + .unwrap(); + assert_eq!(glob.paths, vec![FsPath::new("search/allowed.txt").unwrap()]); + let grep = run_grep( + root.path(), + root.path().join("search"), + GrepRequest { + path: FsPath::new("search").unwrap(), + pattern: "needle".into(), + glob: None, + file_type: None, + case_insensitive: false, + before_context: 0, + after_context: 0, + multiline: false, + output_mode: GrepOutputMode::Content, + offset: 0, + limit: 100, + }, + &policy, + ) + .unwrap(); + assert_eq!(grep.paths, vec![FsPath::new("search/allowed.txt").unwrap()]); + let seen = policy.seen.lock().unwrap(); + assert!( + seen.iter() + .any(|path| path == &root.path().join("search/denied")) + ); + assert!( + !seen + .iter() + .any(|path| path.starts_with(root.path().join("search/denied/deep"))) + ); + assert!( + !seen + .iter() + .any(|path| path == &root.path().join("outside.txt")) + ); +} diff --git a/crates/fs-operation/src/walk.rs b/crates/fs-operation/src/walk.rs new file mode 100644 index 000000000..db5717308 --- /dev/null +++ b/crates/fs-operation/src/walk.rs @@ -0,0 +1,233 @@ +//! Bounded descriptor-provider walk shared by Glob and Grep. Unlike a +//! WalkBuilder predicate, this iterator counts even rejected entries and checks +//! cancellation while scanning them. Denied directories are never enumerated. +use crate::{FsAccessPolicy, FsError, FsTraversalFilter}; +use std::path::{Path, PathBuf}; + +pub(crate) struct Entry { + pub path: PathBuf, + pub kind: Option, +} +pub(crate) enum Walk<'a> { + Plain(ignore::Walk), + Descriptor(DescriptorWalk<'a>), +} +impl<'a> Walk<'a> { + /// A rejected directory has been yielded but not enumerated yet. Keep + /// ignore pruning inside the same bounded descriptor walk as scope pruning. + pub fn skip_current_dir(&mut self) { + if let Self::Descriptor(walk) = self { + walk.pending = None; + } + } + pub fn descriptor(root: &Path, access: &'a dyn FsAccessPolicy) -> Self { + Self::Descriptor(DescriptorWalk { + access, + filter: access.traversal_filter(), + pending: Some((root.to_path_buf(), 0)), + root: Some(root.to_path_buf()), + stack: Vec::new(), + visited: 0, + failed: false, + }) + } +} +impl Iterator for Walk<'_> { + type Item = Result; + fn next(&mut self) -> Option { + match self { + Self::Plain(walk) => loop { + match walk.next()? { + Ok(entry) => { + return Some(Ok(Entry { + path: entry.path().to_path_buf(), + kind: entry.file_type(), + })); + } + Err(_) => continue, + } + }, + Self::Descriptor(walk) => walk.next(), + } + } +} +pub(crate) struct DescriptorWalk<'a> { + access: &'a dyn FsAccessPolicy, + filter: Option, + root: Option, + pending: Option<(PathBuf, usize)>, + stack: Vec<(PathBuf, usize, std::fs::ReadDir)>, + visited: usize, + failed: bool, +} +impl DescriptorWalk<'_> { + fn next_entry(&mut self) -> Result, FsError> { + if let Some(root) = self.root.take() { + self.access + .check_cancelled() + .map_err(|e| FsError::io(&root, e))?; + let resolved = self + .access + .resolve_access_path(&root) + .map_err(|e| FsError::io(&root, e))?; + let metadata = self + .access + .read_metadata(&root, &resolved) + .map_err(|e| FsError::io(&root, e))?; + return Ok(Some(Entry { + path: root, + kind: Some(metadata.file_type()), + })); + } + loop { + if let Some((path, depth)) = self.pending.take() { + self.access + .check_cancelled() + .map_err(|e| FsError::io(&path, e))?; + let selected = self + .filter + .as_ref() + .is_none_or(|filter| filter(&path, true)); + let resolved = self + .access + .resolve_access_path(&path) + .map_err(|e| FsError::io(&path, e))?; + if selected && self.access.can_enumerate_directory(&path, &resolved) { + if depth >= 128 { + return Err(FsError::InvalidArgument( + "descriptor traversal depth exceeds provider limit 128".into(), + )); + } + let entries = self + .access + .open_read_dir(&path, &resolved) + .map_err(|e| FsError::io(&path, e))?; + self.stack.push((path, depth, entries)); + } + } + let Some((parent, depth, entries)) = self.stack.last_mut() else { + return Ok(None); + }; + self.access + .check_cancelled() + .map_err(|e| FsError::io(&*parent, e))?; + let Some(entry) = entries.next() else { + self.stack.pop(); + continue; + }; + self.visited = self.visited.saturating_add(1); + if self.visited > crate::MAX_TRAVERSAL_ENTRIES { + return Err(FsError::InvalidArgument( + "descriptor traversal exceeds provider entry limit".into(), + )); + } + let entry = entry.map_err(|e| FsError::io(&*parent, e))?; + if entry.file_name().to_str().is_none() { + continue; + } + let path = parent.join(entry.file_name()); + let kind = entry.file_type().map_err(|e| FsError::io(&path, e))?; + let selected = self + .filter + .as_ref() + .is_none_or(|filter| filter(&path, kind.is_dir())); + if !selected { + continue; + } + let resolved = match self.access.resolve_access_path(&path) { + Ok(path) => path, + Err(_) => continue, + }; + if !self.access.is_readable_paths(&path, &resolved) { + continue; + } + if kind.is_dir() { + self.pending = Some((path.clone(), depth.saturating_add(1))); + } + return Ok(Some(Entry { + path, + kind: Some(kind), + })); + } + } +} +impl Iterator for DescriptorWalk<'_> { + type Item = Result; + fn next(&mut self) -> Option { + if self.failed { + return None; + } + match self.next_entry() { + Ok(entry) => entry.map(Ok), + Err(error) => { + self.failed = true; + Some(Err(error)) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::sync::{ + Arc, + atomic::{AtomicUsize, Ordering}, + }; + struct RejectFiles { + calls: AtomicUsize, + cancel_at: usize, + } + impl FsAccessPolicy for RejectFiles { + fn is_readable(&self, _: &Path) -> bool { + true + } + fn is_writable(&self, _: &Path) -> bool { + false + } + fn traversal_filter(&self) -> Option { + Some(Arc::new(|_, directory| directory)) + } + fn check_cancelled(&self) -> std::io::Result<()> { + if self.calls.fetch_add(1, Ordering::Relaxed) >= self.cancel_at { + Err(std::io::Error::from(std::io::ErrorKind::Interrupted)) + } else { + Ok(()) + } + } + } + #[test] + fn descriptor_walk_counts_rejected_entries_toward_the_entry_budget() { + let root = tempfile::tempdir().unwrap(); + std::fs::write(root.path().join("rejected"), "must not be opened").unwrap(); + let policy = RejectFiles { + calls: AtomicUsize::new(0), + cancel_at: usize::MAX, + }; + let Walk::Descriptor(mut walk) = Walk::descriptor(root.path(), &policy) else { + panic!() + }; + assert!(walk.next().unwrap().is_ok()); + walk.visited = crate::MAX_TRAVERSAL_ENTRIES; + assert!(matches!( + walk.next().unwrap(), + Err(FsError::InvalidArgument(_)) + )); + } + #[test] + fn descriptor_walk_checks_cancellation_while_scanning_only_rejected_entries() { + let root = tempfile::tempdir().unwrap(); + for index in 0..20 { + std::fs::write(root.path().join(index.to_string()), "must not be opened").unwrap(); + } + let policy = RejectFiles { + calls: AtomicUsize::new(0), + cancel_at: 4, + }; + let mut walk = Walk::descriptor(root.path(), &policy); + assert!(walk.next().unwrap().is_ok()); + assert!( + matches!(walk.next().unwrap(), Err(FsError::Io { source, .. }) if source.kind() == std::io::ErrorKind::Interrupted) + ); + } +} diff --git a/crates/manifest/README.md b/crates/manifest/README.md index 7832b9ead..82ccd0aa7 100644 --- a/crates/manifest/README.md +++ b/crates/manifest/README.md @@ -25,6 +25,8 @@ Does not own: Profiles are reusable recipes; resolved Manifests are runtime contracts. Keep runtime-bound fields such as Worker name, concrete delegated scope, sockets, session pointers, and raw secrets out of reusable Profiles. +`feature.workdir_catalog.enabled` defaults to `true` for read-only Repository/Workdir/attachment reference discovery in WIP mode. It is independent of `feature.manage_workdir` (which defaults disabled), registers no normal Tools, and does not grant filesystem, attachment, or Workspace authority. Permissions remain independent. Standalone Tools-mode profiles may retain this default; only WIP installation consumes the flag. Set it to `false` to opt out. + ## See also - [`../../docs/design/profiles-manifests-prompts.md`](../../docs/design/profiles-manifests-prompts.md) diff --git a/crates/manifest/src/config.rs b/crates/manifest/src/config.rs index 910eacfe1..7407b8b0b 100644 --- a/crates/manifest/src/config.rs +++ b/crates/manifest/src/config.rs @@ -51,8 +51,9 @@ pub struct WorkerManifestConfig { /// is disabled; `Some` requires `default_action` during final resolve. #[serde(default)] pub permissions: Option, - /// Explicit built-in feature/tool-surface enablement. Absent flags resolve - /// disabled after cascade merge. + /// Built-in feature/tool-surface enablement. Absent tool flags resolve + /// disabled after cascade merge; the read-only WIP workdir catalog defaults + /// enabled without granting authority. #[serde(default)] pub feature: FeatureConfigPartial, /// Explicit Model Context Protocol provider declarations. Config parsing @@ -95,6 +96,10 @@ pub struct FeatureConfigPartial { #[serde(default)] pub manage_workdir: Option, #[serde(default)] + pub workdir_catalog: Option, + #[serde(default)] + pub workspace_config: Option, + #[serde(default)] pub ticket: Option, #[serde(default)] pub merge_request: Option, @@ -136,6 +141,16 @@ impl FeatureConfigPartial { other.manage_workdir, FeatureFlagConfigPartial::merge, ), + workdir_catalog: merge_option( + self.workdir_catalog, + other.workdir_catalog, + FeatureFlagConfigPartial::merge, + ), + workspace_config: merge_option( + self.workspace_config, + other.workspace_config, + WorkspaceConfigFeatureConfigPartial::merge, + ), ticket: merge_option(self.ticket, other.ticket, TicketFeatureConfigPartial::merge), merge_request: merge_option( self.merge_request, @@ -165,6 +180,38 @@ impl FeatureFlagConfigPartial { } } +/// Activation only: no Workspace/grant/source/permission fields are accepted. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WorkspaceConfigFeatureConfigPartial { + #[serde(default)] + pub enabled: Option, +} + +impl WorkspaceConfigFeatureConfigPartial { + fn merge(self, other: Self) -> Self { + Self { + enabled: other.enabled.or(self.enabled), + } + } +} + +impl From for FeatureFlagConfig { + fn from(value: WorkspaceConfigFeatureConfigPartial) -> Self { + Self { + enabled: value.enabled.unwrap_or_default(), + } + } +} + +impl From for WorkspaceConfigFeatureConfigPartial { + fn from(value: FeatureFlagConfig) -> Self { + Self { + enabled: Some(value.enabled), + } + } +} + #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct WorkerFeatureConfigPartial { #[serde(default)] @@ -241,6 +288,8 @@ pub struct SubjektivFeatureConfigPartial { #[serde(default)] pub enabled: Option, #[serde(default)] + pub consolidation_tools: Option, + #[serde(default)] pub extraction: Option, } @@ -248,6 +297,7 @@ impl SubjektivFeatureConfigPartial { fn merge(self, other: Self) -> Self { Self { enabled: other.enabled.or(self.enabled), + consolidation_tools: other.consolidation_tools.or(self.consolidation_tools), extraction: merge_option( self.extraction, other.extraction, @@ -388,6 +438,16 @@ impl From for FeatureConfig { .manage_workdir .map(FeatureFlagConfig::from) .unwrap_or_default(), + workdir_catalog: FeatureFlagConfig { + enabled: value + .workdir_catalog + .and_then(|flag| flag.enabled) + .unwrap_or(defaults::WORKDIR_CATALOG_ENABLED), + }, + workspace_config: value + .workspace_config + .map(FeatureFlagConfig::from) + .unwrap_or_default(), ticket: value .ticket .map(TicketFeatureConfig::from) @@ -487,6 +547,7 @@ impl From for ResolvedSubjektivFeatureConfig { Self { profile: SubjektivFeatureProfileConfig { enabled: value.enabled.unwrap_or_default(), + consolidation_tools: value.consolidation_tools.unwrap_or_default(), extraction: MemoryExtractionProfileConfig { enabled: extraction.enabled.unwrap_or(true), model: extraction.model, @@ -506,6 +567,7 @@ impl From for SubjektivFeatureConfigPartial { fn from(value: ResolvedSubjektivFeatureConfig) -> Self { Self { enabled: Some(value.profile.enabled), + consolidation_tools: Some(value.profile.consolidation_tools), extraction: Some(MemoryExtractionProfileConfigPartial { enabled: Some(value.profile.extraction.enabled), model: value.profile.extraction.model, @@ -579,6 +641,8 @@ impl From for FeatureConfigPartial { workspace_worker_discovery: Some(value.workspace_worker_discovery.into()), objective: Some(value.objective.into()), manage_workdir: Some(value.manage_workdir.into()), + workdir_catalog: Some(value.workdir_catalog.into()), + workspace_config: Some(value.workspace_config.into()), ticket: Some(value.ticket.into()), merge_request: Some(value.merge_request.into()), orchestration: Some(value.orchestration.into()), @@ -766,7 +830,8 @@ impl WorkerManifestConfig { /// Base config populated with the in-code per-field defaults listed in /// [`crate::defaults`]. This is not a selectable Profile and does not - /// enable a launch capability surface. Profile and one-file Manifest + /// enable launch authority or normal Tools. The read-only WIP workdir + /// catalog defaults enabled independently. Profile and one-file Manifest /// resolvers start from this layer so every per-field default lives at /// exactly one call site (the `defaults` module). /// @@ -2232,7 +2297,7 @@ worker_max_turns = 7 } #[test] - fn feature_flags_default_disabled_in_resolved_manifest() { + fn tool_feature_flags_default_disabled_except_read_only_workdir_catalog() { let manifest: WorkerManifest = minimal_valid().try_into().unwrap(); assert!(!manifest.feature.task.enabled); assert!(!manifest.feature.memory.profile.enabled); @@ -2240,10 +2305,213 @@ worker_max_turns = 7 assert!(!manifest.feature.sub_worker.enabled); assert!(!manifest.feature.objective.enabled); assert!(!manifest.feature.manage_workdir.enabled); + assert!(manifest.feature.workdir_catalog.enabled); + assert!(!manifest.feature.workspace_config.enabled); assert!(!manifest.feature.ticket.enabled); assert!(!manifest.feature.merge_request.any()); } + #[test] + fn workspace_config_defaults_disabled_for_partial_and_resolved_configs() { + assert!(!FeatureConfig::default().workspace_config.enabled); + assert!(FeatureConfigPartial::default().workspace_config.is_none()); + for source in ["", "[feature]\n", "[feature.workspace_config]\n"] { + let partial = WorkerManifestConfig::from_toml(source).unwrap(); + assert!( + !FeatureConfig::from(partial.feature) + .workspace_config + .enabled + ); + } + for source in ["{}", r#"{"workspace_config": {}}"#] { + let feature: FeatureConfig = serde_json::from_str(source).unwrap(); + assert!(!feature.workspace_config.enabled); + } + } + + #[test] + fn workspace_config_enabled_only_merges_and_roundtrips_snapshots() { + for enabled in [false, true] { + let lower = WorkerManifestConfig::from_toml(&format!( + "[feature.workspace_config]\nenabled = {enabled}\n" + )) + .unwrap(); + let empty = WorkerManifestConfig::from_toml("[feature.workspace_config]\n").unwrap(); + let manifest: WorkerManifest = minimal_valid() + .merge(lower.clone()) + .merge(empty) + .try_into() + .unwrap(); + assert_eq!(manifest.feature.workspace_config.enabled, enabled); + assert!(!manifest.feature.manage_workdir.enabled); + let partial = FeatureConfigPartial::from(manifest.feature.clone()); + assert_eq!( + partial.workspace_config.as_ref().unwrap().enabled, + Some(enabled) + ); + let serialized = toml::to_string(&partial).unwrap(); + let restored: FeatureConfigPartial = toml::from_str(&serialized).unwrap(); + assert_eq!(FeatureConfig::from(restored), manifest.feature); + let json = serde_json::to_value(&manifest.feature).unwrap(); + assert_eq!( + json["workspace_config"], + serde_json::json!({"enabled": enabled}) + ); + assert_eq!( + serde_json::from_value::(json).unwrap(), + manifest.feature + ); + let snapshot = crate::write_persisted_worker_manifest_snapshot(&manifest).unwrap(); + assert_eq!( + crate::read_persisted_worker_manifest_snapshot(snapshot.clone()) + .unwrap() + .feature + .workspace_config + .enabled, + enabled + ); + let mut old_snapshot = snapshot; + old_snapshot["manifest"]["feature"] + .as_object_mut() + .unwrap() + .remove("workspace_config"); + assert!( + !crate::read_persisted_worker_manifest_snapshot(old_snapshot) + .unwrap() + .feature + .workspace_config + .enabled + ); + let upper = WorkerManifestConfig::from_toml(&format!( + "[feature.workspace_config]\nenabled = {}\n", + !enabled + )) + .unwrap(); + let manifest: WorkerManifest = minimal_valid() + .merge(lower) + .merge(upper) + .try_into() + .unwrap(); + assert_eq!(manifest.feature.workspace_config.enabled, !enabled); + } + } + + #[test] + fn workspace_config_rejects_manifest_authority_fields() { + for (field, value) in [ + ("grant_id", "\"grant-1\""), + ("workspace_id", "\"workspace-1\""), + ("access", "\"read_write\""), + ("config_root", "\"/host/config\""), + ("source", "\"config\""), + ] { + let source = format!("[feature.workspace_config]\nenabled = true\n{field} = {value}\n"); + assert!(WorkerManifestConfig::from_toml(&source).is_err(), "{field}"); + assert!( + toml::from_str::(&format!( + "[workspace_config]\nenabled = true\n{field} = {value}\n" + )) + .is_err(), + "{field}" + ); + } + } + + #[test] + fn workdir_catalog_defaults_agree_for_partial_and_resolved_configs() { + assert!(FeatureConfig::default().workdir_catalog.enabled); + assert!(FeatureConfigPartial::default().workdir_catalog.is_none()); + for source in ["", "[feature]\n", "[feature.workdir_catalog]\n"] { + let partial = WorkerManifestConfig::from_toml(source).unwrap(); + let feature = FeatureConfig::from(partial.feature); + assert!(feature.workdir_catalog.enabled, "{source}"); + let value: toml::Value = toml::from_str(source).unwrap(); + let resolved: FeatureConfig = serde_json::from_value( + serde_json::to_value( + value + .get("feature") + .cloned() + .unwrap_or_else(|| toml::Value::Table(Default::default())), + ) + .unwrap(), + ) + .unwrap(); + assert_eq!(feature, resolved); + } + for source in ["{}", r#"{"workdir_catalog": {}}"#] { + let feature: FeatureConfig = serde_json::from_str(source).unwrap(); + assert!(feature.workdir_catalog.enabled); + assert!(!feature.manage_workdir.enabled); + } + } + + #[test] + fn workdir_catalog_is_independent_and_merges_partial_layers() { + for catalog in [false, true] { + for manage in [false, true] { + let source = format!( + "[feature.workdir_catalog]\nenabled = {catalog}\n\ + [feature.manage_workdir]\nenabled = {manage}\n" + ); + let lower = WorkerManifestConfig::from_toml(&source).unwrap(); + let empty = WorkerManifestConfig::from_toml("[feature.workdir_catalog]").unwrap(); + let cfg = minimal_valid().merge(lower.clone()).merge(empty); + let manifest: WorkerManifest = cfg.try_into().unwrap(); + assert_eq!(manifest.feature.workdir_catalog.enabled, catalog); + assert_eq!(manifest.feature.manage_workdir.enabled, manage); + + let override_source = + format!("[feature.workdir_catalog]\nenabled = {}\n", !catalog); + let upper = WorkerManifestConfig::from_toml(&override_source).unwrap(); + let manifest: WorkerManifest = minimal_valid() + .merge(lower) + .merge(upper) + .try_into() + .unwrap(); + assert_eq!(manifest.feature.workdir_catalog.enabled, !catalog); + assert_eq!(manifest.feature.manage_workdir.enabled, manage); + } + } + } + + #[test] + fn workdir_catalog_roundtrips_config_and_persisted_snapshots() { + for enabled in [false, true] { + let mut cfg = minimal_valid(); + cfg.feature.workdir_catalog = Some(FeatureFlagConfigPartial { + enabled: Some(enabled), + }); + let manifest: WorkerManifest = cfg.try_into().unwrap(); + let partial = FeatureConfigPartial::from(manifest.feature.clone()); + assert_eq!( + partial.workdir_catalog.as_ref().unwrap().enabled, + Some(enabled) + ); + let serialized = toml::to_string(&partial).unwrap(); + let partial: FeatureConfigPartial = toml::from_str(&serialized).unwrap(); + assert_eq!(FeatureConfig::from(partial), manifest.feature); + let serialized = serde_json::to_value(&manifest.feature).unwrap(); + assert_eq!(serialized["workdir_catalog"]["enabled"], enabled); + let feature: FeatureConfig = serde_json::from_value(serialized).unwrap(); + assert_eq!(feature, manifest.feature); + + let snapshot = crate::write_persisted_worker_manifest_snapshot(&manifest).unwrap(); + let restored = + crate::read_persisted_worker_manifest_snapshot(snapshot.clone()).unwrap(); + assert_eq!(restored.feature.workdir_catalog.enabled, enabled); + assert!(!restored.feature.manage_workdir.enabled); + + let mut legacy = snapshot; + legacy["manifest"]["feature"] + .as_object_mut() + .unwrap() + .remove("workdir_catalog"); + let restored = crate::read_persisted_worker_manifest_snapshot(legacy).unwrap(); + assert!(restored.feature.workdir_catalog.enabled); + assert!(!restored.feature.manage_workdir.enabled); + } + } + #[test] fn from_toml_parses_explicit_feature_flags() { let cfg = WorkerManifestConfig::from_toml( diff --git a/crates/manifest/src/defaults.rs b/crates/manifest/src/defaults.rs index bf4755b82..8b5b9619e 100644 --- a/crates/manifest/src/defaults.rs +++ b/crates/manifest/src/defaults.rs @@ -6,6 +6,10 @@ //! `pub const`. Both paths read from this module, so changing a //! default requires editing exactly one line. +/// Read-only Repository/Workdir/attachment reference discovery in WIP mode. +/// This enables a reference provider, not normal Tools or any authority grant. +pub const WORKDIR_CATALOG_ENABLED: bool = true; + /// Byte-size cap applied to any tool's `content` output when no /// per-tool override is set. See [`crate::ToolOutputLimits`]. pub const TOOL_OUTPUT_MAX_BYTES: usize = 64 * 1024; diff --git a/crates/manifest/src/lib.rs b/crates/manifest/src/lib.rs index 62634b34f..03e98be88 100644 --- a/crates/manifest/src/lib.rs +++ b/crates/manifest/src/lib.rs @@ -65,8 +65,9 @@ pub struct WorkerManifest { /// permission layer is disabled and tool calls run as before. #[serde(default)] pub permissions: Option, - /// Explicit built-in feature/tool-surface enablement. Omitted feature flags - /// resolve disabled so Profile authors choose the exposed built-in surfaces. + /// Built-in feature/tool-surface enablement. Omitted tool features resolve + /// disabled; the read-only WIP workdir catalog defaults enabled without + /// granting authority or registering normal Tools. #[serde(default)] pub feature: FeatureConfig, /// Explicit external Model Context Protocol provider configuration. This @@ -131,6 +132,19 @@ pub struct FeatureConfig { pub objective: FeatureFlagConfig, #[serde(default)] pub manage_workdir: FeatureFlagConfig, + /// Read-only Repository/Workdir/attachment reference discovery. Defaults + /// enabled, is installed only in WIP mode, and exposes no normal Tools. + /// Independent of `manage_workdir`; permissions and host authority remain + /// separate, and discovering a reference grants no attachment or file access. + #[serde( + default = "default_workdir_catalog", + deserialize_with = "deserialize_workdir_catalog" + )] + pub workdir_catalog: FeatureFlagConfig, + /// Opt-in WIP-only logical Workspace configuration feature. This flag is + /// not an access grant; Backend-owned Worker grants remain authoritative. + #[serde(default, deserialize_with = "deserialize_workspace_config")] + pub workspace_config: FeatureFlagConfig, #[serde(default)] pub ticket: TicketFeatureConfig, #[serde(default)] @@ -139,6 +153,29 @@ pub struct FeatureConfig { pub orchestration: FeatureFlagConfig, } +fn default_workdir_catalog() -> FeatureFlagConfig { + FeatureFlagConfig { + enabled: defaults::WORKDIR_CATALOG_ENABLED, + } +} + +fn deserialize_workdir_catalog<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + let flag = config::FeatureFlagConfigPartial::deserialize(deserializer)?; + Ok(FeatureFlagConfig { + enabled: flag.enabled.unwrap_or(defaults::WORKDIR_CATALOG_ENABLED), + }) +} + +fn deserialize_workspace_config<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + Ok(config::WorkspaceConfigFeatureConfigPartial::deserialize(deserializer)?.into()) +} + impl Default for FeatureConfig { fn default() -> Self { Self { @@ -153,6 +190,8 @@ impl Default for FeatureConfig { workspace_worker_discovery: FeatureFlagConfig::disabled(), objective: FeatureFlagConfig::disabled(), manage_workdir: FeatureFlagConfig::disabled(), + workdir_catalog: default_workdir_catalog(), + workspace_config: FeatureFlagConfig::disabled(), ticket: TicketFeatureConfig::default(), merge_request: MergeRequestFeatureConfig::default(), orchestration: FeatureFlagConfig::disabled(), @@ -363,6 +402,8 @@ impl ResolvedMemoryFeatureConfig { #[serde(default, deny_unknown_fields)] pub struct SubjektivFeatureProfileConfig { pub enabled: bool, + /// Explicit consolidation tool policy; never grants Job or subject authority. + pub consolidation_tools: bool, pub extraction: MemoryExtractionProfileConfig, } @@ -370,6 +411,7 @@ impl Default for SubjektivFeatureProfileConfig { fn default() -> Self { Self { enabled: false, + consolidation_tools: false, extraction: MemoryExtractionProfileConfig::default(), } } diff --git a/crates/manifest/src/profile.rs b/crates/manifest/src/profile.rs index 292411419..7849a58db 100644 --- a/crates/manifest/src/profile.rs +++ b/crates/manifest/src/profile.rs @@ -181,6 +181,8 @@ pub fn validate_profile_execution_target( if feature.flow.enabled { requirements.insert(WorkspaceAuthorityRequirement::Flow); } + // workdir_catalog is a WIP-only reference provider, not Workspace authority + // or normal Tools. It must not prevent standalone Tools-mode launches. if feature.manage_workdir.enabled { requirements.insert(WorkspaceAuthorityRequirement::ManageWorkdir); } @@ -1482,6 +1484,59 @@ mod tests { assert!(!resolved.manifest.feature.manage_workdir.enabled); } + #[test] + fn workdir_catalog_does_not_require_workspace_authority_or_grant_scope() { + let tmp = TempDir::new().unwrap(); + for catalog in [None, Some(false), Some(true)] { + let mut artifact = serde_json::json!({ + "worker": { "mode": "tools" }, + "feature": { "manage_workdir": { "enabled": false } }, + "permissions": { "default_action": "deny" } + }); + if let Some(enabled) = catalog { + artifact["feature"]["workdir_catalog"] = serde_json::json!({ "enabled": enabled }); + } + let resolved = resolve_profile_artifact_value( + artifact, + ProfileSource::Archive { + archive_id: "catalog-test".into(), + source: "test".into(), + }, + tmp.path(), + "catalog-worker", + ) + .unwrap(); + let manifest = resolved.manifest; + assert_eq!( + manifest.feature.workdir_catalog.enabled, + catalog.unwrap_or(true) + ); + assert!(!manifest.feature.manage_workdir.enabled); + assert_eq!(manifest.worker.mode, crate::WorkerMode::Tools); + assert!(manifest.scope.allow.is_empty()); + assert!(manifest.delegation_scope.allow.is_empty()); + assert_eq!( + manifest.permissions.as_ref().unwrap().default_action, + crate::ToolPermissionAction::Deny + ); + validate_profile_execution_target(&manifest, ProfileExecutionTarget::Standalone) + .unwrap(); + + let mut manage_manifest = manifest; + manage_manifest.feature.manage_workdir.enabled = true; + let error = validate_profile_execution_target( + &manage_manifest, + ProfileExecutionTarget::Standalone, + ) + .unwrap_err(); + assert!(matches!( + error, + ProfileError::UnsupportedExecutionTarget { requirements, .. } + if requirements == vec![WorkspaceAuthorityRequirement::ManageWorkdir] + )); + } + } + #[test] fn standalone_rejects_profiles_that_require_workspace_authority() { let tmp = TempDir::new().unwrap(); diff --git a/crates/protocol/Cargo.toml b/crates/protocol/Cargo.toml index 5cfe93f7d..f0f705d35 100644 --- a/crates/protocol/Cargo.toml +++ b/crates/protocol/Cargo.toml @@ -14,6 +14,7 @@ json-schema = ["dep:schemars"] schemars = { workspace = true, optional = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } +thiserror = { workspace = true } sha2.workspace = true tokio = { workspace = true, features = ["io-util"], optional = true } ts-rs = { version = "12.0.1", optional = true } diff --git a/crates/protocol/src/invocation.rs b/crates/protocol/src/invocation.rs new file mode 100644 index 000000000..98c2275d5 --- /dev/null +++ b/crates/protocol/src/invocation.rs @@ -0,0 +1,949 @@ +//! Declarative chat Feature invocation contract. +//! +//! The input spelling is deliberately bounded and unambiguous: +//! `/name(arg=value, "positional value")`. Parentheses terminate arguments, +//! so following prose is never consumed. Strings use JSON-style quoting and +//! escaping. Unquoted values are limited to booleans, integers, enum-like words, +//! and paths without comma/parenthesis/whitespace. This module parses input for +//! clients and validates the structured payload again at the host boundary. + +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt; + +/// UTF-8 byte bound shared by input validation and durable recovery keys. +pub const MAX_FEATURE_INVOCATION_ID_BYTES: usize = 128; +/// Installed descriptors must fit a public completion entry without truncation. +pub const MAX_FEATURE_INVOCATION_DESCRIPTOR_BYTES: usize = 32 * 1024; + +/// Source-qualified identity of one invocation contribution, for example +/// `builtin:attachments/attach`. Display names and aliases are never authority. +#[derive(Debug, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))] +pub struct FeatureInvocationIdentity(pub String); + +impl fmt::Display for FeatureInvocationIdentity { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(&self.0) + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum FeatureInvocationSyntax { + /// `/name(value, key=value)` with JSON-style quoted strings and `)` as the + /// explicit boundary before following natural language. + Parenthesized, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum InvocationArgumentType { + String, + Integer, + Boolean, + Enum { + values: Vec, + }, + /// A Worker-readable path. This does not grant filesystem authority. + WorkerFile, + /// A resource selected and staged by the client. It must never be resolved + /// as a path on the Worker host. + ClientFile, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum InvocationCompletion { + None, + Static { + values: Vec, + }, + WorkerFile, + ClientFile, + /// Host-owned provider selected by source-qualified id. Metadata lookup is + /// side-effect free; implementations never cross this wire boundary. + Provider { + provider: String, + }, +} + +impl Default for InvocationCompletion { + fn default() -> Self { + Self::None + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))] +pub struct InvocationArgumentDescriptor { + pub name: String, + /// Zero-based positional index. `None` means named-only. A positional + /// argument may also be supplied by name; this makes chip editing stable. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub position: Option, + #[serde(default)] + pub required: bool, + pub value_type: InvocationArgumentType, + #[serde(default)] + pub completion: InvocationCompletion, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum InvocationClientAdapter { + /// The client stages a local file and replaces the draft invocation with an + /// `UploadedFile` chip. Local paths are not sent to or persisted by Worker. + Attachment, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))] +pub struct FeatureInvocationDescriptor { + pub identity: FeatureInvocationIdentity, + pub name: String, + #[serde(default)] + pub aliases: Vec, + pub display_name: String, + pub description: String, + pub syntax: FeatureInvocationSyntax, + #[serde(default)] + pub arguments: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub client_adapter: Option, +} + +impl FeatureInvocationDescriptor { + pub fn usage(&self) -> String { + let mut usage = format!("/{}(", self.name); + for (index, argument) in self.arguments.iter().enumerate() { + if index > 0 { + usage.push_str(", "); + } + if argument.position.is_none() { + usage.push_str(&argument.name); + usage.push('='); + } + let marker = match &argument.value_type { + InvocationArgumentType::String => "text", + InvocationArgumentType::Integer => "integer", + InvocationArgumentType::Boolean => "true|false", + InvocationArgumentType::Enum { .. } => "choice", + InvocationArgumentType::WorkerFile => "worker-path", + InvocationArgumentType::ClientFile => "local-file", + }; + if argument.required { + usage.push('<'); + usage.push_str(marker); + usage.push('>'); + } else { + usage.push('['); + usage.push_str(marker); + usage.push(']'); + } + } + usage.push(')'); + usage + } + + pub fn validate(&self) -> Result<(), InvocationValidationError> { + const MAX_PUBLIC_NAMES: usize = 16; + const MAX_ARGUMENTS: usize = 32; + const MAX_METADATA_BYTES: usize = 4096; + validate_invocation_name(&self.name)?; + if self.aliases.len() > MAX_PUBLIC_NAMES || self.arguments.len() > MAX_ARGUMENTS { + return Err(InvocationValidationError::InvalidDescriptor( + "invocation descriptor exceeds collection limits".into(), + )); + } + if self.display_name.len() > MAX_METADATA_BYTES + || self.description.len() > MAX_METADATA_BYTES + { + return Err(InvocationValidationError::InvalidDescriptor( + "invocation metadata exceeds byte limits".into(), + )); + } + if self.identity.0.trim().is_empty() + || !self.identity.0.contains(':') + || self.identity.0.len() > 256 + { + return Err(InvocationValidationError::InvalidDescriptor( + "invocation identity must be source-qualified".into(), + )); + } + let mut names = BTreeSet::new(); + let mut positions = BTreeSet::new(); + for name in std::iter::once(&self.name).chain(self.aliases.iter()) { + validate_invocation_name(name)?; + if !names.insert(name) { + return Err(InvocationValidationError::InvalidDescriptor(format!( + "duplicate invocation name or alias `{name}`" + ))); + } + } + let mut argument_names = BTreeSet::new(); + for argument in &self.arguments { + validate_argument_name(&argument.name)?; + if !argument_names.insert(argument.name.as_str()) { + return Err(InvocationValidationError::InvalidDescriptor(format!( + "duplicate argument `{}`", + argument.name + ))); + } + if let Some(position) = argument.position + && !positions.insert(position) + { + return Err(InvocationValidationError::InvalidDescriptor(format!( + "duplicate positional index `{position}`" + ))); + } + if argument + .description + .as_ref() + .is_some_and(|value| value.len() > MAX_METADATA_BYTES) + { + return Err(InvocationValidationError::InvalidDescriptor(format!( + "argument `{}` description exceeds byte limits", + argument.name + ))); + } + if let InvocationCompletion::Provider { provider } = &argument.completion + && (provider.trim().is_empty() || provider.len() > 256) + { + return Err(InvocationValidationError::InvalidDescriptor(format!( + "argument `{}` has an invalid completion provider", + argument.name + ))); + } + if matches!(&argument.value_type, InvocationArgumentType::Enum { values } + if values.len() > 256 || values.iter().any(|value| value.len() > MAX_METADATA_BYTES)) + || matches!(&argument.completion, InvocationCompletion::Static { values } + if values.len() > 256 || values.iter().any(|value| value.len() > MAX_METADATA_BYTES)) + { + return Err(InvocationValidationError::InvalidDescriptor(format!( + "argument `{}` completion values exceed limits", + argument.name + ))); + } + if matches!(&argument.value_type, InvocationArgumentType::Enum { values } if values.is_empty()) + { + return Err(InvocationValidationError::InvalidDescriptor(format!( + "enum argument `{}` has no values", + argument.name + ))); + } + } + for (expected, actual) in positions.iter().copied().enumerate() { + if actual as usize != expected { + return Err(InvocationValidationError::InvalidDescriptor( + "positional argument indices must be contiguous from zero".into(), + )); + } + } + if serde_json::to_vec(self).map_or(true, |encoded| { + encoded.len() > MAX_FEATURE_INVOCATION_DESCRIPTOR_BYTES + }) { + return Err(InvocationValidationError::InvalidDescriptor( + "invocation descriptor exceeds the public completion byte limit".into(), + )); + } + Ok(()) + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))] +#[serde(tag = "kind", content = "value", rename_all = "snake_case")] +pub enum InvocationValue { + String(String), + Integer(i32), + Boolean(bool), +} + +impl InvocationValue { + pub fn display_input(&self) -> String { + match self { + Self::String(value) => quote_invocation_string(value), + Self::Integer(value) => value.to_string(), + Self::Boolean(value) => value.to_string(), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))] +pub struct InvocationArgumentValue { + pub name: String, + pub value: InvocationValue, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))] +pub struct FeatureInvocation { + /// Stable client-generated id retained across queueing and transport retry. + pub invocation_id: String, + pub identity: FeatureInvocationIdentity, + /// Selected public name, retained for exact draft/history restoration. Host + /// resolution is always by `identity`. + pub name: String, + pub arguments: Vec, +} + +impl FeatureInvocation { + pub fn display_input(&self) -> String { + let mut input = format!("/{}(", self.name); + for (index, argument) in self.arguments.iter().enumerate() { + if index > 0 { + input.push_str(", "); + } + input.push_str(&argument.name); + input.push('='); + input.push_str(&argument.value.display_input()); + } + input.push(')'); + input + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))] +#[serde(rename_all = "snake_case")] +pub enum FeatureInvocationStatus { + Succeeded, + Failed, + OutcomeUnknown, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))] +pub struct FeatureInvocationResult { + pub invocation_id: String, + pub identity: FeatureInvocationIdentity, + pub status: FeatureInvocationStatus, + pub message: String, + /// Bounded model-visible context produced by the handler. Credentials and + /// executable implementation details must not be included. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub context: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ParsedFeatureInvocation { + pub invocation: FeatureInvocation, + /// UTF-8 byte offset immediately after the closing parenthesis. + pub end: usize, +} + +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +pub enum InvocationValidationError { + #[error("invalid invocation descriptor: {0}")] + InvalidDescriptor(String), + #[error("incomplete invocation at byte {0}")] + Incomplete(usize), + #[error("unknown argument `{0}`")] + UnknownArgument(String), + #[error("duplicate argument `{0}`")] + DuplicateArgument(String), + #[error("missing required argument `{0}`")] + MissingRequired(String), + #[error("too many positional arguments")] + TooManyPositional, + #[error("invalid value for `{argument}`: {message}")] + InvalidValue { argument: String, message: String }, + #[error("invocation identity does not match descriptor")] + IdentityMismatch, + #[error("invocation name does not match descriptor")] + NameMismatch, + #[error("invalid invocation syntax at byte {offset}: {message}")] + Syntax { offset: usize, message: String }, +} + +pub fn validate_feature_invocation( + descriptor: &FeatureInvocationDescriptor, + invocation: &FeatureInvocation, +) -> Result<(), InvocationValidationError> { + descriptor.validate()?; + if invocation.identity != descriptor.identity { + return Err(InvocationValidationError::IdentityMismatch); + } + if invocation.name != descriptor.name && !descriptor.aliases.contains(&invocation.name) { + return Err(InvocationValidationError::NameMismatch); + } + if invocation.invocation_id.trim().is_empty() + || invocation.invocation_id.len() > MAX_FEATURE_INVOCATION_ID_BYTES + { + return Err(InvocationValidationError::InvalidValue { + argument: "invocation_id".into(), + message: "must be nonempty and at most 128 UTF-8 bytes".into(), + }); + } + if invocation.arguments.len() > 32 { + return Err(InvocationValidationError::InvalidValue { + argument: "arguments".into(), + message: "too many arguments".into(), + }); + } + let mut values = BTreeMap::new(); + for argument in &invocation.arguments { + if values + .insert(argument.name.as_str(), &argument.value) + .is_some() + { + return Err(InvocationValidationError::DuplicateArgument( + argument.name.clone(), + )); + } + } + for name in values.keys() { + if !descriptor + .arguments + .iter() + .any(|argument| argument.name == **name) + { + return Err(InvocationValidationError::UnknownArgument( + (*name).to_string(), + )); + } + } + for argument in &descriptor.arguments { + match values.get(argument.name.as_str()) { + Some(value) => validate_value(argument, value)?, + None if argument.required => { + return Err(InvocationValidationError::MissingRequired( + argument.name.clone(), + )); + } + None => {} + } + } + Ok(()) +} + +/// Parse one complete parenthesized invocation starting at `start`. Callers +/// decide when input becomes a typed chip; parsing alone never executes it. +pub fn parse_feature_invocation( + input: &str, + start: usize, + descriptor: &FeatureInvocationDescriptor, + invocation_id: impl Into, +) -> Result { + descriptor.validate()?; + if !input.is_char_boundary(start) || input.as_bytes().get(start) != Some(&b'/') { + return Err(InvocationValidationError::Syntax { + offset: start, + message: "expected `/`".into(), + }); + } + let mut cursor = start + 1; + let name_start = cursor; + while let Some(byte) = input.as_bytes().get(cursor) { + if byte.is_ascii_alphanumeric() || matches!(*byte, b'-' | b'_') { + cursor += 1; + } else { + break; + } + } + let name = &input[name_start..cursor]; + if name != descriptor.name && !descriptor.aliases.iter().any(|alias| alias == name) { + return Err(InvocationValidationError::NameMismatch); + } + skip_ws(input, &mut cursor); + if input.as_bytes().get(cursor) != Some(&b'(') { + return Err(InvocationValidationError::Incomplete(cursor)); + } + cursor += 1; + let positional = descriptor + .arguments + .iter() + .filter_map(|argument| argument.position.map(|position| (position, argument))) + .collect::>(); + let mut next_position = 0u16; + let mut parsed = Vec::::new(); + loop { + skip_ws(input, &mut cursor); + match input.as_bytes().get(cursor) { + Some(b')') => { + cursor += 1; + break; + } + None => return Err(InvocationValidationError::Incomplete(cursor)), + _ => {} + } + let token_start = cursor; + let possible_name = parse_identifier(input, &mut cursor); + let after_name = cursor; + skip_ws(input, &mut cursor); + let (argument, raw_value) = if !possible_name.is_empty() + && input.as_bytes().get(cursor) == Some(&b'=') + { + cursor += 1; + skip_ws(input, &mut cursor); + let argument = descriptor + .arguments + .iter() + .find(|argument| argument.name == possible_name) + .ok_or_else(|| InvocationValidationError::UnknownArgument(possible_name.into()))?; + (argument, parse_raw_value(input, &mut cursor)?) + } else { + cursor = token_start; + let argument = positional + .get(&next_position) + .copied() + .ok_or(InvocationValidationError::TooManyPositional)?; + next_position += 1; + (argument, parse_raw_value(input, &mut cursor)?) + }; + if parsed.iter().any(|value| value.name == argument.name) { + return Err(InvocationValidationError::DuplicateArgument( + argument.name.clone(), + )); + } + let value = coerce_value(argument, raw_value)?; + parsed.push(InvocationArgumentValue { + name: argument.name.clone(), + value, + }); + skip_ws(input, &mut cursor); + match input.as_bytes().get(cursor) { + Some(b',') => cursor += 1, + Some(b')') => {} + None => return Err(InvocationValidationError::Incomplete(cursor)), + _ => { + return Err(InvocationValidationError::Syntax { + offset: after_name.max(cursor), + message: "expected `,` or `)`".into(), + }); + } + } + } + let invocation = FeatureInvocation { + invocation_id: invocation_id.into(), + identity: descriptor.identity.clone(), + name: name.to_string(), + arguments: parsed, + }; + validate_feature_invocation(descriptor, &invocation)?; + Ok(ParsedFeatureInvocation { + invocation, + end: cursor, + }) +} + +#[derive(Debug)] +enum RawValue { + Quoted(String), + Bare(String), +} + +fn parse_raw_value(input: &str, cursor: &mut usize) -> Result { + if input.as_bytes().get(*cursor) == Some(&b'"') { + let start = *cursor; + *cursor += 1; + let mut escaped = false; + loop { + let Some(ch) = input[*cursor..].chars().next() else { + return Err(InvocationValidationError::Incomplete(*cursor)); + }; + *cursor += ch.len_utf8(); + if ch == '"' && !escaped { + let value = + serde_json::from_str::(&input[start..*cursor]).map_err(|error| { + InvocationValidationError::Syntax { + offset: start, + message: error.to_string(), + } + })?; + return Ok(RawValue::Quoted(value)); + } + escaped = ch == '\\' && !escaped; + } + } + let start = *cursor; + while let Some(ch) = input[*cursor..].chars().next() { + if ch == ',' || ch == ')' || ch.is_whitespace() { + break; + } + *cursor += ch.len_utf8(); + } + if *cursor == start { + return Err(InvocationValidationError::Syntax { + offset: start, + message: "expected argument value".into(), + }); + } + Ok(RawValue::Bare(input[start..*cursor].to_string())) +} + +fn coerce_value( + descriptor: &InvocationArgumentDescriptor, + raw: RawValue, +) -> Result { + let text = match raw { + RawValue::Quoted(value) | RawValue::Bare(value) => value, + }; + let invalid = |message: &str| InvocationValidationError::InvalidValue { + argument: descriptor.name.clone(), + message: message.into(), + }; + match &descriptor.value_type { + InvocationArgumentType::String + | InvocationArgumentType::WorkerFile + | InvocationArgumentType::ClientFile => Ok(InvocationValue::String(text)), + InvocationArgumentType::Integer => { + let digits = text.strip_prefix('-').unwrap_or(&text); + if digits.is_empty() || !digits.bytes().all(|byte| byte.is_ascii_digit()) { + return Err(invalid("expected a decimal integer")); + } + text.parse::() + .map(InvocationValue::Integer) + .map_err(|_| invalid("expected a signed 32-bit integer")) + } + InvocationArgumentType::Boolean => match text.as_str() { + "true" => Ok(InvocationValue::Boolean(true)), + "false" => Ok(InvocationValue::Boolean(false)), + _ => Err(invalid("expected `true` or `false`")), + }, + InvocationArgumentType::Enum { values } => { + if values.contains(&text) { + Ok(InvocationValue::String(text)) + } else { + Err(invalid("value is not in the declared enum")) + } + } + } +} + +fn validate_value( + descriptor: &InvocationArgumentDescriptor, + value: &InvocationValue, +) -> Result<(), InvocationValidationError> { + let valid = match (&descriptor.value_type, value) { + ( + InvocationArgumentType::String + | InvocationArgumentType::WorkerFile + | InvocationArgumentType::ClientFile, + InvocationValue::String(value), + ) => value.len() <= 16 * 1024, + (InvocationArgumentType::Integer, InvocationValue::Integer(_)) => true, + (InvocationArgumentType::Boolean, InvocationValue::Boolean(_)) => true, + (InvocationArgumentType::Enum { values }, InvocationValue::String(value)) => { + values.contains(value) + } + _ => false, + }; + if valid { + Ok(()) + } else { + Err(InvocationValidationError::InvalidValue { + argument: descriptor.name.clone(), + message: "structured value does not match the declared type".into(), + }) + } +} + +fn validate_invocation_name(name: &str) -> Result<(), InvocationValidationError> { + if name.is_empty() + || !name + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') + || !name.as_bytes()[0].is_ascii_lowercase() + { + return Err(InvocationValidationError::InvalidDescriptor(format!( + "invalid invocation name `{name}`" + ))); + } + Ok(()) +} + +fn validate_argument_name(name: &str) -> Result<(), InvocationValidationError> { + if name.is_empty() + || !name + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_') + || !name.as_bytes()[0].is_ascii_lowercase() + { + return Err(InvocationValidationError::InvalidDescriptor(format!( + "invalid argument name `{name}`" + ))); + } + Ok(()) +} + +fn parse_identifier<'a>(input: &'a str, cursor: &mut usize) -> &'a str { + let start = *cursor; + while let Some(byte) = input.as_bytes().get(*cursor) { + if byte.is_ascii_lowercase() || byte.is_ascii_digit() || *byte == b'_' { + *cursor += 1; + } else { + break; + } + } + &input[start..*cursor] +} + +fn skip_ws(input: &str, cursor: &mut usize) { + while let Some(ch) = input[*cursor..].chars().next() { + if !ch.is_whitespace() { + break; + } + *cursor += ch.len_utf8(); + } +} + +pub fn quote_invocation_string(value: &str) -> String { + // Keep the spelling identical to Web's JSON.stringify, including control + // characters and JSON unicode escapes accepted by both common parsers. + serde_json::to_string(value).expect("serializing a string cannot fail") +} + +#[cfg(test)] +mod tests { + use super::*; + + fn descriptor() -> FeatureInvocationDescriptor { + FeatureInvocationDescriptor { + identity: FeatureInvocationIdentity("builtin:test/run".into()), + name: "run".into(), + aliases: vec!["execute".into()], + display_name: "Run".into(), + description: "test invocation".into(), + syntax: FeatureInvocationSyntax::Parenthesized, + arguments: vec![ + InvocationArgumentDescriptor { + name: "path".into(), + position: Some(0), + required: true, + value_type: InvocationArgumentType::String, + completion: InvocationCompletion::WorkerFile, + description: None, + }, + InvocationArgumentDescriptor { + name: "mode".into(), + position: None, + required: false, + value_type: InvocationArgumentType::Enum { + values: vec!["fast".into(), "safe".into()], + }, + completion: InvocationCompletion::Static { + values: vec!["fast".into(), "safe".into()], + }, + description: None, + }, + ], + client_adapter: None, + } + } + + #[test] + fn parses_unicode_quoted_paths_and_stops_before_natural_language() { + let input = "/run(\"資料/my file / x\", mode=safe) この後は本文"; + let parsed = parse_feature_invocation(input, 0, &descriptor(), "invoke-1").unwrap(); + assert_eq!(&input[parsed.end..], " この後は本文"); + assert_eq!( + parsed.invocation.arguments, + vec![ + InvocationArgumentValue { + name: "path".into(), + value: InvocationValue::String("資料/my file / x".into()), + }, + InvocationArgumentValue { + name: "mode".into(), + value: InvocationValue::String("safe".into()), + }, + ] + ); + } + + #[test] + fn reports_incomplete_unknown_duplicate_missing_and_type_errors() { + assert!(matches!( + parse_feature_invocation("/run(\"x\"", 0, &descriptor(), "1"), + Err(InvocationValidationError::Incomplete(_)) + )); + assert!(matches!( + parse_feature_invocation("/run(path=x, nope=y)", 0, &descriptor(), "1"), + Err(InvocationValidationError::UnknownArgument(name)) if name == "nope" + )); + assert!(matches!( + parse_feature_invocation("/run(path=x, path=y)", 0, &descriptor(), "1"), + Err(InvocationValidationError::DuplicateArgument(name)) if name == "path" + )); + assert!(matches!( + parse_feature_invocation("/run(mode=fast)", 0, &descriptor(), "1"), + Err(InvocationValidationError::MissingRequired(name)) if name == "path" + )); + assert!(matches!( + parse_feature_invocation("/run(x, mode=unknown)", 0, &descriptor(), "1"), + Err(InvocationValidationError::InvalidValue { argument, .. }) if argument == "mode" + )); + } + + #[test] + fn descriptors_cannot_install_with_undiscoverable_oversized_metadata() { + let mut descriptor = descriptor(); + descriptor.arguments[0].completion = InvocationCompletion::Static { + values: vec!["a".repeat(1024); 40], + }; + assert!( + matches!(descriptor.validate(), Err(InvocationValidationError::InvalidDescriptor(message)) if message.contains("completion byte limit")) + ); + } + + #[test] + fn client_and_recovery_invocation_key_bound_is_shared() { + let descriptor = descriptor(); + assert!( + parse_feature_invocation( + "/run(x)", + 0, + &descriptor, + "a".repeat(MAX_FEATURE_INVOCATION_ID_BYTES) + ) + .is_ok() + ); + assert!( + parse_feature_invocation( + "/run(x)", + 0, + &descriptor, + "a".repeat(MAX_FEATURE_INVOCATION_ID_BYTES + 1) + ) + .is_err() + ); + assert!(parse_feature_invocation("/run(x)", 0, &descriptor, "界".repeat(43)).is_err()); + } + + #[test] + fn structured_payload_validation_does_not_trust_client_identity_or_types() { + let descriptor = descriptor(); + let original = parse_feature_invocation("/run(path=x)", 0, &descriptor, "stable") + .unwrap() + .invocation; + let mut changed = original.clone(); + changed.identity = FeatureInvocationIdentity("builtin:other/run".into()); + assert_eq!( + validate_feature_invocation(&descriptor, &changed), + Err(InvocationValidationError::IdentityMismatch) + ); + changed = original.clone(); + changed.name = "unregistered".into(); + assert_eq!( + validate_feature_invocation(&descriptor, &changed), + Err(InvocationValidationError::NameMismatch) + ); + changed = original.clone(); + changed.arguments[0].value = InvocationValue::Boolean(true); + assert!(matches!( + validate_feature_invocation(&descriptor, &changed), + Err(InvocationValidationError::InvalidValue { .. }) + )); + changed = original.clone(); + changed.arguments.push(changed.arguments[0].clone()); + assert!(matches!( + validate_feature_invocation(&descriptor, &changed), + Err(InvocationValidationError::DuplicateArgument(_)) + )); + changed = original; + changed.invocation_id.clear(); + assert!(validate_feature_invocation(&descriptor, &changed).is_err()); + } + + #[test] + fn integer_boolean_and_utf8_offsets_use_the_common_typed_contract() { + let mut descriptor = descriptor(); + descriptor.arguments[0].value_type = InvocationArgumentType::Integer; + descriptor.arguments[1].value_type = InvocationArgumentType::Boolean; + let parsed = parse_feature_invocation( + "本文 /execute(-2147483648, mode=true) 後", + "本文 ".len(), + &descriptor, + "typed", + ) + .unwrap(); + assert_eq!( + parsed.invocation.arguments[0].value, + InvocationValue::Integer(i32::MIN) + ); + assert_eq!( + parsed.invocation.arguments[1].value, + InvocationValue::Boolean(true) + ); + for invalid in ["/run(2147483648)", "/run(1.0)", "/run(1, mode=yes)"] { + assert!(parse_feature_invocation(invalid, 0, &descriptor, "typed").is_err()); + } + assert!(parse_feature_invocation("本文 /run(1)", 1, &descriptor, "typed").is_err()); + assert!(parse_feature_invocation("/run(1)", usize::MAX, &descriptor, "typed").is_err()); + } + + #[test] + fn json_escapes_and_multiple_invocations_have_explicit_boundaries() { + let input = r#"/run("\u8cc7\u6599\/a\b\f", mode=safe) prose /execute("second") end"#; + let first = parse_feature_invocation(input, 0, &descriptor(), "first").unwrap(); + assert_eq!( + first.invocation.arguments[0].value, + InvocationValue::String("資料/a\u{8}\u{c}".into()) + ); + let next = input[first.end..].find("/execute").unwrap() + first.end; + let second = parse_feature_invocation(input, next, &descriptor(), "second").unwrap(); + assert_eq!(&input[second.end..], " end"); + assert!(matches!( + parse_feature_invocation(r#"/run("\uD800")"#, 0, &descriptor(), "bad"), + Err(InvocationValidationError::Syntax { .. }) + )); + let value = "controls \u{0}\u{8}\u{c}"; + let spelling = format!("/run({})", quote_invocation_string(value)); + assert_eq!( + parse_feature_invocation(&spelling, 0, &descriptor(), "control") + .unwrap() + .invocation + .arguments[0] + .value, + InvocationValue::String(value.into()) + ); + } + + #[test] + fn typed_segment_round_trips_without_flattening_identity_or_arguments() { + let parsed = + parse_feature_invocation("/run(\"x y\", mode=fast)", 0, &descriptor(), "1").unwrap(); + let segment = crate::Segment::FeatureInvoke { + invocation: parsed.invocation, + }; + let json = serde_json::to_string(&segment).unwrap(); + assert!(json.contains("\"kind\":\"feature_invoke\"")); + assert!(json.contains("builtin:test/run")); + assert_eq!( + serde_json::from_str::(&json).unwrap(), + segment + ); + } + + #[test] + fn quoting_round_trips_spaces_unicode_slashes_and_escapes() { + let value = "a b/界\\\"\n"; + let input = format!("/run({})", quote_invocation_string(value)); + let parsed = parse_feature_invocation(&input, 0, &descriptor(), "1").unwrap(); + assert_eq!( + parsed.invocation.arguments[0].value, + InvocationValue::String(value.into()) + ); + } +} diff --git a/crates/protocol/src/lib.rs b/crates/protocol/src/lib.rs index e043de416..f7dcd9160 100644 --- a/crates/protocol/src/lib.rs +++ b/crates/protocol/src/lib.rs @@ -1,4 +1,5 @@ pub mod identity; +pub mod invocation; #[cfg(feature = "stream")] pub mod stream; pub mod subscription; @@ -10,6 +11,14 @@ use std::path::PathBuf; use serde::{Deserialize, Serialize}; pub use identity::{WorkerId, WorkerIdParseError}; +pub use invocation::{ + FeatureInvocation, FeatureInvocationDescriptor, FeatureInvocationIdentity, + FeatureInvocationResult, FeatureInvocationStatus, FeatureInvocationSyntax, + InvocationArgumentDescriptor, InvocationArgumentType, InvocationArgumentValue, + InvocationClientAdapter, InvocationCompletion, InvocationValidationError, InvocationValue, + ParsedFeatureInvocation, parse_feature_invocation, quote_invocation_string, + validate_feature_invocation, +}; /// Allocate an opaque idempotency key for one client Submit request. pub fn new_submission_request_id() -> String { @@ -62,6 +71,17 @@ impl Default for AuthenticatedInputSource { } } +// Tracked methods may carry serialized transport provenance, but wire values +// never grant authority. Keep this field recognized under fail-closed decoding +// while replacing it with UntrustedWire, including malformed/forged values. +fn ignore_wire_input_source<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + let _ = serde::de::IgnoredAny::deserialize(deserializer)?; + Ok(AuthenticatedInputSource::UntrustedWire) +} + impl AuthenticatedInputSource { pub fn namespace(&self) -> String { match self { @@ -224,7 +244,12 @@ impl From for WorkerStateSnapshot { #[derive(Debug, Clone, Serialize, Deserialize)] #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))] -#[serde(tag = "method", content = "params", rename_all = "snake_case")] +#[serde( + tag = "method", + content = "params", + rename_all = "snake_case", + deny_unknown_fields +)] pub enum Method { /// Durably accept typed input for immediate activation or the session FIFO. /// Running or Paused Workers queue Submit for a later turn; use Notify for @@ -244,7 +269,7 @@ pub enum Method { SubmitTracked { submission_request_id: String, input: Vec, - #[serde(skip_deserializing, default)] + #[serde(default, deserialize_with = "ignore_wire_input_source")] source: AuthenticatedInputSource, }, /// Agent-controlled input: accept only at an Idle controller boundary, never @@ -254,7 +279,7 @@ pub enum Method { SubmitIfIdle { submission_request_id: String, input: Vec, - #[serde(skip_deserializing, default)] + #[serde(default, deserialize_with = "ignore_wire_input_source")] source: AuthenticatedInputSource, }, /// Human-readable text injected into the target Worker's LLM context as a @@ -272,7 +297,7 @@ pub enum Method { NotifyTracked { notification_request_id: String, message: String, - #[serde(skip_deserializing, default)] + #[serde(default, deserialize_with = "ignore_wire_input_source")] source: AuthenticatedInputSource, }, /// Typed lifecycle report from a child Worker to its direct parent. @@ -336,6 +361,12 @@ pub enum Method { ListCompletions { kind: CompletionKind, prefix: String, + /// Client query generation. Echoed verbatim so identical-prefix ABA + /// responses cannot cross edits, target switches, or permission changes. + #[serde(default, skip_serializing_if = "Option::is_none")] + request_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + context: Option, }, /// List Workers visible to this Worker from durable Worker state and the spawned-child /// registry. This is not a host-wide Worker universe query. @@ -609,6 +640,10 @@ pub enum Segment { /// `[Dir: ]` listings; the flattened user text keeps the literal /// `@` placeholder either way. FileRef { path: String }, + /// Explicitly selected, source-qualified Feature invocation. The host never + /// promotes free text to this variant and validates it against the installed + /// declaration before execution. + FeatureInvoke { invocation: FeatureInvocation }, /// Source-qualified request for the host to start a Flow before committing /// this user input. Runtime resolves it through Workspace authority and /// replaces it with the entered state's instructions before Worker-side @@ -677,6 +712,9 @@ impl Segment { out.push('@'); out.push_str(path); } + Segment::FeatureInvoke { invocation } => { + out.push_str(&invocation.display_input()); + } Segment::Flow { selector } => { out.push_str("[Flow: "); out.push_str(selector); @@ -1348,6 +1386,12 @@ pub enum Event { /// wired up. Completions { kind: CompletionKind, + #[serde(default)] + prefix: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + request_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + context: Option, entries: Vec, }, /// Reply to `Method::ListRewindTargets`. Clients should only open a picker @@ -1460,25 +1504,42 @@ pub enum AlertSource { /// Kind of completion requested by `Method::ListCompletions`. /// -/// Mirrors the completion prefix sigil: `@` → `File`. +/// `File` is the existing `@` path lane. `Feature` lists installed chat +/// invocations. `FeatureArgument` resolves a declared argument using the +/// source-qualified [`CompletionContext`]. #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))] #[serde(rename_all = "snake_case")] pub enum CompletionKind { File, + Feature, + FeatureArgument, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))] +pub struct CompletionContext { + pub invocation: FeatureInvocationIdentity, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub argument: Option, } /// One completion candidate for a prefix query. -/// -/// `value` is a path (file kind). -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default)] #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))] pub struct CompletionEntry { pub value: String, #[serde(default)] pub is_dir: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub description: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub usage: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub invocation: Option, } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] @@ -2114,6 +2175,27 @@ mod tests { ); } + #[test] + fn notify_rejects_typed_intent_instead_of_silently_discarding_it() { + for method in ["notify", "notify_tracked"] { + for key in ["input", "segments", "invocation"] { + let mut params = + serde_json::json!({"notification_request_id":"request", "message":"advisory"}); + params[key] = serde_json::json!([{ "kind":"feature_invoke", "invocation": { "identity":"builtin:test/run" } }]); + assert!( + serde_json::from_value::( + serde_json::json!({"method":method,"params":params}) + ) + .is_err() + ); + } + } + // Text is still text: no decoding or Host scan promotes it to an invoke. + assert!( + matches!(serde_json::from_value::(serde_json::json!({"method":"notify", "params":{"notification_request_id":"text","message":"/run()"}})).unwrap(), Method::Notify { message, .. } if message == "/run()") + ); + } + #[test] fn segment_unknown_variant_decodes_as_unknown() { // A future client sends a segment kind this Worker has never heard of. @@ -2352,6 +2434,8 @@ mod tests { let method = Method::ListCompletions { kind: CompletionKind::File, prefix: "src/".into(), + request_id: None, + context: None, }; let json = serde_json::to_string(&method).unwrap(); let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); @@ -2361,7 +2445,7 @@ mod tests { let decoded: Method = serde_json::from_str(&json).unwrap(); match decoded { - Method::ListCompletions { kind, prefix } => { + Method::ListCompletions { kind, prefix, .. } => { assert_eq!(kind, CompletionKind::File); assert_eq!(prefix, "src/"); } @@ -2369,13 +2453,52 @@ mod tests { } } + #[test] + fn completion_generation_round_trips_for_identical_query_correlation() { + let context = CompletionContext { + invocation: FeatureInvocationIdentity("builtin:test/run".into()), + argument: Some("mode".into()), + }; + for request_id in ["query-1", "query-2"] { + let method = Method::ListCompletions { + kind: CompletionKind::FeatureArgument, + prefix: "s".into(), + request_id: Some(request_id.into()), + context: Some(context.clone()), + }; + let decoded: Method = + serde_json::from_str(&serde_json::to_string(&method).unwrap()).unwrap(); + assert!( + matches!(decoded, Method::ListCompletions { request_id: Some(id), context: Some(restored), .. } if id == request_id && restored == context) + ); + let event = Event::Completions { + kind: CompletionKind::FeatureArgument, + prefix: "s".into(), + request_id: Some(request_id.into()), + context: Some(context.clone()), + entries: Vec::new(), + }; + let decoded: Event = + serde_json::from_str(&serde_json::to_string(&event).unwrap()).unwrap(); + assert!( + matches!(decoded, Event::Completions { request_id: Some(id), context: Some(restored), .. } if id == request_id && restored == context) + ); + } + } + #[test] fn event_completions_format_and_default_is_dir() { let event = Event::Completions { kind: CompletionKind::File, + prefix: "src/".into(), + request_id: None, + context: None, entries: vec![CompletionEntry { value: "src/main.rs".into(), is_dir: false, + description: None, + usage: None, + invocation: None, }], }; let json = serde_json::to_string(&event).unwrap(); @@ -2389,7 +2512,7 @@ mod tests { r#"{"event":"completions","data":{"kind":"file","entries":[{"value":"main.rs"}]}}"#; let decoded: Event = serde_json::from_str(inbound).unwrap(); match decoded { - Event::Completions { kind, entries } => { + Event::Completions { kind, entries, .. } => { assert_eq!(kind, CompletionKind::File); assert_eq!(entries.len(), 1); assert_eq!(entries[0].value, "main.rs"); diff --git a/crates/protocol/src/subscription.rs b/crates/protocol/src/subscription.rs index 642a1528d..5a59175ca 100644 --- a/crates/protocol/src/subscription.rs +++ b/crates/protocol/src/subscription.rs @@ -651,6 +651,10 @@ impl SubscriptionWorkerJob { #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))] pub struct SubscriptionWorker { + /// Opaque Runtime-owned Restore observation identity, never a display-state hash. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[cfg_attr(feature = "typescript", ts(optional = nullable))] + pub restore_observation_token: Option, pub worker_id: SubscriptionWorkerId, /// Set by the Workspace Server when projecting a Runtime-owned Worker to clients. /// Runtime producers leave this unset because the connection identifies the Runtime. @@ -689,6 +693,9 @@ pub struct SubscriptionWorker { impl SubscriptionWorker { pub fn validate(&self) -> Result<(), SubscriptionValidationError> { self.worker_id.validate()?; + if let Some(token) = &self.restore_observation_token { + validate_identifier("restore_observation_token", token, MAX_RESOURCE_ID_BYTES)?; + } if let Some(runtime_id) = &self.runtime_id { validate_identifier("runtime_id", runtime_id, MAX_RESOURCE_ID_BYTES)?; } @@ -962,6 +969,7 @@ mod tests { fn worker(value: &str) -> SubscriptionWorker { SubscriptionWorker { + restore_observation_token: None, worker_id: worker_id(value), runtime_id: None, resource_key: None, @@ -1197,6 +1205,8 @@ mod tests { method: crate::Method::ListCompletions { kind: crate::CompletionKind::File, prefix: "src/".to_string(), + request_id: None, + context: None, }, }, )); diff --git a/crates/protocol/src/typescript.rs b/crates/protocol/src/typescript.rs index c759b0992..201534567 100644 --- a/crates/protocol/src/typescript.rs +++ b/crates/protocol/src/typescript.rs @@ -6,20 +6,23 @@ use ts_rs::{Config, TS}; use crate::{ Alert, AlertLevel, AlertSource, CommandEvent, CommandSnapshot, CommandStatus, CommandStream, CommandStreamSlice, CompactionLifecycle, CompactionLifecycleState, CompactionPhase, - CompactionTrigger, CompletionEntry, CompletionKind, ContextTokenSource, ContextUsage, - ErrorCode, Event, Greeting, InFlightBlock, InFlightCompaction, InFlightSnapshot, + CompactionTrigger, CompletionContext, CompletionEntry, CompletionKind, ContextTokenSource, + ContextUsage, ErrorCode, Event, FeatureInvocation, FeatureInvocationDescriptor, + FeatureInvocationIdentity, FeatureInvocationResult, FeatureInvocationStatus, + FeatureInvocationSyntax, Greeting, InFlightBlock, InFlightCompaction, InFlightSnapshot, InFlightToolCallState, InternalWorkerKind, InternalWorkerRef, InternalWorkerSnapshot, - InvokeKind, MemoryWorkerEvent, Method, PasteArtifactAvailability, PasteArtifactMediaType, - PasteArtifactRef, PendingSubmissionSummary, PendingSubmissionsSnapshot, Permission, - ReasoningConfig, RewindSummary, RewindTarget, RewindTargetId, RunFailureKind, RunResult, - RunResumeSource, RunYieldReason, ScopeRule, Segment, SessionContentPart, - SessionConversationTurn, SessionEntryProvenance, SessionHistoryLineageBoundary, - SessionHistoryPage, SessionMessageRole, SessionSnapshot, SessionSnapshotEntry, - SessionSnapshotEntryData, SessionToolAttachment, SubmissionDisposition, SymlinkPolicy, - ToolResultDisposition, TurnResult, UploadedFileAvailability, UploadedFileRef, WorkerBusyState, - WorkerCommandAcknowledgement, WorkerCommandDisposition, WorkerCommandEnvelope, - WorkerCommandKind, WorkerEvent, WorkerMaintenanceState, WorkerRunState, WorkerState, - WorkerStateSnapshot, WorkerStatus, + InvocationArgumentDescriptor, InvocationArgumentType, InvocationArgumentValue, + InvocationClientAdapter, InvocationCompletion, InvocationValue, InvokeKind, MemoryWorkerEvent, + Method, PasteArtifactAvailability, PasteArtifactMediaType, PasteArtifactRef, + PendingSubmissionSummary, PendingSubmissionsSnapshot, Permission, ReasoningConfig, + RewindSummary, RewindTarget, RewindTargetId, RunFailureKind, RunResult, RunResumeSource, + RunYieldReason, ScopeRule, Segment, SessionContentPart, SessionConversationTurn, + SessionEntryProvenance, SessionHistoryLineageBoundary, SessionHistoryPage, SessionMessageRole, + SessionSnapshot, SessionSnapshotEntry, SessionSnapshotEntryData, SessionToolAttachment, + SubmissionDisposition, SymlinkPolicy, ToolResultDisposition, TurnResult, + UploadedFileAvailability, UploadedFileRef, WorkerBusyState, WorkerCommandAcknowledgement, + WorkerCommandDisposition, WorkerCommandEnvelope, WorkerCommandKind, WorkerEvent, + WorkerMaintenanceState, WorkerRunState, WorkerState, WorkerStateSnapshot, WorkerStatus, subscription::{ EventSubscriptionSelector, MAX_CORRELATION_ID_BYTES, MAX_REJECTION_MESSAGE_BYTES, MAX_RESOURCE_ID_BYTES, MAX_SUBSCRIPTION_COLLECTION_ITEMS, @@ -64,6 +67,19 @@ pub fn generated_protocol_types() -> String { push_decl::(&cfg, &mut output); push_decl::(&cfg, &mut output); push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); push_decl::(&cfg, &mut output); push_decl::(&cfg, &mut output); push_decl::(&cfg, &mut output); @@ -230,6 +246,7 @@ fn rust_serialized_subscription_frame_fixtures() -> Vec { let external_workdir_id = SubscriptionWorkdirId::new("fixture-external-workdir") .expect("fixture external Workdir id must be valid"); let worker = SubscriptionWorker { + restore_observation_token: Some("fixture-restore-observation".into()), worker_id: worker_id.clone(), runtime_id: Some("fixture-runtime".to_string()), resource_key: Some("fixture-worker-resource".to_string()), diff --git a/crates/protocol/src/typescript_validator_template.ts b/crates/protocol/src/typescript_validator_template.ts index 976f9b4bd..fc0da94d0 100644 --- a/crates/protocol/src/typescript_validator_template.ts +++ b/crates/protocol/src/typescript_validator_template.ts @@ -378,6 +378,7 @@ function validateWorkers(workers: SubscriptionWorker[]): boolean { } function validateWorker(worker: SubscriptionWorker): boolean { + if (worker.restore_observation_token != null && !validateIdentifier(worker.restore_observation_token, MAX_RESOURCE_ID_BYTES)) return false; if (!validateIdentifier(worker.worker_id, MAX_RESOURCE_ID_BYTES)) return false; if (worker.runtime_id != null && !validateIdentifier(worker.runtime_id, MAX_RESOURCE_ID_BYTES)) { return false; diff --git a/crates/runtime-api/src/lib.rs b/crates/runtime-api/src/lib.rs index eaf3135ef..242780b6c 100644 --- a/crates/runtime-api/src/lib.rs +++ b/crates/runtime-api/src/lib.rs @@ -420,6 +420,18 @@ pub struct WorkerInput { pub segments: Option>, } +/// Host-authored Job/attempt identity, independent of Profile selection. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct BackendJobExecutionBinding { + pub job_id: String, + pub attempt_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_revision: Option, + #[serde(default)] + pub subjektiv_consolidation: bool, +} + #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] pub struct CreateWorkerRequest { pub worker_id: WorkerId, @@ -448,10 +460,14 @@ pub struct CreateWorkerRequest { /// never activates subjektiv for an ordinary Workspace Worker. #[serde(default, skip_serializing_if = "std::ops::Not::not")] pub subjektiv_attached: bool, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub backend_job: Option, } #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] pub struct WorkerSummary { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub restore_observation_token: Option, pub worker_ref: WorkerRef, pub worker_id: WorkerId, pub status: WorkerStatus, @@ -484,6 +500,62 @@ pub struct WorkerResponse { pub worker: WorkerDetail, } +/// Explicit observed Restore intent. Preparation is authored by the Workspace +/// Backend, not browser/Tool JSON, and applied only after Runtime admission. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct WorkerRestoreRequest { + pub expected_observation_token: String, + pub request_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub preparation: Option, +} + +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct WorkerRestorePreparation { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workspace_api: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workdir_attachments: Option>, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub repository_access: Vec, + /// Read-only references resolved by Workspace only after this operation owns admission. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub repository_access_workdirs: Vec, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct RepositoryAccessWorkdirReference { + pub runtime_id: String, + pub working_directory_id: String, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct WorkerRestoreCoordinationRequest { + pub expected_observation_token: String, + pub request_id: String, + /// None means owner/result recovery only; it can never admit a new intent. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub preparation: Option, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct WorkerRestoreCoordinationResponse { + pub result: Option, + /// Original read-only preparation, only when the existing operation still needs it. + pub preparation: Option, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct WorkingDirectoryRepositoryAccessRequest { + pub working_directory_id: String, + pub materialization: RepositoryMaterializationContext, +} + /// Explicit restore outcome returned for every reachable Runtime restore attempt, /// including preflight rejection and uncertain post-side-effect reconciliation. #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] @@ -567,12 +639,20 @@ pub struct CompletionRequest { pub kind: CompletionKind, #[serde(default)] pub prefix: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub context: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub request_id: Option, } #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] pub struct CompletionResponse { pub kind: CompletionKind, pub prefix: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub context: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub request_id: Option, pub entries: Vec, } @@ -992,11 +1072,18 @@ pub trait RuntimeApi { #[body] request: WorkerLifecycleRequest, ) -> Result; - #[post("/v1/workers/{worker_id}/restore", status = 200, error_status = 400)] + #[post("/v1/workers/{worker_id}/restore/coordinate", status = 200, error_status = 400, additional_error_statuses = [409])] + async fn coordinate_worker_restore( + &self, + #[path] worker_id: String, + #[body] request: WorkerRestoreCoordinationRequest, + ) -> Result; + + #[post("/v1/workers/{worker_id}/restore", status = 200, error_status = 400, additional_error_statuses = [409])] async fn restore_worker( &self, #[path] worker_id: String, - #[body] request: EmptyObjectRequest, + #[body] request: WorkerRestoreRequest, ) -> Result; #[post( @@ -1202,6 +1289,61 @@ pub const REMAINING_RUNTIME_ROUTES: &[RemainingRuntimeRoute] = &[ mod tests { use super::*; + #[test] + fn restore_request_is_mandatory_and_conflict_is_not_an_operation_state() { + for invalid in [ + serde_json::json!({}), + serde_json::json!({"request_id":"r"}), + serde_json::json!({"expected_observation_token":"t"}), + ] { + assert!(serde_json::from_value::(invalid).is_err()); + } + let request: WorkerRestoreRequest = serde_json::from_value( + serde_json::json!({"expected_observation_token":"t", "request_id":"r"}), + ) + .unwrap(); + assert!(request.preparation.is_none()); + for state in [ + "accepted", + "rejected", + "rolled_back", + "reconciliation_required", + ] { + assert!(serde_json::from_value::(serde_json::json!(state)).is_ok()); + } + assert!( + serde_json::from_value::(serde_json::json!( + "restore_observation_conflict" + )) + .is_err() + ); + let conflict = RuntimeApiError::new( + 409, + "restore_observation_conflict", + "refresh before a new intent", + ); + assert_eq!(conflict.status(), 409); + assert_eq!( + serde_json::to_value(conflict).unwrap()["error"]["code"], + "restore_observation_conflict" + ); + } + + #[test] + fn feature_argument_completion_wire_preserves_source_and_argument() { + let json = serde_json::json!({"kind":"feature_argument", "prefix":"資料/", "context": {"invocation":"builtin:test/prepare", "argument":"path"}}); + let request: CompletionRequest = serde_json::from_value(json.clone()).unwrap(); + assert_eq!(request.kind, CompletionKind::FeatureArgument); + assert_eq!( + request.context.as_ref().unwrap().argument.as_deref(), + Some("path") + ); + assert_eq!(serde_json::to_value(request).unwrap(), json); + let legacy: CompletionRequest = + serde_json::from_value(serde_json::json!({"kind":"file"})).unwrap(); + assert!(legacy.context.is_none()); + } + #[derive(Clone)] struct RoundTripService; @@ -1365,10 +1507,17 @@ mod tests { ) -> Result { Err(test_error(501)) } + async fn coordinate_worker_restore( + &self, + _worker_id: String, + _request: WorkerRestoreCoordinationRequest, + ) -> Result { + Err(test_error(501)) + } async fn restore_worker( &self, _worker_id: String, - _request: EmptyObjectRequest, + _request: WorkerRestoreRequest, ) -> Result { Err(test_error(501)) } @@ -1468,7 +1617,7 @@ mod tests { #[test] fn contract_inventory_is_complete_and_unique() { let operations = RuntimeApiMetadata::OPERATIONS; - assert_eq!(operations.len(), 19); + assert_eq!(operations.len(), 20); let mut routes = operations .iter() .map(|operation| (format!("{:?}", operation.method), operation.path)) diff --git a/crates/server-api/src/lib.rs b/crates/server-api/src/lib.rs index a100ca901..0062c3bc5 100644 --- a/crates/server-api/src/lib.rs +++ b/crates/server-api/src/lib.rs @@ -11,6 +11,7 @@ pub use api_macros::axum as server_support; pub use api_macros::reqwest as client_support; pub use api_macros::{ApiContract, BinaryBody, HttpMethod, TransportMetadata, WebSocketOperation}; pub mod repository_openapi_typescript; +mod workspace_config; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; @@ -18,6 +19,7 @@ use webauthn_rs_proto::{ CreationChallengeResponse, PublicKeyCredential, RegisterPublicKeyCredential, RequestChallengeResponse, }; +pub use workspace_config::*; #[allow(dead_code)] #[derive(JsonSchema)] @@ -190,10 +192,32 @@ impl_openapi_schema!( WorkingDirectoryListResponse, WorkingDirectoryDetailResponse, WorkingDirectoryCreateResponse, + CurrentWorkerWorkdirCatalogQuery, + CurrentWorkerWorkdirCatalogResponse, CurrentWorkerWorkdirAttachRequest, CurrentWorkerWorkdirAttachmentResponse, + CurrentWorkerWorkdirAttachmentItem, + CurrentWorkerWorkdirAttachmentListQuery, + CurrentWorkerWorkdirAttachmentListResponse, + CurrentWorkerWorkdirDetachQuery, CurrentWorkerWorkdirOperationRequest, CurrentWorkerWorkdirOperationResponse, + WorkspaceConfigAccess, + WorkspaceConfigNodeKind, + WorkspaceConfigFailureClassification, + WorkspaceConfigAttachRequest, + WorkspaceConfigAttachment, + WorkspaceConfigCurrentResponse, + WorkspaceConfigObserveRequest, + WorkspaceConfigNode, + WorkspaceConfigObserveResponse, + WorkspaceConfigReadRequest, + WorkspaceConfigReadResponse, + WorkspaceConfigCommitRequest, + WorkspaceConfigCommitResponse, + WorkspaceConfigApiError, + WorkspaceConfigGrantCreateRequest, + WorkspaceConfigGrantResponse, ExternalWorkdirGrantCreateRequest, ExternalWorkdirGrantResponse, WorkspaceResponse, @@ -237,6 +261,7 @@ impl_openapi_schema!( MemoryConsolidateStagingRequest, MemoryConsolidationResponse, SubjektivSubjectCreateRequest, + SubjektivSubjectBehaviorUpdateRequest, SubjektivSubjectListQuery, SubjektivSubjectListResponse, SubjektivSubjectResponse, @@ -364,6 +389,7 @@ impl_openapi_schema!( WorkspaceRuntimeListResponse, RuntimeWorkerListResponse, WorkerRestoreResponse, + WorkerRestoreRequest, WorkerControlSubject, WorkerControlRecord, WorkerControlListResponse, @@ -1163,6 +1189,20 @@ pub trait ServerApi { #[path] workspace_id: String, #[path] subject_id: String, ) -> Result; + #[patch( + "/api/w/{workspace_id}/subjektiv/subjects/{subject_id}/behavior", + status = 200, + error_status = 400, + additional_error_statuses = [401, 403, 404, 409, 500], + bearer_auth = true, + browser_auth = true, + )] + async fn subjektiv_subject_behavior_update( + &self, + #[path] workspace_id: String, + #[path] subject_id: String, + #[body] request: SubjektivSubjectBehaviorUpdateRequest, + ) -> Result; #[get( "/api/w/{workspace_id}/subjektiv/subjects/{subject_id}/surface", status = 200, @@ -1513,6 +1553,7 @@ pub trait ServerApi { #[path] workspace_id: String, #[path] runtime_id: String, #[path] worker_id: String, + #[body] request: WorkerRestoreRequest, ) -> Result; #[get( "/api/w/{workspace_id}/worker-observation/sessions", @@ -2038,8 +2079,10 @@ pub trait ServerApi { )] async fn runtime_worker_restore_alias( &self, + #[extension] context: ServerRequestContext, #[path] runtime_id: String, #[path] worker_id: String, + #[body] request: WorkerRestoreRequest, ) -> Result; #[post( @@ -2052,10 +2095,12 @@ pub trait ServerApi { )] async fn runtime_worker_restore( &self, + #[extension] context: ServerRequestContext, #[path] workspace_id: String, #[path] runtime_id: String, #[path] worker_id: String, #[query] query: RestoreTicketAssignmentQuery, + #[body] request: WorkerRestoreRequest, ) -> Result; #[put( @@ -2068,6 +2113,7 @@ pub trait ServerApi { )] async fn runtime_worker_pin( &self, + #[extension] context: ServerRequestContext, #[path] workspace_id: String, #[path] runtime_id: String, #[path] worker_id: String, @@ -2083,6 +2129,7 @@ pub trait ServerApi { )] async fn runtime_worker_unpin( &self, + #[extension] context: ServerRequestContext, #[path] workspace_id: String, #[path] runtime_id: String, #[path] worker_id: String, @@ -2112,6 +2159,7 @@ pub trait ServerApi { )] async fn runtime_cleanup_execute( &self, + #[extension] context: ServerRequestContext, #[path] workspace_id: String, #[path] runtime_id: String, #[body] request: ExecuteRuntimeCleanupRequest, @@ -2127,6 +2175,7 @@ pub trait ServerApi { )] async fn runtime_worker_input_alias( &self, + #[extension] context: ServerRequestContext, #[path] runtime_id: String, #[path] worker_id: String, #[body] request: RuntimeWorkerInputRequest, @@ -2142,6 +2191,7 @@ pub trait ServerApi { )] async fn runtime_worker_input( &self, + #[extension] context: ServerRequestContext, #[path] workspace_id: String, #[path] runtime_id: String, #[path] worker_id: String, @@ -2258,6 +2308,7 @@ pub trait ServerApi { )] async fn runtime_worker_stop_alias( &self, + #[extension] context: ServerRequestContext, #[path] runtime_id: String, #[path] worker_id: String, #[body] request: RuntimeWorkerLifecycleRequest, @@ -2273,6 +2324,7 @@ pub trait ServerApi { )] async fn runtime_worker_stop( &self, + #[extension] context: ServerRequestContext, #[path] workspace_id: String, #[path] runtime_id: String, #[path] worker_id: String, @@ -2289,6 +2341,7 @@ pub trait ServerApi { )] async fn runtime_worker_cancel_alias( &self, + #[extension] context: ServerRequestContext, #[path] runtime_id: String, #[path] worker_id: String, #[body] request: RuntimeWorkerLifecycleRequest, @@ -2304,6 +2357,7 @@ pub trait ServerApi { )] async fn runtime_worker_cancel( &self, + #[extension] context: ServerRequestContext, #[path] workspace_id: String, #[path] runtime_id: String, #[path] worker_id: String, @@ -2549,6 +2603,81 @@ pub trait ServerApi { #[path] working_directory_id: String, #[body] request: WorkingDirectoryRemovalRequest, ) -> Result; + #[get("/api/w/{workspace_id}/workers/self/workspace-config", status = 200, error_status = 400, additional_error_statuses = [401,403,404,409,413,500,503], openapi = false)] + async fn current_worker_workspace_config_get( + &self, + #[extension] context: ServerRequestContext, + #[path] workspace_id: String, + ) -> Result; + #[post("/api/w/{workspace_id}/workers/self/workspace-config", status = 200, error_status = 400, additional_error_statuses = [401,403,404,409,413,500,503], openapi = false)] + async fn current_worker_workspace_config_attach( + &self, + #[extension] context: ServerRequestContext, + #[path] workspace_id: String, + #[body] request: WorkspaceConfigAttachRequest, + ) -> Result; + #[post("/api/w/{workspace_id}/workers/self/workspace-config/observe", status = 200, error_status = 400, additional_error_statuses = [401,403,404,409,413,500,503], openapi = false)] + async fn current_worker_workspace_config_observe( + &self, + #[extension] context: ServerRequestContext, + #[path] workspace_id: String, + #[body] request: WorkspaceConfigObserveRequest, + ) -> Result; + #[post("/api/w/{workspace_id}/workers/self/workspace-config/read", status = 200, error_status = 400, additional_error_statuses = [401,403,404,409,413,500,503], openapi = false)] + async fn current_worker_workspace_config_read( + &self, + #[extension] context: ServerRequestContext, + #[path] workspace_id: String, + #[body] request: WorkspaceConfigReadRequest, + ) -> Result; + #[post("/api/w/{workspace_id}/workers/self/workspace-config/commit", status = 200, error_status = 400, additional_error_statuses = [401,403,404,409,413,500,503], openapi = false)] + async fn current_worker_workspace_config_commit( + &self, + #[extension] context: ServerRequestContext, + #[path] workspace_id: String, + #[body] request: WorkspaceConfigCommitRequest, + ) -> Result; + #[post("/api/w/{workspace_id}/workspace-config-grants", status = 200, error_status = 400, additional_error_statuses = [401,403,404,409,500], bearer_auth = true, browser_auth = true)] + async fn workspace_config_grant_create( + &self, + #[extension] actor: RequestActor, + #[path] workspace_id: String, + #[body] request: WorkspaceConfigGrantCreateRequest, + ) -> Result; + #[delete("/api/w/{workspace_id}/workspace-config-grants/{grant_id}", status = 200, error_status = 400, additional_error_statuses = [401,403,404,409,500], bearer_auth = true, browser_auth = true)] + async fn workspace_config_grant_revoke( + &self, + #[extension] actor: RequestActor, + #[path] workspace_id: String, + #[path] grant_id: String, + ) -> Result; + + #[get( + "/api/w/{workspace_id}/workers/self/workdir-catalog", + status = 200, + error_status = 400, + additional_error_statuses = [401, 403, 404, 500], + openapi = false, + )] + async fn current_worker_workdir_catalog( + &self, + #[extension] context: ServerRequestContext, + #[path] workspace_id: String, + #[query] query: CurrentWorkerWorkdirCatalogQuery, + ) -> Result; + #[get( + "/api/w/{workspace_id}/workers/self/workdir-attachments", + status = 200, + error_status = 400, + additional_error_statuses = [401, 403, 404, 500], + openapi = false, + )] + async fn current_worker_workdir_attachment_list( + &self, + #[extension] context: ServerRequestContext, + #[path] workspace_id: String, + #[query] query: CurrentWorkerWorkdirAttachmentListQuery, + ) -> Result; #[post( "/api/w/{workspace_id}/workers/self/workdir-attachments", status = 200, @@ -2574,6 +2703,7 @@ pub trait ServerApi { #[extension] context: ServerRequestContext, #[path] workspace_id: String, #[path] alias: String, + #[query] query: CurrentWorkerWorkdirDetachQuery, ) -> Result; #[post( "/api/w/{workspace_id}/workers/self/workdir-session/operations", @@ -4579,9 +4709,20 @@ pub enum SubjektivSubjectState { #[serde(deny_unknown_fields)] pub struct SubjektivSubjectCreateRequest { pub role: String, + #[serde(default)] + pub behavior_md: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SubjektivSubjectBehaviorUpdateRequest { + #[schemars(range(min = 0, max = 9_007_199_254_740_991_u64))] + pub expected_behavior_revision: u64, + pub behavior_md: String, } pub const SUBJEKTIV_BROWSER_MAX_LIST_LIMIT: usize = 100; +pub const SUBJEKTIV_MAX_BEHAVIOR_BYTES: usize = 16 * 1024; #[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] #[serde(deny_unknown_fields)] @@ -4609,6 +4750,9 @@ pub struct SubjektivSubjectListResponse { pub struct SubjektivSubjectResponse { pub id: String, pub role: String, + pub behavior_md: String, + #[schemars(range(min = 0, max = 9_007_199_254_740_991_u64))] + pub behavior_revision: u64, pub state: SubjektivSubjectState, #[schemars(range(min = 0, max = 9_007_199_254_740_991_u64))] pub store_revision: u64, @@ -5386,6 +5530,9 @@ pub struct SubjektivSurfacePrepareRequest {} #[serde(deny_unknown_fields)] pub struct SubjektivSurfacePrepareResponse { pub generation_id: String, + /// Existing ready surface at this exact store revision, if no rebuild is needed. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub current_snapshot_id: Option, #[schemars(range(min = 0, max = 9_007_199_254_740_991_u64))] pub store_revision: u64, #[schemars(range(min = 0, max = 9_007_199_254_740_991_usize))] @@ -5442,10 +5589,22 @@ pub struct SubjektivSurfaceFailureResponse { pub status: String, } +#[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct SubjektivResidentContextOutput { + /// Exact user-managed document. Empty means explicitly unset. + pub behavior_md: String, + #[schemars(range(min = 0, max = 9_007_199_254_740_991_u64))] + pub behavior_revision: u64, + /// Independently-fresh generated Memory projection. + pub memory_surface: memory::backend::MemoryResidentSummaryOutput, +} + #[derive(Debug, Clone, Serialize, Deserialize, JsonSchema)] #[serde(tag = "operation", content = "input", rename_all = "snake_case")] pub enum SubjektivMemoryBackendOperation { ResidentSummary(memory::backend::MemoryResidentSummaryOperation), + ResidentContext(memory::backend::MemoryResidentSummaryOperation), Query(SubjektivMemoryQueryRequest), Read(SubjektivMemoryReadRequest), ListRevisions(SubjektivMemoryListRevisionsRequest), @@ -5618,6 +5777,7 @@ pub struct SubjektivMemoryStageExplicitResponse { #[serde(tag = "result", content = "data", rename_all = "snake_case")] pub enum SubjektivMemoryBackendResponse { ResidentSummary(memory::backend::MemoryResidentSummaryOutput), + ResidentContext(SubjektivResidentContextOutput), Query(SubjektivMemoryQueryResponse), Read(SubjektivMemoryReadResponse), ListRevisions(SubjektivMemoryListRevisionsResponse), @@ -6248,6 +6408,7 @@ pub struct Diagnostic { #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[serde(rename_all = "snake_case")] pub enum WorkingDirectoryMaterializerKind { + LogicalWorkspaceConfig, #[default] RuntimeGitClone, ClientHostedExternal, @@ -6387,6 +6548,11 @@ pub struct RuntimeWorkingDirectorySummary { #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] pub enum WorkingDirectorySource { + WorkspaceConfig { + access: WorkspaceConfigAccess, + content_path: String, + purpose: String, + }, Repository { repository_key: String, }, @@ -6754,6 +6920,69 @@ pub struct CurrentWorkerWorkdirAttachRequest { pub working_directory_id: String, } +/// Stable Workdir-ID keyset paging over the authoritative inventory, not a Runtime snapshot. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CurrentWorkerWorkdirCatalogQuery { + /// Defaults to 50; valid range is 1..=100 scanned registry records. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub limit: Option, + /// Exclusive stable Workdir ID. An empty policy-filtered page can still have a next cursor. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cursor: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CurrentWorkerWorkdirCatalogResponse { + pub workspace_id: String, + pub items: Vec, + pub next_cursor: Option, + /// Opaque digest of the complete authoritative inventory and its occupancy. + pub revision: String, +} + +/// Bounded, alias-ordered paging over the current Worker's active attachments. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CurrentWorkerWorkdirAttachmentListQuery { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub limit: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub offset: Option, + /// Exact caller-scoped connection lookup, applied before paging; 1..=128 bytes. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub connection_id: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CurrentWorkerWorkdirAttachmentItem { + pub alias: String, + pub working_directory_id: String, + pub capabilities: workdir::WorkdirSessionCapabilities, + /// Opaque durable identity of this attachment lifetime, not a Workdir ID. + pub connection_id: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CurrentWorkerWorkdirAttachmentListResponse { + pub workspace_id: String, + pub items: Vec, + pub next_offset: Option, + /// Opaque digest of this caller's complete active connection set, independent of query paging/filtering. + pub revision: String, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CurrentWorkerWorkdirDetachQuery { + /// When omitted, retain the normal Tool's unconditional detach contract. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub expected_connection_id: Option, +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] #[serde(deny_unknown_fields)] pub struct CurrentWorkerWorkdirAttachmentResponse { @@ -8233,7 +8462,7 @@ pub struct RuntimeWorkerInputRequest { pub kind: Option, pub content: String, #[serde(default, skip_serializing_if = "Option::is_none")] - pub segments: Option>, + pub segments: Option>, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] @@ -8268,9 +8497,13 @@ pub struct RuntimeWorkerLifecycleResult { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, JsonSchema)] #[serde(deny_unknown_fields)] pub struct RuntimeWorkerCompletionsRequest { - pub kind: serde_json::Value, + pub kind: protocol::CompletionKind, #[serde(default)] pub prefix: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub context: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub request_id: Option, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, JsonSchema)] @@ -8278,9 +8511,13 @@ pub struct RuntimeWorkerCompletionsRequest { pub struct RuntimeWorkerCompletionsResult { pub runtime_id: String, pub worker_id: String, - pub kind: serde_json::Value, + pub kind: protocol::CompletionKind, pub prefix: String, - pub entries: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub context: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub request_id: Option, + pub entries: Vec, #[serde(default)] pub diagnostics: Vec, } @@ -9364,6 +9601,10 @@ pub struct RuntimeWorkerWorkdirAttachmentSummary { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] pub struct WorkerSummary { + /// Opaque Runtime-owned observation fence used by explicit Restore intent. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[cfg_attr(feature = "typescript", ts(optional = nullable))] + pub restore_observation_token: Option, pub runtime_id: String, pub worker_id: String, pub resource_key: String, @@ -9600,6 +9841,16 @@ pub enum WorkerRestoreState { ReconciliationRequired, } +/// One explicit Restore intent. Retries retain both fields; conflicts require a +/// fresh observation and a deliberate new intent, not automatic token replacement. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] +pub struct WorkerRestoreRequest { + pub expected_observation_token: String, + pub request_id: String, +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[serde(deny_unknown_fields)] @@ -9861,9 +10112,11 @@ pub fn memory_api_typescript() -> String { // `ts-rs` solely for this read-only projection. let subjektiv_browser_declarations = r#"export type SubjektivSubjectState = "active" | "retired"; -export type SubjektivSubjectCreateRequest = { role: string, }; +export type SubjektivSubjectCreateRequest = { role: string, behavior_md?: string, }; -export type SubjektivSubjectResponse = { id: string, role: string, state: SubjektivSubjectState, store_revision: number, created_at: string, updated_at: string, current_worker?: | import("./worker-launch-api").WorkerLaunchWorkerSummary | null, }; +export type SubjektivSubjectBehaviorUpdateRequest = { expected_behavior_revision: number, behavior_md: string, }; + +export type SubjektivSubjectResponse = { id: string, role: string, behavior_md: string, behavior_revision: number, state: SubjektivSubjectState, store_revision: number, created_at: string, updated_at: string, current_worker?: | import("./worker-launch-api").WorkerLaunchWorkerSummary | null, }; export type SubjektivSubjectListResponse = { limit: number, items: Array, next_cursor?: string | null, has_more: boolean, }; @@ -9896,8 +10149,11 @@ export type SubjektivMemoryListRevisionsResponse = { memory_id: string, current_ let limits = format!( "export const MEMORY_API_LIMITS = {{\n maxResponseBytes: {MEMORY_API_MAX_RESPONSE_BYTES},\n maxDocumentBytes: {MEMORY_API_MAX_DOCUMENT_BYTES},\n maxCollectionItems: {MEMORY_API_MAX_COLLECTION_ITEMS},\n maxStringBytes: {MEMORY_API_MAX_STRING_BYTES},\n maxIdentifierBytes: {MEMORY_API_MAX_IDENTIFIER_BYTES},\n}} as const;" ); + let subjektiv_limits = format!( + "export const SUBJEKTIV_API_LIMITS = {{\n maxBehaviorBytes: {SUBJEKTIV_MAX_BEHAVIOR_BYTES},\n}} as const;" + ); format!( - "// Generated from server-api. Do not edit by hand.\n// Regenerate: cargo run -q -p server-api --features typescript --example generate_memory_api_types > web/workspace/src/lib/generated/memory-api.ts\n\n{limits}\n\n{}\n\n{subjektiv_browser_declarations}\n", + "// Generated from server-api. Do not edit by hand.\n// Regenerate: cargo run -q -p server-api --features typescript --example generate_memory_api_types > web/workspace/src/lib/generated/memory-api.ts\n\n{limits}\n\n{subjektiv_limits}\n\n{}\n\n{subjektiv_browser_declarations}\n", declarations .into_iter() .map(|declaration| format!("export {declaration}")) @@ -10572,6 +10828,22 @@ pub fn legacy_catalog_typescript() -> String { ToolchainContract::decl(&config), WorkspaceConfigTreeResponse::decl(&config), ConfigCommitRequest::decl(&config), + WorkspaceConfigAccess::decl(&config), + WorkspaceConfigNodeKind::decl(&config), + WorkspaceConfigFailureClassification::decl(&config), + WorkspaceConfigAttachRequest::decl(&config), + WorkspaceConfigAttachment::decl(&config), + WorkspaceConfigCurrentResponse::decl(&config), + WorkspaceConfigObserveRequest::decl(&config), + WorkspaceConfigNode::decl(&config), + WorkspaceConfigObserveResponse::decl(&config), + WorkspaceConfigReadRequest::decl(&config), + WorkspaceConfigReadResponse::decl(&config), + WorkspaceConfigCommitRequest::decl(&config), + WorkspaceConfigCommitResponse::decl(&config), + WorkspaceConfigApiError::decl(&config), + WorkspaceConfigGrantCreateRequest::decl(&config), + WorkspaceConfigGrantResponse::decl(&config), ProfileSettingsResponse::decl(&config), WorkspaceProfileSummary::decl(&config), WorkspaceProfileSourceSummary::decl(&config), @@ -11443,6 +11715,28 @@ mod tests { ); } + #[test] + fn worker_restore_request_requires_observation_and_request_identity() { + let valid = serde_json::json!({"expected_observation_token":"opaque-generation", "request_id":"restore-request"}); + let request: WorkerRestoreRequest = serde_json::from_value(valid.clone()).unwrap(); + assert_eq!(serde_json::to_value(request).unwrap(), valid); + for invalid in [ + serde_json::json!({}), + serde_json::json!({"request_id":"restore-request"}), + serde_json::json!({"expected_observation_token":"opaque-generation"}), + serde_json::json!({"expected_observation_token":"opaque-generation", "request_id":"restore-request", "command_id":701}), + ] { + assert!(serde_json::from_value::(invalid).is_err()); + } + let contract = canonical_openapi_document().unwrap().to_json().unwrap(); + let contract: serde_json::Value = serde_json::from_str(&contract).unwrap(); + let restore = &contract["paths"]["/api/w/{workspace_id}/runtimes/{runtime_id}/workers/{worker_id}/restore"] + ["post"]; + assert_eq!(restore["requestBody"]["required"], true); + assert!(restore["responses"]["409"].is_object()); + assert!(contract["components"]["schemas"]["WorkerRestoreRequest"].is_object()); + } + #[test] fn worker_restore_result_rejects_unknown_fields() { let result = serde_json::from_value::(serde_json::json!({ @@ -11930,6 +12224,135 @@ mod tests { ); } + #[test] + fn current_worker_attachment_contract_keeps_normal_tool_response_closed() { + let lookup = CurrentWorkerWorkdirAttachmentListQuery { + limit: Some(1), + offset: None, + connection_id: Some("opaque-connection".to_string()), + }; + let value = serde_json::json!({"limit": 1, "connection_id": "opaque-connection"}); + assert_eq!(serde_json::to_value(&lookup).unwrap(), value); + assert_eq!( + serde_json::from_value::(value).unwrap(), + lookup + ); + assert_eq!( + serde_json::to_value(CurrentWorkerWorkdirAttachmentListQuery::default()).unwrap(), + serde_json::json!({}) + ); + assert!( + serde_json::from_value::( + serde_json::json!({"connection_id": 123}) + ) + .is_err() + ); + assert!( + serde_json::from_value::( + serde_json::json!({"expected_connection_id": "wrong-field"}) + ) + .is_err() + ); + let query: CurrentWorkerWorkdirDetachQuery = + serde_json::from_value(serde_json::json!({})).unwrap(); + assert_eq!(query.expected_connection_id, None); + assert_eq!(serde_json::to_value(query).unwrap(), serde_json::json!({})); + assert!( + serde_json::from_value::( + serde_json::json!({"connection_id":"wrong-field"}) + ) + .is_err() + ); + let response = CurrentWorkerWorkdirAttachmentResponse { + workspace_id: "workspace".to_string(), + alias: "checkout".to_string(), + working_directory_id: "workdir".to_string(), + capabilities: workdir::WorkdirSessionCapabilities::EMPTY, + attached: false, + }; + let value = serde_json::to_value(&response).unwrap(); + assert_eq!(value.as_object().unwrap().len(), 5); + assert!(value.get("connection_id").is_none()); + assert_eq!( + serde_json::from_value::(value).unwrap(), + response + ); + let list = CurrentWorkerWorkdirAttachmentListResponse { + workspace_id: "workspace".to_string(), + revision: "opaque-revision".to_string(), + next_offset: Some(1), + items: vec![CurrentWorkerWorkdirAttachmentItem { + alias: "checkout".to_string(), + working_directory_id: "workdir".to_string(), + capabilities: workdir::WorkdirSessionCapabilities::READ_ONLY, + connection_id: "opaque".to_string(), + }], + }; + assert_eq!( + serde_json::from_value::( + serde_json::to_value(&list).unwrap() + ) + .unwrap(), + list + ); + } + + #[test] + fn current_worker_workdir_catalog_contract_is_bounded_and_revision_is_required() { + let query = CurrentWorkerWorkdirCatalogQuery { + limit: Some(100), + cursor: Some("stable-id".to_string()), + }; + assert_eq!( + serde_json::from_value::( + serde_json::to_value(&query).unwrap() + ) + .unwrap(), + query + ); + assert_eq!( + serde_json::to_value(CurrentWorkerWorkdirCatalogQuery::default()).unwrap(), + serde_json::json!({}) + ); + assert!( + serde_json::from_value::( + serde_json::json!({"offset": 0}) + ) + .is_err() + ); + assert!( + serde_json::from_value::( + serde_json::json!({"limit": -1}) + ) + .is_err() + ); + let response = CurrentWorkerWorkdirCatalogResponse { + workspace_id: "workspace".to_string(), + items: Vec::new(), + next_cursor: Some("scanned-legacy-row".to_string()), + revision: "opaque".to_string(), + }; + assert_eq!( + serde_json::from_value::( + serde_json::to_value(&response).unwrap() + ) + .unwrap(), + response + ); + assert!( + serde_json::from_value::( + serde_json::json!({"workspace_id": "workspace", "items": [], "next_cursor": null}) + ) + .is_err() + ); + assert!( + serde_json::from_value::( + serde_json::json!({"workspace_id": "workspace", "items": [], "next_offset": null}) + ) + .is_err() + ); + } + #[test] fn workspace_and_repository_response_shapes_round_trip() { let workspace = serde_json::json!({ @@ -12661,6 +13084,45 @@ mod tests { assert!(serde_json::from_value::(private_item).is_err()); } + #[test] + fn structured_invocation_input_round_trips_through_typed_api() { + let json = serde_json::json!({"kind":"user", "content":"", "segments":[{"kind":"feature_invoke", "invocation":{ + "invocation_id":"stable-invoke", "identity":"builtin:test/prepare", "name":"prepare", + "arguments":[{"name":"path", "value":{"kind":"string","value":"資料/a b"}}] + }}]}); + let request: RuntimeWorkerInputRequest = serde_json::from_value(json.clone()).unwrap(); + assert!( + matches!(&request.segments.as_ref().unwrap()[0], protocol::Segment::FeatureInvoke { invocation } + if invocation.identity.0 == "builtin:test/prepare" && invocation.invocation_id == "stable-invoke") + ); + assert_eq!(serde_json::to_value(request).unwrap(), json); + let mut invalid = json; + invalid["segments"][0]["invocation"]["arguments"][0]["value"] = + serde_json::json!({"kind":"integer","value":2147483648_i64}); + assert!(serde_json::from_value::(invalid).is_err()); + } + + #[test] + fn feature_completion_contract_is_typed_and_preserves_argument_context() { + let json = serde_json::json!({"kind":"feature_argument", "prefix":"資料/", "context":{"invocation":"builtin:test/prepare","argument":"path"}}); + let request: RuntimeWorkerCompletionsRequest = + serde_json::from_value(json.clone()).unwrap(); + assert_eq!(request.kind, protocol::CompletionKind::FeatureArgument); + assert_eq!(serde_json::to_value(request).unwrap(), json); + assert!( + serde_json::from_value::( + serde_json::json!({"kind":"unknown"}) + ) + .is_err() + ); + assert!( + serde_json::from_value::( + serde_json::json!({"kind":"feature_argument", "context":{"invocation":123}}) + ) + .is_err() + ); + } + #[test] fn generated_companion_api_contract_is_current() { let root = std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../.."); @@ -12895,6 +13357,13 @@ mod openapi_artifact_tests { // identity, or a one-use Reviewer capability. They remain generated Rust client/Axum // operations, but must not appear as unauthenticated operations in the public OpenAPI. const SIGNED_INTERNAL: &[&str] = &[ + "current_worker_workspace_config_get", + "current_worker_workspace_config_attach", + "current_worker_workspace_config_observe", + "current_worker_workspace_config_read", + "current_worker_workspace_config_commit", + "current_worker_workdir_catalog", + "current_worker_workdir_attachment_list", "current_worker_workdir_attach", "current_worker_workdir_detach", "current_worker_workdir_operation", diff --git a/crates/server-api/src/repository_openapi_typescript.rs b/crates/server-api/src/repository_openapi_typescript.rs index a561bf71c..658133616 100644 --- a/crates/server-api/src/repository_openapi_typescript.rs +++ b/crates/server-api/src/repository_openapi_typescript.rs @@ -872,14 +872,14 @@ fn validate_runtime_operations( "/api/runtimes/{runtime_id}/workers/{worker_id}/restore", "post", "runtime_worker_restore_alias", - None, + Some("WorkerRestoreRequest"), &[("200", "WorkerRestoreResponse")], ), runtime_operation( "/api/w/{workspace_id}/runtimes/{runtime_id}/workers/{worker_id}/restore", "post", "runtime_worker_restore", - None, + Some("WorkerRestoreRequest"), &[("200", "WorkerRestoreResponse")], ), runtime_operation( @@ -1227,8 +1227,16 @@ fn json_content_schema<'a>( } fn schema_reference<'a>(value: &'a Value, context: &str) -> Result<&'a str, GenerationError> { - const ALLOWED_REFERENCE_SIBLINGS: &[&str] = - &["$ref", "$comment", "deprecated", "description", "title"]; + // `default` is a JSON Schema annotation, not a constraint or a replacement + // for the referenced type. Schemars emits it for defaulted enum fields. + const ALLOWED_REFERENCE_SIBLINGS: &[&str] = &[ + "$ref", + "$comment", + "default", + "deprecated", + "description", + "title", + ]; let schema = object(value, context)?; if let Some(keyword) = schema .keys() @@ -2023,6 +2031,27 @@ mod tests { assert!(error.to_string().contains("ambiguous anyOf")); } + #[test] + fn reference_default_annotations_do_not_replace_the_referenced_type() { + let value = serde_json::json!({"$ref":"#/components/schemas/InvocationCompletion", "default":{"kind":"none"}}); + assert_eq!( + schema_reference(&value, "completion").unwrap(), + "#/components/schemas/InvocationCompletion" + ); + let output = generate_runtime_typescript( + &crate::canonical_openapi_document() + .unwrap() + .to_json() + .unwrap(), + ) + .unwrap(); + assert!(output.contains("completion?: InvocationCompletion")); + assert!(output.contains("export type FeatureInvocation =")); + assert!(output.contains( + "export type CompletionKind = \"file\" | \"feature\" | \"feature_argument\"" + )); + } + #[test] fn generator_rejects_lossy_reference_siblings() { let mut document: Value = serde_json::from_str(OPENAPI).unwrap(); diff --git a/crates/server-api/src/workspace_config.rs b/crates/server-api/src/workspace_config.rs new file mode 100644 index 000000000..905c9248c --- /dev/null +++ b/crates/server-api/src/workspace_config.rs @@ -0,0 +1,284 @@ +//! Grant-bound logical Workspace configuration transport. No host filesystem contract. +use super::{ConfigCommitRequest, ConfigContentType}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(rename_all = "snake_case")] +pub enum WorkspaceConfigAccess { + ReadOnly, + ReadWrite, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(rename_all = "snake_case")] +pub enum WorkspaceConfigNodeKind { + Directory, + File, + Missing, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(rename_all = "snake_case")] +pub enum WorkspaceConfigFailureClassification { + NotCommitted, + Unknown, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] +pub struct WorkspaceConfigAttachRequest { + #[serde(default)] + pub alias: Option, + #[serde(default)] + pub access: Option, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] +pub struct WorkspaceConfigAttachment { + pub workspace_id: String, + pub connection_id: String, + pub alias: String, + pub working_directory_id: String, + pub access: WorkspaceConfigAccess, + pub name: String, + pub purpose: String, + pub content_path: String, + pub already_attached: bool, +} +/// Named transport wrapper required by the generated API; JSON remains attachment-or-null. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(transparent)] +pub struct WorkspaceConfigCurrentResponse(pub Option); + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] +pub struct WorkspaceConfigObserveRequest { + pub connection_id: String, + pub paths: Vec, + pub depth: u32, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] +pub struct WorkspaceConfigNode { + pub path: String, + pub kind: WorkspaceConfigNodeKind, + pub validator: String, + pub digest: Option, + pub content_type: Option, + pub operations: Vec, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] +pub struct WorkspaceConfigObserveResponse { + pub connection_id: String, + pub validator: String, + #[cfg_attr(feature = "typescript", ts(type = "number"))] + #[schemars(range(min = 0, max = 9_007_199_254_740_991_u64))] + pub revision: u64, + pub digest: String, + /// Canonical commit entrypoints from this exact metadata snapshot. + pub entrypoints: Vec, + pub nodes: Vec, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] +pub struct WorkspaceConfigReadRequest { + pub connection_id: String, + pub path: String, + pub validator: String, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] +pub struct WorkspaceConfigReadResponse { + pub path: String, + pub content: String, + pub content_type: ConfigContentType, + pub digest: String, + pub validator: String, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] +pub struct WorkspaceConfigCommitRequest { + pub connection_id: String, + pub validator: String, + pub request: ConfigCommitRequest, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] +pub struct WorkspaceConfigCommitResponse { + pub validator: String, + #[cfg_attr(feature = "typescript", ts(type = "number"))] + #[schemars(range(min = 0, max = 9_007_199_254_740_991_u64))] + pub revision: u64, + pub digest: String, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] +pub struct WorkspaceConfigApiError { + pub status: u16, + pub code: String, + pub message: String, + pub classification: WorkspaceConfigFailureClassification, +} +impl api_macros::HttpError for WorkspaceConfigApiError { + fn status_code(&self) -> u16 { + self.status + } +} +impl api_macros::HttpRequestError for WorkspaceConfigApiError { + fn from_request_rejection(status: u16, _message: String) -> Self { + Self { + status, + code: "invalid_request".into(), + message: "Invalid configuration request".into(), + classification: WorkspaceConfigFailureClassification::NotCommitted, + } + } +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] +pub struct WorkspaceConfigGrantCreateRequest { + pub runtime_id: String, + pub worker_id: String, + pub access: WorkspaceConfigAccess, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] +pub struct WorkspaceConfigGrantResponse { + pub grant_id: String, + pub workspace_id: String, + pub runtime_id: String, + pub worker_id: String, + pub working_directory_id: String, + pub access: WorkspaceConfigAccess, + pub revoked: bool, +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::{ApiContract, HttpMethod, ServerApiMetadata}; + + #[test] + fn workspace_config_observation_is_metadata_only_and_has_same_snapshot_cas_inputs() { + let value = serde_json::json!({"connection_id":"c", "validator":"v", "revision":7, "digest":"d", "entrypoints":["main.dcdl"], "nodes":[{"path":"", "kind":"directory", "validator":"v", "digest":null, "content_type":null, "operations":["apply_changes"]}]}); + let response: WorkspaceConfigObserveResponse = + serde_json::from_value(value.clone()).unwrap(); + assert_eq!(serde_json::to_value(response).unwrap(), value); + for field in ["revision", "digest", "entrypoints"] { + let mut missing = value.clone(); + missing.as_object_mut().unwrap().remove(field); + assert!(serde_json::from_value::(missing).is_err()); + } + let mut with_body = value; + with_body["nodes"][0]["content"] = serde_json::json!("must never appear here"); + assert!(serde_json::from_value::(with_body).is_err()); + assert!( + serde_json::from_value::( + serde_json::json!({"runtime_id":"r","worker_id":"w","access":"read_write_command"}) + ) + .is_err() + ); + assert!( + serde_json::from_value::( + serde_json::json!({"grant_id":"invented"}) + ) + .is_err() + ); + assert_eq!( + serde_json::to_value(WorkspaceConfigCurrentResponse(None)).unwrap(), + serde_json::Value::Null + ); + } + + #[test] + fn workspace_config_failure_classification_roundtrips_and_rejections_are_pre_effects() { + for classification in [ + WorkspaceConfigFailureClassification::NotCommitted, + WorkspaceConfigFailureClassification::Unknown, + ] { + let error = WorkspaceConfigApiError { + status: 500, + code: "unavailable".into(), + message: "safe".into(), + classification, + }; + assert_eq!( + serde_json::from_value::( + serde_json::to_value(&error).unwrap() + ) + .unwrap(), + error + ); + assert_eq!(api_macros::HttpError::status_code(&error), 500); + } + let error = + ::from_request_rejection( + 413, + "/private/secret".into(), + ); + assert_eq!( + error.classification, + WorkspaceConfigFailureClassification::NotCommitted + ); + assert!(!error.message.contains("private")); + } + + #[test] + fn workspace_config_generated_routes_are_exact_and_grants_are_separate() { + for (name, method, suffix) in [ + ("get", HttpMethod::Get, ""), + ("attach", HttpMethod::Post, ""), + ("observe", HttpMethod::Post, "/observe"), + ("read", HttpMethod::Post, "/read"), + ("commit", HttpMethod::Post, "/commit"), + ] { + let name = format!("current_worker_workspace_config_{name}"); + let operation = ServerApiMetadata::OPERATIONS + .iter() + .find(|operation| operation.operation_id == name) + .unwrap(); + assert_eq!(operation.method, method); + assert_eq!( + operation.path, + format!("/api/w/{{workspace_id}}/workers/self/workspace-config{suffix}") + ); + } + for (name, method, path) in [ + ( + "workspace_config_grant_create", + HttpMethod::Post, + "/api/w/{workspace_id}/workspace-config-grants", + ), + ( + "workspace_config_grant_revoke", + HttpMethod::Delete, + "/api/w/{workspace_id}/workspace-config-grants/{grant_id}", + ), + ] { + let operation = ServerApiMetadata::OPERATIONS + .iter() + .find(|operation| operation.operation_id == name) + .unwrap(); + assert_eq!(operation.method, method); + assert_eq!(operation.path, path); + } + } +} diff --git a/crates/session-store/src/fs_store.rs b/crates/session-store/src/fs_store.rs index e5babb45c..e9ca34761 100644 --- a/crates/session-store/src/fs_store.rs +++ b/crates/session-store/src/fs_store.rs @@ -383,8 +383,11 @@ impl Store for FsStore { content.push_str(&serde_json::to_string(entry)?); content.push('\n'); } - fs::write(&path, content.as_bytes())?; - Ok(()) + let _guard = self + .append_lock + .lock() + .map_err(|_| std::io::Error::other("session store append lock was poisoned"))?; + crate::worker_session_store::atomic_write_bytes(&path, content.as_bytes()) } fn exists(&self, session_id: SessionId, segment_id: SegmentId) -> Result { diff --git a/crates/session-store/src/store.rs b/crates/session-store/src/store.rs index a32f9038e..8e6f6b720 100644 --- a/crates/session-store/src/store.rs +++ b/crates/session-store/src/store.rs @@ -110,7 +110,14 @@ pub trait Store: Send + Sync { /// segment ID without its session ID (e.g. legacy `--session `). fn lookup_session_of(&self, segment_id: SegmentId) -> Result, StoreError>; - /// Create a new segment within `session_id`, with initial entries. + /// Create or atomically replace a segment within `session_id` with all entries. + /// + /// Readers and crash recovery must observe either the previous complete log + /// or the complete replacement, never a truncated prefix or partial record. + /// Failure before replacement leaves the old log intact; an error after the + /// atomic commit (for example a durability sync failure) may leave the complete + /// replacement. Filesystem implementations must sync replacement bytes before + /// rename and the containing directory afterward before reporting success. fn create_segment( &self, session_id: SessionId, diff --git a/crates/session-store/src/system_item.rs b/crates/session-store/src/system_item.rs index 4d4a28b6a..0f00465cd 100644 --- a/crates/session-store/src/system_item.rs +++ b/crates/session-store/src/system_item.rs @@ -123,6 +123,14 @@ pub enum SystemItem { prompt_provenance: Option, }, + /// Result of an explicitly selected chat Feature invocation. The typed + /// result is retained for clients; `body` is the exact bounded context made + /// visible to the Worker model. + FeatureInvocationResult { + result: protocol::FeatureInvocationResult, + body: String, + }, + /// `@` file reference resolution. `body` is the rendered /// LLM-context text (`[File: ]\n…` for regular files, /// `[Dir: ]\n…` for directory listings, possibly with a @@ -167,6 +175,15 @@ pub enum SystemItem { prompt_provenance: Option, }, + /// A Host-fetched change to user-managed Subject behavior. Each changed + /// revision is appended rather than rewriting prior prompt/session content. + SubjectBehaviorRefresh { + body: String, + behavior_revision: u64, + #[serde(default, skip_serializing_if = "Option::is_none")] + prompt_provenance: Option, + }, + /// Synthetic note inserted after an interrupted turn before the next /// user input. `body` is the exact LLM-context text explaining that the /// previous turn was cut short. @@ -184,6 +201,7 @@ impl SystemItem { match self { SystemItem::Notification { body, .. } => body.clone(), SystemItem::WorkerEvent { body, .. } => body.clone(), + SystemItem::FeatureInvocationResult { body, .. } => body.clone(), SystemItem::FileAttachment { body, .. } => body.clone(), SystemItem::SkillActivation { body, .. } => body.clone(), SystemItem::LegacyKnowledgeIgnored { .. } => String::new(), @@ -192,6 +210,7 @@ impl SystemItem { } SystemItem::TaskReminder { body, .. } => body.clone(), SystemItem::ResidentSummaryRefresh { body, .. } => body.clone(), + SystemItem::SubjectBehaviorRefresh { body, .. } => body.clone(), SystemItem::Interrupt { body, .. } => body.clone(), } } @@ -208,12 +227,14 @@ impl SystemItem { match self { SystemItem::Notification { .. } => "notification", SystemItem::WorkerEvent { .. } => "worker_event", + SystemItem::FeatureInvocationResult { .. } => "feature_invocation_result", SystemItem::FileAttachment { .. } => "file_attachment", SystemItem::SkillActivation { .. } => "skill_activation", SystemItem::LegacyKnowledgeIgnored { .. } => "legacy_knowledge_ignored", SystemItem::LegacyIgnored { .. } => "legacy_ignored", SystemItem::TaskReminder { .. } => "task_reminder", SystemItem::ResidentSummaryRefresh { .. } => "resident_summary_refresh", + SystemItem::SubjectBehaviorRefresh { .. } => "subject_behavior_refresh", SystemItem::Interrupt { .. } => "interrupt", } } @@ -270,6 +291,25 @@ mod tests { )); } + #[test] + fn subject_behavior_refresh_round_trips_with_revision_and_exact_body() { + let item = SystemItem::SubjectBehaviorRefresh { + body: "current Subject context".to_string(), + behavior_revision: 7, + prompt_provenance: None, + }; + let raw = serde_json::to_string(&item).unwrap(); + let parsed: SystemItem = serde_json::from_str(&raw).unwrap(); + assert_eq!(parsed.history_text(), "current Subject context"); + assert!(matches!( + parsed, + SystemItem::SubjectBehaviorRefresh { + behavior_revision: 7, + .. + } + )); + } + #[test] fn notification_history_text_returns_stored_body() { let item = SystemItem::Notification { @@ -363,6 +403,29 @@ mod tests { } } + #[test] + fn invocation_result_retains_typed_outcome_and_exact_context() { + let result = protocol::FeatureInvocationResult { + invocation_id: "stable-invoke".into(), + identity: protocol::FeatureInvocationIdentity("builtin:test/prepare".into()), + status: protocol::FeatureInvocationStatus::OutcomeUnknown, + message: "operation may have committed".into(), + context: None, + }; + let body = "Outcome unknown; do not retry automatically"; + let item = SystemItem::FeatureInvocationResult { + result: result.clone(), + body: body.into(), + }; + let restored: SystemItem = + serde_json::from_str(&serde_json::to_string(&item).unwrap()).unwrap(); + assert_eq!(restored.history_text(), body); + assert_eq!(restored.kind_label(), "feature_invocation_result"); + assert!( + matches!(restored, SystemItem::FeatureInvocationResult { result: saved, .. } if saved == result) + ); + } + #[test] fn round_trip_worker_event() { let item = SystemItem::WorkerEvent { diff --git a/crates/session-store/src/worker_session_store.rs b/crates/session-store/src/worker_session_store.rs index 3711524e5..92b5b2f32 100644 --- a/crates/session-store/src/worker_session_store.rs +++ b/crates/session-store/src/worker_session_store.rs @@ -1166,7 +1166,7 @@ fn atomic_write_json(path: &Path, value: &T) -> Result<(), StoreEr atomic_write_bytes(path, &bytes) } -fn atomic_write_bytes(path: &Path, bytes: &[u8]) -> Result<(), StoreError> { +pub(crate) fn atomic_write_bytes(path: &Path, bytes: &[u8]) -> Result<(), StoreError> { let parent = path .parent() .ok_or_else(|| std::io::Error::other("Worker Session path has no parent"))?; diff --git a/crates/session-store/tests/fs_store_test.rs b/crates/session-store/tests/fs_store_test.rs index 430a685b9..db8ec035c 100644 --- a/crates/session-store/tests/fs_store_test.rs +++ b/crates/session-store/tests/fs_store_test.rs @@ -115,6 +115,100 @@ fn create_segment_writes_all_entries() { assert_eq!(state.session_id, Some(sid)); } +#[test] +fn create_segment_replacement_keeps_open_old_log_and_commits_complete_new_log() { + use std::io::Read; + let dir = tempfile::tempdir().unwrap(); + let store = FsStore::new(dir.path()).unwrap(); + let sid = new_session_id(); + let segid = new_segment_id(); + let old = vec![ + nil_session_start(1, sid), + LogEntry::TurnEnd { + ts: 2, + turn_count: 1, + }, + ]; + store.create_segment(sid, segid, &old).unwrap(); + let path = dir + .path() + .join(sid.to_string()) + .join(format!("{segid}.jsonl")); + let old_bytes = std::fs::read(&path).unwrap(); + let mut old_reader = std::fs::File::open(&path).unwrap(); + let new = vec![ + nil_session_start(1, sid), + LogEntry::Extension { + ts: 3, + domain: "recovery-checkpoint".into(), + payload: serde_json::json!({"receipt": "retained"}), + }, + ]; + store.create_segment(sid, segid, &new).unwrap(); + let mut retained_inode = Vec::new(); + old_reader.read_to_end(&mut retained_inode).unwrap(); + // An in-place fs::write would truncate the inode held by this reader. + assert_eq!(retained_inode, old_bytes); + let reopened = FsStore::new(dir.path()).unwrap(); + assert_eq!( + serde_json::to_value(reopened.read_all(sid, segid).unwrap()).unwrap(), + serde_json::to_value(&new).unwrap() + ); + assert_eq!(reopened.read_entry_count(sid, segid).unwrap(), new.len()); + assert_eq!( + std::fs::read_dir(path.parent().unwrap()).unwrap().count(), + 1, + "successful replacement must not leave temporary files" + ); +} + +#[test] +fn create_segment_atomic_replacement_readers_only_observe_complete_old_or_new() { + let dir = tempfile::tempdir().unwrap(); + let store = FsStore::new(dir.path()).unwrap(); + let sid = new_session_id(); + let segid = new_segment_id(); + let old = vec![ + nil_session_start(1, sid), + LogEntry::Extension { + ts: 2, + domain: "old-evidence".into(), + payload: serde_json::json!({"result": "old"}), + }, + ]; + let new = vec![ + nil_session_start(1, sid), + LogEntry::Extension { + ts: 3, + domain: "new-checkpoint".into(), + payload: serde_json::json!({"result": "new"}), + }, + ]; + store.create_segment(sid, segid, &old).unwrap(); + let old_json = serde_json::to_value(&old).unwrap(); + let new_json = serde_json::to_value(&new).unwrap(); + let barrier = std::sync::Arc::new(std::sync::Barrier::new(2)); + let reader_store = store.clone(); + let reader_barrier = barrier.clone(); + let reader = std::thread::spawn(move || { + reader_barrier.wait(); + for _ in 0..1000 { + let observed = + serde_json::to_value(reader_store.read_all(sid, segid).unwrap()).unwrap(); + assert!( + observed == old_json || observed == new_json, + "replacement must never expose a partial log: {observed}" + ); + } + }); + barrier.wait(); + for _ in 0..50 { + store.create_segment(sid, segid, &new).unwrap(); + store.create_segment(sid, segid, &old).unwrap(); + } + reader.join().unwrap(); +} + #[test] fn list_sessions_and_segments() { let dir = tempfile::tempdir().unwrap(); diff --git a/crates/session-store/tests/session_test.rs b/crates/session-store/tests/session_test.rs index 5ec4fe61d..901c0578f 100644 --- a/crates/session-store/tests/session_test.rs +++ b/crates/session-store/tests/session_test.rs @@ -606,6 +606,70 @@ fn rewound_fork_preserves_uploaded_file_segments_in_snapshot() { ))); } +#[test] +fn feature_invocations_survive_replay_snapshot_and_rewind_without_reparsing() { + let (_dir, store) = make_store(); + let config = RequestConfig::default(); + let (sid, segid) = session_store::create_segment( + &store, + SegmentStartState { + system_prompt: Some("System prompt"), + config: &config, + history: Vec::new(), + user_segments: Vec::new(), + }, + ) + .unwrap(); + let segments = vec![ + Segment::Text { + content: "before ".into(), + }, + Segment::FeatureInvoke { + invocation: protocol::FeatureInvocation { + invocation_id: "stable-request-1".into(), + identity: protocol::FeatureInvocationIdentity("builtin:test/prepare".into()), + name: "prepare".into(), + arguments: vec![protocol::InvocationArgumentValue { + name: "path".into(), + value: protocol::InvocationValue::String("資料/a b".into()), + }], + }, + }, + Segment::Text { + content: " after /prepare(unselected)".into(), + }, + ]; + session_store::save_user_input( + &store, + sid, + segid, + segments.clone(), + annotated(&[Item::user_message(Segment::flatten_to_text(&segments))]), + ) + .unwrap(); + session_store::save_turn_end(&store, sid, segid, 1).unwrap(); + let entries = store.read_all(sid, segid).unwrap(); + assert_eq!( + collect_state(&entries).user_segments, + vec![segments.clone()] + ); + let fork_id = session_store::fork_at(&store, sid, segid, 1).unwrap(); + let fork_entries = store.read_all(sid, fork_id).unwrap(); + assert_eq!( + collect_state(&fork_entries).user_segments, + vec![segments.clone()] + ); + let snapshot = session_store::public_snapshot::project_session_snapshot(sid, &fork_entries); + assert!(snapshot.entries.iter().any(|entry| matches!( + &entry.data, SessionSnapshotEntryData::UserInput { segments: restored } if restored == &segments + ))); + // Original history remains append-only and retains the same exact typed intent. + assert_eq!( + serde_json::to_value(store.read_all(sid, segid).unwrap()).unwrap(), + serde_json::to_value(entries).unwrap() + ); +} + #[tokio::test] async fn session_config_changed_logged() { let (_dir, store) = make_store(); diff --git a/crates/tools/README.md b/crates/tools/README.md index 2503bec6b..d8f72b3fb 100644 --- a/crates/tools/README.md +++ b/crates/tools/README.md @@ -24,6 +24,59 @@ Does not own: A tool implementation must assume model input is untrusted. Permission policy, scope checks, output bounding, and redaction are part of the safety boundary, not optional UI behavior. +## Checkout-native processing + +`execute_checkout_tool` (also exported through `tools::checkout`) accepts the +original `WorkdirSessionRouter`, shared `Tracker`, route-bound alias/generation, +logical target and provider validator. It does not create an attachment or router. +The Host owns native publication and permission identity; provider scope and +capabilities remain authoritative. `Create` uses the `Write` capability/identity. + +| Operation | Arguments (only these keys accepted) | Target | +| --- | --- | --- | +| Read | `offset?`, `limit?` | Existing file; same numbered lines, defaults and byte bound as Tool Read | +| Edit | `old_string`, `new_string`, `replace_all?` | Existing file; prior read/hash and provider validator required | +| Write | `content` | Existing file only; prior read/hash and provider validator required | +| Create | `path`, `content` | Bound parent directory; relative destination strictly below it; create-new, missing parents allowed | +| Glob | `pattern`, `path?` | Bound directory; optional path relative beneath it | +| Grep | `pattern`, `path?`, `glob?`, `type?`, `case_insensitive?`, `-B?`, `-A?`, `-C?`, `multiline?`, `output_mode?`, `head_limit?`, `offset?` | Bound directory; same provider search/options/rendering as Tool Grep | + +Native arguments cannot override `file_path`, `target_workdir`, generation or +validator. Absolute/parent-traversing directory-relative paths are rejected. +Checkout file calls share Tool processing through an optional checked `FileTarget` +and use `checkout_execute`, never a separate Worker-side filesystem engine. +Read records the provider's full content hash **and total line count**, including +partial reads, in the original alias/generation namespace. Normal and native +observations/mutations interoperate. Only successful mutations update tracking. + +Glob/Grep share normal request defaults/options and rendering through `SearchTarget`. +Only native dispatch uses `CheckoutSearchRequest::new` and `checkout_search`. +The constructor's empty scope layers/root output are refined by provider wrappers; +Tools does not synthesize authority or parse/rebase rendered filenames. Ordinary +Tool dispatch remains `session.glob`/`session.grep`, with its existing error mapping. + +`CheckoutToolOutput.paths` contains typed checkout-root-relative search paths +(from `GlobResult.paths` / `GrepResult.paths`) and the created path for Create. +Rendered Grep text is never parsed to construct links. The optional validator is +the provider's returned observation validator, unchanged. Create's result +observation describes the bound parent target, while returned paths and tracker +updates describe the destination. Search checks the bound observation before and +after the bounded checkout provider search, but does **not** guarantee an atomic tree snapshot +or prevent external edits to descendants. File validator/hash checks are inside +the provider's checked execution boundary. Checked native provider conflicts and +native generation/search fences use `ToolError::StructuredConflict` with code +`checkout_stale`; ordinary Tool conflicts retain `ExecutionFailed`. Checked-only +known provider refusals use `checkout_denied` (scope/capability/read-only policy), +`checkout_invalid` (invalid arguments/paths/target kind/search syntax), or +`checkout_unavailable` (closed session/unavailable capacity). A disappeared bound +target is `checkout_stale`. These codes preserve known no-effect rejection for the +Host; ordinary Tool error mappings are unchanged. Ambiguous transport/IO and +operation failures remain `ExecutionFailed`, as does explicit provider +`OutcomeUnknown`; a failure after mutation must not +be automatically retried. A malformed post-mutation provider result is likewise +reported explicitly as outcome unknown. Explicit mismatched Read/Glob/Grep results +are readonly failures classified as `checkout_unavailable`, not stale or unknown. + ## See also - [`../../docs/design/tool-permissions-scope.md`](../../docs/design/tool-permissions-scope.md) diff --git a/crates/tools/src/checkout.rs b/crates/tools/src/checkout.rs new file mode 100644 index 000000000..73e59e5e5 --- /dev/null +++ b/crates/tools/src/checkout.rs @@ -0,0 +1,258 @@ +//! Native checkout entrypoint using the same processing as ordinary filesystem Tools. +//! +//! The Host binds alias, generation, path and validator from the checkout route; none +//! can be overridden by Operation arguments. File reads/mutations dispatch through +//! the provider's checked execution boundary. Search shares normal request and +//! rendering processing but dispatches through checkout_search so provider scope +//! layers and output rebasing are preserved; checks bracket the bound directory +//! observation, not an atomic snapshot of the searched tree. +use std::sync::Arc; + +use agen::tool::{ToolError, ToolExecutionContext, ToolOutput}; +use serde::de::DeserializeOwned; +use serde_json::Value; +use workdir::{WorkdirPath, WorkdirSessionCapability, WorkdirSessionRouter}; + +use crate::{ToolsError, Tracker, file_target::FileTarget}; + +pub struct CheckoutToolOutput { + pub output: ToolOutput, + pub paths: Vec, + pub validator: Option>, +} + +/// Execute an exact named native Operation on a route-bound checkout target. +/// `Create.path` and search `path` are relative to the bound directory; returned +/// paths are typed checkout-root-relative provider paths, never parsed output. +#[allow(clippy::too_many_arguments)] +pub async fn execute_checkout_tool( + router: Arc, + tracker: Tracker, + alias: &str, + generation: u64, + path: WorkdirPath, + validator: Vec, + tool_name: &str, + arguments: Value, + ctx: ToolExecutionContext, +) -> Result { + // Native file arguments come from the same definitions used by ordinary + // Tools and both WIP projections. Route fields are intentionally not in them. + let file_schema = fs_operation::text::TextOperation::from_name(&tool_name.to_lowercase(), true) + .map(|operation| operation.argument_schema()); + let file_fields: Vec<&str> = file_schema + .as_ref() + .and_then(|schema| schema.as_value()["properties"].as_object()) + .map(|properties| properties.keys().map(String::as_str).collect()) + .unwrap_or_default(); + let (capability, allowed): (_, &[&str]) = match tool_name { + "Read" => (WorkdirSessionCapability::Read, &file_fields), + "Edit" => (WorkdirSessionCapability::Edit, &file_fields), + "Write" => (WorkdirSessionCapability::Write, &file_fields), + "Create" => (WorkdirSessionCapability::Write, &["path", "content"]), + "Glob" => (WorkdirSessionCapability::Glob, &["pattern", "path"]), + "Grep" => ( + WorkdirSessionCapability::Grep, + &[ + "pattern", + "path", + "glob", + "type", + "case_insensitive", + "-B", + "-A", + "-C", + "multiline", + "output_mode", + "head_limit", + "offset", + ], + ), + _ => { + return Err(ToolError::InvalidArgument(format!( + "unknown checkout Operation: {tool_name}" + ))); + } + }; + let mut arguments = arguments + .as_object() + .cloned() + .ok_or_else(|| ToolError::InvalidArgument("checkout arguments must be an object".into()))?; + if let Some(key) = arguments + .keys() + .find(|key| !allowed.contains(&key.as_str())) + { + return Err(ToolError::InvalidArgument(format!( + "unknown or route override checkout argument: {key}" + ))); + } + // Do not rebuild a singleton router: this is the original router identity and + // tracker namespace shared with normal Tool calls and its detach/reuse fence. + let selected = crate::routing::resolve_session(&router, Some(alias), capability)?; + if selected.generation != generation { + return Err(stale_checkout( + "checkout attachment generation changed; resolve the checkout again", + )); + } + // Checkout paths are always logical and cannot use Tool-mode absolute scope. + let path = WorkdirPath::new(path.as_str()).map_err(ToolsError::from)?; + let tracker = tracker.scoped_attachment(&selected.alias, selected.generation); + let target = FileTarget { + session: selected.session.clone(), + path: path.clone(), + validator: Some(validator.clone()), + }; + match tool_name { + "Read" | "Edit" | "Write" => { + arguments.insert("file_path".into(), Value::String(path.to_string())); + let arguments = Value::Object(arguments); + match tool_name { + "Read" => { + crate::read::execute_read(target, tracker, decode(arguments, tool_name)?).await + } + "Edit" => { + crate::edit::execute_edit(target, tracker, decode(arguments, tool_name)?, ctx) + .await + } + _ => { + let params: crate::write::WriteParams = decode(arguments, tool_name)?; + crate::write::execute_write(target, tracker, params.write.content, None, ctx) + .await + } + } + } + "Create" => { + #[derive(serde::Deserialize)] + struct CreateParams { + path: String, + content: String, + } + let params: CreateParams = decode(Value::Object(arguments), tool_name)?; + let destination = relative_target(&path, ¶ms.path, true)?; + crate::write::execute_write(target, tracker, params.content, Some(destination), ctx) + .await + } + "Glob" | "Grep" => { + let before = selected + .session + .checkout_observe(path.clone()) + .await + .map_err(crate::file_target::checked_error)?; + if before.validator != validator { + return Err(stale_search()); + } + let mut result = if tool_name == "Glob" { + let params: crate::glob::GlobParams = decode(Value::Object(arguments), tool_name)?; + check_pattern(¶ms.pattern)?; + let search_path = + relative_target(&path, params.path.as_deref().unwrap_or("."), false)?; + crate::glob::execute_glob( + crate::search_target::SearchTarget::checkout(selected.session.clone()), + search_path, + params.pattern, + ) + .await? + } else { + let params: crate::grep::GrepParams = decode(Value::Object(arguments), tool_name)?; + if let Some(glob) = ¶ms.glob { + check_pattern(glob)?; + } + let search_path = + relative_target(&path, params.path.as_deref().unwrap_or("."), false)?; + crate::grep::execute_grep( + crate::search_target::SearchTarget::checkout(selected.session.clone()), + search_path, + params, + ) + .await? + }; + // Reject rather than emit a link/rendered path outside the binding. + // Provider scope is still authoritative for both traversal and content. + for result_path in &result.paths { + let logical = WorkdirPath::new(result_path.as_str()).map_err(ToolsError::from)?; + if !is_beneath(&path, &logical) { + return Err(crate::file_target::readonly_result_error( + "search paths escaped bound directory", + )); + } + } + let after = selected + .session + .checkout_observe(path) + .await + .map_err(crate::file_target::checked_error)?; + if after.validator != before.validator { + return Err(stale_search()); + } + result.validator = Some(after.validator); + Ok(result) + } + _ => unreachable!("Operation names checked above"), + } +} + +fn decode(arguments: Value, name: &str) -> Result { + crate::error::decode_file_value(arguments, name) +} + +fn stale_search() -> ToolError { + stale_checkout("checkout directory changed; resolve it again before searching") +} + +pub(crate) fn stale_checkout(message: impl Into) -> ToolError { + ToolError::StructuredConflict { + code: "checkout_stale".into(), + message: message.into(), + } +} + +fn is_beneath(base: &WorkdirPath, path: &WorkdirPath) -> bool { + base == &WorkdirPath::root() + || std::path::Path::new(path.as_str()).starts_with(std::path::Path::new(base.as_str())) +} + +fn relative_target( + base: &WorkdirPath, + relative: &str, + strict: bool, +) -> Result { + if relative.is_empty() + || relative.starts_with('/') + || relative.contains('\\') + || relative.split('/').any(|part| part == "..") + { + return Err(ToolError::InvalidArgument( + "path must be relative beneath the bound checkout directory, without parent traversal" + .into(), + )); + } + let relative = WorkdirPath::new(relative).map_err(ToolsError::from)?; + let joined = if relative == WorkdirPath::root() { + base.clone() + } else if base == &WorkdirPath::root() { + relative + } else { + WorkdirPath::new(&format!("{}/{}", base.as_str(), relative.as_str())) + .map_err(ToolsError::from)? + }; + if !is_beneath(base, &joined) || (strict && joined == *base) { + return Err(ToolError::InvalidArgument( + "destination must be strictly beneath the bound checkout directory".into(), + )); + } + Ok(joined) +} + +fn check_pattern(pattern: &str) -> Result<(), ToolError> { + // Patterns filter descendants, not an alternate filesystem starting point. + if pattern.starts_with('/') || pattern.split('/').any(|part| part == "..") { + return Err(ToolError::InvalidArgument( + "search glob must stay beneath the bound checkout directory".into(), + )); + } + Ok(()) +} + +#[cfg(test)] +#[path = "checkout_tests.rs"] +mod tests; diff --git a/crates/tools/src/checkout_tests.rs b/crates/tools/src/checkout_tests.rs new file mode 100644 index 000000000..d11203ed3 --- /dev/null +++ b/crates/tools/src/checkout_tests.rs @@ -0,0 +1,562 @@ +use super::*; +use manifest::Scope; +use serde_json::json; +use tempfile::TempDir; +use workdir::{LocalWorkdirSession, WorkdirAttachmentAlias}; + +struct Fixture { + dir: TempDir, + router: Arc, + tracker: Tracker, +} + +impl Fixture { + fn new() -> Self { + let dir = TempDir::new().unwrap(); + let router = Arc::new(WorkdirSessionRouter::new()); + let fixture = Self { + dir, + router, + tracker: Tracker::new(), + }; + fixture.attach("main"); + fixture + } + + fn attach(&self, alias: &str) { + self.router + .attach( + WorkdirAttachmentAlias::new(alias).unwrap(), + Arc::new(LocalWorkdirSession::new( + Scope::writable(self.dir.path()).unwrap(), + self.dir.path().to_path_buf(), + )), + ) + .unwrap(); + } + + async fn observation(&self, alias: &str, path: &str) -> (u64, Vec) { + let selected = self.router.resolve(Some(alias)).unwrap(); + let observation = selected + .session + .checkout_observe(WorkdirPath::new(path).unwrap()) + .await + .unwrap(); + (selected.generation, observation.validator) + } + + async fn native( + &self, + alias: &str, + path: &str, + name: &str, + arguments: Value, + ) -> Result { + let (generation, validator) = self.observation(alias, path).await; + execute_checkout_tool( + self.router.clone(), + self.tracker.clone(), + alias, + generation, + WorkdirPath::new(path).unwrap(), + validator, + name, + arguments, + Default::default(), + ) + .await + } + + async fn normal(&self, name: &str, arguments: Value) -> Result { + let definitions = crate::routed_builtin_tools( + self.router.clone(), + self.tracker.clone(), + Default::default(), + ); + for definition in definitions { + let (meta, tool) = definition(); + if meta.name == name { + return tool + .execute(&arguments.to_string(), Default::default()) + .await; + } + } + panic!("missing Tool {name}") + } +} + +fn assert_checkout_stale(error: ToolError) { + assert_checkout_code(error, "checkout_stale"); +} + +fn assert_checkout_code(error: ToolError, expected: &str) { + match error { + ToolError::StructuredConflict { code, .. } => assert_eq!(code, expected), + other => panic!("expected {expected} structured conflict, got {other:?}"), + } +} + +#[test] +fn checkout_known_refusals_have_typed_codes_without_changing_normal_tools() { + use workdir::{WorkdirError, WorkdirSessionCapability}; + let cases = [ + (WorkdirError::Denied("denied".into()), "checkout_denied"), + ( + WorkdirError::OutOfScope("outside".into()), + "checkout_denied", + ), + ( + WorkdirError::ReadOnly("read only".into()), + "checkout_denied", + ), + ( + WorkdirError::Unsupported(WorkdirSessionCapability::Write), + "checkout_denied", + ), + ( + WorkdirError::InvalidArgument("bad argument".into()), + "checkout_invalid", + ), + ( + WorkdirError::InvalidPath("bad path".into()), + "checkout_invalid", + ), + ( + WorkdirError::Unavailable("budget exhausted".into()), + "checkout_unavailable", + ), + (WorkdirError::NotFound("missing".into()), "checkout_stale"), + (WorkdirError::SessionClosed, "checkout_unavailable"), + ( + WorkdirError::UnsupportedOperation("unsupported".into()), + "checkout_denied", + ), + ( + WorkdirError::UnknownCommand("missing command".into()), + "checkout_invalid", + ), + ( + WorkdirError::RelativePath("relative".into()), + "checkout_invalid", + ), + ( + WorkdirError::IsDirectory("directory".into()), + "checkout_invalid", + ), + ( + WorkdirError::InvalidGlob("bad glob".into()), + "checkout_invalid", + ), + ( + WorkdirError::InvalidRegex("bad regex".into()), + "checkout_invalid", + ), + ]; + for (error, code) in cases { + assert_checkout_code(crate::file_target::checked_error(error), code); + } + assert!(matches!( + ToolError::from(ToolsError::from(WorkdirError::Unavailable( + "unavailable".into() + ))), + ToolError::ExecutionFailed(_) + )); + assert!(matches!( + ToolError::from(ToolsError::from(WorkdirError::Unsupported( + WorkdirSessionCapability::Write + ))), + ToolError::InvalidArgument(_) + )); +} + +#[test] +fn checkout_conflict_classification_preserves_normal_tools_and_unknown_outcomes() { + assert_checkout_stale(crate::file_target::checked_error( + workdir::WorkdirError::Conflict("a.txt".into()), + )); + assert_checkout_stale(stale_search()); + assert!(matches!( + ToolError::from(ToolsError::from(workdir::WorkdirError::Conflict( + "a.txt".into() + ))), + ToolError::ExecutionFailed(_) + )); + assert!(matches!( + crate::file_target::checked_error(workdir::WorkdirError::OutcomeUnknown( + "may have committed".into() + )), + ToolError::ExecutionFailed(_) + )); +} + +#[test] +fn checkout_ambiguous_transport_and_operation_failures_remain_unknown() { + use workdir::WorkdirError; + // Even refusal-looking text in an ambiguous transport error is not proof + // that the provider rejected the request before any effects. + for error in [ + WorkdirError::Transport("denied (ambiguous response)".into()), + WorkdirError::OperationFailed, + ] { + assert!(matches!( + crate::file_target::checked_error(error), + ToolError::ExecutionFailed(_) + )); + } +} + +#[test] +fn checkout_mismatched_readonly_results_are_unavailable_not_stale_or_unknown() { + use workdir::{CheckoutSearchResult, GlobResult, GrepResult}; + let glob = CheckoutSearchResult::Glob(GlobResult { + paths: Vec::new(), + truncated: false, + }); + let grep = CheckoutSearchResult::Grep(GrepResult { + output: String::new(), + match_count: 0, + matched_files: 0, + truncated: false, + paths: Vec::new(), + }); + assert_checkout_code( + crate::search_target::grep_result(glob).err().unwrap(), + "checkout_unavailable", + ); + assert_checkout_code( + crate::search_target::glob_result(grep).err().unwrap(), + "checkout_unavailable", + ); + assert_checkout_code( + crate::file_target::readonly_result_error("Read"), + "checkout_unavailable", + ); +} + +#[test] +fn checkout_relative_path_boundaries() { + let base = WorkdirPath::new("src").unwrap(); + assert_eq!( + relative_target(&base, "nested/a.txt", true) + .unwrap() + .as_str(), + "src/nested/a.txt" + ); + assert_eq!(relative_target(&base, ".", false).unwrap(), base); + assert_eq!( + relative_target(&WorkdirPath::root(), "a.txt", true) + .unwrap() + .as_str(), + "a.txt" + ); + for bad in [ + "", + "..", + "../outside", + "nested/../../outside", + "/absolute", + "a\\..\\outside", + ] { + assert!( + relative_target(&base, bad, false).is_err(), + "accepted {bad}" + ); + } + assert!(relative_target(&base, ".", true).is_err()); + assert!(!is_beneath( + &base, + &WorkdirPath::new("src-other/a.txt").unwrap() + )); + assert!(check_pattern("../*").is_err()); + assert!(check_pattern("/tmp/*").is_err()); + assert!(check_pattern("**/*.rs").is_ok()); +} + +#[tokio::test] +async fn checkout_read_and_normal_edit_share_hash_and_line_count() { + let fixture = Fixture::new(); + std::fs::write(fixture.dir.path().join("a.txt"), "alpha\nbeta\ngamma\n").unwrap(); + let read = fixture + .native("main", "a.txt", "Read", json!({"offset": 1, "limit": 1})) + .await + .unwrap(); + assert!(read.output.summary.contains("[2..2] of 3")); + assert_eq!(read.output.content.as_deref(), Some(" 2\tbeta\n")); + assert!(read.validator.is_some()); + let selected = fixture.router.resolve(Some("main")).unwrap(); + let tracker = fixture + .tracker + .scoped_attachment(&selected.alias, selected.generation); + assert_eq!( + tracker.observed_workdir_line_count(&WorkdirPath::new("a.txt").unwrap()), + Some(3) + ); + fixture.normal("Edit", json!({"target_workdir": "main", "file_path": "a.txt", "old_string": "beta", "new_string": "changed"})).await.unwrap(); + let write = fixture + .native("main", "a.txt", "Write", json!({"content": "new\n"})) + .await + .unwrap(); + assert!(write.output.summary.contains("Overwrote")); + assert_eq!(fixture.tracker.change_stat().deleted, 4); + let reread = fixture + .native("main", "a.txt", "Read", json!({})) + .await + .unwrap(); + assert_eq!(reread.output.content.as_deref(), Some(" 1\tnew\n")); +} + +#[tokio::test] +async fn checkout_normal_read_then_native_edit_uniqueness_and_replace_all() { + let fixture = Fixture::new(); + std::fs::write(fixture.dir.path().join("a.txt"), "x x x\n").unwrap(); + let before_read = fixture + .native( + "main", + "a.txt", + "Edit", + json!({"old_string": "x", "new_string": "y"}), + ) + .await + .err() + .unwrap(); + assert!(before_read.to_string().contains("has not been read")); + fixture + .normal( + "Read", + json!({"target_workdir": "main", "file_path": "a.txt"}), + ) + .await + .unwrap(); + let not_unique = fixture + .native( + "main", + "a.txt", + "Edit", + json!({"old_string": "x", "new_string": "y"}), + ) + .await + .err() + .unwrap(); + assert_checkout_code(not_unique, "checkout_invalid"); + assert_eq!(fixture.tracker.change_stat(), crate::ChangeStat::default()); + let edit = fixture + .native( + "main", + "a.txt", + "Edit", + json!({"old_string": "x", "new_string": "y", "replace_all": true}), + ) + .await + .unwrap(); + assert!(edit.output.summary.contains("3 replacements")); + assert_eq!( + std::fs::read_to_string(fixture.dir.path().join("a.txt")).unwrap(), + "y y y\n" + ); + fixture.normal("Write", json!({"target_workdir": "main", "file_path": "a.txt", "content": "normal after native"})).await.unwrap(); +} + +#[tokio::test] +async fn checkout_alias_and_generation_fences() { + let fixture = Fixture::new(); + fixture.attach("other"); + std::fs::write(fixture.dir.path().join("a.txt"), "same\n").unwrap(); + fixture + .native("main", "a.txt", "Read", json!({})) + .await + .unwrap(); + let wrong_alias = fixture + .native("other", "a.txt", "Write", json!({"content": "not allowed"})) + .await + .err() + .unwrap(); + assert!(wrong_alias.to_string().contains("has not been read")); + let (old_generation, old_validator) = fixture.observation("main", "a.txt").await; + fixture + .router + .detach(&WorkdirAttachmentAlias::new("main").unwrap()) + .await + .unwrap(); + fixture.attach("main"); + let error = execute_checkout_tool( + fixture.router.clone(), + fixture.tracker.clone(), + "main", + old_generation, + WorkdirPath::new("a.txt").unwrap(), + old_validator, + "Read", + json!({}), + Default::default(), + ) + .await + .err() + .unwrap(); + assert!(error.to_string().contains("generation changed")); + assert_checkout_stale(error); + let fresh_generation_no_read = fixture + .native("main", "a.txt", "Write", json!({"content": "not allowed"})) + .await + .err() + .unwrap(); + assert!( + fresh_generation_no_read + .to_string() + .contains("has not been read") + ); +} + +#[tokio::test] +async fn checkout_stale_validator_and_hash_are_independent() { + let fixture = Fixture::new(); + std::fs::write(fixture.dir.path().join("a.txt"), "old\n").unwrap(); + fixture + .native("main", "a.txt", "Read", json!({})) + .await + .unwrap(); + let (generation, validator) = fixture.observation("main", "a.txt").await; + std::fs::write(fixture.dir.path().join("a.txt"), "externally changed\n").unwrap(); + let stale = execute_checkout_tool( + fixture.router.clone(), + fixture.tracker.clone(), + "main", + generation, + WorkdirPath::new("a.txt").unwrap(), + validator, + "Write", + json!({"content": "bad"}), + Default::default(), + ) + .await + .err() + .unwrap(); + assert_checkout_stale(stale); + let hash = fixture + .native("main", "a.txt", "Write", json!({"content": "bad"})) + .await + .err() + .unwrap(); + assert_checkout_stale(hash); + assert_eq!( + std::fs::read_to_string(fixture.dir.path().join("a.txt")).unwrap(), + "externally changed\n" + ); + assert_eq!(fixture.tracker.change_stat(), crate::ChangeStat::default()); +} + +#[tokio::test] +async fn checkout_create_new_missing_parents_never_overwrites() { + let fixture = Fixture::new(); + let created = fixture + .native( + "main", + ".", + "Create", + json!({"path": "nested/deep/a.txt", "content": "new\n"}), + ) + .await + .unwrap(); + assert!(created.output.summary.contains("Created")); + assert_eq!( + created.paths, + vec![WorkdirPath::new("nested/deep/a.txt").unwrap()] + ); + let duplicate = fixture + .native( + "main", + ".", + "Create", + json!({"path": "nested/deep/a.txt", "content": "overwrite"}), + ) + .await; + assert!(duplicate.is_err()); + assert_eq!( + std::fs::read_to_string(fixture.dir.path().join("nested/deep/a.txt")).unwrap(), + "new\n" + ); + // Successful creation updates the same tracker as normal Tool Write. + fixture.normal("Write", json!({"target_workdir": "main", "file_path": "nested/deep/a.txt", "content": "changed"})).await.unwrap(); +} + +#[tokio::test] +async fn checkout_rejects_argument_overrides_and_escapes() { + let fixture = Fixture::new(); + std::fs::write(fixture.dir.path().join("a.txt"), "safe").unwrap(); + for (name, arguments) in [ + ("Read", json!({"file_path": "other"})), + ("Read", json!({"target_workdir": "other"})), + ("Write", json!({"content": "bad", "validator": []})), + ("Create", json!({"path": "../outside", "content": "bad"})), + ("Glob", json!({"pattern": "*", "path": "/absolute"})), + ("Grep", json!({"pattern": "safe", "path": "../outside"})), + ("Other", json!({})), + ] { + let path = if ["Create", "Glob", "Grep"].contains(&name) { + "." + } else { + "a.txt" + }; + let error = fixture + .native("main", path, name, arguments) + .await + .err() + .unwrap(); + assert!( + matches!(error, ToolError::InvalidArgument(_)), + "{name}: {error}" + ); + } +} + +#[tokio::test] +async fn checkout_search_typed_paths_and_normal_rendering() { + let fixture = Fixture::new(); + std::fs::create_dir_all(fixture.dir.path().join("src/nested")).unwrap(); + std::fs::write( + fixture.dir.path().join("src/nested/a.txt"), + "context\nhit\nafter\n", + ) + .unwrap(); + std::fs::write(fixture.dir.path().join("outside.txt"), "hit\n").unwrap(); + let glob = fixture + .native( + "main", + "src", + "Glob", + json!({"pattern": "**/*.txt", "path": "nested"}), + ) + .await + .unwrap(); + assert_eq!( + glob.paths, + vec![WorkdirPath::new("src/nested/a.txt").unwrap()] + ); + let normal = fixture + .normal( + "Glob", + json!({"target_workdir": "main", "pattern": "**/*.txt", "path": "src/nested"}), + ) + .await + .unwrap(); + assert_eq!(glob.output.summary, normal.summary); + assert_eq!(glob.output.content, normal.content); + for mode in ["files_with_matches", "content", "count"] { + let grep = fixture + .native( + "main", + "src", + "Grep", + json!({"pattern": "hit", "output_mode": mode, "-C": 1}), + ) + .await + .unwrap(); + assert_eq!( + grep.paths, + vec![WorkdirPath::new("src/nested/a.txt").unwrap()] + ); + let normal = fixture.normal("Grep", json!({"target_workdir": "main", "path": "src", "pattern": "hit", "output_mode": mode, "-C": 1})).await.unwrap(); + assert_eq!(grep.output.summary, normal.summary); + assert_eq!(grep.output.content, normal.content); + assert!(!grep.output.content.unwrap().contains("outside.txt")); + } +} diff --git a/crates/tools/src/edit.rs b/crates/tools/src/edit.rs index 4a22cb218..805a83087 100644 --- a/crates/tools/src/edit.rs +++ b/crates/tools/src/edit.rs @@ -9,7 +9,7 @@ use serde::Deserialize; use crate::error::ToolsError; use crate::tracker::Tracker; -use workdir::{EditRequest, WorkdirPath, WorkdirSessionHandle, WorkdirSessionRouter}; +use workdir::{WorkdirPath, WorkdirSessionHandle, WorkdirSessionRouter}; const DESCRIPTION: &str = "Replace a substring in an existing file in the selected Workdir attachment. By default \ `old_string` must be unique in the file; set `replace_all: true` to replace \ @@ -23,13 +23,8 @@ pub(crate) struct EditParams { pub target_workdir: Option, /// Logical path relative to the bound Workdir root. pub file_path: String, - /// String to replace. Must be unique in the file unless `replace_all` is true. - pub old_string: String, - /// Replacement string. Must differ from `old_string`. - pub new_string: String, - /// Replace all occurrences. Defaults to false. - #[serde(default)] - pub replace_all: bool, + #[serde(flatten)] + pub edit: fs_operation::text::EditArgs, } pub(crate) struct EditTool { @@ -44,8 +39,7 @@ impl Tool for EditTool { input_json: &str, ctx: agen::tool::ToolExecutionContext, ) -> Result { - let params: EditParams = serde_json::from_str(input_json) - .map_err(|e| ToolError::InvalidArgument(format!("invalid Edit input: {e}")))?; + let params: EditParams = crate::error::decode_file_input(input_json, "Edit")?; let selected = crate::routing::resolve_session( &self.router, @@ -56,56 +50,68 @@ impl Tool for EditTool { .tracker .scoped_attachment(&selected.alias, selected.generation); let path = WorkdirPath::new(¶ms.file_path).map_err(ToolsError::from)?; - tracing::debug!(path = %path, replace_all = params.replace_all, "Edit"); + Ok(execute_edit( + crate::file_target::FileTarget { + session: selected.session, + path, + validator: None, + }, + tracker, + params, + ctx, + ) + .await? + .output) + } +} - if params.old_string.is_empty() { - return Err(ToolError::InvalidArgument( - "old_string must not be empty".into(), - )); - } - if params.old_string == params.new_string { - return Err(ToolError::InvalidArgument( - "old_string and new_string are identical".into(), - )); - } - - let mutation_key = PathBuf::from(path.as_str()); - let _mutation_permit = tracker.acquire_mutation(&mutation_key, &ctx).await; - let expected_hash = tracker.expected_workdir_hash(&path)?; - let result = selected - .session - .edit(EditRequest { - path: path.clone(), - old_string: params.old_string.clone(), - new_string: params.new_string.clone(), - replace_all: params.replace_all, - expected_hash, - }) - .await - .map_err(ToolsError::from)?; - let replacements = result.replacements; - tracker.record_workdir_edit( - &path, - result.content_hash, - replacements, - params.new_string.lines().count(), - params.old_string.lines().count(), - ); - - let summary = format!( - "Edited {} ({} replacement{})", - path, - replacements, - if replacements == 1 { "" } else { "s" } - ); - let preview = make_preview(¶ms.new_string, ¶ms.new_string); - - Ok(ToolOutput { +pub(crate) async fn execute_edit( + target: crate::file_target::FileTarget, + tracker: Tracker, + params: EditParams, + ctx: agen::tool::ToolExecutionContext, +) -> Result { + let path = &target.path; + tracing::debug!(path = %path, replace_all = params.edit.replace_all, "Edit"); + params + .edit + .validate(Default::default()) + .map_err(crate::error::text_error)?; + let mutation_key = PathBuf::from(path.as_str()); + let _mutation_permit = tracker.acquire_mutation(&mutation_key, &ctx).await; + let expected_hash = tracker.expected_workdir_hash(path)?; + let (result, validator) = target + .edit( + params.edit.old_string.clone(), + params.edit.new_string.clone(), + params.edit.replace_all, + expected_hash, + ) + .await?; + let replacements = result.replacements; + tracker.record_workdir_edit( + path, + result.content_hash, + replacements, + params.edit.new_string.lines().count(), + params.edit.old_string.lines().count(), + ); + let summary = format!( + "Edited {} ({} replacement{})", + path, + replacements, + if replacements == 1 { "" } else { "s" } + ); + let preview = make_preview(¶ms.edit.new_string, ¶ms.edit.new_string); + Ok(crate::checkout::CheckoutToolOutput { + output: ToolOutput { summary, content: Some(preview), attachments: Vec::new(), - }) - } + }, + paths: Vec::new(), + validator, + }) } /// Build a small line-numbered snippet centered on the first occurrence of @@ -182,6 +188,27 @@ mod tests { .unwrap(); } + #[tokio::test] + async fn edit_shared_validation_rejects_identical_text_before_read_or_mutation() { + let (dir, fs, tracker) = setup(); + let file = dir.path().join("a.txt"); + std::fs::write(&file, "same").unwrap(); + let (_, tool) = edit_tool(fs, tracker.clone())(); + let error = tool + .execute( + &serde_json::json!({"file_path":"a.txt","old_string":"same","new_string":"same"}) + .to_string(), + Default::default(), + ) + .await + .unwrap_err(); + assert!( + matches!(error, ToolError::InvalidArgument(message) if message == "old_string and new_string are identical") + ); + assert_eq!(std::fs::read_to_string(file).unwrap(), "same"); + assert_eq!(tracker.change_stat(), crate::ChangeStat::default()); + } + #[tokio::test] async fn edit_unique_replacement() { let (dir, fs, tracker) = setup(); diff --git a/crates/tools/src/error.rs b/crates/tools/src/error.rs index 12b1103cb..b25025b7c 100644 --- a/crates/tools/src/error.rs +++ b/crates/tools/src/error.rs @@ -44,7 +44,8 @@ impl From for ToolError { | workdir::WorkdirError::Unavailable(_) | workdir::WorkdirError::OperationFailed | workdir::WorkdirError::Transport(_) - | workdir::WorkdirError::Conflict(_), + | workdir::WorkdirError::Conflict(_) + | workdir::WorkdirError::OutcomeUnknown(_), ) => ToolError::ExecutionFailed(err.to_string()), ToolsError::FileSystem(_) | ToolsError::WorkdirSession(_) @@ -57,11 +58,67 @@ impl From for ToolError { } } +/// Pure text failures are pre-dispatch argument failures, never unknown outcomes. +pub(crate) fn text_error(error: fs_operation::text::TextError) -> ToolError { + ToolError::InvalidArgument(error.to_string()) +} + +pub(crate) fn decode_file_input( + input: &str, + name: &str, +) -> Result { + fs_operation::text::decode_json(input).map_err(|error| decode_error(error, name)) +} + +pub(crate) fn decode_file_value( + value: serde_json::Value, + name: &str, +) -> Result { + fs_operation::text::decode(value).map_err(|error| decode_error(error, name)) +} + +fn decode_error(error: fs_operation::text::TextError, name: &str) -> ToolError { + match error { + fs_operation::text::TextError::Decode(error) => { + ToolError::InvalidArgument(format!("invalid {name} input: {error}")) + } + error => text_error(error), + } +} + #[cfg(test)] mod tests { use super::*; use workdir::http::{WorkdirTransportError, WorkdirTransportErrorCode}; + #[test] + fn flattened_file_tool_arguments_keep_raw_json_duplicate_rejection() { + let error = decode_file_input::( + r#"{"file_path":"a.txt","old_string":"a","old_string":"b","new_string":"c"}"#, + "Edit", + ) + .unwrap_err(); + assert!( + matches!(error, ToolError::InvalidArgument(message) if message.contains("duplicate field `old_string`")) + ); + } + + #[test] + fn checkout_outcome_unknown_remains_execution_failure() { + let error = ToolError::from(ToolsError::WorkdirSession( + workdir::WorkdirError::OutcomeUnknown( + "mutation may have committed; do not retry automatically".into(), + ), + )); + match error { + ToolError::ExecutionFailed(message) => { + assert!(message.contains("mutation may have committed")); + assert!(message.contains("do not retry automatically")); + } + other => panic!("outcome unknown must not become invalid input: {other:?}"), + } + } + #[test] fn local_workdir_content_conflict_is_retryable_execution_failure() { let error = ToolError::from(ToolsError::WorkdirSession( diff --git a/crates/tools/src/file_target.rs b/crates/tools/src/file_target.rs new file mode 100644 index 000000000..17397eea3 --- /dev/null +++ b/crates/tools/src/file_target.rs @@ -0,0 +1,186 @@ +//! Shared provider dispatch for Tool and checkout file processing. +use agen::tool::ToolError; +use workdir::{ + CheckoutOperation, CheckoutOutput, CheckoutRequest, ContentHash, EditRequest, EditResult, + ReadRequest, ReadResult, WorkdirPath, WorkdirSessionHandle, WriteRequest, WriteResult, +}; + +use crate::ToolsError; + +pub(crate) struct FileTarget { + pub session: WorkdirSessionHandle, + pub path: WorkdirPath, + pub validator: Option>, +} + +impl FileTarget { + async fn checked( + &self, + operation: CheckoutOperation, + ) -> Result<(CheckoutOutput, Option>), ToolError> { + let result = self + .session + .checkout_execute(CheckoutRequest { + target: self.path.clone(), + validator: self + .validator + .clone() + .expect("checked dispatch requires validator"), + operation, + }) + .await + .map_err(checked_error)?; + Ok((result.output, Some(result.observation.validator))) + } + + pub async fn read( + &self, + offset: usize, + limit: usize, + max_bytes: usize, + ) -> Result<(ReadResult, Option>), ToolError> { + if self.validator.is_some() { + let (output, validator) = self + .checked(CheckoutOperation::Read { + offset, + limit, + max_bytes, + }) + .await?; + match output { + CheckoutOutput::Read(result) => Ok((result, validator)), + _ => Err(readonly_result_error("Read")), + } + } else { + let result = self + .session + .read(ReadRequest { + path: self.path.clone(), + offset, + limit, + max_bytes, + }) + .await + .map_err(ToolsError::from)?; + Ok((result, None)) + } + } + + pub async fn edit( + &self, + old_string: String, + new_string: String, + replace_all: bool, + expected_hash: ContentHash, + ) -> Result<(EditResult, Option>), ToolError> { + if self.validator.is_some() { + let (output, validator) = self + .checked(CheckoutOperation::Edit { + old_string, + new_string, + replace_all, + expected_hash, + }) + .await?; + match output { + CheckoutOutput::Edit(result) => Ok((result, validator)), + _ => Err(unknown_result("Edit")), + } + } else { + let result = self + .session + .edit(EditRequest { + path: self.path.clone(), + old_string, + new_string, + replace_all, + expected_hash, + }) + .await + .map_err(ToolsError::from)?; + Ok((result, None)) + } + } + + pub async fn write( + &self, + content: Vec, + expected_hash: Option, + create_path: Option, + ) -> Result<(WriteResult, Option>), ToolError> { + if self.validator.is_some() { + let operation = match create_path { + Some(path) => CheckoutOperation::Create { path, content }, + None => CheckoutOperation::Write { + content, + expected_hash: expected_hash + .expect("existing checkout Write requires prior read"), + }, + }; + let (output, validator) = self.checked(operation).await?; + match output { + CheckoutOutput::Write(result) => Ok((result, validator)), + _ => Err(unknown_result("Write/Create")), + } + } else { + let result = self + .session + .write(WriteRequest { + path: self.path.clone(), + content, + expected_hash, + }) + .await + .map_err(ToolsError::from)?; + Ok((result, None)) + } + } +} + +pub(crate) fn checked_error(error: workdir::WorkdirError) -> ToolError { + use workdir::WorkdirError; + let code = match &error { + WorkdirError::Conflict(_) | WorkdirError::NotFound(_) => "checkout_stale", + WorkdirError::Denied { .. } + | WorkdirError::OutOfScope { .. } + | WorkdirError::ReadOnly { .. } + | WorkdirError::Unsupported { .. } + | WorkdirError::UnsupportedOperation { .. } + | WorkdirError::SymlinkOutOfScope { .. } + | WorkdirError::SymlinkDirectoryNotTraversed { .. } => "checkout_denied", + WorkdirError::InvalidArgument { .. } + | WorkdirError::InvalidPath { .. } + | WorkdirError::UnknownCommand { .. } + | WorkdirError::RelativePath { .. } + | WorkdirError::BrokenSymlink { .. } + | WorkdirError::SymlinkTargetIsDirectory { .. } + | WorkdirError::IsDirectory { .. } + | WorkdirError::InvalidGlob { .. } + | WorkdirError::InvalidRegex { .. } => "checkout_invalid", + WorkdirError::Unavailable { .. } | WorkdirError::SessionClosed => "checkout_unavailable", + // IO/transport failures may arrive after effects have begun. Do not infer + // no-effects from their text or strip explicit OutcomeUnknown evidence. + WorkdirError::Io { .. } + | WorkdirError::Transport { .. } + | WorkdirError::OutcomeUnknown { .. } + | WorkdirError::OperationFailed => { + return ToolsError::from(error).into(); + } + }; + ToolError::StructuredConflict { + code: code.into(), + message: error.to_string(), + } +} + +pub(crate) fn readonly_result_error(operation: &str) -> ToolError { + checked_error(workdir::WorkdirError::Unavailable(format!( + "provider returned an invalid readonly checkout {operation} result" + ))) +} + +fn unknown_result(operation: &str) -> ToolError { + ToolError::ExecutionFailed(format!( + "Outcome unknown: provider returned wrong checkout {operation} result after mutation; do not automatically retry" + )) +} diff --git a/crates/tools/src/glob.rs b/crates/tools/src/glob.rs index 0c2914dfd..fe65fc36a 100644 --- a/crates/tools/src/glob.rs +++ b/crates/tools/src/glob.rs @@ -11,15 +11,15 @@ use crate::ToolsError; const RESULT_LIMIT: usize = 1000; #[derive(Debug, Deserialize, JsonSchema)] -struct GlobParams { +pub(crate) struct GlobParams { /// Worker-local alias of the Workdir attachment to use. #[serde(default)] - target_workdir: Option, + pub target_workdir: Option, /// Glob pattern, for example `**/*.rs` or `src/**/test_*.py`. - pattern: String, + pub pattern: String, /// Logical Workdir-relative directory. Defaults to the Workdir root. #[serde(default)] - path: Option, + pub path: Option, } struct GlobTool { @@ -44,42 +44,57 @@ impl Tool for GlobTool { Some(path) => WorkdirPath::new(&path).map_err(ToolsError::from)?, None => WorkdirPath::root(), }; - let pattern = params.pattern; - tracing::debug!(%pattern, %path, "Glob"); - let result = selected - .session - .glob(GlobRequest { - pattern: pattern.clone(), - path, - limit: RESULT_LIMIT, - }) - .await - .map_err(ToolsError::from)?; - let mut body = result - .paths - .iter() - .map(ToString::to_string) - .collect::>() - .join("\n"); - if !body.is_empty() { - body.push('\n'); - } - let summary = if result.paths.is_empty() { - format!("No files found matching {pattern}") - } else if result.truncated { - format!( - "Found {}+ files matching {pattern} (truncated to {RESULT_LIMIT})", - result.paths.len() - ) - } else { - format!("Found {} file(s) matching {pattern}", result.paths.len()) - }; - Ok(ToolOutput { + Ok(execute_glob( + crate::search_target::SearchTarget::tool(selected.session), + path, + params.pattern, + ) + .await? + .output) + } +} + +pub(crate) async fn execute_glob( + target: crate::search_target::SearchTarget, + path: WorkdirPath, + pattern: String, +) -> Result { + tracing::debug!(%pattern, %path, "Glob"); + let result = target + .glob(GlobRequest { + pattern: pattern.clone(), + path, + limit: RESULT_LIMIT, + }) + .await?; + let mut body = result + .paths + .iter() + .map(ToString::to_string) + .collect::>() + .join("\n"); + if !body.is_empty() { + body.push('\n'); + } + let summary = if result.paths.is_empty() { + format!("No files found matching {pattern}") + } else if result.truncated { + format!( + "Found {}+ files matching {pattern} (truncated to {RESULT_LIMIT})", + result.paths.len() + ) + } else { + format!("Found {} file(s) matching {pattern}", result.paths.len()) + }; + Ok(crate::checkout::CheckoutToolOutput { + output: ToolOutput { summary, content: (!body.is_empty()).then_some(body), attachments: Vec::new(), - }) - } + }, + paths: result.paths, + validator: None, + }) } pub fn glob_tool(session: WorkdirSessionHandle) -> ToolDefinition { diff --git a/crates/tools/src/grep.rs b/crates/tools/src/grep.rs index 190b8590e..85257a8cf 100644 --- a/crates/tools/src/grep.rs +++ b/crates/tools/src/grep.rs @@ -22,16 +22,16 @@ enum OutputMode { } #[derive(Debug, Deserialize, JsonSchema)] -struct GrepParams { +pub(crate) struct GrepParams { /// Worker-local alias of the Workdir attachment to use. #[serde(default)] target_workdir: Option, pattern: String, /// Workdir-relative path, or an absolute path covered by readable scope. Defaults to the Workdir root. #[serde(default)] - path: Option, + pub path: Option, #[serde(default)] - glob: Option, + pub glob: Option, #[serde(default, rename = "type")] file_type: Option, #[serde(default)] @@ -70,69 +70,85 @@ impl Tool for GrepTool { params.target_workdir.as_deref(), workdir::WorkdirSessionCapability::Grep, )?; - let path = match params.path { - Some(path) => WorkdirPath::new_scoped(&path).map_err(ToolsError::from)?, + let path = match params.path.as_deref() { + Some(path) => WorkdirPath::new_scoped(path).map_err(ToolsError::from)?, None => WorkdirPath::root(), }; - let mode = match params.output_mode.unwrap_or_default() { - OutputMode::FilesWithMatches => GrepOutputMode::FilesWithMatches, - OutputMode::Content => GrepOutputMode::Content, - OutputMode::Count => GrepOutputMode::Count, - }; - let (before_context, after_context) = params - .context - .map(|context| (context, context)) - .unwrap_or((params.before.unwrap_or(0), params.after.unwrap_or(0))); - let head_limit = params.head_limit.unwrap_or(DEFAULT_HEAD_LIMIT); - let result = selected - .session - .grep(GrepRequest { - pattern: params.pattern, - path, - glob: params.glob, - file_type: params.file_type, - case_insensitive: params.case_insensitive, - before_context, - after_context, - multiline: params.multiline, - output_mode: mode, - limit: head_limit, - offset: params.offset.unwrap_or(0), - }) - .await - .map_err(ToolsError::from)?; + Ok(execute_grep( + crate::search_target::SearchTarget::tool(selected.session), + path, + params, + ) + .await? + .output) + } +} - let summary = if result.match_count == 0 { - match mode { - GrepOutputMode::Content => "No matches".to_owned(), - _ => "No files matched".to_owned(), +pub(crate) async fn execute_grep( + target: crate::search_target::SearchTarget, + path: WorkdirPath, + params: GrepParams, +) -> Result { + let mode = match params.output_mode.unwrap_or_default() { + OutputMode::FilesWithMatches => GrepOutputMode::FilesWithMatches, + OutputMode::Content => GrepOutputMode::Content, + OutputMode::Count => GrepOutputMode::Count, + }; + let (before_context, after_context) = params + .context + .map(|context| (context, context)) + .unwrap_or((params.before.unwrap_or(0), params.after.unwrap_or(0))); + let head_limit = params.head_limit.unwrap_or(DEFAULT_HEAD_LIMIT); + let result = target + .grep(GrepRequest { + pattern: params.pattern, + path, + glob: params.glob, + file_type: params.file_type, + case_insensitive: params.case_insensitive, + before_context, + after_context, + multiline: params.multiline, + output_mode: mode, + limit: head_limit, + offset: params.offset.unwrap_or(0), + }) + .await?; + + let summary = if result.match_count == 0 { + match mode { + GrepOutputMode::Content => "No matches".to_owned(), + _ => "No files matched".to_owned(), + } + } else { + match mode { + GrepOutputMode::FilesWithMatches => { + format!("Found matches in {} file(s)", result.matched_files) } - } else { - match mode { - GrepOutputMode::FilesWithMatches => { - format!("Found matches in {} file(s)", result.matched_files) - } - GrepOutputMode::Count => format!( - "Found matches in {} file(s), {} total line(s)", - result.matched_files, result.match_count - ), - GrepOutputMode::Content => format!( - "{} matching line(s) in {} file(s)", - result.match_count, result.matched_files - ), - } - }; - let summary = if result.truncated { - format!("{summary} (truncated at {head_limit})") - } else { - summary - }; - Ok(ToolOutput { + GrepOutputMode::Count => format!( + "Found matches in {} file(s), {} total line(s)", + result.matched_files, result.match_count + ), + GrepOutputMode::Content => format!( + "{} matching line(s) in {} file(s)", + result.match_count, result.matched_files + ), + } + }; + let summary = if result.truncated { + format!("{summary} (truncated at {head_limit})") + } else { + summary + }; + Ok(crate::checkout::CheckoutToolOutput { + output: ToolOutput { summary, content: (!result.output.is_empty()).then_some(result.output), attachments: Vec::new(), - }) - } + }, + paths: result.paths, + validator: None, + }) } pub fn grep_tool(session: WorkdirSessionHandle) -> ToolDefinition { diff --git a/crates/tools/src/lib.rs b/crates/tools/src/lib.rs index 0aaabe8d6..790dabcf3 100644 --- a/crates/tools/src/lib.rs +++ b/crates/tools/src/lib.rs @@ -9,20 +9,24 @@ //! and command capability, while the Runtime process and OS user remain the //! trusted execution boundary. +pub mod checkout; pub mod error; pub mod tracker; mod bash; mod edit; +mod file_target; mod glob; mod grep; mod read; mod routing; +mod search_target; mod view_image; mod web; mod write; pub use bash::bash_tool; +pub use checkout::{CheckoutToolOutput, execute_checkout_tool}; pub use edit::edit_tool; pub use error::ToolsError; pub use glob::glob_tool; diff --git a/crates/tools/src/read.rs b/crates/tools/src/read.rs index 5a6800b02..f9d49ab1f 100644 --- a/crates/tools/src/read.rs +++ b/crates/tools/src/read.rs @@ -8,7 +8,7 @@ use serde::Deserialize; use crate::error::ToolsError; use crate::tracker::Tracker; -use workdir::{ReadRequest, WorkdirPath, WorkdirSessionHandle, WorkdirSessionRouter}; +use workdir::{WorkdirPath, WorkdirSessionHandle, WorkdirSessionRouter}; const DESCRIPTION: &str = "Read a text file from a Workdir attachment selected by its Worker-local alias. \ Supports offset/limit for large files. Returns line-numbered output (1-based). \ @@ -26,12 +26,8 @@ pub(crate) struct ReadParams { pub target_workdir: Option, /// Workdir-relative path, or an absolute path covered by readable scope. pub file_path: String, - /// 0-based line offset from the start. Defaults to 0. - #[serde(default)] - pub offset: Option, - /// Maximum number of lines to return. Defaults to 2000. - #[serde(default)] - pub limit: Option, + #[serde(flatten)] + pub read: fs_operation::text::LineReadArgs, } pub(crate) struct ReadTool { @@ -46,8 +42,7 @@ impl Tool for ReadTool { input_json: &str, _ctx: agen::tool::ToolExecutionContext, ) -> Result { - let params: ReadParams = serde_json::from_str(input_json) - .map_err(|e| ToolError::InvalidArgument(format!("invalid Read input: {e}")))?; + let params: ReadParams = crate::error::decode_file_input(input_json, "Read")?; let selected = crate::routing::resolve_session( &self.router, params.target_workdir.as_deref(), @@ -56,58 +51,65 @@ impl Tool for ReadTool { let tracker = self .tracker .scoped_attachment(&selected.alias, selected.generation); - let offset = params.offset.unwrap_or(0); - let limit = params.limit.unwrap_or(DEFAULT_LIMIT).max(1); - - let path = WorkdirPath::new_scoped(¶ms.file_path).map_err(ToolsError::from)?; - tracing::debug!(path = %path, offset, limit, "Read"); - - let result = selected - .session - .read(ReadRequest { - path: path.clone(), - offset, - limit, - max_bytes: PROVIDER_BYTE_LIMIT, - }) - .await - .map_err(ToolsError::from)?; - tracker.record_workdir_hash(&path, result.content_hash); - - let text = String::from_utf8_lossy(&result.bytes).into_owned(); - let rendered = render_provider_read( - &text, - result.start_line, - result.total_lines, - result.truncated, - ); - - let summary = if rendered.truncated { - format!( - "Read {} line(s) [{}..{}] of {} from {}", - rendered.line_count, - offset + 1, - offset + rendered.line_count, - rendered.total_lines, - path - ) - } else { - format!("Read {} line(s) from {}", rendered.line_count, path) - }; - - Ok(ToolOutput { - summary, - content: Some(rendered.body), - attachments: Vec::new(), - }) + Ok(execute_read( + crate::file_target::FileTarget { + session: selected.session, + path: WorkdirPath::new_scoped(¶ms.file_path).map_err(ToolsError::from)?, + validator: None, + }, + tracker, + params, + ) + .await? + .output) } } -struct Rendered { - body: String, - line_count: usize, - total_lines: usize, - truncated: bool, +pub(crate) async fn execute_read( + target: crate::file_target::FileTarget, + tracker: Tracker, + params: ReadParams, +) -> Result { + let (offset, limit) = params.read.range(DEFAULT_LIMIT); + let path = &target.path; + tracing::debug!(path = %path, offset, limit, "Read"); + let (result, validator) = target.read(offset, limit, PROVIDER_BYTE_LIMIT).await?; + tracker.record_workdir_observation(path, result.content_hash, result.total_lines); + // Source/response bounds were enforced by the streaming provider. Lossy + // UTF-8 conversion is the existing Tool presentation contract; rendering + // below independently bounds its possible byte expansion. + let text = fs_operation::text::read( + String::from_utf8_lossy(&result.bytes).into_owned(), + Default::default(), + ) + .map_err(crate::error::text_error)?; + let rendered = render_provider_read( + &text.content, + result.start_line, + result.total_lines, + result.truncated, + ); + let summary = if rendered.truncated { + format!( + "Read {} line(s) [{}..{}] of {} from {}", + rendered.line_count, + offset.saturating_add(1), + offset.saturating_add(rendered.line_count), + rendered.total_lines, + path + ) + } else { + format!("Read {} line(s) from {}", rendered.line_count, path) + }; + Ok(crate::checkout::CheckoutToolOutput { + output: ToolOutput { + summary, + content: Some(rendered.body), + attachments: Vec::new(), + }, + paths: Vec::new(), + validator, + }) } fn render_provider_read( @@ -115,45 +117,14 @@ fn render_provider_read( start_line: usize, total_lines: usize, truncated: bool, -) -> Rendered { - use std::fmt::Write as _; - let lines = text.lines().collect::>(); - let mut body = String::with_capacity(text.len().saturating_add(lines.len() * 8)); - for (index, line) in lines.iter().enumerate() { - let _ = writeln!(&mut body, "{:>6}\t{}", start_line + index + 1, line); - } - Rendered { - body, - line_count: lines.len(), +) -> fs_operation::text::Rendered { + fs_operation::text::render_numbered( + text, + start_line, total_lines, - truncated: start_line > 0 || truncated, - } -} - -/// Format a slice of lines from `text` with `cat -n` style 1-based line -/// numbers. Pure function — no I/O, no history touching. -#[cfg(test)] -fn render_numbered(text: &str, offset: usize, limit: usize) -> Rendered { - let all_lines: Vec<&str> = text.lines().collect(); - let total_lines = all_lines.len(); - let start = offset.min(total_lines); - let end = start.saturating_add(limit).min(total_lines); - let slice = &all_lines[start..end]; - let line_count = slice.len(); - - use std::fmt::Write as _; - let mut body = String::with_capacity(text.len().saturating_add(line_count * 8)); - for (i, line) in slice.iter().enumerate() { - let lineno = start + i + 1; - let _ = writeln!(&mut body, "{:>6}\t{}", lineno, line); - } - - Rendered { - body, - line_count, - total_lines, - truncated: start > 0 || end < total_lines, - } + truncated, + PROVIDER_BYTE_LIMIT, + ) } /// Factory for the `Read` tool bound to one compatibility session. @@ -195,6 +166,32 @@ mod tests { (dir, session, Tracker::new()) } + #[test] + fn read_numbering_and_unicode_expansion_keep_rendered_output_bounded() { + for text in [ + "\n".repeat(PROVIDER_BYTE_LIMIT), + "界".repeat(PROVIDER_BYTE_LIMIT), + "\u{1}".repeat(PROVIDER_BYTE_LIMIT), + ] { + let rendered = render_provider_read(&text, 0, text.lines().count(), false); + assert!(rendered.body.len() <= PROVIDER_BYTE_LIMIT); + assert!(serde_json::to_vec(&rendered.body).unwrap().len() < 2 * 1024 * 1024); + } + let rendered = render_provider_read( + &"\n".repeat(PROVIDER_BYTE_LIMIT), + 0, + PROVIDER_BYTE_LIMIT, + false, + ); + assert!(rendered.truncated); + assert!(rendered.line_count < PROVIDER_BYTE_LIMIT); + assert!( + rendered + .body + .contains("truncated at rendered output byte limit") + ); + } + #[tokio::test] async fn read_tool_basic_records_history() { let (dir, fs, tracker) = setup(); diff --git a/crates/tools/src/search_target.rs b/crates/tools/src/search_target.rs new file mode 100644 index 000000000..59110e57d --- /dev/null +++ b/crates/tools/src/search_target.rs @@ -0,0 +1,81 @@ +//! Provider dispatch shared by native and ordinary search processing. +//! Request construction/defaults and rendering remain in Glob/Grep; this only +//! selects the provider surface and verifies the returned readonly result kind. +use agen::tool::ToolError; +use workdir::{ + CheckoutSearchOperation, CheckoutSearchRequest, CheckoutSearchResult, GlobRequest, GlobResult, + GrepRequest, GrepResult, WorkdirSessionHandle, +}; + +use crate::{ToolsError, file_target::checked_error}; + +pub(crate) struct SearchTarget { + session: WorkdirSessionHandle, + checkout: bool, +} + +impl SearchTarget { + pub fn tool(session: WorkdirSessionHandle) -> Self { + Self { + session, + checkout: false, + } + } + + pub fn checkout(session: WorkdirSessionHandle) -> Self { + Self { + session, + checkout: true, + } + } + + pub async fn glob(&self, request: GlobRequest) -> Result { + if self.checkout { + let result = self + .session + .checkout_search(CheckoutSearchRequest::new(CheckoutSearchOperation::Glob( + request, + ))) + .await + .map_err(checked_error)?; + glob_result(result) + } else { + self.session + .glob(request) + .await + .map_err(|error| ToolsError::from(error).into()) + } + } + + pub async fn grep(&self, request: GrepRequest) -> Result { + if self.checkout { + let result = self + .session + .checkout_search(CheckoutSearchRequest::new(CheckoutSearchOperation::Grep( + request, + ))) + .await + .map_err(checked_error)?; + grep_result(result) + } else { + self.session + .grep(request) + .await + .map_err(|error| ToolsError::from(error).into()) + } + } +} + +pub(crate) fn glob_result(result: CheckoutSearchResult) -> Result { + match result { + CheckoutSearchResult::Glob(result) => Ok(result), + _ => Err(crate::file_target::readonly_result_error("Glob")), + } +} + +pub(crate) fn grep_result(result: CheckoutSearchResult) -> Result { + match result { + CheckoutSearchResult::Grep(result) => Ok(result), + _ => Err(crate::file_target::readonly_result_error("Grep")), + } +} diff --git a/crates/tools/src/tracker.rs b/crates/tools/src/tracker.rs index feaaf30ff..52cecabf9 100644 --- a/crates/tools/src/tracker.rs +++ b/crates/tools/src/tracker.rs @@ -256,6 +256,25 @@ impl Tracker { .copied() } + /// Record the full provider content hash and total line count, even for a sliced Read. + /// Keeping both under one lock prevents pairing counts from a different observation. + pub fn record_workdir_observation( + &self, + path: &workdir::WorkdirPath, + hash: workdir::ContentHash, + total_lines: usize, + ) { + let key = self.key(PathBuf::from(path.as_str())); + let mut inner = self.inner.lock().unwrap_or_else(|e| e.into_inner()); + inner.line_counts.insert(key.clone(), total_lines); + inner.hashes.insert(key.clone(), hash); + inner.recency.retain(|candidate| candidate != &key); + inner.recency.push_front(key); + if inner.recency.len() > RECENCY_CAPACITY { + inner.recency.pop_back(); + } + } + pub fn record_workdir_hash(&self, path: &workdir::WorkdirPath, hash: workdir::ContentHash) { let key = self.key(PathBuf::from(path.as_str())); let mut inner = self.inner.lock().unwrap_or_else(|e| e.into_inner()); diff --git a/crates/tools/src/write.rs b/crates/tools/src/write.rs index 8479de806..c6f083cba 100644 --- a/crates/tools/src/write.rs +++ b/crates/tools/src/write.rs @@ -9,10 +9,7 @@ use serde::Deserialize; use crate::error::ToolsError; use crate::tracker::Tracker; -use workdir::{ - StatRequest, WorkdirError, WorkdirPath, WorkdirSessionHandle, WorkdirSessionRouter, - WriteRequest, -}; +use workdir::{StatRequest, WorkdirError, WorkdirPath, WorkdirSessionHandle, WorkdirSessionRouter}; const DESCRIPTION: &str = "Create a new file or overwrite an existing one in the selected Workdir attachment. \ Missing parent directories within scope are created automatically. Existing files must have been read first \ @@ -25,8 +22,8 @@ pub(crate) struct WriteParams { pub target_workdir: Option, /// Logical path relative to the bound Workdir root. pub file_path: String, - /// Full content to write. Overwrites any existing content. - pub content: String, + #[serde(flatten)] + pub write: fs_operation::text::WriteArgs, } pub(crate) struct WriteTool { @@ -41,8 +38,7 @@ impl Tool for WriteTool { input_json: &str, ctx: agen::tool::ToolExecutionContext, ) -> Result { - let params: WriteParams = serde_json::from_str(input_json) - .map_err(|e| ToolError::InvalidArgument(format!("invalid Write input: {e}")))?; + let params: WriteParams = crate::error::decode_file_input(input_json, "Write")?; let selected = crate::routing::resolve_session( &self.router, @@ -53,50 +49,84 @@ impl Tool for WriteTool { .tracker .scoped_attachment(&selected.alias, selected.generation); let path = WorkdirPath::new(¶ms.file_path).map_err(ToolsError::from)?; - tracing::debug!(path = %path, bytes = params.content.len(), "Write"); + Ok(execute_write( + crate::file_target::FileTarget { + session: selected.session, + path, + validator: None, + }, + tracker, + params.write.content, + None, + ctx, + ) + .await? + .output) + } +} - let mutation_key = PathBuf::from(path.as_str()); - let _mutation_permit = tracker.acquire_mutation(&mutation_key, &ctx).await; - let expected_hash = match selected +/// `create_path` is a checked parent-relative create-new target, never an overwrite. +pub(crate) async fn execute_write( + target: crate::file_target::FileTarget, + tracker: Tracker, + content: String, + create_path: Option, + ctx: agen::tool::ToolExecutionContext, +) -> Result { + // Provider size/access checks still run inside the checked save. The shared + // core owns text semantics, not the provider's bounds or lifecycle. + let content = fs_operation::text::write(content, Default::default()) + .map_err(crate::error::text_error)? + .content; + let path = create_path.as_ref().unwrap_or(&target.path); + tracing::debug!(path = %path, bytes = content.len(), "Write"); + let mutation_key = PathBuf::from(path.as_str()); + let _mutation_permit = tracker.acquire_mutation(&mutation_key, &ctx).await; + let expected_hash = if create_path.is_some() { + None + } else if target.validator.is_some() { + // Native Write is existing-file-only, and always requires a prior observation. + Some(tracker.expected_workdir_hash(path)?) + } else { + match target .session .stat(StatRequest { path: path.clone() }) .await { - Ok(_) => Some(tracker.expected_workdir_hash(&path)?), + Ok(_) => Some(tracker.expected_workdir_hash(path)?), Err(WorkdirError::NotFound(_)) => None, Err(error) => return Err(ToolsError::from(error).into()), - }; - - let old_line_count = tracker.observed_workdir_line_count(&path).unwrap_or(0); - let outcome = selected - .session - .write(WriteRequest { - path: path.clone(), - content: params.content.as_bytes().to_vec(), - expected_hash, - }) - .await - .map_err(ToolsError::from)?; - - tracker.record_change(params.content.lines().count(), old_line_count); - tracker.record_workdir_content(&path, params.content.as_bytes()); - - let summary = format!( - "{} {} ({} bytes)", - if outcome.created { - "Created" - } else { - "Overwrote" - }, - path, - outcome.bytes_written - ); - Ok(ToolOutput { + } + }; + let old_line_count = tracker.observed_workdir_line_count(path).unwrap_or(0); + let (outcome, validator) = target + .write( + content.as_bytes().to_vec(), + expected_hash, + create_path.clone(), + ) + .await?; + tracker.record_change(content.lines().count(), old_line_count); + tracker.record_workdir_content(path, content.as_bytes()); + let summary = format!( + "{} {} ({} bytes)", + if outcome.created { + "Created" + } else { + "Overwrote" + }, + path, + outcome.bytes_written + ); + Ok(crate::checkout::CheckoutToolOutput { + output: ToolOutput { summary, content: None, attachments: Vec::new(), - }) - } + }, + paths: create_path.into_iter().collect(), + validator, + }) } /// Factory for the `Write` tool bound to one compatibility session. diff --git a/crates/tools/tests/integration.rs b/crates/tools/tests/integration.rs index 6cf261ccb..c8c55e8d6 100644 --- a/crates/tools/tests/integration.rs +++ b/crates/tools/tests/integration.rs @@ -278,7 +278,7 @@ async fn target_alias_routes_each_tool_and_isolates_read_before_edit() { let bash = call( ®.get("Bash"), - json!({ "target_workdir": "right", "command": "pwd" }), + json!({ "target_workdir": "right", "command": "pwd", "cwd": "." }), ) .await; assert_eq!( @@ -1005,7 +1005,7 @@ async fn bash_inherits_workdir_cwd() { // `pwd` should canonicalize to the workspace root we set up. let (dir, _spill, reg) = setup(); let bash = reg.get("Bash"); - let out = call(&bash, json!({ "command": "pwd" })).await; + let out = call(&bash, json!({ "command": "pwd", "cwd": "." })).await; let body = out.content.unwrap(); let actual = std::fs::canonicalize(body.trim()).unwrap(); let expected = std::fs::canonicalize(dir.path()).unwrap(); @@ -1016,7 +1016,11 @@ async fn bash_inherits_workdir_cwd() { async fn bash_provider_output_exposes_readable_retained_path() { let (_dir, spill, reg) = setup(); let bash = reg.get("Bash"); - let out = call(&bash, json!({ "command": "printf 'x%.0s' {1..20480}" })).await; + let out = call( + &bash, + json!({ "command": "printf 'x%.0s' {1..20480}", "cwd": "." }), + ) + .await; let body = out.content.unwrap(); assert!(body.contains("bounded WorkdirSession command output")); assert!(body.contains("full output saved to")); @@ -1038,7 +1042,7 @@ async fn bash_cancellation_returns_bounded_progress_as_terminal_output() { "printf 'before\\n'; printf 'err-before\\n' >&2; sleep 1; touch {}; printf 'after\\n'", marker.display() ); - let input = serde_json::to_string(&json!({ "command": command })).unwrap(); + let input = serde_json::to_string(&json!({ "command": command, "cwd": "." })).unwrap(); let context = ToolExecutionContext::new("call-heavy", "attempt-heavy", 0); let bash = reg.get("Bash"); let executing = bash.clone(); @@ -1084,7 +1088,7 @@ async fn bash_force_close_cleanup_stops_command_and_keeps_session_reusable() { let (dir, _spill, reg) = setup(); let marker = dir.path().join("must-not-survive-force-close"); let command = format!("sleep 1; touch {}", marker.display()); - let input = serde_json::to_string(&json!({ "command": command })).unwrap(); + let input = serde_json::to_string(&json!({ "command": command, "cwd": "." })).unwrap(); let bash = reg.get("Bash"); let context = ToolExecutionContext::new("call-force", "attempt-force", 0); let (handle, terminal) = ToolExecutionHandle::start(bash.clone(), input, context); @@ -1103,7 +1107,10 @@ async fn bash_force_close_cleanup_stops_command_and_keeps_session_reusable() { ); let output = bash - .execute(r#"{"command":"printf 'reused'"}"#, Default::default()) + .execute( + r#"{"command":"printf 'reused'","cwd":"."}"#, + Default::default(), + ) .await .expect("workdir session remains reusable after cleanup"); assert_eq!(output.content.as_deref(), Some("reused")); diff --git a/crates/tui/Cargo.toml b/crates/tui/Cargo.toml index 9a61c328b..8d1e509c1 100644 --- a/crates/tui/Cargo.toml +++ b/crates/tui/Cargo.toml @@ -31,5 +31,6 @@ async-trait.workspace = true agen.workspace = true [dev-dependencies] +worker-runtime.workspace = true async-trait.workspace = true tempfile = { workspace = true } diff --git a/crates/tui/README.md b/crates/tui/README.md index 9b70d54d7..cbcaf8c8e 100644 --- a/crates/tui/README.md +++ b/crates/tui/README.md @@ -28,3 +28,33 @@ The TUI should display committed events and Worker snapshots rather than inventi - [`../../docs/design/context-history.md`](../../docs/design/context-history.md) - [`../../docs/design/worker-session-state.md`](../../docs/design/worker-session-state.md) + +## Feature invocation editing + +`/` discovers enabled Feature declarations with descriptions and usage. Choose a +name or alias with Tab/Enter, complete its parenthesized arguments, and press +Enter to confirm that selected occurrence as a typed chip. A later Enter sends +the message. Raw slash text and paste content are never dispatched as commands. +Use Alt+Enter beside a Feature chip to rediscover its declaration and reopen its +arguments; Backspace/Delete removes a chip atomically. Typed chips survive local +history, draft restoration, transport retry, and rewind. + +Every completion and chip-re-edit query carries a fresh `request_id`. Replies +must echo that nonce and match the active query context, composer edit/cursor +revision, authority snapshot generation, and Worker target. Identical-prefix ABA +edits, cancelled/reopened queries, permission snapshots, and Worker/view switches +cannot reuse older replies or candidates. Uncorrelated replies are ignored, even +for File queries where the wire field remains optional for legacy callers. + +The declared attachment adapter stages a client-local file in place, replacing +its staging chip with an `UploadedFile` reference when ready. Local file argument +completion runs on the client, not Worker. Alt+Enter beside a failed staging chip +retries with the same upload ID; deleting an unsent chip cancels only its owned +staging resource. Accepted, restored, or history attachments are never deleted by +composer cleanup. Staging blocks Send/Notify and history navigation until it is +ready, retried, or removed. There is no raw `/attach` or `/clear-attachments` +command lane. + +Attachment staging remains Backend-only: the existing Standalone transport has +no client-local file-upload staging API. The TUI reports that limitation and +preserves the draft invocation; it never falls back to raw path dispatch. diff --git a/crates/tui/src/app.rs b/crates/tui/src/app.rs index 1a1f7f156..b856c0867 100644 --- a/crates/tui/src/app.rs +++ b/crates/tui/src/app.rs @@ -35,12 +35,29 @@ pub enum CommandCompletionApply { /// typing inside a `@` / `#` / `/` token. Cleared whenever the trigger /// is invalidated (cursor moved out, whitespace landed inside the /// token, the sigil was deleted, or the candidate was confirmed). +#[derive(Debug, Clone, PartialEq, Eq)] +struct CompletionSnapshot { + input_revision: u64, + generation: u64, + target: String, + worker_view: Option, +} + +struct PendingFeatureEdit { + invocation: protocol::FeatureInvocation, + request_id: String, + snapshot: CompletionSnapshot, +} + pub struct CompletionState { + pub request_id: String, + snapshot: CompletionSnapshot, pub kind: CompletionKind, /// Atom index of the leading sigil (`@` / `#` / `/`). pub prefix_start: usize, /// Text typed after the sigil (sigil itself excluded). pub prefix: String, + pub context: Option, /// Latest candidate set returned by the Worker for `(kind, prefix)`. /// Initially empty until `Event::Completions` lands. pub entries: Vec, @@ -301,6 +318,13 @@ pub struct App { /// Completion popup state, when an `@` / `#` / `/` token is in /// flight. `None` whenever the trigger conditions don't hold. pub completion: Option, + /// Invocation declarations explicitly selected from this composer's `/` + /// completion lane. Only these declarations may turn editable text into a + /// typed FeatureInvoke chip. + selected_feature_invocations: HashMap, + pending_feature_edit: Option, + completion_generation: u64, + completion_target: String, /// Dedicated main-view rewind picker state. pub rewind_picker: Option, rewind_request_pending: bool, @@ -335,6 +359,7 @@ pub struct App { /// Local submit state kept until the accepted run either completes /// normally or reports that the empty assistant turn was rolled back. pending_submit_rollback: Option, + retry_submission: Option, /// Last rolled-back submit that could not be restored because the /// composer already contained unsent user input. last_rolled_back_input: Option>, @@ -382,6 +407,10 @@ impl App { cache: FileCache::new(), assistant_streaming: false, completion: None, + selected_feature_invocations: HashMap::new(), + pending_feature_edit: None, + completion_generation: 0, + completion_target: protocol::new_submission_request_id(), rewind_picker: None, rewind_request_pending: false, rewind_refresh_fence: false, @@ -394,6 +423,7 @@ impl App { input_history: ComposerInputHistory::new(), input_history_store: None, pending_submit_rollback: None, + retry_submission: None, last_rolled_back_input: None, } } @@ -512,6 +542,7 @@ impl App { /// Cycle the presentation-only transcript/task view. Input and control /// methods continue to target the parent Worker regardless of selection. pub fn cycle_worker_view(&mut self) -> bool { + self.invalidate_completion_generation(); if self.internal_workers.is_empty() { self.selected_internal_worker_session_id = None; return false; @@ -571,38 +602,97 @@ impl App { } } + fn completion_snapshot(&self) -> CompletionSnapshot { + CompletionSnapshot { + input_revision: self.input.revision(), + generation: self.completion_generation, + target: self.completion_target.clone(), + worker_view: self.selected_internal_worker_session_id.clone(), + } + } + + pub(crate) fn set_completion_target(&mut self, target: String) { + if self.completion_target != target { + self.completion_target = target; + self.invalidate_completion_generation(); + self.selected_feature_invocations.clear(); + } + } + + /// Authority/snapshot/target changes revoke in-flight queries and visible candidates. + fn invalidate_completion_generation(&mut self) { + self.completion_generation = self.completion_generation.wrapping_add(1); + self.completion = None; + self.pending_feature_edit = None; + } + + #[cfg(test)] + pub(crate) fn completion_request_id(&self) -> Option { + self.completion + .as_ref() + .map(|state| state.request_id.clone()) + .or_else(|| { + self.pending_feature_edit + .as_ref() + .map(|edit| edit.request_id.clone()) + }) + } + /// Re-evaluate the completion popup against the current input. - /// Returns a `Method::ListCompletions` to send when the - /// `(kind, prefix_start, prefix)` triple changed; otherwise `None`. + /// Returns a fresh correlated `Method::ListCompletions` when the prefix, + /// context, token location, or composer/authority/target snapshot changes. + /// An unchanged active query returns `None`. /// Callers should invoke this after every input mutation that could /// move the cursor or change atoms. pub fn refresh_completion(&mut self) -> Option { + self.pending_feature_edit = None; if self.is_command_mode() { self.completion = None; return None; } - match self.input.pending_completion_prefix() { - Some((kind, start, prefix)) => { - let need_query = match &self.completion { - Some(c) => c.kind != kind || c.prefix_start != start || c.prefix != prefix, - None => true, - }; - let entries = match self.completion.take() { - Some(c) if c.kind == kind && c.prefix_start == start => c.entries, - _ => Vec::new(), - }; + let descriptors = self + .selected_feature_invocations + .values() + .cloned() + .collect::>(); + let pending = self + .input + .pending_feature_argument_completion(&descriptors) + .map(|(kind, start, prefix, context)| (kind, start, prefix, Some(context))) + .or_else(|| { + self.input + .pending_completion_prefix() + .map(|(kind, start, prefix)| (kind, start, prefix, None)) + }); + match pending { + Some((kind, start, prefix, context)) => { + let snapshot = self.completion_snapshot(); + if self.completion.as_ref().is_some_and(|state| { + state.kind == kind + && state.prefix_start == start + && state.prefix == prefix + && state.context == context + && state.snapshot == snapshot + }) { + return None; + } + let request_id = protocol::new_submission_request_id(); self.completion = Some(CompletionState { + request_id: request_id.clone(), + snapshot, kind, prefix_start: start, prefix: prefix.clone(), - entries, + context: context.clone(), + entries: Vec::new(), selected: 0, }); - if need_query { - Some(Method::ListCompletions { kind, prefix }) - } else { - None - } + Some(Method::ListCompletions { + kind, + prefix, + context, + request_id: Some(request_id), + }) } None => { self.completion = None; @@ -632,7 +722,7 @@ impl App { } pub fn cancel_completion(&mut self) { - self.completion = None; + self.invalidate_completion_generation(); } /// Tab path: insert the popup-selected entry's value (with a @@ -643,21 +733,132 @@ impl App { /// Returns the follow-up `Method::ListCompletions` to send when /// the new prefix differs from the old one. pub fn apply_completion_text(&mut self) -> Option { + if self + .completion + .as_ref() + .is_some_and(|state| state.snapshot != self.completion_snapshot()) + { + self.cancel_completion(); + return None; + } let state = self.completion.as_ref()?; if state.entries.is_empty() { return None; } - let entry = &state.entries[state.selected]; - let text = if entry.is_dir { - format!("{}/", entry.value) - } else { - entry.value.clone() + let entry = state.entries[state.selected].clone(); + let mut typed_start = state.prefix_start + 1; + let text = match state.kind { + CompletionKind::File => { + if entry.is_dir { + format!("{}/", entry.value) + } else { + entry.value.clone() + } + } + CompletionKind::Feature => { + let descriptor = entry.invocation.clone()?; + if descriptor.validate().is_err() + || !(entry.value == descriptor.name + || descriptor.aliases.contains(&entry.value)) + { + return None; + } + self.selected_feature_invocations + .insert(descriptor.identity.to_string(), descriptor.clone()); + self.input + .replace_feature_name_completion(typed_start, &format!("{}(", entry.value)); + self.input + .select_feature_invocation(state.prefix_start, descriptor); + self.input_history.cancel_browse(); + return self.refresh_completion(); + } + CompletionKind::FeatureArgument => { + if self.input.can_complete_argument_name(state.prefix_start) + && let Some(name) = entry.value.strip_suffix('=') + && state + .context + .as_ref() + .and_then(|context| { + self.selected_feature_invocations + .get(&context.invocation.to_string()) + }) + .is_some_and(|descriptor| { + descriptor + .arguments + .iter() + .any(|argument| argument.name == name) + }) + { + self.input + .replace_argument_completion(typed_start, &entry.value); + self.input_history.cancel_browse(); + return self.refresh_completion(); + } + if state + .context + .as_ref() + .and_then(|context| context.argument.as_ref()) + .is_some() + { + if typed_start > 0 && self.input.char_at(typed_start - 1) == Some('"') { + typed_start -= 1; + } + let context = state.context.as_ref()?; + let descriptor = self + .selected_feature_invocations + .get(&context.invocation.to_string())?; + let argument = descriptor + .arguments + .iter() + .find(|argument| Some(&argument.name) == context.argument.as_ref())?; + match &argument.value_type { + protocol::InvocationArgumentType::Integer => { + entry.value.parse::().ok()?.to_string() + } + protocol::InvocationArgumentType::Boolean + if matches!(entry.value.as_str(), "true" | "false") => + { + entry.value.clone() + } + protocol::InvocationArgumentType::Boolean => return None, + _ => protocol::quote_invocation_string(&if entry.is_dir { + format!("{}/", entry.value.trim_end_matches('/')) + } else { + entry.value.clone() + }), + } + } else { + let context = state.context.as_ref()?; + let descriptor = self + .selected_feature_invocations + .get(&context.invocation.to_string())?; + if !entry.value.ends_with('=') + || !descriptor + .arguments + .iter() + .any(|argument| argument.name == entry.value.trim_end_matches('=')) + { + return None; + } + entry.value.clone() + } + } }; - // `prefix_start` indexes the sigil atom; the text we want to - // replace lives just after it (sigil itself stays). - let typed_start = state.prefix_start + 1; + // `prefix_start` indexes the sigil atom for Feature/file names and one + // atom before the active replacement range for Feature arguments. + let text = + if state.kind == CompletionKind::FeatureArgument && entry.is_dir && text.ends_with('"') + { + text[..text.len() - 1].to_string() + } else { + text + }; self.input_history.cancel_browse(); - self.input.replace_with_text_at(typed_start, &text); + if state.kind == CompletionKind::FeatureArgument { + self.input.replace_argument_completion(typed_start, &text); + } else { + self.input.replace_with_text_at(typed_start, &text); + } self.refresh_completion() } @@ -681,6 +882,14 @@ impl App { /// so a race-y top entry shouldn't block confirmation when the /// typed text matches another entry. pub fn chipify_completion_if_exact_match(&mut self) -> bool { + if self + .completion + .as_ref() + .is_some_and(|state| state.snapshot != self.completion_snapshot()) + { + self.cancel_completion(); + return false; + } let Some(state) = self.completion.as_ref() else { return false; }; @@ -714,6 +923,7 @@ impl App { self.input_history.cancel_browse(); match kind { CompletionKind::File => self.input.replace_with_file_ref(start, value), + CompletionKind::Feature | CompletionKind::FeatureArgument => return false, } self.completion = None; true @@ -729,6 +939,14 @@ impl App { /// for drill-in — chip-ifying a directory on Enter would strand /// the user with no way to inspect children. pub fn chipify_selected_completion_if_committable(&mut self) -> bool { + if self + .completion + .as_ref() + .is_some_and(|state| state.snapshot != self.completion_snapshot()) + { + self.cancel_completion(); + return false; + } let Some(state) = self.completion.as_ref() else { return false; }; @@ -745,12 +963,98 @@ impl App { self.input_history.cancel_browse(); match kind { CompletionKind::File => self.input.replace_with_file_ref(start, value), + CompletionKind::Feature | CompletionKind::FeatureArgument => return false, } self.completion = None; true } + pub fn is_client_file_completion(&self, context: &protocol::CompletionContext) -> bool { + self.selected_feature_invocations + .get(&context.invocation.to_string()) + .is_some_and(|descriptor| { + descriptor.arguments.iter().any(|argument| { + Some(&argument.name) == context.argument.as_ref() + && (matches!( + argument.completion, + protocol::InvocationCompletion::ClientFile + ) || matches!( + argument.value_type, + protocol::InvocationArgumentType::ClientFile + )) + }) + }) + } + + pub fn attachment_invocations(&self) -> Vec<(String, std::path::PathBuf)> { + self.input + .submit_segments() + .into_iter() + .filter_map(|segment| { + let Segment::FeatureInvoke { invocation } = segment else { + return None; + }; + let descriptor = self + .selected_feature_invocations + .get(&invocation.identity.to_string())?; + if descriptor.client_adapter != Some(protocol::InvocationClientAdapter::Attachment) + { + return None; + } + invocation.arguments.iter().find_map(|argument| { + let schema = descriptor + .arguments + .iter() + .find(|schema| schema.name == argument.name)?; + match (&schema.value_type, &argument.value) { + ( + protocol::InvocationArgumentType::ClientFile, + protocol::InvocationValue::String(path), + ) if !path.is_empty() => Some(( + invocation.invocation_id.clone(), + std::path::PathBuf::from(path), + )), + _ => None, + } + }) + }) + .collect() + } + + /// Alt+Enter reopens an adjacent chip using freshly discovered enabled metadata. + pub fn edit_adjacent_feature_invocation(&mut self) -> Option { + let invocation = self.input.adjacent_feature_invocation()?; + let prefix = invocation.name.clone(); + let request_id = protocol::new_submission_request_id(); + self.pending_feature_edit = Some(PendingFeatureEdit { + invocation, + request_id: request_id.clone(), + snapshot: self.completion_snapshot(), + }); + self.completion = None; + Some(Method::ListCompletions { + request_id: Some(request_id), + kind: CompletionKind::Feature, + prefix, + context: None, + }) + } + pub fn submit_input(&mut self) -> Option { + if self.input.has_attachment_stages() { + self.push_error("Attachment is still staging or failed; wait, retry with Alt+Enter, or delete its chip."); + return None; + } + let confirming = self.input.has_selected_feature_invocations(); + if let Err(error) = self.input.finalize_feature_invocations() { + self.push_error(format!("Invalid Feature invocation: {error}")); + return None; + } + if confirming || !self.attachment_invocations().is_empty() { + // Console stages adapters in-place; never record local paths in history or send them. + self.completion = None; + return None; + } let segments = self.input.submit_segments(); if segments_are_blank(&segments) { // Empty Enter only does something meaningful when the Worker @@ -769,15 +1073,23 @@ impl App { } pub fn submit_notify_input(&mut self) -> Option { + if self.input.has_attachment_stages() || self.input.has_selected_feature_invocations() { + self.push_error("Notify accepts text only; confirm or remove Feature invocations and staging chips first."); + return None; + } let segments = self.input.submit_segments(); if segments_are_blank(&segments) { return None; } - if segments - .iter() - .any(|segment| matches!(segment, Segment::UploadedFile { .. })) - { - self.push_error("Notify accepts text only; remove attachments or queue a Submit."); + if segments.iter().any(|segment| { + matches!( + segment, + Segment::UploadedFile { .. } | Segment::FeatureInvoke { .. } + ) + }) { + self.push_error( + "Notify accepts text only; remove attachments or Feature invocations or queue a Submit.", + ); return None; } let message = Segment::flatten_to_text(&segments); @@ -794,7 +1106,8 @@ impl App { return; }; self.pending_submit_rollback = None; - if self.input.is_empty() { + if self.input.is_empty() || self.input.submit_segments() == *input { + self.retry_submission = Some(method.clone()); self.input.replace_with_segments(input); self.completion = None; } else { @@ -816,6 +1129,11 @@ impl App { block_start: self.blocks.len(), turn_before: self.turn_index, }); + if let Some(method @ Method::Submit { .. }) = self.retry_submission.take() { + if matches!(&method, Method::Submit { input, .. } if *input == segments) { + return method; + } + } Method::Submit { submission_request_id: protocol::new_submission_request_id(), input: segments, @@ -862,7 +1180,10 @@ impl App { } pub fn browse_input_history_older(&mut self) -> bool { - if self.input_history.entries.is_empty() { + if self.input.has_attachment_stages() + || self.input.has_selected_feature_invocations() + || self.input_history.entries.is_empty() + { return false; } let draft = self.input.submit_segments(); @@ -1182,7 +1503,21 @@ impl App { } match event { - Event::SubmissionAccepted { .. } | Event::NotificationAccepted { .. } => {} + Event::SubmissionAccepted { + submission_request_id, + .. + } => { + if matches!(&self.retry_submission, Some(Method::Submit { submission_request_id: id, .. }) if *id == submission_request_id) + { + if let Some(Method::Submit { input, .. }) = self.retry_submission.take() { + if self.input.submit_segments() == input { + self.input.clear(); + self.completion = None; + } + } + } + } + Event::NotificationAccepted { .. } => {} Event::SubmissionRejected { message, .. } | Event::NotificationRejected { message, .. } => self.push_error(message), Event::PendingSubmissionsChanged { pending } => { @@ -1577,6 +1912,7 @@ impl App { self.apply_worker_state_snapshot(&state); self.restore_snapshot(&session, greeting, in_flight); self.replace_internal_worker_snapshots(internal_workers); + return self.refresh_completion(); } Event::InternalWorker { worker, @@ -1606,13 +1942,94 @@ impl App { // Command telemetry is an operational Web Console surface. The // TUI continues to render the final Bash ToolResult from history. Event::Command { .. } => {} - Event::Completions { kind, entries } => { - // Apply only if the popup is still on the same - // (kind, prefix) the request was issued for; an - // out-of-date reply (the user typed past it) is dropped. - if let Some(state) = self.completion.as_mut() - && state.kind == kind + Event::Completions { + kind, + prefix, + request_id, + context, + entries, + } => { + let snapshot = self.completion_snapshot(); + if kind == CompletionKind::Feature + && context.is_none() + && self.pending_feature_edit.as_ref().is_some_and(|edit| { + edit.invocation.name == prefix + && request_id.as_ref() == Some(&edit.request_id) + && edit.snapshot == snapshot + }) { + let invocation = self.pending_feature_edit.take().unwrap().invocation; + if let Some(descriptor) = entries + .iter() + .filter_map(|entry| entry.invocation.as_ref()) + .find(|descriptor| { + descriptor.identity == invocation.identity + && descriptor.validate().is_ok() + }) + { + self.selected_feature_invocations + .insert(descriptor.identity.to_string(), descriptor.clone()); + self.input + .edit_feature_invocation(&invocation.invocation_id, descriptor.clone()); + } else { + self.push_error("Feature invocation is unknown or disabled; its typed chip was preserved."); + } + return self.refresh_completion(); + } + // Apply only if the popup is still on the same request context; + // stale argument/provider replies cannot cross invocation scope. + if let Some(state) = self.completion.as_mut() + && request_id.as_ref() == Some(&state.request_id) + && state.snapshot == snapshot + && state.kind == kind + && state.prefix == prefix + && state.context == context + { + let mut entries = entries; + if kind == CompletionKind::Feature { + let mut expanded = Vec::new(); + for entry in entries { + if let Some(descriptor) = &entry.invocation + && entry.value == descriptor.name + && descriptor.validate().is_ok() + { + for name in std::iter::once(&descriptor.name) + .chain(descriptor.aliases.iter()) + .filter(|name| name.starts_with(&prefix)) + { + expanded.push(CompletionEntry { + value: name.clone(), + ..entry.clone() + }); + } + } else { + expanded.push(entry); + } + } + entries = expanded; + } else if kind == CompletionKind::FeatureArgument + && prefix.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_' + }) + && self.input.can_complete_argument_name(state.prefix_start) + && let Some(descriptor) = context.as_ref().and_then(|context| { + self.selected_feature_invocations + .get(&context.invocation.to_string()) + }) + { + for argument in &descriptor.arguments { + let value = format!("{}=", argument.name); + if argument.name.starts_with(&prefix) + && !entries.iter().any(|entry| entry.value == value) + { + entries.push(CompletionEntry { + value, + description: argument.description.clone(), + ..Default::default() + }); + } + } + } state.entries = entries; state.selected = 0; } @@ -1682,6 +2099,7 @@ impl App { ); } Event::Shutdown => { + self.invalidate_completion_generation(); self.mark_orphan_compacts_incomplete(); self.quit = true; } @@ -1890,6 +2308,7 @@ impl App { } pub fn enter_command_mode(&mut self) { + self.invalidate_completion_generation(); self.input_mode = CommandInputMode::Command; self.completion = None; self.command_completion_selected = None; @@ -1897,6 +2316,7 @@ impl App { } pub fn exit_command_mode(&mut self) { + self.invalidate_completion_generation(); self.input_mode = CommandInputMode::Composer; self.command_input.clear(); self.command_completion_selected = None; @@ -2024,6 +2444,7 @@ impl App { } pub fn request_rewind_picker(&mut self) -> Option { + self.invalidate_completion_generation(); // Rewind is a parent Worker control surface. Bring the parent transcript // back into view before presenting diagnostics or the picker. self.selected_internal_worker_session_id = None; @@ -2411,6 +2832,7 @@ impl App { } fn remove_internal_worker(&mut self, worker: InternalWorkerRef, revision: u64) { + self.invalidate_completion_generation(); let session_id = worker.session_id; let Some(index) = self .internal_workers @@ -2495,6 +2917,7 @@ impl App { session: &protocol::SessionSnapshot, greeting: Option, ) { + self.invalidate_completion_generation(); self.run_error_messages.clear(); self.turn_index = 0; self.blocks.clear(); @@ -2615,9 +3038,11 @@ impl App { session_store::SystemItem::WorkerEvent { event, .. } => { self.blocks.push(Block::WorkerEvent { event }); } - session_store::SystemItem::FileAttachment { body, .. } + session_store::SystemItem::FeatureInvocationResult { body, .. } + | session_store::SystemItem::FileAttachment { body, .. } | session_store::SystemItem::SkillActivation { body, .. } | session_store::SystemItem::ResidentSummaryRefresh { body, .. } + | session_store::SystemItem::SubjectBehaviorRefresh { body, .. } | session_store::SystemItem::Interrupt { body, .. } => { self.task_store.apply_system_message_text(&body); self.blocks.push(Block::SystemMessage { text: body }); @@ -3179,7 +3604,7 @@ mod completion_flow_tests { app.insert_char('@'); let method = app.refresh_completion(); match method { - Some(Method::ListCompletions { kind, prefix }) => { + Some(Method::ListCompletions { kind, prefix, .. }) => { assert_eq!(kind, CompletionKind::File); assert_eq!(prefix, ""); } @@ -3196,7 +3621,7 @@ mod completion_flow_tests { app.insert_char('s'); let method = app.refresh_completion(); match method { - Some(Method::ListCompletions { kind, prefix }) => { + Some(Method::ListCompletions { kind, prefix, .. }) => { assert_eq!(kind, CompletionKind::File); assert_eq!(prefix, "s"); } @@ -3204,6 +3629,29 @@ mod completion_flow_tests { } } + #[test] + fn stale_completion_reply_for_prior_prefix_is_ignored() { + let mut app = App::new("test".into()); + for c in "@a".chars() { + app.insert_char(c); + } + let _ = app.refresh_completion(); + app.insert_char('b'); + let _ = app.refresh_completion(); + app.handle_worker_event(Event::Completions { + request_id: app.completion_request_id(), + kind: CompletionKind::File, + prefix: "a".into(), + context: None, + entries: vec![CompletionEntry { + value: "alpha".into(), + ..CompletionEntry::default() + }], + }); + assert_eq!(app.completion.as_ref().unwrap().prefix, "ab"); + assert!(app.completion.as_ref().unwrap().entries.is_empty()); + } + #[test] fn space_after_token_clears_completion_state() { let mut app = App::new("test".into()); @@ -3228,6 +3676,7 @@ mod completion_flow_tests { app.completion.as_mut().unwrap().entries = vec![CompletionEntry { value: "src/main.rs".into(), is_dir: false, + ..CompletionEntry::default() }]; // Tab path: text inserted, popup re-triggered with new prefix // (still File kind since the typed range stays after `@`). @@ -3250,6 +3699,7 @@ mod completion_flow_tests { app.completion.as_mut().unwrap().entries = vec![CompletionEntry { value: "crates".into(), is_dir: true, + ..CompletionEntry::default() }]; let _ = app.apply_completion_text(); // Typed prefix advances to `crates/` so the next query can @@ -3269,6 +3719,7 @@ mod completion_flow_tests { app.completion.as_mut().unwrap().entries = vec![CompletionEntry { value: "src/main.rs".into(), is_dir: false, + ..CompletionEntry::default() }]; assert!(app.chipify_completion_if_exact_match()); assert!(app.completion.is_none()); @@ -3287,6 +3738,7 @@ mod completion_flow_tests { app.completion.as_mut().unwrap().entries = vec![CompletionEntry { value: "src/main.rs".into(), is_dir: false, + ..CompletionEntry::default() }]; // typed = "s", expected = "src/main.rs" → no match, no chip. assert!(!app.chipify_completion_if_exact_match()); @@ -3307,6 +3759,7 @@ mod completion_flow_tests { app.completion.as_mut().unwrap().entries = vec![CompletionEntry { value: "crates".into(), is_dir: true, + ..CompletionEntry::default() }]; assert!(app.chipify_completion_if_exact_match()); let segs = app.input.submit_segments(); @@ -3321,6 +3774,7 @@ mod completion_flow_tests { app.completion.as_mut().unwrap().entries = vec![CompletionEntry { value: "crates".into(), is_dir: true, + ..CompletionEntry::default() }]; assert!(app.chipify_completion_if_exact_match()); let segs = app.input.submit_segments(); @@ -3342,10 +3796,12 @@ mod completion_flow_tests { CompletionEntry { value: "crates/client".into(), is_dir: true, + ..CompletionEntry::default() }, CompletionEntry { value: "crates/agen".into(), is_dir: true, + ..CompletionEntry::default() }, ]; assert!(app.chipify_completion_if_exact_match()); @@ -3366,6 +3822,7 @@ mod completion_flow_tests { app.completion.as_mut().unwrap().entries = vec![CompletionEntry { value: "crates".into(), is_dir: true, + ..CompletionEntry::default() }]; assert!(!app.chipify_selected_completion_if_committable()); // Popup is still active so the caller can fall through to @@ -3386,6 +3843,7 @@ mod completion_flow_tests { app.completion.as_mut().unwrap().entries = vec![CompletionEntry { value: "README.md".into(), is_dir: false, + ..CompletionEntry::default() }]; assert!(app.chipify_selected_completion_if_committable()); app.insert_char(' '); @@ -3408,6 +3866,7 @@ mod completion_flow_tests { app.completion.as_mut().unwrap().entries = vec![CompletionEntry { value: "README.md".into(), is_dir: false, + ..CompletionEntry::default() }]; assert!(app.chipify_selected_completion_if_committable()); assert!(app.completion.is_none()); @@ -3427,6 +3886,7 @@ mod completion_flow_tests { app.completion.as_mut().unwrap().entries = vec![CompletionEntry { value: "crates/client".into(), is_dir: true, + ..CompletionEntry::default() }]; assert!(!app.chipify_completion_if_exact_match()); let segs = app.input.submit_segments(); @@ -3447,10 +3907,12 @@ mod completion_flow_tests { CompletionEntry { value: "src/main.rs.bak".into(), is_dir: false, + ..CompletionEntry::default() }, CompletionEntry { value: "src/main.rs".into(), is_dir: false, + ..CompletionEntry::default() }, ]; // selected stays at 0 (the non-matching one) but find() should @@ -5100,3 +5562,312 @@ fn apply_cache_update( _ => {} } } + +#[cfg(test)] +mod completion_correlation_tests { + use super::*; + + fn reply(request: &Method, value: &str) -> Event { + let Method::ListCompletions { + kind, + prefix, + context, + request_id, + } = request + else { + panic!("completion query expected") + }; + assert!(request_id.as_ref().is_some_and(|id| !id.is_empty())); + Event::Completions { + kind: *kind, + prefix: prefix.clone(), + context: context.clone(), + request_id: request_id.clone(), + entries: vec![CompletionEntry { + value: value.into(), + ..Default::default() + }], + } + } + + fn request_id(request: &Method) -> &str { + let Method::ListCompletions { + request_id: Some(id), + .. + } = request + else { + panic!("correlated completion query expected") + }; + id + } + + fn authority_snapshot(scope: &str) -> Event { + Event::Snapshot { + session: protocol::SessionSnapshot { + entries: Vec::new(), + pending_submissions: Default::default(), + }, + greeting: protocol::Greeting { + worker_name: "test".into(), + cwd: "/tmp".into(), + provider: "test".into(), + model: "test".into(), + reasoning: None, + scope_summary: scope.into(), + tools: vec![], + context_window: 0, + context_tokens: 0, + context_usage: None, + }, + state: protocol::WorkerStateSnapshot::from(WorkerStatus::Idle), + in_flight: Default::default(), + internal_workers: vec![], + } + } + + #[test] + fn identical_file_prefix_aba_rejects_old_nonce_and_accepts_latest_query() { + let mut app = App::new("test".into()); + app.input.insert_str("@same"); + let first = app.refresh_completion().unwrap(); + assert!( + app.refresh_completion().is_none(), + "an unchanged request is not reissued" + ); + app.insert_char('x'); + let second = app.refresh_completion().unwrap(); + app.delete_char_before(); + let third = app.refresh_completion().unwrap(); + assert_ne!(request_id(&first), request_id(&second)); + assert_ne!(request_id(&first), request_id(&third)); + app.handle_worker_event(reply(&first, "old")); + assert!(app.completion.as_ref().unwrap().entries.is_empty()); + app.handle_worker_event(reply(&third, "current")); + app.handle_worker_event(reply(&first, "late old")); + assert_eq!(app.completion.as_ref().unwrap().entries[0].value, "current"); + } + + #[test] + fn identical_argument_context_aba_rejects_reply_before_and_after_refresh() { + let mut app = App::new("test".into()); + crate::invocation_tests::select(&mut app, crate::invocation_tests::descriptor(), "run"); + app.input.insert_str("\"資料/a"); + let first = app.refresh_completion().unwrap(); + app.input.insert_char('x'); + app.input.delete_before(); + // Prefix, context, token location, and cursor are identical; semantic revision is not. + app.handle_worker_event(reply(&first, "old")); + assert!(app.completion.as_ref().unwrap().entries.is_empty()); + let second = app.refresh_completion().unwrap(); + assert_ne!(request_id(&first), request_id(&second)); + app.handle_worker_event(reply(&first, "old")); + assert!(app.completion.as_ref().unwrap().entries.is_empty()); + app.handle_worker_event(reply(&second, "資料/current")); + assert_eq!( + app.completion.as_ref().unwrap().entries[0].value, + "資料/current" + ); + } + + #[test] + fn identical_cursor_aba_and_same_prefix_at_another_location_are_fenced() { + let mut app = App::new("test".into()); + app.input.insert_str("@same @same"); + let last_location = app.refresh_completion().unwrap(); + app.input.move_home(); + for _ in 0..5 { + app.input.move_right(); + } + let first_location = app.refresh_completion().unwrap(); + app.handle_worker_event(reply(&last_location, "wrong location")); + assert!(app.completion.as_ref().unwrap().entries.is_empty()); + app.input.move_end(); + let returned_location = app.refresh_completion().unwrap(); + app.handle_worker_event(reply(&last_location, "old same location")); + app.handle_worker_event(reply(&first_location, "wrong location")); + assert!(app.completion.as_ref().unwrap().entries.is_empty()); + assert_ne!(request_id(&last_location), request_id(&returned_location)); + app.input.move_left(); + app.input.move_right(); + app.handle_worker_event(reply(&returned_location, "cursor ABA before refresh")); + assert!(app.completion.as_ref().unwrap().entries.is_empty()); + let current = app.refresh_completion().unwrap(); + app.handle_worker_event(reply(¤t, "current")); + assert_eq!(app.completion.as_ref().unwrap().entries[0].value, "current"); + } + + #[test] + fn visible_candidates_cannot_be_inserted_after_unrefreshed_aba_edit() { + let mut app = App::new("test".into()); + app.input.insert_str("@same"); + let request = app.refresh_completion().unwrap(); + app.handle_worker_event(reply(&request, "candidate")); + app.input.move_left(); + app.input.move_right(); + assert!(app.apply_completion_text().is_none()); + assert_eq!(app.input.plain_text(), "@same"); + assert!(app.completion.is_none()); + } + + #[test] + fn cancelled_and_reopened_identical_feature_discovery_gets_new_nonce() { + let mut app = App::new("test".into()); + app.input.insert_str("/"); + let first = app.refresh_completion().unwrap(); + app.cancel_completion(); + let current = app.refresh_completion().unwrap(); + assert_ne!(request_id(&first), request_id(¤t)); + app.handle_worker_event(reply(&first, "old feature")); + assert!(app.completion.as_ref().unwrap().entries.is_empty()); + app.handle_worker_event(reply(¤t, "current feature")); + assert_eq!( + app.completion.as_ref().unwrap().entries[0].value, + "current feature" + ); + } + + #[test] + fn permission_snapshot_generation_aba_revokes_queries_even_when_scope_returns_to_original() { + let mut app = App::new("test".into()); + app.handle_worker_event(authority_snapshot("Writable: /tmp")); + crate::invocation_tests::select(&mut app, crate::invocation_tests::descriptor(), "run"); + app.input.insert_str("\"資料/a"); + let first = app.refresh_completion().unwrap(); + let denied = app + .handle_worker_event(authority_snapshot("Readable: /tmp")) + .unwrap(); + let current = app + .handle_worker_event(authority_snapshot("Writable: /tmp")) + .unwrap(); + assert_ne!(request_id(&first), request_id(&denied)); + assert_ne!(request_id(&first), request_id(¤t)); + app.handle_worker_event(reply(&first, "old permission generation")); + app.handle_worker_event(reply(&denied, "intermediate generation")); + assert!(app.completion.as_ref().unwrap().entries.is_empty()); + app.handle_worker_event(reply(¤t, "資料/current")); + assert_eq!( + app.completion.as_ref().unwrap().entries[0].value, + "資料/current" + ); + } + + #[test] + fn actual_worker_target_switch_aba_revokes_identical_queries() { + let mut app = App::new("test".into()); + app.set_completion_target("workspace/runtime/worker-a".into()); + app.input.insert_str("@same"); + let first = app.refresh_completion().unwrap(); + app.set_completion_target("workspace/runtime/worker-b".into()); + let other = app.refresh_completion().unwrap(); + app.set_completion_target("workspace/runtime/worker-a".into()); + let current = app.refresh_completion().unwrap(); + assert_ne!(request_id(&first), request_id(¤t)); + app.handle_worker_event(reply(&first, "worker-a old")); + app.handle_worker_event(reply(&other, "worker-b")); + assert!(app.completion.as_ref().unwrap().entries.is_empty()); + app.handle_worker_event(reply(¤t, "worker-a current")); + assert_eq!( + app.completion.as_ref().unwrap().entries[0].value, + "worker-a current" + ); + } + + #[test] + fn presentation_worker_view_switch_aba_revokes_queries_without_redirecting_parent_controls() { + let mut app = App::new("test".into()); + app.handle_worker_event(Event::InternalWorker { + worker: InternalWorkerRef { + session_id: "child".into(), + name: "child".into(), + parent_session_id: Some("parent".into()), + kind: protocol::InternalWorkerKind::SubWorker, + }, + revision: 1, + event: Box::new(Event::WorkerState { + snapshot: protocol::WorkerStateSnapshot::from(WorkerStatus::Idle), + }), + }); + app.input.insert_str("@same"); + let first = app.refresh_completion().unwrap(); + assert!(app.cycle_worker_view()); + let child_view = app.refresh_completion().unwrap(); + assert!(app.cycle_worker_view()); + let current = app.refresh_completion().unwrap(); + assert!(app.selected_internal_worker_session_id.is_none()); + app.handle_worker_event(reply(&first, "old parent")); + app.handle_worker_event(reply(&child_view, "old child view")); + assert!(app.completion.as_ref().unwrap().entries.is_empty()); + app.handle_worker_event(reply(¤t, "current parent")); + assert_eq!( + app.completion.as_ref().unwrap().entries[0].value, + "current parent" + ); + } + + #[test] + fn feature_chip_reedit_is_nonce_and_snapshot_correlated() { + let descriptor = crate::invocation_tests::descriptor(); + let invocation = protocol::parse_feature_invocation("/run(x)", 0, &descriptor, "stable-id") + .unwrap() + .invocation; + let mut app = App::new("test".into()); + app.input + .replace_with_segments(&[Segment::FeatureInvoke { invocation }]); + let first = app.edit_adjacent_feature_invocation().unwrap(); + app.input.move_home(); + app.input.move_end(); + let mut event = reply(&first, "run"); + if let Event::Completions { entries, .. } = &mut event { + entries[0].invocation = Some(descriptor.clone()); + } + app.handle_worker_event(event); + assert!(matches!( + app.input.submit_segments().as_slice(), + [Segment::FeatureInvoke { .. }] + )); + let current = app.edit_adjacent_feature_invocation().unwrap(); + assert_ne!(request_id(&first), request_id(¤t)); + let mut old = reply(&first, "run"); + if let Event::Completions { entries, .. } = &mut old { + entries[0].invocation = Some(descriptor.clone()); + } + app.handle_worker_event(old); + assert!(matches!( + app.input.submit_segments().as_slice(), + [Segment::FeatureInvoke { .. }] + )); + let mut event = reply(¤t, "run"); + if let Event::Completions { entries, .. } = &mut event { + entries[0].invocation = Some(descriptor); + } + app.handle_worker_event(event); + assert_eq!(app.input.plain_text(), "/run(path=\"x\")"); + app.delete_char_before(); // Reopen the quoted argument prefix. + let arguments = app.refresh_completion().unwrap(); + assert_ne!(request_id(¤t), request_id(&arguments)); + } + + #[test] + fn uncorrelated_legacy_reply_and_wrong_context_are_never_applied_to_correlated_queries() { + let mut app = App::new("test".into()); + app.input.insert_str("@same"); + let request = app.refresh_completion().unwrap(); + let mut missing = reply(&request, "missing nonce"); + if let Event::Completions { request_id, .. } = &mut missing { + *request_id = None; + } + app.handle_worker_event(missing); + assert!(app.completion.as_ref().unwrap().entries.is_empty()); + let mut mismatched = reply(&request, "wrong context"); + if let Event::Completions { context, .. } = &mut mismatched { + *context = Some(protocol::CompletionContext { + invocation: protocol::FeatureInvocationIdentity("plugin:other/run".into()), + argument: None, + }); + } + app.handle_worker_event(mismatched); + assert!(app.completion.as_ref().unwrap().entries.is_empty()); + app.handle_worker_event(reply(&request, "current")); + assert_eq!(app.completion.as_ref().unwrap().entries[0].value, "current"); + } +} diff --git a/crates/tui/src/backend_worker_picker.rs b/crates/tui/src/backend_worker_picker.rs index 962912b63..6821d1c19 100644 --- a/crates/tui/src/backend_worker_picker.rs +++ b/crates/tui/src/backend_worker_picker.rs @@ -24,12 +24,19 @@ use crate::inline_terminal::with_inline_terminal; const MAX_ROWS: usize = 10; const VIEWPORT_LINES: u16 = MAX_ROWS as u16 + 4; +#[derive(Debug)] +enum RestoreAttemptError { + Conflict, + Unknown(io::Error), +} + #[async_trait::async_trait] trait BackendWorkerLifecycle { async fn restore( &self, target: &client::BackendRuntimeTarget, - ) -> Result; + request: client::BackendWorkerRestoreRequest, + ) -> Result; async fn observe( &self, @@ -44,13 +51,19 @@ impl BackendWorkerLifecycle for LiveBackendWorkerLifecycle { async fn restore( &self, target: &client::BackendRuntimeTarget, - ) -> Result { - restore_backend_worker(target).await.map_err(|error| { - io::Error::other(format!( - "failed to restore Backend Worker {}: {error}", - target.display_label() - )) - }) + request: client::BackendWorkerRestoreRequest, + ) -> Result { + restore_backend_worker(target, request) + .await + .map_err(|error| { + if error.is_restore_observation_conflict() { + return RestoreAttemptError::Conflict; + } + RestoreAttemptError::Unknown(io::Error::other(format!( + "failed to restore Backend Worker {}: {error}", + target.display_label() + ))) + }) } async fn observe( @@ -133,8 +146,21 @@ pub(crate) async fn run( let attach_target = target .runtime_target(selected.runtime_id.clone(), selected.worker_id.clone()) .map_err(|error| io::Error::other(error.to_string()))?; - let attach_target = - prepare_selected_worker(&LiveBackendWorkerLifecycle, attach_target, intent).await?; + let restore_request = + selected + .restore_observation_token + .map(|token| client::BackendWorkerRestoreRequest { + expected_observation_token: token, + request_id: uuid::Uuid::now_v7().to_string(), + }); + let attach_target = prepare_selected_worker( + &LiveBackendWorkerLifecycle, + attach_target, + intent, + restore_request, + confirm_restore_recovery, + ) + .await?; return console::run_backend_runtime(attach_target).await; } } @@ -143,10 +169,62 @@ async fn prepare_selected_worker( lifecycle: &impl BackendWorkerLifecycle, mut target: client::BackendRuntimeTarget, intent: BackendWorkerPickerIntent, + restore_request: Option, + mut confirm_recovery: impl FnMut(&str) -> io::Result, ) -> Result { if intent == BackendWorkerPickerIntent::Resume { - let response = lifecycle.restore(&target).await?; - ensure_restore_accepted(&response)?; + let request = restore_request.ok_or_else(|| { + io::Error::other("Worker has no current Restore observation; refresh and select again") + })?; + loop { + let unknown = match lifecycle.restore(&target, request.clone()).await { + Ok(response) if response.result.state == BackendWorkerRestoreState::Accepted => { + break; + } + Ok(response) => { + let error = ensure_restore_accepted(&response).unwrap_err(); + if response.result.state != BackendWorkerRestoreState::ReconciliationRequired { + return Err(error); + } + error + } + Err(RestoreAttemptError::Conflict) => { + return Err(io::Error::other( + "Restore observation changed; refresh the Worker list and select again. No new Restore was started.", + )); + } + Err(RestoreAttemptError::Unknown(error)) => error, + }; + // Refresh is evidence only: GET cannot reconcile a pending Restore. + // A changed observation must never replace this admitted/unknown tuple. + let observation = match lifecycle.observe(&target).await { + Ok(observation) => match observation.observation { + WorkerSessionAvailability::LiveProtocol => "live_protocol".into(), + WorkerSessionAvailability::RetainedSnapshot { .. } => { + "retained_snapshot".into() + } + WorkerSessionAvailability::Unavailable { reason, message } => format!( + "{reason:?}: {}", + message.chars().take(512).collect::() + ), + }, + Err(error) => format!("refresh failed: {error}"), + }; + let detail = format!( + "Restore outcome unknown/pending: {}. Current observation: {}. Recovery request_id={}, expected_observation_token={}. Retry only this exact request; do not start a new Restore intent.", + unknown.to_string().chars().take(512).collect::(), + observation.chars().take(512).collect::(), + request.request_id, + request.expected_observation_token, + ); + // Every subsequent POST requires a new explicit user confirmation. + let retry = confirm_recovery(&detail).map_err(|error| { + io::Error::other(format!("{detail}; recovery prompt failed: {error}")) + })?; + if !retry { + return Err(io::Error::other(detail)); + } + } let observation = lifecycle.observe(&target).await?; require_live_resume_observation(observation.observation)?; return Ok(target); @@ -157,8 +235,34 @@ async fn prepare_selected_worker( Ok(target) } +fn confirm_restore_recovery(detail: &str) -> io::Result { + with_inline_terminal(8, |terminal| { + loop { + terminal.draw(|frame| { + let message = vec![ + Line::from("Restore needs explicit result recovery"), + Line::from("Enter: retry the SAME request Esc/Ctrl-C: cancel recovery"), + Line::from(detail), + ]; + frame.render_widget( + Paragraph::new(message).wrap(ratatui::widgets::Wrap { trim: false }), + frame.area(), + ); + })?; + match poll_event()? { + Some(Action::Submit) => return Ok(true), + Some(Action::Cancel) => return Ok(false), + _ => {} + } + } + }) +} + fn is_resume_candidate(worker: &BackendWorkerSummary) -> bool { - worker.worker_state.is_none() + worker.availability == protocol::subscription::SubscriptionWorkerAvailability::Observed + && worker.state == "stopped" + && worker.worker_state.is_none() + && worker.restore_observation_token.is_some() } fn ensure_restore_accepted(response: &BackendWorkerRestoreResponse) -> Result<(), io::Error> { @@ -526,6 +630,18 @@ fn working_directory_text(worker: &BackendWorkerSummary) -> String { BackendWorkingDirectorySource::Repository { repository_key } => { (repository_key.as_str(), String::new()) } + BackendWorkingDirectorySource::WorkspaceConfig { .. } => { + // Logical config access is not host filesystem or COMMAND authority. + let permissions = &attachment.effective_permissions; + let mut categories = Vec::new(); + if permissions.read { + categories.push("READ"); + } + if permissions.write { + categories.push("EDIT"); + } + ("workspace-config", format!("・{}", categories.join("・"))) + } BackendWorkingDirectorySource::ExternalGrant { .. } => { let permissions = &attachment.effective_permissions; let mut categories = Vec::new(); @@ -558,6 +674,7 @@ mod tests { fn worker(runtime_id: &str, worker_id: &str, profile: Option<&str>) -> BackendWorkerSummary { BackendWorkerSummary { + restore_observation_token: Some("observed-worker-generation".into()), runtime_id: runtime_id.to_string(), worker_id: worker_id.to_string(), resource_key: "W-1".to_string(), @@ -630,7 +747,8 @@ mod tests { async fn restore( &self, _target: &client::BackendRuntimeTarget, - ) -> Result { + _request: client::BackendWorkerRestoreRequest, + ) -> Result { self.calls.lock().unwrap().push("restore"); Ok(self.restore_response.clone()) } @@ -661,6 +779,312 @@ mod tests { serde_json::from_value(value).unwrap() } + // Exercise the existing picker flow against real Runtime admission, not a + // fake request -> operation-ID association. Only execution and transport are + // faked; Runtime owns the pending operation and receipt reconciliation. + #[derive(Default)] + struct PendingRuntimeBackend { + operations: std::sync::Mutex>, + } + + impl worker_runtime::execution::WorkerExecutionBackend for PendingRuntimeBackend { + fn backend_id(&self) -> &str { + "tui-recovery-test" + } + + fn spawn_worker( + &self, + _: worker_runtime::execution::WorkerExecutionSpawnRequest, + ) -> worker_runtime::execution::WorkerExecutionSpawnResult { + worker_runtime::execution::WorkerExecutionSpawnResult::connected( + protocol::WorkerStateSnapshot { + last_command_id: 0, + last_finished_submission_request_id: None, + state: protocol::WorkerState::Idle, + }, + Vec::new(), + ) + } + + fn restore_worker( + &self, + request: worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> worker_runtime::execution::WorkerExecutionSpawnResult { + use worker_runtime::execution::*; + let mut operations = self.operations.lock().unwrap(); + operations.push(request.operation_id); + if operations.len() == 1 { + WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + "unknown execution acknowledgement", + ), + worker_state: None, + workdir_attachments: Vec::new(), + } + } else { + WorkerExecutionSpawnResult::connected( + protocol::WorkerStateSnapshot { + last_command_id: 0, + last_finished_submission_request_id: None, + state: protocol::WorkerState::Idle, + }, + Vec::new(), + ) + } + } + + fn stop_worker( + &self, + _: &worker_runtime::identity::WorkerRef, + ) -> worker_runtime::execution::WorkerExecutionResult { + worker_runtime::execution::WorkerExecutionResult::accepted( + worker_runtime::execution::WorkerExecutionOperation::Stop, + ) + } + + fn dispatch_input( + &self, + _: &worker_runtime::identity::WorkerRef, + _: worker_runtime::interaction::WorkerInput, + ) -> worker_runtime::execution::WorkerExecutionResult { + panic!("Restore must not dispatch input") + } + } + + struct RuntimeRecoveryLifecycle { + runtime: worker_runtime::Runtime, + worker: worker_runtime::identity::WorkerRef, + lose_first_reply: bool, + requests: std::sync::Mutex< + Vec<( + client::BackendRuntimeTarget, + client::BackendWorkerRestoreRequest, + )>, + >, + observations: std::sync::Mutex, + } + + #[async_trait::async_trait] + impl BackendWorkerLifecycle for RuntimeRecoveryLifecycle { + async fn restore( + &self, + target: &client::BackendRuntimeTarget, + request: client::BackendWorkerRestoreRequest, + ) -> Result { + self.requests + .lock() + .unwrap() + .push((target.clone(), request.clone())); + assert_eq!(target.worker_id, self.worker.worker_id.to_string()); + let result = self + .runtime + .restore_worker_operation( + &self.worker, + serde_json::from_value(serde_json::to_value(request).unwrap()).unwrap(), + ) + .map_err(|error| match error { + worker_runtime::error::RuntimeError::RestoreObservationConflict { .. } => { + RestoreAttemptError::Conflict + } + error => RestoreAttemptError::Unknown(io::Error::other(error.to_string())), + })?; + if self.lose_first_reply && self.requests.lock().unwrap().len() == 1 { + return Err(RestoreAttemptError::Unknown(io::Error::new( + io::ErrorKind::ConnectionReset, + "reply lost after admission", + ))); + } + let mut response = restore_response(result.state); + response.runtime_id = target.runtime_id.clone(); + response.worker_id = target.worker_id.clone(); + Ok(response) + } + + async fn observe( + &self, + _: &client::BackendRuntimeTarget, + ) -> Result { + *self.observations.lock().unwrap() += 1; + Ok(session_observation( + serde_json::json!({"availability": "live_protocol"}), + )) + } + } + + fn pending_runtime_lifecycle( + lose_first_reply: bool, + ) -> ( + RuntimeRecoveryLifecycle, + std::sync::Arc, + client::BackendRuntimeTarget, + client::BackendWorkerRestoreRequest, + ) { + use worker_runtime::catalog::*; + let backend = std::sync::Arc::new(PendingRuntimeBackend::default()); + let runtime = + worker_runtime::Runtime::with_execution_backend(Default::default(), backend.clone()) + .unwrap(); + let archive = worker_runtime::profile_archive::ProfileSourceArchive::build( + worker_runtime::profile_archive::ProfileSourceArchiveInput { + id: "tui-recovery-profile".into(), + entrypoints: std::collections::BTreeMap::from([( + "builtin:coder".into(), + "coder.dcdl".into(), + )]), + imports: Default::default(), + sources: std::collections::BTreeMap::from([("coder.dcdl".into(), "{}".into())]), + }, + ) + .unwrap(); + let worker = runtime + .create_worker(CreateWorkerRequest { + worker_id: protocol::WorkerId::now_v7(), + create_fingerprint: "tui-recovery-create".into(), + profile: ProfileSelector::Builtin("builtin:coder".into()), + display_name: None, + profile_source: ProfileSourceArchiveSource::Embedded { archive }, + config_bundle: None, + initial_input: None, + workdir_attachment_requests: Vec::new(), + workdir_attachments: Vec::new(), + worker_observation_enabled: false, + worker_observation_grants: Vec::new(), + workspace_api: None, + memory_settings: Some(manifest::WorkspaceMemorySettingsSnapshot { + workspace_id: "local".into(), + settings_revision: 1, + language: "English".into(), + }), + subjektiv_attached: false, + }) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let request = client::BackendWorkerRestoreRequest { + expected_observation_token: runtime + .worker_detail(&worker.worker_ref) + .unwrap() + .restore_observation_token + .unwrap(), + request_id: "tui-original-request".into(), + }; + let target = client::BackendRuntimeTarget::new( + "http://127.0.0.1:3000", + "workspace-a", + "runtime-a", + worker.worker_id.to_string(), + ); + ( + RuntimeRecoveryLifecycle { + runtime, + worker: worker.worker_ref, + lose_first_reply, + requests: Default::default(), + observations: Default::default(), + }, + backend, + target, + request, + ) + } + + #[tokio::test] + async fn resume_explicit_transport_and_pending_retries_preserve_tuple_and_runtime_operation() { + for lost_reply in [true, false] { + let (lifecycle, backend, target, request) = pending_runtime_lifecycle(lost_reply); + let mut confirmations = 0; + let prepared = prepare_selected_worker( + &lifecycle, + target.clone(), + BackendWorkerPickerIntent::Resume, + Some(request.clone()), + |detail| { + confirmations += 1; + assert_eq!( + backend.operations.lock().unwrap().len(), + 1, + "no POST before explicit retry confirmation" + ); + assert!(detail.contains(&format!("request_id={}", request.request_id))); + assert!(detail.contains(&format!( + "expected_observation_token={}", + request.expected_observation_token + ))); + assert_ne!( + lifecycle + .runtime + .worker_detail(&lifecycle.worker) + .unwrap() + .restore_observation_token + .as_ref(), + Some(&request.expected_observation_token), + "fresh observation must not replace recovery tuple" + ); + Ok(true) + }, + ) + .await + .unwrap(); + assert_eq!(prepared, target); + assert_eq!(confirmations, 1); + assert_eq!( + lifecycle.requests.lock().unwrap().as_slice(), + &[(target.clone(), request.clone()), (target, request)] + ); + let operations = backend.operations.lock().unwrap(); + assert_eq!(operations.len(), 2); + assert_eq!( + operations[0], operations[1], + "Runtime pending lifecycle operation must not be replaced" + ); + assert_eq!( + *lifecycle.observations.lock().unwrap(), + 2, + "GET after uncertainty and after acceptance" + ); + } + } + + #[tokio::test] + async fn resume_cancelled_unknown_recovery_has_no_automatic_retry_and_reports_original_identity() + { + let (lifecycle, backend, target, request) = pending_runtime_lifecycle(true); + let error = prepare_selected_worker( + &lifecycle, + target, + BackendWorkerPickerIntent::Resume, + Some(request.clone()), + |_| Ok(false), + ) + .await + .unwrap_err() + .to_string(); + assert!(error.contains(&request.request_id)); + assert!(error.contains(&request.expected_observation_token)); + assert_eq!(backend.operations.lock().unwrap().len(), 1); + assert_eq!(lifecycle.requests.lock().unwrap().len(), 1); + assert_eq!(*lifecycle.observations.lock().unwrap(), 1); + } + + #[tokio::test] + async fn resume_stale_conflict_never_offers_same_intent_retry() { + let (lifecycle, backend, target, mut request) = pending_runtime_lifecycle(false); + request.expected_observation_token = "stale-token".into(); + let error = prepare_selected_worker( + &lifecycle, + target, + BackendWorkerPickerIntent::Resume, + Some(request), + |_| panic!("stale conflict must not offer unknown-result retry"), + ) + .await + .unwrap_err() + .to_string(); + assert!(error.contains("refresh the Worker list")); + assert!(backend.operations.lock().unwrap().is_empty()); + assert_eq!(*lifecycle.observations.lock().unwrap(), 0); + } + #[tokio::test] async fn resume_restores_before_requiring_live_observation() { let lifecycle = FakeBackendWorkerLifecycle { @@ -677,10 +1101,18 @@ mod tests { "worker-a", ); - let prepared = - prepare_selected_worker(&lifecycle, target, BackendWorkerPickerIntent::Resume) - .await - .unwrap(); + let prepared = prepare_selected_worker( + &lifecycle, + target, + BackendWorkerPickerIntent::Resume, + Some(client::BackendWorkerRestoreRequest { + expected_observation_token: "observed-worker-generation".into(), + request_id: "resume-operation".into(), + }), + |_| Ok(false), + ) + .await + .unwrap(); assert_eq!(*lifecycle.calls.lock().unwrap(), ["restore", "observe"]); assert!(prepared.initial_snapshot.is_none()); @@ -704,10 +1136,19 @@ mod tests { "worker-a", ); - let error = prepare_selected_worker(&lifecycle, target, BackendWorkerPickerIntent::Resume) - .await - .unwrap_err() - .to_string(); + let error = prepare_selected_worker( + &lifecycle, + target, + BackendWorkerPickerIntent::Resume, + Some(client::BackendWorkerRestoreRequest { + expected_observation_token: "observed-worker-generation".into(), + request_id: "resume-operation".into(), + }), + |_| Ok(false), + ) + .await + .unwrap_err() + .to_string(); assert!(error.contains("Rejected")); assert_eq!(*lifecycle.calls.lock().unwrap(), ["restore"]); @@ -724,7 +1165,7 @@ mod tests { assert!(!is_resume_candidate(&live)); assert!(is_resume_candidate(&stopped)); - assert!(is_resume_candidate(&unknown)); + assert!(!is_resume_candidate(&unknown)); } #[test] @@ -860,6 +1301,37 @@ mod tests { assert!(!text.contains("active clean")); } + #[test] + fn logical_config_attachment_labels_editability_without_command_or_host_path() { + for (access, write, expected) in [ + ("read_only", false, "READ"), + ("read_write", true, "READ・EDIT"), + ] { + let mut worker = worker("runtime-a", "worker-b", None); + worker.workdir_attachments = vec![serde_json::from_value(serde_json::json!({ + "alias": "workspace-config", + "effective_permissions": {"read": true, "write": write, "command": false}, + "working_directory": { + "working_directory_id": "logical-config-1", + "display_name": "Workspace configuration", + "source": {"kind": "workspace_config", "access": access, + "content_path": "/workspace-config", "purpose": "Workspace configuration"}, + "materializer_kind": "logical_workspace_config", + "status": "active" + } + })).unwrap()]; + let text = working_directory_text(&worker); + assert_eq!( + text, + format!( + "wd:workspace-config:Workspace configuration・{expected}・logical-config-1" + ) + ); + assert!(!text.contains("COMMAND")); + assert!(!text.contains("/workspace-config")); + } + } + #[test] fn worker_rows_align_identity_name_state_and_workdir_columns() { let mut short = worker("runtime-a", "worker-a", None); diff --git a/crates/tui/src/console/mod.rs b/crates/tui/src/console/mod.rs index 559139b39..03083575d 100644 --- a/crates/tui/src/console/mod.rs +++ b/crates/tui/src/console/mod.rs @@ -121,56 +121,23 @@ fn copy_selection_to_terminal(app: &mut App) -> bool { copy_selection_to_writer(app, &mut stdout) } -type AttachmentUploadResult = Result; - -fn mark_attachments_after_transport_acceptance( - pending: &mut Vec, - awaiting_acceptance: &mut Vec, -) { - awaiting_acceptance.append(pending); +struct AttachmentUploadResult { + id: String, + result: Result, } - -fn reconcile_attachment_submission( - pending: &mut Vec, - awaiting_acceptance: &mut Vec, - event: &Event, -) -> bool { - match event { - Event::UserMessage { segments, .. } => { - let accepted_ids = segments - .iter() - .filter_map(|segment| match segment { - Segment::UploadedFile { file } => Some(file.artifact_id.as_str()), - _ => None, - }) - .collect::>(); - let awaited_ids = awaiting_acceptance - .iter() - .map(|file| file.artifact_id.as_str()) - .collect::>(); - let accepts_exact_submission = !awaited_ids.is_empty() && accepted_ids == awaited_ids; - if accepts_exact_submission { - awaiting_acceptance.clear(); - } - accepts_exact_submission - } - Event::Error { .. } if !awaiting_acceptance.is_empty() => { - pending.append(awaiting_acceptance); - false - } - _ => false, - } +struct StagedUpload { + path: PathBuf, + task: Option>, + reference: Option, } - struct ConsoleConnection { client: Client, standalone_host: Option, backend_target: Option, - pending_attachments: Vec, - awaiting_attachment_acceptance: Vec, - upload_tasks: Vec>, + staged_uploads: HashMap, + typed_submissions: HashMap, + rejected_typed_submit: Option, active_uploads: usize, - upload_ids: HashMap, } async fn upload_client_path( @@ -206,7 +173,20 @@ async fn upload_client_path( "attachment file type is not supported", ) })?; - let bytes = tokio::fs::read(path).await?; + use tokio::io::AsyncReadExt; + let mut bytes = Vec::new(); + tokio::fs::File::open(path) + .await? + .take(10 * 1024 * 1024 + 1) + .read_to_end(&mut bytes) + .await?; + if bytes.len() > 10 * 1024 * 1024 { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "attachment exceeds the 10 MiB limit", + ) + .into()); + } target .upload_file_with_id(upload_id, file_name, media_type, bytes) .await @@ -238,11 +218,10 @@ impl ConsoleConnection { client, standalone_host: Some(host), backend_target: None, - pending_attachments: Vec::new(), - awaiting_attachment_acceptance: Vec::new(), - upload_tasks: Vec::new(), + staged_uploads: HashMap::new(), + typed_submissions: HashMap::new(), + rejected_typed_submit: None, active_uploads: 0, - upload_ids: HashMap::new(), } } @@ -251,11 +230,10 @@ impl ConsoleConnection { client, standalone_host: None, backend_target: Some(target), - pending_attachments: Vec::new(), - awaiting_attachment_acceptance: Vec::new(), - upload_tasks: Vec::new(), + staged_uploads: HashMap::new(), + typed_submissions: HashMap::new(), + rejected_typed_submit: None, active_uploads: 0, - upload_ids: HashMap::new(), } } @@ -268,27 +246,30 @@ impl ConsoleConnection { } async fn send(&mut self, method: &Method) -> Result<(), Box> { - let mut prepared = method.clone(); - let carries_attachments = - matches!(prepared, Method::Submit { .. }) && !self.pending_attachments.is_empty(); - if let Method::Submit { input, .. } = &mut prepared { - input.extend( - self.pending_attachments - .iter() - .cloned() - .map(|file| Segment::UploadedFile { file }), - ); - } - self.client.send(&prepared).await?; - if carries_attachments { - self.mark_attachments_awaiting_acceptance(); + // A failed write may still have delivered a complete request. Track ownership + // before transport I/O and fence deletion until authoritative admission. + if let Method::Submit { + submission_request_id, + input, + } = method + { + if input.iter().any(|segment| { + matches!( + segment, + Segment::UploadedFile { .. } | Segment::FeatureInvoke { .. } + ) + }) { + self.typed_submissions + .insert(submission_request_id.clone(), method.clone()); + } } + self.client.send(method).await?; Ok(()) } - fn start_upload( &mut self, path: PathBuf, + id: String, result_tx: mpsc::UnboundedSender, ) -> Result<(), Box> { let target = self.backend_target.clone().ok_or_else(|| { @@ -297,73 +278,173 @@ impl ConsoleConnection { "client-local file upload is available only for Backend Workers", ) })?; - self.upload_tasks.retain(|task| !task.is_finished()); - let upload_id = self - .upload_ids - .entry(path.clone()) - .or_insert_with(|| uuid::Uuid::now_v7().to_string()) - .clone(); - self.active_uploads = self.active_uploads.saturating_add(1); - self.upload_tasks.push(tokio::spawn(async move { - let result = upload_client_path(&target, &path, &upload_id) + if self + .staged_uploads + .get(&id) + .is_some_and(|upload| upload.task.is_some()) + { + return Ok(()); + } + self.active_uploads += 1; + let upload_id = id.clone(); + let upload_path = path.clone(); + let task = tokio::spawn(async move { + let result = upload_client_path(&target, &upload_path, &upload_id) .await .map_err(|error| error.to_string()); - let _ = result_tx.send(result); - })); + let _ = result_tx.send(AttachmentUploadResult { + id: upload_id, + result, + }); + }); + self.staged_uploads.insert( + id, + StagedUpload { + path, + task: Some(task), + reference: None, + }, + ); Ok(()) } - - fn finish_upload(&mut self) { - self.active_uploads = self.active_uploads.saturating_sub(1); - self.upload_tasks.retain(|task| !task.is_finished()); - } - fn has_active_uploads(&self) -> bool { self.active_uploads != 0 } - fn observe_worker_event(&mut self, event: &Event) -> bool { - let attachments_accepted = reconcile_attachment_submission( - &mut self.pending_attachments, - &mut self.awaiting_attachment_acceptance, - event, - ); - if attachments_accepted { - self.upload_ids.clear(); + match event { + Event::SubmissionAccepted { + submission_request_id, + .. + } => { + if let Some(Method::Submit { input, .. }) = + self.typed_submissions.remove(submission_request_id) + { + self.release_accepted_attachments(&input); + return true; + } + } + Event::SubmissionRejected { + submission_request_id, + .. + } => { + self.rejected_typed_submit = self.typed_submissions.remove(submission_request_id); + } + Event::UserMessage { segments, .. } => { + self.release_accepted_attachments(segments); + self.typed_submissions.retain(|_, method| !matches!(method, Method::Submit { input, .. } if input == segments)); + } + _ => {} } - attachments_accepted + false } - - fn mark_attachments_awaiting_acceptance(&mut self) { - mark_attachments_after_transport_acceptance( - &mut self.pending_attachments, - &mut self.awaiting_attachment_acceptance, - ); + fn release_accepted_attachments(&mut self, segments: &[Segment]) { + let ids = segments + .iter() + .filter_map(|segment| match segment { + Segment::UploadedFile { file } => Some(file.artifact_id.as_str()), + _ => None, + }) + .collect::>(); + self.staged_uploads.retain(|_, upload| { + !upload + .reference + .as_ref() + .is_some_and(|reference| ids.contains(&reference.artifact_id.as_str())) + }); } - - async fn clear_pending_attachments(&mut self) { - for task in self.upload_tasks.drain(..) { + fn attachment_is_submitting(&self, artifact_id: &str) -> bool { + self.typed_submissions.values().any(|method| matches!(method, Method::Submit { input, .. } + if input.iter().any(|segment| matches!(segment, Segment::UploadedFile { file } if file.artifact_id == artifact_id)))) + } + async fn delete_draft_attachment(&mut self, reference: &UploadedFileRef) { + if self.attachment_is_submitting(&reference.artifact_id) { + return; + } + let id = self.staged_uploads.iter().find_map(|(id, upload)| { + upload + .reference + .as_ref() + .filter(|owned| owned.artifact_id == reference.artifact_id) + .map(|_| id.clone()) + }); + // History/accepted/restored files are not this connection's unsent staging. + if let Some(id) = id { + self.cancel_stage(&id).await; + } + } + async fn cancel_stage(&mut self, id: &str) { + let Some(upload) = self.staged_uploads.remove(id) else { + return; + }; + if let Some(task) = upload.task { + self.active_uploads = self.active_uploads.saturating_sub(1); task.abort(); } - self.active_uploads = 0; - let upload_ids = std::mem::take(&mut self.upload_ids) - .into_values() - .collect::>(); - let references = std::mem::take(&mut self.pending_attachments); if let Some(target) = &self.backend_target { - for upload_id in upload_ids { - let _ = target.cancel_file_upload(&upload_id).await; - } - for reference in references { + let _ = target.cancel_file_upload(id).await; + if let Some(reference) = upload.reference { let _ = target.delete_uploaded_file(&reference.artifact_id).await; } } } - + async fn finish_upload(&mut self, app: &mut App, result: AttachmentUploadResult) { + let Some(upload) = self.staged_uploads.get_mut(&result.id) else { + if let (Some(target), Ok(reference)) = (&self.backend_target, result.result) { + if reference.source_entry_id.is_none() { + let _ = target.delete_uploaded_file(&reference.artifact_id).await; + } + } + return; + }; + if upload.task.take().is_some() { + self.active_uploads = self.active_uploads.saturating_sub(1); + } + match result.result { + Ok(reference) => { + if reference.source_entry_id.is_some() { + // An idempotent retry may return an already committed resource. + self.staged_uploads.remove(&result.id); + app.input + .finish_attachment_stage(&result.id, Some(reference)); + return; + } + upload.reference = Some(reference.clone()); + if !app + .input + .finish_attachment_stage(&result.id, Some(reference)) + { + self.cancel_stage(&result.id).await; + } else { + app.completion = None; + app.clear_actionbar_notice(); + } + } + Err(error) => { + if !app.input.finish_attachment_stage(&result.id, None) { + self.cancel_stage(&result.id).await; + } else { + app.push_error(format!( + "Attachment upload failed: {error}. Alt+Enter on the failed chip retries." + )); + } + } + } + } async fn shutdown(&mut self) -> Result<(), Box> { - self.pending_attachments - .append(&mut self.awaiting_attachment_acceptance); - self.clear_pending_attachments().await; + let ids = + self.staged_uploads + .iter() + .filter(|(_, upload)| { + !upload.reference.as_ref().is_some_and(|reference| { + self.attachment_is_submitting(&reference.artifact_id) + }) + }) + .map(|(id, _)| id.clone()) + .collect::>(); + for id in ids { + self.cancel_stage(&id).await; + } + // Disconnect leaves in-flight acceptance ambiguous; never delete those resources. if let Some(host) = self.standalone_host.take() { host.shutdown().await?; } @@ -478,6 +559,15 @@ async fn run_backend_console( let mut terminal = enter_fullscreen()?; let workspace_root = std::env::current_dir().unwrap_or_else(|_| std::path::PathBuf::from(".")); let mut app = App::new_with_persistent_input_history(worker_label, &workspace_root); + app.set_completion_target( + serde_json::to_string(&( + &target.base_url, + &target.workspace_id, + &target.runtime_id, + &target.worker_id, + )) + .expect("serializing a Backend target cannot fail"), + ); app.connected = connected; let mut connection = ConsoleConnection::with_backend_target(client, target); let result = run_loop(&mut terminal, &mut app, &mut connection).await; @@ -852,7 +942,7 @@ where })) } upload = upload_rx.recv() => { - LoopInput::Upload(upload.unwrap_or_else(|| Err("attachment upload queue stopped".into()))) + LoopInput::Upload(upload.unwrap_or_else(|| AttachmentUploadResult { id: String::new(), result: Err("attachment upload queue stopped".into()) })) } event = pod_next, if connected => LoopInput::Worker(event), _ = animation_tick, if animate => LoopInput::Tick, @@ -950,40 +1040,21 @@ async fn run_loop( LoopInput::Terminal(term_event) => { handle_terminal_event(app, client, &upload_tx, term_event?).await?; } - LoopInput::Upload(result) => { - client.finish_upload(); - match result { - Ok(reference) => { - app.flash_actionbar_notice( - format!( - "[{} · {} bytes · ready] Send a message or use /clear-attachments.", - reference.file_name, reference.byte_len - ), - ActionbarNoticeLevel::Info, - ActionbarNoticeSource::Tui, - Duration::from_secs(60 * 60), - ); - if !client - .pending_attachments - .iter() - .any(|pending| pending.artifact_id == reference.artifact_id) - { - client.pending_attachments.push(reference); - } - } - Err(error) => app.push_error(format!("Attachment upload failed: {error}")), - } - } + LoopInput::Upload(result) => client.finish_upload(app, result).await, LoopInput::Worker(event) => match event? { Some(ev) => { if client.observe_worker_event(&ev) { app.clear_actionbar_notice(); } + if let Some(method) = client.rejected_typed_submit.take() { + app.restore_unsent_run(&method); + } if let Some(method) = app.handle_worker_event(ev) { send_console_method(app, client, &method).await?; } } None => { + app.cancel_completion(); app.connected = false; app.mark_orphan_compacts_incomplete(); app.push_error("Connection lost"); @@ -998,25 +1069,48 @@ async fn run_loop( Ok(()) } -fn attachment_command_path(method: &Method) -> Option { - let Method::Submit { input, .. } = method else { - return None; +async fn client_file_completions(prefix: &str) -> Vec { + let (directory, leaf, displayed) = match prefix.rsplit_once('/') { + Some((directory, leaf)) => ( + if directory.is_empty() { "/" } else { directory }, + leaf, + format!("{directory}/"), + ), + None => (".", prefix, String::new()), }; - let [Segment::Text { content }] = input.as_slice() else { - return None; + let Ok(mut reader) = tokio::fs::read_dir(directory).await else { + return Vec::new(); }; - let path = content.strip_prefix("/attach ")?.trim(); - (!path.is_empty()).then(|| PathBuf::from(path)) -} - -fn is_clear_attachments_command(method: &Method) -> bool { - let Method::Submit { input, .. } = method else { - return false; - }; - matches!( - input.as_slice(), - [Segment::Text { content }] if content.trim() == "/clear-attachments" - ) + let mut entries = Vec::new(); + // Keep directory discovery bounded and local; never send a client path to Worker. + let mut visited = 0; + while let Ok(Some(entry)) = reader.next_entry().await { + visited += 1; + if visited > 4096 { + break; + } + let Some(name) = entry.file_name().to_str().map(str::to_owned) else { + continue; + }; + if !name.starts_with(leaf) { + continue; + } + let is_dir = entry + .metadata() + .await + .is_ok_and(|metadata| metadata.is_dir()); + if !is_dir && attachment_media_type(&entry.path()).is_none() { + continue; + } + entries.push(protocol::CompletionEntry { + value: format!("{displayed}{name}"), + is_dir, + ..Default::default() + }); + } + entries.sort_by(|a, b| a.value.cmp(&b.value)); + entries.truncate(100); + entries } async fn send_console_method( @@ -1024,10 +1118,29 @@ async fn send_console_method( client: &mut ConsoleConnection, method: &Method, ) -> Result<(), Box> { + if let Method::ListCompletions { + kind: protocol::CompletionKind::FeatureArgument, + prefix, + context: Some(context), + request_id, + } = method + { + if app.is_client_file_completion(context) { + let entries = client_file_completions(prefix).await; + app.handle_worker_event(Event::Completions { + request_id: request_id.clone(), + kind: protocol::CompletionKind::FeatureArgument, + prefix: prefix.clone(), + context: Some(context.clone()), + entries, + }); + return Ok(()); + } + } if matches!(method, Method::Submit { .. }) && client.has_active_uploads() { app.restore_unsent_run(method); app.flash_actionbar_notice( - "Attachment upload is still in progress; wait or use /clear-attachments.", + "Attachment upload is still in progress; wait for its composer chip.", ActionbarNoticeLevel::Info, ActionbarNoticeSource::Tui, Duration::from_secs(60 * 60), @@ -1035,16 +1148,21 @@ async fn send_console_method( return Ok(()); } - let sends_attachments = - matches!(method, Method::Submit { .. }) && !client.pending_attachments.is_empty(); + let sends_attachments = matches!( + method, + Method::Submit { input, .. } + if input.iter().any(|segment| matches!(segment, Segment::UploadedFile { .. })) + ); if let Err(error) = client.send(method).await { - if sends_attachments { + if sends_attachments + || matches!(method, Method::Submit { input, .. } if input.iter().any(|segment| matches!(segment, Segment::FeatureInvoke { .. }))) + { app.restore_unsent_run(method); app.push_error(format!( - "Attachment submission failed: {error}. Pending attachments were retained; retry Send." + "Typed input submission failed: {error}. The composer was restored; retry Send." )); app.flash_actionbar_notice( - "Attachment submission failed; pending attachments are ready to retry.", + "Typed input submission failed; composer chips are ready to retry.", ActionbarNoticeLevel::Error, ActionbarNoticeSource::Tui, Duration::from_secs(60 * 60), @@ -1072,31 +1190,57 @@ async fn handle_terminal_event( ) -> Result<(), Box> { match event { TermEvent::Key(key) => { + if key.code == KeyCode::Enter + && key.modifiers == KeyModifiers::ALT + && key.kind == crossterm::event::KeyEventKind::Press + { + if let Some(id) = app.input.adjacent_failed_attachment() { + if let Some(upload) = client.staged_uploads.get(&id) { + let path = upload.path.clone(); + client.start_upload(path, id.clone(), upload_tx.clone())?; + app.input.retry_attachment_stage(&id); + } + return Ok(()); + } + } if let Some(method) = handle_key(app, key) { - if let Some(path) = attachment_command_path(&method) { - match client.start_upload(path, upload_tx.clone()) { - Ok(()) => app.flash_actionbar_notice( - "Uploading attachment… Use /clear-attachments to cancel.", + send_console_method(app, client, &method).await?; + } + for (invocation_id, path) in app.attachment_invocations() { + let id = uuid::Uuid::now_v7().to_string(); + let file_name = path + .file_name() + .map(|name| name.to_string_lossy().into_owned()) + .unwrap_or_default(); + match client.start_upload(path, id.clone(), upload_tx.clone()) { + Ok(()) => { + app.input.stage_attachment(&invocation_id, id, file_name); + app.flash_actionbar_notice( + "Uploading attachment; delete its chip to cancel.", ActionbarNoticeLevel::Info, ActionbarNoticeSource::Tui, Duration::from_secs(30), - ), - Err(error) => { - app.push_error(format!("Attachment upload failed: {error}")); - } + ); } - } else if is_clear_attachments_command(&method) { - client.clear_pending_attachments().await; - app.flash_actionbar_notice( - "Removed pending attachments.", - ActionbarNoticeLevel::Info, - ActionbarNoticeSource::Tui, - Duration::from_secs(4), - ); - } else { - send_console_method(app, client, &method).await?; + Err(error) => app.push_error(format!("Attachment upload failed: {error}")), } } + let visible_stages = app.input.attachment_stage_ids(); + let orphaned = client + .staged_uploads + .iter() + .filter(|(id, upload)| upload.reference.is_none() && !visible_stages.contains(id)) + .map(|(id, _)| id.clone()) + .collect::>(); + for id in orphaned { + client.cancel_stage(&id).await; + } + for id in app.input.take_removed_attachment_stages() { + client.cancel_stage(&id).await; + } + for reference in app.input.take_removed_uploaded_files() { + client.delete_draft_attachment(&reference).await; + } } TermEvent::Mouse(mouse) => { handle_mouse(app, mouse); @@ -1171,6 +1315,18 @@ fn apply_composer_edit_action(app: &mut App, action: ComposerEditAction) -> Opti } fn handle_key(app: &mut App, key: KeyEvent) -> Option { + if key.kind == crossterm::event::KeyEventKind::Release + || (key.code == KeyCode::Enter && key.kind == crossterm::event::KeyEventKind::Repeat) + { + return None; + } + if key.code == KeyCode::Enter + && key.modifiers == KeyModifiers::ALT + && !app.is_command_mode() + && app.input.adjacent_feature_invocation().is_some() + { + return app.edit_adjacent_feature_invocation(); + } let ctrl = key.modifiers.contains(KeyModifiers::CONTROL); let shift = key.modifiers.contains(KeyModifiers::SHIFT); let alt = key.modifiers.contains(KeyModifiers::ALT); @@ -1582,23 +1738,387 @@ mod tests { } #[test] - fn client_local_attachment_commands_are_typed_and_do_not_send_the_path() { - let attach = Method::Submit { - submission_request_id: protocol::new_submission_request_id(), - input: vec![Segment::text("/attach /tmp/report.md")], - }; - assert_eq!( - attachment_command_path(&attach), - Some(PathBuf::from("/tmp/report.md")) + fn feature_invocation_alt_enter_reopens_chip_but_preserves_plain_newline_and_key_release_behavior() + { + use crate::invocation_tests::descriptor; + let invocation = protocol::parse_feature_invocation("/run(x)", 0, &descriptor(), "id") + .unwrap() + .invocation; + let mut app = App::new("test".into()); + app.input + .replace_with_segments(&[Segment::FeatureInvoke { invocation }]); + assert!(matches!( + handle_key(&mut app, KeyEvent::new(KeyCode::Enter, KeyModifiers::ALT)), + Some(Method::ListCompletions { + kind: protocol::CompletionKind::Feature, + .. + }) + )); + app.handle_worker_event(Event::Completions { + request_id: app.completion_request_id(), + kind: protocol::CompletionKind::Feature, + prefix: "run".into(), + context: None, + entries: vec![protocol::CompletionEntry { + value: "run".into(), + invocation: Some(descriptor()), + ..Default::default() + }], + }); + assert_eq!(app.input.plain_text(), "/run(path=\"x\")"); + let mut plain = App::new("test".into()); + plain.input.insert_str("committed 日本語"); + handle_key(&mut plain, KeyEvent::new(KeyCode::Enter, KeyModifiers::ALT)); + assert_eq!(plain.input.plain_text(), "committed 日本語\n"); + let before = plain.input.submit_segments(); + handle_key( + &mut plain, + KeyEvent { + code: KeyCode::Enter, + modifiers: KeyModifiers::NONE, + kind: crossterm::event::KeyEventKind::Release, + state: crossterm::event::KeyEventState::NONE, + }, ); - assert!(!is_clear_attachments_command(&attach)); + assert_eq!(plain.input.submit_segments(), before); + } - let clear = Method::Submit { - submission_request_id: protocol::new_submission_request_id(), - input: vec![Segment::text("/clear-attachments")], + fn attachment_test_connection() -> ConsoleConnection { + ConsoleConnection { + client: Client::new(FailOnceSocket { + fail_next_send: false, + }), + standalone_host: None, + backend_target: None, + staged_uploads: HashMap::new(), + typed_submissions: HashMap::new(), + rejected_typed_submit: None, + active_uploads: 0, + } + } + + #[tokio::test] + async fn attachment_staging_completion_replaces_its_chip_not_the_current_cursor() { + use crate::invocation_tests::{descriptor, uploaded_file}; + let mut app = App::new("test".into()); + let invocation = protocol::parse_feature_invocation("/run(x)", 0, &descriptor(), "invoke") + .unwrap() + .invocation; + app.input.replace_with_segments(&[ + Segment::FeatureInvoke { invocation }, + Segment::text(" following prose"), + ]); + app.input + .stage_attachment("invoke", "stage".into(), "a.txt".into()); + app.input.move_home(); + app.input.insert_str("prefix "); + let mut connection = attachment_test_connection(); + connection.staged_uploads.insert( + "stage".into(), + StagedUpload { + path: "a.txt".into(), + task: None, + reference: None, + }, + ); + connection + .finish_upload( + &mut app, + AttachmentUploadResult { + id: "stage".into(), + result: Ok(uploaded_file()), + }, + ) + .await; + assert_eq!( + app.input.submit_segments(), + vec![ + Segment::text("prefix "), + Segment::UploadedFile { + file: uploaded_file() + }, + Segment::text(" following prose") + ] + ); + app.input.insert_str("cursor remains "); + assert!(app.input.plain_text().starts_with("prefix cursor remains ")); + } + + #[tokio::test] + async fn attachment_removed_staging_and_failed_retry_keep_exact_resource_ownership() { + use crate::invocation_tests::{descriptor, uploaded_file}; + let mut app = App::new("test".into()); + let invocation = protocol::parse_feature_invocation("/run(x)", 0, &descriptor(), "invoke") + .unwrap() + .invocation; + app.input + .replace_with_segments(&[Segment::FeatureInvoke { invocation }]); + app.input + .stage_attachment("invoke", "stage".into(), "a.txt".into()); + let mut connection = attachment_test_connection(); + connection.staged_uploads.insert( + "stage".into(), + StagedUpload { + path: "a.txt".into(), + task: None, + reference: None, + }, + ); + connection + .finish_upload( + &mut app, + AttachmentUploadResult { + id: "stage".into(), + result: Err("temporary".into()), + }, + ) + .await; + assert_eq!( + app.input.adjacent_failed_attachment().as_deref(), + Some("stage") + ); + assert_eq!( + connection.staged_uploads["stage"].path, + PathBuf::from("a.txt") + ); + app.input.retry_attachment_stage("stage"); + assert!(app.input.adjacent_failed_attachment().is_none()); + app.input.delete_word_before(); + for id in app.input.take_removed_attachment_stages() { + connection.cancel_stage(&id).await; + } + assert!(connection.staged_uploads.is_empty()); + connection + .finish_upload( + &mut app, + AttachmentUploadResult { + id: "stage".into(), + result: Ok(uploaded_file()), + }, + ) + .await; + assert!(app.input.is_empty()); + assert!(connection.staged_uploads.is_empty()); + } + + #[tokio::test] + async fn attachment_acceptance_transfers_ownership_before_queue_execution_and_history_delete() { + use crate::invocation_tests::uploaded_file; + let mut connection = attachment_test_connection(); + let reference = uploaded_file(); + connection.staged_uploads.insert( + "stage".into(), + StagedUpload { + path: "a.txt".into(), + task: None, + reference: Some(reference.clone()), + }, + ); + let method = Method::Submit { + submission_request_id: "request".into(), + input: vec![Segment::UploadedFile { + file: reference.clone(), + }], }; - assert!(is_clear_attachments_command(&clear)); - assert_eq!(attachment_command_path(&clear), None); + connection.send(&method).await.unwrap(); + connection.delete_draft_attachment(&reference).await; + assert_eq!( + connection.staged_uploads.len(), + 1, + "in-flight resources must be protected" + ); + connection.observe_worker_event(&Event::Error { + code: protocol::ErrorCode::ProviderError, + message: "unrelated provider error".into(), + }); + assert_eq!(connection.typed_submissions.len(), 1); + assert!( + !connection.observe_worker_event(&Event::SubmissionAccepted { + submission_request_id: "other".into(), + submission_id: "other-submission".into(), + disposition: protocol::SubmissionDisposition::Queued + }) + ); + assert!(connection.observe_worker_event(&Event::SubmissionAccepted { + submission_request_id: "request".into(), + submission_id: "submission".into(), + disposition: protocol::SubmissionDisposition::Queued + })); + assert!(connection.staged_uploads.is_empty()); + connection.delete_draft_attachment(&reference).await; + connection.shutdown().await.unwrap(); + assert!( + connection.staged_uploads.is_empty(), + "accepted resources never reenter unsent ownership" + ); + } + + #[tokio::test] + async fn attachment_rejection_is_request_scoped_and_restores_typed_retry() { + use crate::invocation_tests::uploaded_file; + let mut connection = attachment_test_connection(); + let reference = uploaded_file(); + connection.staged_uploads.insert( + "stage".into(), + StagedUpload { + path: "a.txt".into(), + task: None, + reference: Some(reference.clone()), + }, + ); + let first = Method::Submit { + submission_request_id: "first".into(), + input: vec![Segment::UploadedFile { + file: reference.clone(), + }], + }; + let second = Method::Submit { + submission_request_id: "second".into(), + input: vec![Segment::UploadedFile { + file: UploadedFileRef { + artifact_id: "second-artifact".into(), + ..reference.clone() + }, + }], + }; + connection.send(&first).await.unwrap(); + connection.send(&second).await.unwrap(); + connection.observe_worker_event(&Event::SubmissionRejected { + submission_request_id: "first".into(), + message: "rejected".into(), + }); + assert_eq!(connection.typed_submissions.len(), 1); + assert!(connection.typed_submissions.contains_key("second")); + let mut app = App::new("test".into()); + app.restore_unsent_run(&connection.rejected_typed_submit.take().unwrap()); + assert_eq!( + app.input.submit_segments(), + vec![Segment::UploadedFile { + file: reference.clone() + }] + ); + connection.delete_draft_attachment(&reference).await; + assert!(connection.staged_uploads.is_empty()); + } + + #[tokio::test] + async fn attachment_retry_returning_committed_resource_never_acquires_delete_ownership() { + use crate::invocation_tests::{descriptor, uploaded_file}; + let invocation = protocol::parse_feature_invocation("/run(x)", 0, &descriptor(), "invoke") + .unwrap() + .invocation; + let mut app = App::new("test".into()); + app.input + .replace_with_segments(&[Segment::FeatureInvoke { invocation }]); + app.input + .stage_attachment("invoke", "stage".into(), "a.txt".into()); + let mut connection = attachment_test_connection(); + connection.staged_uploads.insert( + "stage".into(), + StagedUpload { + path: "a.txt".into(), + task: None, + reference: None, + }, + ); + let file = UploadedFileRef { + source_entry_id: Some("committed-entry".into()), + ..uploaded_file() + }; + connection + .finish_upload( + &mut app, + AttachmentUploadResult { + id: "stage".into(), + result: Ok(file.clone()), + }, + ) + .await; + assert_eq!( + app.input.submit_segments(), + vec![Segment::UploadedFile { file: file.clone() }] + ); + assert!(connection.staged_uploads.is_empty()); + app.input.delete_before(); + connection.delete_draft_attachment(&file).await; + assert!(connection.staged_uploads.is_empty()); + } + + #[tokio::test] + async fn attachment_local_completion_aba_echoes_nonce_and_ignores_prior_request() { + use crate::invocation_tests::{attachment_descriptor, select}; + let temp = tempfile::tempdir().unwrap(); + std::fs::write(temp.path().join("資料 a.txt"), "data").unwrap(); + let mut app = App::new("test".into()); + select(&mut app, attachment_descriptor(), "add-file"); + let prefix = format!("{}/", temp.path().display()); + app.input + .insert_str(protocol::quote_invocation_string(&prefix).trim_end_matches('"')); + let old = app.refresh_completion().unwrap(); + app.insert_char('x'); + app.delete_char_before(); + let current = app.refresh_completion().unwrap(); + let mut connection = attachment_test_connection(); + connection.client = Client::new(FailOnceSocket { + fail_next_send: true, + }); + send_console_method(&mut app, &mut connection, &old) + .await + .unwrap(); + assert!(app.completion.as_ref().unwrap().entries.is_empty()); + send_console_method(&mut app, &mut connection, ¤t) + .await + .unwrap(); + assert_eq!(app.completion.as_ref().unwrap().entries.len(), 1); + assert!( + app.completion.as_ref().unwrap().entries[0] + .value + .ends_with("資料 a.txt") + ); + } + + #[tokio::test] + async fn attachment_client_file_completion_is_local_bounded_and_safe_for_unicode_paths() { + use crate::invocation_tests::{attachment_descriptor, select}; + let temp = tempfile::tempdir().unwrap(); + std::fs::write(temp.path().join("資料 a.txt"), "data").unwrap(); + std::fs::write(temp.path().join("unsafe.exe"), "data").unwrap(); + std::fs::create_dir(temp.path().join("nested dir")).unwrap(); + let mut app = App::new("test".into()); + select(&mut app, attachment_descriptor(), "add-file"); + let prefix = format!("{}/", temp.path().display()); + app.input + .insert_str(&protocol::quote_invocation_string(&prefix).trim_end_matches('"')); + let method = app.refresh_completion().unwrap(); + let mut connection = attachment_test_connection(); + // If completion is accidentally sent to Worker this deliberately failing transport fails the test. + connection.client = Client::new(FailOnceSocket { + fail_next_send: true, + }); + send_console_method(&mut app, &mut connection, &method) + .await + .unwrap(); + let entries = &app.completion.as_ref().unwrap().entries; + assert_eq!(entries.len(), 2); + assert!( + entries + .iter() + .any(|entry| entry.value.ends_with("資料 a.txt")) + ); + assert!( + entries + .iter() + .any(|entry| entry.is_dir && entry.value.ends_with("nested dir")) + ); + assert!( + !entries + .iter() + .any(|entry| entry.value.ends_with("unsafe.exe")) + ); + } + + #[test] + fn attachment_media_types_remain_restricted() { assert_eq!( attachment_media_type(Path::new("report.webp")), Some("image/webp") @@ -1652,15 +2172,15 @@ mod tests { }), standalone_host: None, backend_target: None, - pending_attachments: vec![file.clone()], - awaiting_attachment_acceptance: Vec::new(), - upload_tasks: Vec::new(), + staged_uploads: HashMap::new(), + typed_submissions: HashMap::new(), + rejected_typed_submit: None, active_uploads: 0, - upload_ids: HashMap::new(), }; let mut app = App::new("worker".into()); app.input.insert_str("inspect"); + app.input.insert_uploaded_file(file.clone()); connection.active_uploads = 1; let blocked = app.submit_input().unwrap(); @@ -1668,36 +2188,41 @@ mod tests { send_console_method(&mut app, &mut connection, &blocked) .await .unwrap(); - assert_eq!(app.input.plain_text(), "inspect"); - assert_eq!(connection.pending_attachments, vec![file.clone()]); - assert!(connection.awaiting_attachment_acceptance.is_empty()); + assert!(matches!(app.input.submit_segments().as_slice(), + [Segment::Text { content }, Segment::UploadedFile { file: restored }] + if content == "inspect" && restored == &file)); + assert!(connection.staged_uploads.is_empty()); + assert!(connection.typed_submissions.is_empty()); connection.active_uploads = 0; let failed = app.submit_input().unwrap(); send_console_method(&mut app, &mut connection, &failed) .await .unwrap(); - assert_eq!(app.input.plain_text(), "inspect"); - assert_eq!(connection.pending_attachments, vec![file.clone()]); - assert!(connection.awaiting_attachment_acceptance.is_empty()); + assert!(matches!(app.input.submit_segments().as_slice(), + [Segment::Text { content }, Segment::UploadedFile { file: restored }] + if content == "inspect" && restored == &file)); + assert!(connection.staged_uploads.is_empty()); + assert_eq!(connection.typed_submissions.len(), 1); let retry = app.submit_input().unwrap(); + assert_eq!( + serde_json::to_value(&retry).unwrap(), + serde_json::to_value(&failed).unwrap() + ); send_console_method(&mut app, &mut connection, &retry) .await .unwrap(); assert!(app.input.is_empty()); - assert!(connection.pending_attachments.is_empty()); - assert_eq!( - connection.awaiting_attachment_acceptance, - vec![file.clone()] - ); + assert!(connection.staged_uploads.is_empty()); + assert_eq!(connection.typed_submissions.len(), 1); connection.observe_worker_event(&Event::UserMessage { entry_id: None, segments: vec![Segment::text("inspect"), Segment::UploadedFile { file }], }); - assert!(connection.pending_attachments.is_empty()); - assert!(connection.awaiting_attachment_acceptance.is_empty()); + assert!(connection.staged_uploads.is_empty()); + assert!(connection.typed_submissions.is_empty()); } #[tokio::test] @@ -1718,15 +2243,15 @@ mod tests { }), standalone_host: None, backend_target: None, - pending_attachments: vec![file.clone()], - awaiting_attachment_acceptance: Vec::new(), - upload_tasks: Vec::new(), + staged_uploads: HashMap::new(), + typed_submissions: HashMap::new(), + rejected_typed_submit: None, active_uploads: 0, - upload_ids: HashMap::new(), }; let mut app = App::new("worker".into()); app.set_worker_status(WorkerStatus::Running); app.input.insert_str("queued inspect"); + app.input.insert_uploaded_file(file.clone()); let method = app .submit_input() .expect("running Submit is sent immediately"); @@ -1735,63 +2260,46 @@ mod tests { .await .unwrap(); - assert_eq!(app.input.plain_text(), "queued inspect"); - assert_eq!(connection.pending_attachments, vec![file]); - assert!(connection.awaiting_attachment_acceptance.is_empty()); + assert!(matches!(app.input.submit_segments().as_slice(), + [Segment::Text { content }, Segment::UploadedFile { file: restored }] + if content == "queued inspect" && restored == &file)); + assert!(connection.staged_uploads.is_empty()); + assert_eq!(connection.typed_submissions.len(), 1); + assert_eq!( + serde_json::to_value(app.submit_input().unwrap()).unwrap(), + serde_json::to_value(&method).unwrap() + ); } #[test] - fn partial_attachment_echo_does_not_acknowledge_an_atomic_submission() { - let first = UploadedFileRef { - artifact_id: "artifact-1".into(), - file_name: "one.txt".into(), - media_type: "text/plain".into(), - created_at_ms: 1, - availability: UploadedFileAvailability::Available, - byte_len: 1, - sha256: "a".repeat(64), - source_entry_id: None, - }; - let second = UploadedFileRef { - artifact_id: "artifact-2".into(), - file_name: "two.txt".into(), - ..first.clone() - }; - let mut pending = Vec::new(); - let mut awaiting = vec![first.clone(), second.clone()]; - let expected = awaiting.clone(); - - for segments in [ - vec![Segment::UploadedFile { - file: first.clone(), - }], - vec![ - Segment::UploadedFile { - file: second.clone(), - }, - Segment::UploadedFile { - file: first.clone(), - }, - ], - vec![ - Segment::UploadedFile { - file: first.clone(), - }, - Segment::UploadedFile { - file: first.clone(), - }, - ], - ] { - assert!(!reconcile_attachment_submission( - &mut pending, - &mut awaiting, - &Event::UserMessage { - entry_id: None, - segments, - }, - )); - assert_eq!(awaiting, expected); - assert!(pending.is_empty()); + fn typed_retry_late_acceptance_clears_only_the_matching_restored_draft() { + for changed in [false, true] { + let mut app = App::new("worker".into()); + app.input + .insert_uploaded_file(crate::invocation_tests::uploaded_file()); + let method = app.submit_input().unwrap(); + app.restore_unsent_run(&method); + if changed { + app.input.insert_str("new edit"); + } + let before = app.input.submit_segments(); + let Method::Submit { + submission_request_id, + .. + } = method + else { + unreachable!() + }; + app.handle_worker_event(Event::SubmissionAccepted { + submission_request_id, + submission_id: "durable".into(), + disposition: protocol::SubmissionDisposition::Queued, + }); + if changed { + assert_eq!(app.input.submit_segments(), before); + } else { + assert!(app.input.is_empty()); + } } } @@ -1935,7 +2443,12 @@ mod tests { sha256: "a".repeat(64), source_entry_id: None, }; - upload_tx.send(Ok(file.clone())).unwrap(); + upload_tx + .send(AttachmentUploadResult { + id: "upload-1".into(), + result: Ok(file.clone()), + }) + .unwrap(); match next_loop_input( &mut terminal_rx, @@ -1947,7 +2460,13 @@ mod tests { ) .await { - LoopInput::Upload(Ok(received)) => assert_eq!(received, file), + LoopInput::Upload(AttachmentUploadResult { + id, + result: Ok(received), + }) => { + assert_eq!(id, "upload-1"); + assert_eq!(received, file); + } _ => panic!("expected attachment upload result"), } } @@ -3008,6 +3527,7 @@ mod tests { app.completion.as_mut().unwrap().entries = vec![protocol::CompletionEntry { value: "src/main.rs".into(), is_dir: false, + ..protocol::CompletionEntry::default() }]; app.handle_worker_event(Event::InternalWorker { worker: protocol::InternalWorkerRef { diff --git a/crates/tui/src/input.rs b/crates/tui/src/input.rs index 2c1368456..f7fec6a76 100644 --- a/crates/tui/src/input.rs +++ b/crates/tui/src/input.rs @@ -115,12 +115,31 @@ impl FlowRefAtom { } } +#[derive(Debug, Clone)] +pub struct FeatureInvokeAtom { + pub invocation: protocol::FeatureInvocation, +} + +impl FeatureInvokeAtom { + pub fn label(&self) -> String { + self.invocation.display_input() + } +} + #[derive(Debug, Clone)] pub enum Atom { Char(char), + Unknown, + AttachmentStage { + id: String, + file_name: String, + failed: bool, + }, Paste(PasteRef), PasteArtifact(protocol::PasteArtifactRef), + UploadedFile(protocol::UploadedFileRef), FileRef(FileRefAtom), + FeatureInvoke(FeatureInvokeAtom), FlowRef(FlowRefAtom), } @@ -130,6 +149,23 @@ impl Atom { fn chip(&self) -> Option<(Style, String)> { match self { Atom::Char(_) => None, + Atom::Unknown => Some(( + Style::default().fg(Color::Red), + "[unknown input segment]".into(), + )), + Atom::AttachmentStage { + file_name, failed, .. + } => Some(( + Style::default().fg(Color::Cyan), + format!( + "[Attached {file_name} | {}]", + if *failed { + "failed; Alt+Enter retries" + } else { + "uploading" + } + ), + )), Atom::Paste(p) => Some((Style::default().fg(Color::Magenta), p.label())), Atom::PasteArtifact(artifact) => Some(( Style::default().fg(Color::Magenta), @@ -143,7 +179,17 @@ impl Atom { artifact.created_at_ms ), )), + Atom::UploadedFile(file) => Some(( + Style::default().fg(Color::Cyan), + format!( + "[Attached {} | {} bytes, ready]", + file.file_name, file.byte_len + ), + )), Atom::FileRef(r) => Some((Style::default().fg(Color::Cyan), r.label())), + Atom::FeatureInvoke(invocation) => { + Some((Style::default().fg(Color::Yellow), invocation.label())) + } Atom::FlowRef(r) => Some((Style::default().fg(Color::Yellow), r.label())), } } @@ -173,9 +219,14 @@ enum WordKind { fn atom_class(atom: &Atom) -> AtomClass { match atom { Atom::Char(c) => char_class(*c), - Atom::Paste(_) | Atom::PasteArtifact(_) | Atom::FileRef(_) | Atom::FlowRef(_) => { - AtomClass::Chip - } + Atom::Unknown + | Atom::AttachmentStage { .. } + | Atom::Paste(_) + | Atom::PasteArtifact(_) + | Atom::UploadedFile(_) + | Atom::FileRef(_) + | Atom::FeatureInvoke(_) + | Atom::FlowRef(_) => AtomClass::Chip, } } @@ -195,12 +246,53 @@ fn char_class(c: char) -> AtomClass { } } +fn first_unquoted_char(value: &str, target: char) -> Option { + unquoted_char_indices(value).find_map(|(index, ch)| (ch == target).then_some(index)) +} + +fn last_unquoted_char(value: &str, target: char) -> Option { + unquoted_char_indices(value) + .filter_map(|(index, ch)| (ch == target).then_some(index)) + .last() +} + +fn unquoted_char_indices(value: &str) -> impl Iterator + '_ { + let mut quoted = false; + let mut escaped = false; + value.char_indices().filter(move |(_, ch)| { + if !escaped && *ch == '"' { + quoted = !quoted; + escaped = false; + return false; + } + let visible = !quoted; + escaped = quoted && !escaped && *ch == '\\'; + if *ch != '\\' { + escaped = false; + } + visible + }) +} + +#[derive(Clone)] +struct SelectedInvocation { + start: usize, + descriptor: protocol::FeatureInvocationDescriptor, + invocation_id: String, +} + pub struct InputBuffer { + /// Monotonic semantic edit/cursor revision, including ABA changes. + revision: u64, atoms: Vec, + selected_invocations: Vec, + removed_attachment_stages: Vec, /// Insertion point in `0..=atoms.len()`. cursor: usize, /// Top wrapped row of the visible composer viewport. scroll_offset: usize, + /// Uploaded draft chips removed by explicit editing and awaiting client-side cleanup. + removed_uploaded_files: Vec, /// Monotonic counter reused across the TUI process lifetime. next_paste_id: u32, } @@ -208,20 +300,34 @@ pub struct InputBuffer { impl Default for InputBuffer { fn default() -> Self { Self { + revision: 0, atoms: Vec::new(), + selected_invocations: Vec::new(), + removed_attachment_stages: Vec::new(), cursor: 0, scroll_offset: 0, + removed_uploaded_files: Vec::new(), next_paste_id: 1, } } } impl InputBuffer { + pub fn revision(&self) -> u64 { + self.revision + } + + fn bump_revision(&mut self) { + self.revision = self.revision.wrapping_add(1); + } + pub fn new() -> Self { Self::default() } pub fn clear(&mut self) { + self.bump_revision(); + self.selected_invocations.clear(); self.atoms.clear(); self.cursor = 0; self.scroll_offset = 0; @@ -243,6 +349,8 @@ impl InputBuffer { /// by [`submit_segments`](Self::submit_segments), preserving typed chips /// and placing the cursor at the end of the restored input. pub fn replace_with_segments(&mut self, segments: &[protocol::Segment]) { + self.bump_revision(); + self.selected_invocations.clear(); self.atoms.clear(); for segment in segments { match segment { @@ -267,24 +375,24 @@ impl InputBuffer { self.atoms.push(Atom::PasteArtifact(artifact.clone())); } protocol::Segment::UploadedFile { file } => { - self.atoms.extend( - format!("[Attached file: {}]", file.file_name) - .chars() - .map(Atom::Char), - ); + self.atoms.push(Atom::UploadedFile(file.clone())); } protocol::Segment::FileRef { path } => { self.atoms .push(Atom::FileRef(FileRefAtom { path: path.clone() })); } + protocol::Segment::FeatureInvoke { invocation } => { + self.atoms.push(Atom::FeatureInvoke(FeatureInvokeAtom { + invocation: invocation.clone(), + })); + } protocol::Segment::Flow { selector } => { self.atoms.push(Atom::FlowRef(FlowRefAtom { selector: selector.clone(), })); } protocol::Segment::Unknown => { - self.atoms - .extend("[unknown input segment]".chars().map(Atom::Char)); + self.atoms.push(Atom::Unknown); } } } @@ -292,6 +400,8 @@ impl InputBuffer { } pub fn insert_char(&mut self, c: char) { + self.bump_revision(); + self.adjust_selection(self.cursor, self.cursor, 1); self.atoms.insert(self.cursor, Atom::Char(c)); self.cursor += 1; } @@ -307,6 +417,10 @@ impl InputBuffer { for atom in &self.atoms { match atom { Atom::Char(c) => text.push(*c), + Atom::Unknown => text.push_str("[unknown input segment]"), + Atom::AttachmentStage { file_name, .. } => { + text.push_str(&format!("[Staging {file_name}]")) + } Atom::Paste(paste) => text.push_str(&paste.content), Atom::PasteArtifact(artifact) => { text.push_str(&protocol::Segment::flatten_to_text(&[ @@ -315,7 +429,11 @@ impl InputBuffer { }, ])) } + Atom::UploadedFile(file) => { + text.push_str(&format!("[Attached file: {}]", file.file_name)); + } Atom::FileRef(file) => text.push_str(&file.path), + Atom::FeatureInvoke(invocation) => text.push_str(&invocation.label()), Atom::FlowRef(flow) => text.push_str(&flow.selector), } } @@ -327,6 +445,7 @@ impl InputBuffer { } pub fn insert_paste(&mut self, content: String) { + self.bump_revision(); let measurement = measure_paste(&content); if measurement.presentation() == PastePresentation::Text { self.insert_str(&content); @@ -335,6 +454,7 @@ impl InputBuffer { let id = self.next_paste_id; self.next_paste_id = self.next_paste_id.wrapping_add(1); + self.adjust_selection(self.cursor, self.cursor, 1); self.atoms.insert( self.cursor, Atom::Paste(PasteRef { @@ -347,14 +467,236 @@ impl InputBuffer { self.cursor += 1; } + #[cfg(test)] + pub fn insert_uploaded_file(&mut self, file: protocol::UploadedFileRef) { + self.bump_revision(); + self.adjust_selection(self.cursor, self.cursor, 1); + self.atoms.insert(self.cursor, Atom::UploadedFile(file)); + self.cursor += 1; + } + + // Selections belong to an exact occurrence, never to all text matching a name. + fn adjust_selection(&mut self, start: usize, end: usize, inserted: usize) { + self.selected_invocations.retain_mut(|selection| { + if start < end && (start..end).contains(&selection.start) { + return false; + } + if selection.start >= end { + selection.start = selection.start - (end - start) + inserted; + } + true + }); + } + + pub fn select_feature_invocation( + &mut self, + start: usize, + descriptor: protocol::FeatureInvocationDescriptor, + ) { + self.bump_revision(); + self.selected_invocations + .retain(|selection| selection.start != start); + self.selected_invocations.push(SelectedInvocation { + start, + descriptor, + invocation_id: protocol::new_submission_request_id(), + }); + } + + pub fn has_selected_feature_invocations(&self) -> bool { + !self.selected_invocations.is_empty() + } + + pub fn retry_attachment_stage(&mut self, id: &str) { + self.bump_revision(); + for atom in &mut self.atoms { + if let Atom::AttachmentStage { + id: stage_id, + failed, + .. + } = atom + { + if stage_id == id { + *failed = false; + } + } + } + } + + pub fn finalize_feature_invocations(&mut self) -> Result<(), String> { + // Validate first so invalid arguments cannot partially chip a draft. + let mut parsed = Vec::new(); + for selection in &self.selected_invocations { + let text = self.atoms[selection.start..] + .iter() + .take_while(|atom| matches!(atom, Atom::Char(_))) + .filter_map(|atom| { + if let Atom::Char(ch) = atom { + Some(*ch) + } else { + None + } + }) + .collect::(); + let invocation = protocol::parse_feature_invocation( + &text, + 0, + &selection.descriptor, + selection.invocation_id.clone(), + ) + .map_err(|error| error.to_string())?; + parsed.push(( + selection.start, + text[..invocation.end].chars().count(), + invocation.invocation, + )); + } + parsed.sort_by_key(|(start, _, _)| std::cmp::Reverse(*start)); + for (start, count, invocation) in parsed { + self.replace_atoms( + start, + start + count, + vec![Atom::FeatureInvoke(FeatureInvokeAtom { invocation })], + ); + } + Ok(()) + } + + fn replace_atoms(&mut self, start: usize, end: usize, atoms: Vec) { + self.bump_revision(); + let inserted = atoms.len(); + self.adjust_selection(start, end, inserted); + let removed = self.atoms.splice(start..end, atoms).collect::>(); + for atom in removed { + self.record_removed_atom(atom); + } + if self.cursor >= end { + self.cursor = self.cursor - (end - start) + inserted; + } else if self.cursor > start { + self.cursor = start + inserted; + } + } + + pub fn adjacent_feature_invocation(&self) -> Option { + [self.cursor.checked_sub(1), Some(self.cursor)] + .into_iter() + .flatten() + .find_map(|index| match self.atoms.get(index) { + Some(Atom::FeatureInvoke(atom)) => Some(atom.invocation.clone()), + _ => None, + }) + } + + pub fn edit_feature_invocation( + &mut self, + id: &str, + descriptor: protocol::FeatureInvocationDescriptor, + ) -> bool { + let Some((index, invocation)) = + self.atoms + .iter() + .enumerate() + .find_map(|(index, atom)| match atom { + Atom::FeatureInvoke(atom) if atom.invocation.invocation_id == id => { + Some((index, atom.invocation.clone())) + } + _ => None, + }) + else { + return false; + }; + if invocation.identity != descriptor.identity { + return false; + } + let text = invocation.display_input(); + self.replace_atoms(index, index + 1, text.chars().map(Atom::Char).collect()); + self.cursor = index + text.chars().count() - 1; + self.selected_invocations.push(SelectedInvocation { + start: index, + descriptor, + invocation_id: invocation.invocation_id, + }); + true + } + + pub fn stage_attachment(&mut self, invocation_id: &str, id: String, file_name: String) -> bool { + let Some(index) = self.atoms.iter().position(|atom| matches!(atom, Atom::FeatureInvoke(atom) if atom.invocation.invocation_id == invocation_id)) else { return false; }; + self.replace_atoms( + index, + index + 1, + vec![Atom::AttachmentStage { + id, + file_name, + failed: false, + }], + ); + true + } + + pub fn has_attachment_stages(&self) -> bool { + self.atoms + .iter() + .any(|atom| matches!(atom, Atom::AttachmentStage { .. })) + } + + pub fn attachment_stage_ids(&self) -> Vec { + self.atoms + .iter() + .filter_map(|atom| { + if let Atom::AttachmentStage { id, .. } = atom { + Some(id.clone()) + } else { + None + } + }) + .collect() + } + + pub fn adjacent_failed_attachment(&self) -> Option { + [self.cursor.checked_sub(1), Some(self.cursor)] + .into_iter() + .flatten() + .find_map(|index| match self.atoms.get(index) { + Some(Atom::AttachmentStage { + id, failed: true, .. + }) => Some(id.clone()), + _ => None, + }) + } + + pub fn finish_attachment_stage( + &mut self, + id: &str, + file: Option, + ) -> bool { + self.bump_revision(); + let Some(index) = self.atoms.iter().position( + |atom| matches!(atom, Atom::AttachmentStage { id: stage_id, .. } if stage_id == id), + ) else { + return false; + }; + if let Some(file) = file { + self.atoms[index] = Atom::UploadedFile(file); + } else if let Atom::AttachmentStage { failed, .. } = &mut self.atoms[index] { + *failed = true; + } + true + } + + pub fn take_removed_attachment_stages(&mut self) -> Vec { + std::mem::take(&mut self.removed_attachment_stages) + } + /// Replace `atoms[start..self.cursor]` (the in-flight `@` / /// active `@` file token) with the corresponding chip atom /// and place the cursor right after the chip. Used by the completion /// confirm path. pub fn replace_with_file_ref(&mut self, start: usize, path: String) { - self.atoms.drain(start..self.cursor); - self.atoms - .insert(start, Atom::FileRef(FileRefAtom { path })); + self.replace_atoms( + start, + self.cursor, + vec![Atom::FileRef(FileRefAtom { path })], + ); self.cursor = start + 1; } @@ -363,14 +705,177 @@ impl InputBuffer { /// completion path: the popup-selected entry is inserted as raw /// text (not a chip) so the user can keep typing — e.g. drill into /// a directory whose value ends with `/`. - pub fn replace_with_text_at(&mut self, start: usize, text: &str) { - self.atoms.drain(start..self.cursor); - let mut idx = start; - for c in text.chars() { - self.atoms.insert(idx, Atom::Char(c)); - idx += 1; + pub fn char_at(&self, index: usize) -> Option { + match self.atoms.get(index) { + Some(Atom::Char(ch)) => Some(*ch), + _ => None, } - self.cursor = idx; + } + + pub fn replace_with_text_at(&mut self, start: usize, text: &str) { + self.replace_atoms(start, self.cursor, text.chars().map(Atom::Char).collect()); + self.cursor = start + text.chars().count(); + } + + pub fn can_complete_argument_name(&self, prefix_start: usize) -> bool { + let mut before = prefix_start; + while self.char_at(before).is_some_and(char::is_whitespace) { + let Some(previous) = before.checked_sub(1) else { + return false; + }; + before = previous; + } + matches!(self.char_at(before), Some('(' | ',')) + } + + pub fn replace_feature_name_completion(&mut self, start: usize, text: &str) { + let mut end = self.cursor; + while self + .char_at(end) + .is_some_and(|ch| ch.is_ascii_alphanumeric() || ch == '-') + { + end += 1; + } + if self.char_at(end) == Some('(') { + end += 1; + } + self.replace_atoms(start, end, text.chars().map(Atom::Char).collect()); + self.cursor = start + text.chars().count(); + } + + pub fn replace_argument_completion(&mut self, start: usize, text: &str) { + // Replace the whole value including an existing closing quote when editing its middle. + let mut end = self.cursor; + let quoted = self.char_at(start) == Some('"'); + let mut escaped = false; + while let Some(ch) = self.char_at(end) { + if !quoted && (ch == ',' || ch == ')' || ch.is_whitespace()) { + break; + } + end += 1; + if quoted && !escaped && ch == '"' { + break; + } + escaped = ch == '\\' && !escaped; + if ch != '\\' { + escaped = false; + } + } + if quoted && self.char_at(end.saturating_sub(1)) != Some('"') { + end = self.cursor; + } + self.replace_atoms(start, end, text.chars().map(Atom::Char).collect()); + self.cursor = start + text.chars().count(); + } + + pub fn pending_feature_argument_completion( + &self, + descriptors: &[protocol::FeatureInvocationDescriptor], + ) -> Option<( + protocol::CompletionKind, + usize, + String, + protocol::CompletionContext, + )> { + if self.cursor == 0 { + return None; + } + let selection = self + .selected_invocations + .iter() + .filter(|selected| selected.start < self.cursor) + .max_by_key(|selected| selected.start)?; + let run_start = selection.start; + let descriptor = &selection.descriptor; + if !descriptors + .iter() + .any(|candidate| candidate.identity == descriptor.identity) + { + return None; + } + let invocation = self.atoms[run_start..self.cursor] + .iter() + .map(|atom| match atom { + Atom::Char(ch) => Some(*ch), + _ => None, + }) + .collect::>()?; + let open = invocation.find('(')?; + let arguments = &invocation[open + 1..]; + if first_unquoted_char(arguments, ')').is_some() { + return None; + } + let current_start = last_unquoted_char(arguments, ',').map_or(0, |index| index + 1); + let current = &arguments[current_start..]; + let leading = current.len() - current.trim_start().len(); + let current = ¤t[leading..]; + let (argument, prefix_offset, mut prefix) = + if let Some(equal) = first_unquoted_char(current, '=') { + let name = current[..equal].trim(); + let mut value = ¤t[equal + 1..]; + let mut offset = equal + 1; + let value_leading = value.len() - value.trim_start().len(); + value = value.trim_start(); + offset += value_leading; + if let Some(unquoted) = value.strip_prefix('"') { + value = unquoted; + offset += 1; + } + (Some(name.to_string()), offset, value.to_string()) + } else { + let mut previous_start = 0; + let mut position = 0; + for comma in unquoted_char_indices(&arguments[..current_start]) + .filter_map(|(index, ch)| (ch == ',').then_some(index)) + { + if first_unquoted_char(&arguments[previous_start..comma], '=').is_none() { + position += 1; + } + previous_start = comma + 1; + } + let positional = descriptor + .arguments + .iter() + .find(|argument| argument.position == Some(position)); + if (current.is_empty() && positional.is_none()) + || (current.bytes().all(|byte| { + byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'_' + }) && positional.is_none()) + { + (None, 0, current.to_string()) + } else { + let mut value = current; + let mut offset = 0; + if let Some(unquoted) = value.strip_prefix('"') { + value = unquoted; + offset = 1; + } + ( + positional.map(|argument| argument.name.clone()), + offset, + value.to_string(), + ) + } + }; + if prefix_offset > 0 && current[..prefix_offset].ends_with('"') { + if serde_json::from_str::(&format!("\"{}", prefix)).is_ok() { + return None; + } + prefix = serde_json::from_str::(&format!("\"{}\"", prefix)).ok()?; + } + let prefix_start = run_start + + invocation[..open + 1].chars().count() + + arguments[..current_start + leading].chars().count() + + current[..prefix_offset].chars().count(); + Some(( + protocol::CompletionKind::FeatureArgument, + prefix_start.saturating_sub(1), + prefix, + protocol::CompletionContext { + invocation: descriptor.identity.clone(), + argument, + }, + )) } /// If the cursor is currently inside a `@` / @@ -398,7 +903,9 @@ impl InputBuffer { } let kind = match c { '@' => Some(protocol::CompletionKind::File), - + '/' if !typed.contains(['(', ')', '=', ',']) => { + Some(protocol::CompletionKind::Feature) + } _ => None, }; if let Some(k) = kind { @@ -422,17 +929,29 @@ impl InputBuffer { None } + pub fn take_removed_uploaded_files(&mut self) -> Vec { + std::mem::take(&mut self.removed_uploaded_files) + } + + fn record_removed_atom(&mut self, atom: Atom) { + match atom { + Atom::UploadedFile(file) => self.removed_uploaded_files.push(file), + Atom::AttachmentStage { id, .. } => self.removed_attachment_stages.push(id), + _ => {} + } + } + pub fn delete_before(&mut self) { if self.cursor == 0 { return; } self.cursor -= 1; - self.atoms.remove(self.cursor); + self.replace_atoms(self.cursor, self.cursor + 1, Vec::new()); } pub fn delete_after(&mut self) { if self.cursor < self.atoms.len() { - self.atoms.remove(self.cursor); + self.replace_atoms(self.cursor, self.cursor + 1, Vec::new()); } } @@ -442,14 +961,16 @@ impl InputBuffer { let end = self.cursor; self.move_word_left(); let start = self.cursor; - self.atoms.drain(start..end); + self.replace_atoms(start, end, Vec::new()); } pub fn move_left(&mut self) { + self.bump_revision(); self.cursor = self.cursor.saturating_sub(1); } pub fn move_right(&mut self) { + self.bump_revision(); self.cursor = (self.cursor + 1).min(self.atoms.len()); } @@ -457,6 +978,7 @@ impl InputBuffer { /// atoms sharing the same [`AtomClass`] — so `Word(Hiragana)` next to /// `Word(Han)` are separate blocks, and a `Paste` atom is its own block. pub fn move_word_left(&mut self) { + self.bump_revision(); while self.cursor > 0 && atom_class(&self.atoms[self.cursor - 1]) == AtomClass::Sep { self.cursor -= 1; } @@ -471,6 +993,7 @@ impl InputBuffer { /// Move forward by one word. Mirror of [`move_word_left`]. pub fn move_word_right(&mut self) { + self.bump_revision(); while self.cursor < self.atoms.len() && atom_class(&self.atoms[self.cursor]) == AtomClass::Sep { @@ -486,6 +1009,7 @@ impl InputBuffer { } pub fn move_start(&mut self) { + self.bump_revision(); self.cursor = 0; } @@ -534,11 +1058,13 @@ impl InputBuffer { } pub fn move_home(&mut self) { + self.bump_revision(); let (ranges, line, _) = self.logical_line_and_col(); self.cursor = ranges[line].0; } pub fn move_end(&mut self) { + self.bump_revision(); let (ranges, line, _) = self.logical_line_and_col(); self.cursor = ranges[line].1; } @@ -546,6 +1072,7 @@ impl InputBuffer { /// Move one logical line up, preserving column (atom count from /// current line start). No-op if already on the first line. pub fn move_up(&mut self) { + self.bump_revision(); let (ranges, line, col) = self.logical_line_and_col(); if line == 0 { return; @@ -556,6 +1083,7 @@ impl InputBuffer { /// Move one logical line down, preserving column. pub fn move_down(&mut self) { + self.bump_revision(); let (ranges, line, col) = self.logical_line_and_col(); let Some(&(start, end)) = ranges.get(line + 1) else { return; @@ -580,6 +1108,12 @@ impl InputBuffer { for a in &self.atoms { match a { Atom::Char(c) => buf.push(*c), + Atom::Unknown => { + flush_text(&mut buf, &mut out); + out.push(protocol::Segment::Unknown); + } + // Staging is client-owned and never a submit payload. + Atom::AttachmentStage { .. } => {} Atom::Paste(p) => { flush_text(&mut buf, &mut out); out.push(protocol::Segment::Paste { @@ -595,12 +1129,22 @@ impl InputBuffer { artifact: artifact.clone(), }); } + Atom::UploadedFile(file) => { + flush_text(&mut buf, &mut out); + out.push(protocol::Segment::UploadedFile { file: file.clone() }); + } Atom::FileRef(r) => { flush_text(&mut buf, &mut out); out.push(protocol::Segment::FileRef { path: r.path.clone(), }); } + Atom::FeatureInvoke(invocation) => { + flush_text(&mut buf, &mut out); + out.push(protocol::Segment::FeatureInvoke { + invocation: invocation.invocation.clone(), + }); + } Atom::FlowRef(r) => { flush_text(&mut buf, &mut out); out.push(protocol::Segment::Flow { @@ -1531,9 +2075,14 @@ mod word_motion_tests { for a in &buf.atoms { match a { Atom::Char(c) => out.push(*c), - Atom::Paste(_) | Atom::PasteArtifact(_) | Atom::FileRef(_) | Atom::FlowRef(_) => { - out.push_str("

") - } + Atom::Unknown + | Atom::AttachmentStage { .. } + | Atom::Paste(_) + | Atom::PasteArtifact(_) + | Atom::UploadedFile(_) + | Atom::FileRef(_) + | Atom::FeatureInvoke(_) + | Atom::FlowRef(_) => out.push_str("

"), } } out @@ -1621,3 +2170,81 @@ mod word_motion_tests { assert_eq!(cursor(&buf), 7); // after "った" } } + +#[cfg(test)] +mod feature_invocation_input_tests { + use super::*; + use protocol::{ + FeatureInvocationDescriptor, FeatureInvocationIdentity, FeatureInvocationSyntax, + InvocationArgumentDescriptor, InvocationArgumentType, InvocationCompletion, Segment, + UploadedFileAvailability, UploadedFileRef, + }; + + fn descriptor() -> FeatureInvocationDescriptor { + FeatureInvocationDescriptor { + identity: FeatureInvocationIdentity("builtin:test/run".into()), + name: "run".into(), + aliases: Vec::new(), + display_name: "Run".into(), + description: "test".into(), + syntax: FeatureInvocationSyntax::Parenthesized, + arguments: vec![InvocationArgumentDescriptor { + name: "path".into(), + position: Some(0), + required: true, + value_type: InvocationArgumentType::String, + completion: InvocationCompletion::None, + description: None, + }], + client_adapter: None, + } + } + + #[test] + fn selected_invocations_become_typed_atoms_and_preserve_following_prose() { + let mut input = InputBuffer::new(); + input.insert_str("before /run(\"資料/a b\") after /run(\"x/y\") done"); + input.select_feature_invocation(7, descriptor()); + input.select_feature_invocation( + "before /run(\"資料/a b\") after ".chars().count(), + descriptor(), + ); + input.finalize_feature_invocations().unwrap(); + let segments = input.submit_segments(); + assert_eq!( + segments + .iter() + .filter(|segment| matches!(segment, Segment::FeatureInvoke { .. })) + .count(), + 2 + ); + assert!(matches!(segments.last(), Some(Segment::Text { content }) if content == " done")); + + let mut unselected = InputBuffer::new(); + unselected.insert_str("/run(\"x\")"); + unselected.finalize_feature_invocations().unwrap(); + assert!(matches!( + unselected.submit_segments().as_slice(), + [Segment::Text { content }] if content == "/run(\"x\")" + )); + } + + #[test] + fn deleting_an_uploaded_file_chip_reports_exact_staged_resource() { + let file = UploadedFileRef { + artifact_id: "artifact-1".into(), + file_name: "report.md".into(), + media_type: "text/markdown".into(), + created_at_ms: 1, + availability: UploadedFileAvailability::Available, + byte_len: 4, + sha256: "a".repeat(64), + source_entry_id: None, + }; + let mut input = InputBuffer::new(); + input.insert_uploaded_file(file.clone()); + input.delete_before(); + assert_eq!(input.take_removed_uploaded_files(), vec![file]); + assert!(input.take_removed_uploaded_files().is_empty()); + } +} diff --git a/crates/tui/src/invocation_tests.rs b/crates/tui/src/invocation_tests.rs new file mode 100644 index 000000000..b7f6a8a40 --- /dev/null +++ b/crates/tui/src/invocation_tests.rs @@ -0,0 +1,653 @@ +//! T-696 client contract regressions, exercised through real composer selection. +use crate::{app::App, input::InputBuffer}; +use protocol::*; + +pub(crate) fn descriptor() -> FeatureInvocationDescriptor { + FeatureInvocationDescriptor { + identity: FeatureInvocationIdentity("plugin:test/run".into()), + name: "run".into(), + aliases: vec!["execute".into()], + display_name: "Run".into(), + description: "Run a declared action".into(), + syntax: FeatureInvocationSyntax::Parenthesized, + arguments: vec![ + InvocationArgumentDescriptor { + name: "path".into(), + position: Some(0), + required: true, + value_type: InvocationArgumentType::WorkerFile, + completion: InvocationCompletion::WorkerFile, + description: None, + }, + InvocationArgumentDescriptor { + name: "mode".into(), + position: None, + required: false, + value_type: InvocationArgumentType::Enum { + values: vec!["safe".into(), "fast".into()], + }, + completion: InvocationCompletion::Static { + values: vec!["safe".into(), "fast".into()], + }, + description: None, + }, + InvocationArgumentDescriptor { + name: "count".into(), + position: None, + required: false, + value_type: InvocationArgumentType::Integer, + completion: InvocationCompletion::None, + description: None, + }, + InvocationArgumentDescriptor { + name: "force".into(), + position: None, + required: false, + value_type: InvocationArgumentType::Boolean, + completion: InvocationCompletion::None, + description: None, + }, + ], + client_adapter: None, + } +} + +pub(crate) fn attachment_descriptor() -> FeatureInvocationDescriptor { + let mut descriptor = descriptor(); + // No hardcoded builtin identity, public name, or argument name in adapter dispatch. + descriptor.identity = FeatureInvocationIdentity("plugin:files/add".into()); + descriptor.name = "add-file".into(); + descriptor.aliases.clear(); + descriptor.arguments.truncate(1); + descriptor.arguments[0].name = "resource".into(); + descriptor.arguments[0].value_type = InvocationArgumentType::ClientFile; + descriptor.arguments[0].completion = InvocationCompletion::ClientFile; + descriptor.client_adapter = Some(InvocationClientAdapter::Attachment); + descriptor +} + +pub(crate) fn uploaded_file() -> UploadedFileRef { + UploadedFileRef { + artifact_id: "artifact-owned".into(), + file_name: "資料 a.txt".into(), + media_type: "text/plain".into(), + created_at_ms: 1, + availability: UploadedFileAvailability::Available, + byte_len: 4, + sha256: "a".repeat(64), + source_entry_id: None, + } +} + +pub(crate) fn select( + app: &mut App, + descriptor: FeatureInvocationDescriptor, + name: &str, +) -> Option { + app.insert_char('/'); + assert!(matches!( + app.refresh_completion(), + Some(Method::ListCompletions { + kind: CompletionKind::Feature, + .. + }) + )); + app.handle_worker_event(Event::Completions { + request_id: app.completion_request_id(), + kind: CompletionKind::Feature, + prefix: String::new(), + context: None, + entries: vec![CompletionEntry { + value: name.into(), + invocation: Some(descriptor), + ..Default::default() + }], + }); + app.apply_completion_text() +} + +fn complete(app: &mut App, value: &str, is_dir: bool) -> Option { + let state = app.completion.as_ref().expect("argument completion"); + app.handle_worker_event(Event::Completions { + request_id: app.completion_request_id(), + kind: state.kind, + prefix: state.prefix.clone(), + context: state.context.clone(), + entries: vec![CompletionEntry { + value: value.into(), + is_dir, + ..Default::default() + }], + }); + app.apply_completion_text() +} + +#[test] +fn feature_invocation_discovery_alias_selection_and_confirmation_are_occurrence_scoped() { + let mut app = App::new("test".into()); + app.input.insert_str("説明 "); + assert!( + matches!(select(&mut app, descriptor(), "execute"), Some(Method::ListCompletions { + kind: CompletionKind::FeatureArgument, context: Some(CompletionContext { argument: Some(argument), .. }), .. + }) if argument == "path") + ); + app.input + .insert_str("\"資料/a b\") after /execute(\"not selected\")"); + assert!( + app.submit_input().is_none(), + "first Enter confirms typed chips, not Send" + ); + let segments = app.input.submit_segments(); + assert!( + matches!(&segments[1], Segment::FeatureInvoke { invocation } if invocation.identity == descriptor().identity && invocation.name == "execute") + ); + assert!( + matches!(segments.last(), Some(Segment::Text { content }) if content == " after /execute(\"not selected\")") + ); + assert!(matches!(app.submit_input(), Some(Method::Submit { input, .. }) if input == segments)); +} + +#[test] +fn feature_invocation_raw_text_paste_and_unknown_payloads_do_not_dispatch() { + let mut app = App::new("test".into()); + app.input + .insert_str("/run(x) /attach(/tmp/a.txt) /clear-attachments"); + assert!( + matches!(app.submit_input(), Some(Method::Submit { input, .. }) if matches!(input.as_slice(), [Segment::Text { .. }])) + ); + let unknown: Segment = + serde_json::from_str(r#"{"kind":"future_invocation","identity":"plugin:test/run"}"#) + .unwrap(); + app.input.replace_with_segments(&[unknown]); + assert_eq!(app.input.submit_segments(), vec![Segment::Unknown]); + let mut input = InputBuffer::new(); + input.insert_paste("/run(x) ".repeat(20)); + input.finalize_feature_invocations().unwrap(); + assert!(matches!( + input.submit_segments().as_slice(), + [Segment::Paste { .. }] + )); +} + +#[test] +fn feature_invocation_argument_insertion_quotes_unicode_spaces_escapes_and_keeps_prose() { + let mut app = App::new("test".into()); + app.input.insert_str("界 "); + select(&mut app, descriptor(), "run"); + let value = "資料/a b/quote\"slash\\end"; + complete(&mut app, value, false); + app.input.insert_str(", mode=sa"); + app.refresh_completion(); + complete(&mut app, "safe", false); + app.input.insert_str(", count=1"); + app.refresh_completion(); + complete(&mut app, "12", false); + app.input.insert_str(", force=f"); + app.refresh_completion(); + complete(&mut app, "false", false); + app.input.insert_str(") trailing prose"); + assert!(app.submit_input().is_none()); + let segments = app.input.submit_segments(); + let Segment::FeatureInvoke { invocation } = &segments[1] else { + panic!("typed chip expected") + }; + assert_eq!( + invocation.arguments[0].value, + InvocationValue::String(value.into()) + ); + assert_eq!(invocation.arguments[2].value, InvocationValue::Integer(12)); + assert_eq!( + invocation.arguments[3].value, + InvocationValue::Boolean(false) + ); + assert!( + matches!(segments.last(), Some(Segment::Text { content }) if content == " trailing prose") + ); +} + +#[test] +fn feature_invocation_argument_names_and_directory_drill_in_are_safe() { + let mut app = App::new("test".into()); + select(&mut app, descriptor(), "run"); + assert!( + matches!(complete(&mut app, "資料 dir", true), Some(Method::ListCompletions { prefix, .. }) if prefix == "資料 dir/") + ); + complete(&mut app, "資料 dir/file name.txt", false); + app.input.insert_str(", mo"); + app.refresh_completion(); + assert!( + app.completion + .as_ref() + .unwrap() + .context + .as_ref() + .unwrap() + .argument + .is_none() + ); + assert!( + matches!(complete(&mut app, "mode=", false), Some(Method::ListCompletions { context: Some(CompletionContext { argument: Some(argument), .. }), .. }) if argument == "mode") + ); + complete(&mut app, "fast", false); + app.input.insert_char(')'); + assert!(app.submit_input().is_none()); + assert!(matches!( + app.input.submit_segments().as_slice(), + [Segment::FeatureInvoke { .. }] + )); +} + +#[test] +fn feature_invocation_completion_replaces_middle_value_without_suffix_corruption() { + let mut input = InputBuffer::new(); + input.insert_str("界 /run(path= \"資料/old name\", mode=safe) prose"); + input.select_feature_invocation(2, descriptor()); + input.move_home(); + for _ in 0.."界 /run(path= \"資料/ol".chars().count() { + input.move_right(); + } + let (_, before, prefix, context) = input + .pending_feature_argument_completion(&[descriptor()]) + .unwrap(); + assert_eq!(prefix, "資料/ol"); + assert_eq!(context.argument.as_deref(), Some("path")); + let start = before; // preceding atom is the opening quote + input.replace_argument_completion(start, "e_invocation_string("資料/new name")); + assert_eq!( + input.plain_text(), + "界 /run(path= \"資料/new name\", mode=safe) prose" + ); + input.finalize_feature_invocations().unwrap(); + assert!( + matches!(input.submit_segments().last(), Some(Segment::Text { content }) if content == " prose") + ); +} + +#[test] +fn feature_invocation_validation_is_atomic_for_multiple_selected_calls() { + for invalid in [ + "/run()", + "/run(x, nope=y)", + "/run(x, mode=unknown)", + "/run(x, count=oops)", + "/run(x, force=yes)", + "/run(x, path=y)", + "/run(\"unfinished", + ] { + let mut input = InputBuffer::new(); + input.insert_str("/run(ok) then "); + let second = input.plain_text().chars().count(); + input.insert_str(invalid); + input.select_feature_invocation(0, descriptor()); + input.select_feature_invocation(second, descriptor()); + let before = input.submit_segments(); + assert!(input.finalize_feature_invocations().is_err(), "{invalid}"); + assert_eq!(input.submit_segments(), before); + } +} + +#[test] +fn feature_invocation_selection_tracks_cursor_edits_and_does_not_survive_deleting_its_sigil() { + let mut input = InputBuffer::new(); + input.insert_str("/run(x)"); + input.select_feature_invocation(0, descriptor()); + input.move_home(); + input.insert_str("prefix "); + input.finalize_feature_invocations().unwrap(); + assert!(matches!( + &input.submit_segments()[1], + Segment::FeatureInvoke { .. } + )); + input.replace_with_segments(&[Segment::text("/run(y)")]); + input.select_feature_invocation(0, descriptor()); + input.move_home(); + input.delete_after(); + input.insert_char('/'); + input.finalize_feature_invocations().unwrap(); + assert_eq!(input.submit_segments(), vec![Segment::text("/run(y)")]); +} + +#[test] +fn feature_invocation_chip_reedit_preserves_identity_id_and_typed_arguments() { + let parsed = parse_feature_invocation( + "/execute(\"a b\", mode=safe, count=2)", + 0, + &descriptor(), + "stable-id", + ) + .unwrap() + .invocation; + let segments = vec![ + Segment::FeatureInvoke { + invocation: parsed.clone(), + }, + Segment::UploadedFile { + file: uploaded_file(), + }, + Segment::Flow { + selector: "review".into(), + }, + Segment::FileRef { + path: "src/".into(), + }, + ]; + let mut input = InputBuffer::new(); + input.replace_with_segments(&segments); + assert_eq!(input.submit_segments(), segments); + assert!(input.edit_feature_invocation("stable-id", descriptor())); + input.finalize_feature_invocations().unwrap(); + assert_eq!(input.submit_segments(), segments); + assert!(!input.edit_feature_invocation("stable-id", attachment_descriptor())); +} + +#[test] +fn feature_invocation_reedit_unknown_disabled_or_stale_keeps_original_chip() { + let invocation = parse_feature_invocation("/run(x)", 0, &descriptor(), "id") + .unwrap() + .invocation; + let mut app = App::new("test".into()); + let segments = vec![Segment::FeatureInvoke { invocation }]; + app.input.replace_with_segments(&segments); + assert!(matches!( + app.edit_adjacent_feature_invocation(), + Some(Method::ListCompletions { .. }) + )); + app.handle_worker_event(Event::Completions { + request_id: app.completion_request_id(), + kind: CompletionKind::Feature, + prefix: "run".into(), + context: None, + entries: Vec::new(), + }); + assert_eq!(app.input.submit_segments(), segments); + app.edit_adjacent_feature_invocation(); + app.input.move_home(); + app.refresh_completion(); + app.handle_worker_event(Event::Completions { + request_id: app.completion_request_id(), + kind: CompletionKind::Feature, + prefix: "run".into(), + context: None, + entries: vec![CompletionEntry { + value: "run".into(), + invocation: Some(descriptor()), + ..Default::default() + }], + }); + assert_eq!(app.input.submit_segments(), segments); +} + +#[test] +fn feature_invocation_generic_attachment_adapter_never_records_or_sends_local_path() { + let mut app = App::new("test".into()); + select(&mut app, attachment_descriptor(), "add-file"); + app.input.insert_str("\"/tmp/資料 a.txt\") keep this prose"); + assert!(app.submit_input().is_none()); + assert_eq!(app.input_history_len(), 0); + let adapters = app.attachment_invocations(); + assert_eq!(adapters.len(), 1); + assert_eq!(adapters[0].1.to_str().unwrap(), "/tmp/資料 a.txt"); + app.input + .stage_attachment(&adapters[0].0, "staging-id".into(), "資料 a.txt".into()); + assert!(app.submit_input().is_none()); + assert!(app.submit_notify_input().is_none()); + app.input + .finish_attachment_stage("staging-id", Some(uploaded_file())); + let Method::Submit { input, .. } = app.submit_input().unwrap() else { + panic!("submit expected") + }; + assert!(matches!(&input[0], Segment::UploadedFile { .. })); + assert!(!serde_json::to_string(&input).unwrap().contains("/tmp/")); + assert!( + matches!(input.last(), Some(Segment::Text { content }) if content == " keep this prose") + ); +} + +#[test] +fn feature_invocation_staging_deletion_and_late_completion_do_not_resurrect_chips() { + for delete in [0, 1, 2] { + let invocation = parse_feature_invocation("/run(x)", 0, &descriptor(), "invoke") + .unwrap() + .invocation; + let mut input = InputBuffer::new(); + input.replace_with_segments(&[Segment::FeatureInvoke { invocation }]); + input.stage_attachment("invoke", "stage".into(), "a.txt".into()); + match delete { + 0 => input.delete_before(), + 1 => { + input.move_left(); + input.delete_after(); + } + _ => input.delete_word_before(), + } + assert_eq!(input.take_removed_attachment_stages(), vec!["stage"]); + assert!(!input.finish_attachment_stage("stage", Some(uploaded_file()))); + assert!(input.is_empty()); + } +} + +#[test] +fn feature_invocation_typed_history_draft_transport_and_rewind_restore_without_flattening() { + let invocation = parse_feature_invocation( + "/run(\"資料/a b\", mode=safe)", + 0, + &descriptor(), + "stable-id", + ) + .unwrap() + .invocation; + let segments = vec![ + Segment::text("before "), + Segment::FeatureInvoke { invocation }, + Segment::UploadedFile { + file: uploaded_file(), + }, + Segment::Paste { + id: 2, + chars: 4, + lines: 1, + content: "data".into(), + }, + Segment::FileRef { + path: "src/".into(), + }, + Segment::Flow { + selector: "review".into(), + }, + ]; + let mut app = App::new("test".into()); + app.input.replace_with_segments(&segments); + let method = app.submit_input().unwrap(); + app.restore_unsent_run(&method); + assert_eq!(app.input.submit_segments(), segments); + app.input.clear(); + let draft = vec![ + Segment::text("draft "), + Segment::UploadedFile { + file: uploaded_file(), + }, + ]; + app.input.replace_with_segments(&draft); + app.input.move_home(); + assert!(app.browse_input_history_older()); + assert_eq!(app.input.submit_segments(), segments); + assert!(app.browse_input_history_newer()); + assert_eq!(app.input.submit_segments(), draft); + app.input.clear(); + app.handle_worker_event(Event::RewindApplied { + session: SessionSnapshot { + entries: vec![], + pending_submissions: PendingSubmissionsSnapshot::default(), + }, + input: segments.clone(), + summary: RewindSummary { + truncated_to_entries: 1, + discarded_entries: 2, + tool_side_effect_warning: false, + }, + }); + assert_eq!(app.input.submit_segments(), segments); +} + +#[test] +fn feature_invocation_stale_argument_context_and_unvalidated_discovery_entries_are_ignored() { + let mut app = App::new("test".into()); + select(&mut app, descriptor(), "run"); + let state = app.completion.as_ref().unwrap(); + let prefix = state.prefix.clone(); + let mut context = state.context.clone().unwrap(); + context.invocation = FeatureInvocationIdentity("plugin:other/run".into()); + app.handle_worker_event(Event::Completions { + request_id: app.completion_request_id(), + kind: CompletionKind::FeatureArgument, + prefix, + context: Some(context), + entries: vec![CompletionEntry { + value: "unsafe".into(), + ..Default::default() + }], + }); + assert!(app.completion.as_ref().unwrap().entries.is_empty()); + let mut app = App::new("test".into()); + app.insert_char('/'); + app.refresh_completion(); + let mut invalid = descriptor(); + invalid.identity = FeatureInvocationIdentity("unqualified".into()); + app.handle_worker_event(Event::Completions { + request_id: app.completion_request_id(), + kind: CompletionKind::Feature, + prefix: String::new(), + context: None, + entries: vec![CompletionEntry { + value: "run".into(), + invocation: Some(invalid), + ..Default::default() + }], + }); + assert!(app.apply_completion_text().is_none()); + assert_eq!(app.input.plain_text(), "/"); +} + +#[test] +fn feature_invocation_multiple_selected_calls_preserve_prose_and_source_qualified_identity() { + let mut app = App::new("test".into()); + select(&mut app, descriptor(), "run"); + app.input.insert_str("x) prose "); + let mut other = descriptor(); + other.identity = FeatureInvocationIdentity("plugin:other/second".into()); + other.name = "second".into(); + other.aliases.clear(); + select(&mut app, other.clone(), "second"); + app.input.insert_str("\"y z\") done"); + assert!(app.submit_input().is_none()); + let segments = app.input.submit_segments(); + assert!( + matches!(&segments[0], Segment::FeatureInvoke { invocation } if invocation.identity == descriptor().identity) + ); + assert!(matches!(&segments[1], Segment::Text { content } if content == " prose ")); + assert!( + matches!(&segments[2], Segment::FeatureInvoke { invocation } if invocation.identity == other.identity) + ); + assert!(matches!(&segments[3], Segment::Text { content } if content == " done")); +} + +#[test] +fn feature_invocation_range_replacement_reports_each_removed_upload_and_stage() { + let invocation = parse_feature_invocation("/run(x)", 0, &descriptor(), "id") + .unwrap() + .invocation; + let mut input = InputBuffer::new(); + input.replace_with_segments(&[ + Segment::UploadedFile { + file: uploaded_file(), + }, + Segment::text(" "), + Segment::FeatureInvoke { invocation }, + ]); + input.stage_attachment("id", "stage".into(), "a.txt".into()); + input.replace_with_text_at(0, "replacement"); + assert_eq!(input.take_removed_uploaded_files(), vec![uploaded_file()]); + assert_eq!(input.take_removed_attachment_stages(), vec!["stage"]); + assert_eq!(input.plain_text(), "replacement"); +} + +#[test] +fn feature_invocation_argument_positions_ignore_previous_named_arguments() { + let mut input = InputBuffer::new(); + input.insert_str("/run(mode=safe, \"資料/a"); + input.select_feature_invocation(0, descriptor()); + let (_, _, prefix, context) = input + .pending_feature_argument_completion(&[descriptor()]) + .unwrap(); + assert_eq!(context.argument.as_deref(), Some("path")); + assert_eq!(prefix, "資料/a"); +} + +#[test] +fn feature_invocation_name_completion_replaces_suffix_at_middle_cursor() { + let mut app = App::new("test".into()); + app.input.insert_str("/run() prose"); + app.input.move_home(); + app.input.move_right(); + app.input.move_right(); + app.refresh_completion(); + app.handle_worker_event(Event::Completions { + request_id: app.completion_request_id(), + kind: CompletionKind::Feature, + prefix: "r".into(), + context: None, + entries: vec![CompletionEntry { + value: "run".into(), + invocation: Some(descriptor()), + ..Default::default() + }], + }); + app.apply_completion_text(); + assert_eq!(app.input.plain_text(), "/run() prose"); +} + +#[test] +fn feature_invocation_discovery_expands_aliases_from_host_metadata() { + let mut app = App::new("test".into()); + app.input.insert_str("/exe"); + app.refresh_completion(); + app.handle_worker_event(Event::Completions { + request_id: app.completion_request_id(), + kind: CompletionKind::Feature, + prefix: "exe".into(), + context: None, + entries: vec![CompletionEntry { + value: "run".into(), + invocation: Some(descriptor()), + ..Default::default() + }], + }); + assert_eq!(app.completion.as_ref().unwrap().entries[0].value, "execute"); + app.apply_completion_text(); + assert_eq!(app.input.plain_text(), "/execute("); +} + +#[test] +fn feature_invocation_named_arguments_are_suggested_alongside_positional_values() { + let mut app = App::new("test".into()); + select(&mut app, descriptor(), "run"); + app.input.insert_str("pa"); + app.refresh_completion(); + let state = app.completion.as_ref().unwrap(); + app.handle_worker_event(Event::Completions { + request_id: app.completion_request_id(), + kind: state.kind, + prefix: state.prefix.clone(), + context: state.context.clone(), + entries: Vec::new(), + }); + assert_eq!(app.completion.as_ref().unwrap().entries[0].value, "path="); + app.apply_completion_text(); + assert_eq!(app.input.plain_text(), "/run(path="); + app.input.insert_str("\"資料/path\")"); + assert!(app.submit_input().is_none()); + assert!(matches!( + app.input.submit_segments().as_slice(), + [Segment::FeatureInvoke { .. }] + )); +} diff --git a/crates/tui/src/lib.rs b/crates/tui/src/lib.rs index 5bc9ca090..359063b6e 100644 --- a/crates/tui/src/lib.rs +++ b/crates/tui/src/lib.rs @@ -13,6 +13,8 @@ mod console; mod e2e_observer; mod inline_terminal; mod input; +#[cfg(test)] +mod invocation_tests; pub mod keys; mod markdown; mod scroll; diff --git a/crates/tui/src/ui.rs b/crates/tui/src/ui.rs index e7f453b57..de5492ac3 100644 --- a/crates/tui/src/ui.rs +++ b/crates/tui/src/ui.rs @@ -366,6 +366,23 @@ fn task_status_mark(status: TaskStatus) -> (&'static str, Style) { /// taller than that single row); `Clear` blanks the cells first so /// underlying text doesn't bleed through. The popup width matches the /// widest visible label, capped at the input-area width. +fn completion_entry_label(entry: &CompletionEntry) -> String { + if let Some(invocation) = &entry.invocation { + format!( + "/{} — {} — {}", + entry.value, + invocation.description, + invocation.usage() + ) + } else { + let suffix = if entry.is_dir { "/" } else { "" }; + match &entry.description { + Some(description) => format!("{}{suffix} — {description}", entry.value), + None => format!("{}{suffix}", entry.value), + } + } +} + fn draw_completion_popup(frame: &mut Frame, state: &CompletionState, input_area: Rect) { let entries = &state.entries; if entries.is_empty() || input_area.y == 0 { @@ -380,16 +397,10 @@ fn draw_completion_popup(frame: &mut Frame, state: &CompletionState, input_area: }; let view_end = (view_start + visible).min(entries.len()); - let label_for = |entry: &CompletionEntry| { - let mut s = entry.value.clone(); - if entry.is_dir { - s.push('/'); - } - s - }; + let label_for = |entry: &CompletionEntry| completion_entry_label(entry); let max_label = entries[view_start..view_end] .iter() - .map(|e| label_for(e).chars().count() as u16) + .map(|e| label_for(e).width().min(u16::MAX as usize) as u16) .max() .unwrap_or(0); let popup_w = max_label.saturating_add(2).min(input_area.width).max(1); @@ -1341,6 +1352,10 @@ fn chip_span_for(seg: &Segment, fallback: Style) -> (Style, String) { ), ), Segment::FileRef { path } => (Style::default().fg(Color::Cyan), format!("@{path}")), + Segment::FeatureInvoke { invocation } => ( + Style::default().fg(Color::Yellow), + invocation.display_input(), + ), Segment::Flow { selector } => ( Style::default().fg(Color::Yellow), format!("[Flow: {selector}]"), @@ -1375,6 +1390,7 @@ fn segment_display_text(seg: &Segment) -> String { file.availability.as_str() ), Segment::FileRef { path } => format!("@{path}"), + Segment::FeatureInvoke { invocation } => invocation.display_input(), Segment::Flow { selector } => format!("[Flow: {selector}]"), Segment::Unknown => "[unknown segment]".to_owned(), } @@ -2186,6 +2202,28 @@ mod tests { }); } + #[test] + fn feature_invocation_completion_labels_include_alias_description_usage_and_argument_help() { + let descriptor = crate::invocation_tests::descriptor(); + let entry = CompletionEntry { + value: "execute".into(), + invocation: Some(descriptor.clone()), + ..Default::default() + }; + let label = completion_entry_label(&entry); + assert!(label.starts_with("/execute")); + assert!(label.contains(&descriptor.description)); + assert!(label.contains(&descriptor.usage())); + assert_eq!( + completion_entry_label(&CompletionEntry { + value: "mode=".into(), + description: Some("execution mode".into()), + ..Default::default() + }), + "mode= — execution mode" + ); + } + #[test] fn run_status_line_matches_console_metrics_and_spinner_frame() { let now = Instant::now(); diff --git a/crates/workdir/CHECKOUT_PROVIDER.md b/crates/workdir/CHECKOUT_PROVIDER.md new file mode 100644 index 000000000..0e7a9827b --- /dev/null +++ b/crates/workdir/CHECKOUT_PROVIDER.md @@ -0,0 +1,167 @@ +# Request-time checkout provider boundary + +`workdir::checkout` (also crate-root reexports) exposes `CheckoutObservation`, +`CheckoutRequest`, `CheckoutOperation`, `CheckoutOutput`, and `CheckoutResult`. +`WorkdirSession::checkout_observe(path)` defaults to unsupported; execution also +fails closed by default. Observation is metadata-only, with no recursive tree +walk. Directory discovery/search remains the ordinary bounded List/Glob/Grep +provider engine. GrepResult.paths carries exact typed provider paths appearing +in the returned report, respecting its offset/limit/truncation; filenames are +never parsed from rendered text. List/Glob/Grep skip unrepresentable non-UTF8 +paths, preserving valid Unicode exactly without normalization. Native callers supply their own attachment generation/namespace +fence; this provider validator is not a substitute for that fence. + +* Read: line `offset`, `limit`, `max_bytes`; same `ReadResult` (bytes, start line, + total lines, full-source content hash, truncation) as normal Tools. Formatting + line numbers and recording the returned hash in caller read-history are caller + responsibilities. The validator is **not** permission to bypass read-before-write. +* Write: replace existing file with `content` and required `expected_hash`. +* Edit: existing UTF-8 file, unique `old_string` unless `replace_all`, required + `expected_hash`; same match/replacement/result-size checks as Tools. +* Create: target must be an observed directory; `path` is **checkout-root + relative**, strictly below target, not target-relative. No existing file can + be overwritten. Intermediate directories and destination must be writable. +* Every result observation describes the bound request target. Create returns + the updated **parent directory** observation from the retained pinned target, + not the created destination. Read returns the read state's observation. + +## Shared implementation + +`fs_operation::CheckedTarget` is an `FsAccessPolicy` adapter around pinned target, +save-parent, and provider-root descriptors. It calls the ordinary +`run_read_bounded`, `run_edit`, and `run_write` functions, including full-source +hashing, UTF-8/string matching, bounds, and atomic temp-save primitive; there is +no second content-operation engine. Shared text reads now check descriptor +metadata before/after reading. Validator bytes encode device/inode/size/mode/ +mtime/ctime with nanoseconds, excluding atime and host paths; same-content inode +replacement is distinguishable. Successful native Read rechecks the named target +and pinned descriptor before returning hash and observation. + +Local sessions serialize checked observe/execute and ordinary trait Read/Write/ +Edit using one nonqueueing session lock. Expected validator comparison happens +inside that boundary. The adapter retains handles across reading and save, +rechecks named target + save-parent identity at save boundary, and writes a new +temporary inode then atomically renames it, isolating existing hardlink aliases. +External sessions retain the approved descriptor root after pathname rename or +replacement. Checkout resolution rejects symlinks, special files, mount crossings, +absolute/traversal paths and unavailable descriptor primitives, rather than +weakening confinement. This currently requires Linux `openat2` and `/proc/self/fd`. +Ordinary Tools preserve their existing explicit symlink-policy behavior. + +Scoped sessions use their existing scope mutex, current validity/source +capability checks, provider authorization and write leases; Create also checks +intermediate paths. Observation capabilities are attenuated per path and live +write leases, never broadened to the whole attachment. ReadOnly removes mutation +capabilities and rejects mutations. Routed operations use existing active-op +accounting and detach fencing. Remote HTTP forwards exact operation/result types +and validates returned operation/path pairing. External DTOs validate checkout +path/validator/content/result bounds. No provider endpoint or host root is added +to the observation. + +## Effects and limits + +Absent paths during checkout observation return `NotFound` (including a missing +ancestor or a non-directory ancestor); native discovery can return no route. +An execution against a previously observed target that disappeared instead +returns `Conflict`, consistently at pinning, shared-engine, and final-check +boundaries. Conflict means stale validator/hash/name or existing Create destination before +save effects. `WorkdirError::OutcomeUnknown` has a separate sanitized wire code +`outcome_unknown` (HTTP 500), never an automatic retry. Partial parent creation, +possible commit failure, or failed post-save observation is outcome unknown. +Remote lost/malformed/mismatched mutation responses are also outcome unknown. +Create failures conservatively report unknown once parent creation may begin, +even if a particular failure happened without a persisted effect. + +Bounds: path <=4096 UTF-8 bytes/128 components, validator <=256 bytes; Read source +64 MiB, retained response <=1 MiB (or stricter provider read limits), Write/Edit +preimage and result <=8 MiB (or stricter provider limit), Edit replacements +<=10,000 (or stricter provider limit). One checked operation per Local session, +with bounded retained handles and no waiting queue. Cancellation/deadline checks +use the existing provider guard, or a 30-second cooperative guard for Local +checkout operations. Blocking kernel/filesystem calls are not preempted; this +is **not** a hard wall-time guarantee for hung storage. Transport request-body +bounds before serde allocation remain the hosting transport's responsibility. + +### External-editor limitations (not a magic advisory-lock guarantee) + +No advisory `flock` is claimed to exclude arbitrary editors. Provider-session +serialization does not lock other sessions/processes. Metadata checks detect +concurrent reads/edits up to each check, but are not a transactional snapshot +against privileged actors forging metadata. Linux offers no atomic +compare-identity-and-replace-by-name primitive here: an uncooperative external +rename/change between the final validator check and rename can still race the +save. The pinned handles prevent following replacements/symlinks into arbitrary +objects, and deterministic substitution before the save boundary is rejected, +but the final syscall gap is an explicit limitation. Callers must not interpret +this as linearizability against all external filesystem mutations. Likewise, +post-save observation is the then-current named state, not an exclusion of an +external edit after save. Crash durability of parent directories is not claimed. + +Host generation reuse, native permission identity, read-history updates, model +Operation publication, transport hosting (Runtime/Backend/External), and durable +Ticket/MR review remain the parent integration's responsibilities. + +## Scoped checkout search boundary + +`CheckoutSearchRequest::new(CheckoutSearchOperation::{List, Glob, Grep}(request))` +uses the current session's logical root as its result frame. The typed result is +`CheckoutSearchResult::{List, Glob, Grep}`. The request's internal context fields +are `scope_layers: Vec>` and `output_root: WorkdirPath`. +Rules union within each layer; layers intersect with each other, the current +provider authority, and the output-root boundary. Wrappers contribute their own +scope; caller-supplied layers can only narrow, never replace it. Each wrapper +translates all incoming coordinates once and forwards the composed output root. +Provider-rendered Grep text and every typed result path are already relative to +that final root; wrappers must not prefix cwd onto returned paths again. + +Local/External use the same shared `run_list`, `run_glob`, `run_grep` engines. +`SearchAccess` gates metadata/content opens and enumeration. Shared List checks +`can_enumerate_directory`; Glob/Grep use the shared bounded descriptor walk and +owned `traversal_filter` policy to prune before descending/opening directories. +The walk anchors at the actual search directory, not at a broader provider +root. It counts rejected entries too, checks cancellation while scanning them, +and limits descent to 128 levels (bounded retained directory handles). Existing +provider-owned ignore loading, glob/type selection, grep context/offset/limit, +rendering, and source/result limits remain in their shared engines. Unreadable +ignore files are not opened to discover out-of-scope contents. + +Descriptor search loads `.ignore`, `.gitignore` and an authorized directory's +`.git/info/exclude` through the same `FsAccessPolicy` opens as other sources. +It preserves anchored rules, negation, nearest-directory precedence within a +source class, `.ignore` precedence over Git rules, and nested repository +boundaries. Ignored directories are pruned before enumeration or loading their +own rules; explicitly selected ignored search roots are still traversable. +Only the current ancestor matchers are retained, never a second tree/store. +Glob requires an authorized `.git` directory/file marker for Git rules. Grep +retains the existing descriptor-provider contract of applying `.gitignore` +without a marker and rejecting unavailable/symlinked ignore opens; Glob skips +unavailable ignore sources like its ordinary walker. Scope-denied sources are +skipped by both before open. Bounds/cancellation failures are never retried. + +Configuration discovery cannot expand provider authority. Ambient ancestors +outside the search result/provider root, global Git configuration/ignore files, +and linked-worktree `.git`/`commondir` administration pointers are not followed. +A `.git` file activates repository matching without treating it as a directory +or opening the host path it contains. In-root readable ignore rules remain +active, including inherited rules above a nested base. Descriptor ignore text +must be UTF-8, bounded to 1 MiB/file, 16 KiB/line and 8 MiB aggregate per search; +limit violations fail rather than returning silently incomplete filtering. +Grep's separate candidate-content budget remains 64 MiB. Ordinary unscoped +path-backed Tools keep their original discovery policy. + +Scoped ordinary trait List/Glob/Grep use this boundary as well, returning +session-logical coordinates. Unscoped normal Tools continue using their original +trait/provider behavior. A provider without checkout search support returns +UnsupportedOperation; there is no broad traversal fallback. Native/scoped +searches intentionally fail closed on symlink paths, including logical-policy +aliases which normal unscoped Tools may follow. A visible symlink may therefore +cause a native List failure rather than expanding authority. Descriptor support +is currently Linux-only. Search remains request-time traversal, not a snapshot +or transactional exclusion of external editors. + +HTTP operation/result variant: `CheckoutSearch`. External DTO validation rejects +absolute/out-of-root paths, empty scope layers, oversized paths/patterns/context/ +results and invalid result paths. Generic request ceilings are 16 layers and 256 +aggregate rules; External transport further limits aggregate rules to 64. Search +result limits follow the existing shared List/Glob/Grep wire bounds. Runtime and +Backend forwarding outside this crate remain parent integration work. diff --git a/crates/workdir/src/checkout.rs b/crates/workdir/src/checkout.rs new file mode 100644 index 000000000..f524ae61c --- /dev/null +++ b/crates/workdir/src/checkout.rs @@ -0,0 +1,1015 @@ +//! Request-time checkout contracts. Validators are opaque provider observations, +//! not content hashes. Callers must separately retain Read's content hash for +//! read-before-mutation. Create destinations are checkout-root relative. +use crate::{ + ContentHash, EditResult, EntryKind, ReadResult, WorkdirError, WorkdirPath, + WorkdirSessionCapabilities, WorkdirSessionCapability, WriteResult, +}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CheckoutObservation { + pub path: WorkdirPath, + pub kind: EntryKind, + pub size: u64, + pub validator: Vec, + pub capabilities: WorkdirSessionCapabilities, +} +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CheckoutRequest { + pub target: WorkdirPath, + pub validator: Vec, + pub operation: CheckoutOperation, +} +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(tag = "operation", rename_all = "snake_case", deny_unknown_fields)] +pub enum CheckoutOperation { + Read { + offset: usize, + limit: usize, + max_bytes: usize, + }, + Write { + content: Vec, + expected_hash: ContentHash, + }, + Edit { + old_string: String, + new_string: String, + replace_all: bool, + expected_hash: ContentHash, + }, + Create { + path: WorkdirPath, + content: Vec, + }, +} +impl CheckoutOperation { + pub fn capability(&self) -> WorkdirSessionCapability { + match self { + Self::Read { .. } => WorkdirSessionCapability::Read, + Self::Edit { .. } => WorkdirSessionCapability::Edit, + Self::Write { .. } | Self::Create { .. } => WorkdirSessionCapability::Write, + } + } +} +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde( + tag = "operation", + content = "result", + rename_all = "snake_case", + deny_unknown_fields +)] +pub enum CheckoutOutput { + Read(ReadResult), + Write(WriteResult), + Edit(EditResult), +} +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CheckoutResult { + /// Post-operation state of the bound request target. For Create this is the + /// parent directory, not the destination file (whose path the caller owns). + pub observation: CheckoutObservation, + pub output: CheckoutOutput, +} + +/// One provider-side traversal. Rules union within a layer and intersect across +/// layers, current provider authority and output_root. Wrappers add layers; +/// callers normally use new() and never supply provider coordinates. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct CheckoutSearchRequest { + pub operation: CheckoutSearchOperation, + pub scope_layers: Vec>, + pub output_root: WorkdirPath, +} +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde( + tag = "operation", + content = "request", + rename_all = "snake_case", + deny_unknown_fields +)] +pub enum CheckoutSearchOperation { + List(crate::ListRequest), + Glob(crate::GlobRequest), + Grep(crate::GrepRequest), +} +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[serde( + tag = "operation", + content = "result", + rename_all = "snake_case", + deny_unknown_fields +)] +pub enum CheckoutSearchResult { + List(crate::ListResult), + Glob(crate::GlobResult), + Grep(crate::GrepResult), +} +impl CheckoutSearchOperation { + pub fn path(&self) -> &WorkdirPath { + match self { + Self::List(r) => &r.path, + Self::Glob(r) => &r.path, + Self::Grep(r) => &r.path, + } + } + pub fn path_mut(&mut self) -> &mut WorkdirPath { + match self { + Self::List(r) => &mut r.path, + Self::Glob(r) => &mut r.path, + Self::Grep(r) => &mut r.path, + } + } + pub fn capability(&self) -> WorkdirSessionCapability { + match self { + Self::List(_) => WorkdirSessionCapability::Read, + Self::Glob(_) => WorkdirSessionCapability::Glob, + Self::Grep(_) => WorkdirSessionCapability::Grep, + } + } +} +impl CheckoutSearchRequest { + pub fn new(operation: CheckoutSearchOperation) -> Self { + Self { + operation, + scope_layers: Vec::new(), + output_root: WorkdirPath::root(), + } + } + pub(crate) fn validate(&self) -> Result<(), WorkdirError> { + validate_path(&self.output_root)?; + validate_path(self.operation.path())?; + if !std::path::Path::new(self.operation.path().as_str()) + .starts_with(self.output_root.as_str()) + { + return Err(WorkdirError::Denied( + "checkout search must stay beneath its output root".into(), + )); + } + if self.scope_layers.len() > 16 + || self.scope_layers.iter().map(Vec::len).sum::() > 256 + || self.scope_layers.iter().any(Vec::is_empty) + { + return Err(WorkdirError::InvalidArgument( + "checkout search scope exceeds provider bounds".into(), + )); + } + for layer in &self.scope_layers { + for rule in layer { + validate_path(&rule.target)?; + } + } + let valid = match &self.operation { + CheckoutSearchOperation::List(r) => { + r.limit <= crate::external::MAX_EXTERNAL_RESULT_ITEMS + } + CheckoutSearchOperation::Glob(r) => { + r.limit <= crate::external::MAX_EXTERNAL_RESULT_ITEMS && r.pattern.len() <= 4096 + } + CheckoutSearchOperation::Grep(r) => { + r.limit <= crate::external::MAX_EXTERNAL_RESULT_ITEMS + && r.offset <= 1_000_000 + && r.pattern.len() <= 4096 + && r.glob.as_ref().is_none_or(|s| s.len() <= 4096) + && r.file_type.as_ref().is_none_or(|s| s.len() <= 128) + && r.before_context <= 100 + && r.after_context <= 100 + } + }; + if valid { + Ok(()) + } else { + Err(WorkdirError::InvalidArgument( + "checkout search exceeds provider bounds".into(), + )) + } + } +} + +pub(crate) fn validate_path(path: &WorkdirPath) -> Result<(), WorkdirError> { + WorkdirPath::new(path.as_str())?; + if path.as_str().len() > 4096 || std::path::Path::new(path.as_str()).components().count() > 128 + { + return Err(WorkdirError::InvalidArgument( + "checkout path exceeds provider bounds".into(), + )); + } + Ok(()) +} + +/// Check strict containment without converting a directory-relative path into +/// ambient authority. Absolute paths are forbidden even if otherwise scoped. +pub fn create_relative_path( + target: &WorkdirPath, + path: &WorkdirPath, +) -> Result { + validate_path(target)?; + validate_path(path)?; + let target = std::path::Path::new(target.as_str()); + let path = std::path::Path::new(path.as_str()); + if target.is_absolute() || path.is_absolute() { + return Err(WorkdirError::InvalidArgument( + "checkout paths must be root relative".into(), + )); + } + let relative = path.strip_prefix(target).map_err(|_| { + WorkdirError::InvalidArgument( + "Create destination must be strictly beneath its target".into(), + ) + })?; + if relative.as_os_str().is_empty() + || relative + .components() + .any(|c| !matches!(c, std::path::Component::Normal(_))) + { + return Err(WorkdirError::InvalidArgument( + "Create destination must be strictly beneath its target".into(), + )); + } + Ok(relative.to_path_buf()) +} + +#[cfg(all(test, target_os = "linux"))] +mod tests { + use super::*; + use crate::{LocalWorkdirSession, WorkdirSession}; + use std::sync::Arc; + fn path(p: &str) -> WorkdirPath { + WorkdirPath::new(p).unwrap() + } + fn session(root: &std::path::Path) -> LocalWorkdirSession { + LocalWorkdirSession::new(manifest::Scope::writable(root).unwrap(), root.to_path_buf()) + } + async fn checked_read( + session: &dyn WorkdirSession, + observation: CheckoutObservation, + ) -> CheckoutResult { + session + .checkout_execute(CheckoutRequest { + target: observation.path, + validator: observation.validator, + operation: CheckoutOperation::Read { + offset: 0, + limit: 20, + max_bytes: 1024, + }, + }) + .await + .unwrap() + } + fn hash(result: &CheckoutResult) -> ContentHash { + match &result.output { + CheckoutOutput::Read(r) => r.content_hash, + _ => panic!("not Read"), + } + } + #[tokio::test] + async fn checkout_read_edit_write_create_share_hash_and_string_contracts() { + let root = tempfile::tempdir().unwrap(); + std::fs::write(root.path().join("a"), "alpha\nbeta\n").unwrap(); + let session = session(root.path()); + let read = checked_read(&session, session.checkout_observe(path("a")).await.unwrap()).await; + let edited = session + .checkout_execute(CheckoutRequest { + target: path("a"), + validator: read.observation.validator.clone(), + operation: CheckoutOperation::Edit { + old_string: "beta".into(), + new_string: "gamma".into(), + replace_all: false, + expected_hash: hash(&read), + }, + }) + .await + .unwrap(); + assert!(matches!( + edited.output, + CheckoutOutput::Edit(EditResult { + replacements: 1, + .. + }) + )); + assert_eq!( + std::fs::read_to_string(root.path().join("a")).unwrap(), + "alpha\ngamma\n" + ); + let read = checked_read(&session, edited.observation).await; + // A validator alone does not bypass read-history's expected content hash. + assert!(matches!( + session + .checkout_execute(CheckoutRequest { + target: path("a"), + validator: read.observation.validator.clone(), + operation: CheckoutOperation::Write { + content: b"bad".to_vec(), + expected_hash: [0; 32] + } + }) + .await, + Err(WorkdirError::Conflict(_)) + )); + let written = session + .checkout_execute(CheckoutRequest { + target: path("a"), + validator: read.observation.validator.clone(), + operation: CheckoutOperation::Write { + content: b"saved".to_vec(), + expected_hash: hash(&read), + }, + }) + .await + .unwrap(); + assert_ne!(written.observation.validator, read.observation.validator); + let directory = session.checkout_observe(WorkdirPath::root()).await.unwrap(); + let created = session + .checkout_execute(CheckoutRequest { + target: directory.path, + validator: directory.validator, + operation: CheckoutOperation::Create { + path: path("nested/new"), + content: b"new".to_vec(), + }, + }) + .await + .unwrap(); + assert_eq!(created.observation.path, WorkdirPath::root()); + assert_eq!(created.observation.kind, EntryKind::Directory); + assert_eq!( + created.observation, + session.checkout_observe(WorkdirPath::root()).await.unwrap() + ); + assert_eq!( + std::fs::read(root.path().join("nested/new")).unwrap(), + b"new" + ); + assert!(matches!( + created.output, + CheckoutOutput::Write(WriteResult { created: true, .. }) + )); + } + #[tokio::test] + async fn checkout_observe_missing_routes_are_not_found_but_deleted_execution_is_conflict() { + let root = tempfile::tempdir().unwrap(); + std::fs::write(root.path().join("file"), "original").unwrap(); + let local = session(root.path()); + let external = LocalWorkdirSession::external_read_write( + crate::Workdir::new("external-missing"), + root.path(), + crate::BoundedReadLimits::EXTERNAL_DEFAULT, + ) + .unwrap(); + let scoped = crate::WorkdirToolBroker::new(Arc::new(local.clone())); + for provider in [ + &local as &dyn WorkdirSession, + &external, + &*scoped.tool_session(), + ] { + for missing in ["missing", "missing-parent/child", "file/child"] { + let error = provider.checkout_observe(path(missing)).await.unwrap_err(); + assert!( + matches!(&error, WorkdirError::NotFound(logical) if logical == std::path::Path::new(missing)), + "{missing}: {error:?}" + ); + assert_eq!( + crate::http::WorkdirTransportError::from_workdir_error(&error).code, + crate::http::WorkdirTransportErrorCode::NotFound + ); + } + std::fs::write(root.path().join("observed"), "original").unwrap(); + let read = checked_read( + provider, + provider.checkout_observe(path("observed")).await.unwrap(), + ) + .await; + std::fs::remove_file(root.path().join("observed")).unwrap(); + for operation in [ + CheckoutOperation::Read { + offset: 0, + limit: 10, + max_bytes: 100, + }, + CheckoutOperation::Write { + content: b"new".to_vec(), + expected_hash: hash(&read), + }, + CheckoutOperation::Edit { + old_string: "original".into(), + new_string: "new".into(), + replace_all: false, + expected_hash: hash(&read), + }, + ] { + let error = provider + .checkout_execute(CheckoutRequest { + target: path("observed"), + validator: read.observation.validator.clone(), + operation, + }) + .await + .unwrap_err(); + assert!(matches!(error, WorkdirError::Conflict(_)), "{error:?}"); + assert_eq!( + crate::http::WorkdirTransportError::from_workdir_error(&error).code, + crate::http::WorkdirTransportErrorCode::Conflict + ); + } + assert!(matches!( + provider.checkout_observe(path("observed")).await, + Err(WorkdirError::NotFound(_)) + )); + std::fs::create_dir(root.path().join("directory")).unwrap(); + let directory = provider.checkout_observe(path("directory")).await.unwrap(); + std::fs::remove_dir(root.path().join("directory")).unwrap(); + let error = provider + .checkout_execute(CheckoutRequest { + target: directory.path, + validator: directory.validator, + operation: CheckoutOperation::Create { + path: path("directory/new"), + content: b"new".to_vec(), + }, + }) + .await + .unwrap_err(); + assert!(matches!(error, WorkdirError::Conflict(_))); + assert!(!root.path().join("directory").exists()); + // An observed parent replaced with a regular file makes the old + // descendant execution stale, while discovery reports no route. + std::fs::create_dir(root.path().join("parent")).unwrap(); + std::fs::write(root.path().join("parent/child"), "original").unwrap(); + let child = provider + .checkout_observe(path("parent/child")) + .await + .unwrap(); + std::fs::remove_dir_all(root.path().join("parent")).unwrap(); + std::fs::write(root.path().join("parent"), "replacement").unwrap(); + assert!(matches!( + provider.checkout_observe(path("parent/child")).await, + Err(WorkdirError::NotFound(_)) + )); + assert!(matches!( + provider + .checkout_execute(CheckoutRequest { + target: child.path, + validator: child.validator, + operation: CheckoutOperation::Read { + offset: 0, + limit: 10, + max_bytes: 100 + } + }) + .await, + Err(WorkdirError::Conflict(_)) + )); + std::fs::remove_file(root.path().join("parent")).unwrap(); + } + } + + #[tokio::test] + async fn checkout_inode_substitution_and_stale_content_are_conflicts() { + let root = tempfile::tempdir().unwrap(); + std::fs::write(root.path().join("a"), "same").unwrap(); + let session = session(root.path()); + let original = + checked_read(&session, session.checkout_observe(path("a")).await.unwrap()).await; + std::fs::rename(root.path().join("a"), root.path().join("old")).unwrap(); + std::fs::write(root.path().join("a"), "same").unwrap(); + let new = session.checkout_observe(path("a")).await.unwrap(); + assert_ne!(original.observation.validator, new.validator); + assert!(matches!( + session + .checkout_execute(CheckoutRequest { + target: path("a"), + validator: original.observation.validator.clone(), + operation: CheckoutOperation::Write { + content: b"wrong".to_vec(), + expected_hash: hash(&original) + } + }) + .await, + Err(WorkdirError::Conflict(_)) + )); + assert_eq!( + std::fs::read_to_string(root.path().join("a")).unwrap(), + "same" + ); + std::fs::write(root.path().join("a"), "changed").unwrap(); + assert!(matches!( + session + .checkout_execute(CheckoutRequest { + target: path("a"), + validator: new.validator, + operation: CheckoutOperation::Read { + offset: 0, + limit: 20, + max_bytes: 100 + } + }) + .await, + Err(WorkdirError::Conflict(_)) + )); + } + #[tokio::test] + async fn checkout_hardlink_save_isolates_alias_and_rejects_symlinks_special_files() { + let root = tempfile::tempdir().unwrap(); + std::fs::write(root.path().join("a"), "original").unwrap(); + std::fs::hard_link(root.path().join("a"), root.path().join("alias")).unwrap(); + std::os::unix::fs::symlink("a", root.path().join("link")).unwrap(); + let _socket = std::os::unix::net::UnixListener::bind(root.path().join("socket")).unwrap(); + let session = session(root.path()); + assert!(session.checkout_observe(path("link")).await.is_err()); + assert!(session.checkout_observe(path("socket")).await.is_err()); + let read = checked_read(&session, session.checkout_observe(path("a")).await.unwrap()).await; + session + .checkout_execute(CheckoutRequest { + target: path("a"), + validator: read.observation.validator.clone(), + operation: CheckoutOperation::Write { + content: b"saved".to_vec(), + expected_hash: hash(&read), + }, + }) + .await + .unwrap(); + assert_eq!( + std::fs::read_to_string(root.path().join("alias")).unwrap(), + "original" + ); + } + #[tokio::test] + async fn checkout_create_is_strictly_beneath_directory_and_does_not_clobber() { + let root = tempfile::tempdir().unwrap(); + std::fs::create_dir(root.path().join("d")).unwrap(); + std::fs::write(root.path().join("d/existing"), "keep").unwrap(); + let session = session(root.path()); + for destination in ["sibling", "d", "directory/file"] { + let observation = session.checkout_observe(path("d")).await.unwrap(); + assert!( + session + .checkout_execute(CheckoutRequest { + target: path("d"), + validator: observation.validator, + operation: CheckoutOperation::Create { + path: path(destination), + content: vec![1] + } + }) + .await + .is_err() + ); + } + assert!(WorkdirPath::new("d/../sibling").is_err()); + assert!( + create_relative_path(&path("d"), &WorkdirPath::new_scoped("/d/new").unwrap()).is_err() + ); + let observation = session.checkout_observe(path("d")).await.unwrap(); + assert!( + session + .checkout_execute(CheckoutRequest { + target: path("d"), + validator: observation.validator, + operation: CheckoutOperation::Create { + path: path("d/existing"), + content: vec![1] + } + }) + .await + .is_err() + ); + assert_eq!( + std::fs::read_to_string(root.path().join("d/existing")).unwrap(), + "keep" + ); + let observation = session.checkout_observe(path("d")).await.unwrap(); + std::fs::write(root.path().join("d/concurrent"), "created externally").unwrap(); + assert!(matches!( + session + .checkout_execute(CheckoutRequest { + target: path("d"), + validator: observation.validator, + operation: CheckoutOperation::Create { + path: path("d/new"), + content: vec![1] + } + }) + .await, + Err(WorkdirError::Conflict(_)) + )); + assert!(!root.path().join("d/new").exists()); + } + #[tokio::test] + async fn checkout_external_approved_descriptor_survives_root_path_swap() { + let parent = tempfile::tempdir().unwrap(); + let selected = parent.path().join("selected"); + std::fs::create_dir(&selected).unwrap(); + std::fs::write(selected.join("a"), "approved").unwrap(); + let session = LocalWorkdirSession::external_read_write( + crate::Workdir::new("external"), + &selected, + crate::BoundedReadLimits::EXTERNAL_DEFAULT, + ) + .unwrap(); + let original = session.checkout_observe(path("a")).await.unwrap(); + let approved = parent.path().join("approved"); + std::fs::rename(&selected, &approved).unwrap(); + std::fs::create_dir(&selected).unwrap(); + std::fs::write(selected.join("a"), "replacement").unwrap(); + let read = checked_read(&session, original).await; + assert!(matches!(&read.output, CheckoutOutput::Read(r) if r.bytes == b"approved")); + session + .checkout_execute(CheckoutRequest { + target: path("a"), + validator: read.observation.validator.clone(), + operation: CheckoutOperation::Write { + content: b"saved".to_vec(), + expected_hash: hash(&read), + }, + }) + .await + .unwrap(); + assert_eq!( + std::fs::read_to_string(approved.join("a")).unwrap(), + "saved" + ); + assert_eq!( + std::fs::read_to_string(selected.join("a")).unwrap(), + "replacement" + ); + } + #[tokio::test] + async fn checkout_scope_readonly_write_leases_and_revocation_attenuate_authority() { + use crate::{ + WorkdirToolBroker, WorkdirToolScope, WorkdirToolScopePermission as Permission, + WorkdirToolScopeRule, + }; + let root = tempfile::tempdir().unwrap(); + std::fs::create_dir(root.path().join("child")).unwrap(); + std::fs::write(root.path().join("child/a"), "child").unwrap(); + std::fs::write(root.path().join("outside"), "outside").unwrap(); + let source = Arc::new(session(root.path())); + let broker = WorkdirToolBroker::new(source.clone()); + let lease = broker + .scope(WorkdirToolScope { + rules: vec![WorkdirToolScopeRule { + target: path("child"), + permission: Permission::Write, + recursive: true, + symlink_policy: Default::default(), + }], + cwd: path("child"), + command: false, + }) + .await + .unwrap(); + let child = lease.tool_session(); + assert!(child.checkout_observe(path("outside")).await.is_err()); + let observation = child.checkout_observe(path("a")).await.unwrap(); + assert!( + observation + .capabilities + .supports(WorkdirSessionCapability::Write) + ); + let parent_observation = broker.checkout_observe(path("child/a")).await.unwrap(); + assert!( + !parent_observation + .capabilities + .supports(WorkdirSessionCapability::Write) + ); + let read = checked_read(&*child, observation).await; + assert!( + broker + .checkout_execute(CheckoutRequest { + target: path("child/a"), + validator: parent_observation.validator, + operation: CheckoutOperation::Write { + content: b"blocked".to_vec(), + expected_hash: hash(&read) + } + }) + .await + .is_err() + ); + let readonly = crate::ReadOnlyWorkdirSession::new(child.clone()); + assert!( + !readonly + .checkout_observe(path("a")) + .await + .unwrap() + .capabilities + .supports(WorkdirSessionCapability::Write) + ); + assert!( + readonly + .checkout_execute(CheckoutRequest { + target: path("a"), + validator: read.observation.validator.clone(), + operation: CheckoutOperation::Edit { + old_string: "child".into(), + new_string: "bad".into(), + replace_all: false, + expected_hash: hash(&read) + } + }) + .await + .is_err() + ); + let directory = child.checkout_observe(WorkdirPath::root()).await.unwrap(); + let created = child + .checkout_execute(CheckoutRequest { + target: WorkdirPath::root(), + validator: directory.validator, + operation: CheckoutOperation::Create { + path: path("fresh"), + content: b"fresh".to_vec(), + }, + }) + .await + .unwrap(); + assert_eq!(created.observation.path, WorkdirPath::root()); + assert_eq!(created.observation.kind, EntryKind::Directory); + assert_eq!( + created.observation, + child.checkout_observe(WorkdirPath::root()).await.unwrap() + ); + assert_eq!( + std::fs::read(root.path().join("child/fresh")).unwrap(), + b"fresh" + ); + lease.close().await.unwrap(); + assert!(matches!( + child + .checkout_execute(CheckoutRequest { + target: path("a"), + validator: read.observation.validator.clone(), + operation: CheckoutOperation::Read { + offset: 0, + limit: 10, + max_bytes: 100 + } + }) + .await, + Err(WorkdirError::SessionClosed) + )); + } + #[tokio::test] + async fn shared_search_skips_non_utf8_names_without_colliding_with_unicode_paths() { + use crate::{GlobRequest, GrepOutputMode, GrepRequest, ListRequest}; + use std::os::unix::ffi::OsStringExt; + let root = tempfile::tempdir().unwrap(); + std::fs::write( + root.path().join(std::ffi::OsString::from_vec( + b"replacement\xff.txt".to_vec(), + )), + "needle invalid", + ) + .unwrap(); + std::fs::write(root.path().join("replacement\u{fffd}.txt"), "needle valid").unwrap(); + std::fs::write(root.path().join("é.txt"), "needle composed").unwrap(); + std::fs::write(root.path().join("e\u{301}.txt"), "needle decomposed").unwrap(); + let invalid_dir = root + .path() + .join(std::ffi::OsString::from_vec(b"directory\xff".to_vec())); + std::fs::create_dir(&invalid_dir).unwrap(); + std::fs::write(invalid_dir.join("child.txt"), "needle hidden invalid child").unwrap(); + std::fs::create_dir(root.path().join("directory\u{fffd}")).unwrap(); + std::fs::write( + root.path().join("directory\u{fffd}/child.txt"), + "needle visible valid child", + ) + .unwrap(); + let local = session(root.path()); + let external = LocalWorkdirSession::external_read_only( + crate::Workdir::new("external-search"), + root.path(), + crate::BoundedReadLimits::EXTERNAL_DEFAULT, + ) + .unwrap(); + for provider in [&local as &dyn WorkdirSession, &external] { + let listed = provider + .list(ListRequest { + path: WorkdirPath::root(), + limit: 100, + }) + .await + .unwrap(); + assert_eq!(listed.entries.len(), 4); + assert_eq!( + listed + .entries + .iter() + .filter(|e| e.path == path("replacement\u{fffd}.txt")) + .count(), + 1 + ); + let globbed = provider + .glob(GlobRequest { + path: WorkdirPath::root(), + pattern: "**/*.txt".into(), + limit: 100, + }) + .await + .unwrap(); + let mut expected = vec![ + path("directory\u{fffd}/child.txt"), + path("replacement\u{fffd}.txt"), + path("é.txt"), + path("e\u{301}.txt"), + ]; + expected.sort_by(|a, b| a.as_str().cmp(b.as_str())); + assert_eq!(globbed.paths, expected); + for mode in [ + GrepOutputMode::Content, + GrepOutputMode::FilesWithMatches, + GrepOutputMode::Count, + ] { + let grep = provider + .grep(GrepRequest { + path: WorkdirPath::root(), + pattern: "needle".into(), + glob: None, + file_type: None, + case_insensitive: false, + before_context: 0, + after_context: 0, + multiline: false, + output_mode: mode, + offset: 0, + limit: 100, + }) + .await + .unwrap(); + let mut paths = grep.paths; + paths.sort_by(|a, b| a.as_str().cmp(b.as_str())); + assert_eq!(paths, expected); + assert_eq!(grep.matched_files, 4); + assert_eq!(grep.match_count, 4); + assert!(!grep.output.contains("needle invalid")); + assert!(!grep.output.contains("hidden invalid child")); + } + } + } + + #[tokio::test] + async fn shared_grep_typed_paths_follow_returned_modes_offset_limit_and_odd_names() { + use crate::{GrepOutputMode, GrepRequest}; + let root = tempfile::tempdir().unwrap(); + for name in ["a.txt", "b.txt", "c.txt", "line\nfile:1.txt"] { + std::fs::write(root.path().join(name), "needle").unwrap(); + } + let session = session(root.path()); + for mode in [ + GrepOutputMode::Content, + GrepOutputMode::FilesWithMatches, + GrepOutputMode::Count, + ] { + let request = GrepRequest { + path: WorkdirPath::root(), + pattern: "needle".into(), + glob: None, + file_type: None, + case_insensitive: false, + before_context: 0, + after_context: 0, + multiline: false, + output_mode: mode, + offset: 1, + limit: 1, + }; + let result = session.grep(request.clone()).await.unwrap(); + assert_eq!(result.paths.len(), 1); + assert_eq!(result.matched_files, 1); + assert!(result.output.contains(result.paths[0].as_str())); + let empty = session + .grep(GrepRequest { + limit: 0, + ..request.clone() + }) + .await + .unwrap(); + assert!(empty.output.is_empty()); + assert!(empty.paths.is_empty()); + let direct = session + .grep(GrepRequest { + path: path("line\nfile:1.txt"), + offset: 0, + ..request + }) + .await + .unwrap(); + assert_eq!(direct.paths, vec![path("line\nfile:1.txt")]); + } + let legacy: crate::GrepResult = serde_json::from_value( + serde_json::json!({"output":"", "match_count":0,"matched_files":0,"truncated":false}), + ) + .unwrap(); + assert!(legacy.paths.is_empty()); + } + + #[tokio::test] + async fn checkout_partial_parent_creation_is_outcome_unknown() { + let root = tempfile::tempdir().unwrap(); + let session = session(root.path()); + let observation = session.checkout_observe(WorkdirPath::root()).await.unwrap(); + // The final name cannot be saved, but an earlier mkdir has effects. + let destination = path(&format!("made/{}", "x".repeat(300))); + let error = session + .checkout_execute(CheckoutRequest { + target: observation.path, + validator: observation.validator, + operation: CheckoutOperation::Create { + path: destination, + content: b"new".to_vec(), + }, + }) + .await + .unwrap_err(); + assert!(matches!(error, WorkdirError::OutcomeUnknown(_))); + assert!(root.path().join("made").is_dir()); + } + + #[tokio::test] + async fn checkout_http_dispatch_and_external_dto_bounds() { + use crate::external::{ExternalWorkdirOperation, ExternalWorkdirOperationResult}; + use crate::http::{ + WorkdirSessionOperation as Op, WorkdirSessionOperationResult as Result, + WorkdirTransportError, WorkdirTransportErrorCode, dispatch_workdir_session_operation, + }; + let root = tempfile::tempdir().unwrap(); + std::fs::write(root.path().join("a"), "contents").unwrap(); + let session = session(root.path()); + let Result::CheckoutObserve(observation) = + dispatch_workdir_session_operation(&session, Op::CheckoutObserve(path("a"))) + .await + .unwrap() + else { + panic!() + }; + let request = CheckoutRequest { + target: path("a"), + validator: observation.validator, + operation: CheckoutOperation::Read { + offset: 0, + limit: 10, + max_bytes: 100, + }, + }; + assert!(ExternalWorkdirOperation::try_from(Op::CheckoutExecute(request.clone())).is_ok()); + let response = + dispatch_workdir_session_operation(&session, Op::CheckoutExecute(request.clone())) + .await + .unwrap(); + assert!(matches!(response, Result::CheckoutExecute(_))); + assert!(ExternalWorkdirOperationResult::try_from(response.clone()).is_ok()); + assert_eq!( + serde_json::from_value::(serde_json::to_value(response).unwrap()) + .unwrap() + .clone(), + dispatch_workdir_session_operation(&session, Op::CheckoutExecute(request.clone())) + .await + .unwrap() + ); + let mut oversized = request.clone(); + oversized.validator = vec![0; 257]; + assert!(ExternalWorkdirOperation::try_from(Op::CheckoutExecute(oversized)).is_err()); + let mut oversized = request; + oversized.operation = CheckoutOperation::Read { + offset: 0, + limit: 1, + max_bytes: 1024 * 1024 + 1, + }; + assert!(ExternalWorkdirOperation::try_from(Op::CheckoutExecute(oversized)).is_err()); + let grep = crate::GrepResult { + paths: vec![path("a")], + output: "a\n".into(), + match_count: 1, + matched_files: 1, + truncated: false, + }; + assert!(ExternalWorkdirOperationResult::try_from(Result::Grep(grep.clone())).is_ok()); + let invalid = crate::GrepResult { + paths: vec![WorkdirPath::new_scoped("/outside").unwrap()], + ..grep.clone() + }; + assert!(ExternalWorkdirOperationResult::try_from(Result::Grep(invalid)).is_err()); + let oversized = crate::GrepResult { + paths: vec![path("a"); crate::external::MAX_EXTERNAL_RESULT_ITEMS + 1], + ..grep + }; + assert!(ExternalWorkdirOperationResult::try_from(Result::Grep(oversized)).is_err()); + let unknown = WorkdirTransportError::from_workdir_error(&WorkdirError::OutcomeUnknown( + "/secret/host/path".into(), + )); + assert_eq!(unknown.code, WorkdirTransportErrorCode::OutcomeUnknown); + assert!(!unknown.message.contains("secret")); + assert!(matches!( + unknown.into_workdir_error(), + WorkdirError::OutcomeUnknown(_) + )); + } +} diff --git a/crates/workdir/src/checkout_search_tests.rs b/crates/workdir/src/checkout_search_tests.rs new file mode 100644 index 000000000..0de5ab0db --- /dev/null +++ b/crates/workdir/src/checkout_search_tests.rs @@ -0,0 +1,817 @@ +#![cfg(target_os = "linux")] +use crate::*; +use std::sync::Arc; +fn path(value: &str) -> WorkdirPath { + WorkdirPath::new(value).unwrap() +} +fn rule(value: &str, recursive: bool) -> WorkdirToolScopeRule { + WorkdirToolScopeRule { + target: path(value), + recursive, + permission: WorkdirToolScopePermission::Read, + symlink_policy: Default::default(), + } +} +fn scope(rules: Vec, cwd: &str) -> WorkdirToolScope { + WorkdirToolScope { + rules, + cwd: path(cwd), + command: false, + } +} +fn grep(mode: GrepOutputMode) -> GrepRequest { + GrepRequest { + path: WorkdirPath::root(), + pattern: "needle".into(), + glob: None, + file_type: None, + case_insensitive: false, + before_context: 0, + after_context: 0, + multiline: false, + output_mode: mode, + offset: 0, + limit: 100, + } +} +fn fixture() -> tempfile::TempDir { + let root = tempfile::tempdir().unwrap(); + std::fs::create_dir_all(root.path().join("child/keep")).unwrap(); + std::fs::create_dir_all(root.path().join("child/denied/deep")).unwrap(); + std::fs::write(root.path().join("child/a.txt"), "needle allowed direct").unwrap(); + std::fs::write( + root.path().join("child/keep/yes.txt"), + "needle allowed subtree", + ) + .unwrap(); + std::fs::write( + root.path().join("child/denied/deep/secret.txt"), + "needle secret", + ) + .unwrap(); + // Opening this out-of-scope source would exhaust the shared grep source bound. + std::fs::File::create(root.path().join("child/denied/huge.txt")) + .unwrap() + .set_len(65 * 1024 * 1024) + .unwrap(); + std::fs::write(root.path().join("outside.txt"), "needle outside").unwrap(); + root +} +async fn verify_sparse(session: &dyn WorkdirSession) { + let listed = session + .list(ListRequest { + path: WorkdirPath::root(), + limit: 100, + }) + .await + .unwrap(); + assert_eq!( + listed + .entries + .iter() + .map(|entry| entry.path.clone()) + .collect::>(), + vec![path("denied"), path("keep"), path("a.txt")] + ); + let empty = session + .list(ListRequest { + path: path("denied"), + limit: 100, + }) + .await + .unwrap(); + assert!(empty.entries.is_empty()); + assert_eq!(empty.total_entries, 0); + let globbed = session + .glob(GlobRequest { + path: WorkdirPath::root(), + pattern: "**/*.txt".into(), + limit: 100, + }) + .await + .unwrap(); + assert_eq!(globbed.paths, vec![path("a.txt"), path("keep/yes.txt")]); + for mode in [ + GrepOutputMode::Content, + GrepOutputMode::Count, + GrepOutputMode::FilesWithMatches, + ] { + let result = session.grep(grep(mode)).await.unwrap(); + let mut paths = result.paths; + paths.sort_by(|a, b| a.as_str().cmp(b.as_str())); + assert_eq!(paths, vec![path("a.txt"), path("keep/yes.txt")]); + assert!(!result.output.contains("child/")); + assert!(!result.output.contains("secret")); + assert!(!result.output.contains("outside")); + assert_eq!(result.matched_files, 2); + } + let CheckoutSearchResult::List(native) = session + .checkout_search(CheckoutSearchRequest::new(CheckoutSearchOperation::List( + ListRequest { + path: WorkdirPath::root(), + limit: 100, + }, + ))) + .await + .unwrap() + else { + panic!() + }; + assert_eq!(listed, native); +} +#[tokio::test] +async fn checkout_search_nested_sparse_nonrecursive_scopes_rebase_and_exclude_content() { + let root = fixture(); + let source = Arc::new(LocalWorkdirSession::new( + manifest::Scope::writable(root.path()).unwrap(), + root.path().to_path_buf(), + )); + let parent = WorkdirToolBroker::new(source); + let first = parent + .scope(scope(vec![rule("child", true)], "child")) + .await + .unwrap(); + let nested = first + .scope(scope( + vec![rule("child", false), rule("child/keep", true)], + "child", + )) + .await + .unwrap(); + verify_sparse(&*nested.tool_session()).await; + // Nested incoming layers cannot replace the session's own sparse layer. + let mut request = CheckoutSearchRequest::new(CheckoutSearchOperation::Glob(GlobRequest { + path: WorkdirPath::root(), + pattern: "**/*.txt".into(), + limit: 100, + })); + request.scope_layers.push(vec![rule("", true)]); + let CheckoutSearchResult::Glob(result) = nested + .tool_session() + .checkout_search(request) + .await + .unwrap() + else { + panic!() + }; + assert_eq!(result.paths, vec![path("a.txt"), path("keep/yes.txt")]); + nested.close().await.unwrap(); + assert!(matches!( + nested + .tool_session() + .checkout_search(CheckoutSearchRequest::new(CheckoutSearchOperation::List( + ListRequest { + path: WorkdirPath::root(), + limit: 100 + } + ))) + .await, + Err(WorkdirError::SessionClosed) + )); +} +#[tokio::test] +async fn checkout_search_composes_actual_nested_wrappers_and_output_root() { + let root = fixture(); + let parent = WorkdirToolBroker::new(Arc::new(LocalWorkdirSession::new( + manifest::Scope::writable(root.path()).unwrap(), + root.path().to_path_buf(), + ))); + let first = parent + .scope(scope(vec![rule("child", true)], "child")) + .await + .unwrap(); + // Two broker wrappers, not only leases that share a source. + let stacked = WorkdirToolBroker::new(first.tool_session()); + let mut request = + CheckoutSearchRequest::new(CheckoutSearchOperation::Grep(grep(GrepOutputMode::Content))); + *request.operation.path_mut() = path("keep"); + request.output_root = path("keep"); + request.scope_layers = vec![vec![rule("keep", false)]]; + let CheckoutSearchResult::Grep(result) = stacked.checkout_search(request).await.unwrap() else { + panic!() + }; + assert_eq!(result.paths, vec![path("yes.txt")]); + assert!(result.output.starts_with("yes.txt\n")); + // The outer context is relative to the inner session; no duplicated child/. + let CheckoutSearchResult::List(result) = stacked + .checkout_search(CheckoutSearchRequest::new(CheckoutSearchOperation::List( + ListRequest { + path: path("keep"), + limit: 100, + }, + ))) + .await + .unwrap() + else { + panic!() + }; + assert_eq!(result.entries[0].path, path("keep/yes.txt")); + // A second actual scoped wrapper has its own cwd and nonrecursive layer. + // Logical rules avoid requesting resolved authorization from a wrapper + // that conservatively does not expose host-side resolution authority. + std::fs::create_dir(root.path().join("child/keep/deep")).unwrap(); + std::fs::write( + root.path().join("child/keep/deep/secret"), + "needle hidden nested", + ) + .unwrap(); + let mut own_rule = rule("keep", false); + own_rule.symlink_policy = manifest::SymlinkPolicy::Logical; + let second = stacked.scope(scope(vec![own_rule], "keep")).await.unwrap(); + let result = second + .tool_session() + .grep(grep(GrepOutputMode::Content)) + .await + .unwrap(); + assert_eq!(result.paths, vec![path("yes.txt")]); + assert!(result.output.starts_with("yes.txt\n")); + assert!(!result.output.contains("hidden nested")); + let entries = second + .tool_session() + .list(ListRequest { + path: WorkdirPath::root(), + limit: 100, + }) + .await + .unwrap() + .entries; + assert_eq!( + entries + .iter() + .map(|entry| entry.path.clone()) + .collect::>(), + vec![path("deep"), path("yes.txt")] + ); + assert!( + second + .tool_session() + .list(ListRequest { + path: path("deep"), + limit: 100 + }) + .await + .unwrap() + .entries + .is_empty() + ); +} +#[tokio::test] +async fn checkout_search_external_keeps_approved_root_and_sparse_authority() { + let root = fixture(); + let external = LocalWorkdirSession::external_read_only( + Workdir::new("external-search"), + root.path(), + BoundedReadLimits::EXTERNAL_DEFAULT, + ) + .unwrap(); + let broker = WorkdirToolBroker::new(Arc::new(external)); + let lease = broker + .scope(scope( + vec![rule("child", false), rule("child/keep", true)], + "child", + )) + .await + .unwrap(); + let parent = tempfile::tempdir().unwrap(); + let approved = parent.path().join("approved"); + std::fs::rename(root.path(), &approved).unwrap(); + std::fs::create_dir(root.path()).unwrap(); + std::fs::create_dir(root.path().join("child")).unwrap(); + std::fs::write( + root.path().join("child/replacement.txt"), + "needle replacement", + ) + .unwrap(); + verify_sparse(&*lease.tool_session()).await; +} +#[tokio::test] +async fn checkout_search_scope_layers_intersect_before_list_and_grep() { + let root = fixture(); + let source = LocalWorkdirSession::new( + manifest::Scope::writable(root.path()).unwrap(), + root.path().to_path_buf(), + ); + let mut request = CheckoutSearchRequest::new(CheckoutSearchOperation::List(ListRequest { + path: path("child"), + limit: 100, + })); + request.output_root = path("child"); + request.scope_layers = vec![ + vec![rule("child", false), rule("child/keep", true)], + vec![rule("child", false)], + ]; + let CheckoutSearchResult::List(result) = source.checkout_search(request.clone()).await.unwrap() + else { + panic!() + }; + assert_eq!(result.entries.len(), 3); + request.operation = CheckoutSearchOperation::Grep(GrepRequest { + path: path("child"), + ..grep(GrepOutputMode::Content) + }); + let CheckoutSearchResult::Grep(result) = source.checkout_search(request).await.unwrap() else { + panic!() + }; + assert_eq!(result.paths, vec![path("a.txt")]); + assert!(!result.output.contains("subtree")); +} +#[tokio::test] +async fn checkout_search_wire_dto_bounds_and_provider_capabilities() { + use crate::external::{ExternalWorkdirOperation, ExternalWorkdirOperationResult}; + use crate::http::{WorkdirSessionOperation as Op, WorkdirSessionOperationResult as Output}; + let root = fixture(); + let source = LocalWorkdirSession::new( + manifest::Scope::writable(root.path()).unwrap(), + root.path().to_path_buf(), + ); + let request = CheckoutSearchRequest::new(CheckoutSearchOperation::List(ListRequest { + path: path("child/keep"), + limit: 100, + })); + let wire = Op::CheckoutSearch(request.clone()); + assert!(ExternalWorkdirOperation::try_from(wire.clone()).is_ok()); + assert_eq!( + serde_json::from_value::(serde_json::to_value(&wire).unwrap()).unwrap(), + wire + ); + let result = crate::http::dispatch_workdir_session_operation(&source, wire) + .await + .unwrap(); + assert!(matches!( + &result, + Output::CheckoutSearch(CheckoutSearchResult::List(_)) + )); + assert!(ExternalWorkdirOperationResult::try_from(result.clone()).is_ok()); + assert_eq!( + serde_json::from_value::(serde_json::to_value(&result).unwrap()).unwrap(), + result + ); + let mut bad = request.clone(); + bad.output_root = WorkdirPath::new_scoped("/host").unwrap(); + assert!(ExternalWorkdirOperation::try_from(Op::CheckoutSearch(bad)).is_err()); + let mut bad = request.clone(); + bad.scope_layers = vec![vec![rule("", true)]; 17]; + assert!(ExternalWorkdirOperation::try_from(Op::CheckoutSearch(bad)).is_err()); + let mut bad = request.clone(); + bad.scope_layers = vec![vec![rule("", true); 257]]; + assert!(ExternalWorkdirOperation::try_from(Op::CheckoutSearch(bad)).is_err()); + let mut bad = request.clone(); + bad.scope_layers = vec![vec![rule("", true); 65]]; + assert!(ExternalWorkdirOperation::try_from(Op::CheckoutSearch(bad)).is_err()); + let mut bad = request; + bad.scope_layers = vec![Vec::new()]; + assert!(ExternalWorkdirOperation::try_from(Op::CheckoutSearch(bad)).is_err()); + let bad_result = Output::CheckoutSearch(CheckoutSearchResult::Glob(GlobResult { + paths: vec![WorkdirPath::new_scoped("/host").unwrap()], + truncated: false, + })); + assert!(ExternalWorkdirOperationResult::try_from(bad_result).is_err()); +} + +fn write_ignore_fixture(root: &std::path::Path, relative: &str, contents: &str) { + let file = root.join(relative); + std::fs::create_dir_all(file.parent().unwrap()).unwrap(); + std::fs::write(file, contents).unwrap(); +} + +async fn verify_ignore_search_parity(session: &dyn WorkdirSession, base: &str, expected: &[&str]) { + let request = GlobRequest { + path: path(base), + pattern: "**/*.txt".into(), + limit: 100, + }; + // On a Local session this is the actual ordinary WalkBuilder-backed path, + // not a broker wrapper that would route both calls through checkout_search. + let ordinary = session.glob(request.clone()).await.unwrap(); + assert_eq!( + ordinary.paths, + expected.iter().map(|value| path(value)).collect::>(), + "ordinary Glob fixture semantics at {base:?}" + ); + assert!(!ordinary.truncated); + let CheckoutSearchResult::Glob(native) = session + .checkout_search(CheckoutSearchRequest::new(CheckoutSearchOperation::Glob( + request, + ))) + .await + .unwrap() + else { + panic!("expected native Glob result") + }; + assert_eq!(native, ordinary, "native Glob parity at {base:?}"); + + // Content mode renders paths in sorted order, avoiding dependence on the + // different walkers' visitation order. All fixture content has one match. + let request = GrepRequest { + path: path(base), + ..grep(GrepOutputMode::Content) + }; + let ordinary = session.grep(request.clone()).await.unwrap(); + assert_eq!( + ordinary.paths, + expected.iter().map(|value| path(value)).collect::>(), + "ordinary Grep fixture semantics at {base:?}" + ); + assert_eq!(ordinary.matched_files, expected.len()); + assert_eq!(ordinary.match_count, expected.len()); + assert!(!ordinary.truncated); + let CheckoutSearchResult::Grep(native) = session + .checkout_search(CheckoutSearchRequest::new(CheckoutSearchOperation::Grep( + request, + ))) + .await + .unwrap() + else { + panic!("expected native Grep result") + }; + assert_eq!(native, ordinary, "native Grep parity at {base:?}"); +} + +#[tokio::test] +async fn checkout_search_local_glob_inherits_root_and_nested_ignore_rules() { + // Exercise the two ignore sources independently so rule precedence cannot + // conceal either missing source. A .git marker activates .gitignore rules. + for ignore_name in [".ignore", ".gitignore"] { + let root = tempfile::tempdir().unwrap(); + std::fs::create_dir(root.path().join(".git")).unwrap(); + write_ignore_fixture( + root.path(), + ignore_name, + "root-only.txt\n/root-anchored.txt\n*.drop.txt\n!keep.drop.txt\ninherited.txt\npruned/\n", + ); + write_ignore_fixture( + root.path(), + &format!("nested/{ignore_name}"), + "/nested-anchored.txt\nnested-only.txt\n!inherited.txt\nignored-dir/\n!ignored-dir/rescue.txt\n", + ); + for relative in [ + "visible.txt", + "root-only.txt", + "root-anchored.txt", + "discard.drop.txt", + "keep.drop.txt", + "inherited.txt", + "pruned/rescue.txt", + "nested/visible.txt", + "nested/root-only.txt", + "nested/root-anchored.txt", + "nested/nested-only.txt", + "nested/nested-anchored.txt", + "nested/discard.drop.txt", + "nested/keep.drop.txt", + "nested/inherited.txt", + "nested/pruned/rescue.txt", + "nested/ignored-dir/rescue.txt", + "nested/deep/nested-anchored.txt", + ] { + write_ignore_fixture(root.path(), relative, "needle fixture\n"); + } + let source = LocalWorkdirSession::new( + manifest::Scope::writable(root.path()).unwrap(), + root.path().to_path_buf(), + ); + verify_ignore_search_parity( + &source, + "", + &[ + "keep.drop.txt", + "nested/deep/nested-anchored.txt", + "nested/inherited.txt", + "nested/keep.drop.txt", + "nested/root-anchored.txt", + "nested/visible.txt", + "visible.txt", + ], + ) + .await; + // Starting explicitly below the provider root must still inherit the + // root's rules and retain provider-relative (not base-relative) paths. + verify_ignore_search_parity( + &source, + "nested", + &[ + "nested/deep/nested-anchored.txt", + "nested/inherited.txt", + "nested/keep.drop.txt", + "nested/root-anchored.txt", + "nested/visible.txt", + ], + ) + .await; + verify_ignore_search_parity(&source, "nested/deep", &["nested/deep/nested-anchored.txt"]) + .await; + } +} + +#[tokio::test] +async fn checkout_search_local_glob_preserves_ignore_source_precedence_across_levels() { + let root = tempfile::tempdir().unwrap(); + std::fs::create_dir(root.path().join(".git")).unwrap(); + write_ignore_fixture( + root.path(), + ".ignore", + "priority.txt\n!ignore-wins.txt\nroot-ignore.txt\n", + ); + write_ignore_fixture( + root.path(), + ".gitignore", + "ignore-wins.txt\nrevived-git.txt\n", + ); + write_ignore_fixture(root.path(), "nested/.ignore", "!root-ignore.txt\n"); + write_ignore_fixture( + root.path(), + "nested/.gitignore", + "!priority.txt\n!revived-git.txt\n", + ); + for relative in [ + "priority.txt", + "ignore-wins.txt", + "root-ignore.txt", + "revived-git.txt", + "nested/priority.txt", + "nested/ignore-wins.txt", + "nested/root-ignore.txt", + "nested/revived-git.txt", + ] { + write_ignore_fixture(root.path(), relative, "needle fixture\n"); + } + let source = LocalWorkdirSession::new( + manifest::Scope::writable(root.path()).unwrap(), + root.path().to_path_buf(), + ); + // .ignore outranks .gitignore even when the .gitignore is deeper. Within + // the same source kind, a deeper negation can override an ancestor rule. + verify_ignore_search_parity( + &source, + "", + &[ + "ignore-wins.txt", + "nested/ignore-wins.txt", + "nested/revived-git.txt", + "nested/root-ignore.txt", + ], + ) + .await; + verify_ignore_search_parity( + &source, + "nested", + &[ + "nested/ignore-wins.txt", + "nested/revived-git.txt", + "nested/root-ignore.txt", + ], + ) + .await; +} + +#[tokio::test] +async fn checkout_search_scoped_broker_skips_denied_ancestor_ignore_files() { + let root = tempfile::tempdir().unwrap(); + std::fs::create_dir(root.path().join(".git")).unwrap(); + for (relative, contents) in [ + (".ignore", "root-ignore-visible.txt\n"), + (".gitignore", "root-git-visible.txt\n"), + ("parent/.ignore", "parent-ignore-visible.txt\n"), + ("parent/.gitignore", "parent-git-visible.txt\n"), + ("parent/nested/.ignore", "allowed-ignore.txt\n"), + ] { + write_ignore_fixture(root.path(), relative, contents); + } + for name in [ + "root-ignore-visible.txt", + "root-git-visible.txt", + "parent-ignore-visible.txt", + "parent-git-visible.txt", + "allowed-ignore.txt", + ] { + write_ignore_fixture( + root.path(), + &format!("parent/nested/{name}"), + "needle fixture\n", + ); + } + let source = Arc::new(LocalWorkdirSession::new( + manifest::Scope::writable(root.path()).unwrap(), + root.path().to_path_buf(), + )); + // Without a scope layer, ordinary Local traversal really does inherit + // these ancestor rules. Each denied source suppresses a different file. + assert!( + source + .glob(GlobRequest { + path: path("parent/nested"), + pattern: "**/*.txt".into(), + limit: 100, + }) + .await + .unwrap() + .paths + .is_empty() + ); + // Make accidental consumption observable even if a matcher were to drop + // the valid patterns: these denied sparse sources exceed the shared Grep + // source budget. The allowed subtree remains tiny. + for relative in [ + ".ignore", + ".gitignore", + "parent/.ignore", + "parent/.gitignore", + ] { + std::fs::OpenOptions::new() + .write(true) + .open(root.path().join(relative)) + .unwrap() + .set_len(65 * 1024 * 1024) + .unwrap(); + } + let broker = WorkdirToolBroker::new(source.clone()); + let lease = broker + .scope(scope(vec![rule("parent/nested", true)], "parent/nested")) + .await + .unwrap(); + // Broker .glob/.grep use the native path under a scope. Compare those + // public entry points with explicit checkout_search and assert rebasing. + verify_ignore_search_parity( + &*lease.tool_session(), + "", + &[ + "parent-git-visible.txt", + "parent-ignore-visible.txt", + "root-git-visible.txt", + "root-ignore-visible.txt", + ], + ) + .await; + // Also exercise a Local native request with an unre-based provider root: + // allowed descendants do not grant authority to ancestor ignore sources. + let mut request = CheckoutSearchRequest::new(CheckoutSearchOperation::Glob(GlobRequest { + path: path("parent/nested"), + pattern: "**/*.txt".into(), + limit: 100, + })); + request.scope_layers = vec![vec![rule("parent/nested", true)]]; + let CheckoutSearchResult::Glob(result) = source.checkout_search(request).await.unwrap() else { + panic!("expected native Glob result") + }; + assert_eq!( + result.paths, + vec![ + path("parent/nested/parent-git-visible.txt"), + path("parent/nested/parent-ignore-visible.txt"), + path("parent/nested/root-git-visible.txt"), + path("parent/nested/root-ignore-visible.txt"), + ] + ); + assert!(!result.truncated); + lease.close().await.unwrap(); +} + +#[tokio::test] +async fn checkout_search_native_skips_symlinked_ignore_sources_without_outside_effects() { + let root = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + std::fs::create_dir(root.path().join(".git")).unwrap(); + std::fs::create_dir(root.path().join("nested")).unwrap(); + // Following any of these sources would hide an otherwise visible file. + // The native descriptor provider must safely skip rejected ignore opens, + // including when the source belongs to an explicitly selected base. + for (relative, hidden) in [ + (".ignore", "root-ignore-visible.txt"), + (".gitignore", "root-git-visible.txt"), + ("nested/.ignore", "nested-ignore-visible.txt"), + ("nested/.gitignore", "nested-git-visible.txt"), + ] { + let target = outside.path().join(relative.replace('/', "-")); + let contents = format!("{hidden}\n"); + std::fs::write(&target, &contents).unwrap(); + std::os::unix::fs::symlink(&target, root.path().join(relative)).unwrap(); + let visible = if relative.starts_with("nested/") { + format!("nested/{hidden}") + } else { + hidden.to_string() + }; + write_ignore_fixture(root.path(), &visible, "needle fixture\n"); + } + write_ignore_fixture(outside.path(), "secret.txt", "needle outside\n"); + let source = LocalWorkdirSession::new( + manifest::Scope::writable(root.path()).unwrap(), + root.path().to_path_buf(), + ); + for (base, expected) in [ + ( + "", + vec![ + path("nested/nested-git-visible.txt"), + path("nested/nested-ignore-visible.txt"), + path("root-git-visible.txt"), + path("root-ignore-visible.txt"), + ], + ), + ( + "nested", + vec![ + path("nested/nested-git-visible.txt"), + path("nested/nested-ignore-visible.txt"), + ], + ), + ] { + let CheckoutSearchResult::Glob(result) = source + .checkout_search(CheckoutSearchRequest::new(CheckoutSearchOperation::Glob( + GlobRequest { + path: path(base), + pattern: "**/*.txt".into(), + limit: 100, + }, + ))) + .await + .unwrap() + else { + panic!("expected native Glob result") + }; + assert_eq!(result.paths, expected); + assert!(!result.truncated); + // Grep's existing descriptor contract rejects an unauthorized ignore + // open, rather than ignoring its error as Glob does. Both behaviors + // preserve confinement; do not relax Grep merely to match Glob. + let error = source + .checkout_search(CheckoutSearchRequest::new(CheckoutSearchOperation::Grep( + GrepRequest { + path: path(base), + ..grep(GrepOutputMode::Content) + }, + ))) + .await + .unwrap_err(); + assert!(matches!(error, WorkdirError::OutOfScope(_))); + } + for (relative, hidden) in [ + (".ignore", "root-ignore-visible.txt"), + (".gitignore", "root-git-visible.txt"), + ("nested/.ignore", "nested-ignore-visible.txt"), + ("nested/.gitignore", "nested-git-visible.txt"), + ] { + assert!( + std::fs::symlink_metadata(root.path().join(relative)) + .unwrap() + .file_type() + .is_symlink() + ); + assert_eq!( + std::fs::read_to_string(outside.path().join(relative.replace('/', "-"))).unwrap(), + format!("{hidden}\n") + ); + } + assert_eq!( + std::fs::read_to_string(outside.path().join("secret.txt")).unwrap(), + "needle outside\n" + ); +} + +#[tokio::test] +async fn checkout_search_preserves_current_provider_scope_and_rejects_symlink_escape() { + let root = fixture(); + let current = manifest::Scope::from_config(&manifest::ScopeConfig { + allow: vec![manifest::ScopeRule { + target: root.path().join("child"), + permission: manifest::Permission::Read, + recursive: false, + symlink_policy: Default::default(), + }], + deny: Vec::new(), + }) + .unwrap(); + let source = LocalWorkdirSession::new(current, root.path().to_path_buf()); + let mut request = CheckoutSearchRequest::new(CheckoutSearchOperation::Grep(GrepRequest { + path: path("child"), + ..grep(GrepOutputMode::Content) + })); + request.output_root = path("child"); + // A broad caller layer cannot enlarge the provider's nonrecursive authority. + request.scope_layers = vec![vec![rule("", true)]]; + let CheckoutSearchResult::Grep(result) = source.checkout_search(request).await.unwrap() else { + panic!() + }; + assert_eq!(result.paths, vec![path("a.txt")]); + let CheckoutSearchResult::List(result) = source + .checkout_search(CheckoutSearchRequest { + operation: CheckoutSearchOperation::List(ListRequest { + path: path("child/denied"), + limit: 100, + }), + output_root: path("child"), + scope_layers: Vec::new(), + }) + .await + .unwrap() + else { + panic!() + }; + assert!(result.entries.is_empty()); + std::os::unix::fs::symlink("../outside.txt", root.path().join("child/link.txt")).unwrap(); + let mut request = CheckoutSearchRequest::new(CheckoutSearchOperation::Grep(GrepRequest { + path: path("child/link.txt"), + ..grep(GrepOutputMode::Content) + })); + request.output_root = path("child"); + assert!(source.checkout_search(request).await.is_err()); +} diff --git a/crates/workdir/src/external.rs b/crates/workdir/src/external.rs index c94582867..1b37cda23 100644 --- a/crates/workdir/src/external.rs +++ b/crates/workdir/src/external.rs @@ -149,6 +149,56 @@ fn validate_external_operation(operation: &WorkdirSessionOperation) -> Result<() WorkdirSessionOperation::ScopeRulesOverlap(request) => { is_root_relative(&request.left.target) && is_root_relative(&request.right.target) } + WorkdirSessionOperation::CheckoutSearch(request) => { + request.validate().is_ok() + && request.scope_layers.iter().map(Vec::len).sum::() + <= MAX_EXTERNAL_SCOPE_RULES + && validate_external_operation(&match &request.operation { + crate::CheckoutSearchOperation::List(r) => { + WorkdirSessionOperation::List(r.clone()) + } + crate::CheckoutSearchOperation::Glob(r) => { + WorkdirSessionOperation::Glob(r.clone()) + } + crate::CheckoutSearchOperation::Grep(r) => { + WorkdirSessionOperation::Grep(r.clone()) + } + }) + .is_ok() + } + WorkdirSessionOperation::CheckoutObserve(path) => { + is_root_relative(path) && crate::checkout::validate_path(path).is_ok() + } + WorkdirSessionOperation::CheckoutExecute(request) => { + is_root_relative(&request.target) + && crate::checkout::validate_path(&request.target).is_ok() + && !request.validator.is_empty() + && request.validator.len() <= 256 + && match &request.operation { + crate::CheckoutOperation::Read { + limit, max_bytes, .. + } => { + *limit <= 1_000_000 + && *max_bytes <= BoundedReadLimits::EXTERNAL_DEFAULT.max_response_bytes + } + crate::CheckoutOperation::Write { content, .. } => { + content.len() <= MAX_EXTERNAL_WRITE_BYTES + } + crate::CheckoutOperation::Edit { + old_string, + new_string, + .. + } => old_string + .len() + .checked_add(new_string.len()) + .is_some_and(|size| size <= MAX_EXTERNAL_WRITE_BYTES), + crate::CheckoutOperation::Create { path, content } => { + is_root_relative(path) + && crate::checkout::create_relative_path(&request.target, path).is_ok() + && content.len() <= MAX_EXTERNAL_WRITE_BYTES + } + } + } WorkdirSessionOperation::Stat(request) => is_root_relative(&request.path), WorkdirSessionOperation::Read(request) => { is_root_relative(&request.path) @@ -272,10 +322,59 @@ fn result_paths_fit<'a>(paths: impl IntoIterator) -> boo .is_some_and(|retained| retained <= fs_operation::MAX_RESULT_PATH_BYTES) } +fn valid_checkout_observation(observation: &crate::CheckoutObservation) -> bool { + is_root_relative(&observation.path) + && !observation.validator.is_empty() + && observation.validator.len() <= 256 + && matches!( + observation.kind, + crate::EntryKind::File | crate::EntryKind::Directory + ) + && (observation.kind == crate::EntryKind::Directory + || observation.size <= BoundedReadLimits::EXTERNAL_DEFAULT.max_source_bytes) +} + fn validate_external_result(result: &WorkdirSessionOperationResult) -> Result<(), String> { let result_is_valid = match result { WorkdirSessionOperationResult::AuthorizeScope | WorkdirSessionOperationResult::ScopeRulesOverlap { .. } => true, + WorkdirSessionOperationResult::CheckoutSearch(result) => { + validate_external_result(&match result { + crate::CheckoutSearchResult::List(r) => { + WorkdirSessionOperationResult::List(r.clone()) + } + crate::CheckoutSearchResult::Glob(r) => { + WorkdirSessionOperationResult::Glob(r.clone()) + } + crate::CheckoutSearchResult::Grep(r) => { + WorkdirSessionOperationResult::Grep(r.clone()) + } + }) + .is_ok() + } + WorkdirSessionOperationResult::CheckoutObserve(observation) => { + valid_checkout_observation(observation) + } + WorkdirSessionOperationResult::CheckoutExecute(result) => { + valid_checkout_observation(&result.observation) + && match &result.output { + crate::CheckoutOutput::Read(read) => { + read.path == result.observation.path + && validate_external_result(&WorkdirSessionOperationResult::Read( + read.clone(), + )) + .is_ok() + } + crate::CheckoutOutput::Write(write) => validate_external_result( + &WorkdirSessionOperationResult::Write(write.clone()), + ) + .is_ok(), + crate::CheckoutOutput::Edit(edit) => { + validate_external_result(&WorkdirSessionOperationResult::Edit(edit.clone())) + .is_ok() + } + } + } WorkdirSessionOperationResult::Stat(result) => is_root_relative(&result.path), WorkdirSessionOperationResult::Read(result) => { is_root_relative(&result.path) @@ -307,6 +406,9 @@ fn validate_external_result(result: &WorkdirSessionOperationResult) -> Result<() result.output.len() <= BoundedReadLimits::EXTERNAL_DEFAULT.max_response_bytes && result.match_count <= MAX_EXTERNAL_RESULT_ITEMS && result.matched_files <= MAX_EXTERNAL_RESULT_ITEMS + && result.paths.len() <= MAX_EXTERNAL_RESULT_ITEMS + && result.paths.iter().all(is_root_relative) + && result_paths_fit(&result.paths) } WorkdirSessionOperationResult::Write(result) => { result.bytes_written <= MAX_EXTERNAL_WRITE_BYTES @@ -821,6 +923,7 @@ mod tests { assert!(ExternalWorkdirOperation::try_from(unbounded).is_err()); let oversized_result = WorkdirSessionOperationResult::Grep(GrepResult { + paths: Vec::new(), output: "x".repeat(BoundedReadLimits::EXTERNAL_DEFAULT.max_response_bytes + 1), match_count: 1, matched_files: 1, diff --git a/crates/workdir/src/http.rs b/crates/workdir/src/http.rs index 4017056a0..1050ae3d9 100644 --- a/crates/workdir/src/http.rs +++ b/crates/workdir/src/http.rs @@ -64,6 +64,9 @@ pub struct OpenWorkdirSessionResponse { pub enum WorkdirSessionOperation { AuthorizeScope(WorkdirScopeAuthorizationRequest), ScopeRulesOverlap(WorkdirScopeOverlapRequest), + CheckoutSearch(crate::CheckoutSearchRequest), + CheckoutObserve(crate::WorkdirPath), + CheckoutExecute(crate::CheckoutRequest), Stat(StatRequest), Read(ReadRequest), ReadBytes(ReadBytesRequest), @@ -96,6 +99,9 @@ pub struct WorkdirSessionOperationRequest { pub enum WorkdirSessionOperationResult { AuthorizeScope, ScopeRulesOverlap { overlaps: bool }, + CheckoutSearch(crate::CheckoutSearchResult), + CheckoutObserve(crate::CheckoutObservation), + CheckoutExecute(crate::CheckoutResult), Stat(StatResult), Read(ReadResult), ReadBytes(ReadBytesResult), @@ -129,6 +135,18 @@ pub async fn dispatch_workdir_session_operation( .scope_rules_overlap(request) .await .map(|overlaps| WorkdirSessionOperationResult::ScopeRulesOverlap { overlaps }), + WorkdirSessionOperation::CheckoutSearch(request) => session + .checkout_search(request) + .await + .map(WorkdirSessionOperationResult::CheckoutSearch), + WorkdirSessionOperation::CheckoutObserve(path) => session + .checkout_observe(path) + .await + .map(WorkdirSessionOperationResult::CheckoutObserve), + WorkdirSessionOperation::CheckoutExecute(request) => session + .checkout_execute(request) + .await + .map(WorkdirSessionOperationResult::CheckoutExecute), WorkdirSessionOperation::Stat(request) => session .stat(request) .await @@ -186,6 +204,7 @@ pub async fn dispatch_workdir_session_operation( pub enum WorkdirTransportErrorCode { NotFound, Conflict, + OutcomeUnknown, Unsupported, InvalidRequest, Denied, @@ -208,6 +227,7 @@ impl WorkdirTransportErrorCode { match self { Self::NotFound => "not_found", Self::Conflict => "conflict", + Self::OutcomeUnknown => "outcome_unknown", Self::Unsupported => "unsupported", Self::InvalidRequest => "invalid_request", Self::Denied => "denied", @@ -239,7 +259,7 @@ impl WorkdirTransportErrorCode { | Self::IsDirectory | Self::SymlinkDirectoryNotTraversed => 400, Self::Unavailable => 503, - Self::Io | Self::Internal => 500, + Self::Io | Self::Internal | Self::OutcomeUnknown => 500, Self::Transport => 502, } } @@ -256,6 +276,10 @@ impl WorkdirTransportError { pub fn from_workdir_error(error: &WorkdirError) -> Self { use WorkdirTransportErrorCode as Code; let (code, message) = match error { + WorkdirError::OutcomeUnknown(_) => ( + Code::OutcomeUnknown, + "Workdir operation may have effects; inspect before retrying", + ), WorkdirError::NotFound(_) => (Code::NotFound, "Workdir path was not found"), WorkdirError::Conflict(_) => ( Code::Conflict, @@ -330,6 +354,9 @@ impl WorkdirTransportError { match self.code { Code::NotFound => WorkdirError::NotFound("".into()), Code::Conflict => WorkdirError::Conflict(self.message), + Code::OutcomeUnknown => WorkdirError::OutcomeUnknown( + "Workdir operation may have effects; inspect before retrying".into(), + ), Code::Unsupported => WorkdirError::UnsupportedOperation(self.message), Code::InvalidRequest => WorkdirError::InvalidArgument(self.message), Code::Denied => WorkdirError::Denied(self.message), @@ -564,6 +591,114 @@ mod client { } } + async fn checkout_search( + &self, + request: crate::CheckoutSearchRequest, + ) -> Result { + request.validate()?; + let expected = request.operation.capability(); + match self + .operate(WorkdirSessionOperation::CheckoutSearch(request)) + .await? + { + WorkdirSessionOperationResult::CheckoutSearch(result) + if matches!( + (&result, expected), + ( + crate::CheckoutSearchResult::List(_), + crate::WorkdirSessionCapability::Read + ) | ( + crate::CheckoutSearchResult::Glob(_), + crate::WorkdirSessionCapability::Glob + ) | ( + crate::CheckoutSearchResult::Grep(_), + crate::WorkdirSessionCapability::Grep + ) + ) => + { + crate::external::ExternalWorkdirOperationResult::try_from( + WorkdirSessionOperationResult::CheckoutSearch(result.clone()), + ) + .map_err(|_| Self::mismatch("bounded checkout_search"))?; + Ok(result) + } + _ => Err(Self::mismatch("checkout_search")), + } + } + + async fn checkout_observe( + &self, + path: crate::WorkdirPath, + ) -> Result { + match self + .operate(WorkdirSessionOperation::CheckoutObserve(path.clone())) + .await? + { + WorkdirSessionOperationResult::CheckoutObserve(mut result) + if result.path == path + && !result.validator.is_empty() + && result.validator.len() <= 256 => + { + result.capabilities = result.capabilities.intersection(self.capabilities()); + Ok(result) + } + _ => Err(Self::mismatch("checkout_observe")), + } + } + async fn checkout_execute( + &self, + request: crate::CheckoutRequest, + ) -> Result { + let mutation = request.operation.capability() != crate::WorkdirSessionCapability::Read; + let expected_path = request.target.clone(); + let expected_output = request.operation.capability(); + match self + .operate(WorkdirSessionOperation::CheckoutExecute(request)) + .await + { + Ok(WorkdirSessionOperationResult::CheckoutExecute(mut result)) + if result.observation.path == expected_path + && !result.observation.validator.is_empty() + && result.observation.validator.len() <= 256 + && match (&result.output, expected_output) { + ( + crate::CheckoutOutput::Read(read), + crate::WorkdirSessionCapability::Read, + ) => { + read.path == expected_path + && read.bytes.len() + <= crate::BoundedReadLimits::EXTERNAL_DEFAULT + .max_response_bytes + } + ( + crate::CheckoutOutput::Write(_), + crate::WorkdirSessionCapability::Write, + ) + | ( + crate::CheckoutOutput::Edit(_), + crate::WorkdirSessionCapability::Edit, + ) => true, + _ => false, + } => + { + result.observation.capabilities = result + .observation + .capabilities + .intersection(self.capabilities()); + Ok(result) + } + Ok(_) if mutation => Err(WorkdirError::OutcomeUnknown( + "mismatched response after checkout mutation".into(), + )), + Ok(_) => Err(Self::mismatch("checkout_execute")), + Err(WorkdirError::Unavailable(_) | WorkdirError::Transport(_)) if mutation => { + Err(WorkdirError::OutcomeUnknown( + "checkout transport failed; effects may have occurred".into(), + )) + } + Err(error) => Err(error), + } + } async fn stat(&self, request: StatRequest) -> Result { match self.operate(WorkdirSessionOperation::Stat(request)).await? { WorkdirSessionOperationResult::Stat(result) => Ok(result), diff --git a/crates/workdir/src/lib.rs b/crates/workdir/src/lib.rs index a390b1084..29f265784 100644 --- a/crates/workdir/src/lib.rs +++ b/crates/workdir/src/lib.rs @@ -5,6 +5,11 @@ //! bound to one Worker. Tools consume sessions; they do not own Workdir //! materialization or cleanup. +pub mod checkout; +pub use checkout::{ + CheckoutObservation, CheckoutOperation, CheckoutOutput, CheckoutRequest, CheckoutResult, + CheckoutSearchOperation, CheckoutSearchRequest, CheckoutSearchResult, +}; pub mod external; pub mod http; mod local; @@ -13,6 +18,8 @@ mod router; mod scope; pub mod workspace; +#[cfg(test)] +mod checkout_search_tests; #[cfg(test)] mod external_local_tests; @@ -219,6 +226,31 @@ pub trait WorkdirSession: std::fmt::Debug + Send + Sync { )) } + async fn checkout_search( + &self, + _request: CheckoutSearchRequest, + ) -> Result { + Err(WorkdirError::UnsupportedOperation( + "scoped checkout traversal".into(), + )) + } + + async fn checkout_observe( + &self, + _path: WorkdirPath, + ) -> Result { + Err(WorkdirError::UnsupportedOperation( + "checkout observation".into(), + )) + } + async fn checkout_execute( + &self, + _request: CheckoutRequest, + ) -> Result { + Err(WorkdirError::UnsupportedOperation( + "checked checkout execution".into(), + )) + } async fn stat(&self, request: StatRequest) -> Result; async fn read(&self, request: ReadRequest) -> Result; async fn read_bytes( @@ -289,6 +321,10 @@ pub enum WorkdirError { #[error("{0}")] Conflict(String), + /// Effects may have happened. Do not automatically retry the operation. + #[error("Workdir operation outcome is unknown: {0}")] + OutcomeUnknown(String), + #[error("unknown Workdir session command: {0}")] UnknownCommand(String), diff --git a/crates/workdir/src/local.rs b/crates/workdir/src/local.rs index 33850729b..417f6fe88 100644 --- a/crates/workdir/src/local.rs +++ b/crates/workdir/src/local.rs @@ -204,7 +204,7 @@ impl CommandTelemetry { } } -#[derive(Debug)] +#[derive(Debug, Clone)] struct ScopeAccess { scope: Arc, root: PathBuf, @@ -314,13 +314,12 @@ impl fs_operation::FsAccessPolicy for ScopeAccess { "path is not a directory", )); } - let root = self - .pinned_root - .as_ref() - .expect("confined traversal requires a pinned root") - .try_clone()?; - let path = PathBuf::from(format!("/proc/self/fd/{}", root.as_raw_fd())); - return Ok(Some(fs_operation::FsTraversalRoot::new(path, root))); + let path = PathBuf::from(format!("/proc/self/fd/{}", directory.as_raw_fd())); + return Ok(Some(fs_operation::FsTraversalRoot::new_at( + path, + directory, + resolved.to_path_buf(), + ))); } #[cfg(not(target_os = "linux"))] Err(std::io::Error::new( @@ -443,6 +442,118 @@ impl fs_operation::FsAccessPolicy for ScopeAccess { } } +/// A read-only, conjunctively narrowed descriptor policy for native searches. +#[derive(Debug, Clone)] +struct SearchAccess { + inner: ScopeAccess, + layers: Vec>, + output_root: PathBuf, + source_allows: Arc>, +} +impl SearchAccess { + fn logical_path(&self, path: &Path) -> Option { + WorkdirPath::new(path.strip_prefix(&self.inner.root).ok()?.to_str()?).ok() + } + fn read_allowed(&self, logical: &Path, resolved: &Path) -> bool { + use fs_operation::FsAccessPolicy; + let Some(path) = self.logical_path(logical) else { + return false; + }; + logical.starts_with(&self.output_root) + && self.inner.is_readable_paths(logical, resolved) + && self.layers.iter().all(|layer| { + layer.iter().any(|rule| { + crate::scope::rule_allows_path(rule, &path, WorkdirToolScopePermission::Read) + }) + }) + } + fn enumeration_allowed(&self, logical: &Path, resolved: &Path) -> bool { + self.read_allowed(logical, resolved) + && self.source_allows.iter().any(|rule| { + if rule.recursive { + logical.starts_with(&rule.target) || rule.target.starts_with(logical) + } else { + logical == rule.target || rule.target.starts_with(logical) + } + }) + && self.layers.iter().all(|layer| { + layer.iter().any(|rule| { + let target = self.inner.root.join(rule.target.as_str()); + if rule.recursive { + logical.starts_with(&target) || target.starts_with(logical) + } else { + logical == target || target.starts_with(logical) + } + }) + }) + } + fn require_read(&self, logical: &Path, resolved: &Path) -> std::io::Result<()> { + if self.read_allowed(logical, resolved) { + Ok(()) + } else { + Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "checkout search path denied", + )) + } + } +} +impl fs_operation::FsAccessPolicy for SearchAccess { + fn is_readable(&self, path: &Path) -> bool { + self.read_allowed(path, path) + } + fn is_writable(&self, _: &Path) -> bool { + false + } + fn is_readable_paths(&self, logical: &Path, resolved: &Path) -> bool { + self.read_allowed(logical, resolved) + } + fn can_enumerate_directory(&self, logical: &Path, resolved: &Path) -> bool { + self.enumeration_allowed(logical, resolved) + } + fn traversal_filter(&self) -> Option { + let policy = self.clone(); + Some(Arc::new(move |path, directory| { + if directory { + policy.enumeration_allowed(path, path) + } else { + policy.read_allowed(path, path) + } + })) + } + fn check_cancelled(&self) -> std::io::Result<()> { + self.inner.check_cancelled() + } + fn resolve_access_path(&self, logical: &Path) -> std::io::Result { + self.inner.resolve_access_path(logical) + } + fn open_traversal_root( + &self, + logical: &Path, + resolved: &Path, + ) -> std::io::Result> { + self.require_read(logical, resolved)?; + self.inner.open_traversal_root(logical, resolved) + } + fn open_read_file(&self, logical: &Path, resolved: &Path) -> std::io::Result { + self.require_read(logical, resolved)?; + self.inner.open_read_file(logical, resolved) + } + fn read_metadata(&self, logical: &Path, resolved: &Path) -> std::io::Result { + self.require_read(logical, resolved)?; + self.inner.read_metadata(logical, resolved) + } + fn open_read_dir(&self, logical: &Path, resolved: &Path) -> std::io::Result { + if !self.enumeration_allowed(logical, resolved) { + return Err(std::io::Error::new( + std::io::ErrorKind::PermissionDenied, + "checkout directory enumeration denied", + )); + } + self.inner.open_read_dir(logical, resolved) + } +} + fn path_sets_overlap( left: &Path, left_recursive: bool, @@ -491,6 +602,7 @@ struct LocalWorkdirSessionInner { reject_symlinks: bool, closed: AtomicBool, close_lock: Mutex<()>, + checkout_lock: StdMutex<()>, next_command_id: AtomicU64, commands: Mutex>, command_telemetry: CommandTelemetry, @@ -911,6 +1023,7 @@ impl LocalWorkdirSession { reject_symlinks, closed: AtomicBool::new(false), close_lock: Mutex::new(()), + checkout_lock: StdMutex::new(()), next_command_id: AtomicU64::new(1), commands: Mutex::new(HashMap::new()), command_telemetry: CommandTelemetry::new(), @@ -1129,6 +1242,69 @@ impl LocalWorkdirSession { Ok(()) } + fn checkout_access(&self) -> ScopeAccess { + let guard = self + .operation_guard + .clone() + .unwrap_or_else(|| OperationGuard { + cancelled: Arc::new(AtomicBool::new(false)), + deadline: Instant::now() + Duration::from_secs(30), + }); + ScopeAccess::new( + self.inner.scope.snapshot(), + &self.inner.root, + self.inner.pinned_root.clone(), + self.inner.reject_symlinks, + Some(guard), + ) + } + + fn checkout_root(&self) -> Result, WorkdirError> { + match &self.inner.pinned_root { + Some(root) => Ok(root.clone()), + None => fs_operation::open_root_no_symlinks(&self.inner.root) + .map(Arc::new) + .map_err(|e| WorkdirError::io(Path::new(""), e)), + } + } + + fn checkout_observation( + &self, + path: WorkdirPath, + pinned: &fs_operation::CheckedTarget<'_>, + ) -> Result { + let metadata = pinned + .metadata() + .map_err(|e| WorkdirError::io(Path::new(path.as_str()), e))?; + if fs_operation::identity_validator(&metadata).map_err(|e| checkout_io(&path, e))? + != pinned.validator() + { + return Err(WorkdirError::Conflict( + "checkout changed during observation".into(), + )); + } + let access = self.scope_access(); + let absolute = self.resolve(&path); + let capabilities = if manifest::Scope::is_writable(&access.scope, &absolute) { + self.capabilities() + .intersection(WorkdirSessionCapabilities::READ_WRITE) + } else { + self.capabilities() + .intersection(WorkdirSessionCapabilities::READ_ONLY) + }; + Ok(crate::CheckoutObservation { + path, + kind: if metadata.is_dir() { + crate::EntryKind::Directory + } else { + crate::EntryKind::File + }, + size: metadata.len(), + validator: pinned.validator().to_vec(), + capabilities, + }) + } + fn scope_access(&self) -> ScopeAccess { ScopeAccess::new( self.inner.scope.snapshot(), @@ -1236,6 +1412,255 @@ impl WorkdirSession for LocalWorkdirSession { )) } + async fn checkout_search( + &self, + mut request: crate::CheckoutSearchRequest, + ) -> Result { + use crate::{CheckoutSearchOperation as Op, CheckoutSearchResult as Output}; + self.ensure_capability(request.operation.capability())?; + request.validate()?; + let _serial = self + .inner + .checkout_lock + .try_lock() + .map_err(|_| WorkdirError::Unavailable("checkout provider busy".into()))?; + let provider_root = self.checkout_root()?; + let mut inner = self.checkout_access(); + inner.pinned_root = Some(provider_root); + inner.reject_symlinks = true; + let output_root = self.resolve(&request.output_root); + let logical = request.operation.path().clone(); + let relative = Path::new(logical.as_str()) + .strip_prefix(request.output_root.as_str()) + .map_err(|_| WorkdirError::Denied("checkout search escaped output root".into()))?; + *request.operation.path_mut() = WorkdirPath::new(relative.to_str().ok_or_else(|| { + WorkdirError::InvalidPath("checkout search path is not UTF8".into()) + })?)?; + let source_allows = Arc::new(inner.scope.allow_rules()); + let access = SearchAccess { + inner, + layers: request.scope_layers, + output_root: output_root.clone(), + source_allows, + }; + let result = match request.operation { + Op::List(r) => fs_operation::run_list(&output_root, r, &access).map(Output::List), + Op::Glob(r) => { + fs_operation::run_glob(&output_root, &self.resolve(&logical), r, &access) + .map(Output::Glob) + } + Op::Grep(r) => fs_operation::run_grep(&output_root, self.resolve(&logical), r, &access) + .map(Output::Grep), + }; + result + .map_err(WorkdirError::from) + .map_err(|error| sanitize_error(error, &logical)) + } + + async fn checkout_observe( + &self, + path: WorkdirPath, + ) -> Result { + self.ensure_capability(WorkdirSessionCapability::Read)?; + crate::checkout::validate_path(&path)?; + let _serial = self + .inner + .checkout_lock + .try_lock() + .map_err(|_| WorkdirError::Unavailable("checkout provider busy".into()))?; + let root = self.checkout_root().map_err(|error| match error { + WorkdirError::Io { source, .. } => checkout_observe_io(&path, source), + error => error, + })?; + let access = self.checkout_access(); + let pinned = fs_operation::CheckedTarget::pin( + &root, + Path::new(path.as_str()), + &self.resolve(&path), + &access, + false, + ) + .map_err(|e| checkout_observe_io(&path, e))?; + self.checkout_observation(path.clone(), &pinned) + .map_err(|error| match error { + WorkdirError::Io { source, .. } => checkout_observe_io(&path, source), + error => error, + }) + } + + async fn checkout_execute( + &self, + request: crate::CheckoutRequest, + ) -> Result { + use crate::{CheckoutOperation as Op, CheckoutOutput as Output}; + self.ensure_capability(request.operation.capability())?; + crate::checkout::validate_path(&request.target)?; + if request.validator.is_empty() || request.validator.len() > 256 { + return Err(WorkdirError::InvalidArgument( + "invalid checkout validator".into(), + )); + } + let _serial = self + .inner + .checkout_lock + .try_lock() + .map_err(|_| WorkdirError::Unavailable("checkout provider busy".into()))?; + let path = request.target; + let root = self + .checkout_root() + .map_err(|error| checkout_execute_error(&path, error))?; + let access = self.checkout_access(); + let mut pinned = fs_operation::CheckedTarget::pin( + &root, + Path::new(path.as_str()), + &self.resolve(&path), + &access, + request.operation.capability() != WorkdirSessionCapability::Read, + ) + .map_err(|e| checkout_io(&path, e))?; + if pinned.validator() != request.validator { + return Err(WorkdirError::Conflict( + "checkout observation changed; observe and read again".into(), + )); + } + let is_create = matches!(&request.operation, Op::Create { .. }); + let output = match request.operation { + Op::Read { + offset, + limit, + max_bytes, + } => { + if limit > 1_000_000 + || max_bytes > BoundedReadLimits::EXTERNAL_DEFAULT.max_response_bytes + { + return Err(WorkdirError::InvalidArgument( + "checkout Read exceeds provider bounds".into(), + )); + } + fs_operation::run_read_bounded( + &self.inner.root, + ReadRequest { + path: path.clone(), + offset, + limit, + max_bytes, + }, + &pinned, + self.inner + .read_limits + .unwrap_or(BoundedReadLimits::EXTERNAL_DEFAULT), + ) + .map(Output::Read) + } + Op::Write { + content, + expected_hash, + } => fs_operation::run_write( + &self.inner.root, + WriteRequest { + path: path.clone(), + content, + expected_hash: Some(expected_hash), + }, + &pinned, + ) + .map(Output::Write), + Op::Edit { + old_string, + new_string, + replace_all, + expected_hash, + } => { + if old_string.len().saturating_add(new_string.len()) > 8 * 1024 * 1024 { + return Err(WorkdirError::InvalidArgument( + "checkout Edit exceeds provider bounds".into(), + )); + } + fs_operation::run_edit( + &self.inner.root, + EditRequest { + path: path.clone(), + old_string, + new_string, + replace_all, + expected_hash, + }, + &pinned, + ) + .map(Output::Edit) + } + Op::Create { + path: destination, + content, + } => { + let suffix = crate::checkout::create_relative_path(&path, &destination)?; + let create = pinned.create_access(suffix, self.resolve(&destination)); + fs_operation::run_write( + &self.inner.root, + WriteRequest { + path: destination, + content, + expected_hash: None, + }, + &create, + ) + .map(Output::Write) + } + }; + let output = output.map_err(|error| { + if pinned.effects_possible() { + WorkdirError::OutcomeUnknown( + "checkout save or parent creation may have effects; inspect before retry" + .into(), + ) + } else { + checkout_execute_error(&path, WorkdirError::from(error)) + } + })?; + if matches!(&output, Output::Read(_)) { + pinned.confirm().map_err(|e| checkout_io(&path, e))?; + return Ok(crate::CheckoutResult { + observation: self + .checkout_observation(path.clone(), &pinned) + .map_err(|error| checkout_execute_error(&path, error))?, + output, + }); + } + if is_create { + pinned.refresh_directory_observation().map_err(|_| { + WorkdirError::OutcomeUnknown( + "checkout Create may have effects but parent observation failed".into(), + ) + })?; + let observation = self.checkout_observation(path, &pinned).map_err(|_| { + WorkdirError::OutcomeUnknown( + "checkout Create may have effects but parent observation failed".into(), + ) + })?; + return Ok(crate::CheckoutResult { + observation, + output, + }); + } + let after = fs_operation::CheckedTarget::pin( + &root, + Path::new(path.as_str()), + &self.resolve(&path), + &access, + true, + ) + .map_err(|_| { + WorkdirError::OutcomeUnknown("checkout saved but post-save observation failed".into()) + })?; + let observation = self.checkout_observation(path, &after).map_err(|_| { + WorkdirError::OutcomeUnknown("checkout saved but post-save observation failed".into()) + })?; + Ok(crate::CheckoutResult { + observation, + output, + }) + } + async fn stat(&self, request: StatRequest) -> Result { self.ensure_capability(WorkdirSessionCapability::Read)?; let logical = request.path.clone(); @@ -1247,6 +1672,11 @@ impl WorkdirSession for LocalWorkdirSession { } async fn read(&self, request: ReadRequest) -> Result { + let _serial = self + .inner + .checkout_lock + .try_lock() + .map_err(|_| WorkdirError::Unavailable("filesystem provider busy".into()))?; self.ensure_capability(WorkdirSessionCapability::Read)?; let logical = request.path.clone(); self.validate_operation_path(&request.path)?; @@ -1277,6 +1707,11 @@ impl WorkdirSession for LocalWorkdirSession { } async fn write(&self, request: WriteRequest) -> Result { + let _serial = self + .inner + .checkout_lock + .try_lock() + .map_err(|_| WorkdirError::Unavailable("filesystem provider busy".into()))?; self.ensure_capability(WorkdirSessionCapability::Write)?; let logical = request.path.clone(); self.validate_operation_path(&request.path)?; @@ -1287,6 +1722,11 @@ impl WorkdirSession for LocalWorkdirSession { } async fn edit(&self, request: EditRequest) -> Result { + let _serial = self + .inner + .checkout_lock + .try_lock() + .map_err(|_| WorkdirError::Unavailable("filesystem provider busy".into()))?; self.ensure_capability(WorkdirSessionCapability::Edit)?; let logical = request.path.clone(); self.validate_operation_path(&request.path)?; @@ -1562,6 +2002,44 @@ fn command_output_page(output: &CommandOutput, cursor: usize, limit: usize) -> C } } +// Discovery of an absent route is not a stale-validator execution attempt. +fn checkout_observe_io(path: &WorkdirPath, error: std::io::Error) -> WorkdirError { + if matches!( + error.kind(), + std::io::ErrorKind::NotFound | std::io::ErrorKind::NotADirectory + ) { + WorkdirError::NotFound(PathBuf::from(path.as_str())) + } else { + checkout_io(path, error) + } +} + +// All pre-effect execution stages preserve the same stale-name classification, +// including FsError::NotFound produced by the shared engine after pinning. +fn checkout_execute_error(path: &WorkdirPath, error: WorkdirError) -> WorkdirError { + match error { + WorkdirError::NotFound(_) => { + WorkdirError::Conflict("checkout observation changed; observe and read again".into()) + } + WorkdirError::Io { source, .. } => checkout_io(path, source), + error => sanitize_error(error, path), + } +} + +fn checkout_io(path: &WorkdirPath, error: std::io::Error) -> WorkdirError { + if matches!( + error.kind(), + std::io::ErrorKind::WouldBlock + | std::io::ErrorKind::NotFound + | std::io::ErrorKind::NotADirectory + | std::io::ErrorKind::AlreadyExists + ) { + WorkdirError::Conflict("checkout observation changed; observe and read again".into()) + } else { + sanitize_error(WorkdirError::io(Path::new(path.as_str()), error), path) + } +} + fn sanitize_error(error: WorkdirError, logical: &WorkdirPath) -> WorkdirError { let path = PathBuf::from(logical.as_str()); match error { @@ -2083,6 +2561,47 @@ mod tests { use std::fs; use tempfile::TempDir; + #[test] + fn checkout_error_mapping_distinguishes_absence_from_stale_execution_at_all_boundaries() { + let path = WorkdirPath::new("missing").unwrap(); + for kind in [ + std::io::ErrorKind::NotFound, + std::io::ErrorKind::NotADirectory, + ] { + assert!( + matches!(checkout_observe_io(&path, std::io::Error::from(kind)), WorkdirError::NotFound(p) if p == Path::new("missing")) + ); + assert!(matches!( + checkout_io(&path, std::io::Error::from(kind)), + WorkdirError::Conflict(_) + )); + assert!(matches!( + checkout_execute_error( + &path, + WorkdirError::io(Path::new(""), std::io::Error::from(kind)) + ), + WorkdirError::Conflict(_) + )); + } + let shared_error = + WorkdirError::from(fs_operation::FsError::NotFound(PathBuf::from("missing"))); + assert!(matches!( + checkout_execute_error(&path, shared_error), + WorkdirError::Conflict(_) + )); + assert!(matches!( + checkout_observe_io(&path, std::io::Error::from(std::io::ErrorKind::WouldBlock)), + WorkdirError::Conflict(_) + )); + assert!(matches!( + checkout_execute_error( + &path, + WorkdirError::OutcomeUnknown("possible save effects".into()) + ), + WorkdirError::OutcomeUnknown(_) + )); + } + fn make_fs(dir: &TempDir) -> LocalWorkdirSession { LocalWorkdirSession::new( Scope::writable(dir.path()).unwrap(), diff --git a/crates/workdir/src/router.rs b/crates/workdir/src/router.rs index 270f0ec90..02e00004d 100644 --- a/crates/workdir/src/router.rs +++ b/crates/workdir/src/router.rs @@ -248,6 +248,29 @@ impl WorkdirSession for RoutedWorkdirSession { self.state.session.scope_rules_overlap(request).await } + async fn checkout_search( + &self, + request: crate::CheckoutSearchRequest, + ) -> Result { + let _active = self.state.enter()?; + self.state.session.checkout_search(request).await + } + + async fn checkout_observe( + &self, + path: crate::WorkdirPath, + ) -> Result { + let _active = self.state.enter()?; + self.state.session.checkout_observe(path).await + } + async fn checkout_execute( + &self, + request: crate::CheckoutRequest, + ) -> Result { + let _active = self.state.enter()?; + self.state.session.checkout_execute(request).await + } + async fn stat(&self, request: StatRequest) -> Result { let _active = self.state.enter()?; self.state.session.stat(request).await @@ -650,6 +673,42 @@ mod tests { )) } + #[tokio::test] + async fn checkout_routes_use_active_operation_detach_fence_and_old_handles_stay_closed() { + let root = TempDir::new().unwrap(); + std::fs::write(root.path().join("a"), "contents").unwrap(); + let router = WorkdirSessionRouter::new(); + let alias = WorkdirAttachmentAlias::new("checkout").unwrap(); + router.attach(alias.clone(), session("wd", &root)).unwrap(); + let old = router.resolve(Some("checkout")).unwrap().session; + old.checkout_observe(crate::WorkdirPath::new("a").unwrap()) + .await + .unwrap(); + let state = router.sessions.read().unwrap().get(&alias).unwrap().clone(); + let active = state.enter().unwrap(); + assert!(router.begin_detach(&alias).await.is_err()); + drop(active); + router.detach(&alias).await.unwrap(); + assert!( + old.checkout_observe(crate::WorkdirPath::new("a").unwrap()) + .await + .is_err() + ); + router.attach(alias, session("wd-new", &root)).unwrap(); + router + .resolve(Some("checkout")) + .unwrap() + .session + .checkout_observe(crate::WorkdirPath::new("a").unwrap()) + .await + .unwrap(); + assert!( + old.checkout_observe(crate::WorkdirPath::new("a").unwrap()) + .await + .is_err() + ); + } + #[tokio::test] async fn router_holds_multiple_aliases_without_implicit_primary() { let left = TempDir::new().unwrap(); diff --git a/crates/workdir/src/scope.rs b/crates/workdir/src/scope.rs index a58e0b01d..2f798d4b5 100644 --- a/crates/workdir/src/scope.rs +++ b/crates/workdir/src/scope.rs @@ -994,6 +994,138 @@ impl WorkdirSession for ScopedWorkdirSession { self.capabilities } + async fn checkout_search( + &self, + mut request: crate::CheckoutSearchRequest, + ) -> Result { + let _scope_guard = self.scope_lock.lock().await; + request.validate()?; + let path = self + .resolve_operation_path(request.operation.path(), WorkdirToolScopePermission::Read) + .await?; + self.ensure_read(&path, request.operation.capability())?; + *request.operation.path_mut() = path; + request.output_root = self.resolve_path(&request.output_root)?; + for layer in &mut request.scope_layers { + for rule in layer { + rule.target = self.resolve_path(&rule.target)?; + } + } + if let Some(rules) = &self.scope { + request.scope_layers.push(rules.clone()); + } + request.validate()?; + // Source results already use the composed output_root, not this source's + // cwd. Rewriting them here would duplicate cwd through nested wrappers. + let result = self.source.checkout_search(request).await?; + self.ensure_active()?; + Ok(result) + } + + async fn checkout_observe( + &self, + caller_path: crate::WorkdirPath, + ) -> Result { + let _scope_guard = self.scope_lock.lock().await; + let path = self + .resolve_operation_path(&caller_path, WorkdirToolScopePermission::Read) + .await?; + self.ensure_read(&path, WorkdirSessionCapability::Read)?; + let mut observation = self.source.checkout_observe(path.clone()).await?; + observation.capabilities = observation.capabilities.intersection(self.capabilities()); + // Capability bits must not advertise authority over a read-only path or + // a path whose write authority is currently leased to a child. + if self + .resolve_operation_path(&caller_path, WorkdirToolScopePermission::Write) + .await + .is_err() + || self + .ensure_write(&path, WorkdirSessionCapability::Write) + .is_err() + { + observation.capabilities = observation + .capabilities + .intersection(WorkdirSessionCapabilities::READ_ONLY); + } + observation.path = caller_path; + Ok(observation) + } + async fn checkout_execute( + &self, + mut request: crate::CheckoutRequest, + ) -> Result { + let _scope_guard = self.scope_lock.lock().await; + let caller_path = request.target.clone(); + let capability = request.operation.capability(); + let permission = if capability == WorkdirSessionCapability::Read { + WorkdirToolScopePermission::Read + } else { + WorkdirToolScopePermission::Write + }; + let path = self + .resolve_operation_path(&request.target, permission) + .await?; + if permission == WorkdirToolScopePermission::Read { + self.ensure_read(&path, capability)?; + } else { + self.ensure_write(&path, capability)?; + } + let result_path = caller_path; + if let crate::CheckoutOperation::Create { + path: destination, .. + } = &mut request.operation + { + let suffix = crate::checkout::create_relative_path(&request.target, destination)?; + let mut intermediate = std::path::PathBuf::from(request.target.as_str()); + let mut components = suffix.components().peekable(); + while let Some(component) = components.next() { + intermediate.push(component.as_os_str()); + if components.peek().is_some() { + let intermediate = + crate::WorkdirPath::new(intermediate.to_str().ok_or_else(|| { + WorkdirError::InvalidPath("non-UTF8 Create path".into()) + })?)?; + let resolved = self + .resolve_operation_path(&intermediate, WorkdirToolScopePermission::Write) + .await?; + self.ensure_write(&resolved, WorkdirSessionCapability::Write)?; + } + } + let resolved = self + .resolve_operation_path(destination, WorkdirToolScopePermission::Write) + .await?; + self.ensure_write(&resolved, WorkdirSessionCapability::Write)?; + *destination = resolved; + } + request.target = path; + let mut result = self.source.checkout_execute(request).await?; + result.observation.path = result_path.clone(); + result.observation.capabilities = result + .observation + .capabilities + .intersection(self.capabilities()); + if self + .resolve_operation_path(&result_path, WorkdirToolScopePermission::Write) + .await + .is_err() + || self + .ensure_write( + &self.resolve_path(&result_path)?, + WorkdirSessionCapability::Write, + ) + .is_err() + { + result.observation.capabilities = result + .observation + .capabilities + .intersection(WorkdirSessionCapabilities::READ_ONLY); + } + if let crate::CheckoutOutput::Read(read) = &mut result.output { + read.path = result_path; + } + Ok(result) + } + async fn stat(&self, mut request: StatRequest) -> Result { let path = self .resolve_operation_path(&request.path, WorkdirToolScopePermission::Read) @@ -1048,6 +1180,19 @@ impl WorkdirSession for ScopedWorkdirSession { } async fn list(&self, mut request: ListRequest) -> Result { + if self.scope.is_some() || !self.cwd.is_root() { + return match self + .checkout_search(crate::CheckoutSearchRequest::new( + crate::CheckoutSearchOperation::List(request), + )) + .await? + { + crate::CheckoutSearchResult::List(result) => Ok(result), + _ => Err(WorkdirError::Unavailable( + "mismatched checkout search result".into(), + )), + }; + } let path = self .resolve_operation_path(&request.path, WorkdirToolScopePermission::Read) .await?; @@ -1057,6 +1202,19 @@ impl WorkdirSession for ScopedWorkdirSession { } async fn glob(&self, mut request: GlobRequest) -> Result { + if self.scope.is_some() || !self.cwd.is_root() { + return match self + .checkout_search(crate::CheckoutSearchRequest::new( + crate::CheckoutSearchOperation::Glob(request), + )) + .await? + { + crate::CheckoutSearchResult::Glob(result) => Ok(result), + _ => Err(WorkdirError::Unavailable( + "mismatched checkout search result".into(), + )), + }; + } let path = self .resolve_operation_path(&request.path, WorkdirToolScopePermission::Read) .await?; @@ -1066,6 +1224,19 @@ impl WorkdirSession for ScopedWorkdirSession { } async fn grep(&self, mut request: GrepRequest) -> Result { + if self.scope.is_some() || !self.cwd.is_root() { + return match self + .checkout_search(crate::CheckoutSearchRequest::new( + crate::CheckoutSearchOperation::Grep(request), + )) + .await? + { + crate::CheckoutSearchResult::Grep(result) => Ok(result), + _ => Err(WorkdirError::Unavailable( + "mismatched checkout search result".into(), + )), + }; + } let path = self .resolve_operation_path(&request.path, WorkdirToolScopePermission::Read) .await?; @@ -1284,6 +1455,44 @@ impl WorkdirSession for ReadOnlyWorkdirSession { self.inner.scope_rules_overlap(request).await } + async fn checkout_search( + &self, + request: crate::CheckoutSearchRequest, + ) -> Result { + if !self + .capabilities() + .intersection(self.inner.capabilities()) + .supports(request.operation.capability()) + { + return Err(WorkdirError::Unsupported(request.operation.capability())); + } + self.inner.checkout_search(request).await + } + + async fn checkout_observe( + &self, + path: crate::WorkdirPath, + ) -> Result { + let mut observation = self.inner.checkout_observe(path).await?; + observation.capabilities = observation + .capabilities + .intersection(WorkdirSessionCapabilities::READ_ONLY); + Ok(observation) + } + async fn checkout_execute( + &self, + request: crate::CheckoutRequest, + ) -> Result { + if request.operation.capability() != WorkdirSessionCapability::Read { + return Err(WorkdirError::Unsupported(request.operation.capability())); + } + let mut result = self.inner.checkout_execute(request).await?; + result.observation.capabilities = result + .observation + .capabilities + .intersection(WorkdirSessionCapabilities::READ_ONLY); + Ok(result) + } async fn stat(&self, request: StatRequest) -> Result { self.inner.stat(request).await } @@ -1654,6 +1863,53 @@ mod tests { (WorkdirToolBroker::new(source), receiver, release) } + #[tokio::test] + async fn checkout_search_unsupported_source_never_falls_back_to_broad_traversal() { + let root = TempDir::new().unwrap(); + fs::create_dir(root.path().join("child")).unwrap(); + fs::write(root.path().join("child/a"), "needle").unwrap(); + let (parent, _, _) = blocking_session(root.path()); + // Unscoped normal Tools still use their established provider contract. + assert_eq!( + parent + .list(ListRequest { + path: fs_path("child"), + limit: 100 + }) + .await + .unwrap() + .entries + .len(), + 1 + ); + let child = parent + .scope(request("child", WorkdirToolScopePermission::Read)) + .await + .unwrap(); + assert!(matches!( + child + .tool_session() + .list(ListRequest { + path: FsPath::root(), + limit: 100 + }) + .await, + Err(WorkdirError::UnsupportedOperation(_)) + )); + assert!(matches!( + child + .tool_session() + .checkout_search(crate::CheckoutSearchRequest::new( + crate::CheckoutSearchOperation::List(ListRequest { + path: FsPath::root(), + limit: 100 + }) + )) + .await, + Err(WorkdirError::UnsupportedOperation(_)) + )); + } + fn request(path: &str, permission: WorkdirToolScopePermission) -> WorkdirToolScope { WorkdirToolScope { rules: vec![WorkdirToolScopeRule { diff --git a/crates/worker-runtime/src/catalog.rs b/crates/worker-runtime/src/catalog.rs index ed393fca9..2b00cac60 100644 --- a/crates/worker-runtime/src/catalog.rs +++ b/crates/worker-runtime/src/catalog.rs @@ -258,6 +258,8 @@ impl std::fmt::Debug for WorkspaceApiRef { } } +pub use worker::BackendJobExecutionBinding; + /// Canonical Runtime Worker creation request. /// /// Browser/product launch semantics are resolved by a backend before this @@ -302,6 +304,9 @@ pub struct CreateWorkerRequest { /// may enable subjektiv policy, but only this host-owned bit activates it. #[serde(default, skip_serializing_if = "is_false")] pub subjektiv_attached: bool, + /// Backend-only immutable Job/attempt capability. Never inferred from Profile policy. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub backend_job: Option, } /// Last persisted Worker lifecycle status. @@ -334,6 +339,8 @@ pub(crate) enum WorkerRestoreIntent { /// Lightweight persisted Worker identity projection. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct WorkerSummary { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub restore_observation_token: Option, pub worker_ref: WorkerRef, pub worker_id: WorkerId, pub status: WorkerStatus, @@ -359,6 +366,8 @@ pub struct WorkerSummary { /// Full persisted Worker identity and lifecycle detail. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct WorkerDetail { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub restore_observation_token: Option, pub worker_ref: WorkerRef, pub worker_id: WorkerId, pub status: WorkerStatus, diff --git a/crates/worker-runtime/src/error.rs b/crates/worker-runtime/src/error.rs index bbbea4435..ef9609cc0 100644 --- a/crates/worker-runtime/src/error.rs +++ b/crates/worker-runtime/src/error.rs @@ -12,6 +12,9 @@ pub enum RuntimeError { #[error("initial worker input must be user input, got {kind}")] InvalidInitialInputKind { kind: String }, + #[error("worker {worker_id} Restore observation conflicts with the current operation")] + RestoreObservationConflict { worker_id: WorkerId }, + #[error("worker {worker_id} was not found")] WorkerNotFound { worker_id: WorkerId }, diff --git a/crates/worker-runtime/src/execution.rs b/crates/worker-runtime/src/execution.rs index a832d5350..86af999e1 100644 --- a/crates/worker-runtime/src/execution.rs +++ b/crates/worker-runtime/src/execution.rs @@ -207,34 +207,6 @@ impl WorkerExecutionResult { } } -/// Opaque per-Worker execution handle returned by a backend. -/// -/// The handle is a typed token for routing calls back into the same backend. It -/// intentionally contains no socket path, process id, credential, manifest path, -/// or session path. -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct WorkerExecutionHandle { - worker_ref: WorkerRef, - backend_id: String, -} - -impl WorkerExecutionHandle { - pub fn new(worker_ref: WorkerRef, backend_id: impl Into) -> Self { - Self { - worker_ref, - backend_id: backend_id.into(), - } - } - - pub fn worker_ref(&self) -> &WorkerRef { - &self.worker_ref - } - - pub fn backend_id(&self) -> &str { - &self.backend_id - } -} - /// Runtime hooks available to an execution backend for one Worker. #[cfg(feature = "ws-server")] type WorkerObservationPublisher = Arc< @@ -348,7 +320,17 @@ impl WorkerExecutionContext { let Some(event) = event else { return Ok(()); }; - (self.observation_publisher)(self.worker_ref.clone(), event)?; + if let Err(error) = (self.observation_publisher)(self.worker_ref.clone(), event.clone()) + { + let mut state = self + .publication_state + .lock() + .map_err(|_| RuntimeError::StatePoisoned)?; + if let WorkerExecutionPublicationState::Activating(events) = &mut *state { + events.push_front(event); + } + return Err(error); + } } } @@ -443,7 +425,6 @@ pub struct WorkerExecutionRestoreRequest { pub struct WorkerExecutionStopRequest { pub operation_id: WorkerLifecycleOperationId, pub worker_ref: WorkerRef, - pub handle: Option, } /// Runtime-side request to refresh the latest Workspace Config before Worker creation. @@ -466,7 +447,6 @@ pub enum WorkspaceConfigFetchResult { #[derive(Clone, Debug)] pub enum WorkerExecutionSpawnResult { Connected { - handle: WorkerExecutionHandle, worker_state: protocol::WorkerStateSnapshot, workdir_attachments: Vec, }, @@ -475,11 +455,10 @@ pub enum WorkerExecutionSpawnResult { /// Live work started, failed, and all operation-owned resources were joined/released. RolledBack(WorkerExecutionResult), /// The operation crossed the live side-effect boundary and could not prove - /// either commit or complete cleanup. Optional handle/state fields preserve + /// either commit or complete cleanup. Optional state fields preserve /// concrete execution evidence for retry reconciliation. ReconciliationRequired { result: WorkerExecutionResult, - handle: Option, worker_state: Option, workdir_attachments: Vec, }, @@ -490,12 +469,10 @@ pub enum WorkerExecutionSpawnResult { impl WorkerExecutionSpawnResult { pub fn connected( - handle: WorkerExecutionHandle, worker_state: protocol::WorkerStateSnapshot, workdir_attachments: Vec, ) -> Self { Self::Connected { - handle, worker_state, workdir_attachments, } @@ -521,6 +498,57 @@ pub struct WorkerSessionAttachmentRequest { pub attachment_id: String, } +/// One real protocol connection, bound to the execution that was attached. +/// This is not a routing token and is never persisted as Worker authority. +#[cfg(feature = "ws-server")] +pub struct WorkerProtocolTransport { + pub(crate) worker_ref: WorkerRef, + pub snapshot: protocol::Event, + pub events: tokio::sync::mpsc::Receiver, + dispatch: + Arc Result, WorkerExecutionResult> + Send + Sync>, + validate: Arc Result<(), WorkerExecutionResult> + Send + Sync>, +} + +#[cfg(feature = "ws-server")] +impl WorkerProtocolTransport { + pub fn new( + worker_ref: WorkerRef, + snapshot: protocol::Event, + events: tokio::sync::mpsc::Receiver, + dispatch: Arc< + dyn Fn(Method) -> Result, WorkerExecutionResult> + Send + Sync, + >, + validate: Arc Result<(), WorkerExecutionResult> + Send + Sync>, + ) -> Self { + Self { + worker_ref, + snapshot, + events, + dispatch, + validate, + } + } + + pub fn dispatch(&self, method: Method) -> Result, WorkerExecutionResult> { + (self.dispatch)(method) + } + + pub fn validate(&self) -> Result<(), WorkerExecutionResult> { + (self.validate)() + } +} + +#[cfg(feature = "ws-server")] +impl fmt::Debug for WorkerProtocolTransport { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("WorkerProtocolTransport") + .finish_non_exhaustive() + } +} + +/// The connection selects this backend; WorkerRef identifies an execution, +/// never a capability or authorization token. pub trait WorkerExecutionBackend: Send + Sync + 'static { fn backend_id(&self) -> &str; @@ -580,8 +608,9 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static { fn activate_restored_worker( &self, _operation_id: WorkerLifecycleOperationId, - _handle: &WorkerExecutionHandle, + worker_ref: &WorkerRef, ) -> Result<(), String> { + let _ = worker_ref; Ok(()) } @@ -673,20 +702,17 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static { Ok(()) } - fn dispatch_input( - &self, - handle: &WorkerExecutionHandle, - input: WorkerInput, - ) -> WorkerExecutionResult; + fn dispatch_input(&self, worker_ref: &WorkerRef, input: WorkerInput) -> WorkerExecutionResult; fn upload_file( &self, - _handle: &WorkerExecutionHandle, + worker_ref: &WorkerRef, _file_name: &str, _media_type: &str, _content: &[u8], _context: Option<&session_store::UploadedFileUploadContext>, ) -> Result { + let _ = worker_ref; Err(WorkerExecutionResult::unsupported( WorkerExecutionOperation::UploadFile, "execution backend does not support file upload", @@ -695,20 +721,18 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static { fn delete_uploaded_file( &self, - _handle: &WorkerExecutionHandle, + worker_ref: &WorkerRef, _artifact_id: &str, ) -> WorkerExecutionResult { + let _ = worker_ref; WorkerExecutionResult::unsupported( WorkerExecutionOperation::DeleteUploadedFile, "execution backend does not support uploaded-file deletion", ) } - fn dispatch_method( - &self, - _handle: &WorkerExecutionHandle, - _method: Method, - ) -> WorkerExecutionResult { + fn dispatch_method(&self, worker_ref: &WorkerRef, _method: Method) -> WorkerExecutionResult { + let _ = worker_ref; WorkerExecutionResult::unsupported( WorkerExecutionOperation::ProtocolMethod, "execution backend does not support direct Worker protocol methods", @@ -717,31 +741,29 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static { fn worker_completions( &self, - _handle: &WorkerExecutionHandle, + worker_ref: &WorkerRef, _kind: protocol::CompletionKind, _prefix: &str, + _context: Option<&protocol::CompletionContext>, ) -> Vec { + let _ = worker_ref; Vec::new() } fn stop_worker_operation(&self, request: WorkerExecutionStopRequest) -> WorkerExecutionResult { - let Some(handle) = request.handle.as_ref() else { - return WorkerExecutionResult::unsupported( - WorkerExecutionOperation::Stop, - "execution backend cannot reconcile a stopped Worker without a live handle", - ); - }; - self.stop_worker(handle) + self.stop_worker(&request.worker_ref) } - fn stop_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult { + fn stop_worker(&self, worker_ref: &WorkerRef) -> WorkerExecutionResult { + let _ = worker_ref; WorkerExecutionResult::unsupported( WorkerExecutionOperation::Stop, "execution backend does not support stopping workers", ) } - fn cancel_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult { + fn cancel_worker(&self, worker_ref: &WorkerRef) -> WorkerExecutionResult { + let _ = worker_ref; WorkerExecutionResult::unsupported( WorkerExecutionOperation::Cancel, "execution backend does not support cancelling workers", @@ -749,7 +771,20 @@ pub trait WorkerExecutionBackend: Send + Sync + 'static { } #[cfg(feature = "ws-server")] - fn worker_snapshot(&self, _handle: &WorkerExecutionHandle) -> Option { + fn attach_worker_protocol( + self: Arc, + worker_ref: &WorkerRef, + ) -> Result { + let _ = worker_ref; + Err(WorkerExecutionResult::unsupported( + WorkerExecutionOperation::ProtocolMethod, + "execution backend does not support bound Worker protocol connections", + )) + } + + #[cfg(feature = "ws-server")] + fn worker_snapshot(&self, worker_ref: &WorkerRef) -> Option { + let _ = worker_ref; None } } @@ -827,9 +862,10 @@ impl WorkerExecutionBackendRef { pub(crate) fn activate_restored_worker( &self, operation_id: WorkerLifecycleOperationId, - handle: &WorkerExecutionHandle, + worker_ref: &WorkerRef, ) -> Result<(), String> { - self.backend.activate_restored_worker(operation_id, handle) + self.backend + .activate_restored_worker(operation_id, worker_ref) } pub(crate) fn restore_worker( @@ -895,55 +931,62 @@ impl WorkerExecutionBackendRef { pub(crate) fn dispatch_input( &self, - handle: &WorkerExecutionHandle, + worker_ref: &WorkerRef, input: WorkerInput, ) -> WorkerExecutionResult { - self.backend.dispatch_input(handle, input) + self.backend.dispatch_input(worker_ref, input) } pub(crate) fn upload_file( &self, - handle: &WorkerExecutionHandle, + worker_ref: &WorkerRef, file_name: &str, media_type: &str, content: &[u8], context: Option<&session_store::UploadedFileUploadContext>, ) -> Result { self.backend - .upload_file(handle, file_name, media_type, content, context) + .upload_file(worker_ref, file_name, media_type, content, context) } pub(crate) fn delete_uploaded_file( &self, - handle: &WorkerExecutionHandle, + worker_ref: &WorkerRef, artifact_id: &str, ) -> WorkerExecutionResult { - self.backend.delete_uploaded_file(handle, artifact_id) + self.backend.delete_uploaded_file(worker_ref, artifact_id) } pub(crate) fn dispatch_method( &self, - handle: &WorkerExecutionHandle, + worker_ref: &WorkerRef, method: Method, ) -> WorkerExecutionResult { - self.backend.dispatch_method(handle, method) + self.backend.dispatch_method(worker_ref, method) } #[cfg(feature = "ws-server")] - pub(crate) fn worker_snapshot( + pub(crate) fn attach_worker_protocol( &self, - handle: &WorkerExecutionHandle, - ) -> Option { - self.backend.worker_snapshot(handle) + worker_ref: &WorkerRef, + ) -> Result { + Arc::clone(&self.backend).attach_worker_protocol(worker_ref) + } + + #[cfg(feature = "ws-server")] + pub(crate) fn worker_snapshot(&self, worker_ref: &WorkerRef) -> Option { + self.backend.worker_snapshot(worker_ref) } pub(crate) fn worker_completions( &self, - handle: &WorkerExecutionHandle, + worker_ref: &WorkerRef, kind: protocol::CompletionKind, prefix: &str, + context: Option<&protocol::CompletionContext>, ) -> Vec { - self.backend.worker_completions(handle, kind, prefix) + self.backend + .worker_completions(worker_ref, kind, prefix, context) } pub(crate) fn stop_worker_operation( @@ -953,12 +996,12 @@ impl WorkerExecutionBackendRef { self.backend.stop_worker_operation(request) } - pub(crate) fn stop_worker(&self, handle: &WorkerExecutionHandle) -> WorkerExecutionResult { - self.backend.stop_worker(handle) + pub(crate) fn stop_worker(&self, worker_ref: &WorkerRef) -> WorkerExecutionResult { + self.backend.stop_worker(worker_ref) } - pub(crate) fn cancel_worker(&self, handle: &WorkerExecutionHandle) -> WorkerExecutionResult { - self.backend.cancel_worker(handle) + pub(crate) fn cancel_worker(&self, worker_ref: &WorkerRef) -> WorkerExecutionResult { + self.backend.cancel_worker(worker_ref) } } diff --git a/crates/worker-runtime/src/fs_store.rs b/crates/worker-runtime/src/fs_store.rs index c2fb52bf7..0e5c8b5c5 100644 --- a/crates/worker-runtime/src/fs_store.rs +++ b/crates/worker-runtime/src/fs_store.rs @@ -493,7 +493,12 @@ pub(crate) struct PersistedWorkerRestoreOperation { #[serde(deny_unknown_fields)] pub(crate) struct PersistedWorkerStopOperation { pub(crate) operation_id: WorkerLifecycleOperationId, - pub(crate) request: CreateWorkerRequest, + /// Restore resources remain operation-owned until stop confirms cleanup. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) pending_restore: Option, + /// Stop can reconcile an identity whose execution metadata is unavailable. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub(crate) request: Option, pub(crate) binding: Option, pub(crate) restore_intent: WorkerRestoreIntent, pub(crate) last_settled_status: WorkerStatus, @@ -515,6 +520,7 @@ pub(crate) enum PersistedWorkerExecutionState { #[derive(Clone, Debug)] pub(crate) struct PersistedWorkerRecord { + pub(crate) restore_guard: crate::runtime::RestoreGuard, pub(crate) worker_ref: WorkerRef, pub(crate) worker_id: WorkerId, pub(crate) profile: ProfileSelector, @@ -1814,6 +1820,8 @@ impl RuntimeSnapshot { #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct WorkerAggregateRecord { + #[serde(default)] + restore_guard: crate::runtime::RestoreGuard, schema_version: u32, worker_ref: WorkerRef, worker_id: WorkerId, @@ -1869,6 +1877,7 @@ impl WorkerAggregateRecord { }; Self { schema_version: SCHEMA_VERSION, + restore_guard: worker.restore_guard.clone(), worker_ref: worker.worker_ref.clone(), worker_id: worker.worker_id, profile: worker.profile.clone(), @@ -1897,6 +1906,7 @@ impl WorkerAggregateRecord { } fn validate(self, path: &Path) -> Result { + let restore_guard = self.restore_guard; let last_finished_submission_request_id = self.last_finished_submission_request_id; let identity = WorkerIdentityRecord { schema_version: self.schema_version, @@ -1942,7 +1952,7 @@ impl WorkerAggregateRecord { }); } ( - &restore.request, + Some(&restore.request), &restore.binding, if restore.last_settled_status.is_active() { WorkerRestoreIntent::Automatic @@ -1953,7 +1963,7 @@ impl WorkerAggregateRecord { ) } PersistedWorkerLifecycleOperation::Stop(stop) => { - if !stop.last_settled_status.is_active() { + if !stop.last_settled_status.is_active() && stop.pending_restore.is_none() { return Err(RuntimeError::StoreCorrupt { operation: "read Worker stop reconciliation", path: path.to_path_buf(), @@ -1961,8 +1971,18 @@ impl WorkerAggregateRecord { .to_string(), }); } + if stop.request.is_none() + && (stop.binding.is_some() || stop.pending_restore.is_some()) + { + return Err(RuntimeError::StoreCorrupt { + operation: "read Worker stop reconciliation", + path: path.to_path_buf(), + message: "metadata-free stop cannot carry restore authority" + .to_string(), + }); + } ( - &stop.request, + stop.request.as_ref(), &stop.binding, stop.restore_intent, stop.last_settled_status, @@ -1978,18 +1998,21 @@ impl WorkerAggregateRecord { .to_string(), }); } - WorkerExecutionRecord { - schema_version: SCHEMA_VERSION, - request: request.clone(), - binding: binding.clone(), - restore_intent, + if let Some(request) = request { + WorkerExecutionRecord { + schema_version: SCHEMA_VERSION, + request: request.clone(), + binding: binding.clone(), + restore_intent, + } + .validate_for_schema(&identity, path, SCHEMA_VERSION)?; } - .validate_for_schema(&identity, path, SCHEMA_VERSION)?; PersistedWorkerExecutionState::ReconciliationRequired(operation) } WorkerExecutionStateRecord::Unavailable => PersistedWorkerExecutionState::Unavailable, }; let mut persisted = identity.into_persisted(execution_state); + persisted.restore_guard = restore_guard; persisted.last_finished_submission_request_id = last_finished_submission_request_id; Ok(persisted) } @@ -2163,6 +2186,7 @@ impl WorkerIdentityRecord { self.workdir_attachments }; PersistedWorkerRecord { + restore_guard: crate::runtime::RestoreGuard::default(), worker_ref: self.worker_ref, worker_id: self.worker_id, profile: self.profile, diff --git a/crates/worker-runtime/src/http_server.rs b/crates/worker-runtime/src/http_server.rs index b4967073d..0326f899a 100644 --- a/crates/worker-runtime/src/http_server.rs +++ b/crates/worker-runtime/src/http_server.rs @@ -521,19 +521,8 @@ pub struct RuntimeHttpUploadedFileDeleteResponse { pub deleted: bool, } -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct RuntimeHttpWorkerCompletionsRequest { - pub kind: protocol::CompletionKind, - #[serde(default)] - pub prefix: String, -} - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct RuntimeHttpWorkerCompletionsResponse { - pub kind: protocol::CompletionKind, - pub prefix: String, - pub entries: Vec, -} +pub type RuntimeHttpWorkerCompletionsRequest = runtime_api::CompletionRequest; +pub type RuntimeHttpWorkerCompletionsResponse = runtime_api::CompletionResponse; /// Worker lifecycle request body used by stop/cancel endpoints. #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] @@ -1600,78 +1589,30 @@ async fn worker_protocol_ws_session( input_source: Option, mut socket: WebSocket, ) { - let mut cursor = match query.cursor.as_deref() { - Some(raw) => match WorkerObservationCursor::decode(raw) { - Some(cursor) => cursor, - None => { - let event = - protocol_error_event(format!("malformed worker observation cursor: {raw}")); - let _ = send_protocol_event(&mut socket, &event).await; - return; - } - }, - None => match runtime.worker_observation_cursor_now(&worker_ref) { - Ok(cursor) => cursor, - Err(error) => { - let event = protocol_error_event(error.to_string()); - let _ = send_protocol_event(&mut socket, &event).await; - return; - } - }, - }; - // Observation cursors are process-local. After a Runtime restart (or - // bounded backlog expiry), the current Worker snapshot is authoritative - // and replay resumes from the current in-memory tail. - if runtime - .read_worker_observation_events(&worker_ref, cursor) - .is_err() + if let Some(raw) = query.cursor.as_deref() + && WorkerObservationCursor::decode(raw).is_none() { - cursor = match runtime.worker_observation_cursor_now(&worker_ref) { - Ok(cursor) => cursor, - Err(error) => { - let event = protocol_error_event(error.to_string()); - let _ = send_protocol_event(&mut socket, &event).await; - return; - } - }; - } - - let mut receiver = match runtime.subscribe_worker_observation() { - Ok(receiver) => receiver, - Err(error) => { - let event = - protocol_error_event(format!("runtime observation bus unavailable: {error}")); - let _ = send_protocol_event(&mut socket, &event).await; - return; - } - }; - - let snapshot = match runtime.worker_observation_snapshot(&worker_ref) { - Ok(snapshot) => snapshot, - Err(error) => { - let event = protocol_error_event(error.to_string()); - let _ = send_protocol_event(&mut socket, &event).await; - return; - } - }; - if !send_protocol_event(&mut socket, &snapshot).await { + let event = protocol_error_event("malformed worker observation cursor"); + let _ = send_protocol_event(&mut socket, &event).await; return; } - - match runtime.read_worker_observation_events(&worker_ref, cursor) { - Ok(backlog) => { - for event in backlog { - cursor = WorkerObservationCursor::new(event.sequence); - if !send_protocol_event(&mut socket, &event.payload).await { - return; - } - } - } + // The complete Controller snapshot reconstitutes this protocol connection. + // A WorkerRef-wide catalog cursor cannot attribute historical operational + // events to this captured execution; never append that bus after its snapshot. + let attached = match scope.as_ref() { + Some(scope) => runtime.attach_worker_protocol_scoped(scope, &worker_ref), + None => runtime.attach_worker_protocol(&worker_ref), + }; + let mut transport = match attached { + Ok(transport) => transport, Err(error) => { let event = protocol_error_event(error.to_string()); let _ = send_protocol_event(&mut socket, &event).await; return; } + }; + if !send_protocol_event(&mut socket, &transport.snapshot).await { + return; } loop { @@ -1684,9 +1625,13 @@ async fn worker_protocol_ws_session( authorize_runtime_protocol_method(method, input_source.as_ref()); let result = match scope.as_ref() { Some(scope) => { - runtime.send_protocol_method_scoped(scope, &worker_ref, method) + runtime.send_connected_protocol_method_scoped( + scope, &worker_ref, &transport, method, + ) } - None => runtime.send_protocol_method(&worker_ref, method), + None => runtime.send_connected_protocol_method( + &worker_ref, &transport, method, + ), }; match result { Ok(events) => { @@ -1701,6 +1646,10 @@ async fn worker_protocol_ws_session( if !send_protocol_event(&mut socket, &event).await { return; } + if transport.validate().is_err() { + let _ = socket.send(WsMessage::Close(None)).await; + return; + } } } }, @@ -1727,23 +1676,20 @@ async fn worker_protocol_ws_session( } } } - event = receiver.recv() => { + event = transport.events.recv() => { match event { - Ok(event) if event.worker_ref == worker_ref && event.sequence > cursor.sequence => { - cursor = WorkerObservationCursor::new(event.sequence); - if !send_protocol_event(&mut socket, &event.payload).await { + Some(event) => { + let terminal = matches!(event, protocol::Event::Shutdown); + if !send_protocol_event(&mut socket, &event).await { + return; + } + if terminal { + let _ = socket.send(WsMessage::Close(None)).await; return; } } - Ok(_) => {} - Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => { - let event = protocol_error_event("runtime observation backlog was overrun"); - let _ = send_protocol_event(&mut socket, &event).await; - return; - } - Err(tokio::sync::broadcast::error::RecvError::Closed) => { - let event = protocol_error_event("runtime observation bus closed"); - let _ = send_protocol_event(&mut socket, &event).await; + None => { + let _ = socket.send(WsMessage::Close(None)).await; return; } } @@ -1865,15 +1811,21 @@ async fn worker_completions( &worker_ref, request.kind, &request.prefix, + request.context.as_ref(), + ), + None => state.runtime.worker_completions( + &worker_ref, + request.kind, + &request.prefix, + request.context.as_ref(), ), - None => state - .runtime - .worker_completions(&worker_ref, request.kind, &request.prefix), } .map_err(RuntimeHttpRestError::runtime)?; Ok(Json(RuntimeHttpWorkerCompletionsResponse { kind: request.kind, prefix: request.prefix, + context: request.context, + request_id: request.request_id, entries, })) } @@ -2350,6 +2302,7 @@ fn required_runtime_permission(method: &Method, path: &str) -> Option<&'static s } if path.ends_with("/input") || path.ends_with("/restore") + || path.ends_with("/restore/coordinate") || (path.contains("/attachments") && *method != Method::GET) || path.contains("/workdir-attachments") { @@ -2562,6 +2515,7 @@ fn status_for_runtime_error(error: &RuntimeError) -> StatusCode { StatusCode::CONFLICT } RuntimeError::RuntimeStopped + | RuntimeError::RestoreObservationConflict { .. } | RuntimeError::RuntimeStoreAlreadyOpen { .. } | RuntimeError::WorkerExecutionUnavailable { .. } | RuntimeError::ExecutionBackendUnavailable { .. } @@ -2586,6 +2540,9 @@ fn status_for_runtime_error(error: &RuntimeError) -> StatusCode { fn code_for_runtime_error(error: &RuntimeError) -> String { match error { RuntimeError::RuntimeStopped => "runtime_stopped".to_string(), + RuntimeError::RestoreObservationConflict { .. } => { + "restore_observation_conflict".to_string() + } RuntimeError::RuntimeStoreAlreadyOpen { .. } => "runtime_store_already_open".to_string(), RuntimeError::WorkerNotFound { .. } => "worker_not_found".to_string(), RuntimeError::WorkerExecutionUnavailable { .. } => { @@ -2646,9 +2603,8 @@ mod tests { ConfigBundle, ConfigBundleMetadata, ConfigBundleProvenance, ConfigProfileDescriptor, }; use crate::execution::{ - WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation, - WorkerExecutionRestoreRequest, WorkerExecutionResult, WorkerExecutionSpawnRequest, - WorkerExecutionSpawnResult, + WorkerExecutionBackend, WorkerExecutionOperation, WorkerExecutionRestoreRequest, + WorkerExecutionResult, WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult, }; use crate::management::RuntimeOptions; use crate::retention::{DiagnosticsDisposition, SessionDisposition}; @@ -3023,6 +2979,7 @@ mod tests { assert_eq!(error.error.code, "worker_not_found"); } + #[cfg(feature = "ws-server")] #[test] fn runtime_protocol_replaces_serialized_tracked_source() { let wire = serde_json::to_string(&protocol::Method::SubmitTracked { @@ -3050,6 +3007,7 @@ mod tests { )); } + #[cfg(feature = "ws-server")] #[test] fn runtime_protocol_uses_transport_authenticated_account_source() { let mut headers = HeaderMap::new(); @@ -3171,6 +3129,7 @@ mod tests { language: "English".to_string(), }), subjektiv_attached: false, + backend_job: None, } } @@ -3403,12 +3362,9 @@ mod tests { } fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult { - WorkerExecutionSpawnResult::Connected { - handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()), - worker_state: protocol::WorkerStateSnapshot { - ..protocol::WorkerStatus::Idle.into() - }, - workdir_attachments: request + WorkerExecutionSpawnResult::connected( + protocol::WorkerStatus::Idle.into(), + request .workdir_attachments .iter() .map( @@ -3418,25 +3374,22 @@ mod tests { }, ) .collect(), - } + ) } fn restore_worker( &self, request: WorkerExecutionRestoreRequest, ) -> WorkerExecutionSpawnResult { - WorkerExecutionSpawnResult::Connected { - handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()), - worker_state: protocol::WorkerStateSnapshot { - ..protocol::WorkerStatus::Idle.into() - }, - workdir_attachments: request.previous_workdir_attachments, - } + WorkerExecutionSpawnResult::connected( + protocol::WorkerStatus::Idle.into(), + request.previous_workdir_attachments, + ) } fn dispatch_input( &self, - _handle: &WorkerExecutionHandle, + _worker_ref: &WorkerRef, input: WorkerInput, ) -> WorkerExecutionResult { if let Some(submission_id) = input.submission_request_id { @@ -3451,7 +3404,57 @@ mod tests { } } - fn stop_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult { + fn stop_worker(&self, _worker_ref: &WorkerRef) -> WorkerExecutionResult { + WorkerExecutionResult::accepted(WorkerExecutionOperation::Stop) + } + } + + #[cfg(feature = "fs-store")] + #[derive(Default)] + struct RejectingRestoreBackend { + restores: Mutex>, + inputs: Mutex>, + stops: Mutex>, + } + + #[cfg(feature = "fs-store")] + impl WorkerExecutionBackend for RejectingRestoreBackend { + fn backend_id(&self) -> &str { + "http-test" + } + + fn spawn_worker( + &self, + _request: WorkerExecutionSpawnRequest, + ) -> WorkerExecutionSpawnResult { + panic!("a saved Worker must be restored, not spawned again"); + } + + fn restore_worker( + &self, + request: WorkerExecutionRestoreRequest, + ) -> WorkerExecutionSpawnResult { + self.restores.lock().unwrap().push(request.worker_ref); + WorkerExecutionSpawnResult::Rejected(WorkerExecutionResult::rejected( + WorkerExecutionOperation::Restore, + "repository access must be reacquired", + )) + } + + fn dispatch_input( + &self, + worker_ref: &WorkerRef, + _input: WorkerInput, + ) -> WorkerExecutionResult { + self.inputs.lock().unwrap().push(worker_ref.clone()); + WorkerExecutionResult::rejected( + WorkerExecutionOperation::Input, + "Worker execution is unavailable after restore rejection", + ) + } + + fn stop_worker(&self, worker_ref: &WorkerRef) -> WorkerExecutionResult { + self.stops.lock().unwrap().push(worker_ref.clone()); WorkerExecutionResult::accepted(WorkerExecutionOperation::Stop) } } @@ -3511,6 +3514,56 @@ mod tests { serde_json::from_slice(&body).unwrap() } + #[tokio::test] + async fn rest_feature_completions_echo_each_nonce_and_qualified_context() { + let runtime = + Runtime::with_execution_backend(RuntimeOptions::default(), Arc::new(AcceptingBackend)) + .unwrap(); + runtime + .store_config_bundle(test_bundle(ProfileSelector::Builtin( + "builtin:coder".into(), + ))) + .unwrap(); + let token = "completion-token"; + let app = runtime_http_router(runtime, token.into()); + let created = authed_json_request( + app.clone(), + Method::POST, + "/v1/workers", + token, + &task_request("completion"), + ) + .await; + assert_eq!(created.status(), StatusCode::OK); + let created: RuntimeHttpWorkerResponse = read_json(created).await; + let context = protocol::CompletionContext { + invocation: protocol::FeatureInvocationIdentity("builtin:test/prepare".into()), + argument: Some("path".into()), + }; + for nonce in ["first", "second", "first"] { + let request = runtime_api::CompletionRequest { + kind: protocol::CompletionKind::FeatureArgument, + prefix: "資料/".into(), + context: Some(context.clone()), + request_id: Some(nonce.into()), + }; + let response = authed_json_request( + app.clone(), + Method::POST, + &format!("/v1/workers/{}/completions", created.worker.worker_id), + token, + &request, + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + let response: runtime_api::CompletionResponse = read_json(response).await; + assert_eq!(response.kind, request.kind); + assert_eq!(response.prefix, request.prefix); + assert_eq!(response.context, request.context); + assert_eq!(response.request_id, request.request_id); + } + } + #[tokio::test] async fn rest_command_api_delegates_to_runtime() { let runtime = @@ -3595,7 +3648,17 @@ mod tests { client .restore_worker( generated_worker_id.clone(), - runtime_api::EmptyObjectRequest::default(), + runtime_api::WorkerRestoreRequest { + expected_observation_token: client + .get_worker(generated_worker_id.clone()) + .await + .unwrap() + .worker + .restore_observation_token + .unwrap(), + request_id: "generated-restore".into(), + preparation: Some(runtime_api::WorkerRestorePreparation::default()), + }, ) .await .unwrap(); @@ -3739,18 +3802,86 @@ mod tests { let stop: RuntimeHttpWorkerLifecycleResponse = read_json(response).await; assert_eq!(stop.ack.worker_ref, created.worker.worker_ref); - let response = authed_empty_request( + let restore_uri = format!("/v1/workers/{}/restore", created.worker.worker_id); + let missing = authed_empty_request(app.clone(), Method::POST, &restore_uri, token).await; + assert_eq!(missing.status(), StatusCode::BAD_REQUEST); + let stale = runtime_api::WorkerRestoreRequest { + expected_observation_token: created.worker.restore_observation_token.clone().unwrap(), + request_id: "stale-http-restore".into(), + preparation: None, + }; + let response = + authed_json_request(app.clone(), Method::POST, &restore_uri, token, &stale).await; + assert_eq!(response.status(), StatusCode::CONFLICT); + let conflict: RuntimeHttpErrorResponse = read_json(response).await; + assert_eq!(conflict.error.code, "restore_observation_conflict"); + + let mut restore_request = runtime.test_restore_request(&created.worker.worker_ref); + let coordinate_uri = format!("{restore_uri}/coordinate"); + let mut coordination = runtime_api::WorkerRestoreCoordinationRequest { + expected_observation_token: restore_request.expected_observation_token.clone(), + request_id: restore_request.request_id.clone(), + preparation: None, + }; + let before_lookup = runtime.worker_detail(&created.worker.worker_ref).unwrap(); + let response = authed_json_request( app.clone(), Method::POST, - &format!("/v1/workers/{}/restore", created.worker.worker_id), + &coordinate_uri, token, + &coordination, + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + let missing_owner: runtime_api::WorkerRestoreCoordinationResponse = + read_json(response).await; + assert!(missing_owner.result.is_none() && missing_owner.preparation.is_none()); + assert_eq!( + runtime.worker_detail(&created.worker.worker_ref).unwrap(), + before_lookup + ); + let response = authed_json_request( + app.clone(), + Method::POST, + &coordinate_uri, + "wrong-token", + &coordination, + ) + .await; + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + coordination.preparation = Some(runtime_api::WorkerRestorePreparation::default()); + let response = authed_json_request( + app.clone(), + Method::POST, + &coordinate_uri, + token, + &coordination, + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + let admitted: runtime_api::WorkerRestoreCoordinationResponse = read_json(response).await; + assert!(admitted.result.is_none() && admitted.preparation.is_some()); + assert_eq!( + runtime + .worker_detail(&created.worker.worker_ref) + .unwrap() + .status, + crate::catalog::WorkerStatus::Stopped + ); + restore_request.preparation = admitted.preparation; + let response = authed_json_request( + app.clone(), + Method::POST, + &restore_uri, + token, + &restore_request, ) .await; assert_eq!(response.status(), StatusCode::OK); let restored: runtime_api::WorkerRestoreResponse = read_json(response).await; assert_eq!(restored.state, WorkerRestoreState::Accepted); assert_eq!( - restored.worker.unwrap().status, + restored.worker.as_ref().unwrap().status, runtime_api::WorkerStatus::Idle ); @@ -3763,6 +3894,47 @@ mod tests { .await; assert_eq!(response.status(), StatusCode::OK); + coordination.preparation = None; + let response = authed_json_request( + app.clone(), + Method::POST, + &coordinate_uri, + token, + &coordination, + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + let recovered: runtime_api::WorkerRestoreCoordinationResponse = read_json(response).await; + assert!(recovered.preparation.is_none()); + assert_eq!(recovered.result, Some(restored.clone())); + + restore_request.preparation = Some(serde_json::from_value(serde_json::json!({ + "workspace_api": { "workspace_id": "must-not-bind", "base_url": "https://fresh.invalid.example" }, + "workdir_attachments": [], + "repository_access": [] + })).unwrap()); + let response = authed_json_request( + app.clone(), + Method::POST, + &restore_uri, + token, + &restore_request, + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + let replayed: runtime_api::WorkerRestoreResponse = read_json(response).await; + assert_eq!( + serde_json::to_value(replayed).unwrap(), + serde_json::to_value(restored).unwrap() + ); + assert_eq!( + runtime + .worker_detail(&created.worker.worker_ref) + .unwrap() + .status, + crate::catalog::WorkerStatus::Stopped + ); + let response = authed_empty_request( app.clone(), Method::POST, @@ -3786,6 +3958,193 @@ mod tests { assert_eq!(summary.runtime.stopped_worker_count, 1); } + #[cfg(feature = "fs-store")] + #[tokio::test] + async fn rest_saved_active_worker_restore_rejection_is_observable_and_can_be_stopped_removed() { + let temp = tempfile::tempdir().unwrap(); + let options = crate::fs_store::FsRuntimeStoreOptions::new(temp.path().join("runtime")); + let runtime = Runtime::with_fs_store_and_execution_backend( + options.clone(), + Arc::new(AcceptingBackend), + ) + .unwrap(); + runtime + .store_config_bundle(test_bundle(ProfileSelector::Builtin( + "builtin:coder".to_string(), + ))) + .unwrap(); + let mut request = task_request("saved active Worker"); + request.workspace_api = Some(WorkspaceApiRef { + workspace_id: "local".to_string(), + base_url: "http://127.0.0.1:8787".to_string(), + }); + let created = runtime + .create_worker_scoped(&RuntimeWorkspaceScope::new("local", "local-token"), request) + .unwrap(); + assert_eq!(created.status, crate::catalog::WorkerStatus::Idle); + assert!(created.worker_state.is_some()); + drop(runtime); + + let backend = Arc::new(RejectingRestoreBackend::default()); + let runtime = + Runtime::with_fs_store_and_execution_backend(options, backend.clone()).unwrap(); + // Startup attempted automatic restore without turning rejection into a + // fabricated live Worker or silently changing its saved lifecycle status. + assert_eq!( + *backend.restores.lock().unwrap(), + vec![created.worker_ref.clone()], + "startup diagnostics: {:?}; Worker: {:?}", + runtime.diagnostics().unwrap(), + runtime.worker_detail(&created.worker_ref) + ); + let token = "local-token"; + let app = runtime_http_router(runtime.clone(), token.to_string()); + let worker_uri = format!("/v1/workers/{}", created.worker_id); + + let response = authed_empty_request(app.clone(), Method::GET, &worker_uri, token).await; + assert_eq!(response.status(), StatusCode::OK); + let detail: RuntimeHttpWorkerResponse = read_json(response).await; + assert_eq!(detail.worker.worker_ref, created.worker_ref); + assert_eq!(detail.worker.status, crate::catalog::WorkerStatus::Idle); + assert!(detail.worker.execution_metadata_available); + assert!(detail.worker.worker_state.is_none()); + + let response = authed_empty_request(app.clone(), Method::GET, "/v1/workers", token).await; + assert_eq!(response.status(), StatusCode::OK); + let listed: RuntimeHttpWorkersResponse = read_json(response).await; + assert_eq!(listed.workers.len(), 1); + assert_eq!(listed.workers[0].worker_ref, created.worker_ref); + assert_eq!(listed.workers[0].status, crate::catalog::WorkerStatus::Idle); + assert!(listed.workers[0].worker_state.is_none()); + + let response = authed_json_request( + app.clone(), + Method::POST, + &format!("{worker_uri}/restore"), + token, + &runtime.test_restore_request(&created.worker_ref), + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + let rejected: runtime_api::WorkerRestoreResponse = read_json(response).await; + assert_eq!(rejected.state, WorkerRestoreState::Rejected); + assert_eq!( + rejected.reason_code.as_deref(), + Some("worker_restore_rejected") + ); + assert!(rejected.worker.is_none()); + assert_eq!( + *backend.restores.lock().unwrap(), + vec![created.worker_ref.clone(), created.worker_ref.clone()] + ); + + let response = authed_json_request( + app.clone(), + Method::POST, + &format!("{worker_uri}/input"), + token, + &WorkerInput::user("must not be accepted by a fabricated execution"), + ) + .await; + assert_eq!(response.status(), StatusCode::CONFLICT); + let error: RuntimeHttpErrorResponse = read_json(response).await; + assert_eq!(error.error.code, "worker_execution_rejected"); + assert_eq!( + *backend.inputs.lock().unwrap(), + vec![created.worker_ref.clone()] + ); + + #[cfg(feature = "ws-server")] + { + use tokio_tungstenite::tungstenite::Message; + use tokio_tungstenite::tungstenite::client::IntoClientRequest; + + assert!(matches!( + runtime.worker_observation_snapshot(&created.worker_ref), + Err(RuntimeError::WorkerExecutionUnavailable { worker_id, .. }) + if worker_id == created.worker_id + )); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let server_app = app.clone(); + let server = tokio::spawn(async move { + axum::serve(listener, server_app).await.unwrap(); + }); + let mut request = format!("ws://{address}{worker_uri}/protocol/ws") + .into_client_request() + .unwrap(); + request.headers_mut().insert( + tokio_tungstenite::tungstenite::http::header::AUTHORIZATION, + format!("Bearer {token}").parse().unwrap(), + ); + let (mut stream, _) = tokio_tungstenite::connect_async(request).await.unwrap(); + let frame = tokio::time::timeout(std::time::Duration::from_secs(5), stream.next()) + .await + .expect("unavailable snapshot must produce a protocol error promptly") + .unwrap() + .unwrap(); + let Message::Text(text) = frame else { + panic!("expected protocol error frame, not a fabricated snapshot"); + }; + let event: protocol::Event = serde_json::from_str(&text).unwrap(); + assert!(matches!(event, protocol::Event::Error { .. })); + drop(stream); + server.abort(); + let _ = server.await; + } + + // Saved active status still requires a normal stop before removal, + // even though automatic restore never connected a live execution. + let response = authed_empty_request(app.clone(), Method::DELETE, &worker_uri, token).await; + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + let error: RuntimeHttpErrorResponse = read_json(response).await; + assert_eq!(error.error.code, "invalid_request"); + assert!(backend.stops.lock().unwrap().is_empty()); + + let response = authed_empty_request( + app.clone(), + Method::POST, + &format!("{worker_uri}/stop"), + token, + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + let stopped: RuntimeHttpWorkerLifecycleResponse = read_json(response).await; + assert_eq!(stopped.ack.worker_ref, created.worker_ref); + assert_eq!(stopped.ack.status, crate::catalog::WorkerStatus::Stopped); + assert_eq!( + *backend.stops.lock().unwrap(), + vec![created.worker_ref.clone()] + ); + + let response = authed_empty_request(app.clone(), Method::GET, &worker_uri, token).await; + assert_eq!(response.status(), StatusCode::OK); + let detail: RuntimeHttpWorkerResponse = read_json(response).await; + assert_eq!(detail.worker.status, crate::catalog::WorkerStatus::Stopped); + + let response = authed_empty_request(app.clone(), Method::DELETE, &worker_uri, token).await; + assert_eq!(response.status(), StatusCode::OK); + let removed: RuntimeHttpWorkerDeleteResponse = read_json(response).await; + assert_eq!(removed.worker.worker_id, created.worker_id); + assert!(removed.worker.deleted); + // Removal asks the backend to reconfirm shutdown even for a saved + // Stopped Worker; it must use the same WorkerRef as the original stop. + assert_eq!( + *backend.stops.lock().unwrap(), + vec![created.worker_ref.clone(), created.worker_ref.clone()] + ); + assert!( + !temp + .path() + .join("runtime/workers") + .join(created.worker_id.to_string()) + .exists() + ); + + let response = authed_empty_request(app, Method::GET, &worker_uri, token).await; + assert_eq!(response.status(), StatusCode::NOT_FOUND); + } + #[tokio::test] async fn local_token_placeholder_rejects_missing_bearer_token() { let app = runtime_http_router(Runtime::new_memory(), "local-token".to_string()); @@ -3995,8 +4354,9 @@ mod ws_tests { ConfigBundle, ConfigBundleMetadata, ConfigBundleProvenance, ConfigProfileDescriptor, }; use crate::execution::{ - WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation, - WorkerExecutionResult, WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult, + WorkerExecutionBackend, WorkerExecutionContext, WorkerExecutionOperation, + WorkerExecutionRestoreRequest, WorkerExecutionResult, WorkerExecutionSpawnRequest, + WorkerExecutionSpawnResult, WorkerProtocolTransport, }; use crate::management::RuntimeOptions; use futures::{SinkExt, StreamExt}; @@ -4006,7 +4366,133 @@ mod ws_tests { use tokio_tungstenite::tungstenite::client::IntoClientRequest; use tokio_tungstenite::tungstenite::http::header as ws_header; - struct WsBackend; + struct WsController { + commands: tokio::sync::mpsc::UnboundedReceiver, + connections: Vec>, + } + + struct WsExecution { + snapshot: protocol::Event, + context: WorkerExecutionContext, + methods: tokio::sync::mpsc::UnboundedSender, + controller: Mutex, + } + + impl WsExecution { + fn validate(&self) -> Result<(), WorkerExecutionResult> { + if self.methods.is_closed() { + Err(WorkerExecutionResult::rejected( + WorkerExecutionOperation::ProtocolMethod, + "mock Controller method channel is closed", + )) + } else { + Ok(()) + } + } + + fn dispatch( + &self, + method: protocol::Method, + ) -> Result, WorkerExecutionResult> { + let mut controller = self.controller.lock().unwrap(); + self.validate()?; + match method { + protocol::Method::ListCompletions { + kind, + prefix, + request_id, + context, + } => Ok(vec![protocol::Event::Completions { + kind, + prefix, + request_id, + context, + entries: Vec::new(), + }]), + protocol::Method::Shutdown { .. } => { + controller.commands.close(); + controller.connections.clear(); + Ok(Vec::new()) + } + method => { + self.methods.send(method).map_err(|error| { + WorkerExecutionResult::rejected( + WorkerExecutionOperation::ProtocolMethod, + error.to_string(), + ) + })?; + Ok(Vec::new()) + } + } + } + + fn close(&self) { + let mut controller = self.controller.lock().unwrap(); + controller.commands.close(); + controller.connections.clear(); + } + } + + #[derive(Default)] + struct WsBackend { + executions: Mutex>>, + } + + impl WsBackend { + fn insert_execution(&self, worker_ref: WorkerRef, context: WorkerExecutionContext) { + let (methods, commands) = tokio::sync::mpsc::unbounded_channel(); + let execution = Arc::new(WsExecution { + snapshot: ws_snapshot(&worker_ref), + context, + methods, + controller: Mutex::new(WsController { + commands, + connections: Vec::new(), + }), + }); + if let Some(previous) = self + .executions + .lock() + .unwrap() + .insert(worker_ref, execution) + { + previous.close(); + } + } + + fn publish( + &self, + worker_ref: &WorkerRef, + event: protocol::Event, + ) -> crate::observation::WorkerObservationEvent { + let execution = self + .executions + .lock() + .unwrap() + .get(worker_ref) + .unwrap() + .clone(); + let observation = execution + .context + .publish_protocol_event(event.clone()) + .unwrap(); + // The fixture publishes separately to Runtime history and to this + // Controller's attached streams, just like the backend event relay. + execution + .controller + .lock() + .unwrap() + .connections + .retain(|sender| match sender.try_send(event.clone()) { + Ok(()) => true, + Err(tokio::sync::mpsc::error::TrySendError::Closed(_)) => false, + Err(tokio::sync::mpsc::error::TrySendError::Full(_)) => { + panic!("mock protocol event receiver lagged") + } + }); + observation + } + } impl WorkerExecutionBackend for WsBackend { fn backend_id(&self) -> &str { @@ -4014,12 +4500,10 @@ mod ws_tests { } fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult { - WorkerExecutionSpawnResult::Connected { - handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()), - worker_state: protocol::WorkerStateSnapshot { - ..protocol::WorkerStatus::Idle.into() - }, - workdir_attachments: request + self.insert_execution(request.worker_ref, request.context); + WorkerExecutionSpawnResult::connected( + protocol::WorkerStatus::Idle.into(), + request .workdir_attachments .iter() .map( @@ -4029,12 +4513,12 @@ mod ws_tests { }, ) .collect(), - } + ) } fn dispatch_input( &self, - _handle: &WorkerExecutionHandle, + _worker_ref: &WorkerRef, input: WorkerInput, ) -> WorkerExecutionResult { if let Some(submission_id) = input.submission_request_id { @@ -4049,40 +4533,91 @@ mod ws_tests { } } - fn worker_snapshot(&self, handle: &WorkerExecutionHandle) -> Option { - Some(protocol::Event::Snapshot { - session: protocol::SessionSnapshot { - pending_submissions: protocol::PendingSubmissionsSnapshot::default(), - entries: Vec::new(), - }, - greeting: protocol::Greeting { - worker_name: handle.worker_ref().worker_id.to_string(), - cwd: String::new(), - provider: "ws-test".to_string(), - model: "ws-test".to_string(), - scope_summary: "WebSocket test execution snapshot".to_string(), - tools: Vec::new(), - context_window: 0, - context_tokens: 0, - reasoning: None, - context_usage: None, - }, - state: protocol::WorkerStateSnapshot::initial(), - in_flight: protocol::InFlightSnapshot { - blocks: Vec::new(), - commands: Vec::new(), - compaction: None, - }, - internal_workers: Vec::new(), - }) + fn worker_snapshot(&self, worker_ref: &WorkerRef) -> Option { + self.executions + .lock() + .unwrap() + .get(worker_ref) + .map(|execution| execution.snapshot.clone()) } - fn dispatch_method( + fn restore_worker( &self, - _handle: &WorkerExecutionHandle, - _method: protocol::Method, - ) -> WorkerExecutionResult { - WorkerExecutionResult::accepted(WorkerExecutionOperation::ProtocolMethod) + request: WorkerExecutionRestoreRequest, + ) -> WorkerExecutionSpawnResult { + self.insert_execution(request.worker_ref, request.context); + WorkerExecutionSpawnResult::connected( + protocol::WorkerStatus::Idle.into(), + request.previous_workdir_attachments, + ) + } + + fn stop_worker(&self, worker_ref: &WorkerRef) -> WorkerExecutionResult { + if let Some(execution) = self.executions.lock().unwrap().remove(worker_ref) { + execution.close(); + } + WorkerExecutionResult::accepted(WorkerExecutionOperation::Stop) + } + + fn attach_worker_protocol( + self: Arc, + worker_ref: &WorkerRef, + ) -> Result { + let execution = self + .executions + .lock() + .unwrap() + .get(worker_ref) + .cloned() + .ok_or_else(|| { + WorkerExecutionResult::rejected( + WorkerExecutionOperation::ProtocolMethod, + "mock Controller is unavailable", + ) + })?; + let (sender, events) = tokio::sync::mpsc::channel(32); + { + let mut controller = execution.controller.lock().unwrap(); + execution.validate()?; + controller.connections.push(sender); + } + let dispatcher = execution.clone(); + let validator = execution.clone(); + Ok(WorkerProtocolTransport::new( + worker_ref.clone(), + execution.snapshot.clone(), + events, + Arc::new(move |method| dispatcher.dispatch(method)), + Arc::new(move || validator.validate()), + )) + } + } + + fn ws_snapshot(worker_ref: &WorkerRef) -> protocol::Event { + protocol::Event::Snapshot { + session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), + entries: Vec::new(), + }, + greeting: protocol::Greeting { + worker_name: worker_ref.worker_id.to_string(), + cwd: String::new(), + provider: "ws-test".to_string(), + model: "ws-test".to_string(), + scope_summary: "WebSocket test execution snapshot".to_string(), + tools: Vec::new(), + context_window: 0, + context_tokens: 0, + reasoning: None, + context_usage: None, + }, + state: protocol::WorkerStateSnapshot::initial(), + in_flight: protocol::InFlightSnapshot { + blocks: Vec::new(), + commands: Vec::new(), + compaction: None, + }, + internal_workers: Vec::new(), } } @@ -4151,13 +4686,14 @@ mod ws_tests { language: "English".to_string(), }), subjektiv_attached: false, + backend_job: None, } } - async fn spawn_runtime_server() -> (Runtime, WorkerRef, String) { + async fn spawn_runtime_server() -> (Runtime, Arc, WorkerRef, String) { + let backend = Arc::new(WsBackend::default()); let runtime = - Runtime::with_execution_backend(RuntimeOptions::default(), Arc::new(WsBackend)) - .unwrap(); + Runtime::with_execution_backend(RuntimeOptions::default(), backend.clone()).unwrap(); runtime .store_config_bundle(ws_test_bundle(ProfileSelector::Builtin( "builtin:companion".to_string(), @@ -4181,6 +4717,7 @@ mod ws_tests { }); ( runtime, + backend, worker.worker_ref.clone(), format!( "ws://{addr}/v1/workers/{}/protocol/ws", @@ -4203,7 +4740,11 @@ mod ws_tests { tokio_tungstenite::MaybeTlsStream, >, ) -> protocol::Event { - let message = stream.next().await.unwrap().unwrap(); + let message = tokio::time::timeout(std::time::Duration::from_secs(5), stream.next()) + .await + .expect("Worker protocol frame must arrive promptly") + .unwrap() + .unwrap(); let Message::Text(text) = message else { panic!("expected text frame"); }; @@ -4215,7 +4756,11 @@ mod ws_tests { tokio_tungstenite::MaybeTlsStream, >, ) -> SubscriptionFrame { - let message = stream.next().await.unwrap().unwrap(); + let message = tokio::time::timeout(std::time::Duration::from_secs(5), stream.next()) + .await + .expect("Worker protocol frame must arrive promptly") + .unwrap() + .unwrap(); let Message::Text(text) = message else { panic!("expected text frame"); }; @@ -4245,7 +4790,7 @@ mod ws_tests { #[tokio::test] async fn runtime_protocol_ws_subscribes_and_filters_worker_lifecycle() { - let (runtime, worker_ref, worker_url) = spawn_runtime_server().await; + let (runtime, backend, worker_ref, worker_url) = spawn_runtime_server().await; let other = runtime .create_worker_scoped( &RuntimeWorkspaceScope::new("local", "local-token"), @@ -4302,22 +4847,18 @@ mod ws_tests { )); snapshot }; - runtime - .observe_worker_event( - &other.worker_ref, - protocol::Event::WorkerState { - snapshot: running_snapshot(&other.worker_ref), - }, - ) - .unwrap(); - runtime - .observe_worker_event( - &worker_ref, - protocol::Event::WorkerState { - snapshot: running_snapshot(&worker_ref), - }, - ) - .unwrap(); + backend.publish( + &other.worker_ref, + protocol::Event::WorkerState { + snapshot: running_snapshot(&other.worker_ref), + }, + ); + backend.publish( + &worker_ref, + protocol::Event::WorkerState { + snapshot: running_snapshot(&worker_ref), + }, + ); let event = next_subscription_frame(&mut stream).await; assert!(matches!( event.payload, @@ -4364,7 +4905,7 @@ mod ws_tests { #[tokio::test] async fn protocol_ws_connect_sends_snapshot_and_live_worker_events() { - let (runtime, worker_ref, url) = spawn_runtime_server().await; + let (_runtime, backend, worker_ref, url) = spawn_runtime_server().await; let (mut stream, _) = connect_async(authed_ws_request(&url)).await.unwrap(); assert!(matches!( @@ -4372,14 +4913,12 @@ mod ws_tests { protocol::Event::Snapshot { .. } )); - runtime - .observe_worker_event( - &worker_ref, - protocol::Event::TextDelta { - text: "started".into(), - }, - ) - .unwrap(); + backend.publish( + &worker_ref, + protocol::Event::TextDelta { + text: "started".into(), + }, + ); assert!(matches!( next_frame(&mut stream).await, protocol::Event::TextDelta { .. } @@ -4387,25 +4926,118 @@ mod ws_tests { } #[tokio::test] - async fn protocol_ws_cursor_resume_is_duplicate_safe_and_filters_workers() { - let (runtime, worker_ref, url) = spawn_runtime_server().await; - let other = runtime.create_worker(ws_create_request()).unwrap(); - let first = runtime - .observe_worker_event( - &worker_ref, - protocol::Event::TextDelta { - text: "started".into(), - }, - ) + async fn protocol_ws_connected_dispatch_preserves_transport_source_and_completion_reply() { + let (_runtime, backend, worker_ref, url) = spawn_runtime_server().await; + let mut request = authed_ws_request(&url); + request.headers_mut().insert( + protocol::AUTHENTICATED_ACCOUNT_ID_HEADER, + "account-1".parse().unwrap(), + ); + let (mut stream, _) = connect_async(request).await.unwrap(); + let _ = next_frame(&mut stream).await; + let notification = protocol::Method::NotifyTracked { + notification_request_id: "notification-1".into(), + message: "hello".into(), + source: protocol::AuthenticatedInputSource::Account { + account_id: "forged".into(), + }, + }; + let completions = protocol::Method::ListCompletions { + kind: protocol::CompletionKind::Feature, + prefix: "builtin:".into(), + request_id: Some("completion-1".into()), + context: None, + }; + for method in [notification, completions] { + stream + .send(Message::Text( + serde_json::to_string(&method).unwrap().into(), + )) + .await + .unwrap(); + } + assert!(matches!( + next_frame(&mut stream).await, + protocol::Event::Completions { request_id: Some(request_id), prefix, .. } + if request_id == "completion-1" && prefix == "builtin:" + )); + // The reply orders this assertion after dispatch of the preceding frame. + let execution = backend + .executions + .lock() + .unwrap() + .get(&worker_ref) + .unwrap() + .clone(); + let method = execution + .controller + .lock() + .unwrap() + .commands + .try_recv() .unwrap(); + assert!(matches!( + method, + protocol::Method::NotifyTracked { + source: protocol::AuthenticatedInputSource::Account { account_id }, + .. + } if account_id == "account-1" + )); + } + + #[tokio::test] + async fn protocol_ws_stop_closes_old_stream_and_restore_requires_new_connection() { + let (runtime, backend, worker_ref, url) = spawn_runtime_server().await; + let (mut old_stream, _) = connect_async(authed_ws_request(&url)).await.unwrap(); + let _ = next_frame(&mut old_stream).await; + runtime.stop_worker(&worker_ref, None).unwrap(); runtime - .observe_worker_event( - &other.worker_ref, - protocol::Event::TextDelta { - text: "other".into(), - }, - ) + .restore_worker(&worker_ref, runtime.test_restore_request(&worker_ref)) .unwrap(); + backend.publish( + &worker_ref, + protocol::Event::TextDelta { + text: "new execution".into(), + }, + ); + let closed = tokio::time::timeout(std::time::Duration::from_secs(5), old_stream.next()) + .await + .expect("the old protocol connection must close on stop"); + assert!(matches!(closed, Some(Ok(Message::Close(_))) | None)); + + let (mut new_stream, _) = connect_async(authed_ws_request(&url)).await.unwrap(); + assert!(matches!( + next_frame(&mut new_stream).await, + protocol::Event::Snapshot { .. } + )); + backend.publish( + &worker_ref, + protocol::Event::TextDone { + text: "restored execution".into(), + }, + ); + assert!(matches!( + next_frame(&mut new_stream).await, + protocol::Event::TextDone { text } if text == "restored execution" + )); + } + + #[tokio::test] + async fn protocol_ws_cursor_resume_is_duplicate_safe_and_filters_workers() { + let (runtime, backend, worker_ref, url) = spawn_runtime_server().await; + let other = runtime.create_worker(ws_create_request()).unwrap(); + let first = backend.publish( + &worker_ref, + protocol::Event::TextDelta { + text: "started".into(), + }, + ); + backend.publish( + &other.worker_ref, + protocol::Event::TextDelta { + text: "other".into(), + }, + ); let resume_url = format!("{url}?cursor={}", first.cursor); let (mut stream, _) = connect_async(authed_ws_request(&resume_url)).await.unwrap(); @@ -4414,23 +5046,58 @@ mod ws_tests { protocol::Event::Snapshot { .. } )); - runtime - .observe_worker_event( - &worker_ref, - protocol::Event::TextDone { - text: "done".into(), - }, - ) - .unwrap(); + backend.publish( + &worker_ref, + protocol::Event::TextDone { + text: "done".into(), + }, + ); assert!(matches!( next_frame(&mut stream).await, protocol::Event::TextDone { .. } )); } + #[tokio::test] + async fn protocol_ws_old_cursor_never_replays_stopped_execution_after_new_snapshot() { + let (runtime, backend, worker_ref, url) = spawn_runtime_server().await; + let old = backend.publish( + &worker_ref, + protocol::Event::TextDelta { + text: "old cursor".into(), + }, + ); + backend.publish( + &worker_ref, + protocol::Event::TextDone { + text: "old output".into(), + }, + ); + backend.publish(&worker_ref, protocol::Event::Shutdown); + runtime.stop_worker(&worker_ref, None).unwrap(); + runtime + .restore_worker(&worker_ref, runtime.test_restore_request(&worker_ref)) + .unwrap(); + let resume_url = format!("{url}?cursor={}", old.cursor); + let (mut stream, _) = connect_async(authed_ws_request(&resume_url)).await.unwrap(); + assert!(matches!( + next_frame(&mut stream).await, + protocol::Event::Snapshot { .. } + )); + backend.publish( + &worker_ref, + protocol::Event::TextDone { + text: "new output".into(), + }, + ); + assert!( + matches!(next_frame(&mut stream).await, protocol::Event::TextDone { text } if text == "new output") + ); + } + #[tokio::test] async fn protocol_ws_reports_malformed_cursor_and_method_frame() { - let (_runtime, _worker_ref, url) = spawn_runtime_server().await; + let (_runtime, _backend, _worker_ref, url) = spawn_runtime_server().await; let malformed_url = format!("{url}?cursor=bad"); let (mut malformed, _) = connect_async(authed_ws_request(&malformed_url)) .await diff --git a/crates/worker-runtime/src/http_server/runtime_management_api.rs b/crates/worker-runtime/src/http_server/runtime_management_api.rs index c6992e09e..b92e22731 100644 --- a/crates/worker-runtime/src/http_server/runtime_management_api.rs +++ b/crates/worker-runtime/src/http_server/runtime_management_api.rs @@ -462,21 +462,74 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi { Ok(runtime_api::WorkerLifecycleResponse { ack }) } + async fn coordinate_worker_restore( + &self, + worker_id: String, + request: runtime_api::WorkerRestoreCoordinationRequest, + ) -> Result { + let worker_ref = worker_ref_for(&self.state.runtime, worker_id).map_err(api_error)?; + let scope = + auth_workspace_scope(&self.state, auth_extension().as_ref()).map_err(api_error)?; + let runtime = self.state.runtime.clone(); + let (result, preparation) = tokio::task::spawn_blocking(move || { + runtime.coordinate_worker_restore_operation(&worker_ref, request, scope.as_ref()) + }) + .await + .map_err(|error| { + runtime_api::RuntimeApiError::new(500, "runtime_restore_task_failed", error.to_string()) + })? + .map_err(RuntimeHttpRestError::runtime) + .map_err(api_error)?; + let result = result + .map(|result| -> Result<_, runtime_api::RuntimeApiError> { + Ok(runtime_api::WorkerRestoreResponse { + state: match result.state { + server_api::WorkerRestoreState::Accepted => { + runtime_api::WorkerRestoreState::Accepted + } + server_api::WorkerRestoreState::Rejected => { + runtime_api::WorkerRestoreState::Rejected + } + server_api::WorkerRestoreState::RolledBack => { + runtime_api::WorkerRestoreState::RolledBack + } + server_api::WorkerRestoreState::ReconciliationRequired => { + runtime_api::WorkerRestoreState::ReconciliationRequired + } + }, + worker: result.worker.map(response).transpose()?, + reason_code: result.reason_code, + message: result.message, + }) + }) + .transpose()?; + Ok(runtime_api::WorkerRestoreCoordinationResponse { + result, + preparation, + }) + } + async fn restore_worker( &self, worker_id: String, - _request: runtime_api::EmptyObjectRequest, + request: runtime_api::WorkerRestoreRequest, ) -> Result { let worker_ref = worker_ref_for(&self.state.runtime, worker_id).map_err(api_error)?; - let result = match auth_workspace_scope(&self.state, auth_extension().as_ref()) - .map_err(api_error)? - { - Some(scope) => self - .state - .runtime - .restore_worker_operation_scoped(&scope, &worker_ref), - None => self.state.runtime.restore_worker_operation(&worker_ref), - } + let scope = + auth_workspace_scope(&self.state, auth_extension().as_ref()).map_err(api_error)?; + let runtime = self.state.runtime.clone(); + let result = tokio::task::spawn_blocking(move || match scope { + Some(scope) => runtime.restore_worker_operation_scoped(&scope, &worker_ref, request), + None => runtime.restore_worker_operation(&worker_ref, request), + }) + .await + .map_err(|error| { + runtime_api::RuntimeApiError::new( + StatusCode::INTERNAL_SERVER_ERROR.as_u16(), + "runtime_restore_task_failed", + error.to_string(), + ) + })? .map_err(RuntimeHttpRestError::runtime) .map_err(api_error)?; Ok(runtime_api::WorkerRestoreResponse { @@ -562,15 +615,19 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi { let entries = match auth_workspace_scope(&self.state, auth_extension().as_ref()) .map_err(api_error)? { - Some(scope) => { - self.state - .runtime - .worker_completions_scoped(&scope, &worker_ref, kind, &prefix) - } - None => self - .state - .runtime - .worker_completions(&worker_ref, kind, &prefix), + Some(scope) => self.state.runtime.worker_completions_scoped( + &scope, + &worker_ref, + kind, + &prefix, + value.context.as_ref(), + ), + None => self.state.runtime.worker_completions( + &worker_ref, + kind, + &prefix, + value.context.as_ref(), + ), } .map_err(RuntimeHttpRestError::runtime) .map_err(api_error)?; @@ -578,6 +635,8 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi { Ok(runtime_api::CompletionResponse { kind, prefix, + context: value.context, + request_id: value.request_id, entries, }) } diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index 96622fb2f..543d89244 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -13,11 +13,10 @@ use crate::diagnostics::{DiagnosticSeverity, RuntimeDiagnostic}; use crate::error::RuntimeError; use crate::execution::WorkerExecutionRestoreRequest; use crate::execution::{ - WorkerExecutionBackend, WorkerExecutionBackendRef, WorkerExecutionHandle, - WorkerExecutionOperation, WorkerExecutionResult, WorkerExecutionSpawnRequest, - WorkerExecutionSpawnResult, WorkerExecutionStopRequest, WorkerLifecycleOperationId, - WorkerSessionHistoryRequest, WorkerSessionObservationRequest, WorkspaceConfigFetchRequest, - WorkspaceConfigFetchResult, + WorkerExecutionBackend, WorkerExecutionBackendRef, WorkerExecutionOperation, + WorkerExecutionResult, WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult, + WorkerExecutionStopRequest, WorkerLifecycleOperationId, WorkerSessionHistoryRequest, + WorkerSessionObservationRequest, WorkspaceConfigFetchRequest, WorkspaceConfigFetchResult, }; #[cfg(feature = "fs-store")] use crate::fs_store::{ @@ -146,7 +145,7 @@ impl From for WorkerRestoreMode { /// Runtime-internal restore result consumed by both embedded and HTTP /// providers. Diagnostics are stable and path-free; detailed backend errors /// remain in Runtime logs rather than crossing the API boundary. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct RuntimeWorkerRestoreResult { pub state: WorkerRestoreState, pub worker: Option, @@ -164,12 +163,16 @@ impl RuntimeWorkerRestoreResult { } } - fn failed(state: WorkerRestoreState, reason_code: &'static str, message: &'static str) -> Self { + fn failed( + state: WorkerRestoreState, + reason_code: &'static str, + message: impl Into, + ) -> Self { Self { state, worker: None, reason_code: Some(reason_code.to_string()), - message: Some(message.to_string()), + message: Some(message.into()), } } @@ -375,6 +378,8 @@ impl Runtime { state }; state.execution_backend = execution_backend; + // Initialize legacy generations durably before exposing an observation. + state.persist_workers()?; let runtime = Self { inner: Arc::new(Mutex::new(state)), worker_operations: Arc::new(Mutex::new(BTreeMap::new())), @@ -893,7 +898,9 @@ impl Runtime { message: "persisted Worker restore request is unavailable".to_string(), } })?; - if existing_request.create_fingerprint != request.create_fingerprint { + if existing_request.create_fingerprint != request.create_fingerprint + || existing_request.backend_job != request.backend_job + { return Err(RuntimeError::InvalidRequest(format!( "worker {} was already created with a different fingerprint", request.worker_id @@ -1063,7 +1070,9 @@ impl Runtime { message: "persisted Worker restore request is unavailable".to_string(), } })?; - if existing_request.create_fingerprint != request.create_fingerprint { + if existing_request.create_fingerprint != request.create_fingerprint + || existing_request.backend_job != request.backend_job + { return Err(RuntimeError::InvalidRequest(format!( "worker {} was already created with a different fingerprint", request.worker_id @@ -1091,6 +1100,7 @@ impl Runtime { let durable_request = durable_create_worker_request(&request); let record = WorkerRecord { + restore_guard: RestoreGuard::default(), worker_ref: worker_ref.clone(), worker_id: worker_id.clone(), status: WorkerStatus::Stopped, @@ -1118,7 +1128,6 @@ impl Runtime { restore_candidate_context: None, workdir_attachments: Vec::new(), logical_workdir_attachments: Vec::new(), - execution_handle: None, internal_workers: InternalWorkerActivityProjection::default(), }; state.workers.insert(worker_id, record); @@ -1136,12 +1145,11 @@ impl Runtime { }; let spawn_result = backend.spawn_worker(spawn_request); - let (handle, initial_worker_state, workdir_attachments) = match spawn_result { + let (initial_worker_state, workdir_attachments) = match spawn_result { WorkerExecutionSpawnResult::Connected { - handle, worker_state, workdir_attachments, - } => (handle, worker_state, workdir_attachments), + } => (worker_state, workdir_attachments), WorkerExecutionSpawnResult::Rejected(result) | WorkerExecutionSpawnResult::RolledBack(result) | WorkerExecutionSpawnResult::Errored(result) => { @@ -1156,21 +1164,18 @@ impl Runtime { } WorkerExecutionSpawnResult::ReconciliationRequired { result, - handle, worker_state, workdir_attachments, } => { - let retain_result = - if let (Some(handle), Some(worker_state)) = (handle, worker_state) { - self.retain_create_execution_evidence( - &worker_ref, - handle, - worker_state, - workdir_attachments, - ) - } else { - self.retain_create_reconciliation_pending(&worker_ref, workdir_attachments) - }; + let retain_result = if let Some(worker_state) = worker_state { + self.retain_create_execution_evidence( + &worker_ref, + worker_state, + workdir_attachments, + ) + } else { + self.retain_create_reconciliation_pending(&worker_ref, workdir_attachments) + }; if let Err(error) = retain_result { tracing::error!( worker_id = %worker_ref.worker_id, @@ -1202,9 +1207,9 @@ impl Runtime { .filter(|request_id| !request_id.trim().is_empty()) .unwrap_or_else(|| Uuid::now_v7().to_string()); initial_input.submission_request_id = Some(expected_submission_id.clone()); - let dispatch_result = backend.dispatch_input(&handle, initial_input.clone()); + let dispatch_result = backend.dispatch_input(&worker_ref, initial_input.clone()); if !dispatch_result.is_accepted() { - self.cleanup_connected_failed_create(&backend, &worker_ref, &handle)?; + self.cleanup_connected_failed_create(&backend, &worker_ref)?; return Err(RuntimeError::WorkerExecutionRejected { worker_id: worker_ref.worker_id.clone(), operation: dispatch_result.operation, @@ -1218,7 +1223,7 @@ impl Runtime { .as_ref() .is_some_and(|ack| ack.submission_request_id == expected_submission_id); if !has_durable_acceptance { - self.cleanup_connected_failed_create(&backend, &worker_ref, &handle)?; + self.cleanup_connected_failed_create(&backend, &worker_ref)?; let result = WorkerExecutionResult::rejected( WorkerExecutionOperation::Input, "execution backend accepted initial input without a durable submission acknowledgement", @@ -1233,33 +1238,31 @@ impl Runtime { } let detail = match self.commit_created_worker( &worker_ref, - handle.clone(), initial_worker_state.clone(), workdir_attachments, dispatch_result, ) { Ok(detail) => detail, Err(error) => { - self.cleanup_connected_failed_create(&backend, &worker_ref, &handle)?; + self.cleanup_connected_failed_create(&backend, &worker_ref)?; return Err(error); } }; if let Err(error) = self.record_input_observation(&worker_ref, initial_input) { - self.cleanup_connected_failed_create(&backend, &worker_ref, &handle)?; + self.cleanup_connected_failed_create(&backend, &worker_ref)?; return Err(error); } Ok(detail) } else { match self.commit_created_worker( &worker_ref, - handle.clone(), initial_worker_state, workdir_attachments, WorkerExecutionResult::accepted(WorkerExecutionOperation::Spawn), ) { Ok(detail) => Ok(detail), Err(error) => { - self.cleanup_connected_failed_create(&backend, &worker_ref, &handle)?; + self.cleanup_connected_failed_create(&backend, &worker_ref)?; Err(error) } } @@ -1516,8 +1519,7 @@ impl Runtime { worker_ref: &WorkerRef, attachments: Vec, ) -> Result { - self.ensure_worker_in_workspace(scope, worker_ref)?; - self.replace_worker_workdir_attachments(worker_ref, attachments) + self.replace_worker_workdir_attachments_with_scope(worker_ref, attachments, Some(scope)) } /// Persist the Workspace-authoritative logical attachment set used by live @@ -1526,12 +1528,32 @@ impl Runtime { pub fn replace_worker_workdir_attachments( &self, worker_ref: &WorkerRef, - mut attachments: Vec, + attachments: Vec, + ) -> Result { + self.replace_worker_workdir_attachments_with_scope(worker_ref, attachments, None) + } + + fn replace_worker_workdir_attachments_with_scope( + &self, + worker_ref: &WorkerRef, + attachments: Vec, + scope: Option<&RuntimeWorkspaceScope>, ) -> Result { let operation_lock = self.worker_operation_lock(worker_ref.worker_id)?; let _operation_guard = operation_lock .lock() .map_err(|_| RuntimeError::StatePoisoned)?; + if let Some(scope) = scope { + self.ensure_worker_in_workspace(scope, worker_ref)?; + } + self.replace_worker_workdir_attachments_under_lock(worker_ref, attachments) + } + + fn replace_worker_workdir_attachments_under_lock( + &self, + worker_ref: &WorkerRef, + mut attachments: Vec, + ) -> Result { validate_logical_workdir_attachments(&attachments)?; attachments.sort_by(|left, right| left.alias.cmp(&right.alias)); @@ -1626,10 +1648,7 @@ impl Runtime { self.ensure_worker_in_workspace(scope, worker_ref)?; let backend = { let state = self.lock()?; - let worker = state.worker(worker_ref)?; - if worker.execution_handle.is_some() && worker.status.is_active() { - return Ok(runtime_api::WorkerSessionAvailability::LiveProtocol); - } + state.worker(worker_ref)?; state.execution_backend.clone() }; Ok(match backend { @@ -1708,13 +1727,14 @@ impl Runtime { }) } - /// Attach a live execution through a workspace-scoped Runtime authorization context. + /// Restore admission is fenced by the same lock as every lifecycle mutation. pub fn restore_worker_scoped( &self, scope: &RuntimeWorkspaceScope, worker_ref: &WorkerRef, + request: runtime_api::WorkerRestoreRequest, ) -> Result { - self.restore_worker_operation_scoped(scope, worker_ref)? + self.restore_worker_operation_scoped(scope, worker_ref, request)? .into_legacy_result() } @@ -1722,41 +1742,681 @@ impl Runtime { &self, scope: &RuntimeWorkspaceScope, worker_ref: &WorkerRef, + request: runtime_api::WorkerRestoreRequest, ) -> Result { - self.restore_worker_operation_with_scope(worker_ref, Some(scope)) + self.restore_worker_operation_with_scope(worker_ref, request, Some(scope)) } - /// Attach a live execution to a persisted Worker definition. - /// - /// Every lifecycle mutation for a Worker is serialized by the same operation - /// lock. A concurrent exact restore waits for the first caller and then - /// converges on its already-installed execution rather than spawning another - /// controller. - pub fn restore_worker(&self, worker_ref: &WorkerRef) -> Result { - self.restore_worker_operation(worker_ref)? + pub fn restore_worker( + &self, + worker_ref: &WorkerRef, + request: runtime_api::WorkerRestoreRequest, + ) -> Result { + self.restore_worker_operation(worker_ref, request)? .into_legacy_result() } pub fn restore_worker_operation( &self, worker_ref: &WorkerRef, + request: runtime_api::WorkerRestoreRequest, ) -> Result { - self.restore_worker_operation_with_scope(worker_ref, None) + self.restore_worker_operation_with_scope(worker_ref, request, None) + } + + pub fn coordinate_worker_restore_operation( + &self, + worker_ref: &WorkerRef, + request: runtime_api::WorkerRestoreCoordinationRequest, + scope: Option<&RuntimeWorkspaceScope>, + ) -> Result< + ( + Option, + Option, + ), + RuntimeError, + > { + self.dispatch_restore_worker_with_scope( + worker_ref, + runtime_api::WorkerRestoreRequest { + expected_observation_token: request.expected_observation_token, + request_id: request.request_id, + preparation: request.preparation, + }, + scope, + true, + ) } fn restore_worker_operation_with_scope( &self, worker_ref: &WorkerRef, + request: runtime_api::WorkerRestoreRequest, scope: Option<&RuntimeWorkspaceScope>, ) -> Result { + self.dispatch_restore_worker_with_scope(worker_ref, request, scope, false)? + .0 + .ok_or_else(|| RuntimeError::InvalidRequest("Restore preparation is required".into())) + } + + fn dispatch_restore_worker_with_scope( + &self, + worker_ref: &WorkerRef, + mut request: runtime_api::WorkerRestoreRequest, + scope: Option<&RuntimeWorkspaceScope>, + coordinate: bool, + ) -> Result< + ( + Option, + Option, + ), + RuntimeError, + > { + let mut preparation_authorized = !coordinate && request.preparation.is_some(); let operation_lock = self.worker_operation_lock(worker_ref.worker_id)?; let _operation_guard = operation_lock .lock() .map_err(|_| RuntimeError::StatePoisoned)?; if let Some(scope) = scope { - self.ensure_worker_in_workspace(scope, worker_ref)?; + let mut state = self.lock()?; + if !state + .worker(worker_ref)? + .belongs_to_workspace(&scope.workspace_id) + { + return Err(RuntimeError::WorkerNotFound { + worker_id: worker_ref.worker_id, + }); + } + // Authorization is read-only until token admission succeeds. + state.ensure_workspace_owner(scope, false)?; } - self.restore_worker_under_lock(worker_ref, WorkerRestoreMode::Explicit) + if request.request_id.trim().is_empty() + || request.request_id.len() > 512 + || request.expected_observation_token.is_empty() + || request.expected_observation_token.len() > 256 + { + return Err(RuntimeError::InvalidRequest( + "Restore requires a nonempty request_id and expected_observation_token".to_string(), + )); + } + use sha2::Digest; + // Preparation contains ephemeral Workspace/resource handles, not public + // intent. Retries must use the first admitted owner's durable payload. + // The owner map is scoped to this Worker, so bind only the external fence. + let bytes = serde_json::to_vec(&(&request.expected_observation_token, &request.request_id)) + .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?; + let fingerprint = sha2::Sha256::digest(bytes) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::(); + { + let mut state = self.lock()?; + let mut candidate = state.worker(worker_ref)?.clone(); + let conflict = || RuntimeError::RestoreObservationConflict { + worker_id: worker_ref.worker_id, + }; + if let Some(mut owner) = candidate + .restore_guard + .owners + .get(&request.request_id) + .cloned() + { + if owner.fingerprint != fingerprint { + return Err(conflict()); + } + // Replay is historical: it must not attach a controller after a later stop. + if let Some(receipt) = &owner.receipt { + return Ok((Some(receipt.clone()), None)); + } + if candidate.restore_guard.active_request_id.as_deref() != Some(&request.request_id) + || candidate + .pending_restore + .is_none_or(|pending| pending.operation_id != owner.operation_id) + { + return Err(conflict()); + } + if coordinate && !owner.preparation_applied { + if let Some(preparation) = owner.request.preparation.as_mut() { + if preparation.repository_access_workdirs.is_empty() + && !preparation.repository_access.is_empty() + { + preparation.repository_access_workdirs = preparation + .repository_access + .iter() + .map(|access| runtime_api::RepositoryAccessWorkdirReference { + runtime_id: access.materialization.runtime_id.clone(), + working_directory_id: access.working_directory_id.clone(), + }) + .collect(); + candidate + .restore_guard + .owners + .insert(request.request_id.clone(), owner.clone()); + match state.persist_worker_record(&candidate) { + Ok(()) => { + state + .workers + .insert(worker_ref.worker_id, candidate.clone()); + } + Err(error @ RuntimeError::StoreCommitOutcomeUnknown { .. }) => { + state.workers.insert(worker_ref.worker_id, candidate); + return Ok(( + Some(RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::ReconciliationRequired, + "worker_restore_preparation_commit_unknown", + sanitize_worker_create_failure_message( + &error.to_string(), + ), + )), + None, + )); + } + Err(error) => { + return Ok(( + Some(RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::ReconciliationRequired, + "worker_restore_preparation_commit_failed", + sanitize_worker_create_failure_message( + &error.to_string(), + ), + )), + None, + )); + } + } + } + } + } + if !owner.preparation_applied { + if let (Some(original), Some(incoming)) = + (&owner.request.preparation, &request.preparation) + { + // Server-regenerated bindings are not part of the public fingerprint. + // Reuse admitted bindings and accept only resource completion for those references. + let references_match = incoming.repository_access.len() + == original.repository_access_workdirs.len() + && original.repository_access_workdirs.iter().all(|reference| { + incoming + .repository_access + .iter() + .filter(|access| { + reference.working_directory_id + == access.working_directory_id + && reference.runtime_id + == access.materialization.runtime_id + }) + .count() + == 1 + }); + let snapshot_matches = original.repository_access.is_empty() + || (original.repository_access.len() + == incoming.repository_access.len() + && original + .repository_access + .iter() + .zip(&incoming.repository_access) + .all(|(original, incoming)| { + let mut normalized = incoming.clone(); + if let (Some(old), Some(new)) = ( + &original.materialization.ssh, + &mut normalized.materialization.ssh, + ) { + new.secret_resource = old.secret_resource.clone(); + new.expires_at_epoch_seconds = + old.expires_at_epoch_seconds; + } + &normalized == original + })); + if !original.repository_access_workdirs.is_empty() { + preparation_authorized = + !coordinate && references_match && snapshot_matches; + } + if references_match + && snapshot_matches + && original.repository_access != incoming.repository_access + { + candidate + .restore_guard + .owners + .get_mut(&request.request_id) + .unwrap() + .request + .preparation + .as_mut() + .unwrap() + .repository_access = incoming.repository_access.clone(); + match state.persist_worker_record(&candidate) { + Ok(()) => { + state + .workers + .insert(worker_ref.worker_id, candidate.clone()); + } + Err(error @ RuntimeError::StoreCommitOutcomeUnknown { .. }) => { + state.workers.insert(worker_ref.worker_id, candidate); + return Ok(( + Some(RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::ReconciliationRequired, + "worker_restore_preparation_commit_unknown", + sanitize_worker_create_failure_message( + &error.to_string(), + ), + )), + None, + )); + } + Err(error) => { + return Ok(( + Some(RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::ReconciliationRequired, + "worker_restore_preparation_commit_failed", + sanitize_worker_create_failure_message( + &error.to_string(), + ), + )), + None, + )); + } + } + } + } + } + } else { + // Startup/older persisted pending operations have a lifecycle + // owner but no public request receipt. Bind recovery to that + // exact operation; never allocate replacement execution or apply + // candidate preparation. The current observation still fences + // stale callers, and a second request cannot steal this receipt. + let recovering_pending = candidate.pending_restore.is_some() + && candidate.pending_stop.is_none() + && candidate.restore_guard.active_request_id.is_none(); + if coordinate && request.preparation.is_none() && !recovering_pending { + if candidate.restore_guard.generation != request.expected_observation_token + || candidate.restore_guard.active_request_id.is_some() + || candidate.has_pending_lifecycle_operation() + { + return Err(conflict()); + } + return Ok((None, None)); + } + if coordinate + && request + .preparation + .as_ref() + .is_some_and(|prep| !prep.repository_access.is_empty()) + { + return Err(RuntimeError::InvalidRequest( + "Admission accepts read-only preparation references only".into(), + )); + } + state.ensure_running()?; + if candidate.restore_guard.generation != request.expected_observation_token + || candidate.restore_guard.active_request_id.is_some() + || (candidate.has_pending_lifecycle_operation() && !recovering_pending) + { + return Err(conflict()); + } + if (scope.is_some() || candidate.workspace_id.is_some()) + && request.preparation.is_none() + && !recovering_pending + { + return Err(RuntimeError::InvalidRequest( + "New Workspace Restore requires preparation".into(), + )); + } + if let Some(scope) = scope { + state.ensure_workspace_owner(scope, true)?; + state.persist_runtime_snapshot()?; + } + let pending = candidate + .pending_restore + .unwrap_or_else(|| PendingWorkerRestore { + operation_id: WorkerLifecycleOperationId::new(), + mode: WorkerRestoreMode::Explicit, + last_settled_status: candidate.status, + }); + let operation_id = pending.operation_id; + candidate.pending_restore = Some(pending); + if recovering_pending { + request.preparation = None; + } + candidate.restore_guard.rotate(); + candidate.restore_guard.active_request_id = Some(request.request_id.clone()); + candidate.restore_guard.owners.insert( + request.request_id.clone(), + RestoreRequestOwner { + fingerprint, + operation_id, + request, + preparation_applied: recovering_pending, + execution_started: recovering_pending, + receipt: None, + }, + ); + match state.persist_worker_record(&candidate) { + Ok(()) => {} + Err(error @ RuntimeError::StoreCommitOutcomeUnknown { .. }) => { + state.workers.insert(worker_ref.worker_id, candidate); + return Ok(( + Some(RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::ReconciliationRequired, + "worker_restore_admission_commit_unknown", + sanitize_worker_create_failure_message(&error.to_string()), + )), + None, + )); + } + Err(error) => return Err(error), + } + state.workers.insert(worker_ref.worker_id, candidate); + state.publish_worker_upsert(worker_ref.worker_id)?; + } + } + if coordinate { + let state = self.lock()?; + let candidate = state.worker(worker_ref)?; + let id = candidate + .restore_guard + .active_request_id + .as_ref() + .expect("admitted request"); + let owner = &candidate.restore_guard.owners[id]; + if !owner.preparation_applied + && !owner.execution_started + && owner.request.preparation.is_some() + { + return Ok((None, owner.request.preparation.clone())); + } + } + self.restore_admitted_worker_under_lock_with_preparation( + worker_ref, + WorkerRestoreMode::Explicit, + preparation_authorized, + ) + .map(|result| (Some(result), None)) + } + + #[cfg(test)] + pub(crate) fn test_restore_request( + &self, + worker_ref: &WorkerRef, + ) -> runtime_api::WorkerRestoreRequest { + let state = self.lock().unwrap(); + let worker = state.worker(worker_ref).unwrap(); + if let Some(id) = &worker.restore_guard.active_request_id { + return worker.restore_guard.owners[id].request.clone(); + } + runtime_api::WorkerRestoreRequest { + expected_observation_token: worker.restore_guard.generation.clone(), + request_id: uuid::Uuid::now_v7().to_string(), + preparation: worker + .workspace_id + .as_ref() + .map(|_| runtime_api::WorkerRestorePreparation::default()), + } + } + + /// Private startup and exact pending-owner reconciliation use the durable payload. + fn restore_admitted_worker_under_lock( + &self, + worker_ref: &WorkerRef, + mode: WorkerRestoreMode, + ) -> Result { + self.restore_admitted_worker_under_lock_with_preparation(worker_ref, mode, false) + } + + fn restore_admitted_worker_under_lock_with_preparation( + &self, + worker_ref: &WorkerRef, + mode: WorkerRestoreMode, + preparation_authorized: bool, + ) -> Result { + let owner = { + let state = self.lock()?; + let worker = state.worker(worker_ref)?; + match worker.restore_guard.active_request_id.as_ref() { + Some(id) => { + let owner = worker.restore_guard.owners.get(id).ok_or_else(|| { + RuntimeError::InvalidRequest( + "Active Restore request has no persisted owner".into(), + ) + })?; + if worker + .pending_restore + .is_some_and(|pending| pending.operation_id != owner.operation_id) + { + return Err(RuntimeError::RestoreObservationConflict { + worker_id: worker_ref.worker_id, + }); + } + Some(owner.clone()) + } + None => None, + } + }; + if let Some(owner) = owner { + // A settled owner must never turn into a fresh execution, including + // when startup encounters an old active-request marker. + if let Some(receipt) = owner.receipt { + return Ok(receipt); + } + if owner.execution_started { + let committed_detail = { + let state = self.lock()?; + let worker = state.worker(worker_ref)?; + if worker.pending_restore.is_none() && worker.pending_stop.is_none() { + if !worker.execution_bound || !worker.status.is_active() { + return Ok(RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::ReconciliationRequired, + "worker_restore_terminal_receipt_missing", + "Restore execution was started but its terminal receipt is unavailable", + )); + } + Some(worker.detail()) + } else { + None + } + }; + if let Some(detail) = committed_detail { + // Recover a receipt gap without launching execution again. + let result = RuntimeWorkerRestoreResult::accepted(detail); + self.finish_restore_receipt(worker_ref, &result)?; + return Ok(result); + } + } + if !owner.preparation_applied { + let uncertain = |message: String| { + RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::ReconciliationRequired, + "worker_restore_preparation_reconciliation_required", + message, + ) + }; + if owner.request.preparation.as_ref().is_some_and(|prep| { + prep.repository_access.len() < prep.repository_access_workdirs.len() + || (!prep.repository_access_workdirs.is_empty() && !preparation_authorized) + }) { + return Ok(uncertain( + "Admitted Restore is awaiting Workspace resource completion".into(), + )); + } + // Complete idempotent binding/access effects under the original pending_restore ID. + let mut effects_started = false; + if let Err(error) = self.apply_restore_preparation_under_lock( + worker_ref, + owner.request.preparation, + owner.operation_id, + &mut effects_started, + ) { + if effects_started + || matches!(error, RuntimeError::StoreCommitOutcomeUnknown { .. }) + { + return Ok(uncertain(sanitize_worker_create_failure_message( + &error.to_string(), + ))); + } + let result = RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::Rejected, + "worker_restore_preparation_rejected", + sanitize_worker_create_failure_message(&error.to_string()), + ); + self.finish_restore_receipt(worker_ref, &result)?; + return Ok(result); + } + let mut state = self.lock()?; + let mut candidate = state.worker(worker_ref)?.clone(); + candidate + .restore_guard + .owners + .get_mut(&owner.request.request_id) + .expect("admitted owner") + .preparation_applied = true; + if let Err(error) = state.persist_worker_record(&candidate) { + // Keep the started (not applied) record in memory. Even a + // known failed completion write follows possible effects. + return Ok(uncertain(sanitize_worker_create_failure_message( + &error.to_string(), + ))); + } + state.workers.insert(worker_ref.worker_id, candidate); + } + } + let result = self.restore_worker_under_lock(worker_ref, mode)?; + self.finish_restore_receipt(worker_ref, &result)?; + Ok(result) + } + + fn finish_restore_receipt( + &self, + worker_ref: &WorkerRef, + result: &RuntimeWorkerRestoreResult, + ) -> Result<(), RuntimeError> { + if result.state == WorkerRestoreState::ReconciliationRequired { + return Ok(()); + } + let mut state = self.lock()?; + let mut candidate = state.worker(worker_ref)?.clone(); + if candidate.restore_guard.active_request_id.is_some() { + candidate.restore_guard.finish(result.clone()); + candidate.pending_restore = None; + state.persist_worker_record(&candidate)?; + state.workers.insert(worker_ref.worker_id, candidate); + } + Ok(()) + } + + fn apply_restore_preparation_under_lock( + &self, + worker_ref: &WorkerRef, + preparation: Option, + operation_id: WorkerLifecycleOperationId, + effects_started: &mut bool, + ) -> Result<(), RuntimeError> { + let Some(preparation) = preparation else { + return Ok(()); + }; + // Validate all read-only structure before any binding effect. + if let Some(attachments) = &preparation.workdir_attachments { + let attachments: Vec = serde_json::from_value( + serde_json::to_value(attachments) + .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?, + ) + .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?; + validate_logical_workdir_attachments(&attachments)?; + } + // DTO conversion and every Runtime preparation effect occur AFTER admission. + if let Some(api) = preparation.workspace_api { + let api: WorkspaceApiRef = serde_json::from_value( + serde_json::to_value(api) + .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?, + ) + .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?; + if self.lock()?.worker(worker_ref)?.workspace_id.as_deref() + != Some(api.workspace_id.as_str()) + { + return Err(RuntimeError::InvalidRequest( + "Prepared Workspace API does not match Worker Workspace identity".into(), + )); + } + let mut state = self.lock()?; + let mut candidate = state.worker(worker_ref)?.clone(); + if candidate.pending_stop.is_some() + || candidate + .pending_restore + .is_none_or(|pending| pending.operation_id != operation_id) + { + return Err(RuntimeError::RestoreObservationConflict { + worker_id: worker_ref.worker_id, + }); + } + let persisted = candidate.request.as_mut().ok_or_else(|| { + RuntimeError::InvalidRequest("Restore specification is missing".into()) + })?; + if persisted.workspace_api.as_ref().is_some_and(|existing| { + existing.workspace_id != api.workspace_id + || existing.base_url.trim_end_matches('/') != api.base_url.trim_end_matches('/') + }) { + return Err(RuntimeError::InvalidRequest( + "Workspace API preparation cannot change Worker identity or base URL".into(), + )); + } + persisted.workspace_api = Some(api); + *effects_started = true; + match state.persist_worker_record(&candidate) { + Ok(()) => { + state.workers.insert(worker_ref.worker_id, candidate); + } + Err(error @ RuntimeError::StoreCommitOutcomeUnknown { .. }) => { + state.workers.insert(worker_ref.worker_id, candidate); + return Err(error); + } + Err(error) => return Err(error), + } + } + if let Some(attachments) = preparation.workdir_attachments { + let attachments: Vec = serde_json::from_value( + serde_json::to_value(attachments) + .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?, + ) + .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?; + validate_logical_workdir_attachments(&attachments)?; + *effects_started = true; + self.replace_worker_workdir_attachments_under_lock(worker_ref, attachments)?; + } + for access in preparation.repository_access { + if preparation + .repository_access_workdirs + .iter() + .any(|reference| { + reference.working_directory_id == access.working_directory_id + && reference.runtime_id == access.materialization.runtime_id + }) + { + // Workspace completed these at the Workdir owner Runtime after snapshot persistence. + // Never send cross-Runtime credentials to this Worker's Runtime. + continue; + } + let access: WorkingDirectoryRepositoryAccessRequest = serde_json::from_value( + serde_json::to_value(access) + .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?, + ) + .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?; + *effects_started = true; + let runtime = self.clone(); + // The public embedded interface is synchronous. A separate IO executor + // avoids nested Tokio block_on while retaining the lifecycle lock. + std::thread::spawn(move || { + let executor = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?; + executor.block_on( + runtime.authorize_working_directory_repository_access_from_resource(access), + ) + }) + .join() + .map_err(|_| RuntimeError::StatePoisoned)??; + } + Ok(()) } fn restore_worker_under_lock( @@ -1764,7 +2424,17 @@ impl Runtime { worker_ref: &WorkerRef, mode: WorkerRestoreMode, ) -> Result { - let fresh_operation_id = WorkerLifecycleOperationId::new(); + let fresh_operation_id = { + let state = self.lock()?; + let worker = state.worker(worker_ref)?; + worker + .restore_guard + .active_request_id + .as_ref() + .and_then(|id| worker.restore_guard.owners.get(id)) + .map(|owner| owner.operation_id) + .unwrap_or_else(WorkerLifecycleOperationId::new) + }; let (backend, request, operation_id, operation_mode, reconcile, candidate_context) = { let state = self.lock()?; state.ensure_running()?; @@ -1777,38 +2447,17 @@ impl Runtime { )); } let pending_restore = worker.pending_restore; - let reconcile = pending_restore.is_some(); + let reconcile = pending_restore.is_some() + && worker + .restore_guard + .active_request_id + .as_ref() + .and_then(|id| worker.restore_guard.owners.get(id)) + .is_none_or(|owner| owner.execution_started); let operation_id = pending_restore .map(|pending| pending.operation_id) .unwrap_or(fresh_operation_id); let operation_mode = pending_restore.map(|pending| pending.mode).unwrap_or(mode); - if !reconcile && worker.execution_handle.is_some() { - if worker.status.is_active() { - return Ok(RuntimeWorkerRestoreResult::accepted(worker.detail())); - } - return Ok(RuntimeWorkerRestoreResult::failed( - WorkerRestoreState::ReconciliationRequired, - "worker_restore_cleanup_pending", - "Worker restore cleanup or reconciliation is still pending", - )); - } - // An active persisted Worker can still have no live handle when its - // startup restore was rejected before side effects (for example, - // before Workspace Backend renewed operation-scoped Repository - // access). Preserve its automatic intent so an explicit restore can - // serve as the authorized retry for that state. - if !reconcile - && mode == WorkerRestoreMode::Explicit - && worker.execution_metadata_available - && worker.status != WorkerStatus::Stopped - && worker.restore_intent != WorkerRestoreIntent::Automatic - { - return Ok(RuntimeWorkerRestoreResult::failed( - WorkerRestoreState::Rejected, - "worker_restore_not_stopped", - "Worker restore requires a stopped Worker", - )); - } if !reconcile && mode == WorkerRestoreMode::Automatic && (!worker.status.is_active() @@ -1861,11 +2510,39 @@ impl Runtime { ) }; - if !reconcile && let Err(_result) = backend.preflight_restore(&request) { + if !reconcile && let Err(result) = backend.preflight_restore(&request) { + // A preflight may attest an already-connected Controller snapshot. + // Repeated restore is read-only; do not replace its operation owner. + if result.outcome == crate::execution::WorkerExecutionOutcome::Busy + && let Some(snapshot) = result.worker_state.as_ref() + { + let unfinished_context = { + self.lock()? + .worker(worker_ref)? + .restore_candidate_context + .clone() + }; + if let Some(context) = unfinished_context { + // The prior aggregate commit succeeded, but observation + // activation can still be unfinished after a transient error. + if let Err(error) = + self.activate_restore_candidate_context(worker_ref, &context) + { + return Ok(RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::ReconciliationRequired, + "worker_restore_observation_activation_pending", + sanitize_worker_create_failure_message(&error.to_string()), + )); + } + } + let mut detail = self.lock()?.worker(worker_ref)?.detail(); + detail.worker_state = Some(snapshot.clone()); + return Ok(RuntimeWorkerRestoreResult::accepted(detail)); + } return Ok(RuntimeWorkerRestoreResult::failed( WorkerRestoreState::Rejected, "worker_restore_preflight_rejected", - "Worker restore preflight rejected the current request", + sanitize_worker_create_failure_message(&result.message_or_default()), )); } @@ -1894,20 +2571,19 @@ impl Runtime { }; match restore_result { WorkerExecutionSpawnResult::Connected { - handle, worker_state, workdir_attachments, } => { let commit = self.commit_restored_worker_execution( worker_ref, - handle.clone(), worker_state.clone(), WorkerStatus::Idle, workdir_attachments.clone(), ); match commit { Ok(worker) => { - if let Err(error) = backend.activate_restored_worker(operation_id, &handle) + if let Err(error) = + backend.activate_restored_worker(operation_id, worker_ref) { tracing::error!( worker_id = %worker_ref.worker_id, @@ -1915,12 +2591,19 @@ impl Runtime { "Worker restore committed but Backend observation activation failed" ); } - if let Err(error) = candidate_context.activate_candidate() { + if let Err(error) = + self.activate_restore_candidate_context(worker_ref, &candidate_context) + { tracing::error!( worker_id = %worker_ref.worker_id, error = %error, "Worker restore committed but buffered protocol observations could not be activated" ); + return Ok(RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::ReconciliationRequired, + "worker_restore_observation_activation_pending", + sanitize_worker_create_failure_message(&error.to_string()), + )); } Ok(RuntimeWorkerRestoreResult::accepted(worker)) } @@ -1932,7 +2615,6 @@ impl Runtime { ); if let Err(retain_error) = self.retain_restore_execution_evidence( worker_ref, - handle, worker_state, workdir_attachments, candidate_context.clone(), @@ -1949,74 +2631,78 @@ impl Runtime { "Worker restore commit outcome is uncertain; reread or retry the Worker restore", )) } - Err(error) => { - match self.cleanup_failed_restore(&backend, worker_ref, &handle) { - Ok(()) => { - candidate_context.discard_candidate(); - tracing::warn!( - worker_id = %worker_ref.worker_id, - error = %error, - "Worker restore commit failed and was rolled back" - ); - if let Err(settle_error) = self.settle_restore_rollback(worker_ref) - { - tracing::error!( - worker_id = %worker_ref.worker_id, - error = %settle_error, - "Worker restore cleanup completed but its durable operation could not be settled" - ); - return Ok(RuntimeWorkerRestoreResult::failed( - WorkerRestoreState::ReconciliationRequired, - "worker_restore_rollback_commit_failed", - "Worker restore cleanup completed but durable reconciliation is still required", - )); - } - Ok(RuntimeWorkerRestoreResult::failed( - WorkerRestoreState::RolledBack, - "worker_restore_commit_rolled_back", - "Worker restore commit failed and cleanup completed", - )) - } - Err(cleanup_error) => { + Err(error) => match self.cleanup_failed_restore(&backend, worker_ref) { + Ok(()) => { + self.discard_restore_candidate_context(worker_ref, &candidate_context)?; + tracing::warn!( + worker_id = %worker_ref.worker_id, + error = %error, + "Worker restore commit failed and was rolled back" + ); + let receipt = RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::RolledBack, + "worker_restore_commit_rolled_back", + "Worker restore commit failed and cleanup completed", + ); + if let Err(settle_error) = + self.settle_restore_rollback(worker_ref, &receipt) + { tracing::error!( worker_id = %worker_ref.worker_id, - error = %error, - cleanup_error = %cleanup_error, - "Worker restore requires reconciliation after commit and cleanup failures" + error = %settle_error, + "Worker restore cleanup completed but its durable operation could not be settled" ); - if let Err(retain_error) = self.retain_restore_execution_evidence( - worker_ref, - handle, - worker_state, - workdir_attachments, - candidate_context.clone(), - ) { - tracing::error!( - worker_id = %worker_ref.worker_id, - error = %retain_error, - "failed to persist restore execution evidence; in-process reconciliation remains required" - ); - } - Ok(RuntimeWorkerRestoreResult::failed( + return Ok(RuntimeWorkerRestoreResult::failed( WorkerRestoreState::ReconciliationRequired, - "worker_restore_reconciliation_required", - "Worker restore result is uncertain; reread or retry the Worker restore", - )) + "worker_restore_rollback_commit_failed", + "Worker restore cleanup completed but durable reconciliation is still required", + )); } + Ok(receipt) } - } + Err(cleanup_error) => { + tracing::error!( + worker_id = %worker_ref.worker_id, + error = %error, + cleanup_error = %cleanup_error, + "Worker restore requires reconciliation after commit and cleanup failures" + ); + if let Err(retain_error) = self.retain_restore_execution_evidence( + worker_ref, + worker_state, + workdir_attachments, + candidate_context.clone(), + ) { + tracing::error!( + worker_id = %worker_ref.worker_id, + error = %retain_error, + "failed to persist restore execution evidence; in-process reconciliation remains required" + ); + } + Ok(RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::ReconciliationRequired, + "worker_restore_reconciliation_required", + "Worker restore result is uncertain; reread or retry the Worker restore", + )) + } + }, } } - WorkerExecutionSpawnResult::Rejected(_result) => { + WorkerExecutionSpawnResult::Rejected(result) => { if reconcile { return Ok(RuntimeWorkerRestoreResult::failed( WorkerRestoreState::ReconciliationRequired, "worker_restore_reconciliation_rejected", - "Worker restore reconciliation was not completed", + sanitize_worker_create_failure_message(&result.message_or_default()), )); } - candidate_context.discard_candidate(); - if let Err(error) = self.settle_restore_rollback(worker_ref) { + self.discard_restore_candidate_context(worker_ref, &candidate_context)?; + let receipt = RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::Rejected, + "worker_restore_rejected", + sanitize_worker_create_failure_message(&result.message_or_default()), + ); + if let Err(error) = self.settle_restore_rollback(worker_ref, &receipt) { tracing::error!( worker_id = %worker_ref.worker_id, error = %error, @@ -2028,16 +2714,20 @@ impl Runtime { "Worker restore was rejected but durable reconciliation is still required", )); } - Ok(RuntimeWorkerRestoreResult::failed( - WorkerRestoreState::Rejected, - "worker_restore_rejected", - "Worker restore was rejected before live work started", - )) + Ok(receipt) } WorkerExecutionSpawnResult::RolledBack(_result) => { - candidate_context.discard_candidate(); + let receipt = RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::RolledBack, + "worker_restore_rolled_back", + sanitize_worker_create_failure_message(&_result.message_or_default()), + ); + self.discard_restore_candidate_context(worker_ref, &candidate_context)?; #[cfg(feature = "fs-store")] - if let Err(error) = self.lock()?.record_restore_failure(worker_ref, _result) { + if let Err(error) = self + .lock()? + .record_restore_failure(worker_ref, &_result, &receipt) + { tracing::error!( worker_id = %worker_ref.worker_id, error = %error, @@ -2050,23 +2740,17 @@ impl Runtime { )); } #[cfg(not(feature = "fs-store"))] - self.settle_restore_rollback(worker_ref)?; - Ok(RuntimeWorkerRestoreResult::failed( - WorkerRestoreState::RolledBack, - "worker_restore_rolled_back", - "Worker restore failed after live work started; cleanup completed", - )) + self.settle_restore_rollback(worker_ref, &receipt)?; + Ok(receipt) } WorkerExecutionSpawnResult::ReconciliationRequired { result: _result, - handle, worker_state, workdir_attachments, } => { - if let (Some(handle), Some(worker_state)) = (handle, worker_state) { + if let Some(worker_state) = worker_state { if let Err(retain_error) = self.retain_restore_execution_evidence( worker_ref, - handle, worker_state, workdir_attachments, candidate_context.clone(), @@ -2105,13 +2789,39 @@ impl Runtime { Ok(RuntimeWorkerRestoreResult::failed( WorkerRestoreState::ReconciliationRequired, "worker_restore_backend_outcome_unknown", - "Worker restore backend outcome is uncertain; reread or retry the Worker restore", + sanitize_worker_create_failure_message(&_result.message_or_default()), )) } } } - fn ensure_worker_execution(&self, worker_ref: &WorkerRef) -> Result<(), RuntimeError> { + fn activate_restore_candidate_context( + &self, + worker_ref: &WorkerRef, + context: &crate::execution::WorkerExecutionContext, + ) -> Result<(), RuntimeError> { + context.activate_candidate()?; + self.lock()? + .worker_mut(worker_ref)? + .restore_candidate_context = None; + Ok(()) + } + + fn discard_restore_candidate_context( + &self, + worker_ref: &WorkerRef, + context: &crate::execution::WorkerExecutionContext, + ) -> Result<(), RuntimeError> { + // Cleanup/rejection proved no live candidate remains. Detach its + // discarded publication owner even if journal settlement later fails. + self.lock()? + .worker_mut(worker_ref)? + .restore_candidate_context = None; + context.discard_candidate(); + Ok(()) + } + + fn ensure_worker_operation_settled(&self, worker_ref: &WorkerRef) -> Result<(), RuntimeError> { let state = self.lock()?; let worker = state.worker(worker_ref)?; if let Some(operation) = worker.pending_lifecycle_operation_name() { @@ -2120,18 +2830,7 @@ impl Runtime { message: format!("worker {operation} reconciliation is pending"), }); } - if worker.execution_handle.is_some() { - return Ok(()); - } - let message = if worker.status == WorkerStatus::Stopped { - "stopped worker requires an explicit restore" - } else { - "worker has no live execution handle" - }; - Err(RuntimeError::WorkerExecutionUnavailable { - worker_id: worker_ref.worker_id, - message: message.to_string(), - }) + Ok(()) } /// Accept input into a Worker through a workspace-scoped Runtime authorization context. @@ -2141,16 +2840,31 @@ impl Runtime { worker_ref: &WorkerRef, input: WorkerInput, ) -> Result { - self.ensure_worker_in_workspace(scope, worker_ref)?; - self.send_input(worker_ref, input) + self.send_input_with_scope(worker_ref, input, Some(scope)) } /// Accept input into a Worker. pub fn send_input( &self, worker_ref: &WorkerRef, - mut input: WorkerInput, + input: WorkerInput, ) -> Result { + self.send_input_with_scope(worker_ref, input, None) + } + + fn send_input_with_scope( + &self, + worker_ref: &WorkerRef, + mut input: WorkerInput, + scope: Option<&RuntimeWorkspaceScope>, + ) -> Result { + let operation_lock = self.worker_operation_lock(worker_ref.worker_id)?; + let _operation_guard = operation_lock + .lock() + .map_err(|_| RuntimeError::StatePoisoned)?; + if let Some(scope) = scope { + self.ensure_worker_in_workspace(scope, worker_ref)?; + } validate_worker_input(&input)?; if matches!( input.kind, @@ -2190,32 +2904,21 @@ impl Runtime { )); } } - self.ensure_worker_execution(worker_ref)?; - let (backend, handle) = { + self.ensure_worker_operation_settled(worker_ref)?; + let backend = { let state = self.lock()?; state.ensure_running()?; state.ensure_worker_ref(worker_ref)?; - let worker = state.worker(worker_ref)?; - if !worker.status.is_active() { - return Err(RuntimeError::InvalidRequest(format!( - "worker {} is not running", - worker_ref.worker_id - ))); - } - let backend = state.execution_backend.clone(); - let handle = worker.execution_handle.clone(); - match (backend, handle) { - (Some(backend), Some(handle)) => (backend, handle), - _ => { - return Err(RuntimeError::WorkerExecutionUnavailable { - worker_id: worker_ref.worker_id.clone(), - message: "worker has no live execution handle".to_string(), - }); + state.worker(worker_ref)?; + state.execution_backend.clone().ok_or_else(|| { + RuntimeError::WorkerExecutionUnavailable { + worker_id: worker_ref.worker_id, + message: "runtime has no execution backend".to_string(), } - } + })? }; - let dispatch_result = backend.dispatch_input(&handle, input.clone()); + let dispatch_result = backend.dispatch_input(worker_ref, input.clone()); if !dispatch_result.is_accepted() { self.record_execution_result(worker_ref, dispatch_result.clone())?; return Err(RuntimeError::WorkerExecutionRejected { @@ -2300,8 +3003,14 @@ impl Runtime { media_type: &str, content: &[u8], ) -> Result { - self.ensure_worker_in_workspace(scope, worker_ref)?; - self.upload_worker_file(worker_ref, file_name, media_type, content) + self.upload_worker_file_inner( + worker_ref, + file_name, + media_type, + content, + None, + Some(scope), + ) } pub fn upload_worker_file( @@ -2311,7 +3020,7 @@ impl Runtime { media_type: &str, content: &[u8], ) -> Result { - self.upload_worker_file_inner(worker_ref, file_name, media_type, content, None) + self.upload_worker_file_inner(worker_ref, file_name, media_type, content, None, None) } pub fn upload_worker_file_with_context_scoped( @@ -2323,8 +3032,14 @@ impl Runtime { content: &[u8], context: &session_store::UploadedFileUploadContext, ) -> Result { - self.ensure_worker_in_workspace(scope, worker_ref)?; - self.upload_worker_file_inner(worker_ref, file_name, media_type, content, Some(context)) + self.upload_worker_file_inner( + worker_ref, + file_name, + media_type, + content, + Some(context), + Some(scope), + ) } pub fn upload_worker_file_with_context( @@ -2335,7 +3050,14 @@ impl Runtime { content: &[u8], context: &session_store::UploadedFileUploadContext, ) -> Result { - self.upload_worker_file_inner(worker_ref, file_name, media_type, content, Some(context)) + self.upload_worker_file_inner( + worker_ref, + file_name, + media_type, + content, + Some(context), + None, + ) } fn upload_worker_file_inner( @@ -2345,27 +3067,30 @@ impl Runtime { media_type: &str, content: &[u8], context: Option<&session_store::UploadedFileUploadContext>, + scope: Option<&RuntimeWorkspaceScope>, ) -> Result { - let (backend, handle) = { + let operation_lock = self.worker_operation_lock(worker_ref.worker_id)?; + let _operation_guard = operation_lock + .lock() + .map_err(|_| RuntimeError::StatePoisoned)?; + if let Some(scope) = scope { + self.ensure_worker_in_workspace(scope, worker_ref)?; + } + self.ensure_worker_operation_settled(worker_ref)?; + let backend = { let state = self.lock()?; state.ensure_running()?; state.ensure_worker_ref(worker_ref)?; - let worker = state.worker(worker_ref)?; - match ( - state.execution_backend.clone(), - worker.execution_handle.clone(), - ) { - (Some(backend), Some(handle)) => (backend, handle), - _ => { - return Err(RuntimeError::WorkerExecutionUnavailable { - worker_id: worker_ref.worker_id.clone(), - message: "worker has no live execution handle".to_string(), - }); + state.worker(worker_ref)?; + state.execution_backend.clone().ok_or_else(|| { + RuntimeError::WorkerExecutionUnavailable { + worker_id: worker_ref.worker_id, + message: "runtime has no execution backend".to_string(), } - } + })? }; backend - .upload_file(&handle, file_name, media_type, content, context) + .upload_file(worker_ref, file_name, media_type, content, context) .map_err(|result| RuntimeError::WorkerExecutionRejected { worker_id: worker_ref.worker_id.clone(), operation: result.operation, @@ -2382,8 +3107,7 @@ impl Runtime { worker_ref: &WorkerRef, artifact_id: &str, ) -> Result<(), RuntimeError> { - self.ensure_worker_in_workspace(scope, worker_ref)?; - self.delete_worker_uploaded_file(worker_ref, artifact_id) + self.delete_worker_uploaded_file_with_scope(worker_ref, artifact_id, Some(scope)) } pub fn delete_worker_uploaded_file( @@ -2391,25 +3115,36 @@ impl Runtime { worker_ref: &WorkerRef, artifact_id: &str, ) -> Result<(), RuntimeError> { - let (backend, handle) = { + self.delete_worker_uploaded_file_with_scope(worker_ref, artifact_id, None) + } + + fn delete_worker_uploaded_file_with_scope( + &self, + worker_ref: &WorkerRef, + artifact_id: &str, + scope: Option<&RuntimeWorkspaceScope>, + ) -> Result<(), RuntimeError> { + let operation_lock = self.worker_operation_lock(worker_ref.worker_id)?; + let _operation_guard = operation_lock + .lock() + .map_err(|_| RuntimeError::StatePoisoned)?; + if let Some(scope) = scope { + self.ensure_worker_in_workspace(scope, worker_ref)?; + } + self.ensure_worker_operation_settled(worker_ref)?; + let backend = { let state = self.lock()?; state.ensure_running()?; state.ensure_worker_ref(worker_ref)?; - let worker = state.worker(worker_ref)?; - match ( - state.execution_backend.clone(), - worker.execution_handle.clone(), - ) { - (Some(backend), Some(handle)) => (backend, handle), - _ => { - return Err(RuntimeError::WorkerExecutionUnavailable { - worker_id: worker_ref.worker_id.clone(), - message: "worker has no live execution handle".to_string(), - }); + state.worker(worker_ref)?; + state.execution_backend.clone().ok_or_else(|| { + RuntimeError::WorkerExecutionUnavailable { + worker_id: worker_ref.worker_id, + message: "runtime has no execution backend".to_string(), } - } + })? }; - let result = backend.delete_uploaded_file(&handle, artifact_id); + let result = backend.delete_uploaded_file(worker_ref, artifact_id); if result.is_accepted() { Ok(()) } else { @@ -2430,9 +3165,9 @@ impl Runtime { worker_ref: &WorkerRef, kind: protocol::CompletionKind, prefix: &str, + context: Option<&protocol::CompletionContext>, ) -> Result, RuntimeError> { - self.ensure_worker_in_workspace(scope, worker_ref)?; - self.worker_completions(worker_ref, kind, prefix) + self.worker_completions_with_scope(worker_ref, kind, prefix, context, Some(scope)) } /// Return live completion entries for the Worker composer. @@ -2441,23 +3176,39 @@ impl Runtime { worker_ref: &WorkerRef, kind: protocol::CompletionKind, prefix: &str, + context: Option<&protocol::CompletionContext>, ) -> Result, RuntimeError> { - let (backend, handle) = { + self.worker_completions_with_scope(worker_ref, kind, prefix, context, None) + } + + fn worker_completions_with_scope( + &self, + worker_ref: &WorkerRef, + kind: protocol::CompletionKind, + prefix: &str, + context: Option<&protocol::CompletionContext>, + scope: Option<&RuntimeWorkspaceScope>, + ) -> Result, RuntimeError> { + let operation_lock = self.worker_operation_lock(worker_ref.worker_id)?; + let _operation_guard = operation_lock + .lock() + .map_err(|_| RuntimeError::StatePoisoned)?; + if let Some(scope) = scope { + self.ensure_worker_in_workspace(scope, worker_ref)?; + } + let backend = { let state = self.lock()?; state.ensure_worker_ref(worker_ref)?; let worker = state.worker(worker_ref)?; if worker.has_pending_lifecycle_operation() { return Ok(Vec::new()); } - ( - state.execution_backend.clone(), - worker.execution_handle.clone(), - ) + state.execution_backend.clone() }; - let Some((backend, handle)) = backend.zip(handle) else { + let Some(backend) = backend else { return Ok(Vec::new()); }; - Ok(backend.worker_completions(&handle, kind, prefix)) + Ok(backend.worker_completions(worker_ref, kind, prefix, context)) } /// Accept a protocol method through a workspace-scoped Runtime authorization context. @@ -2467,8 +3218,7 @@ impl Runtime { worker_ref: &WorkerRef, method: Method, ) -> Result, RuntimeError> { - self.ensure_worker_in_workspace(scope, worker_ref)?; - self.send_protocol_method(worker_ref, method) + self.send_protocol_method_with_scope(worker_ref, method, Some(scope)) } /// Accept a protocol method for a Worker through a Backend/runtime transport. @@ -2482,41 +3232,68 @@ impl Runtime { worker_ref: &WorkerRef, method: Method, ) -> Result, RuntimeError> { - if let Method::ListCompletions { kind, prefix } = method { - let entries = self.worker_completions(worker_ref, kind, &prefix)?; - return Ok(vec![Event::Completions { kind, entries }]); + self.send_protocol_method_with_scope(worker_ref, method, None) + } + + fn send_protocol_method_with_scope( + &self, + worker_ref: &WorkerRef, + method: Method, + scope: Option<&RuntimeWorkspaceScope>, + ) -> Result, RuntimeError> { + if let Method::ListCompletions { + kind, + prefix, + request_id, + context, + } = method + { + let entries = self.worker_completions_with_scope( + worker_ref, + kind, + &prefix, + context.as_ref(), + scope, + )?; + return Ok(vec![Event::Completions { + kind, + prefix, + request_id, + context, + entries, + }]); } if matches!(&method, Method::Shutdown { .. }) { - self.stop_worker(worker_ref, Some("worker protocol shutdown".to_string()))?; + self.stop_worker_with_scope( + worker_ref, + Some("worker protocol shutdown".to_string()), + scope, + )?; return Ok(Vec::new()); } - self.ensure_worker_execution(worker_ref)?; + let operation_lock = self.worker_operation_lock(worker_ref.worker_id)?; + let _operation_guard = operation_lock + .lock() + .map_err(|_| RuntimeError::StatePoisoned)?; + if let Some(scope) = scope { + self.ensure_worker_in_workspace(scope, worker_ref)?; + } + self.ensure_worker_operation_settled(worker_ref)?; - let (backend, handle) = { + let backend = { let state = self.lock()?; state.ensure_running()?; state.ensure_worker_ref(worker_ref)?; - let worker = state.worker(worker_ref)?; - if !worker.status.is_active() { - return Err(RuntimeError::InvalidRequest(format!( - "worker {} is not running", - worker_ref.worker_id - ))); - } - let backend = state.execution_backend.clone(); - let handle = worker.execution_handle.clone(); - match (backend, handle) { - (Some(backend), Some(handle)) => (backend, handle), - _ => { - return Err(RuntimeError::WorkerExecutionUnavailable { - worker_id: worker_ref.worker_id.clone(), - message: "worker has no live execution handle".to_string(), - }); + state.worker(worker_ref)?; + state.execution_backend.clone().ok_or_else(|| { + RuntimeError::WorkerExecutionUnavailable { + worker_id: worker_ref.worker_id, + message: "runtime has no execution backend".to_string(), } - } + })? }; - let dispatch_result = backend.dispatch_method(&handle, method); + let dispatch_result = backend.dispatch_method(worker_ref, method); if !dispatch_result.is_accepted() { self.record_execution_result(worker_ref, dispatch_result.clone())?; return Err(RuntimeError::WorkerExecutionRejected { @@ -2532,10 +3309,138 @@ impl Runtime { Ok(Vec::new()) } + /// Attach one protocol transport under the same admission/authorization lock + /// as lifecycle operations. The backend owns the real Controller endpoint; + /// this session is never a Worker catalog field or a routing token. + #[cfg(feature = "ws-server")] + pub fn attach_worker_protocol( + &self, + worker_ref: &WorkerRef, + ) -> Result { + self.attach_worker_protocol_with_scope(worker_ref, None) + } + + #[cfg(feature = "ws-server")] + fn attach_worker_protocol_with_scope( + &self, + worker_ref: &WorkerRef, + scope: Option<&RuntimeWorkspaceScope>, + ) -> Result { + let operation_lock = self.worker_operation_lock(worker_ref.worker_id)?; + let _operation_guard = operation_lock + .lock() + .map_err(|_| RuntimeError::StatePoisoned)?; + if let Some(scope) = scope { + self.ensure_worker_in_workspace(scope, worker_ref)?; + } + self.ensure_worker_operation_settled(worker_ref)?; + let backend = { + let state = self.lock()?; + state.ensure_running()?; + state.worker(worker_ref)?; + state.execution_backend.clone().ok_or_else(|| { + RuntimeError::WorkerExecutionUnavailable { + worker_id: worker_ref.worker_id, + message: "runtime has no execution backend".to_string(), + } + })? + }; + let transport = backend + .attach_worker_protocol(worker_ref) + .map_err(|result| RuntimeError::WorkerExecutionRejected { + worker_id: worker_ref.worker_id, + operation: result.operation, + outcome: result.outcome, + message: result.message_or_default(), + result, + })?; + if &transport.worker_ref != worker_ref { + return Err(RuntimeError::WorkerExecutionUnavailable { + worker_id: worker_ref.worker_id, + message: "protocol endpoint target does not match the authorized Worker".into(), + }); + } + Ok(transport) + } + + #[cfg(feature = "ws-server")] + pub fn attach_worker_protocol_scoped( + &self, + scope: &RuntimeWorkspaceScope, + worker_ref: &WorkerRef, + ) -> Result { + self.attach_worker_protocol_with_scope(worker_ref, Some(scope)) + } + + #[cfg(feature = "ws-server")] + pub fn send_connected_protocol_method_scoped( + &self, + scope: &RuntimeWorkspaceScope, + worker_ref: &WorkerRef, + transport: &crate::execution::WorkerProtocolTransport, + method: Method, + ) -> Result, RuntimeError> { + self.send_connected_protocol_method_with_scope(worker_ref, transport, method, Some(scope)) + } + + #[cfg(feature = "ws-server")] + pub fn send_connected_protocol_method( + &self, + worker_ref: &WorkerRef, + transport: &crate::execution::WorkerProtocolTransport, + method: Method, + ) -> Result, RuntimeError> { + self.send_connected_protocol_method_with_scope(worker_ref, transport, method, None) + } + + #[cfg(feature = "ws-server")] + fn send_connected_protocol_method_with_scope( + &self, + worker_ref: &WorkerRef, + transport: &crate::execution::WorkerProtocolTransport, + method: Method, + scope: Option<&RuntimeWorkspaceScope>, + ) -> Result, RuntimeError> { + // Authorization must address the endpoint's actual target, not a + // different WorkerRef supplied beside a transport captured elsewhere. + if &transport.worker_ref != worker_ref { + return Err(RuntimeError::WorkerNotFound { + worker_id: worker_ref.worker_id, + }); + } + let operation_lock = self.worker_operation_lock(worker_ref.worker_id)?; + let _operation_guard = operation_lock + .lock() + .map_err(|_| RuntimeError::StatePoisoned)?; + if let Some(scope) = scope { + self.ensure_worker_in_workspace(scope, worker_ref)?; + } + self.lock()?.ensure_running()?; + self.ensure_worker_operation_settled(worker_ref)?; + let rejection = |result: WorkerExecutionResult| RuntimeError::WorkerExecutionRejected { + worker_id: worker_ref.worker_id, + operation: result.operation, + outcome: result.outcome, + message: result.message_or_default(), + result, + }; + // Validate the captured endpoint before ANY durable mutation. Even if + // this socket processes terminal closure late, it cannot stop or send + // to the next execution under the same WorkerRef. + transport.validate().map_err(rejection)?; + if matches!(&method, Method::Shutdown { .. }) { + let operation_id = WorkerLifecycleOperationId::new(); + self.begin_stop_operation(worker_ref, operation_id)?; + let events = transport.dispatch(method).map_err(rejection)?; + self.commit_stopped_worker(worker_ref, operation_id)?; + return Ok(events); + } + transport.dispatch(method).map_err(rejection) + } + fn commit_created_worker( &self, worker_ref: &WorkerRef, - handle: WorkerExecutionHandle, initial_worker_state: protocol::WorkerStateSnapshot, workdir_attachments: Vec, result: WorkerExecutionResult, @@ -2543,7 +3448,7 @@ impl Runtime { let mut state = self.lock()?; let detail = { let worker = state.worker_mut(worker_ref)?; - worker.execution_handle = Some(handle); + worker.execution_metadata_available = true; worker.execution_bound = true; worker.status = WorkerStatus::Idle; @@ -2572,15 +3477,14 @@ impl Runtime { &self, backend: &WorkerExecutionBackendRef, worker_ref: &WorkerRef, - handle: &WorkerExecutionHandle, ) -> Result<(), RuntimeError> { - let stop_result = backend.stop_worker(handle); + let stop_result = backend.stop_worker(worker_ref); if stop_result.is_accepted() { return self.rollback_failed_create(worker_ref); } let mut state = self.lock()?; let record = state.worker_mut(worker_ref)?; - record.execution_handle = Some(handle.clone()); + record.worker_state = stop_result.worker_state.clone(); state.persist_runtime_snapshot()?; state.persist_worker(&worker_ref.worker_id)?; @@ -2615,7 +3519,7 @@ impl Runtime { result: WorkerExecutionResult, ) -> Result<(), RuntimeError> { // Accepted dispatch without a state snapshot is transport evidence only; - // the revisioned protocol stream remains live authority. Test/detached + // the Controller protocol stream remains live authority. Test/detached // backends may return an exact full snapshot as their acknowledgement. if !result.is_accepted() { return Ok(()); @@ -2637,27 +3541,24 @@ impl Runtime { worker_ref: &WorkerRef, operation: WorkerExecutionOperation, ) -> Result<(), RuntimeError> { - let Some((backend, handle)) = ({ + let backend = { let state = self.lock()?; state.ensure_worker_ref(worker_ref)?; let worker = state.worker(worker_ref)?; if !worker.status.is_active() { return Ok(()); } - match ( - state.execution_backend.clone(), - worker.execution_handle.clone(), - ) { - (Some(backend), Some(handle)) => Some((backend, handle)), - _ => None, - } - }) else { - return Ok(()); + state.execution_backend.clone().ok_or_else(|| { + RuntimeError::WorkerExecutionUnavailable { + worker_id: worker_ref.worker_id, + message: "runtime has no execution backend".to_string(), + } + })? }; let result = match operation { - WorkerExecutionOperation::Stop => backend.stop_worker(&handle), - WorkerExecutionOperation::Cancel => backend.cancel_worker(&handle), + WorkerExecutionOperation::Stop => backend.stop_worker(worker_ref), + WorkerExecutionOperation::Cancel => backend.cancel_worker(worker_ref), WorkerExecutionOperation::Spawn | WorkerExecutionOperation::Restore | WorkerExecutionOperation::Input @@ -2712,42 +3613,42 @@ impl Runtime { self.ensure_worker_in_workspace(scope, worker_ref)?; } - let (backend, handle, pending_stop, status) = { + let (backend, pending_stop, pending_restore, status) = { let state = self.lock()?; state.ensure_running()?; let worker = state.worker(worker_ref)?; - if worker.pending_restore.is_some() { - return Err(RuntimeError::InvalidRequest(format!( - "worker {} has pending restore reconciliation", - worker_ref.worker_id - ))); - } ( state.execution_backend.clone(), - worker.execution_handle.clone(), worker.pending_stop, + worker.pending_restore, worker.status, ) }; - if status == WorkerStatus::Stopped && pending_stop.is_none() { - let state = self.lock()?; - let worker = state.worker(worker_ref)?; - return Ok(WorkerLifecycleAck { + if status == WorkerStatus::Stopped && pending_stop.is_none() && pending_restore.is_none() { + let mut state = self.lock()?; + let mut worker = state.worker(worker_ref)?.clone(); + worker.restore_guard.rotate(); + worker + .restore_guard + .finish(RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::RolledBack, + "worker_restore_superseded_by_stop", + "Worker Restore was superseded by Stop", + )); + state.persist_worker_record(&worker)?; + let ack = WorkerLifecycleAck { worker_ref: worker_ref.clone(), status: worker.status, worker_state: worker.worker_state.clone(), - }); + }; + state.workers.insert(worker_ref.worker_id, worker); + state.publish_worker_upsert(worker_ref.worker_id)?; + return Ok(ack); } let backend = backend.ok_or_else(|| RuntimeError::WorkerExecutionUnavailable { worker_id: worker_ref.worker_id, message: "runtime has no execution backend".to_string(), })?; - if handle.is_none() && pending_stop.is_none() { - return Err(RuntimeError::WorkerExecutionUnavailable { - worker_id: worker_ref.worker_id, - message: "worker has no execution handle".to_string(), - }); - } let operation_id = match pending_stop { Some(pending) => pending.operation_id, None => { @@ -2759,7 +3660,6 @@ impl Runtime { let result = backend.stop_worker_operation(WorkerExecutionStopRequest { operation_id, worker_ref: worker_ref.clone(), - handle, }); if !result.is_accepted() { return Err(RuntimeError::WorkerExecutionRejected { @@ -2782,8 +3682,7 @@ impl Runtime { worker_ref: &WorkerRef, reason: Option, ) -> Result { - self.ensure_worker_in_workspace(scope, worker_ref)?; - self.cancel_worker(worker_ref, reason) + self.cancel_worker_with_scope(worker_ref, reason, Some(scope)) } /// Cancel the current run while keeping the Worker session available. @@ -2792,6 +3691,22 @@ impl Runtime { worker_ref: &WorkerRef, reason: Option, ) -> Result { + self.cancel_worker_with_scope(worker_ref, reason, None) + } + + fn cancel_worker_with_scope( + &self, + worker_ref: &WorkerRef, + reason: Option, + scope: Option<&RuntimeWorkspaceScope>, + ) -> Result { + let operation_lock = self.worker_operation_lock(worker_ref.worker_id)?; + let _operation_guard = operation_lock + .lock() + .map_err(|_| RuntimeError::StatePoisoned)?; + if let Some(scope) = scope { + self.ensure_worker_in_workspace(scope, worker_ref)?; + } { let state = self.lock()?; state.ensure_running()?; @@ -2850,7 +3765,7 @@ impl Runtime { if let Some(scope) = scope { self.ensure_worker_in_workspace(scope, worker_ref)?; } - let (backend, execution_handle) = { + let backend = { let state = self.lock()?; state.ensure_running()?; state.ensure_worker_ref(worker_ref)?; @@ -2861,27 +3776,16 @@ impl Runtime { worker_ref.worker_id ))); } - if worker.status.is_active() - && (worker.execution_handle.is_some() - || (!worker.execution_metadata_available - && !worker.workdir_attachments.is_empty())) - { + if worker.status.is_active() { return Err(RuntimeError::InvalidRequest(format!( "worker {} is active or requires reconciliation and must be stopped before deletion", worker_ref.worker_id ))); } - ( - state.execution_backend.clone(), - worker.execution_handle.clone(), - ) + state.execution_backend.clone() }; - if let Some(handle) = execution_handle { - let backend = backend.ok_or_else(|| RuntimeError::ExecutionBackendUnavailable { - message: "Worker deletion requires its execution backend to confirm shutdown" - .to_string(), - })?; - let result = backend.stop_worker(&handle); + if let Some(backend) = backend { + let result = backend.stop_worker(worker_ref); if !result.is_accepted() { return Err(RuntimeError::WorkerExecutionRejected { worker_id: worker_ref.worker_id, @@ -2902,10 +3806,7 @@ impl Runtime { worker_ref.worker_id ))); } - if worker.status.is_active() - && (worker.execution_handle.is_some() - || (!worker.execution_metadata_available && !worker.workdir_attachments.is_empty())) - { + if worker.status.is_active() { return Err(RuntimeError::InvalidRequest(format!( "worker {} became active or reconciliation-required before deletion", worker_ref.worker_id @@ -2964,16 +3865,17 @@ impl Runtime { &self, worker_ref: &WorkerRef, ) -> Result { - let (backend, handle) = { + let operation_lock = self.worker_operation_lock(worker_ref.worker_id)?; + let _operation_guard = operation_lock + .lock() + .map_err(|_| RuntimeError::StatePoisoned)?; + let backend = { let state = self.lock()?; - let worker = state.worker(worker_ref)?; - ( - state.execution_backend.clone(), - worker.execution_handle.clone(), - ) + state.worker(worker_ref)?; + state.execution_backend.clone() }; - if let (Some(backend), Some(handle)) = (backend, handle) { - if let Some(snapshot) = backend.worker_snapshot(&handle) { + if let Some(backend) = backend { + if let Some(snapshot) = backend.worker_snapshot(worker_ref) { return Ok(snapshot); } } @@ -3138,6 +4040,26 @@ impl Runtime { #[cfg(feature = "fs-store")] fn restore_persisted_worker_executions(&self) -> Result<(), RuntimeError> { + let candidates = { + let state = self.lock()?; + if state.execution_backend.is_none() { + return Ok(()); + } + state + .workers + .values() + .filter(|worker| { + worker.restore_guard.active_request_id.is_none() + && worker.pending_restore.is_none() + && worker.pending_stop.is_none() + && worker.execution_bound + && worker.status.is_active() + && worker.restore_intent == WorkerRestoreIntent::Automatic + }) + .map(|worker| worker.worker_ref.clone()) + .collect::>() + }; + let pending_stops = { let state = self.lock()?; if state.execution_backend.is_none() { @@ -3164,74 +4086,83 @@ impl Runtime { } } - let pending_restores = { - let state = self.lock()?; - state - .workers - .values() - .filter_map(|worker| { - worker - .pending_restore - .map(|pending| (worker.worker_ref.clone(), pending.mode)) - }) - .collect::>() - }; + let pending_restores = + { + let state = self.lock()?; + state + .workers + .values() + .filter(|worker| worker.pending_stop.is_none()) + .filter_map(|worker| { + worker + .pending_restore + .map(|pending| (worker.worker_ref.clone(), pending.mode)) + .or_else(|| { + worker.restore_guard.active_request_id.as_ref().map(|_| { + (worker.worker_ref.clone(), WorkerRestoreMode::Explicit) + }) + }) + }) + .collect::>() + }; for (worker_ref, mode) in pending_restores { let operation_lock = self.worker_operation_lock(worker_ref.worker_id)?; let _operation_guard = operation_lock .lock() .map_err(|_| RuntimeError::StatePoisoned)?; - if let Err(error) = self.restore_worker_under_lock(&worker_ref, mode) { - tracing::warn!( - worker_id = %worker_ref.worker_id, - error = %error, - "Worker restore reconciliation remains pending after Runtime startup" - ); - } + let result = self.restore_admitted_worker_under_lock(&worker_ref, mode)?; + self.record_startup_restore_result(&worker_ref, &result)?; } - let candidates = { - let state = self.lock()?; - if state.execution_backend.is_none() { - return Ok(()); - } - state - .workers - .values() - .filter(|worker| { - worker.execution_handle.is_none() - && worker.pending_restore.is_none() - && worker.pending_stop.is_none() - && worker.execution_bound - && worker.status.is_active() - && worker.restore_intent == WorkerRestoreIntent::Automatic - }) - .map(|worker| worker.worker_ref.clone()) - .collect::>() - }; - for worker_ref in candidates { let operation_lock = self.worker_operation_lock(worker_ref.worker_id)?; let _operation_guard = operation_lock .lock() .map_err(|_| RuntimeError::StatePoisoned)?; - match self.restore_worker_under_lock(&worker_ref, WorkerRestoreMode::Automatic) { - Ok(_) => {} - Err(RuntimeError::WorkerExecutionRejected { .. }) => { - // The failed restore and fail-closed terminal status were - // persisted by restore_worker_under_lock. - } - Err(error) => return Err(error), - } + let result = + self.restore_admitted_worker_under_lock(&worker_ref, WorkerRestoreMode::Automatic)?; + self.record_startup_restore_result(&worker_ref, &result)?; } Ok(()) } + #[cfg(feature = "fs-store")] + fn record_startup_restore_result( + &self, + worker_ref: &WorkerRef, + result: &RuntimeWorkerRestoreResult, + ) -> Result<(), RuntimeError> { + match result.state { + WorkerRestoreState::Accepted => return Ok(()), + WorkerRestoreState::Rejected + | WorkerRestoreState::RolledBack + | WorkerRestoreState::ReconciliationRequired => {} + } + let mut state = self.lock()?; + let diagnostic_id = state.next_diagnostic_id; + state.next_diagnostic_id = state.next_diagnostic_id.saturating_add(1); + let message = sanitize_worker_create_failure_message( + result + .message + .as_deref() + .unwrap_or("Worker restore did not commit"), + ); + tracing::warn!(worker_id = %worker_ref.worker_id, restore_state = ?result.state, reason = ?result.reason_code, %message, "Worker startup restore did not commit"); + state.diagnostics.push(RuntimeDiagnostic { + id: diagnostic_id, + severity: DiagnosticSeverity::Warning, + code: "worker_execution_restore_failed".to_string(), + message, + worker_ref: Some(worker_ref.clone()), + }); + state.persist_runtime_snapshot() + } + fn reconcile_pending_stop_under_lock( &self, worker_ref: &WorkerRef, ) -> Result<(), RuntimeError> { - let (backend, handle, operation_id) = { + let (backend, operation_id) = { let state = self.lock()?; let worker = state.worker(worker_ref)?; let pending = worker.pending_stop.ok_or_else(|| { @@ -3246,16 +4177,11 @@ impl Runtime { message: "runtime has no execution backend".to_string(), } })?; - ( - backend, - worker.execution_handle.clone(), - pending.operation_id, - ) + (backend, pending.operation_id) }; let result = backend.stop_worker_operation(WorkerExecutionStopRequest { operation_id, worker_ref: worker_ref.clone(), - handle, }); if !result.is_accepted() { return Err(RuntimeError::WorkerExecutionRejected { @@ -3277,18 +4203,19 @@ impl Runtime { ) -> Result<(), RuntimeError> { let mut state = self.lock()?; let mut candidate = state.worker(worker_ref)?.clone(); - if candidate.has_pending_lifecycle_operation() { + if candidate.pending_stop.is_some() { return Err(RuntimeError::InvalidRequest(format!( - "worker {} already has pending lifecycle reconciliation", + "worker {} already has pending stop reconciliation", worker_ref.worker_id ))); } - if !candidate.status.is_active() || candidate.request.is_none() { + if !candidate.status.is_active() && candidate.pending_restore.is_none() { return Err(RuntimeError::InvalidRequest(format!( "worker {} cannot begin a stop operation from its current state", worker_ref.worker_id ))); } + candidate.restore_guard.rotate(); candidate.pending_stop = Some(PendingWorkerStop { operation_id, last_settled_status: candidate.status, @@ -3323,10 +4250,22 @@ impl Runtime { ))); } candidate.pending_stop = None; + candidate.pending_restore = None; + if let Some(context) = candidate.restore_candidate_context.take() { + context.discard_candidate(); + } + candidate.restore_guard.rotate(); candidate.status = WorkerStatus::Stopped; candidate.worker_state = None; candidate.restore_intent = WorkerRestoreIntent::Explicit; - candidate.execution_handle = None; + + candidate + .restore_guard + .finish(RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::RolledBack, + "worker_restore_superseded_by_stop", + "Worker Restore was superseded by Stop", + )); candidate.internal_workers.clear(); state.persist_worker_record(&candidate)?; let ack = WorkerLifecycleAck { @@ -3356,7 +4295,11 @@ impl Runtime { let mut state = self.lock()?; state.ensure_worker_ref(worker_ref)?; let mut candidate = state.worker(worker_ref)?.clone(); - if candidate.has_pending_lifecycle_operation() { + if candidate.pending_stop.is_some() + || candidate + .pending_restore + .is_some_and(|pending| pending.operation_id != operation_id) + { return Err(RuntimeError::InvalidRequest(format!( "worker {} already has pending lifecycle reconciliation", worker_ref.worker_id @@ -3368,6 +4311,18 @@ impl Runtime { worker_ref.worker_id ))); } + if let Some(id) = &candidate.restore_guard.active_request_id { + let owner = candidate.restore_guard.owners.get_mut(id).ok_or_else(|| { + RuntimeError::InvalidRequest("Active Restore request has no persisted owner".into()) + })?; + if owner.operation_id != operation_id || owner.receipt.is_some() { + return Err(RuntimeError::RestoreObservationConflict { + worker_id: worker_ref.worker_id, + }); + } + owner.execution_started = true; + } + candidate.restore_guard.rotate(); candidate.pending_restore = Some(PendingWorkerRestore { operation_id, mode, @@ -3387,9 +4342,15 @@ impl Runtime { } } - fn settle_restore_rollback(&self, worker_ref: &WorkerRef) -> Result<(), RuntimeError> { + fn settle_restore_rollback( + &self, + worker_ref: &WorkerRef, + receipt: &RuntimeWorkerRestoreResult, + ) -> Result<(), RuntimeError> { let mut state = self.lock()?; let mut candidate = state.worker(worker_ref)?.clone(); + candidate.restore_guard.rotate(); + candidate.restore_guard.finish(receipt.clone()); candidate.pending_restore = None; candidate.restore_candidate_context = None; state.persist_worker_record(&candidate)?; @@ -3400,7 +4361,7 @@ impl Runtime { fn commit_restored_worker_execution( &self, worker_ref: &WorkerRef, - handle: WorkerExecutionHandle, + worker_state: protocol::WorkerStateSnapshot, status: WorkerStatus, workdir_attachments: Vec, @@ -3408,18 +4369,12 @@ impl Runtime { let mut state = self.lock()?; state.ensure_worker_ref(worker_ref)?; let mut candidate = state.worker(worker_ref)?.clone(); - if candidate.execution_handle.is_some() { - return Err(RuntimeError::InvalidRequest(format!( - "worker {} already has a current execution", - worker_ref.worker_id - ))); - } - candidate.execution_handle = Some(handle); + candidate.restore_guard.rotate(); candidate.execution_metadata_available = true; candidate.execution_bound = true; candidate.pending_restore = None; candidate.pending_stop = None; - candidate.restore_candidate_context = None; + // Keep the actual publication owner until buffered observations activate. candidate.status = status; let mut worker_state = worker_state; if worker_state.last_finished_submission_request_id.is_none() { @@ -3438,6 +4393,9 @@ impl Runtime { // Validate the exact subscription projection before persistence. The // subsequent publish uses the same pure projection while the Worker // operation lock keeps this candidate stable. + candidate.restore_guard.rotate(); + let accepted = RuntimeWorkerRestoreResult::accepted(candidate.detail()); + candidate.restore_guard.finish(accepted); let _subscription_projection = state.subscription_worker(&candidate)?; // Persist the complete candidate before making it observable. A failed @@ -3460,13 +4418,14 @@ impl Runtime { fn retain_create_execution_evidence( &self, worker_ref: &WorkerRef, - handle: WorkerExecutionHandle, + worker_state: protocol::WorkerStateSnapshot, workdir_attachments: Vec, ) -> Result<(), RuntimeError> { let mut state = self.lock()?; let mut candidate = state.worker(worker_ref)?.clone(); - candidate.execution_handle = Some(handle); + + candidate.restore_guard.rotate(); candidate.execution_metadata_available = true; candidate.execution_bound = true; candidate.status = WorkerStatus::Idle; @@ -3482,7 +4441,7 @@ impl Runtime { fn retain_restore_execution_evidence( &self, worker_ref: &WorkerRef, - handle: WorkerExecutionHandle, + _worker_state: protocol::WorkerStateSnapshot, workdir_attachments: Vec, candidate_context: crate::execution::WorkerExecutionContext, @@ -3495,7 +4454,8 @@ impl Runtime { worker_ref.worker_id ))); } - candidate.execution_handle = Some(handle); + + candidate.restore_guard.rotate(); candidate.execution_metadata_available = true; candidate.execution_bound = true; candidate.restore_candidate_context = Some(candidate_context); @@ -3512,7 +4472,8 @@ impl Runtime { ) -> Result<(), RuntimeError> { let mut state = self.lock()?; let mut candidate = state.worker(worker_ref)?.clone(); - candidate.execution_handle = None; + + candidate.restore_guard.rotate(); candidate.execution_metadata_available = false; candidate.execution_bound = true; candidate.status = WorkerStatus::Idle; @@ -3553,9 +4514,8 @@ impl Runtime { &self, backend: &WorkerExecutionBackendRef, worker_ref: &WorkerRef, - handle: &WorkerExecutionHandle, ) -> Result<(), RuntimeError> { - let result = backend.stop_worker(handle); + let result = backend.stop_worker(worker_ref); if !result.is_accepted() { return Err(RuntimeError::WorkerExecutionRejected { worker_id: worker_ref.worker_id, @@ -4005,11 +4965,17 @@ impl RuntimeState { ) } PersistedWorkerLifecycleOperation::Stop(operation) => ( - Some(operation.request), - true, + operation.request.clone(), + operation.request.is_some(), operation.binding.is_some(), operation.restore_intent, - None, + operation + .pending_restore + .map(|restore| PendingWorkerRestore { + operation_id: restore.operation_id, + mode: restore.mode.into(), + last_settled_status: restore.last_settled_status, + }), Some(PendingWorkerStop { operation_id: operation.operation_id, last_settled_status: operation.last_settled_status, @@ -4037,6 +5003,15 @@ impl RuntimeState { workers.insert( worker_id, WorkerRecord { + restore_guard: { + let mut guard = worker.restore_guard; + // A prior active controller is not live in this process. + // Stopped observations remain stable across restart. + if worker.status.is_active() && execution_bound { + guard.rotate(); + } + guard + }, worker_ref: worker.worker_ref, worker_id: worker.worker_id, status: worker.status, @@ -4058,7 +5033,6 @@ impl RuntimeState { restore_candidate_context: None, workdir_attachments: worker.workdir_attachments, logical_workdir_attachments, - execution_handle: None, internal_workers: InternalWorkerActivityProjection::default(), }, ); @@ -4394,6 +5368,7 @@ impl RuntimeState { ProfileSelector::Builtin(name) | ProfileSelector::Named(name) => Some(name.clone()), }; Ok(SubscriptionWorker { + restore_observation_token: Some(worker.restore_guard.generation.clone()), worker_id, runtime_id: None, resource_key: None, @@ -4538,12 +5513,10 @@ impl RuntimeState { fn record_restore_failure( &mut self, worker_ref: &WorkerRef, - result: WorkerExecutionResult, + result: &WorkerExecutionResult, + receipt: &RuntimeWorkerRestoreResult, ) -> Result<(), RuntimeError> { - let message = result - .message - .clone() - .unwrap_or_else(|| "worker execution restore failed".to_string()); + let message = sanitize_worker_create_failure_message(&result.message_or_default()); let diagnostic_id = self.next_diagnostic_id; let workspace_id = self .workers @@ -4551,11 +5524,13 @@ impl RuntimeState { .and_then(|worker| worker.workspace_id.clone()) .unwrap_or_else(|| "".to_string()); let mut candidate = self.worker(worker_ref)?.clone(); - candidate.execution_handle = None; + candidate.execution_bound = false; + candidate.restore_guard.finish(receipt.clone()); candidate.pending_restore = None; candidate.pending_stop = None; candidate.restore_candidate_context = None; + candidate.restore_guard.rotate(); candidate.status = WorkerStatus::Stopped; candidate.restore_intent = WorkerRestoreIntent::Explicit; candidate.internal_workers.clear(); @@ -4934,8 +5909,52 @@ struct PendingWorkerStop { last_settled_status: WorkerStatus, } +/// Durable Restore admission/replay metadata. Never discard receipts on stop. +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +pub(crate) struct RestoreRequestOwner { + fingerprint: String, + operation_id: WorkerLifecycleOperationId, + request: runtime_api::WorkerRestoreRequest, + preparation_applied: bool, + #[serde(default)] + execution_started: bool, + receipt: Option, +} + +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +pub(crate) struct RestoreGuard { + generation: String, + active_request_id: Option, + owners: BTreeMap, +} + +impl Default for RestoreGuard { + fn default() -> Self { + Self { + generation: uuid::Uuid::now_v7().to_string(), + active_request_id: None, + owners: BTreeMap::new(), + } + } +} + +impl RestoreGuard { + fn rotate(&mut self) { + self.generation = uuid::Uuid::now_v7().to_string(); + } + + fn finish(&mut self, result: RuntimeWorkerRestoreResult) { + if let Some(id) = self.active_request_id.take() + && let Some(owner) = self.owners.get_mut(&id) + { + owner.receipt = Some(result); + } + } +} + #[derive(Debug, Clone)] struct WorkerRecord { + restore_guard: RestoreGuard, worker_ref: WorkerRef, worker_id: WorkerId, status: WorkerStatus, @@ -4957,12 +5976,14 @@ struct WorkerRecord { restore_candidate_context: Option, workdir_attachments: Vec, logical_workdir_attachments: Vec, - execution_handle: Option, internal_workers: InternalWorkerActivityProjection, } impl WorkerRecord { fn apply_worker_state(&mut self, incoming: &protocol::WorkerStateSnapshot) { + if self.worker_state.as_ref().map(|state| &state.state) != Some(&incoming.state) { + self.restore_guard.rotate(); + } if let Some(request_id) = &incoming.last_finished_submission_request_id { self.last_finished_submission_request_id = Some(request_id.clone()); } @@ -4989,6 +6010,7 @@ impl WorkerRecord { fn summary(&self) -> WorkerSummary { WorkerSummary { + restore_observation_token: Some(self.restore_guard.generation.clone()), worker_ref: self.worker_ref.clone(), worker_id: self.worker_id, status: self.status, @@ -5006,6 +6028,7 @@ impl WorkerRecord { fn detail(&self) -> WorkerDetail { WorkerDetail { + restore_observation_token: Some(self.restore_guard.generation.clone()), worker_ref: self.worker_ref.clone(), worker_id: self.worker_id, status: self.status, @@ -5024,16 +6047,26 @@ impl WorkerRecord { #[cfg(feature = "fs-store")] fn persisted_record(&self) -> PersistedWorkerRecord { - debug_assert!(self.pending_restore.is_none() || self.pending_stop.is_none()); - let execution_state = if let (Some(pending), Some(request)) = - (self.pending_stop, self.request.clone()) - { + let execution_state = if let Some(pending) = self.pending_stop { PersistedWorkerExecutionState::ReconciliationRequired( PersistedWorkerLifecycleOperation::Stop(PersistedWorkerStopOperation { operation_id: pending.operation_id, - request, - binding: self - .execution_bound + pending_restore: self.pending_restore.map(|restore| { + PersistedWorkerRestoreOperation { + operation_id: restore.operation_id, + mode: restore.mode.into(), + request: self + .request + .clone() + .expect("pending restore retains its operation specification"), + binding: self + .execution_bound + .then_some(PersistedWorkerExecutionBinding {}), + last_settled_status: restore.last_settled_status, + } + }), + request: self.request.clone(), + binding: (self.execution_bound && self.request.is_some()) .then_some(PersistedWorkerExecutionBinding {}), restore_intent: self.restore_intent, last_settled_status: pending.last_settled_status, @@ -5067,6 +6100,7 @@ impl WorkerRecord { } }; PersistedWorkerRecord { + restore_guard: self.restore_guard.clone(), worker_ref: self.worker_ref.clone(), worker_id: self.worker_id, profile: self.profile.clone(), @@ -5212,6 +6246,16 @@ fn validate_logical_workdir_attachments( } fn validate_create_worker_request(request: &CreateWorkerRequest) -> Result<(), RuntimeError> { + if let Some(binding) = &request.backend_job { + binding + .validate() + .map_err(|message| RuntimeError::InvalidRequest(message.into()))?; + if request.workspace_api.is_none() + || (binding.subjektiv_consolidation && !request.subjektiv_attached) + { + return Err(RuntimeError::InvalidRequest("Backend Job capability requires Workspace authority and consolidation requires trusted subject attachment".into())); + } + } if request.create_fingerprint.trim().is_empty() { return Err(RuntimeError::InvalidRequest( "create_fingerprint must not be empty".to_string(), @@ -5309,6 +6353,9 @@ fn runtime_worker_create_failure_fields( error: &RuntimeError, ) -> (&'static str, Option, Option) { match error { + RuntimeError::RestoreObservationConflict { .. } => { + ("restore_observation_conflict", None, None) + } RuntimeError::RuntimeStopped => ("runtime_stopped", None, None), RuntimeError::RuntimeStoreAlreadyOpen { .. } => ("runtime_store_already_open", None, None), RuntimeError::InvalidInitialInputKind { .. } => ("invalid_initial_input_kind", None, None), @@ -5490,7 +6537,29 @@ fn validate_create_workspace_scope( Ok(()) } -fn validate_worker_input(input: &WorkerInput) -> Result<(), RuntimeError> { +pub(crate) fn validate_worker_input(input: &WorkerInput) -> Result<(), RuntimeError> { + if input.segments.as_ref().is_some_and(|segments| { + segments + .iter() + .any(|segment| matches!(segment, protocol::Segment::Unknown)) + }) { + return Err(RuntimeError::InvalidRequest( + "unsupported typed input must not be discarded; use a compatible client and host" + .into(), + )); + } + if !matches!( + input.kind, + WorkerInputKind::User | WorkerInputKind::UserIfIdle + ) && input.segments.as_ref().is_some_and(|segments| { + segments + .iter() + .any(|segment| !matches!(segment, protocol::Segment::Text { .. })) + }) { + return Err(RuntimeError::InvalidRequest( + "Notify and control inputs cannot carry typed submission intent; use Submit".into(), + )); + } let has_segments = input .segments .as_ref() @@ -5559,8 +6628,7 @@ mod tests { ConfigDeclarationKind, ConfigProfileDescriptor, }; use crate::execution::{ - WorkerExecutionBackend, WorkerExecutionContext, WorkerExecutionHandle, - WorkerExecutionRestoreRequest, + WorkerExecutionBackend, WorkerExecutionContext, WorkerExecutionRestoreRequest, }; use crate::working_directory::WorkingDirectoryDiagnostic; use async_trait::async_trait; @@ -6058,9 +7126,95 @@ mod tests { language: "English".to_string(), }), subjektiv_attached: false, + backend_job: None, } } + #[cfg(feature = "fs-store")] + #[test] + fn backend_job_binding_survives_restart_and_restore_without_profile_inference() { + let dir = tempfile::tempdir().unwrap(); + let options = crate::fs_store::FsRuntimeStoreOptions { + root: dir.path().join("runtime"), + runtime_id: "job-runtime".into(), + display_name: None, + }; + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = + Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()).unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let mut request = task_request("bound Job"); + request.workspace_api = Some(WorkspaceApiRef { + workspace_id: "local".into(), + base_url: "https://backend.invalid".into(), + }); + request.backend_job = Some(crate::catalog::BackendJobExecutionBinding { + job_id: "job-1".into(), + attempt_id: "attempt-1".into(), + input_revision: Some("revision-1".into()), + subjektiv_consolidation: false, + }); + let owner = scope("local", "server-job-test"); + let created = runtime + .create_worker_scoped(&owner, request.clone()) + .unwrap(); + assert_eq!( + backend.job_bindings.lock().unwrap().as_slice(), + &[request.backend_job.clone()] + ); + let mut changed = request.clone(); + changed.backend_job.as_mut().unwrap().attempt_id = "attempt-other".into(); + assert!(runtime.create_worker_scoped(&owner, changed).is_err()); + drop(runtime); + let restored_backend = Arc::new(TestExecutionBackend::default()); + let restored = + Runtime::with_fs_store_and_execution_backend(options, restored_backend.clone()) + .unwrap(); + assert_eq!( + restored + .lock() + .unwrap() + .worker(&created.worker_ref) + .unwrap() + .request + .as_ref() + .unwrap() + .backend_job, + request.backend_job + ); + restored + .restore_worker( + &created.worker_ref, + restored.test_restore_request(&created.worker_ref), + ) + .unwrap(); + assert_eq!( + restored_backend.job_bindings.lock().unwrap().as_slice(), + &[request.backend_job] + ); + } + + #[test] + fn backend_job_binding_requires_workspace_and_consolidation_attachment() { + let mut request = task_request("Job binding validation"); + request.backend_job = Some(crate::catalog::BackendJobExecutionBinding { + job_id: "job-1".into(), + attempt_id: "attempt-1".into(), + input_revision: None, + subjektiv_consolidation: true, + }); + assert!(validate_create_worker_request(&request).is_err()); + request.workspace_api = Some(WorkspaceApiRef { + workspace_id: "local".into(), + base_url: "https://backend.invalid".into(), + }); + assert!(validate_create_worker_request(&request).is_err()); + request.subjektiv_attached = true; + assert!(validate_create_worker_request(&request).is_ok()); + request.backend_job.as_mut().unwrap().attempt_id.clear(); + assert!(validate_create_worker_request(&request).is_err()); + } + #[test] fn durable_worker_request_omits_repository_credentials() { let mut request = task_request("worker-secret-redaction"); @@ -6483,6 +7637,7 @@ mod tests { #[derive(Default)] struct TestExecutionBackend { spawn_result: Mutex>, + job_bindings: Mutex>>, dispatch_result: Mutex>, stop_result: Mutex>, stop_gate: Mutex>>, @@ -6500,6 +7655,13 @@ mod tests { workspace_config_results: Mutex>, contexts: Mutex>, dispatched_inputs: Mutex>, + completion_queries: Mutex< + Vec<( + protocol::CompletionKind, + String, + Option, + )>, + >, repository_accesses: Mutex>, repository_access_available: AtomicBool, working_directory_requests: Mutex>, @@ -6592,6 +7754,10 @@ mod tests { } fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult { + self.job_bindings + .lock() + .unwrap() + .push(request.request.backend_job.clone()); if let Some(result) = self.spawn_result.lock().unwrap().take() { return result; } @@ -6604,7 +7770,6 @@ mod tests { .unwrap() .insert(request.worker_ref.worker_id.clone(), request.context); WorkerExecutionSpawnResult::Connected { - handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()), worker_state: protocol::WorkerStateSnapshot { ..protocol::WorkerStatus::Idle.into() }, @@ -6621,10 +7786,41 @@ mod tests { } } + fn worker_session( + &self, + request: WorkerSessionObservationRequest, + ) -> runtime_api::WorkerSessionAvailability { + if self + .contexts + .lock() + .unwrap() + .contains_key(&request.worker_ref.worker_id) + { + runtime_api::WorkerSessionAvailability::LiveProtocol + } else { + runtime_api::WorkerSessionAvailability::Unavailable { + reason: runtime_api::WorkerSessionUnavailableReason::StorageUnavailable, + message: "test backend has no retained session".to_string(), + } + } + } + fn preflight_restore( &self, - _request: &WorkerExecutionRestoreRequest, + request: &WorkerExecutionRestoreRequest, ) -> Result<(), WorkerExecutionResult> { + if self + .contexts + .lock() + .unwrap() + .contains_key(&request.worker_ref.worker_id) + { + return Err(WorkerExecutionResult::busy( + WorkerExecutionOperation::Restore, + "Worker is already connected", + ) + .with_worker_state(protocol::WorkerStatus::Idle.into())); + } match self.preflight_restore_result.lock().unwrap().clone() { Some(result) => Err(result), None => Ok(()), @@ -6635,6 +7831,10 @@ mod tests { &self, request: WorkerExecutionRestoreRequest, ) -> WorkerExecutionSpawnResult { + self.job_bindings + .lock() + .unwrap() + .push(request.request.backend_job.clone()); *self.restore_count.lock().unwrap() += 1; self.restore_operation_ids .lock() @@ -6660,7 +7860,6 @@ mod tests { .unwrap() .insert(request.worker_ref.worker_id.clone(), request.context); WorkerExecutionSpawnResult::Connected { - handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()), worker_state: protocol::WorkerStateSnapshot { ..protocol::WorkerStatus::Idle.into() }, @@ -6677,11 +7876,39 @@ mod tests { } } + fn worker_completions( + &self, + _worker_ref: &WorkerRef, + kind: protocol::CompletionKind, + prefix: &str, + context: Option<&protocol::CompletionContext>, + ) -> Vec { + self.completion_queries + .lock() + .unwrap() + .push((kind, prefix.into(), context.cloned())); + vec![protocol::CompletionEntry { + value: "資料/a b".into(), + ..Default::default() + }] + } + fn dispatch_input( &self, - _handle: &WorkerExecutionHandle, + worker_ref: &WorkerRef, input: WorkerInput, ) -> WorkerExecutionResult { + if !self + .contexts + .lock() + .unwrap() + .contains_key(&worker_ref.worker_id) + { + return WorkerExecutionResult::rejected( + WorkerExecutionOperation::Input, + "WorkerRef does not reference a live Worker execution", + ); + } let request_id = input.submission_request_id.clone(); let input_kind = input.kind.clone(); self.dispatched_inputs.lock().unwrap().push(input); @@ -6725,35 +7952,35 @@ mod tests { .lock() .unwrap() .push(request.operation_id); - let handle = request.handle.unwrap_or_else(|| { - WorkerExecutionHandle::new(request.worker_ref, self.backend_id()) - }); - self.stop_worker(&handle) + self.stop_worker(&request.worker_ref) } - fn stop_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult { + fn stop_worker(&self, worker_ref: &WorkerRef) -> WorkerExecutionResult { *self.stop_count.lock().unwrap() += 1; let stop_gate = self.stop_gate.lock().unwrap().clone(); if let Some(gate) = stop_gate { gate.enter_and_wait(); } - self.stop_result - .lock() - .unwrap() - .take() - .unwrap_or_else(|| WorkerExecutionResult::accepted(WorkerExecutionOperation::Stop)) + let result = + self.stop_result.lock().unwrap().take().unwrap_or_else(|| { + WorkerExecutionResult::accepted(WorkerExecutionOperation::Stop) + }); + if result.is_accepted() { + self.contexts.lock().unwrap().remove(&worker_ref.worker_id); + } + result } - fn cancel_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult { + fn cancel_worker(&self, _worker_ref: &WorkerRef) -> WorkerExecutionResult { WorkerExecutionResult::accepted(WorkerExecutionOperation::Cancel) } #[cfg(feature = "ws-server")] - fn worker_snapshot(&self, handle: &WorkerExecutionHandle) -> Option { + fn worker_snapshot(&self, worker_ref: &WorkerRef) -> Option { self.snapshots .lock() .unwrap() - .get(&handle.worker_ref().worker_id) + .get(&worker_ref.worker_id) .cloned() } } @@ -6816,6 +8043,108 @@ mod tests { .block_on(subscription.recv()) } + #[test] + fn unknown_typed_intent_is_rejected_before_backend_dispatch() { + let (runtime, backend) = runtime_and_backend(); + let worker = runtime + .create_worker(task_request("unsupported input")) + .unwrap(); + let mut input = WorkerInput::user("normal prose"); + input.segments = Some(vec![ + protocol::Segment::text("normal prose"), + protocol::Segment::Unknown, + ]); + assert!(matches!( + runtime.send_input(&worker.worker_ref, input), + Err(RuntimeError::InvalidRequest(_)) + )); + assert!(backend.dispatched_inputs.lock().unwrap().is_empty()); + } + + #[test] + fn notify_rejects_typed_feature_intent_before_backend_dispatch() { + let (runtime, backend) = runtime_and_backend(); + let worker = runtime.create_worker(task_request("notification")).unwrap(); + let invoke = protocol::Segment::FeatureInvoke { + invocation: protocol::FeatureInvocation { + invocation_id: "not-executed".into(), + identity: protocol::FeatureInvocationIdentity("builtin:test/run".into()), + name: "run".into(), + arguments: Vec::new(), + }, + }; + let mut input = WorkerInput::user("/run()"); + input.kind = WorkerInputKind::Notify; + input.segments = Some(vec![invoke]); + assert!(matches!( + runtime.send_input(&worker.worker_ref, input), + Err(RuntimeError::InvalidRequest(_)) + )); + assert!(backend.dispatched_inputs.lock().unwrap().is_empty()); + let mut text = WorkerInput::user("/run() advisory text"); + text.kind = WorkerInputKind::Notify; + assert!( + runtime + .send_input(&worker.worker_ref, text) + .unwrap() + .notification + .is_some() + ); + assert_eq!(backend.dispatched_inputs.lock().unwrap().len(), 1); + } + + #[test] + fn feature_argument_completions_preserve_context_and_workspace_authority() { + let (runtime, backend) = runtime_and_backend(); + let owner = scope("workspace-a", "server-a"); + let worker = runtime + .create_worker_scoped(&owner, scoped_task_request("completion", "workspace-a")) + .unwrap(); + let context = protocol::CompletionContext { + invocation: protocol::FeatureInvocationIdentity("builtin:test/prepare".into()), + argument: Some("path".into()), + }; + let events = runtime + .send_protocol_method_scoped( + &owner, + &worker.worker_ref, + Method::ListCompletions { + kind: protocol::CompletionKind::FeatureArgument, + prefix: "資料/".into(), + request_id: Some("query-current".into()), + context: Some(context.clone()), + }, + ) + .unwrap(); + assert!( + matches!(&events[0], Event::Completions { kind: protocol::CompletionKind::FeatureArgument, prefix, request_id: Some(request_id), context: Some(restored), entries } + if request_id == "query-current" && prefix == "資料/" && restored == &context && entries[0].value == "資料/a b") + ); + assert_eq!( + *backend.completion_queries.lock().unwrap(), + vec![( + protocol::CompletionKind::FeatureArgument, + "資料/".into(), + Some(context.clone()) + )] + ); + let outsider = scope("workspace-b", "server-b"); + assert!( + runtime + .worker_completions_scoped( + &outsider, + &worker.worker_ref, + protocol::CompletionKind::FeatureArgument, + "", + Some(&context) + ) + .is_err() + ); + assert_eq!(backend.completion_queries.lock().unwrap().len(), 1); + assert!(backend.dispatched_inputs.lock().unwrap().is_empty()); + } + + #[cfg(feature = "ws-server")] #[test] fn runtime_worker_subscription_has_gap_free_snapshot_and_live_updates() { let runtime = runtime_with_backend(); @@ -7261,7 +8590,12 @@ mod tests { vec![attachment.clone()], ) .unwrap(); - runtime.restore_worker(&worker.worker_ref).unwrap(); + runtime + .restore_worker( + &worker.worker_ref, + runtime.test_restore_request(&worker.worker_ref), + ) + .unwrap(); let state = runtime.lock().unwrap(); assert_eq!( state @@ -7459,7 +8793,12 @@ mod tests { ); runtime.stop_worker(&detail.worker_ref, None).unwrap(); - runtime.restore_worker(&detail.worker_ref).unwrap(); + runtime + .restore_worker( + &detail.worker_ref, + runtime.test_restore_request(&detail.worker_ref), + ) + .unwrap(); assert_eq!( backend.config_bundles.lock().unwrap().as_slice(), &[Some(bundle), None] @@ -7519,7 +8858,12 @@ mod tests { .unwrap(); runtime.stop_worker(&detail.worker_ref, None).unwrap(); runtime.lock().unwrap().config_bundles.clear(); - runtime.restore_worker(&detail.worker_ref).unwrap(); + runtime + .restore_worker( + &detail.worker_ref, + runtime.test_restore_request(&detail.worker_ref), + ) + .unwrap(); assert_eq!( backend.config_bundles.lock().unwrap().as_slice(), &[Some(bundle), None] @@ -7539,7 +8883,12 @@ mod tests { .unwrap() .config_bundles .insert(replacement.metadata.id.clone(), replacement); - runtime.restore_worker(&detail.worker_ref).unwrap(); + runtime + .restore_worker( + &detail.worker_ref, + runtime.test_restore_request(&detail.worker_ref), + ) + .unwrap(); assert_eq!( backend.config_bundles.lock().unwrap().as_slice(), &[Some(bundle), None] @@ -7656,7 +9005,12 @@ mod tests { let restore_runtime = runtime.clone(); let restore_ref = created.worker_ref.clone(); - let restoring = std::thread::spawn(move || restore_runtime.restore_worker(&restore_ref)); + let restoring = std::thread::spawn(move || { + restore_runtime.restore_worker( + &restore_ref, + restore_runtime.test_restore_request(&restore_ref), + ) + }); assert!(gate.wait_for_entered(1, std::time::Duration::from_secs(2))); let remove_runtime = runtime.clone(); let remove_ref = created.worker_ref.clone(); @@ -7757,6 +9111,62 @@ mod tests { ); } + #[cfg(feature = "fs-store")] + #[test] + fn fs_store_metadata_free_stop_retains_operation_across_unknown_commit_restart() { + let dir = tempfile::tempdir().unwrap(); + let options = crate::fs_store::FsRuntimeStoreOptions { + root: dir.path().join("runtime"), + runtime_id: "test-runtime".into(), + display_name: None, + }; + let runtime = Runtime::with_fs_store_and_execution_backend( + options.clone(), + Arc::new(TestExecutionBackend::default()), + ) + .unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("metadata-free stop")) + .unwrap(); + { + let mut state = runtime.lock().unwrap(); + let record = state.worker_mut(&worker.worker_ref).unwrap(); + record.request = None; + record.execution_metadata_available = false; + record.execution_bound = false; + state.persist_worker(&worker.worker_id).unwrap(); + } + runtime_store(&runtime) + .fail_next_worker_write(crate::fs_store::AtomicWriteFault::AfterRename); + assert!(matches!( + runtime.stop_worker(&worker.worker_ref, None), + Err(RuntimeError::StoreCommitOutcomeUnknown { .. }) + )); + let operation_id = runtime + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .pending_stop + .unwrap() + .operation_id; + drop(runtime); + let backend = Arc::new(TestExecutionBackend::default()); + let reopened = + Runtime::with_fs_store_and_execution_backend(options, backend.clone()).unwrap(); + let stopped = reopened.worker_detail(&worker.worker_ref).unwrap(); + assert_eq!(stopped.status, WorkerStatus::Stopped); + assert!(!stopped.execution_metadata_available); + assert!(stopped.worker_state.is_none()); + assert_eq!(*backend.restore_count.lock().unwrap(), 0); + assert_eq!( + backend.stop_operation_ids.lock().unwrap().as_slice(), + &[operation_id] + ); + assert!(reopened.delete_worker(&worker.worker_ref).unwrap().deleted); + } + #[cfg(feature = "fs-store")] #[test] fn fs_store_reconciles_pending_stop_before_automatic_restore() { @@ -7878,7 +9288,9 @@ mod tests { let restore_runtime = runtime.clone(); let restore_ref = created.worker_ref.clone(); - let restoring = std::thread::spawn(move || restore_runtime.restore_worker(&restore_ref)); + let request = runtime.test_restore_request(&restore_ref); + let restoring = + std::thread::spawn(move || restore_runtime.restore_worker(&restore_ref, request)); std::thread::sleep(std::time::Duration::from_millis(100)); assert_eq!( *backend.restore_count.lock().unwrap(), @@ -7888,8 +9300,18 @@ mod tests { gate.release(); stopping.join().unwrap().unwrap(); - let restored = restoring.join().unwrap().unwrap(); + assert!(matches!( + restoring.join().unwrap(), + Err(RuntimeError::RestoreObservationConflict { .. }) + )); assert_eq!(*backend.stop_count.lock().unwrap(), 1); + assert_eq!(*backend.restore_count.lock().unwrap(), 0); + let restored = runtime + .restore_worker( + &created.worker_ref, + runtime.test_restore_request(&created.worker_ref), + ) + .unwrap(); assert_eq!(*backend.restore_count.lock().unwrap(), 1); assert_eq!(restored.status, WorkerStatus::Idle); assert_eq!( @@ -7898,6 +9320,95 @@ mod tests { ); } + #[cfg(feature = "fs-store")] + #[test] + fn stop_preserves_pending_restore_owner_across_unknown_commit_and_restart() { + for mode in [WorkerRestoreMode::Automatic, WorkerRestoreMode::Explicit] { + let dir = tempfile::tempdir().unwrap(); + let options = crate::fs_store::FsRuntimeStoreOptions { + root: dir.path().join("runtime"), + runtime_id: "stop-pending-restore".to_string(), + display_name: None, + }; + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = + Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()) + .unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("stop pending restore")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + if mode == WorkerRestoreMode::Automatic { + let mut state = runtime.lock().unwrap(); + let record = state.worker_mut(&worker.worker_ref).unwrap(); + record.status = WorkerStatus::Idle; + record.restore_intent = WorkerRestoreIntent::Automatic; + state.persist_worker(&worker.worker_id).unwrap(); + } + *backend.restore_result.lock().unwrap() = + Some(WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + "cleanup owner is pending", + ), + worker_state: None, + workdir_attachments: Vec::new(), + }); + let restored = runtime + .restore_worker_under_lock(&worker.worker_ref, mode) + .unwrap(); + assert_eq!(restored.state, WorkerRestoreState::ReconciliationRequired); + let restore_id = runtime + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .pending_restore + .unwrap() + .operation_id; + runtime_store(&runtime) + .fail_next_worker_write(crate::fs_store::AtomicWriteFault::AfterRename); + assert!(matches!( + runtime.stop_worker(&worker.worker_ref, None), + Err(RuntimeError::StoreCommitOutcomeUnknown { .. }) + )); + let state = runtime.lock().unwrap(); + let record = state.worker(&worker.worker_ref).unwrap(); + let stop_id = record.pending_stop.unwrap().operation_id; + assert_eq!(record.pending_restore.unwrap().operation_id, restore_id); + drop(state); + drop(runtime); + let retry_backend = Arc::new(TestExecutionBackend::default()); + let retry = + Runtime::with_fs_store_and_execution_backend(options, retry_backend.clone()) + .unwrap(); + assert_eq!( + *retry_backend.restore_count.lock().unwrap(), + 0, + "Stop must not launch a Controller to clean up restore" + ); + assert_eq!( + retry_backend.stop_operation_ids.lock().unwrap().as_slice(), + &[stop_id] + ); + assert_eq!( + retry.worker_detail(&worker.worker_ref).unwrap().status, + WorkerStatus::Stopped + ); + assert!( + retry + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .pending_restore + .is_none() + ); + assert!(retry.delete_worker(&worker.worker_ref).unwrap().deleted); + } + } + #[test] fn concurrent_exact_restores_share_one_execution() { let backend = Arc::new(TestExecutionBackend::default()); @@ -7912,15 +9423,17 @@ mod tests { let gate = Arc::new(RestoreGate::default()); *backend.restore_gate.lock().unwrap() = Some(gate.clone()); + let request = runtime.test_restore_request(&created.worker_ref); let start = Arc::new(Barrier::new(3)); let mut threads = Vec::new(); for _ in 0..2 { let runtime = runtime.clone(); let worker_ref = created.worker_ref.clone(); let start = start.clone(); + let request = request.clone(); threads.push(std::thread::spawn(move || { start.wait(); - runtime.restore_worker(&worker_ref) + runtime.restore_worker(&worker_ref, request) })); } start.wait(); @@ -7942,6 +9455,1089 @@ mod tests { assert!(runtime.worker_operations.lock().unwrap().is_empty()); } + #[test] + fn restore_guard_coordination_is_authorized_and_none_cannot_admit_new_intent() { + let (runtime, backend) = runtime_and_backend(); + let scope = scope("workspace-a", "server-a"); + let worker = runtime + .create_worker_scoped(&scope, scoped_task_request("coordinate", "workspace-a")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let observed = runtime.worker_detail(&worker.worker_ref).unwrap(); + let mut request = runtime.test_restore_request(&worker.worker_ref); + request.preparation = None; + let coordination = runtime_api::WorkerRestoreCoordinationRequest { + expected_observation_token: request.expected_observation_token.clone(), + request_id: request.request_id.clone(), + preparation: None, + }; + assert_eq!( + runtime + .coordinate_worker_restore_operation( + &worker.worker_ref, + coordination.clone(), + Some(&scope) + ) + .unwrap(), + (None, None) + ); + assert_eq!(runtime.worker_detail(&worker.worker_ref).unwrap(), observed); + assert!( + runtime + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .restore_guard + .owners + .is_empty() + ); + assert!(matches!( + runtime.restore_worker_operation_scoped(&scope, &worker.worker_ref, request.clone()), + Err(RuntimeError::InvalidRequest(_)) + )); + let wrong = RuntimeWorkspaceScope::new("workspace-a", "different-server"); + assert!( + runtime + .coordinate_worker_restore_operation( + &worker.worker_ref, + coordination.clone(), + Some(&wrong) + ) + .is_err() + ); + let mut admitted = coordination.clone(); + admitted.preparation = Some(runtime_api::WorkerRestorePreparation::default()); + assert!( + runtime + .coordinate_worker_restore_operation(&worker.worker_ref, admitted, Some(&scope)) + .unwrap() + .1 + .is_some() + ); + let operation_id = runtime + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .pending_restore + .unwrap() + .operation_id; + assert!( + runtime + .coordinate_worker_restore_operation( + &worker.worker_ref, + coordination.clone(), + Some(&scope) + ) + .unwrap() + .1 + .is_some() + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 0); + request.preparation = Some(runtime_api::WorkerRestorePreparation::default()); + let receipt = runtime + .restore_worker_operation_scoped(&scope, &worker.worker_ref, request) + .unwrap(); + assert_eq!(receipt.state, WorkerRestoreState::Accepted); + assert_eq!( + backend.restore_operation_ids.lock().unwrap().as_slice(), + &[operation_id] + ); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + assert!( + runtime + .coordinate_worker_restore_operation( + &worker.worker_ref, + coordination.clone(), + Some(&wrong) + ) + .is_err() + ); + assert_eq!( + runtime + .coordinate_worker_restore_operation(&worker.worker_ref, coordination, Some(&scope)) + .unwrap() + .0, + Some(receipt) + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 1); + assert_eq!( + runtime.worker_detail(&worker.worker_ref).unwrap().status, + WorkerStatus::Stopped + ); + } + + #[test] + fn restore_guard_resource_snapshot_requires_workspace_completion_before_execution() { + let (runtime, backend) = runtime_and_backend(); + let scope = scope("workspace-a", "server-a"); + let worker = runtime + .create_worker_scoped( + &scope, + scoped_task_request("resource coordination", "workspace-a"), + ) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let mut request = runtime.test_restore_request(&worker.worker_ref); + let mut preparation = runtime_api::WorkerRestorePreparation::default(); + preparation.repository_access_workdirs.push( + runtime_api::RepositoryAccessWorkdirReference { + runtime_id: "peer-runtime".into(), + working_directory_id: "peer-workdir".into(), + }, + ); + let mut coordination = runtime_api::WorkerRestoreCoordinationRequest { + expected_observation_token: request.expected_observation_token.clone(), + request_id: request.request_id.clone(), + preparation: Some(preparation.clone()), + }; + runtime + .coordinate_worker_restore_operation( + &worker.worker_ref, + coordination.clone(), + Some(&scope), + ) + .unwrap(); + let operation_id = runtime + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .pending_restore + .unwrap() + .operation_id; + // Model a retained resource snapshot before the peer authorization callback completes. + preparation + .repository_access + .push(runtime_api::WorkingDirectoryRepositoryAccessRequest { + working_directory_id: "peer-workdir".into(), + materialization: runtime_api::RepositoryMaterializationContext { + workspace_id: "workspace-a".into(), + runtime_id: "peer-runtime".into(), + operation_id: format!("worker-restore:{}", request.request_id), + config_revision: 1, + config_projection_digest: "snapshot".into(), + ssh: None, + }, + }); + coordination.preparation = Some(preparation.clone()); + assert!( + runtime + .coordinate_worker_restore_operation(&worker.worker_ref, coordination, Some(&scope)) + .unwrap() + .1 + .is_some() + ); + request.preparation = None; + assert_eq!( + runtime + .restore_worker_operation_scoped(&scope, &worker.worker_ref, request.clone()) + .unwrap() + .state, + WorkerRestoreState::ReconciliationRequired + ); + assert_eq!( + runtime + .restore_admitted_worker_under_lock(&worker.worker_ref, WorkerRestoreMode::Explicit) + .unwrap() + .state, + WorkerRestoreState::ReconciliationRequired + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 0); + request.preparation = Some(runtime_api::WorkerRestorePreparation::default()); + assert_eq!( + runtime + .restore_worker_operation_scoped(&scope, &worker.worker_ref, request.clone()) + .unwrap() + .state, + WorkerRestoreState::ReconciliationRequired + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 0); + request.preparation = Some(preparation); + assert_eq!( + runtime + .restore_worker_operation_scoped(&scope, &worker.worker_ref, request) + .unwrap() + .state, + WorkerRestoreState::Accepted + ); + assert_eq!( + backend.restore_operation_ids.lock().unwrap().as_slice(), + &[operation_id] + ); + } + + #[test] + fn restore_guard_stale_preparation_has_no_effects() { + let (runtime, backend) = runtime_and_backend(); + let worker = runtime + .create_worker(task_request("stale preparation")) + .unwrap(); + let mut stale = runtime.test_restore_request(&worker.worker_ref); + stale.preparation = Some(serde_json::from_value(serde_json::json!({ + "workspace_api": { "workspace_id": "must-not-bind", "base_url": "https://invalid.example" }, + "workdir_attachments": [], + "repository_access": [] + })).unwrap()); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let before = runtime.worker_detail(&worker.worker_ref).unwrap(); + assert!(matches!( + runtime.restore_worker_operation(&worker.worker_ref, stale), + Err(RuntimeError::RestoreObservationConflict { .. }) + )); + assert_eq!(runtime.worker_detail(&worker.worker_ref).unwrap(), before); + let state = runtime.lock().unwrap(); + let record = state.worker(&worker.worker_ref).unwrap(); + assert!(record.request.as_ref().unwrap().workspace_api.is_none()); + assert!(record.restore_guard.owners.is_empty()); + assert_eq!(*backend.restore_count.lock().unwrap(), 0); + assert!(backend.repository_accesses.lock().unwrap().is_empty()); + } + + #[test] + fn restore_guard_lifecycle_aba_and_historical_replay_after_stop() { + let (runtime, backend) = runtime_and_backend(); + let worker = runtime.create_worker(task_request("restore ABA")).unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let request = runtime.test_restore_request(&worker.worker_ref); + let mut other_intent = request.clone(); + other_intent.request_id = "other-intent".into(); + let accepted = runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!(accepted.state, WorkerRestoreState::Accepted); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + assert!(matches!( + runtime.restore_worker_operation(&worker.worker_ref, other_intent), + Err(RuntimeError::RestoreObservationConflict { .. }) + )); + let stopped = runtime.worker_detail(&worker.worker_ref).unwrap(); + assert_ne!( + stopped.restore_observation_token.as_deref(), + Some(request.expected_observation_token.as_str()) + ); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap(), + accepted + ); + assert_eq!(runtime.worker_detail(&worker.worker_ref).unwrap(), stopped); + assert_eq!(*backend.restore_count.lock().unwrap(), 1); + let mut altered = request; + altered.expected_observation_token = stopped.restore_observation_token.unwrap(); + assert!(matches!( + runtime.restore_worker_operation(&worker.worker_ref, altered), + Err(RuntimeError::RestoreObservationConflict { .. }) + )); + } + + #[test] + fn restore_guard_protocol_state_aba_rotates_generation() { + let (runtime, backend) = runtime_and_backend(); + let worker = runtime.create_worker(task_request("state ABA")).unwrap(); + let request = runtime.test_restore_request(&worker.worker_ref); + { + let mut state = runtime.lock().unwrap(); + for status in [ + protocol::WorkerStatus::Running, + protocol::WorkerStatus::Idle, + ] { + assert!(state.project_protocol_event_to_worker_state( + &worker.worker_ref, + &protocol::Event::WorkerState { + snapshot: status.into() + } + )); + state.persist_worker(&worker.worker_id).unwrap(); + } + } + assert_eq!( + runtime + .worker_detail(&worker.worker_ref) + .unwrap() + .worker_state + .unwrap() + .state, + protocol::WorkerState::Idle + ); + assert!(matches!( + runtime.restore_worker_operation(&worker.worker_ref, request), + Err(RuntimeError::RestoreObservationConflict { .. }) + )); + assert_eq!(*backend.restore_count.lock().unwrap(), 0); + } + + #[test] + fn restore_guard_distinct_contenders_are_fenced_before_preparation() { + let (runtime, backend) = runtime_and_backend(); + let worker = runtime + .create_worker(task_request("distinct contenders")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let request = runtime.test_restore_request(&worker.worker_ref); + let mut contender = request.clone(); + contender.request_id = "losing-request".into(); + contender.preparation = Some(serde_json::from_value(serde_json::json!({ + "workspace_api": { "workspace_id": "must-not-bind", "base_url": "https://invalid.example" }, + "repository_access": [] + })).unwrap()); + let gate = Arc::new(RestoreGate::default()); + backend.restore_gate.lock().unwrap().replace(gate.clone()); + let first_runtime = runtime.clone(); + let first_ref = worker.worker_ref.clone(); + let first = + std::thread::spawn(move || first_runtime.restore_worker_operation(&first_ref, request)); + assert!(gate.wait_for_entered(1, std::time::Duration::from_secs(2))); + let second_runtime = runtime.clone(); + let second_ref = worker.worker_ref.clone(); + let second = std::thread::spawn(move || { + second_runtime.restore_worker_operation(&second_ref, contender) + }); + gate.release(); + assert_eq!( + first.join().unwrap().unwrap().state, + WorkerRestoreState::Accepted + ); + assert!(matches!( + second.join().unwrap(), + Err(RuntimeError::RestoreObservationConflict { .. }) + )); + assert_eq!(*backend.restore_count.lock().unwrap(), 1); + assert!( + runtime + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .request + .as_ref() + .unwrap() + .workspace_api + .is_none() + ); + } + + #[test] + fn restore_guard_applies_preparation_once_and_replays_changed_candidate() { + let (runtime, backend) = runtime_and_backend(); + let scope = scope("workspace-a", "server-a"); + let worker = runtime + .create_worker_scoped( + &scope, + scoped_task_request("prepared restore", "workspace-a"), + ) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let mut request = runtime.test_restore_request(&worker.worker_ref); + request.preparation = Some(serde_json::from_value(serde_json::json!({ + "workspace_api": { "workspace_id": "workspace-a", "base_url": "https://workspace.example/workspace-a" }, + "workdir_attachments": [], + "repository_access": [] + })).unwrap()); + let receipt = runtime + .restore_worker_operation_scoped(&scope, &worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!(receipt.state, WorkerRestoreState::Accepted); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let logical = vec![LogicalWorkdirAttachment { + alias: workdir::WorkdirAttachmentAlias::new("changed-after-stop").unwrap(), + working_directory_id: "workdir-later".into(), + capabilities: workdir::WorkdirSessionCapabilities::READ_ONLY, + }]; + runtime + .replace_worker_workdir_attachments(&worker.worker_ref, logical.clone()) + .unwrap(); + assert_eq!( + runtime + .restore_worker_operation_scoped(&scope, &worker.worker_ref, request.clone()) + .unwrap(), + receipt + ); + assert_eq!( + runtime + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .logical_workdir_attachments, + logical + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 1); + request.preparation = Some(serde_json::from_value(serde_json::json!({ + "workspace_api": { "workspace_id": "must-not-bind", "base_url": "https://fresh.invalid.example" }, + "workdir_attachments": [], + "repository_access": [] + })).unwrap()); + assert_eq!( + runtime + .restore_worker_operation_scoped(&scope, &worker.worker_ref, request.clone()) + .unwrap(), + receipt + ); + let state = runtime.lock().unwrap(); + let record = state.worker(&worker.worker_ref).unwrap(); + assert_eq!(record.logical_workdir_attachments, logical); + let owner = &record.restore_guard.owners[&request.request_id]; + assert_eq!( + owner + .request + .preparation + .as_ref() + .unwrap() + .workspace_api + .as_ref() + .unwrap() + .base_url, + "https://workspace.example/workspace-a" + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 1); + } + + #[cfg(feature = "fs-store")] + #[test] + fn restore_guard_admission_retry_uses_first_persisted_preparation() { + let dir = tempfile::tempdir().unwrap(); + let options = FsRuntimeStoreOptions::new(dir.path().join("runtime")) + .with_runtime_id("guard-first-preparation"); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = + Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()).unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let scope = scope("workspace-a", "server-a"); + let mut create = scoped_task_request("first preparation", "workspace-a"); + create.workspace_api.as_mut().unwrap().base_url = "https://first.example".into(); + let worker = runtime.create_worker_scoped(&scope, create).unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let mut request = runtime.test_restore_request(&worker.worker_ref); + request.preparation = Some(serde_json::from_value(serde_json::json!({ + "workspace_api": { "workspace_id": "workspace-a", "base_url": "https://first.example" }, + "repository_access": [] + })).unwrap()); + runtime_store(&runtime) + .fail_next_worker_write(crate::fs_store::AtomicWriteFault::AfterRename); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap() + .state, + WorkerRestoreState::ReconciliationRequired + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 0); + drop(runtime); + // No backend on restart: admission survives, preparation has not run. + let resumed = Runtime::with_fs_store(options).unwrap(); + resumed.lock().unwrap().execution_backend = + Some(crate::execution::WorkerExecutionBackendRef::new(backend.clone()).unwrap()); + request.preparation = Some(serde_json::from_value(serde_json::json!({ + "workspace_api": { "workspace_id": "must-not-bind", "base_url": "https://fresh.invalid.example" }, + "repository_access": [] + })).unwrap()); + assert!( + resumed.worker_detail(&worker.worker_ref).is_ok(), + "{:?}", + resumed.lock().unwrap().diagnostics + ); + let receipt = resumed + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!(receipt.state, WorkerRestoreState::Accepted); + assert_eq!(*backend.restore_count.lock().unwrap(), 1); + let state = resumed.lock().unwrap(); + let record = state.worker(&worker.worker_ref).unwrap(); + assert_eq!( + record + .request + .as_ref() + .unwrap() + .workspace_api + .as_ref() + .unwrap() + .base_url, + "https://first.example" + ); + let owner = &record.restore_guard.owners[&request.request_id]; + assert_eq!( + owner + .request + .preparation + .as_ref() + .unwrap() + .workspace_api + .as_ref() + .unwrap() + .base_url, + "https://first.example" + ); + assert_eq!(owner.receipt.as_ref(), Some(&receipt)); + } + + #[cfg(feature = "fs-store")] + #[test] + fn restore_guard_interrupted_preparation_replays_original_pending_operation() { + let dir = tempfile::tempdir().unwrap(); + let options = FsRuntimeStoreOptions::new(dir.path().join("runtime")) + .with_runtime_id("guard-partial-preparation"); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = + Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()).unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let scope = scope("workspace-a", "server-a"); + let mut create = scoped_task_request("interrupted preparation", "workspace-a"); + create.workspace_api = None; + let worker = runtime.create_worker_scoped(&scope, create).unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let mut request = runtime.test_restore_request(&worker.worker_ref); + request.preparation = Some(serde_json::from_value(serde_json::json!({ + "workspace_api": { "workspace_id": "workspace-a", "base_url": "https://partially-applied.example" }, + "workdir_attachments": [], "repository_access": [] + })).unwrap()); + let (_, preparation) = runtime + .coordinate_worker_restore_operation( + &worker.worker_ref, + runtime_api::WorkerRestoreCoordinationRequest { + expected_observation_token: request.expected_observation_token.clone(), + request_id: request.request_id.clone(), + preparation: request.preparation.clone(), + }, + Some(&scope), + ) + .unwrap(); + assert!(preparation.is_some()); + let operation_id = runtime + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .pending_restore + .unwrap() + .operation_id; + runtime_store(&runtime) + .fail_next_worker_write(crate::fs_store::AtomicWriteFault::AfterRename); + request.preparation = None; + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap() + .state, + WorkerRestoreState::ReconciliationRequired + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 0); + assert_eq!( + runtime + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .pending_restore + .unwrap() + .operation_id, + operation_id + ); + drop(runtime); + let resumed = Runtime::with_fs_store(options).unwrap(); + resumed.lock().unwrap().execution_backend = + Some(crate::execution::WorkerExecutionBackendRef::new(backend.clone()).unwrap()); + let receipt = resumed + .restore_worker_operation(&worker.worker_ref, request) + .unwrap(); + assert_eq!(receipt.state, WorkerRestoreState::Accepted); + assert_eq!(*backend.restore_count.lock().unwrap(), 1); + assert_eq!( + backend.restore_operation_ids.lock().unwrap().as_slice(), + &[operation_id] + ); + assert!( + resumed + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .pending_restore + .is_none() + ); + } + + #[cfg(feature = "fs-store")] + #[test] + fn restore_guard_unowned_pending_operation_recovers_exact_lifecycle_owner_after_restart() { + let dir = tempfile::tempdir().unwrap(); + let options = FsRuntimeStoreOptions::new(dir.path().join("runtime")) + .with_runtime_id("guard-unowned-pending"); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = + Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()).unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("recover existing pending operation")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + *backend.restore_result.lock().unwrap() = + Some(WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + "unknown acknowledgement", + ), + worker_state: None, + workdir_attachments: Vec::new(), + }); + // Existing startup/old-contract pending operation: no public receipt owner. + assert_eq!( + runtime + .restore_worker_under_lock(&worker.worker_ref, WorkerRestoreMode::Explicit) + .unwrap() + .state, + WorkerRestoreState::ReconciliationRequired + ); + let operation_id = runtime + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .pending_restore + .unwrap() + .operation_id; + drop(runtime); + let path = dir + .path() + .join("runtime/workers") + .join(worker.worker_id.to_string()) + .join("worker.json"); + let mut aggregate: serde_json::Value = + serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + aggregate.as_object_mut().unwrap().remove("restore_guard"); + std::fs::write(&path, serde_json::to_vec_pretty(&aggregate).unwrap()).unwrap(); + let resumed = Runtime::with_fs_store(options).unwrap(); + resumed.lock().unwrap().execution_backend = + Some(crate::execution::WorkerExecutionBackendRef::new(backend.clone()).unwrap()); + let request = runtime_api::WorkerRestoreCoordinationRequest { + expected_observation_token: resumed + .worker_detail(&worker.worker_ref) + .unwrap() + .restore_observation_token + .unwrap(), + request_id: "explicit-pending-recovery".into(), + preparation: None, + }; + assert_eq!( + resumed + .coordinate_worker_restore_operation(&worker.worker_ref, request.clone(), None) + .unwrap() + .0 + .unwrap() + .state, + WorkerRestoreState::ReconciliationRequired + ); + let count = *backend.restore_count.lock().unwrap(); + let mut competitor = request.clone(); + competitor.request_id = "different-intent".into(); + assert!(matches!( + resumed.coordinate_worker_restore_operation(&worker.worker_ref, competitor, None), + Err(RuntimeError::RestoreObservationConflict { .. }) + )); + assert_eq!(*backend.restore_count.lock().unwrap(), count); + *backend.restore_result.lock().unwrap() = None; + assert_eq!( + resumed + .coordinate_worker_restore_operation(&worker.worker_ref, request.clone(), None) + .unwrap() + .0 + .unwrap() + .state, + WorkerRestoreState::Accepted + ); + assert!( + backend + .restore_operation_ids + .lock() + .unwrap() + .iter() + .all(|id| *id == operation_id) + ); + assert!( + resumed + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .pending_restore + .is_none() + ); + resumed.stop_worker(&worker.worker_ref, None).unwrap(); + let count = *backend.restore_count.lock().unwrap(); + assert_eq!( + resumed + .coordinate_worker_restore_operation(&worker.worker_ref, request, None) + .unwrap() + .0 + .unwrap() + .state, + WorkerRestoreState::Accepted + ); + assert_eq!(*backend.restore_count.lock().unwrap(), count); + assert_eq!( + resumed.worker_detail(&worker.worker_ref).unwrap().status, + WorkerStatus::Stopped + ); + } + + #[cfg(feature = "fs-store")] + #[test] + fn restore_guard_unknown_preparation_write_retries_same_pending_operation() { + let dir = tempfile::tempdir().unwrap(); + let options = FsRuntimeStoreOptions::new(dir.path().join("runtime")) + .with_runtime_id("guard-unknown-preparation"); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = + Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()).unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("unknown preparation commit")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let mut request = runtime.test_restore_request(&worker.worker_ref); + request.preparation = Some( + serde_json::from_value(serde_json::json!({ + "workdir_attachments": [], "repository_access": [] + })) + .unwrap(), + ); + runtime_store(&runtime) + .fail_next_worker_write(crate::fs_store::AtomicWriteFault::AfterRename); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap() + .state, + WorkerRestoreState::ReconciliationRequired + ); + // This write interrupts idempotent attachment completion after admission. + runtime_store(&runtime) + .fail_next_worker_write(crate::fs_store::AtomicWriteFault::AfterRename); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap() + .state, + WorkerRestoreState::ReconciliationRequired + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 0); + drop(runtime); + let resumed_backend = Arc::new(TestExecutionBackend::default()); + let resumed = + Runtime::with_fs_store_and_execution_backend(options, resumed_backend.clone()).unwrap(); + request.preparation = None; + assert_eq!( + resumed + .restore_worker_operation(&worker.worker_ref, request) + .unwrap() + .state, + WorkerRestoreState::Accepted + ); + assert_eq!(*resumed_backend.restore_count.lock().unwrap(), 1); + assert_eq!( + resumed.worker_detail(&worker.worker_ref).unwrap().status, + WorkerStatus::Idle + ); + } + + #[cfg(feature = "fs-store")] + #[test] + fn restore_guard_terminal_commit_is_atomic_and_settled_owner_never_resumes() { + let dir = tempfile::tempdir().unwrap(); + let options = FsRuntimeStoreOptions::new(dir.path().join("runtime")) + .with_runtime_id("guard-terminal-commit"); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = + Runtime::with_fs_store_and_execution_backend(options.clone(), backend).unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("atomic terminal receipt")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let request = runtime.test_restore_request(&worker.worker_ref); + let receipt = runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap(); + let aggregate: serde_json::Value = serde_json::from_slice( + &std::fs::read( + dir.path() + .join("runtime/workers") + .join(worker.worker_id.to_string()) + .join("worker.json"), + ) + .unwrap(), + ) + .unwrap(); + assert_eq!(aggregate["status"], "idle"); + assert!(aggregate["restore_guard"]["active_request_id"].is_null()); + assert_eq!( + aggregate["restore_guard"]["owners"][&request.request_id]["receipt"], + serde_json::to_value(&receipt).unwrap() + ); + // Simulate a legacy/inconsistent active marker on an already-settled + // owner. Startup must not silently create a new execution for it. + { + let mut state = runtime.lock().unwrap(); + let mut candidate = state.worker(&worker.worker_ref).unwrap().clone(); + candidate.restore_guard.active_request_id = Some(request.request_id.clone()); + state.persist_worker_record(&candidate).unwrap(); + state.workers.insert(worker.worker_id, candidate); + } + drop(runtime); + let resumed_backend = Arc::new(TestExecutionBackend::default()); + let resumed = + Runtime::with_fs_store_and_execution_backend(options, resumed_backend.clone()).unwrap(); + assert_eq!(*resumed_backend.restore_count.lock().unwrap(), 0); + assert_eq!( + resumed + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap(), + receipt + ); + // Also exercise the accepted-aggregate/receipt-pending crash gap. + { + let mut state = resumed.lock().unwrap(); + let mut candidate = state.worker(&worker.worker_ref).unwrap().clone(); + let owner = candidate + .restore_guard + .owners + .get_mut(&request.request_id) + .unwrap(); + assert!(owner.execution_started); + owner.receipt = None; + candidate.restore_guard.active_request_id = Some(request.request_id.clone()); + state.persist_worker_record(&candidate).unwrap(); + state.workers.insert(worker.worker_id, candidate); + } + drop(resumed); + let gap_backend = Arc::new(TestExecutionBackend::default()); + let gap_resumed = Runtime::with_fs_store_and_execution_backend( + FsRuntimeStoreOptions::new(dir.path().join("runtime")) + .with_runtime_id("guard-terminal-commit"), + gap_backend.clone(), + ) + .unwrap(); + assert_eq!(*gap_backend.restore_count.lock().unwrap(), 0); + assert!(gap_backend.restore_operation_ids.lock().unwrap().is_empty()); + assert_eq!( + gap_resumed + .restore_worker_operation(&worker.worker_ref, request) + .unwrap() + .state, + WorkerRestoreState::Accepted + ); + } + + #[test] + fn restore_guard_terminal_rejection_replays_without_repreparation() { + let (runtime, backend) = runtime_and_backend(); + let worker = runtime + .create_worker(task_request("rejected owner")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let request = runtime.test_restore_request(&worker.worker_ref); + backend + .restore_result + .lock() + .unwrap() + .replace(WorkerExecutionSpawnResult::Rejected( + WorkerExecutionResult::rejected( + WorkerExecutionOperation::Restore, + "injected rejection", + ), + )); + let receipt = runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!(receipt.state, WorkerRestoreState::Rejected); + backend.restore_result.lock().unwrap().take(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request) + .unwrap(), + receipt + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 1); + assert_eq!( + runtime.worker_detail(&worker.worker_ref).unwrap().status, + WorkerStatus::Stopped + ); + } + + #[cfg(feature = "fs-store")] + #[test] + fn restore_guard_pending_owner_survives_restart_and_reuses_operation_id() { + let dir = tempfile::tempdir().unwrap(); + let options = + FsRuntimeStoreOptions::new(dir.path().join("runtime")).with_runtime_id("guard-pending"); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = + Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()).unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("durable pending owner")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let request = runtime.test_restore_request(&worker.worker_ref); + backend + .restore_result + .lock() + .unwrap() + .replace(WorkerExecutionSpawnResult::Errored( + WorkerExecutionResult::rejected(WorkerExecutionOperation::Restore, "uncertain"), + )); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap() + .state, + WorkerRestoreState::ReconciliationRequired + ); + let operation_id = backend.restore_operation_ids.lock().unwrap()[0]; + drop(runtime); + let restarted = Runtime::with_fs_store(options.clone()).unwrap(); + let mut other = request.clone(); + other.request_id = "not-persisted-owner".into(); + other.expected_observation_token = restarted + .worker_detail(&worker.worker_ref) + .unwrap() + .restore_observation_token + .unwrap(); + assert!(matches!( + restarted.restore_worker_operation(&worker.worker_ref, other), + Err(RuntimeError::RestoreObservationConflict { .. }) + )); + drop(restarted); + let resumed_backend = Arc::new(TestExecutionBackend::default()); + let resumed = + Runtime::with_fs_store_and_execution_backend(options, resumed_backend.clone()).unwrap(); + assert_eq!( + resumed_backend + .restore_operation_ids + .lock() + .unwrap() + .as_slice(), + &[operation_id] + ); + let receipt = resumed + .restore_worker_operation(&worker.worker_ref, request) + .unwrap(); + assert_eq!(receipt.state, WorkerRestoreState::Accepted); + assert_eq!(*resumed_backend.restore_count.lock().unwrap(), 1); + } + + #[test] + fn restore_guard_pending_owner_is_only_reconciliation_intent() { + let (runtime, backend) = runtime_and_backend(); + let worker = runtime + .create_worker(task_request("pending owner")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let mut request = runtime.test_restore_request(&worker.worker_ref); + request.preparation = Some( + serde_json::from_value(serde_json::json!({ + "workdir_attachments": [], "repository_access": [] + })) + .unwrap(), + ); + backend + .restore_result + .lock() + .unwrap() + .replace(WorkerExecutionSpawnResult::Errored( + WorkerExecutionResult::rejected(WorkerExecutionOperation::Restore, "uncertain"), + )); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap() + .state, + WorkerRestoreState::ReconciliationRequired + ); + // A retry may carry newly minted preparation. It must reconcile the + // first persisted owner, not apply this invalid replacement payload. + request.preparation = Some(serde_json::from_value(serde_json::json!({ + "workspace_api": { "workspace_id": "must-not-bind", "base_url": "https://fresh.invalid.example" }, + "repository_access": [] + })).unwrap()); + let mut fresh = runtime.test_restore_request(&worker.worker_ref); + fresh.request_id = "new-owner".into(); + fresh.expected_observation_token = runtime + .worker_detail(&worker.worker_ref) + .unwrap() + .restore_observation_token + .unwrap(); + assert!(matches!( + runtime.restore_worker_operation(&worker.worker_ref, fresh), + Err(RuntimeError::RestoreObservationConflict { .. }) + )); + backend.restore_result.lock().unwrap().take(); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request) + .unwrap() + .state, + WorkerRestoreState::Accepted + ); + let ids = backend.restore_operation_ids.lock().unwrap(); + assert_eq!(ids.len(), 2); + assert_eq!(ids[0], ids[1]); + } + + #[cfg(feature = "fs-store")] + #[test] + fn restore_guard_generation_and_terminal_receipt_survive_restart() { + let dir = tempfile::tempdir().unwrap(); + let options = + FsRuntimeStoreOptions::new(dir.path().join("runtime")).with_runtime_id("guard-test"); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = + Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()).unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("durable replay")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let request = runtime.test_restore_request(&worker.worker_ref); + let receipt = runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let generation = runtime + .worker_detail(&worker.worker_ref) + .unwrap() + .restore_observation_token; + drop(runtime); + let restarted = Runtime::with_fs_store(options).unwrap(); + assert_eq!( + restarted + .worker_detail(&worker.worker_ref) + .unwrap() + .restore_observation_token, + generation + ); + assert_eq!( + restarted + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap(), + receipt + ); + let mut stale = request; + stale.request_id = "stale-after-restart".into(); + assert!(matches!( + restarted.restore_worker_operation(&worker.worker_ref, stale), + Err(RuntimeError::RestoreObservationConflict { .. }) + )); + assert_eq!( + restarted.worker_detail(&worker.worker_ref).unwrap().status, + WorkerStatus::Stopped + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 1); + } + #[test] fn operation_lock_registry_reclaims_idle_worker_entries() { let runtime = runtime_with_backend(); @@ -7954,6 +10550,60 @@ mod tests { assert!(runtime.worker_operations.lock().unwrap().is_empty()); } + #[test] + fn scoped_attachment_replacement_revalidates_workspace_after_worker_lock() { + let runtime = Arc::new(runtime_with_backend()); + runtime.store_config_bundle(test_bundle()).unwrap(); + let workspace_scope = scope("workspace-a", "server-a"); + let worker = runtime + .create_worker_scoped( + &workspace_scope, + scoped_task_request("attachment scope race", "workspace-a"), + ) + .unwrap(); + let lease = runtime.worker_operation_lock(worker.worker_id).unwrap(); + let guard = lease.lock().unwrap(); + let (started_tx, started_rx) = std::sync::mpsc::channel(); + let replacing_runtime = runtime.clone(); + let worker_ref = worker.worker_ref.clone(); + let replacing = std::thread::spawn(move || { + started_tx.send(()).unwrap(); + replacing_runtime.replace_worker_workdir_attachments_scoped( + &workspace_scope, + &worker_ref, + Vec::new(), + ) + }); + started_rx.recv().unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2); + while Arc::strong_count(&lease.operation_lock) < 3 { + assert!( + std::time::Instant::now() < deadline, + "attachment replacement did not reach its Worker lock" + ); + std::thread::yield_now(); + } + { + let mut state = runtime.lock().unwrap(); + state.worker_mut(&worker.worker_ref).unwrap().workspace_id = Some("workspace-b".into()); + } + drop(guard); + drop(lease); + assert!(matches!( + replacing.join().unwrap(), + Err(RuntimeError::WorkerNotFound { .. }) + )); + assert!( + runtime + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .logical_workdir_attachments + .is_empty() + ); + } + #[test] fn scoped_stop_revalidates_workspace_after_waiting_for_worker_lock() { let (runtime, backend) = runtime_and_backend(); @@ -7979,6 +10629,14 @@ mod tests { stopping_runtime.stop_worker_scoped(&stopping_scope, &stopping_ref, None) }); started_rx.recv().unwrap(); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(2); + while Arc::strong_count(&lease.operation_lock) < 3 { + assert!( + std::time::Instant::now() < deadline, + "scoped operation did not reach its Worker lock" + ); + std::thread::yield_now(); + } { let mut state = runtime.lock().unwrap(); state.worker_mut(&worker.worker_ref).unwrap().workspace_id = @@ -7993,6 +10651,209 @@ mod tests { assert!(runtime.worker_operations.lock().unwrap().is_empty()); } + #[cfg(all(feature = "fs-store", feature = "ws-server"))] + #[test] + fn rejected_restore_settlement_failure_detaches_discarded_publication_owner() { + for rolled_back in [false, true] { + let dir = tempfile::tempdir().unwrap(); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = Arc::new( + Runtime::with_fs_store_and_execution_backend( + crate::fs_store::FsRuntimeStoreOptions { + root: dir.path().join("runtime"), + runtime_id: "test-runtime".into(), + display_name: None, + }, + backend.clone(), + ) + .unwrap(), + ); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("discarded context settlement")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let failure = WorkerExecutionResult::rejected( + WorkerExecutionOperation::Restore, + "injected restore rejection", + ); + backend + .restore_result + .lock() + .unwrap() + .replace(if rolled_back { + WorkerExecutionSpawnResult::RolledBack(failure) + } else { + WorkerExecutionSpawnResult::Rejected(failure) + }); + let gate = Arc::new(RestoreGate::default()); + backend.restore_gate.lock().unwrap().replace(gate.clone()); + let restoring_runtime = runtime.clone(); + let worker_ref = worker.worker_ref.clone(); + let restoring = std::thread::spawn(move || { + restoring_runtime.restore_worker_operation( + &worker_ref, + restoring_runtime.test_restore_request(&worker_ref), + ) + }); + assert!(gate.wait_for_entered(1, std::time::Duration::from_secs(2))); + let operation_id = runtime + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .pending_restore + .unwrap() + .operation_id; + runtime_store(&runtime) + .fail_next_worker_write(crate::fs_store::AtomicWriteFault::BeforeRename); + gate.release(); + assert_eq!( + restoring.join().unwrap().unwrap().state, + WorkerRestoreState::ReconciliationRequired + ); + { + let state = runtime.lock().unwrap(); + let record = state.worker(&worker.worker_ref).unwrap(); + assert_eq!(record.pending_restore.unwrap().operation_id, operation_id); + assert!(record.restore_candidate_context.is_none()); + } + backend.restore_result.lock().unwrap().take(); + backend.restore_gate.lock().unwrap().take(); + backend + .restore_event + .lock() + .unwrap() + .replace(protocol::Event::TextDelta { + text: "new candidate".into(), + }); + let cursor = runtime + .worker_observation_cursor_now(&worker.worker_ref) + .unwrap(); + assert_eq!( + runtime + .restore_worker_operation( + &worker.worker_ref, + runtime.test_restore_request(&worker.worker_ref) + ) + .unwrap() + .state, + WorkerRestoreState::Accepted + ); + backend + .publish_text_delta(&worker.worker_ref, "live candidate") + .unwrap(); + let events = runtime + .read_worker_observation_events(&worker.worker_ref, cursor) + .unwrap(); + assert_eq!(events.len(), 2); + assert_eq!( + backend.restore_operation_ids.lock().unwrap().as_slice(), + &[operation_id, operation_id] + ); + } + } + + #[cfg(feature = "ws-server")] + #[test] + fn committed_restore_retries_retained_observation_activation() { + let (runtime, backend) = runtime_and_backend(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("activation retry")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let cursor = runtime + .worker_observation_cursor_now(&worker.worker_ref) + .unwrap(); + let actual = runtime.execution_context(worker.worker_ref.clone()); + let fail_once = Arc::new(AtomicBool::new(true)); + let failure = fail_once.clone(); + let context = WorkerExecutionContext::candidate( + worker.worker_ref.clone(), + Arc::new(move |_, event| { + if failure.swap(false, Ordering::SeqCst) { + return Err(RuntimeError::InvalidRequest( + "injected publication failure".into(), + )); + } + actual.publish_observation(event) + }), + ); + let operation_id = WorkerLifecycleOperationId::new(); + runtime + .begin_restore_operation( + &worker.worker_ref, + operation_id, + WorkerRestoreMode::Explicit, + context, + ) + .unwrap(); + backend + .restore_event + .lock() + .unwrap() + .replace(protocol::Event::TextDelta { + text: "first".into(), + }); + let first = runtime + .restore_admitted_worker_under_lock(&worker.worker_ref, WorkerRestoreMode::Explicit) + .unwrap(); + assert_eq!(first.state, WorkerRestoreState::ReconciliationRequired); + assert!( + runtime + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .restore_candidate_context + .is_some() + ); + assert!( + runtime + .read_worker_observation_events(&worker.worker_ref, cursor) + .unwrap() + .is_empty() + ); + backend + .publish_text_delta(&worker.worker_ref, "second") + .unwrap(); + let retry = runtime + .restore_worker_operation( + &worker.worker_ref, + runtime.test_restore_request(&worker.worker_ref), + ) + .unwrap(); + assert_eq!(retry.state, WorkerRestoreState::Accepted); + assert_eq!( + *backend.restore_count.lock().unwrap(), + 1, + "activation retry must not replace the execution" + ); + assert!( + runtime + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .restore_candidate_context + .is_none() + ); + backend + .publish_text_delta(&worker.worker_ref, "third") + .unwrap(); + let texts = runtime + .read_worker_observation_events(&worker.worker_ref, cursor) + .unwrap() + .into_iter() + .filter_map(|event| match event.payload { + protocol::Event::TextDelta { text } => Some(text), + _ => None, + }) + .collect::>(); + assert_eq!(texts, ["first", "second", "third"]); + } + #[cfg(feature = "ws-server")] #[test] fn restore_candidate_observation_is_hidden_until_aggregate_commit() { @@ -8020,8 +10881,12 @@ mod tests { let restoring_runtime = runtime.clone(); let restoring_ref = worker.worker_ref.clone(); - let restoring = - std::thread::spawn(move || restoring_runtime.restore_worker_operation(&restoring_ref)); + let restoring = std::thread::spawn(move || { + restoring_runtime.restore_worker_operation( + &restoring_ref, + restoring_runtime.test_restore_request(&restoring_ref), + ) + }); assert!(gate.wait_for_entered(1, std::time::Duration::from_secs(2))); let pending = runtime.worker_detail(&worker.worker_ref).unwrap(); @@ -8062,7 +10927,12 @@ mod tests { .unwrap(); runtime.stop_worker(&created.worker_ref, None).unwrap(); - let restored = runtime.restore_worker(&created.worker_ref).unwrap(); + let restored = runtime + .restore_worker( + &created.worker_ref, + runtime.test_restore_request(&created.worker_ref), + ) + .unwrap(); let worker_state = restored .worker_state @@ -8176,7 +11046,6 @@ mod tests { WorkerExecutionOperation::Spawn, "Workdir cleanup could not be proven", ), - handle: None, worker_state: None, workdir_attachments: vec![WorkingDirectoryAttachmentStatus { alias: workdir::WorkdirAttachmentAlias::new("checkout").unwrap(), @@ -8420,7 +11289,6 @@ mod tests { fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult { WorkerExecutionSpawnResult::Connected { - handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()), worker_state: protocol::WorkerStateSnapshot { ..protocol::WorkerStatus::Idle.into() }, @@ -8439,7 +11307,7 @@ mod tests { fn dispatch_input( &self, - _handle: &WorkerExecutionHandle, + _worker_ref: &WorkerRef, input: WorkerInput, ) -> WorkerExecutionResult { WorkerExecutionResult::accepted_submission( @@ -8477,6 +11345,114 @@ mod tests { ); } + #[cfg(feature = "ws-server")] + #[test] + fn connected_protocol_cannot_authorize_a_different_transport_target() { + let (runtime, _) = runtime_and_backend(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let a_scope = scope("workspace-a", "server-a"); + let b_scope = scope("workspace-b", "server-b"); + let a = runtime + .create_worker_scoped(&a_scope, scoped_task_request("a", "workspace-a")) + .unwrap(); + let b = runtime + .create_worker_scoped(&b_scope, scoped_task_request("b", "workspace-b")) + .unwrap(); + let (_, events) = tokio::sync::mpsc::channel(1); + let transport = crate::execution::WorkerProtocolTransport::new( + a.worker_ref.clone(), + protocol::Event::TextDelta { + text: "endpoint a".into(), + }, + events, + Arc::new(|_| panic!("scope for B must not authorize endpoint A")), + Arc::new(|| Ok(())), + ); + let method = Method::Shutdown { + command: protocol::WorkerCommandEnvelope { command_id: 1 }, + }; + assert!(matches!( + runtime.send_connected_protocol_method_scoped( + &b_scope, + &b.worker_ref, + &transport, + method + ), + Err(RuntimeError::WorkerNotFound { .. }) + )); + for worker_ref in [&a.worker_ref, &b.worker_ref] { + let state = runtime.lock().unwrap(); + let record = state.worker(worker_ref).unwrap(); + assert!(record.pending_stop.is_none()); + assert_eq!(record.status, WorkerStatus::Idle); + } + } + + #[cfg(feature = "ws-server")] + #[test] + fn closed_protocol_transport_cannot_journal_shutdown_or_dispatch_into_restored_worker() { + let (runtime, backend) = runtime_and_backend(); + let worker = runtime + .create_worker(task_request("old protocol endpoint")) + .unwrap(); + let (_, events) = tokio::sync::mpsc::channel(1); + let transport = crate::execution::WorkerProtocolTransport::new( + worker.worker_ref.clone(), + protocol::Event::TextDelta { + text: "old transport".to_string(), + }, + events, + Arc::new(|_| panic!("closed transport must be rejected before dispatch")), + Arc::new(|| { + Err(WorkerExecutionResult::rejected( + WorkerExecutionOperation::ProtocolMethod, + "original Controller transport is closed", + )) + }), + ); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + runtime + .restore_worker( + &worker.worker_ref, + runtime.test_restore_request(&worker.worker_ref), + ) + .unwrap(); + let stop_count = *backend.stop_count.lock().unwrap(); + let detail = runtime.worker_detail(&worker.worker_ref).unwrap(); + for method in [ + Method::Submit { + submission_request_id: "old-request".to_string(), + input: Vec::new(), + }, + Method::ListCompletions { + kind: protocol::CompletionKind::File, + prefix: "".to_string(), + request_id: None, + context: None, + }, + Method::Shutdown { + command: test_command(), + }, + ] { + assert!(matches!( + runtime.send_connected_protocol_method(&worker.worker_ref, &transport, method), + Err(RuntimeError::WorkerExecutionRejected { .. }) + )); + } + assert_eq!(*backend.stop_count.lock().unwrap(), stop_count); + assert!(backend.dispatched_inputs.lock().unwrap().is_empty()); + let state = runtime.lock().unwrap(); + assert!( + state + .worker(&worker.worker_ref) + .unwrap() + .pending_stop + .is_none() + ); + assert_eq!(runtime.worker_operations.lock().unwrap().len(), 0); + assert_eq!(state.worker(&worker.worker_ref).unwrap().detail(), detail); + } + #[test] fn protocol_shutdown_transitions_worker_to_stopped() { let runtime = runtime_with_backend(); @@ -8550,11 +11526,19 @@ mod tests { assert!(matches!( runtime.send_input(&detail.worker_ref, WorkerInput::user("do not wake")), - Err(RuntimeError::WorkerExecutionUnavailable { .. }) + Err(RuntimeError::WorkerExecutionRejected { + outcome: crate::execution::WorkerExecutionOutcome::Rejected, + .. + }) )); assert_eq!(*backend.restore_count.lock().unwrap(), 0); - runtime.restore_worker(&detail.worker_ref).unwrap(); + runtime + .restore_worker( + &detail.worker_ref, + runtime.test_restore_request(&detail.worker_ref), + ) + .unwrap(); runtime .send_input(&detail.worker_ref, WorkerInput::user("wake up")) .unwrap(); @@ -8600,7 +11584,12 @@ mod tests { runtime .stop_worker(&detail.worker_ref, Some("restore test".to_string())) .unwrap(); - runtime.restore_worker(&detail.worker_ref).unwrap(); + runtime + .restore_worker( + &detail.worker_ref, + runtime.test_restore_request(&detail.worker_ref), + ) + .unwrap(); assert_eq!( backend.dispatched_inputs.lock().unwrap().len(), @@ -9204,7 +12193,7 @@ mod tests { #[cfg(feature = "fs-store")] #[test] - fn fs_store_restores_active_worker_execution_handles() { + fn fs_store_restores_active_worker_executions() { let root = fs_store_root("execution-restore"); let runtime = Runtime::with_fs_store_and_execution_backend( crate::fs_store::FsRuntimeStoreOptions { @@ -9327,6 +12316,124 @@ mod tests { let _ = std::fs::remove_dir_all(root); } + #[cfg(feature = "fs-store")] + #[test] + fn active_startup_preflight_rejection_is_observable_bounded_and_stop_reconciles_absence() { + let dir = tempfile::tempdir().unwrap(); + let options = crate::fs_store::FsRuntimeStoreOptions { + root: dir.path().join("runtime"), + runtime_id: "active-preflight-rejection".to_string(), + display_name: None, + }; + let runtime = Runtime::with_fs_store_and_execution_backend( + options.clone(), + Arc::new(TestExecutionBackend::default()), + ) + .unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("saved active preflight rejection")) + .unwrap(); + drop(runtime); + let backend = Arc::new(TestExecutionBackend::default()); + let reason = format!( + "saved Manifest attachment mismatch token credential-value /private/path {}", + "details ".repeat(100) + ); + *backend.preflight_restore_result.lock().unwrap() = Some(WorkerExecutionResult::rejected( + WorkerExecutionOperation::Restore, + reason, + )); + let runtime = + Runtime::with_fs_store_and_execution_backend(options, backend.clone()).unwrap(); + let detail = runtime.worker_detail(&worker.worker_ref).unwrap(); + assert_eq!(detail.status, WorkerStatus::Idle); + assert!(detail.worker_state.is_none()); + assert_eq!(*backend.restore_count.lock().unwrap(), 0); + let diagnostics = runtime.diagnostics().unwrap(); + let diagnostic = diagnostics + .iter() + .find(|entry| entry.worker_ref.as_ref() == Some(&worker.worker_ref)) + .unwrap(); + assert!( + diagnostic + .message + .starts_with("saved Manifest attachment mismatch") + ); + assert!(!diagnostic.message.contains("credential-value")); + assert!(!diagnostic.message.contains("/private/path")); + assert!(diagnostic.message.chars().count() <= WORKER_CREATE_FAILURE_MESSAGE_LIMIT + 1); + let before = std::fs::read( + dir.path() + .join("runtime/workers") + .join(worker.worker_id.to_string()) + .join("worker.json"), + ) + .unwrap(); + assert_eq!( + runtime + .restore_worker_operation( + &worker.worker_ref, + runtime.test_restore_request(&worker.worker_ref) + ) + .unwrap() + .state, + WorkerRestoreState::Rejected + ); + let mut before: serde_json::Value = serde_json::from_slice(&before).unwrap(); + let mut after: serde_json::Value = serde_json::from_slice( + &std::fs::read( + dir.path() + .join("runtime/workers") + .join(worker.worker_id.to_string()) + .join("worker.json"), + ) + .unwrap(), + ) + .unwrap(); + let old_guard = before + .as_object_mut() + .unwrap() + .remove("restore_guard") + .unwrap(); + let guard = after + .as_object_mut() + .unwrap() + .remove("restore_guard") + .unwrap(); + assert_ne!(guard["generation"], old_guard["generation"]); + assert!(guard["active_request_id"].is_null()); + assert_eq!( + guard["owners"] + .as_object() + .unwrap() + .values() + .next() + .unwrap()["receipt"]["state"], + "rejected" + ); + assert_eq!( + after, before, + "preflight changes only durable intent/receipt metadata, never execution" + ); + assert!(matches!( + runtime.send_input( + &worker.worker_ref, + WorkerInput::notify("no implicit restore") + ), + Err(RuntimeError::WorkerExecutionRejected { + outcome: crate::execution::WorkerExecutionOutcome::Rejected, + .. + }) + )); + let stopped = runtime.stop_worker(&worker.worker_ref, None).unwrap(); + assert_eq!(stopped.status, WorkerStatus::Stopped); + assert!(stopped.worker_state.is_none()); + assert_eq!(*backend.restore_count.lock().unwrap(), 0); + assert_eq!(backend.stop_operation_ids.lock().unwrap().len(), 1); + assert!(runtime.delete_worker(&worker.worker_ref).unwrap().deleted); + } + #[cfg(feature = "fs-store")] #[test] fn fs_store_explicitly_retries_rejected_automatic_restore() { @@ -9374,7 +12481,10 @@ mod tests { restoring_backend.restore_result.lock().unwrap().take(); let retry = restored - .restore_worker_operation(&worker.worker_ref) + .restore_worker_operation( + &worker.worker_ref, + restored.test_restore_request(&worker.worker_ref), + ) .unwrap(); assert_eq!(retry.state, WorkerRestoreState::Accepted); @@ -9409,7 +12519,10 @@ mod tests { )); let result = runtime - .restore_worker_operation(&worker.worker_ref) + .restore_worker_operation( + &worker.worker_ref, + runtime.test_restore_request(&worker.worker_ref), + ) .unwrap(); assert_eq!(result.state, WorkerRestoreState::RolledBack); @@ -9426,7 +12539,10 @@ mod tests { backend.restore_result.lock().unwrap().take(); let retry = runtime - .restore_worker_operation(&worker.worker_ref) + .restore_worker_operation( + &worker.worker_ref, + runtime.test_restore_request(&worker.worker_ref), + ) .unwrap(); assert_eq!(retry.state, WorkerRestoreState::Accepted); assert_eq!(*backend.restore_count.lock().unwrap(), 2); @@ -9451,13 +12567,41 @@ mod tests { )); let result = runtime - .restore_worker_operation(&created.worker_ref) + .restore_worker_operation( + &created.worker_ref, + runtime.test_restore_request(&created.worker_ref), + ) .unwrap(); assert_eq!(result.state, WorkerRestoreState::Rejected); assert!(result.worker.is_none()); assert_eq!(*backend.restore_count.lock().unwrap(), 0); - assert_eq!(runtime.worker_detail(&created.worker_ref).unwrap(), before); + let mut after = runtime.worker_detail(&created.worker_ref).unwrap(); + assert_ne!( + after.restore_observation_token, + before.restore_observation_token + ); + after.restore_observation_token = before.restore_observation_token.clone(); + assert_eq!( + after, before, + "admission may rotate the fence but must not change execution state" + ); + let state = runtime.lock().unwrap(); + let record = state.worker(&created.worker_ref).unwrap(); + assert!(record.pending_restore.is_none()); + assert!(record.restore_guard.active_request_id.is_none()); + assert_eq!(record.restore_guard.owners.len(), 1); + assert_eq!( + record + .restore_guard + .owners + .values() + .next() + .unwrap() + .receipt + .as_ref(), + Some(&result) + ); } #[test] @@ -9474,14 +12618,16 @@ mod tests { WorkerExecutionOperation::Restore, "controller handshake outcome was not observable", ), - handle: None, worker_state: None, workdir_attachments: Vec::new(), }, ); let result = runtime - .restore_worker_operation(&created.worker_ref) + .restore_worker_operation( + &created.worker_ref, + runtime.test_restore_request(&created.worker_ref), + ) .unwrap(); assert_eq!(result.state, WorkerRestoreState::ReconciliationRequired); @@ -9511,7 +12657,10 @@ mod tests { ); backend.restore_result.lock().unwrap().take(); let retry = runtime - .restore_worker_operation(&created.worker_ref) + .restore_worker_operation( + &created.worker_ref, + runtime.test_restore_request(&created.worker_ref), + ) .unwrap(); assert_eq!(retry.state, WorkerRestoreState::Accepted); assert_eq!(*backend.restore_count.lock().unwrap(), 2); @@ -9641,7 +12790,10 @@ mod tests { ); let restored_detail = restored - .restore_worker(&restorable.worker_ref) + .restore_worker( + &restorable.worker_ref, + restored.test_restore_request(&restorable.worker_ref), + ) .expect("explicit restore reconstructs execution metadata"); assert!(restored_detail.execution_metadata_available); restored @@ -9743,11 +12895,19 @@ mod tests { ); assert!(matches!( restored.send_input(&worker.worker_ref, WorkerInput::user("implicit restore")), - Err(RuntimeError::WorkerExecutionUnavailable { .. }) + Err(RuntimeError::WorkerExecutionRejected { + outcome: crate::execution::WorkerExecutionOutcome::Rejected, + .. + }) )); assert_eq!(*backend.restore_count.lock().unwrap(), 0); - restored.restore_worker(&worker.worker_ref).unwrap(); + restored + .restore_worker( + &worker.worker_ref, + restored.test_restore_request(&worker.worker_ref), + ) + .unwrap(); assert_eq!(*backend.restore_count.lock().unwrap(), 1); assert_eq!( restored.worker_detail(&worker.worker_ref).unwrap().status, @@ -9794,6 +12954,10 @@ mod tests { serde_json::from_slice(&std::fs::read(&worker_path).expect("worker aggregate")) .expect("worker aggregate json"); worker_identity["schema_version"] = serde_json::json!(7); + worker_identity + .as_object_mut() + .unwrap() + .remove("restore_guard"); let mut worker_execution = worker_identity .as_object_mut() .expect("worker aggregate object") @@ -9881,11 +13045,17 @@ mod tests { runtime_store(&runtime) .fail_next_worker_write(crate::fs_store::AtomicWriteFault::AfterRename); - let result = runtime - .restore_worker_operation(&worker.worker_ref) - .unwrap(); + let result = runtime.begin_restore_operation( + &worker.worker_ref, + WorkerLifecycleOperationId::new(), + WorkerRestoreMode::Explicit, + runtime.restore_candidate_execution_context(worker.worker_ref.clone()), + ); - assert_eq!(result.state, WorkerRestoreState::ReconciliationRequired); + assert!(matches!( + result, + Err(RuntimeError::StoreCommitOutcomeUnknown { .. }) + )); assert_eq!(*backend.restore_count.lock().unwrap(), 0); assert_eq!(*backend.stop_count.lock().unwrap(), 1); assert_eq!( @@ -9939,7 +13109,10 @@ mod tests { ) .unwrap(); let retry = restarted - .restore_worker_operation(&worker.worker_ref) + .restore_worker_operation( + &worker.worker_ref, + restarted.test_restore_request(&worker.worker_ref), + ) .unwrap(); assert_eq!(retry.state, WorkerRestoreState::Accepted); assert_eq!(*restarted_backend.restore_count.lock().unwrap(), 1); @@ -10014,8 +13187,14 @@ mod tests { .unwrap() .operation_id; restoring_backend.restore_result.lock().unwrap().take(); + // An explicit observed recovery may bind a receipt to an unowned + // startup pending operation, but must preserve its lifecycle ID. It + // cannot allocate a new execution operation or overwrite preparation. let retry = restored - .restore_worker_operation(&worker.worker_ref) + .restore_worker_operation( + &worker.worker_ref, + restored.test_restore_request(&worker.worker_ref), + ) .unwrap(); assert_eq!(retry.state, WorkerRestoreState::Accepted); assert_eq!(*restoring_backend.restore_count.lock().unwrap(), 2); diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index 293a2ea5f..d338848a8 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -2,7 +2,7 @@ //! `worker` crate controller/run lifecycle. //! //! The adapter intentionally owns real `WorkerHandle`s internally and exposes -//! only the opaque `worker-runtime` execution handle to callers. Browser/API +//! operations addressed by `WorkerRef` to callers. Browser/API //! projections therefore keep the existing runtime redaction boundary: no raw //! socket paths, session paths, manifests, credentials, or handles leave this //! module. @@ -22,11 +22,13 @@ use crate::catalog::{ WorkingDirectoryStatusKind, }; use crate::config_bundle::{ConfigBundle, workspace_config_etag}; +#[cfg(feature = "ws-server")] +use crate::execution::WorkerProtocolTransport; use crate::execution::{ - WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation, - WorkerExecutionRestoreRequest, WorkerExecutionResult, WorkerExecutionSpawnRequest, - WorkerExecutionSpawnResult, WorkerExecutionStopRequest, WorkerSessionObservationRequest, - WorkspaceConfigFetchRequest, WorkspaceConfigFetchResult, + WorkerExecutionBackend, WorkerExecutionOperation, WorkerExecutionRestoreRequest, + WorkerExecutionResult, WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult, + WorkerExecutionStopRequest, WorkerSessionObservationRequest, WorkspaceConfigFetchRequest, + WorkspaceConfigFetchResult, }; use crate::identity::WorkerRef; use crate::interaction::{WorkerInput, WorkerInputKind}; @@ -83,7 +85,6 @@ fn rollback_materialized_spawn_workdirs( cleanup_failures.join("; ") ), ), - handle: None, worker_state: None, workdir_attachments: uncertain_attachments, }) @@ -96,6 +97,12 @@ fn next_internal_command( .read() .map_err(|_| "worker state lock is poisoned".to_string())? .clone(); + Ok(next_internal_command_for_snapshot(&snapshot)) +} + +fn next_internal_command_for_snapshot( + snapshot: &protocol::WorkerStateSnapshot, +) -> WorkerCommandEnvelope { let floor = snapshot.last_command_id.saturating_add(1); let command_id = NEXT_INTERNAL_COMMAND_ID .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| { @@ -103,7 +110,7 @@ fn next_internal_command( }) .unwrap_or(floor) .max(floor); - Ok(WorkerCommandEnvelope::new(command_id)) + WorkerCommandEnvelope::new(command_id) } use session_store::{ CombinedStore, Store, WorkerAggregateStore, WorkerMetadataStore, WorkerSessionStore, @@ -214,6 +221,23 @@ pub trait RuntimeWorkerFactory: Send + Sync + 'static { request: WorkerExecutionRestoreRequest, ) -> Result; + /// Prove cleanup of a restore future that completed without returning a + /// controller. An error is not absence evidence; unsupported factories keep + /// the operation-owned resource record for retry. + async fn cleanup_failed_restore( + &self, + _request: &WorkerExecutionRestoreRequest, + ) -> Result<(), String> { + Err("Runtime Worker factory cannot prove failed restore cleanup".to_string()) + } + + /// Verify/release factory-owned resources when the backend registry has no + /// Controller. Factories retaining resources outside returned controllers + /// must override this; it must never start a new Controller. + async fn reconcile_stopped_worker(&self, _worker_ref: &WorkerRef) -> Result<(), String> { + Ok(()) + } + fn activate_restored_controller( &self, _worker_ref: &WorkerRef, @@ -418,6 +442,17 @@ pub struct ProfileRuntimeWorkerFactory { workspace_request_clients: Arc>, embedded_worker_mutation_dispatcher: Option>, controller_transport: WorkerControllerTransport, + failed_restore_sessions: Arc< + Mutex< + HashMap< + WorkerRef, + ( + crate::execution::WorkerLifecycleOperationId, + Arc, + ), + >, + >, + >, } impl ProfileRuntimeWorkerFactory { @@ -434,6 +469,7 @@ impl ProfileRuntimeWorkerFactory { workspace_request_clients: Arc::new(HashMap::new()), embedded_worker_mutation_dispatcher: None, controller_transport: WorkerControllerTransport::InProcess, + failed_restore_sessions: Arc::new(Mutex::new(HashMap::new())), } } @@ -516,6 +552,87 @@ impl ProfileRuntimeWorkerFactory { .join(format!("restore-{operation_id}"))) } + // Called only after the backend has excluded actual and pending executions + // under its per-Worker operation lock. Persisted run artifacts survive a + // process restart even when the in-memory failed-session map does not. + fn stopped_restore_run_dirs(&self, worker_ref: &WorkerRef) -> Result, String> { + let aggregate = self.worker_aggregate_dir(worker_ref)?; + let root = aggregate + .parent() + .expect("Worker aggregate has configured parent"); + let runs = aggregate.join("runs"); + for path in [root, aggregate.as_path(), runs.as_path()] { + match std::fs::symlink_metadata(path) { + Ok(metadata) if metadata.is_dir() && !metadata.file_type().is_symlink() => {} + Ok(_) => { + return Err(format!( + "unsafe restore artifact directory: {}", + path.display() + )); + } + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(error) => { + return Err(format!( + "inspect restore artifact directory {}: {error}", + path.display() + )); + } + } + } + let mut candidates = Vec::new(); + for entry in std::fs::read_dir(&runs) + .map_err(|error| format!("read restore artifact directory: {error}"))? + { + let entry = entry.map_err(|error| format!("read restore artifact entry: {error}"))?; + let name = entry.file_name(); + let name = name + .to_str() + .ok_or_else(|| "invalid restore artifact run name".to_string())?; + let Some(operation) = name.strip_prefix("restore-") else { + continue; + }; + let id = uuid::Uuid::parse_str(operation) + .map_err(|_| format!("invalid restore artifact operation name: {name}"))?; + if id.to_string() != operation { + return Err(format!( + "noncanonical restore artifact operation name: {name}" + )); + } + let path = entry.path(); + Self::validate_restore_artifact_tree(&path, true)?; + candidates.push(path); + } + candidates.sort(); + Ok(candidates) + } + + fn validate_restore_artifact_tree(path: &Path, require_directory: bool) -> Result<(), String> { + let metadata = std::fs::symlink_metadata(path) + .map_err(|error| format!("inspect restore artifact {}: {error}", path.display()))?; + if metadata.file_type().is_symlink() || (require_directory && !metadata.is_dir()) { + return Err(format!("unsafe restore artifact: {}", path.display())); + } + if metadata.is_dir() { + for entry in std::fs::read_dir(path) + .map_err(|error| format!("read restore artifact {}: {error}", path.display()))? + { + let entry = entry.map_err(|error| format!("read restore artifact: {error}"))?; + Self::validate_restore_artifact_tree(&entry.path(), false)?; + } + } + Ok(()) + } + + fn remove_restore_run_artifacts(run_dir: &Path) -> Result<(), String> { + match std::fs::symlink_metadata(run_dir) { + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(format!("inspect restore run artifacts: {error}")), + Ok(_) => Self::validate_restore_artifact_tree(run_dir, true)?, + } + std::fs::remove_dir_all(run_dir) + .map_err(|error| format!("failed to clean restore run artifacts: {error}")) + } + fn runtime_worker_name_for_ref(worker_ref: &crate::identity::WorkerRef) -> String { format!("worker-runtime-{}", worker_ref.worker_id) } @@ -558,6 +675,25 @@ impl ProfileRuntimeWorkerFactory { .map_err(|err| format!("failed to build restore fallback manifest: {err}"))?; Ok((manifest, PromptCatalogSource::builtins_only())) } + fn manifest_for_restore( + metadata: &session_store::WorkerMetadata, + request: &CreateWorkerRequest, + ) -> Result<(manifest::WorkerManifest, PromptCatalogSource), String> { + let (fallback, loader) = Self::restore_fallback_manifest(&metadata.worker_name)?; + let manifest = match metadata.resolved_manifest_snapshot.clone() { + Some(snapshot) => manifest::read_persisted_worker_manifest_snapshot(snapshot) + .map_err(|error| format!("failed to read saved Worker Manifest: {error}"))?, + None if request.workspace_api.is_some() || request.subjektiv_attached => { + return Err("Workspace Worker metadata has no saved Manifest; replacement Worker is required".to_string()); + } + None => fallback, + }; + // Saved policy is execution authority. Rebinding current settings here + // would hide revision and attachment mismatches. + validate_worker_memory_settings(&manifest, request)?; + Ok((manifest, loader)) + } + fn observe_bundle_prompt_projection( &self, bundle: &crate::config_bundle::ConfigBundle, @@ -950,10 +1086,26 @@ fn restored_workdir_router( Ok(router) } +fn validate_backend_job_profile( + manifest: &manifest::WorkerManifest, + request: &CreateWorkerRequest, +) -> Result<(), String> { + if let Some(binding) = &request.backend_job { + let policy = &manifest.feature.subjektiv.profile; + if policy.consolidation_tools != binding.subjektiv_consolidation + || (binding.subjektiv_consolidation && policy.extraction.enabled) + { + return Err("Backend Job consolidation_tools must match its trusted consolidation grant and extraction must be disabled".into()); + } + } + Ok(()) +} + fn bind_workspace_memory_settings( manifest: &mut manifest::WorkerManifest, request: &CreateWorkerRequest, ) -> Result<(), String> { + validate_backend_job_profile(manifest, request)?; let Some(snapshot) = request.memory_settings.as_ref() else { if request.workspace_api.is_some() || request.subjektiv_attached { return Err( @@ -1007,15 +1159,20 @@ fn validate_worker_memory_settings( manifest: &manifest::WorkerManifest, request: &CreateWorkerRequest, ) -> Result<(), String> { + validate_backend_job_profile(manifest, request)?; let Some(expected) = request.memory_settings.as_ref() else { - if request.subjektiv_attached { + if request.subjektiv_attached || request.workspace_api.is_some() { return Err( - "subject-attached Worker restore is missing its trusted settings snapshot" - .to_string(), + "Workspace Worker restore is missing its trusted settings snapshot".to_string(), ); } return Ok(()); }; + if let Some(workspace_api) = request.workspace_api.as_ref() + && expected.workspace_id != workspace_api.workspace_id + { + return Err("trusted Memory settings do not match Workspace API scope".to_string()); + } manifest .feature .memory @@ -1279,6 +1436,11 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { } })?; let worker = prepared.worker_mut(); + if let Some(binding) = request.request.backend_job.as_ref() { + worker + .bind_backend_job(binding.clone()) + .map_err(str::to_string)?; + } validate_worker_memory_settings(worker.manifest(), &request.request)?; worker .finalize_subjektiv_session_attribution( @@ -1368,8 +1530,6 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { request: &WorkerExecutionRestoreRequest, ) -> Result<(), String> { let worker_name = Self::runtime_worker_name_for_ref(&request.worker_ref); - let (mut manifest, _) = Self::restore_fallback_manifest(&worker_name)?; - bind_workspace_memory_settings(&mut manifest, &request.request)?; let worker_aggregate_dir = self.worker_aggregate_dir(&request.worker_ref)?; if !worker_aggregate_dir.is_dir() { return Err("Persisted Worker aggregate metadata is unavailable".to_string()); @@ -1383,6 +1543,7 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { .read_by_name(&worker_name) .map_err(|error| format!("failed to read Worker metadata: {error}"))? .ok_or_else(|| "Persisted Worker metadata is unavailable".to_string())?; + let (manifest, loader) = Self::manifest_for_restore(&metadata, &request.request)?; if let Some(active) = metadata.active.as_ref() && let Some(segment_id) = active.segment_id { @@ -1398,17 +1559,35 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { .read_all(active.session_id, segment_id) .map_err(|error| format!("failed to read Worker Session segment: {error}"))?; } - if request.request.workspace_api.is_some() + if let Some(api) = request.request.workspace_api.as_ref() && metadata .active .as_ref() .is_some_and(|active| active.segment_id.is_none()) - && request.config_bundle.is_none() { - return Err( - "Pending Workspace Worker restore requires current profile launch authority" - .to_string(), - ); + let bundle = request.config_bundle.as_ref().ok_or_else(|| { + "Pending Workspace Worker restore requires operation-owned launch material" + .to_string() + })?; + crate::config_bundle::validate_config_bundle(bundle) + .map_err(|error| format!("invalid pending Worker launch material: {error}"))?; + if bundle.metadata.workspace_id != api.workspace_id { + return Err(format!( + "Workspace Prompt projection scope mismatch: expected {}, got {}", + api.workspace_id, bundle.metadata.workspace_id + )); + } + let prompt_catalog = bundle.prompt_catalog.as_ref().ok_or_else(|| { + "pending Workspace Worker restore requires a saved Workspace Prompt projection" + .to_string() + })?; + // Validate the exact launch catalog without publishing it into the + // current projection cache or allocating live Worker resources. + worker::SystemPromptTemplate::parse( + &manifest.engine.instruction, + loader.with_effective_catalog(prompt_catalog.clone()), + ) + .map_err(|error| format!("invalid pending Worker launch Prompt: {error}"))?; } Ok(()) } @@ -1417,6 +1596,7 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { &self, request: WorkerExecutionRestoreRequest, ) -> Result { + self.preflight_restore(&request).await?; let worker_name = Self::runtime_worker_name_for_ref(&request.worker_ref); let only_binding = (request.workdir_attachments.len() == 1) .then(|| request.workdir_attachments.values().next()) @@ -1454,8 +1634,6 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { self.embedded_worker_mutation_dispatcher.as_ref(), Some(self.prompt_projection_cache.clone()), ); - let (mut manifest, loader) = Self::restore_fallback_manifest(&worker_name)?; - bind_workspace_memory_settings(&mut manifest, &request.request)?; let worker_aggregate_dir = self.worker_aggregate_dir(&request.worker_ref)?; let session_dir = worker_aggregate_dir.join("session"); @@ -1474,6 +1652,11 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { ) }, )?; + let metadata = worker_metadata_store + .read_by_name(&worker_name) + .map_err(|error| format!("failed to read Worker metadata: {error}"))? + .ok_or_else(|| "Persisted Worker metadata is unavailable".to_string())?; + let (manifest, loader) = Self::manifest_for_restore(&metadata, &request.request)?; let store = CombinedStore::new(session_store, worker_metadata_store); let mut worker = match Worker::restore_from_worker_metadata_with_context( @@ -1523,6 +1706,11 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { } Err(err) => return Err(format!("failed to restore Worker from metadata: {err}")), }; + if let Some(binding) = request.request.backend_job.as_ref() { + worker + .bind_backend_job(binding.clone()) + .map_err(str::to_string)?; + } validate_worker_memory_settings(worker.manifest(), &request.request)?; worker .finalize_subjektiv_session_attribution( @@ -1573,6 +1761,7 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { let workspace_client = worker.workspace_client_handle(); let run_dir = self.worker_restore_run_dir(&request.worker_ref, request.operation_id)?; let bash_output_dir = bash_output_dir_for_worker_id(&request.worker_ref.worker_id); + let cleanup_sessions = worker.workdir_sessions(); let started = PreparedWorker::new( worker, WorkerBootstrapLayout::RuntimeManagedRun { @@ -1587,10 +1776,24 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { WorkerBootstrapError::Worker(source) => { format!("failed to prepare restored Worker: {source}") } - WorkerBootstrapError::Controller { source, .. } => format!( - "failed to spawn restored Worker controller in {}: {source}", - run_dir.display() - ), + WorkerBootstrapError::Controller { + source, + cleanup_failed, + } => { + if cleanup_failed { + self.failed_restore_sessions + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert( + request.worker_ref.clone(), + (request.operation_id, cleanup_sessions), + ); + } + format!( + "failed to spawn restored Worker controller in {}: {source}", + run_dir.display() + ) + } })?; let (handle, shutdown_rx, controller_task) = (started.handle, started.shutdown, started.controller_task); @@ -1605,6 +1808,66 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { }) } + async fn cleanup_failed_restore( + &self, + request: &WorkerExecutionRestoreRequest, + ) -> Result<(), String> { + let retained = self + .failed_restore_sessions + .lock() + .map_err(|_| "failed restore sessions lock is poisoned".to_string())? + .get(&request.worker_ref) + .cloned(); + if let Some((operation_id, sessions)) = retained { + if operation_id != request.operation_id { + return Err("failed restore sessions belong to another operation".to_string()); + } + sessions + .close_all() + .await + .map_err(|error| format!("failed restore session cleanup: {error}"))?; + } + // restore_controller starts the Controller only as its final fallible + // step. A completed error leaves no live Controller; remove only this + // operation's disposable run artifacts, never Session/metadata authority. + self.stopped_restore_run_dirs(&request.worker_ref)?; + let run_dir = self.worker_restore_run_dir(&request.worker_ref, request.operation_id)?; + Self::remove_restore_run_artifacts(&run_dir)?; + self.failed_restore_sessions + .lock() + .map_err(|_| "failed restore sessions lock is poisoned".to_string())? + .remove(&request.worker_ref); + Ok(()) + } + + async fn reconcile_stopped_worker(&self, worker_ref: &WorkerRef) -> Result<(), String> { + let retained = self + .failed_restore_sessions + .lock() + .map_err(|_| "failed restore sessions lock is poisoned".to_string())? + .get(worker_ref) + .cloned(); + if let Some((_, sessions)) = retained.as_ref() { + sessions + .close_all() + .await + .map_err(|error| format!("failed restore session cleanup: {error}"))?; + } + // A restart loses the map, not the operation-owned filesystem artifacts. + // Validate every candidate before deleting any, and retain map/journal + // retry authority whenever validation or cleanup remains unproven. + for run_dir in self.stopped_restore_run_dirs(worker_ref)? { + Self::remove_restore_run_artifacts(&run_dir)?; + } + if retained.is_some() { + self.failed_restore_sessions + .lock() + .map_err(|_| "failed restore sessions lock is poisoned".to_string())? + .remove(worker_ref); + } + Ok(()) + } + fn activate_restored_controller( &self, worker_ref: &WorkerRef, @@ -1618,23 +1881,167 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { #[derive(Clone)] struct RuntimeExecutionTaskScope { - tasks: Arc>>, + tasks: Arc>>>, + shutdown_admission: Arc>>, + // Actual unconnected Controllers whose cleanup failed, owned by this + // execution's structured scope. No WorkerRef index or presence mirror. + cleanup_candidates: Arc>>>, +} + +// Evidence for one exact command on this actual Controller. Keep the send task +// and receiver across bounded waits: unknown admission never authorizes resend. +struct RuntimeShutdownAdmission { + command_id: u64, + send: Option>>, + events: tokio::sync::broadcast::Receiver, + acknowledgement: Option, + uncertainty: Option, +} + +impl RuntimeShutdownAdmission { + async fn wait(&mut self, timeout: Duration) -> Result<(), String> { + let result = tokio::time::timeout(timeout, async { + if let Some(send) = self.send.as_mut() { + match send.await { + Ok(Ok(())) => { self.send.take(); } + Ok(Err(message)) => { + self.send.take(); + self.uncertainty = Some(message.clone()); + return Err(message); + } + Err(error) => { + self.send.take(); + let message = format!("Shutdown send task failed: {error}; admission is unknown"); + self.uncertainty = Some(message.clone()); + return Err(message); + } + } + } + if self.acknowledgement.is_some() { + return Ok(()); + } + loop { + match self.events.recv().await { + Ok(Event::CommandAcknowledged { acknowledgement }) + if acknowledgement.command_id == self.command_id + && acknowledgement.command == protocol::WorkerCommandKind::Shutdown => { + self.acknowledgement = Some(acknowledgement); + self.uncertainty = None; + return Ok(()); + } + Ok(_) => {} + Err(error) => { + let message = format!("Shutdown command {} acknowledgement unavailable: {error}; admission is unknown", self.command_id); + self.uncertainty = Some(message.clone()); + return Err(message); + } + } + } + }).await; + match result { + Ok(result) => result, + Err(_) => { + let message = format!( + "Shutdown command {} acknowledgement timed out; admission is unknown and waiter is retained", + self.command_id + ); + self.uncertainty = Some(message.clone()); + Err(message) + } + } + } } struct RuntimeExecutionTask { name: &'static str, - task: tokio::task::JoinHandle<()>, + completion: tokio::sync::Mutex, abort_before_join: bool, } +// The scope retains the actual handle while awaiting it and retains its abnormal +// completion after it has been consumed. Neither cancellation of a cleanup +// future nor another Stop attempt can turn an unknown cleanup into an empty scope. +struct RuntimeExecutionTaskCompletion { + task: Option>, + failure: Option, +} + impl RuntimeExecutionTaskScope { fn new(controller_task: tokio::task::JoinHandle<()>) -> Self { - Self { - tasks: Arc::new(Mutex::new(vec![RuntimeExecutionTask { - name: "controller", - task: controller_task, - abort_before_join: false, - }])), + let scope = Self { + tasks: Arc::new(Mutex::new(Vec::new())), + shutdown_admission: Arc::new(tokio::sync::Mutex::new(None)), + cleanup_candidates: Arc::new(Mutex::new(Vec::new())), + }; + scope.push("controller", controller_task, false); + scope + } + + fn has_cleanup_candidates(&self) -> Result { + self.cleanup_candidates + .lock() + .map(|candidates| !candidates.is_empty()) + .map_err(|_| "execution cleanup candidate lock is poisoned".to_string()) + } + + async fn request_shutdown( + &self, + handle: &WorkerHandle, + shutdown_requested: &AtomicBool, + timeout: Duration, + ) -> Result<(), String> { + let mut retained = self.shutdown_admission.lock().await; + if handle.protocol_is_closed() { + // Endpoint closure alone is not cleanup proof. Drain our send task + // too, then let the existing completion + all-task join barrier prove it. + if let Some(pending) = retained.as_mut() { + if let Some(send) = pending.send.as_mut() { + let _ = tokio::time::timeout(timeout, send).await.map_err(|_| { + "Shutdown send task still pending on closed endpoint".to_string() + })?; + pending.send.take(); + } + } + shutdown_requested.store(true, Ordering::Release); + return Ok(()); + } + if retained.is_none() { + if shutdown_requested.load(Ordering::Acquire) { + return Err("Shutdown admission evidence missing; cleanup is unproven".to_string()); + } + // The bridge cache can lag admission of already queued commands. + // Rejection retries use the real Controller snapshot, never that cache. + let command = next_internal_command_for_snapshot(&handle.shared_state.snapshot()); + let events = handle.subscribe(); + let worker = handle.clone(); + *retained = Some(RuntimeShutdownAdmission { + command_id: command.command_id, + send: Some(tokio::spawn(async move { + worker + .send(Method::Shutdown { command }) + .await + .map_err(|error| format!("failed to send Shutdown: {error}")) + })), + events, + acknowledgement: None, + uncertainty: None, + }); + shutdown_requested.store(true, Ordering::Release); + } + let pending = retained.as_mut().unwrap(); + pending.wait(timeout).await?; + let acknowledgement = pending.acknowledgement.as_ref().unwrap(); + match acknowledgement.disposition { + protocol::WorkerCommandDisposition::Accepted => Ok(()), + disposition => { + let message = format!( + "Shutdown command {} rejected: {disposition:?}; stop remains retryable", + acknowledgement.command_id + ); + retained.take(); + shutdown_requested.store(false, Ordering::Release); + Err(message) + } } } @@ -1643,70 +2050,128 @@ impl RuntimeExecutionTaskScope { Ok(tasks) => tasks, Err(poisoned) => poisoned.into_inner(), }; - tasks.push(RuntimeExecutionTask { + tasks.push(Arc::new(RuntimeExecutionTask { name, - task, + completion: tokio::sync::Mutex::new(RuntimeExecutionTaskCompletion { + task: Some(task), + failure: None, + }), abort_before_join, - }); + })); } - fn abort_all(&self) { - let mut tasks = match self.tasks.lock() { - Ok(tasks) => tasks, - Err(poisoned) => poisoned.into_inner(), + fn retain_completion_failure(&self, message: String) -> Result<(), String> { + self.tasks + .lock() + .map_err(|_| "execution task registry lock is poisoned".to_string())? + .push(Arc::new(RuntimeExecutionTask { + name: "controller shutdown completion", + completion: tokio::sync::Mutex::new(RuntimeExecutionTaskCompletion { + task: None, + failure: Some(message), + }), + abort_before_join: false, + })); + Ok(()) + } + + async fn confirm_shutdown_and_join( + &self, + shutdown: &Arc>>, + ) -> Result<(), String> { + // Do not consume a completion receiver if its failure cannot be retained. + drop( + self.tasks + .lock() + .map_err(|_| "execution task registry lock is poisoned".to_string())?, + ); + let confirmation = { + let mut guard = shutdown.lock().await; + if let Some(receiver) = guard.as_mut() { + match tokio::time::timeout(Duration::from_secs(5), receiver).await { + Ok(Ok(())) => { + guard.take(); + Ok(()) + } + Ok(Err(_)) => { + let message = + "Worker shutdown completion channel closed; cleanup is unproven" + .to_string(); + self.retain_completion_failure(message.clone())?; + guard.take(); + Err(message) + } + Err(_) => Err( + "Worker shutdown confirmation timed out; stop remains retryable" + .to_string(), + ), + } + } else { + Ok(()) + } }; - for task in tasks.drain(..) { - task.task.abort(); - } + // Drain bridges/relays even after an abnormal completion, while keeping + // the failed completion and all timed-out handles in their owning scope. + self.join().await?; + confirmation } async fn join(&self) -> Result<(), String> { + let tasks = self + .tasks + .lock() + .map_err(|_| "execution task registry lock is poisoned".to_string())? + .clone(); let mut first_failure = None; - let mut pending = Vec::new(); - loop { - let next = { - let mut tasks = self - .tasks - .lock() - .map_err(|_| "execution task registry lock is poisoned".to_string())?; - if tasks.is_empty() { - None - } else { - Some(tasks.remove(0)) - } - }; - let Some(mut task) = next else { - break; - }; - let name = task.name; - if task.abort_before_join { - task.task.abort(); + for task in tasks { + let mut completion = task.completion.lock().await; + if let Some(message) = completion.failure.as_ref() { + first_failure.get_or_insert_with(|| message.clone()); + continue; } - match tokio::time::timeout(Duration::from_secs(5), &mut task.task).await { - Ok(Ok(())) => {} - Ok(Err(error)) if task.abort_before_join && error.is_cancelled() => {} - Ok(Err(error)) => { - first_failure.get_or_insert_with(|| { - format!("{name} task failed while stopping Worker: {error}") - }); + if let Some(handle) = completion.task.as_mut() { + if task.abort_before_join { + handle.abort(); } - Err(_) => { - first_failure.get_or_insert_with(|| { - format!("{name} task did not stop before timeout; stop remains retryable") - }); - pending.push(task); + match tokio::time::timeout(Duration::from_secs(5), handle).await { + Ok(Ok(())) => { + completion.task.take(); + } + Ok(Err(error)) if task.abort_before_join && error.is_cancelled() => { + completion.task.take(); + } + Ok(Err(error)) => { + let message = format!( + "{} task failed while stopping Worker: {error}; cleanup is unproven", + task.name + ); + completion.task.take(); + completion.failure = Some(message.clone()); + first_failure.get_or_insert(message); + continue; + } + Err(_) => { + first_failure.get_or_insert_with(|| { + format!( + "{} task did not stop before timeout; stop remains retryable", + task.name + ) + }); + continue; + } } } - } - if !pending.is_empty() { - let mut tasks = match self.tasks.lock() { - Ok(tasks) => tasks, - Err(poisoned) => poisoned.into_inner(), - }; - tasks.extend(pending); + drop(completion); + self.tasks + .lock() + .map_err(|_| "execution task registry lock is poisoned".to_string())? + .retain(|retained| !Arc::ptr_eq(retained, &task)); } match first_failure { Some(message) => Err(message), + None if self.has_cleanup_candidates()? => Err( + "unconnected Controller cleanup remains unproven; scope is retained".to_string(), + ), None => Ok(()), } } @@ -1725,14 +2190,59 @@ struct RuntimeWorkerExecution { workdir_attachments: Vec, } -/// `worker-runtime` execution backend backed by real `worker` crate Workers. +// Actual operation-owned resources, not a second Worker catalog or presence +// marker. Timeout never drops the factory future or a late Controller result. +struct PendingRuntimeRestore { + request: WorkerExecutionRestoreRequest, + task: tokio::sync::Mutex>>, + result: Mutex>>, + failure: Mutex>, +} + +impl PendingRuntimeRestore { + async fn wait(&self, timeout: Duration) -> Result<(), String> { + let mut task = self.task.lock().await; + if let Some(message) = self + .failure + .lock() + .map_err(|_| "restore factory failure lock is poisoned".to_string())? + .as_ref() + { + return Err(message.clone()); + } + if let Some(handle) = task.as_mut() { + match tokio::time::timeout(timeout, handle).await { + Ok(Ok(())) => { + task.take(); + } + Ok(Err(error)) => { + task.take(); + let message = + format!("restore factory task failed: {error}; cleanup is unproven"); + *self + .failure + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) = Some(message.clone()); + return Err(message); + } + Err(_) => { + return Err("restore factory is still pending; cleanup is unproven".to_string()); + } + } + } + Ok(()) + } +} + +/// Worker execution backend backed by real worker crate Workers. pub struct WorkerRuntimeExecutionBackend { backend_id: String, factory: Arc, working_directory_materializer: Option>, runtime: Mutex>, workers: Mutex>, - restore_lock: Mutex<()>, + pending_restores: Mutex>>, + worker_locks: Mutex>>>, spawn_restore_timeout: Duration, } @@ -1752,7 +2262,8 @@ where working_directory_materializer: None, runtime: Mutex::new(Some(runtime)), workers: Mutex::new(HashMap::new()), - restore_lock: Mutex::new(()), + pending_restores: Mutex::new(HashMap::new()), + worker_locks: Mutex::new(HashMap::new()), spawn_restore_timeout: SPAWN_RESTORE_TASK_TIMEOUT, }) } @@ -1776,6 +2287,17 @@ where self } + fn worker_lock(&self, worker_ref: &WorkerRef) -> Result>, String> { + let mut locks = self + .worker_locks + .lock() + .map_err(|_| "worker operation lock registry is poisoned".to_string())?; + Ok(locks + .entry(worker_ref.clone()) + .or_insert_with(|| Arc::new(Mutex::new(()))) + .clone()) + } + fn wait_for_runtime_task(receiver: mpsc::Receiver>) -> Result { receiver .recv_timeout(RUNTIME_TASK_TIMEOUT) @@ -1816,6 +2338,26 @@ where Self::wait_for_runtime_task(rx) } + // Cleanup contains its own bounded waits. Do not return while an outer + // waiter still owns removed task handles: a retry must see complete scope + // ownership, rather than mistake an in-flight join for an empty scope. + fn run_joined_on_adapter_runtime(&self, task: Fut) -> Result + where + T: Send + 'static, + Fut: Future> + Send + 'static, + { + let (tx, rx) = mpsc::sync_channel(1); + self.spawn_on_adapter_runtime(async move { + let result = match tokio::spawn(task).await { + Ok(result) => result, + Err(error) => Err(format!("worker adapter cleanup task failed: {error}")), + }; + let _ = tx.send(result); + })?; + rx.recv() + .map_err(|error| format!("worker adapter cleanup did not complete: {error}"))? + } + fn run_cancellable_on_adapter_runtime( &self, timeout: Duration, @@ -1852,9 +2394,38 @@ where .map_err(|err| format!("worker adapter task did not complete: {err}"))? } + fn pending_restore( + &self, + worker_ref: &WorkerRef, + ) -> Result>, String> { + self.pending_restores + .lock() + .map(|pending| pending.get(worker_ref).cloned()) + .map_err(|_| "pending restore resources lock is poisoned".to_string()) + } + + fn wait_pending_restore( + &self, + pending: &Arc, + timeout: Duration, + ) -> Result<(), String> { + let pending = Arc::clone(pending); + self.run_cancellable_on_adapter_runtime(timeout + Duration::from_secs(1), async move { + pending.wait(timeout).await + }) + } + + fn remove_pending_restore(&self, worker_ref: &WorkerRef) -> Result<(), String> { + self.pending_restores + .lock() + .map_err(|_| "pending restore resources lock is poisoned".to_string())? + .remove(worker_ref); + Ok(()) + } + fn get_execution( &self, - handle: &WorkerExecutionHandle, + worker_ref: &WorkerRef, ) -> Result< ( WorkerHandle, @@ -1863,16 +2434,6 @@ where ), WorkerExecutionResult, > { - if handle.backend_id() != self.backend_id() { - return Err(WorkerExecutionResult::rejected( - WorkerExecutionOperation::Input, - format!( - "execution handle belongs to backend {}, not {}", - handle.backend_id(), - self.backend_id() - ), - )); - } let workers = self.workers.lock().map_err(|_| { WorkerExecutionResult::errored( WorkerExecutionOperation::Input, @@ -1880,7 +2441,7 @@ where ) })?; workers - .get(handle.worker_ref()) + .get(worker_ref) .map(|execution| { ( execution.handle.clone(), @@ -1891,11 +2452,115 @@ where .ok_or_else(|| { WorkerExecutionResult::rejected( WorkerExecutionOperation::Input, - "execution handle does not reference a live Worker", + "WorkerRef does not reference a live Worker execution", ) }) } + #[cfg(feature = "ws-server")] + fn validate_protocol_execution_under_lock( + &self, + worker_ref: &WorkerRef, + execution: &RuntimeWorkerExecution, + ) -> Result<(), WorkerExecutionResult> { + if execution.shutdown_requested.load(Ordering::Acquire) + || execution.handle.protocol_is_closed() + { + return Err(WorkerExecutionResult::rejected( + WorkerExecutionOperation::ProtocolMethod, + "attached Worker protocol endpoint is closed", + )); + } + if self + .pending_restore(worker_ref) + .map_err(|message| { + WorkerExecutionResult::errored(WorkerExecutionOperation::ProtocolMethod, message) + })? + .is_some() + { + return Err(WorkerExecutionResult::busy( + WorkerExecutionOperation::ProtocolMethod, + "Worker restore still owns pending resources", + )); + } + let workers = self.workers.lock().map_err(|_| { + WorkerExecutionResult::errored( + WorkerExecutionOperation::ProtocolMethod, + "worker adapter registry lock is poisoned", + ) + })?; + if workers + .get(worker_ref) + .is_none_or(|current| !current.handle.same_controller(&execution.handle)) + { + return Err(WorkerExecutionResult::rejected( + WorkerExecutionOperation::ProtocolMethod, + "attached Worker protocol endpoint is no longer current", + )); + } + Ok(()) + } + + #[cfg(feature = "ws-server")] + fn dispatch_protocol_execution_under_lock( + &self, + worker_ref: &WorkerRef, + execution: &RuntimeWorkerExecution, + method: Method, + ) -> Result, WorkerExecutionResult> { + self.validate_protocol_execution_under_lock(worker_ref, execution)?; + match method { + Method::ListCompletions { + request_id, + kind, + prefix, + context, + } => { + let handle = execution.handle.clone(); + self.run_cancellable_on_adapter_runtime(USER_INPUT_TASK_TIMEOUT, async move { + let entries = handle + .completion_entries(kind, &prefix, context.as_ref()) + .await; + Ok(vec![Event::Completions { + request_id, + kind, + prefix, + context, + entries, + }]) + }) + .map_err(|message| { + WorkerExecutionResult::errored( + WorkerExecutionOperation::ProtocolMethod, + message, + ) + }) + } + Method::Shutdown { .. } => { + // Identity was checked against the captured Controller while + // this same operation lock is held. Never unlock/rebind here. + let result = self.stop_worker_under_lock(worker_ref); + if result.is_accepted() { + Ok(Vec::new()) + } else { + Err(result) + } + } + method => { + let result = self.send_method( + WorkerExecutionOperation::ProtocolMethod, + execution.handle.clone(), + method, + ); + if result.is_accepted() { + Ok(Vec::new()) + } else { + Err(result) + } + } + } + } + fn send_method( &self, operation: WorkerExecutionOperation, @@ -2058,40 +2723,45 @@ where worker_ref: &crate::identity::WorkerRef, handle: WorkerHandle, shutdown: Arc>>, + shutdown_requested: Arc, tasks: RuntimeExecutionTaskScope, worker_state: Arc>, workspace_client: Option>, restore_operation_id: Option, workdir_attachments: Vec, - ) -> Result { + ) { + // Retention must not return the candidate as an error and drop it. Even + // a poisoned registry keeps its existing actual resource ownership. let mut workers = self .workers .lock() - .map_err(|_| "worker adapter registry lock is poisoned".to_string())?; - if workers.contains_key(worker_ref) { - return Err("Worker already has a retained execution handle".to_string()); - } + .unwrap_or_else(std::sync::PoisonError::into_inner); let workspace_id = workspace_client .as_ref() .and_then(|client| client.workspace_id().map(str::to_string)); - workers.insert( - worker_ref.clone(), - RuntimeWorkerExecution { - handle, - shutdown, - shutdown_requested: Arc::new(AtomicBool::new(false)), - tasks, - worker_state, - workspace_client, - workspace_id, - restore_operation_id, - workdir_attachments, - }, - ); - Ok(WorkerExecutionHandle::new( - worker_ref.clone(), - self.backend_id(), - )) + let candidate = RuntimeWorkerExecution { + handle, + shutdown, + shutdown_requested, + tasks, + worker_state, + workspace_client, + workspace_id, + restore_operation_id, + workdir_attachments, + }; + if let Some(existing) = workers.get(worker_ref) { + // Keep the actual endpoint, admission waiter, shutdown receiver and + // all task handles without replacing the registered Controller. + existing + .tasks + .cleanup_candidates + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .push(Arc::new(candidate)); + } else { + workers.insert(worker_ref.clone(), candidate); + } } fn cleanup_unconnected_controller( @@ -2099,36 +2769,19 @@ where handle: &WorkerHandle, shutdown: &Arc>>, tasks: &RuntimeExecutionTaskScope, - worker_state: &Arc>, + shutdown_requested: &Arc, + _worker_state: &Arc>, ) -> Result<(), String> { - let command = next_internal_command(worker_state)?; let handle = handle.clone(); + let shutdown_requested = Arc::clone(shutdown_requested); let shutdown = shutdown.clone(); let tasks_for_join = tasks.clone(); - let cleanup = self.run_on_adapter_runtime(async move { - handle - .send(Method::Shutdown { command }) - .await - .map_err(|error| format!("failed to request controller cleanup: {error}"))?; - let mut guard = shutdown.lock().await; - if let Some(mut receiver) = guard.take() { - match tokio::time::timeout(Duration::from_secs(5), &mut receiver).await { - Ok(Ok(())) => {} - Ok(Err(_)) => { - return Err("controller cleanup completion channel closed".to_string()); - } - Err(_) => { - *guard = Some(receiver); - return Err("controller cleanup confirmation timed out".to_string()); - } - } - } - drop(guard); - tasks_for_join.join().await + let cleanup = self.run_joined_on_adapter_runtime(async move { + tasks_for_join + .request_shutdown(&handle, &shutdown_requested, Duration::from_secs(5)) + .await?; + tasks_for_join.confirm_shutdown_and_join(&shutdown).await }); - if cleanup.is_err() { - tasks.abort_all(); - } cleanup } @@ -2143,6 +2796,36 @@ where restore_operation_id: Option, workdir_attachments: BTreeMap, workspace_client: Option>, + ) -> WorkerExecutionSpawnResult { + self.connect_controller_scope( + operation, + worker_ref, + bridge_context, + handle, + shutdown, + RuntimeExecutionTaskScope::new(controller_task), + Arc::new(AtomicBool::new(false)), + restore_operation_id, + workdir_attachments, + workspace_client, + ) + } + + // Own the actual Controller scope before doing any fallible connection work. + // The split also permits testing duplicate cleanup with an already-pending + // real admission waiter, rather than relying on a scheduler race. + fn connect_controller_scope( + &self, + operation: WorkerExecutionOperation, + worker_ref: WorkerRef, + bridge_context: crate::execution::WorkerExecutionContext, + handle: WorkerHandle, + shutdown: Arc>>, + tasks: RuntimeExecutionTaskScope, + shutdown_requested: Arc, + restore_operation_id: Option, + workdir_attachments: BTreeMap, + workspace_client: Option>, ) -> WorkerExecutionSpawnResult { #[cfg(feature = "ws-server")] let streams = subscribe_worker_protocol_session(&handle); @@ -2154,7 +2837,6 @@ where #[cfg(not(feature = "ws-server"))] let worker_state_snapshot = handle.shared_state.snapshot(); let worker_state = Arc::new(RwLock::new(worker_state_snapshot)); - let tasks = RuntimeExecutionTaskScope::new(controller_task); let workdir_attachment_statuses = workdir_attachments .iter() .map( @@ -2164,6 +2846,50 @@ where }, ) .collect::>(); + let mut workers = self + .workers + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + if workers.contains_key(&worker_ref) { + drop(workers); + let cleanup = self.cleanup_unconnected_controller( + &handle, + &shutdown, + &tasks, + &shutdown_requested, + &worker_state, + ); + let result = WorkerExecutionResult::busy( + operation, + match &cleanup { + Ok(()) => "Worker is already connected to execution backend".to_string(), + Err(cleanup) => format!( + "Worker is already connected to execution backend; controller cleanup failed: {cleanup}" + ), + }, + ); + return match cleanup { + Ok(()) => WorkerExecutionSpawnResult::RolledBack(result), + Err(_) => { + self.retain_uncertain_unconnected_controller( + &worker_ref, + handle, + shutdown, + shutdown_requested, + tasks, + worker_state, + workspace_client, + restore_operation_id, + workdir_attachment_statuses.clone(), + ); + WorkerExecutionSpawnResult::ReconciliationRequired { + result, + worker_state: None, + workdir_attachments: workdir_attachment_statuses, + } + } + }; + } #[cfg(feature = "ws-server")] { let mut events = streams.events; @@ -2209,10 +2935,12 @@ where }) { Ok(task) => task, Err(message) => { + drop(workers); let cleanup = self.cleanup_unconnected_controller( &handle, &shutdown, &tasks, + &shutdown_requested, &worker_state, ); let result = WorkerExecutionResult::errored( @@ -2224,36 +2952,27 @@ where } }, ); - return if operation == WorkerExecutionOperation::Restore { - match cleanup { - Ok(()) => WorkerExecutionSpawnResult::RolledBack(result), - Err(_) => { - let retained_state = - worker_state.read().ok().map(|state| state.clone()); - let retained_workdir_attachments = - workdir_attachment_statuses.clone(); - let retained_handle = self - .retain_uncertain_unconnected_controller( - &worker_ref, - handle, - shutdown, - tasks, - worker_state, - workspace_client, - restore_operation_id, - retained_workdir_attachments.clone(), - ) - .ok(); - WorkerExecutionSpawnResult::ReconciliationRequired { - result, - handle: retained_handle, - worker_state: retained_state, - workdir_attachments: retained_workdir_attachments, - } + return match cleanup { + Ok(()) => WorkerExecutionSpawnResult::RolledBack(result), + Err(_) => { + let retained_state = worker_state.read().ok().map(|state| state.clone()); + self.retain_uncertain_unconnected_controller( + &worker_ref, + handle, + shutdown, + shutdown_requested, + tasks, + worker_state, + workspace_client, + restore_operation_id, + workdir_attachment_statuses.clone(), + ); + WorkerExecutionSpawnResult::ReconciliationRequired { + result, + worker_state: retained_state, + workdir_attachments: workdir_attachment_statuses, } } - } else { - WorkerExecutionSpawnResult::Errored(result) }; } }; @@ -2264,41 +2983,6 @@ where let _ = bridge_context; } - let mut workers = self - .workers - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()); - if workers.contains_key(&worker_ref) { - let existing_worker_state = workers - .get(&worker_ref) - .and_then(|worker| worker.worker_state.read().ok().map(|state| state.clone())); - let existing_handle = WorkerExecutionHandle::new(worker_ref.clone(), self.backend_id()); - drop(workers); - let cleanup = - self.cleanup_unconnected_controller(&handle, &shutdown, &tasks, &worker_state); - let result = WorkerExecutionResult::busy( - operation, - match &cleanup { - Ok(()) => "Worker is already connected to execution backend".to_string(), - Err(cleanup) => format!( - "Worker is already connected to execution backend; controller cleanup failed: {cleanup}" - ), - }, - ); - return if operation == WorkerExecutionOperation::Restore { - match cleanup { - Ok(()) => WorkerExecutionSpawnResult::RolledBack(result), - Err(_) => WorkerExecutionSpawnResult::ReconciliationRequired { - result, - handle: Some(existing_handle), - worker_state: existing_worker_state, - workdir_attachments: Vec::new(), - }, - } - } else { - WorkerExecutionSpawnResult::Rejected(result) - }; - } let connected_worker_state = worker_state .read() .unwrap_or_else(std::sync::PoisonError::into_inner) @@ -2311,7 +2995,7 @@ where RuntimeWorkerExecution { handle, shutdown, - shutdown_requested: Arc::new(AtomicBool::new(false)), + shutdown_requested, tasks, worker_state, workspace_client, @@ -2322,11 +3006,176 @@ where ); WorkerExecutionSpawnResult::Connected { - handle: WorkerExecutionHandle::new(worker_ref, self.backend_id()), worker_state: connected_worker_state, workdir_attachments: workdir_attachment_statuses, } } + + // Called under the same per-Worker operation lock as connect/Stop. Each + // candidate remains in its actual owning scope until its own cleanup proves + // success. A failed child does not prevent joining the other Controllers. + fn cleanup_execution_resources( + &self, + execution: &RuntimeWorkerExecution, + ) -> Result<(), String> { + let candidates = execution + .tasks + .cleanup_candidates + .lock() + .map_err(|_| "execution cleanup candidate lock is poisoned".to_string())? + .clone(); + let mut first_failure = None; + for candidate in candidates { + match self.cleanup_execution_resources(&candidate) { + Ok(()) => { + execution + .tasks + .cleanup_candidates + .lock() + .map_err(|_| "execution cleanup candidate lock is poisoned".to_string())? + .retain(|retained| !Arc::ptr_eq(retained, &candidate)); + } + Err(message) => { + first_failure.get_or_insert(message); + } + } + } + let handle = execution.handle.clone(); + let shutdown_requested = execution.shutdown_requested.clone(); + let shutdown = execution.shutdown.clone(); + let tasks = execution.tasks.clone(); + let cleanup = self.run_joined_on_adapter_runtime(async move { + tasks + .request_shutdown(&handle, &shutdown_requested, Duration::from_secs(5)) + .await?; + tasks.confirm_shutdown_and_join(&shutdown).await?; + handle.delete_uncommitted_uploaded_files().map_err(|error| { + format!("uploaded_file_cleanup_failed: {error}; stop remains retryable") + }) + }); + if let Err(message) = cleanup { + first_failure.get_or_insert(message); + } + match first_failure { + Some(message) => Err(message), + None => Ok(()), + } + } + + // Caller owns the per-Worker operation lock for this entire cleanup. + fn stop_worker_under_lock(&self, worker_ref: &WorkerRef) -> WorkerExecutionResult { + let pending = match self.pending_restore(worker_ref) { + Ok(pending) => pending, + Err(message) => { + return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); + } + }; + if let Some(pending) = pending { + if let Err(message) = self.wait_pending_restore(&pending, Duration::from_secs(5)) { + return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); + } + if self.workers.lock().is_err() { + return WorkerExecutionResult::errored( + WorkerExecutionOperation::Stop, + "worker adapter registry lock is poisoned", + ); + } + let mut retained = match pending.result.lock() { + Ok(retained) => retained, + Err(_) => { + return WorkerExecutionResult::errored( + WorkerExecutionOperation::Stop, + "restore result lock is poisoned", + ); + } + }; + match retained.as_ref() { + Some(Ok(_)) => { + let controller = match retained.take().unwrap() { + Ok(controller) => controller, + Err(_) => unreachable!(), + }; + let state = Arc::new(RwLock::new(controller.handle.shared_state.snapshot())); + self.retain_uncertain_unconnected_controller( + worker_ref, + controller.handle, + controller.shutdown, + Arc::new(AtomicBool::new(false)), + RuntimeExecutionTaskScope::new(controller.controller_task), + state, + Some(controller.workspace_client), + Some(pending.request.operation_id), + Vec::new(), + ); + } + Some(Err(_)) => { + let factory = Arc::clone(&self.factory); + let request = pending.request.clone(); + if let Err(message) = self + .run_cancellable_on_adapter_runtime(RUNTIME_TASK_TIMEOUT, async move { + factory.cleanup_failed_restore(&request).await + }) + { + return WorkerExecutionResult::errored( + WorkerExecutionOperation::Stop, + message, + ); + } + } + None => { + return WorkerExecutionResult::errored( + WorkerExecutionOperation::Stop, + "restore factory has no completion evidence; cleanup is unproven", + ); + } + } + drop(retained); + if let Err(message) = self.remove_pending_restore(worker_ref) { + return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); + } + } + let execution = match self.workers.lock() { + Ok(workers) => workers.get(worker_ref).cloned(), + Err(_) => { + return WorkerExecutionResult::errored( + WorkerExecutionOperation::Stop, + "worker adapter registry lock is poisoned", + ); + } + }; + let Some(execution) = execution else { + // The execution backend cleanup may have committed before the + // Runtime catalog commit failed. Treat the retry as converged so + // the Runtime can durably finish its Stopped transition. + let factory = Arc::clone(&self.factory); + let worker_ref = worker_ref.clone(); + return match self.run_cancellable_on_adapter_runtime(RUNTIME_TASK_TIMEOUT, async move { + factory.reconcile_stopped_worker(&worker_ref).await + }) { + Ok(()) => WorkerExecutionResult::accepted(WorkerExecutionOperation::Stop), + Err(message) => { + WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message) + } + }; + }; + + if let Err(message) = self.cleanup_execution_resources(&execution) { + return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); + } + + match self.workers.lock() { + Ok(mut workers) => { + workers.remove(worker_ref); + } + Err(_) => { + return WorkerExecutionResult::errored( + WorkerExecutionOperation::Stop, + "worker adapter registry lock is poisoned; cleanup commit remains retryable", + ); + } + } + WorkerExecutionResult::accepted(WorkerExecutionOperation::Stop) + } } impl Drop for WorkerRuntimeExecutionBackend { @@ -2376,6 +3225,27 @@ where &self, request: WorkerSessionObservationRequest, ) -> runtime_api::WorkerSessionAvailability { + let unavailable = |message: String| runtime_api::WorkerSessionAvailability::Unavailable { + reason: runtime_api::WorkerSessionUnavailableReason::StorageUnavailable, + message, + }; + let operation_lock = match self.worker_lock(&request.worker_ref) { + Ok(lock) => lock, + Err(message) => return unavailable(message), + }; + let _operation_guard = match operation_lock.lock() { + Ok(guard) => guard, + Err(_) => return unavailable("worker operation lock is poisoned".to_string()), + }; + let workers = match self.workers.lock() { + Ok(workers) => workers, + Err(_) => return unavailable("worker adapter registry lock is poisoned".to_string()), + }; + if workers.contains_key(&request.worker_ref) { + return runtime_api::WorkerSessionAvailability::LiveProtocol; + } + drop(workers); + match self.factory.retained_session_snapshot(&request.worker_ref) { Ok(retained) => runtime_api::WorkerSessionAvailability::RetainedSnapshot { identity: runtime_api::RetainedSessionIdentity { @@ -2461,6 +3331,12 @@ where request: crate::execution::WorkerSessionAttachmentRequest, ) -> Result { + let operation_lock = self + .worker_lock(&request.worker_ref) + .map_err(|_| session_store::RetainedAttachmentReadError::StorageUnavailable)?; + let _operation_guard = operation_lock + .lock() + .map_err(|_| session_store::RetainedAttachmentReadError::StorageUnavailable)?; let live = self .workers .lock() @@ -2596,21 +3472,56 @@ where } fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult { - let _start_guard = self - .restore_lock - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()); - if self - .workers - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .contains_key(&request.worker_ref) - { + let operation_lock = match self.worker_lock(&request.worker_ref) { + Ok(lock) => lock, + Err(message) => { + return WorkerExecutionSpawnResult::Rejected(WorkerExecutionResult::errored( + WorkerExecutionOperation::Spawn, + message, + )); + } + }; + let _operation_guard = match operation_lock.lock() { + Ok(guard) => guard, + Err(_) => { + let message = "worker operation lock is poisoned".to_string(); + return WorkerExecutionSpawnResult::Rejected(WorkerExecutionResult::errored( + WorkerExecutionOperation::Spawn, + message, + )); + } + }; + match self.pending_restore(&request.worker_ref) { + Ok(None) => {} + Ok(Some(_)) => { + return WorkerExecutionSpawnResult::Rejected(WorkerExecutionResult::busy( + WorkerExecutionOperation::Spawn, + "Worker restore still owns pending resources", + )); + } + Err(message) => { + return WorkerExecutionSpawnResult::Rejected(WorkerExecutionResult::errored( + WorkerExecutionOperation::Spawn, + message, + )); + } + } + let workers = match self.workers.lock() { + Ok(workers) => workers, + Err(_) => { + return WorkerExecutionSpawnResult::Rejected(WorkerExecutionResult::errored( + WorkerExecutionOperation::Spawn, + "worker adapter registry lock is poisoned", + )); + } + }; + if workers.contains_key(&request.worker_ref) { return WorkerExecutionSpawnResult::Rejected(WorkerExecutionResult::busy( WorkerExecutionOperation::Spawn, "Worker is already connected to execution backend", )); } + drop(workers); let mut request = request; let Some(materializer) = self.working_directory_materializer.as_ref().or_else(|| { @@ -2786,16 +3697,69 @@ where &self, request: &WorkerExecutionRestoreRequest, ) -> Result<(), WorkerExecutionResult> { - let workers = self - .workers - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()); - if workers.contains_key(&request.worker_ref) { + let operation_lock = match self.worker_lock(&request.worker_ref) { + Ok(lock) => lock, + Err(message) => { + return Err(WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + message, + )); + } + }; + let _operation_guard = match operation_lock.lock() { + Ok(guard) => guard, + Err(_) => { + let message = "worker operation lock is poisoned".to_string(); + return Err(WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + message, + )); + } + }; + if self + .pending_restore(&request.worker_ref) + .map_err(|message| { + WorkerExecutionResult::errored(WorkerExecutionOperation::Restore, message) + })? + .is_some() + { return Err(WorkerExecutionResult::busy( WorkerExecutionOperation::Restore, - "Worker is already connected to execution backend", + "Worker restore still owns pending resources", )); } + let workers = self.workers.lock().map_err(|_| { + WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + "worker adapter registry lock is poisoned", + ) + })?; + if let Some(existing) = workers.get(&request.worker_ref) { + let result = WorkerExecutionResult::busy( + WorkerExecutionOperation::Restore, + "Worker is already connected to execution backend", + ); + if existing.shutdown_requested.load(Ordering::Acquire) + || existing.tasks.has_cleanup_candidates().map_err(|message| { + WorkerExecutionResult::errored(WorkerExecutionOperation::Restore, message) + })? + { + return Err(result); + } + let snapshot = existing + .worker_state + .read() + .map_err(|_| { + WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + "worker state lock is poisoned", + ) + })? + .clone(); + // Read-only AlreadyConnected evidence. The Runtime may accept this + // exact restore without journaling or replacing the current owner. + return Err(result.with_worker_state(snapshot)); + } drop(workers); if !request.previous_workdir_attachments.is_empty() { @@ -2878,18 +3842,51 @@ where &self, mut request: WorkerExecutionRestoreRequest, ) -> WorkerExecutionSpawnResult { - // Serialize backend restore starts so duplicate detection is completed - // before any controller or bridge side effect. Runtime also serializes - // per Worker; this guard protects direct/concurrent backend callers. - let _restore_guard = self - .restore_lock - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()); - let workers = self - .workers - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()); + let operation_lock = match self.worker_lock(&request.worker_ref) { + Ok(lock) => lock, + Err(message) => { + return WorkerExecutionSpawnResult::Rejected(WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + message, + )); + } + }; + let _operation_guard = match operation_lock.lock() { + Ok(guard) => guard, + Err(_) => { + let message = "worker operation lock is poisoned".to_string(); + return WorkerExecutionSpawnResult::Rejected(WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + message, + )); + } + }; + let workers = self.workers.lock().map_err(|_| { + WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + "worker adapter registry lock is poisoned", + ) + }); + let workers = match workers { + Ok(workers) => workers, + Err(result) => return WorkerExecutionSpawnResult::Rejected(result), + }; if let Some(existing) = workers.get(&request.worker_ref) { + let pending_cleanup = existing.tasks.has_cleanup_candidates(); + if existing.shutdown_requested.load(Ordering::Acquire) + || !matches!(pending_cleanup.as_ref(), Ok(false)) + { + return WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + pending_cleanup.err().unwrap_or_else(|| { + "Worker still owns unproven Controller cleanup".to_string() + }), + ), + worker_state: None, + workdir_attachments: existing.workdir_attachments.clone(), + }; + } if existing.restore_operation_id == Some(request.operation_id) { let worker_state = existing .worker_state @@ -2897,10 +3894,6 @@ where .unwrap_or_else(std::sync::PoisonError::into_inner) .clone(); return WorkerExecutionSpawnResult::Connected { - handle: WorkerExecutionHandle::new( - request.worker_ref.clone(), - self.backend_id(), - ), worker_state, workdir_attachments: existing.workdir_attachments.clone(), }; @@ -2995,79 +3988,212 @@ where } request.workdir_attachments = workdir_attachments.clone(); - let factory = self.factory.clone(); - let bridge_context = request.context.clone(); let worker_ref = request.worker_ref.clone(); let operation_id = request.operation_id; - let restore_result = self - .run_cancellable_on_adapter_runtime(self.spawn_restore_timeout, async move { - factory.restore_controller(request).await - }); - - let controller = match restore_result { - Ok(controller) => controller, + let pending = match self.pending_restore(&worker_ref) { + Ok(Some(pending)) if pending.request.operation_id == operation_id => pending, + Ok(Some(_)) => { + return WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::busy( + WorkerExecutionOperation::Restore, + "another restore operation still owns pending resources", + ), + worker_state: None, + workdir_attachments: Vec::new(), + }; + } + Ok(None) => { + let factory = Arc::clone(&self.factory); + let preflight_request = request.clone(); + if let Err(message) = self.run_on_adapter_runtime(async move { + factory.preflight_restore(&preflight_request).await + }) { + return WorkerExecutionSpawnResult::Rejected(WorkerExecutionResult::rejected( + WorkerExecutionOperation::Restore, + message, + )); + } + let pending = Arc::new(PendingRuntimeRestore { + request: request.clone(), + task: tokio::sync::Mutex::new(None), + result: Mutex::new(None), + failure: Mutex::new(None), + }); + let mut resources = match self.pending_restores.lock() { + Ok(resources) => resources, + Err(_) => { + return WorkerExecutionSpawnResult::Rejected( + WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + "pending restore resources lock is poisoned", + ), + ); + } + }; + resources.insert(worker_ref.clone(), Arc::clone(&pending)); + let factory = Arc::clone(&self.factory); + let result_owner = Arc::clone(&pending); + let task = self.spawn_on_adapter_runtime(async move { + let result = factory.restore_controller(request).await; + let mut retained = result_owner + .result + .lock() + .unwrap_or_else(std::sync::PoisonError::into_inner); + *retained = Some(result); + }); + match task { + Ok(task) => { + *pending.task.try_lock().expect("new restore task lock") = Some(task) + } + Err(message) => { + resources.remove(&worker_ref); + return WorkerExecutionSpawnResult::Rejected( + WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + message, + ), + ); + } + } + drop(resources); + pending + } Err(message) => { return WorkerExecutionSpawnResult::ReconciliationRequired { result: WorkerExecutionResult::errored( WorkerExecutionOperation::Restore, message, ), - handle: None, worker_state: None, workdir_attachments: Vec::new(), }; } }; + if let Err(message) = self.wait_pending_restore(&pending, self.spawn_restore_timeout) { + return WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::errored(WorkerExecutionOperation::Restore, message), + worker_state: None, + workdir_attachments: Vec::new(), + }; + } + let controller = { + let mut retained = match pending.result.lock() { + Ok(retained) => retained, + Err(_) => { + return WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + "restore result lock is poisoned", + ), + worker_state: None, + workdir_attachments: Vec::new(), + }; + } + }; + match retained.as_ref() { + Some(Ok(_)) => match retained.take().unwrap() { + Ok(controller) => controller, + Err(_) => unreachable!(), + }, + Some(Err(message)) => { + return WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + message.clone(), + ), + worker_state: None, + workdir_attachments: Vec::new(), + }; + } + None => { + return WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + "restore factory completion has no result; cleanup is unproven", + ), + worker_state: None, + workdir_attachments: Vec::new(), + }; + } + } + }; - self.connect_handle( + let result = self.connect_handle( WorkerExecutionOperation::Restore, - worker_ref, - bridge_context, + worker_ref.clone(), + pending.request.context.clone(), controller.handle, controller.shutdown, controller.controller_task, Some(operation_id), - workdir_attachments, + pending.request.workdir_attachments.clone(), Some(controller.workspace_client), - ) + ); + if let Err(message) = self.remove_pending_restore(&worker_ref) { + return WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::errored(WorkerExecutionOperation::Restore, message), + worker_state: None, + workdir_attachments: Vec::new(), + }; + } + result } fn activate_restored_worker( &self, operation_id: crate::execution::WorkerLifecycleOperationId, - handle: &WorkerExecutionHandle, + worker_ref: &WorkerRef, ) -> Result<(), String> { - if handle.backend_id() != self.backend_id() { - return Err(format!( - "execution handle belongs to backend {}, not {}", - handle.backend_id(), - self.backend_id() - )); - } + let operation_lock = match self.worker_lock(worker_ref) { + Ok(lock) => lock, + Err(message) => return Err(message), + }; + let _operation_guard = match operation_lock.lock() { + Ok(guard) => guard, + Err(_) => { + let message = "worker operation lock is poisoned".to_string(); + return Err(message); + } + }; let workers = self .workers .lock() .map_err(|_| "worker adapter registry lock is poisoned".to_string())?; let execution = workers - .get(handle.worker_ref()) + .get(worker_ref) .ok_or_else(|| "restored Worker execution is not registered".to_string())?; if execution.restore_operation_id != Some(operation_id) { return Err("restored Worker operation identity does not match".to_string()); } self.factory.activate_restored_controller( - handle.worker_ref(), + worker_ref, execution.workspace_id.as_deref(), &execution.handle, ); Ok(()) } - fn dispatch_input( - &self, - handle: &WorkerExecutionHandle, - input: WorkerInput, - ) -> WorkerExecutionResult { - let (worker, worker_state, _workspace_client) = match self.get_execution(handle) { + fn dispatch_input(&self, worker_ref: &WorkerRef, input: WorkerInput) -> WorkerExecutionResult { + let operation_lock = match self.worker_lock(worker_ref) { + Ok(lock) => lock, + Err(message) => { + return WorkerExecutionResult::errored(WorkerExecutionOperation::Input, message); + } + }; + let _operation_guard = match operation_lock.lock() { + Ok(guard) => guard, + Err(_) => { + let message = "worker operation lock is poisoned".to_string(); + return WorkerExecutionResult::errored(WorkerExecutionOperation::Input, message); + } + }; + if let Err(error) = crate::runtime::validate_worker_input(&input) { + return WorkerExecutionResult::rejected( + WorkerExecutionOperation::Input, + error.to_string(), + ); + } + let (worker, worker_state, _workspace_client) = match self.get_execution(worker_ref) { Ok(execution) => execution, Err(mut result) => { result.operation = WorkerExecutionOperation::Input; @@ -3167,13 +4293,32 @@ where fn upload_file( &self, - handle: &WorkerExecutionHandle, + worker_ref: &WorkerRef, file_name: &str, media_type: &str, content: &[u8], context: Option<&session_store::UploadedFileUploadContext>, ) -> Result { - let (worker, _, _) = self.get_execution(handle).map_err(|mut result| { + let operation_lock = match self.worker_lock(worker_ref) { + Ok(lock) => lock, + Err(message) => { + return Err(WorkerExecutionResult::errored( + WorkerExecutionOperation::UploadFile, + message, + )); + } + }; + let _operation_guard = match operation_lock.lock() { + Ok(guard) => guard, + Err(_) => { + let message = "worker operation lock is poisoned".to_string(); + return Err(WorkerExecutionResult::errored( + WorkerExecutionOperation::UploadFile, + message, + )); + } + }; + let (worker, _, _) = self.get_execution(worker_ref).map_err(|mut result| { result.operation = WorkerExecutionOperation::UploadFile; result })?; @@ -3193,10 +4338,29 @@ where fn delete_uploaded_file( &self, - handle: &WorkerExecutionHandle, + worker_ref: &WorkerRef, artifact_id: &str, ) -> WorkerExecutionResult { - let (worker, _, _) = match self.get_execution(handle) { + let operation_lock = match self.worker_lock(worker_ref) { + Ok(lock) => lock, + Err(message) => { + return WorkerExecutionResult::errored( + WorkerExecutionOperation::DeleteUploadedFile, + message, + ); + } + }; + let _operation_guard = match operation_lock.lock() { + Ok(guard) => guard, + Err(_) => { + let message = "worker operation lock is poisoned".to_string(); + return WorkerExecutionResult::errored( + WorkerExecutionOperation::DeleteUploadedFile, + message, + ); + } + }; + let (worker, _, _) = match self.get_execution(worker_ref) { Ok(execution) => execution, Err(mut result) => { result.operation = WorkerExecutionOperation::DeleteUploadedFile; @@ -3212,12 +4376,27 @@ where } } - fn dispatch_method( - &self, - handle: &WorkerExecutionHandle, - method: Method, - ) -> WorkerExecutionResult { - let (worker, _worker_state, _workspace_client) = match self.get_execution(handle) { + fn dispatch_method(&self, worker_ref: &WorkerRef, method: Method) -> WorkerExecutionResult { + let operation_lock = match self.worker_lock(worker_ref) { + Ok(lock) => lock, + Err(message) => { + return WorkerExecutionResult::errored( + WorkerExecutionOperation::ProtocolMethod, + message, + ); + } + }; + let _operation_guard = match operation_lock.lock() { + Ok(guard) => guard, + Err(_) => { + let message = "worker operation lock is poisoned".to_string(); + return WorkerExecutionResult::errored( + WorkerExecutionOperation::ProtocolMethod, + message, + ); + } + }; + let (worker, _worker_state, _workspace_client) = match self.get_execution(worker_ref) { Ok(execution) => execution, Err(mut result) => { result.operation = WorkerExecutionOperation::ProtocolMethod; @@ -3229,109 +4408,41 @@ where } fn stop_worker_operation(&self, request: WorkerExecutionStopRequest) -> WorkerExecutionResult { - let handle = request - .handle - .unwrap_or_else(|| WorkerExecutionHandle::new(request.worker_ref, self.backend_id())); - self.stop_worker(&handle) + self.stop_worker(&request.worker_ref) } - fn stop_worker(&self, handle: &WorkerExecutionHandle) -> WorkerExecutionResult { - if handle.backend_id() != self.backend_id() { - return WorkerExecutionResult::rejected( - WorkerExecutionOperation::Stop, - format!( - "execution handle belongs to backend {}, not {}", - handle.backend_id(), - self.backend_id() - ), - ); - } - let execution = match self.workers.lock() { - Ok(workers) => workers.get(handle.worker_ref()).cloned(), + fn stop_worker(&self, worker_ref: &WorkerRef) -> WorkerExecutionResult { + let operation_lock = match self.worker_lock(worker_ref) { + Ok(lock) => lock, + Err(message) => { + return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); + } + }; + let _operation_guard = match operation_lock.lock() { + Ok(guard) => guard, Err(_) => { - return WorkerExecutionResult::errored( - WorkerExecutionOperation::Stop, - "worker adapter registry lock is poisoned", - ); + let message = "worker operation lock is poisoned".to_string(); + return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); } }; - let Some(execution) = execution else { - // The execution backend cleanup may have committed before the - // Runtime catalog commit failed. Treat the retry as converged so - // the Runtime can durably finish its Stopped transition. - return WorkerExecutionResult::accepted(WorkerExecutionOperation::Stop); - }; - - let first_request = !execution.shutdown_requested.swap(true, Ordering::AcqRel); - let result = if first_request { - let command = match next_internal_command(&execution.worker_state) { - Ok(command) => command, - Err(error) => { - execution.shutdown_requested.store(false, Ordering::Release); - return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, error); - } - }; - let result = self.send_method( - WorkerExecutionOperation::Stop, - execution.handle.clone(), - Method::Shutdown { command }, - ); - if result.outcome != crate::execution::WorkerExecutionOutcome::Accepted { - execution.shutdown_requested.store(false, Ordering::Release); - return result; - } - result - } else { - WorkerExecutionResult::accepted(WorkerExecutionOperation::Stop) - }; - - let shutdown = execution.shutdown.clone(); - let tasks = execution.tasks.clone(); - let shutdown_wait = self.run_on_adapter_runtime(async move { - { - let mut guard = shutdown.lock().await; - if let Some(mut receiver) = guard.take() { - match tokio::time::timeout(Duration::from_secs(5), &mut receiver).await { - Ok(Ok(())) => {} - Ok(Err(_)) => { - return Err("Worker shutdown completion channel closed".to_string()); - } - Err(_) => { - *guard = Some(receiver); - return Err( - "Worker shutdown confirmation timed out; stop remains retryable" - .to_string(), - ); - } - } - } - } - tasks.join().await - }); - if let Err(message) = shutdown_wait { - return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); - } - - if let Err(error) = execution.handle.delete_uncommitted_uploaded_files() { - return WorkerExecutionResult::errored( - WorkerExecutionOperation::Stop, - format!("uploaded_file_cleanup_failed: {error}; stop remains retryable"), - ); - } - - match self.workers.lock() { - Ok(mut workers) => { - workers.remove(handle.worker_ref()); - } - Err(poisoned) => { - poisoned.into_inner().remove(handle.worker_ref()); - } - } - result + self.stop_worker_under_lock(worker_ref) } - fn cancel_worker(&self, handle: &WorkerExecutionHandle) -> WorkerExecutionResult { - let (worker, worker_state, _workspace_client) = match self.get_execution(handle) { + fn cancel_worker(&self, worker_ref: &WorkerRef) -> WorkerExecutionResult { + let operation_lock = match self.worker_lock(worker_ref) { + Ok(lock) => lock, + Err(message) => { + return WorkerExecutionResult::errored(WorkerExecutionOperation::Cancel, message); + } + }; + let _operation_guard = match operation_lock.lock() { + Ok(guard) => guard, + Err(_) => { + let message = "worker operation lock is poisoned".to_string(); + return WorkerExecutionResult::errored(WorkerExecutionOperation::Cancel, message); + } + }; + let (worker, worker_state, _workspace_client) = match self.get_execution(worker_ref) { Ok(execution) => execution, Err(mut result) => { result.operation = WorkerExecutionOperation::Cancel; @@ -3352,32 +4463,161 @@ where } #[cfg(feature = "ws-server")] - fn worker_snapshot(&self, handle: &WorkerExecutionHandle) -> Option { - if handle.backend_id() != self.backend_id() { - return None; - } + fn attach_worker_protocol( + self: Arc, + worker_ref: &WorkerRef, + ) -> Result { + let operation_lock = self.worker_lock(worker_ref).map_err(|message| { + WorkerExecutionResult::errored(WorkerExecutionOperation::ProtocolMethod, message) + })?; + let _operation_guard = operation_lock.lock().map_err(|_| { + WorkerExecutionResult::errored( + WorkerExecutionOperation::ProtocolMethod, + "worker operation lock is poisoned", + ) + })?; + let execution = self + .workers + .lock() + .map_err(|_| { + WorkerExecutionResult::errored( + WorkerExecutionOperation::ProtocolMethod, + "worker adapter registry lock is poisoned", + ) + })? + .get(worker_ref) + .cloned() + .ok_or_else(|| { + WorkerExecutionResult::rejected( + WorkerExecutionOperation::ProtocolMethod, + "Worker has no live protocol endpoint", + ) + })?; + self.validate_protocol_execution_under_lock(worker_ref, &execution)?; + let streams = subscribe_worker_protocol_session(&execution.handle); + let snapshot = streams.snapshot_event; + let (tx, events) = tokio::sync::mpsc::channel(128); + let relay_handle = execution.handle.clone(); + let task = self + .spawn_on_adapter_runtime(async move { + for alert in streams.alert_snapshot { + tokio::select! { + biased; + _ = relay_handle.protocol_closed() => return, + sent = tx.send(Event::Alert(alert)) => if sent.is_err() { return; }, + } + } + let mut protocol_events = streams.events; + let mut log_entries = streams.log_entries; + loop { + let event = tokio::select! { + biased; + _ = relay_handle.protocol_closed() => break, + _ = tx.closed() => break, + event = protocol_events.recv() => match event { + Ok(event) => Some(event), + Err(_) => break, + }, + entry = log_entries.recv() => match entry { + Ok(entry) => live_log_entry_event(entry), + Err(_) => break, + }, + }; + if let Some(event) = event { + tokio::select! { + biased; + _ = relay_handle.protocol_closed() => break, + sent = tx.send(event) => if sent.is_err() { break; }, + } + } + } + }) + .map_err(|message| { + WorkerExecutionResult::errored(WorkerExecutionOperation::ProtocolMethod, message) + })?; + // Stop owns the Controller and every attached relay in this same scope. + // Its abort/join barrier closes the bounded stream before cleanup succeeds. + execution.tasks.push("protocol transport", task, true); + + let dispatcher_backend = Arc::clone(&self); + let dispatcher_lock = Arc::clone(&operation_lock); + let dispatcher_execution = execution.clone(); + let dispatcher_worker_ref = worker_ref.clone(); + let dispatch = Arc::new(move |method| { + let _guard = dispatcher_lock.lock().map_err(|_| { + WorkerExecutionResult::errored( + WorkerExecutionOperation::ProtocolMethod, + "worker operation lock is poisoned", + ) + })?; + dispatcher_backend.dispatch_protocol_execution_under_lock( + &dispatcher_worker_ref, + &dispatcher_execution, + method, + ) + }); + let validator_backend = Arc::clone(&self); + let validator_lock = Arc::clone(&operation_lock); + let validator_worker_ref = worker_ref.clone(); + let validate = Arc::new(move || { + let _guard = validator_lock.lock().map_err(|_| { + WorkerExecutionResult::errored( + WorkerExecutionOperation::ProtocolMethod, + "worker operation lock is poisoned", + ) + })?; + validator_backend + .validate_protocol_execution_under_lock(&validator_worker_ref, &execution) + }); + Ok(WorkerProtocolTransport::new( + worker_ref.clone(), + snapshot, + events, + dispatch, + validate, + )) + } + + #[cfg(feature = "ws-server")] + fn worker_snapshot(&self, worker_ref: &WorkerRef) -> Option { + let operation_lock = match self.worker_lock(worker_ref) { + Ok(lock) => lock, + Err(_) => return None, + }; + let _operation_guard = match operation_lock.lock() { + Ok(guard) => guard, + Err(_) => return None, + }; let workers = self.workers.lock().ok()?; workers - .get(handle.worker_ref()) + .get(worker_ref) .map(|execution| execution.handle.snapshot_event()) } fn worker_completions( &self, - handle: &WorkerExecutionHandle, + worker_ref: &WorkerRef, kind: protocol::CompletionKind, prefix: &str, + context: Option<&protocol::CompletionContext>, ) -> Vec { - if handle.backend_id() != self.backend_id() { - return Vec::new(); - } + let operation_lock = match self.worker_lock(worker_ref) { + Ok(lock) => lock, + Err(_) => return Vec::new(), + }; + let _operation_guard = match operation_lock.lock() { + Ok(guard) => guard, + Err(_) => return Vec::new(), + }; let Ok(workers) = self.workers.lock() else { return Vec::new(); }; workers - .get(handle.worker_ref()) + .get(worker_ref) .map(|execution| { - futures::executor::block_on(execution.handle.completion_entries(kind, prefix)) + futures::executor::block_on( + execution.handle.completion_entries(kind, prefix, context), + ) }) .unwrap_or_default() } @@ -3403,6 +4643,7 @@ mod tests { use crate::identity::WorkerId; use crate::identity::WorkerRef; use crate::management::RuntimeOptions; + #[cfg(feature = "ws-server")] use crate::observation::WorkerObservationCursor; use crate::working_directory::RuntimeGitMaterializer; use agen::Engine; @@ -3503,6 +4744,1419 @@ mod tests { (session_id, segment_id, expected_entry_id) } + const SAVED_SUBJEKTIV_SYSTEM_PROMPT: &str = "Committed saved Subject Worker system prompt"; + + async fn saved_subjektiv_restore_fixture( + root: &Path, + materialize_head: bool, + ) -> WorkerExecutionRestoreRequest { + let worker_ref = WorkerRef::new(WorkerId::now_v7()); + let worker_name = ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&worker_ref); + let aggregate_dir = root.join("workers").join(worker_ref.worker_id.to_string()); + let session_store = WorkerSessionStore::new(aggregate_dir.join("session")).unwrap(); + let store = WorkerAggregateStore::new(&aggregate_dir, &worker_name).unwrap(); + let mut saved = WorkerManifest::from_toml(&format!( + r#" + [worker] + name = "{worker_name}" + [model] + scheme = "anthropic" + model_id = "saved-test-model" + auth = {{ kind = "none" }} + [engine] + instruction = "default" + max_tokens = 100 + [[scope.allow]] + target = "{}" + permission = "read" + recursive = true + symlink_policy = "resolved" + [feature.memory.profile] + enabled = false + [feature.subjektiv.profile] + enabled = true + [feature.subjektiv.profile.extraction] + enabled = false + "#, + root.display() + )) + .unwrap(); + Scope::from_config(&saved.scope) + .expect("saved fixture must declare a valid resolved scope"); + let mut request = create_request("saved subject policy"); + request.worker_id = worker_ref.worker_id; + request.workspace_api = Some(WorkspaceApiRef { + workspace_id: "workspace-saved-subjektiv".to_string(), + base_url: "http://workspace.invalid".to_string(), + }); + request.subjektiv_attached = true; + request.memory_settings = Some(manifest::WorkspaceMemorySettingsSnapshot { + workspace_id: "workspace-saved-subjektiv".to_string(), + settings_revision: 17, + language: "English".to_string(), + }); + bind_workspace_memory_settings(&mut saved, &request).unwrap(); + let client = MockClient::new(Vec::new()); + let mut engine = + Engine::<_, agen::state::Mutable, worker::SessionHistoryMetadata>::new_annotated( + client.clone(), + ); + engine.set_system_prompt(SAVED_SUBJEKTIV_SYSTEM_PROMPT); + let mut worker = Worker::new( + saved.clone(), + engine, + CombinedStore::new(session_store.clone(), store.clone()), + WorkerWorkspaceContext::unavailable( + Some(WorkspaceId::new("workspace-saved-subjektiv").unwrap()), + "saved session fixture has no network client", + ), + WorkerFilesystemAuthority::None, + Scope::empty(), + ) + .await + .unwrap(); + worker.enable_worker_metadata_write_through().unwrap(); + let session_id = worker.session_id(); + let segment_id = worker.segment_id(); + if materialize_head { + // Use the Worker's normal typed startup writer, not the retained + // public-history fixture (which intentionally has no system prompt). + worker.materialize_durable_session_head().await.unwrap(); + let entries = session_store.read_all(session_id, segment_id).unwrap(); + assert!( + matches!(entries.as_slice(), [LogEntry::AnnotatedSegmentStart { + session_id: committed_session_id, + system_prompt: Some(prompt), + history, + forked_from: None, + compacted_from: None, + .. + }] if *committed_session_id == session_id + && prompt == SAVED_SUBJEKTIV_SYSTEM_PROMPT + && history.is_empty()) + ); + assert_eq!( + session_store::collect_state(&entries) + .system_prompt + .as_deref(), + Some(SAVED_SUBJEKTIV_SYSTEM_PROMPT) + ); + } else { + assert!(matches!( + session_store.exists(session_id, segment_id), + Err(session_store::StoreError::Corrupt { message, .. }) + if message.contains("no materialized Session") + )); + } + assert_eq!(client.call_count.load(Ordering::SeqCst), 0); + drop(worker); + let mut metadata = store.read_by_name(&worker_name).unwrap().unwrap(); + let (manifest, _) = ProfileRuntimeWorkerFactory::manifest_for_restore(&metadata, &request) + .expect("fixture must persist valid saved Manifest authority"); + assert_eq!( + manifest::write_persisted_worker_manifest_snapshot(&manifest).unwrap(), + manifest::write_persisted_worker_manifest_snapshot(&saved).unwrap() + ); + assert_eq!( + metadata.active.as_ref().unwrap().session_id, + session_id, + "metadata write-through must preserve the real Worker Session" + ); + assert_eq!( + metadata.active.as_ref().unwrap().segment_id, + materialize_head.then_some(segment_id) + ); + assert_eq!( + metadata.workspace_id.as_deref(), + Some("workspace-saved-subjektiv") + ); + // Attribution already committed before restart; the real restore needs + // no network fixture or replacement current Profile. + metadata.subjektiv_session_attribution = + Some(session_store::SubjektivSessionAttributionState::Confirmed { + session_id, + subject_id: "saved-subject".to_string(), + }); + store.write(&metadata).unwrap(); + WorkerExecutionRestoreRequest { + operation_id: crate::execution::WorkerLifecycleOperationId::new(), + worker_ref: worker_ref.clone(), + request, + workspace_scope: Some(crate::runtime::RuntimeWorkspaceScope::new( + "workspace-saved-subjektiv", + "server-main", + )), + context: test_execution_context(worker_ref), + previous_workdir_attachments: Vec::new(), + logical_workdir_attachments: Vec::new(), + workdir_attachments: BTreeMap::new(), + config_bundle: None, + } + } + + async fn shutdown_profile_controller(controller: RuntimeWorkerController) { + let state = Arc::new(RwLock::new(controller.handle.shared_state.snapshot())); + controller + .handle + .send(Method::Shutdown { + command: next_internal_command(&state).unwrap(), + }) + .await + .unwrap(); + if let Some(receiver) = controller.shutdown.lock().await.take() { + tokio::time::timeout(Duration::from_secs(5), receiver) + .await + .unwrap() + .unwrap(); + } + tokio::time::timeout(Duration::from_secs(5), controller.controller_task) + .await + .unwrap() + .unwrap(); + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn profile_backend_existing_restore_attestation_is_read_only_under_concurrency() { + let root = tempfile::tempdir().unwrap(); + let runtime_store = root.path().join("runtime"); + let backend = Arc::new( + WorkerRuntimeExecutionBackend::new( + ProfileRuntimeWorkerFactory::new(root.path()) + .with_runtime_store_dir(&runtime_store) + .with_runtime_id("runtime-subjektiv-test"), + ) + .unwrap(), + ); + let fixture_root = runtime_store.clone(); + let request = backend + .run_on_adapter_runtime(async move { + Ok(saved_subjektiv_restore_fixture(&fixture_root, true).await) + }) + .unwrap(); + backend.preflight_restore(&request).unwrap(); + let expected = match backend.restore_worker(request.clone()) { + WorkerExecutionSpawnResult::Connected { worker_state, .. } => worker_state, + other => panic!("real Profile backend restore failed: {other:?}"), + }; + assert!(matches!( + backend.worker_session(WorkerSessionObservationRequest { + worker_ref: request.worker_ref.clone(), + }), + runtime_api::WorkerSessionAvailability::LiveProtocol + )); + let current = backend + .workers + .lock() + .unwrap() + .get(&request.worker_ref) + .unwrap() + .clone(); + let authority_files = || { + let mut files = persisted_files(&runtime_store); + files.retain(|path, _| { + !path + .components() + .any(|component| component.as_os_str() == "runs") + }); + files + }; + let files_before = authority_files(); + let runs = backend + .factory + .worker_aggregate_dir(&request.worker_ref) + .unwrap() + .join("runs"); + let run_directories = || { + fs::read_dir(&runs) + .unwrap() + .map(|entry| entry.unwrap().path()) + .collect::>() + }; + let runs_before = run_directories(); + std::thread::scope(|threads| { + for _ in 0..8 { + let backend = Arc::clone(&backend); + let request = request.clone(); + let expected = expected.clone(); + threads.spawn(move || { + for _ in 0..2 { + let connected = backend.preflight_restore(&request).unwrap_err(); + assert_eq!( + connected.outcome, + crate::execution::WorkerExecutionOutcome::Busy + ); + assert_eq!(connected.worker_state, Some(expected.clone())); + match backend.reconcile_restore(request.clone()) { + WorkerExecutionSpawnResult::Connected { worker_state, .. } => { + assert_eq!(worker_state, expected) + } + other => { + panic!("repeated exact operation was not connected: {other:?}") + } + } + } + }); + } + }); + let mut repeated_request = request.clone(); + repeated_request.operation_id = crate::execution::WorkerLifecycleOperationId::new(); + let connected = backend.preflight_restore(&repeated_request).unwrap_err(); + assert_eq!( + connected.outcome, + crate::execution::WorkerExecutionOutcome::Busy + ); + assert_eq!(connected.worker_state, Some(expected)); + let workers = backend.workers.lock().unwrap(); + let unchanged = workers.get(&request.worker_ref).unwrap(); + assert!(Arc::ptr_eq( + &unchanged.handle.shared_state, + ¤t.handle.shared_state + )); + assert!(Arc::ptr_eq(&unchanged.worker_state, ¤t.worker_state)); + assert_eq!(unchanged.restore_operation_id, Some(request.operation_id)); + assert_eq!(workers.len(), 1); + drop(workers); + assert_eq!(authority_files(), files_before); + assert_eq!(run_directories(), runs_before); + assert!( + backend + .pending_restore(&request.worker_ref) + .unwrap() + .is_none() + ); + current.shutdown_requested.store(true, Ordering::Release); + let pending_cleanup = backend.preflight_restore(&request).unwrap_err(); + assert_eq!( + pending_cleanup.outcome, + crate::execution::WorkerExecutionOutcome::Busy + ); + assert!( + pending_cleanup.worker_state.is_none(), + "pending cleanup is not AlreadyConnected evidence" + ); + current.shutdown_requested.store(false, Ordering::Release); + assert!(backend.stop_worker(&request.worker_ref).is_accepted()); + assert!(matches!( + backend.worker_session(WorkerSessionObservationRequest { + worker_ref: request.worker_ref, + }), + runtime_api::WorkerSessionAvailability::RetainedSnapshot { .. } + )); + } + + #[cfg(feature = "ws-server")] + #[test] + #[serial_test::serial(worker_allocation)] + fn protocol_transport_stays_bound_to_stopped_execution_after_restore() { + let root = tempfile::tempdir().unwrap(); + let runtime_store = root.path().join("runtime"); + let backend = Arc::new( + WorkerRuntimeExecutionBackend::new( + ProfileRuntimeWorkerFactory::new(root.path()) + .with_runtime_store_dir(&runtime_store) + .with_runtime_id("runtime-subjektiv-test"), + ) + .unwrap(), + ); + let fixture_root = runtime_store.clone(); + let mut request = backend + .run_on_adapter_runtime(async move { + Ok(saved_subjektiv_restore_fixture(&fixture_root, true).await) + }) + .unwrap(); + assert!(matches!( + backend.restore_worker(request.clone()), + WorkerExecutionSpawnResult::Connected { .. } + )); + let mut transport_a = Arc::clone(&backend) + .attach_worker_protocol(&request.worker_ref) + .unwrap(); + assert!(matches!(&transport_a.snapshot, Event::Snapshot { .. })); + transport_a.validate().unwrap(); + let completion = |id: &str| Method::ListCompletions { + request_id: Some(id.to_string()), + kind: protocol::CompletionKind::File, + prefix: String::new(), + context: None, + }; + assert!( + matches!(transport_a.dispatch(completion("a-completion")).unwrap().as_slice(), + [Event::Completions { request_id: Some(id), .. }] if id == "a-completion") + ); + assert!( + transport_a + .dispatch(Method::Shutdown { + command: WorkerCommandEnvelope::new(1), + }) + .unwrap() + .is_empty() + ); + assert!( + transport_a.events.is_closed(), + "stop must join the connection relay before success" + ); + while transport_a.events.try_recv().is_ok() {} + assert!(matches!( + transport_a.events.try_recv(), + Err(tokio::sync::mpsc::error::TryRecvError::Disconnected) + )); + request.operation_id = crate::execution::WorkerLifecycleOperationId::new(); + assert!(matches!( + backend.restore_worker(request.clone()), + WorkerExecutionSpawnResult::Connected { .. } + )); + let current_b = backend + .workers + .lock() + .unwrap() + .get(&request.worker_ref) + .unwrap() + .clone(); + let snapshot_b = current_b.handle.shared_state.snapshot(); + assert!(transport_a.validate().is_err()); + for delayed in [ + Method::Cancel { + command: WorkerCommandEnvelope::new(900), + }, + completion("delayed-a-completion"), + Method::Shutdown { + command: WorkerCommandEnvelope::new(901), + }, + ] { + let rejected = transport_a.dispatch(delayed).unwrap_err(); + assert_eq!( + rejected.outcome, + crate::execution::WorkerExecutionOutcome::Rejected + ); + assert!(rejected.worker_state.is_none()); + } + assert_eq!(current_b.handle.shared_state.snapshot(), snapshot_b); + assert_eq!( + backend + .workers + .lock() + .unwrap() + .get(&request.worker_ref) + .unwrap() + .restore_operation_id, + Some(request.operation_id) + ); + let transport_b = Arc::clone(&backend) + .attach_worker_protocol(&request.worker_ref) + .unwrap(); + transport_b.validate().unwrap(); + assert!( + matches!(transport_b.dispatch(completion("b-completion")).unwrap().as_slice(), + [Event::Completions { request_id: Some(id), .. }] if id == "b-completion") + ); + let command_id = snapshot_b.last_command_id.saturating_add(1); + transport_b + .dispatch(Method::Cancel { + command: WorkerCommandEnvelope::new(command_id), + }) + .unwrap(); + let mut b_events = transport_b.events; + backend.run_on_adapter_runtime(async move { + tokio::time::timeout(Duration::from_secs(5), async { + while let Some(event) = b_events.recv().await { + if matches!(event, Event::CommandAcknowledged { acknowledgement } if acknowledgement.command_id == command_id) { + return Ok(()); + } + } + Err("fresh B protocol stream closed before command acknowledgement".to_string()) + }).await.map_err(|_| "fresh B protocol command acknowledgement timed out".to_string())? + }).unwrap(); + assert!( + matches!( + transport_a.events.try_recv(), + Err(tokio::sync::mpsc::error::TryRecvError::Disconnected) + ), + "A must receive no restored B protocol events" + ); + assert!(backend.stop_worker(&request.worker_ref).is_accepted()); + } + + fn profile_stop_fixture( + root: &Path, + ) -> (WorkerRuntimeExecutionBackend, WorkerExecutionRestoreRequest) { + let runtime_store = root.join("runtime"); + let backend = WorkerRuntimeExecutionBackend::new( + ProfileRuntimeWorkerFactory::new(root) + .with_runtime_store_dir(&runtime_store) + .with_runtime_id("runtime-subjektiv-test"), + ) + .unwrap(); + let request = backend + .run_on_adapter_runtime(async move { + Ok(saved_subjektiv_restore_fixture(&runtime_store, true).await) + }) + .unwrap(); + assert!(matches!( + backend.restore_worker(request.clone()), + WorkerExecutionSpawnResult::Connected { .. } + )); + (backend, request) + } + + // Queue a real Cancel first so the exact generated Shutdown ID is either + // already used by another kind (Conflict) or below the Controller floor. + // Install the same owned evidence as request_shutdown before enqueueing. + async fn prepare_rejected_shutdown( + execution: &RuntimeWorkerExecution, + stale: bool, + release: Option>, + ) -> u64 { + let command = next_internal_command_for_snapshot(&execution.handle.shared_state.snapshot()); + let cancel_id = command.command_id + u64::from(stale); + let mut cancel_events = execution.handle.subscribe(); + execution + .handle + .send(Method::Cancel { + command: WorkerCommandEnvelope::new(cancel_id), + }) + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if matches!(cancel_events.recv().await.unwrap(), Event::CommandAcknowledged { acknowledgement } + if acknowledgement.command_id == cancel_id + && acknowledgement.command == protocol::WorkerCommandKind::Cancel) { + break; + } + } + }).await.unwrap(); + let events = execution.handle.subscribe(); + let handle = execution.handle.clone(); + *execution.tasks.shutdown_admission.lock().await = Some(RuntimeShutdownAdmission { + command_id: command.command_id, + send: Some(tokio::spawn(async move { + if let Some(release) = release { + release.await.map_err(|error| error.to_string())?; + } + handle + .send(Method::Shutdown { command }) + .await + .map_err(|error| error.to_string()) + })), + events, + acknowledgement: None, + uncertainty: None, + }); + execution.shutdown_requested.store(true, Ordering::Release); + command.command_id + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn stop_rejected_shutdown_resets_request_and_retries_from_actual_snapshot() { + for stale in [false, true] { + let root = tempfile::tempdir().unwrap(); + let (backend, request) = profile_stop_fixture(root.path()); + let execution = backend + .workers + .lock() + .unwrap() + .get(&request.worker_ref) + .unwrap() + .clone(); + let preparing = execution.clone(); + let rejected_id = backend + .run_on_adapter_runtime(async move { + Ok(prepare_rejected_shutdown(&preparing, stale, None).await) + }) + .unwrap(); + // Explicitly stale bridge cache must not determine the retry ID. + execution.worker_state.write().unwrap().last_command_id = 0; + let result = backend.stop_worker(&request.worker_ref); + assert!(!result.is_accepted()); + let message = result.message.unwrap(); + assert!( + message.contains(if stale { "StaleCommandId" } else { "Conflict" }), + "{message}" + ); + assert!(!execution.shutdown_requested.load(Ordering::Acquire)); + assert!( + backend + .workers + .lock() + .unwrap() + .contains_key(&request.worker_ref) + ); + assert!(backend.stop_worker(&request.worker_ref).is_accepted()); + assert!(execution.handle.shared_state.snapshot().last_command_id > rejected_id); + assert!(backend.workers.lock().unwrap().is_empty()); + assert!(execution.tasks.tasks.lock().unwrap().is_empty()); + } + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn stop_late_shutdown_rejection_after_timeout_retains_waiter_without_double_enqueue() { + let root = tempfile::tempdir().unwrap(); + let (backend, request) = profile_stop_fixture(root.path()); + let execution = backend + .workers + .lock() + .unwrap() + .get(&request.worker_ref) + .unwrap() + .clone(); + let checking = execution.clone(); + backend + .run_on_adapter_runtime(async move { + let (release, gate) = tokio::sync::oneshot::channel(); + let command_id = prepare_rejected_shutdown(&checking, false, Some(gate)).await; + for _ in 0..2 { + let error = checking + .tasks + .request_shutdown( + &checking.handle, + &checking.shutdown_requested, + Duration::from_millis(10), + ) + .await + .unwrap_err(); + assert!(error.contains("timed out"), "{error}"); + assert!(checking.shutdown_requested.load(Ordering::Acquire)); + let retained = checking.tasks.shutdown_admission.lock().await; + let pending = retained.as_ref().unwrap(); + assert_eq!(pending.command_id, command_id); + assert!( + pending + .send + .as_ref() + .is_some_and(|task| !task.is_finished()) + ); + assert!(pending.acknowledgement.is_none()); + } + // A mismatched ID or command kind is not this Shutdown's verdict. + for (id, kind) in [ + (command_id + 1, protocol::WorkerCommandKind::Shutdown), + (command_id, protocol::WorkerCommandKind::Cancel), + ] { + checking + .handle + .send_event(Event::CommandAcknowledged { + acknowledgement: protocol::WorkerCommandAcknowledgement { + command_id: id, + command: kind, + disposition: protocol::WorkerCommandDisposition::Accepted, + state: checking.handle.shared_state.snapshot(), + }, + }) + .unwrap(); + } + release.send(()).unwrap(); + let error = checking + .tasks + .request_shutdown( + &checking.handle, + &checking.shutdown_requested, + Duration::from_secs(5), + ) + .await + .unwrap_err(); + assert!(error.contains("Conflict"), "{error}"); + assert!(!checking.shutdown_requested.load(Ordering::Acquire)); + assert!(checking.tasks.shutdown_admission.lock().await.is_none()); + Ok(()) + }) + .unwrap(); + assert!(backend.stop_worker(&request.worker_ref).is_accepted()); + assert!(execution.tasks.tasks.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn shutdown_ack_timeout_then_late_rejection_keeps_exact_receiver() { + let (events, receiver) = tokio::sync::broadcast::channel(4); + let mut pending = RuntimeShutdownAdmission { + command_id: 37, + send: None, + events: receiver, + acknowledgement: None, + uncertainty: None, + }; + assert!( + pending + .wait(Duration::from_millis(10)) + .await + .unwrap_err() + .contains("timed out") + ); + assert!(pending.uncertainty.as_ref().unwrap().contains("timed out")); + let acknowledgement = protocol::WorkerCommandAcknowledgement { + command_id: 37, + command: protocol::WorkerCommandKind::Shutdown, + disposition: protocol::WorkerCommandDisposition::StaleCommandId, + state: protocol::WorkerStateSnapshot::initial(), + }; + events + .send(Event::CommandAcknowledged { + acknowledgement: acknowledgement.clone(), + }) + .unwrap(); + pending.wait(Duration::from_secs(1)).await.unwrap(); + assert_eq!(pending.acknowledgement, Some(acknowledgement)); + } + + #[tokio::test] + async fn shutdown_ack_lag_and_closed_retain_correlated_unknown_evidence() { + let (events, receiver) = tokio::sync::broadcast::channel(1); + let mut pending = RuntimeShutdownAdmission { + command_id: 38, + send: None, + events: receiver, + acknowledgement: None, + uncertainty: None, + }; + for _ in 0..2 { + events.send(Event::Shutdown).unwrap(); + } + assert!(pending.wait(Duration::from_secs(1)).await.is_err()); + assert!(pending.uncertainty.is_some()); + assert!(pending.acknowledgement.is_none()); + drop(events); + assert!(pending.wait(Duration::from_secs(1)).await.is_err()); + assert_eq!(pending.command_id, 38); + assert!(pending.uncertainty.is_some()); + assert!(pending.acknowledgement.is_none()); + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn unconnected_cleanup_rejected_shutdown_retains_scope_for_retry() { + let root = tempfile::tempdir().unwrap(); + let (backend, request) = profile_stop_fixture(root.path()); + let execution = backend + .workers + .lock() + .unwrap() + .get(&request.worker_ref) + .unwrap() + .clone(); + let preparing = execution.clone(); + backend + .run_on_adapter_runtime(async move { + prepare_rejected_shutdown(&preparing, true, None).await; + Ok(()) + }) + .unwrap(); + let cleanup = || { + backend.cleanup_unconnected_controller( + &execution.handle, + &execution.shutdown, + &execution.tasks, + &execution.shutdown_requested, + &execution.worker_state, + ) + }; + assert!(cleanup().unwrap_err().contains("StaleCommandId")); + assert!(!execution.shutdown_requested.load(Ordering::Acquire)); + assert!(!execution.tasks.tasks.lock().unwrap().is_empty()); + assert!(cleanup().is_ok()); + assert!(execution.tasks.tasks.lock().unwrap().is_empty()); + assert!(backend.stop_worker(&request.worker_ref).is_accepted()); + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn duplicate_connect_rejected_shutdown_retains_actual_candidate_until_stop_retry() { + for operation in [ + WorkerExecutionOperation::Spawn, + WorkerExecutionOperation::Restore, + ] { + let root = tempfile::tempdir().unwrap(); + let candidate_root = tempfile::tempdir().unwrap(); + let (backend, request) = profile_stop_fixture(root.path()); + let primary = backend + .workers + .lock() + .unwrap() + .get(&request.worker_ref) + .unwrap() + .clone(); + let (candidate_backend, candidate_request) = + profile_stop_fixture(candidate_root.path()); + let candidate = candidate_backend + .workers + .lock() + .unwrap() + .remove(&candidate_request.worker_ref) + .unwrap(); + assert!(!primary.handle.same_controller(&candidate.handle)); + let preparing = candidate.clone(); + candidate_backend + .run_on_adapter_runtime(async move { + prepare_rejected_shutdown(&preparing, true, None).await; + Ok(()) + }) + .unwrap(); + let result = { + let lock = backend.worker_lock(&request.worker_ref).unwrap(); + let _guard = lock.lock().unwrap(); + backend.connect_controller_scope( + operation, + request.worker_ref.clone(), + request.context.clone(), + candidate.handle.clone(), + candidate.shutdown.clone(), + candidate.tasks.clone(), + candidate.shutdown_requested.clone(), + Some(request.operation_id), + BTreeMap::new(), + candidate.workspace_client.clone(), + ) + }; + match result { + WorkerExecutionSpawnResult::ReconciliationRequired { + result, + worker_state, + .. + } => { + assert!(result.message.unwrap().contains("StaleCommandId")); + assert!( + worker_state.is_none(), + "existing snapshot is not candidate cleanup proof" + ); + } + other => panic!( + "duplicate cleanup failure was classified as side-effect-free: {other:?}" + ), + } + let retained = primary.tasks.cleanup_candidates.lock().unwrap()[0].clone(); + assert!(retained.handle.same_controller(&candidate.handle)); + assert!(Arc::ptr_eq(&retained.shutdown, &candidate.shutdown)); + assert!(Arc::ptr_eq(&retained.tasks.tasks, &candidate.tasks.tasks)); + assert!(Arc::ptr_eq( + &retained.tasks.shutdown_admission, + &candidate.tasks.shutdown_admission + )); + let registered = backend + .workers + .lock() + .unwrap() + .get(&request.worker_ref) + .unwrap() + .clone(); + assert!(registered.handle.same_controller(&primary.handle)); + assert_eq!(backend.workers.lock().unwrap().len(), 1); + assert!( + backend + .preflight_restore(&request) + .unwrap_err() + .worker_state + .is_none() + ); + assert!(matches!( + backend.restore_worker(request.clone()), + WorkerExecutionSpawnResult::ReconciliationRequired { .. } + )); + + // Fail candidate cleanup once more during Stop. The main Controller + // must still join, but that cannot turn the failed child into Stopped. + let preparing = candidate.clone(); + candidate_backend + .run_on_adapter_runtime(async move { + prepare_rejected_shutdown(&preparing, false, None).await; + Ok(()) + }) + .unwrap(); + let stopped = backend.stop_worker(&request.worker_ref); + assert!(!stopped.is_accepted()); + assert!(stopped.message.unwrap().contains("Conflict")); + assert!(primary.handle.protocol_is_closed()); + assert!(primary.tasks.tasks.lock().unwrap().is_empty()); + let checking_scope = primary.tasks.clone(); + let incomplete_scope = + backend.run_on_adapter_runtime(async move { checking_scope.join().await }); + assert!( + incomplete_scope + .unwrap_err() + .contains("unconnected Controller cleanup remains unproven") + ); + assert!( + backend + .workers + .lock() + .unwrap() + .contains_key(&request.worker_ref) + ); + assert_eq!(primary.tasks.cleanup_candidates.lock().unwrap().len(), 1); + assert!(!candidate.tasks.tasks.lock().unwrap().is_empty()); + assert!(backend.stop_worker(&request.worker_ref).is_accepted()); + assert!(candidate.tasks.tasks.lock().unwrap().is_empty()); + assert!(primary.tasks.cleanup_candidates.lock().unwrap().is_empty()); + assert!(backend.workers.lock().unwrap().is_empty()); + } + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn duplicate_connect_timeout_retains_candidate_waiter_and_no_false_stop() { + let root = tempfile::tempdir().unwrap(); + let candidate_root = tempfile::tempdir().unwrap(); + let (backend, request) = profile_stop_fixture(root.path()); + let primary = backend + .workers + .lock() + .unwrap() + .get(&request.worker_ref) + .unwrap() + .clone(); + let (candidate_backend, candidate_request) = profile_stop_fixture(candidate_root.path()); + let candidate = candidate_backend + .workers + .lock() + .unwrap() + .remove(&candidate_request.worker_ref) + .unwrap(); + let (release, gate) = tokio::sync::oneshot::channel(); + let preparing = candidate.clone(); + let command_id = candidate_backend + .run_on_adapter_runtime(async move { + Ok(prepare_rejected_shutdown(&preparing, false, Some(gate)).await) + }) + .unwrap(); + let result = { + let lock = backend.worker_lock(&request.worker_ref).unwrap(); + let _guard = lock.lock().unwrap(); + backend.connect_controller_scope( + WorkerExecutionOperation::Restore, + request.worker_ref.clone(), + request.context.clone(), + candidate.handle.clone(), + candidate.shutdown.clone(), + candidate.tasks.clone(), + candidate.shutdown_requested.clone(), + Some(request.operation_id), + BTreeMap::new(), + candidate.workspace_client.clone(), + ) + }; + assert!(matches!( + result, + WorkerExecutionSpawnResult::ReconciliationRequired { .. } + )); + let stopped = backend.stop_worker(&request.worker_ref); + assert!(!stopped.is_accepted()); + assert!(stopped.message.unwrap().contains("timed out")); + assert!(primary.handle.protocol_is_closed()); + let retained = primary.tasks.cleanup_candidates.lock().unwrap()[0].clone(); + assert!(retained.handle.same_controller(&candidate.handle)); + assert!(Arc::ptr_eq( + &retained.tasks.shutdown_admission, + &candidate.tasks.shutdown_admission + )); + let checking = retained.clone(); + backend + .run_on_adapter_runtime(async move { + let admission = checking.tasks.shutdown_admission.lock().await; + let pending = admission.as_ref().unwrap(); + assert_eq!( + pending.command_id, command_id, + "unknown admission must not enqueue another command" + ); + assert!( + pending + .send + .as_ref() + .is_some_and(|task| !task.is_finished()) + ); + assert!(pending.uncertainty.as_ref().unwrap().contains("timed out")); + assert!(pending.acknowledgement.is_none()); + Ok(()) + }) + .unwrap(); + assert!( + backend + .workers + .lock() + .unwrap() + .contains_key(&request.worker_ref) + ); + release.send(()).unwrap(); + let rejected = backend.stop_worker(&request.worker_ref); + assert!(!rejected.is_accepted()); + assert!(rejected.message.unwrap().contains("Conflict")); + assert!(!candidate.shutdown_requested.load(Ordering::Acquire)); + assert_eq!(primary.tasks.cleanup_candidates.lock().unwrap().len(), 1); + assert!(backend.stop_worker(&request.worker_ref).is_accepted()); + assert!(candidate.tasks.tasks.lock().unwrap().is_empty()); + assert!(primary.tasks.tasks.lock().unwrap().is_empty()); + assert!(primary.tasks.cleanup_candidates.lock().unwrap().is_empty()); + assert!(backend.workers.lock().unwrap().is_empty()); + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn pending_restore_join_failure_never_becomes_absence_on_stop_retry() { + let root = tempfile::tempdir().unwrap(); + let runtime_store = root.path().join("runtime"); + let backend = WorkerRuntimeExecutionBackend::new( + ProfileRuntimeWorkerFactory::new(root.path()) + .with_runtime_store_dir(&runtime_store) + .with_runtime_id("runtime-subjektiv-test"), + ) + .unwrap(); + let request = backend + .run_on_adapter_runtime(async move { + Ok(saved_subjektiv_restore_fixture(&runtime_store, true).await) + }) + .unwrap(); + let failed = backend + .spawn_on_adapter_runtime(async { + panic!("factory scope failed before publishing result") + }) + .unwrap(); + let pending = Arc::new(PendingRuntimeRestore { + request: request.clone(), + task: tokio::sync::Mutex::new(Some(failed)), + result: Mutex::new(None), + failure: Mutex::new(None), + }); + backend + .pending_restores + .lock() + .unwrap() + .insert(request.worker_ref.clone(), pending.clone()); + for _ in 0..3 { + let stopped = backend.stop_worker(&request.worker_ref); + assert!(!stopped.is_accepted()); + assert!( + stopped + .message + .unwrap() + .contains("restore factory task failed") + ); + assert!(pending.failure.lock().unwrap().is_some()); + assert!(pending.result.lock().unwrap().is_none()); + assert!(Arc::ptr_eq( + &backend + .pending_restore(&request.worker_ref) + .unwrap() + .unwrap(), + &pending + )); + assert!(backend.workers.lock().unwrap().is_empty()); + } + assert!(matches!( + backend.restore_worker(request), + WorkerExecutionSpawnResult::ReconciliationRequired { .. } + )); + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn stop_joins_healthy_self_terminated_controller_with_closed_endpoint() { + let root = tempfile::tempdir().unwrap(); + let (backend, request) = profile_stop_fixture(root.path()); + let execution = backend + .workers + .lock() + .unwrap() + .get(&request.worker_ref) + .unwrap() + .clone(); + let handle = execution.handle.clone(); + let controller = execution.tasks.tasks.lock().unwrap()[0].clone(); + backend + .run_on_adapter_runtime(async move { + handle + .send(Method::Shutdown { + command: WorkerCommandEnvelope::new(1), + }) + .await + .map_err(|error| error.to_string())?; + tokio::time::timeout(Duration::from_secs(5), async { + loop { + if handle.protocol_is_closed() + && controller + .completion + .lock() + .await + .task + .as_ref() + .unwrap() + .is_finished() + { + break; + } + tokio::task::yield_now().await; + } + }) + .await + .map_err(|error| error.to_string())?; + Ok(()) + }) + .unwrap(); + assert!(!execution.shutdown_requested.load(Ordering::Acquire)); + assert!(backend.stop_worker(&request.worker_ref).is_accepted()); + assert!(backend.workers.lock().unwrap().is_empty()); + assert!(execution.tasks.tasks.lock().unwrap().is_empty()); + assert!(backend.stop_worker(&request.worker_ref).is_accepted()); + } + + #[test] + #[serial_test::serial(worker_allocation)] + fn stop_and_unconnected_cleanup_retain_abnormal_controller_completion_across_retries() { + let root = tempfile::tempdir().unwrap(); + let (backend, request) = profile_stop_fixture(root.path()); + let execution = backend + .workers + .lock() + .unwrap() + .get(&request.worker_ref) + .unwrap() + .clone(); + let controller = execution.tasks.tasks.lock().unwrap()[0].clone(); + let aborted_controller = Arc::clone(&controller); + backend + .run_on_adapter_runtime(async move { + aborted_controller + .completion + .lock() + .await + .task + .as_ref() + .unwrap() + .abort(); + tokio::time::timeout(Duration::from_secs(5), async { + while !aborted_controller + .completion + .lock() + .await + .task + .as_ref() + .unwrap() + .is_finished() + { + tokio::task::yield_now().await; + } + }) + .await + .map_err(|error| error.to_string())?; + Ok(()) + }) + .unwrap(); + for _ in 0..3 { + let cleanup = backend + .cleanup_unconnected_controller( + &execution.handle, + &execution.shutdown, + &execution.tasks, + &execution.shutdown_requested, + &execution.worker_state, + ) + .unwrap_err(); + assert!(cleanup.contains("controller task failed"), "{cleanup}"); + let stopped = backend.stop_worker(&request.worker_ref); + assert_eq!( + stopped.outcome, + crate::execution::WorkerExecutionOutcome::Errored + ); + assert!(stopped.message.unwrap().contains("controller task failed")); + let workers = backend.workers.lock().unwrap(); + let retained = workers + .get(&request.worker_ref) + .expect("unknown cleanup must retain owner"); + assert!(retained.handle.same_controller(&execution.handle)); + assert!(Arc::ptr_eq( + &retained.tasks.tasks.lock().unwrap()[0], + &controller + )); + } + let retained = execution.tasks.tasks.lock().unwrap().clone(); + assert_eq!( + retained.len(), + 2, + "failed task and failed shutdown proof remain; relays drain" + ); + } + + #[tokio::test] + #[serial_test::serial(worker_allocation)] + async fn profile_factory_saved_subjektiv_manifest_survives_restart_and_explicit_restore() { + let root = tempfile::tempdir().unwrap(); + let runtime_store = root.path().join("runtime"); + let mut request = saved_subjektiv_restore_fixture(&runtime_store, true).await; + let worker_name = + ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&request.worker_ref); + let aggregate_dir = runtime_store + .join("workers") + .join(request.worker_ref.worker_id.to_string()); + let metadata_store = WorkerAggregateStore::new(&aggregate_dir, &worker_name).unwrap(); + let saved_metadata = metadata_store.read_by_name(&worker_name).unwrap().unwrap(); + let saved_active = saved_metadata.active.as_ref().unwrap(); + let session_store = WorkerSessionStore::new(aggregate_dir.join("session")).unwrap(); + for _ in 0..2 { + // Fresh factory: exercise the same saved authority after both a + // restart and another explicit restore, not a mock Controller path. + let factory = + ProfileRuntimeWorkerFactory::new(root.path().join("nonexistent-current-profiles")) + .with_runtime_store_dir(&runtime_store) + .with_runtime_id("runtime-subjektiv-test"); + let before = persisted_files(&runtime_store); + factory.preflight_restore(&request).await.unwrap(); + assert_eq!( + persisted_files(&runtime_store), + before, + "preflight must be read-only" + ); + assert!(factory.observation_hub.workers.lock().unwrap().is_empty()); + let controller = factory + .restore_controller(request.clone()) + .await + .unwrap_or_else(|error| panic!("saved Subjektiv restore failed: {error}")); + assert_eq!( + controller.handle.shared_state.catalog_status(), + WorkerStatus::Idle + ); + shutdown_profile_controller(controller).await; + let restored_metadata = metadata_store.read_by_name(&worker_name).unwrap().unwrap(); + assert_eq!(restored_metadata.active, saved_metadata.active); + assert_eq!( + restored_metadata.resolved_manifest_snapshot, + saved_metadata.resolved_manifest_snapshot + ); + assert_eq!( + restored_metadata.subjektiv_session_attribution, + saved_metadata.subjektiv_session_attribution + ); + let entries = session_store + .read_all(saved_active.session_id, saved_active.segment_id.unwrap()) + .unwrap(); + assert!( + matches!(entries.first(), Some(LogEntry::AnnotatedSegmentStart { + system_prompt: Some(prompt), + .. + }) if prompt == SAVED_SUBJEKTIV_SYSTEM_PROMPT) + ); + assert_eq!( + session_store::collect_state(&entries) + .system_prompt + .as_deref(), + Some(SAVED_SUBJEKTIV_SYSTEM_PROMPT), + "actual restore must replay the committed prompt rather than re-render it" + ); + request.operation_id = crate::execution::WorkerLifecycleOperationId::new(); + } + assert!(!root.path().join("nonexistent-current-profiles").exists()); + } + + #[tokio::test] + #[serial_test::serial(worker_allocation)] + async fn profile_factory_pending_subjektiv_restore_requires_operation_owned_launch_material() { + let root = tempfile::tempdir().unwrap(); + let runtime_store = root.path().join("runtime"); + let mut request = saved_subjektiv_restore_fixture(&runtime_store, false).await; + let factory = ProfileRuntimeWorkerFactory::new(root.path()) + .with_runtime_store_dir(&runtime_store) + .with_runtime_id("runtime-subjektiv-test"); + let worker_name = + ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&request.worker_ref); + let store = WorkerAggregateStore::new( + factory.worker_aggregate_dir(&request.worker_ref).unwrap(), + &worker_name, + ) + .unwrap(); + let metadata = store.read_by_name(&worker_name).unwrap().unwrap(); + let pending_session_id = metadata.active.as_ref().unwrap().session_id; + assert!(metadata.active.as_ref().unwrap().segment_id.is_none()); + let before = persisted_files(&runtime_store); + let error = factory.preflight_restore(&request).await.unwrap_err(); + assert!( + error.contains("Pending Workspace Worker restore requires"), + "{error}" + ); + assert_eq!(persisted_files(&runtime_store), before); + let mut bundle = test_bundle(); + bundle.metadata.workspace_id = "workspace-saved-subjektiv".to_string(); + let builtins = worker::PromptCatalog::builtins_only().unwrap(); + let projection = builtins.projection(); + let mut templates = projection.templates.clone(); + templates.insert( + "default".to_string(), + "OPERATION-OWNED-PENDING-SYSTEM-PROMPT".to_string(), + ); + templates.insert( + "internal.notify_wrapper".to_string(), + "SAVED {{ message }}".to_string(), + ); + let mut catalog = worker::EffectivePromptCatalog::new( + templates, + 17, + projection.schema_fingerprint.clone(), + projection.toolchain_fingerprint.clone(), + ) + .unwrap(); + catalog.source_digest = "saved-source".to_string(); + bundle.prompt_catalog = Some(catalog); + let bundle = bundle.with_computed_digest(); + let mut missing_projection = request.clone(); + let mut missing_projection_bundle = bundle.clone(); + missing_projection_bundle.prompt_catalog = None; + missing_projection.config_bundle = Some(missing_projection_bundle.with_computed_digest()); + let mut wrong_workspace = request.clone(); + let mut wrong_workspace_bundle = bundle.clone(); + wrong_workspace_bundle.metadata.workspace_id = "another-workspace".to_string(); + wrong_workspace.config_bundle = Some(wrong_workspace_bundle.with_computed_digest()); + let mut missing_instruction = request.clone(); + let mut missing_instruction_bundle = bundle.clone(); + let mut templates = projection.templates.clone(); + templates.remove("default"); + missing_instruction_bundle.prompt_catalog = Some( + worker::EffectivePromptCatalog::new( + templates, + 17, + projection.schema_fingerprint.clone(), + projection.toolchain_fingerprint.clone(), + ) + .unwrap(), + ); + missing_instruction.config_bundle = Some(missing_instruction_bundle.with_computed_digest()); + for (invalid, expected) in [ + (request.clone(), "requires operation-owned launch material"), + ( + missing_projection, + "requires a saved Workspace Prompt projection", + ), + ( + wrong_workspace, + "Workspace Prompt projection scope mismatch", + ), + (missing_instruction, "invalid pending Worker launch Prompt"), + ] { + let error = factory.preflight_restore(&invalid).await.unwrap_err(); + assert!(error.contains(expected), "{error}"); + let error = match factory.restore_controller(invalid).await { + Ok(controller) => { + shutdown_profile_controller(controller).await; + panic!("invalid pending launch authority started a Controller"); + } + Err(error) => error, + }; + assert!(error.contains(expected), "{error}"); + assert_eq!(persisted_files(&runtime_store), before); + assert!( + factory + .prompt_projection_cache + .active("workspace-saved-subjektiv") + .unwrap() + .is_none() + ); + assert!(factory.observation_hub.workers.lock().unwrap().is_empty()); + } + request.config_bundle = Some(bundle); + factory.preflight_restore(&request).await.unwrap(); + assert_eq!(persisted_files(&runtime_store), before); + assert!( + factory + .prompt_projection_cache + .active("workspace-saved-subjektiv") + .unwrap() + .is_none() + ); + let controller = factory + .restore_controller(request.clone()) + .await + .unwrap_or_else(|error| panic!("pending saved Subjektiv restore failed: {error}")); + assert_eq!( + controller.handle.shared_state.catalog_status(), + WorkerStatus::Idle + ); + shutdown_profile_controller(controller).await; + let restored = store.read_by_name(&worker_name).unwrap().unwrap(); + assert_eq!( + restored.resolved_manifest_snapshot, metadata.resolved_manifest_snapshot, + "pending launch material must not replace the saved Manifest" + ); + assert_eq!( + restored.subjektiv_session_attribution, metadata.subjektiv_session_attribution, + "pending restore must retain committed Subject attribution" + ); + let active = restored.active.unwrap(); + assert_eq!(active.session_id, pending_session_id); + let segment_id = active + .segment_id + .expect("operation-owned launch must commit the initial Segment before exposure"); + let entries = WorkerSessionStore::new( + factory + .worker_aggregate_dir(&request.worker_ref) + .unwrap() + .join("session"), + ) + .unwrap() + .read_all(active.session_id, segment_id) + .unwrap(); + assert!( + matches!(entries.first(), Some(LogEntry::AnnotatedSegmentStart { + session_id, + system_prompt: Some(prompt), + history, + forked_from: None, + compacted_from: None, + .. + }) if *session_id == pending_session_id + && prompt.contains("OPERATION-OWNED-PENDING-SYSTEM-PROMPT") + && history.is_empty()) + ); + assert!( + session_store::collect_state(&entries) + .system_prompt + .is_some() + ); + } + + #[tokio::test] + #[serial_test::serial(worker_allocation)] + async fn profile_factory_saved_manifest_mismatch_rejected_before_live_work() { + let root = tempfile::tempdir().unwrap(); + let runtime_store = root.path().join("runtime"); + let request = saved_subjektiv_restore_fixture(&runtime_store, true).await; + let factory = ProfileRuntimeWorkerFactory::new(root.path()) + .with_runtime_store_dir(&runtime_store) + .with_runtime_id("runtime-subjektiv-test"); + let before = persisted_files(&runtime_store); + let mut wrong_revision = request.clone(); + wrong_revision + .request + .memory_settings + .as_mut() + .unwrap() + .settings_revision = 18; + let mut detached = request.clone(); + detached.request.subjektiv_attached = false; + let mut missing_settings = request.clone(); + missing_settings.request.memory_settings = None; + for (invalid, expected) in [ + (wrong_revision, "subjektiv settings snapshot mismatch"), + (detached, "unauthorized subjektiv attachment"), + (missing_settings, "missing its trusted settings snapshot"), + ] { + let error = factory.preflight_restore(&invalid).await.unwrap_err(); + assert!(error.contains(expected), "{error}"); + let error = match factory.restore_controller(invalid).await { + Ok(controller) => { + shutdown_profile_controller(controller).await; + panic!("invalid restore started a Controller") + } + Err(error) => error, + }; + assert!(error.contains(expected), "{error}"); + assert_eq!(persisted_files(&runtime_store), before); + assert!(factory.observation_hub.workers.lock().unwrap().is_empty()); + } + let run_dir = factory + .worker_aggregate_dir(&request.worker_ref) + .unwrap() + .join("runs"); + assert!( + !run_dir.exists(), + "mismatch must reject before creating any run resources" + ); + } + #[test] fn profile_factory_reads_retained_session_from_canonical_worker_aggregate() { let root = tempfile::tempdir().unwrap(); @@ -3861,7 +6515,7 @@ mod tests { } #[tokio::test] - async fn execution_scope_drains_remaining_tasks_after_join_failure_and_allows_retry() { + async fn execution_scope_drains_remaining_tasks_and_retains_abnormal_completion() { let failed = tokio::spawn(async { panic!("injected controller failure") }); let scope = RuntimeExecutionTaskScope::new(failed); scope.push( @@ -3872,8 +6526,81 @@ mod tests { let error = scope.join().await.unwrap_err(); assert!(error.contains("controller task failed")); - assert!(scope.tasks.lock().unwrap().is_empty()); + let retained = scope.tasks.lock().unwrap().clone(); + assert_eq!(retained.len(), 1, "only the failed completion must remain"); + assert_eq!(retained[0].name, "controller"); + assert!(retained[0].completion.lock().await.failure.is_some()); + for _ in 0..3 { + assert_eq!(scope.join().await.unwrap_err(), error); + } + } + + #[tokio::test] + async fn execution_scope_retains_closed_shutdown_completion_after_other_tasks_drain() { + let scope = RuntimeExecutionTaskScope::new(tokio::spawn(async {})); + scope.push( + "protocol bridge", + tokio::spawn(std::future::pending()), + true, + ); + let (sender, receiver) = tokio::sync::oneshot::channel(); + drop(sender); + let shutdown = Arc::new(tokio::sync::Mutex::new(Some(receiver))); + for _ in 0..3 { + let error = scope + .confirm_shutdown_and_join(&shutdown) + .await + .unwrap_err(); + assert!(error.contains("completion channel closed"), "{error}"); + assert_eq!(scope.tasks.lock().unwrap().len(), 1); + } + assert!(shutdown.lock().await.is_none()); + } + + #[tokio::test] + async fn cancelled_execution_scope_join_keeps_actual_task_owned_for_retry() { + let release = Arc::new(tokio::sync::Notify::new()); + let task_release = Arc::clone(&release); + let scope = RuntimeExecutionTaskScope::new(tokio::spawn(async move { + task_release.notified().await; + })); + let owned = scope.tasks.lock().unwrap()[0].clone(); + let joining_scope = scope.clone(); + let joining = tokio::spawn(async move { joining_scope.join().await }); + tokio::time::timeout(Duration::from_secs(5), async { + while owned.completion.try_lock().is_ok() { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + joining.abort(); + assert!(joining.await.unwrap_err().is_cancelled()); + assert!(Arc::ptr_eq(&scope.tasks.lock().unwrap()[0], &owned)); + assert!(owned.completion.lock().await.task.is_some()); + release.notify_one(); scope.join().await.unwrap(); + assert!(scope.tasks.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn execution_scope_timeout_retains_exact_task_until_completion_is_proven() { + let release = Arc::new(tokio::sync::Notify::new()); + let task_release = Arc::clone(&release); + let scope = RuntimeExecutionTaskScope::new(tokio::spawn(async move { + task_release.notified().await; + })); + let owned = scope.tasks.lock().unwrap()[0].clone(); + let error = scope.join().await.unwrap_err(); + assert!( + error.contains("task did not stop before timeout"), + "{error}" + ); + assert!(Arc::ptr_eq(&scope.tasks.lock().unwrap()[0], &owned)); + assert!(owned.completion.lock().await.task.is_some()); + release.notify_one(); + scope.join().await.unwrap(); + assert!(scope.tasks.lock().unwrap().is_empty()); } fn adapter_command( @@ -4521,6 +7248,58 @@ mod tests { workspace_api: None, memory_settings: None, subjektiv_attached: false, + backend_job: None, + } + } + + #[test] + fn backend_job_profile_policy_matches_trusted_grant_without_implicit_changes() { + let mut manifest = WorkerManifest::from_toml( + r#" + [worker] + name = "job-policy-test" + pwd = "./" + [model] + scheme = "anthropic" + model_id = "chosen-model" + auth = { kind = "none" } + [engine] + instruction = "default" + [[scope.allow]] + target = "./" + permission = "read" + "#, + ) + .unwrap(); + let mut request = create_request("policy"); + for granted in [false, true] { + request.backend_job = Some(crate::catalog::BackendJobExecutionBinding { + job_id: "job-1".into(), + attempt_id: "attempt-1".into(), + input_revision: None, + subjektiv_consolidation: granted, + }); + for tools in [false, true] { + for extraction in [false, true] { + manifest.feature.subjektiv.profile.consolidation_tools = tools; + manifest.feature.subjektiv.profile.extraction.enabled = extraction; + let expected = tools == granted && (!granted || !extraction); + let before = serde_json::to_value(&manifest).unwrap(); + assert_eq!( + validate_backend_job_profile(&manifest, &request).is_ok(), + expected + ); + assert_eq!( + bind_workspace_memory_settings(&mut manifest, &request).is_ok(), + expected + ); + assert_eq!( + validate_worker_memory_settings(&manifest, &request).is_ok(), + expected + ); + assert_eq!(serde_json::to_value(&manifest).unwrap(), before); + } + } } } @@ -4614,6 +7393,564 @@ mod tests { } } + #[cfg(feature = "fs-store")] + struct RestartArtifactFailureFactory { + inner: MockFactory, + runtime_store: PathBuf, + restore_id: Arc>>, + } + + #[cfg(feature = "fs-store")] + #[async_trait] + impl RuntimeWorkerFactory for RestartArtifactFailureFactory { + async fn spawn_controller( + &self, + request: WorkerExecutionSpawnRequest, + ) -> Result { + self.inner.spawn_controller(request).await + } + + async fn restore_controller( + &self, + request: WorkerExecutionRestoreRequest, + ) -> Result { + *self.restore_id.lock().unwrap() = Some(request.operation_id); + let runs = self + .runtime_store + .join("workers") + .join(request.worker_ref.worker_id.to_string()) + .join("runs"); + fs::create_dir_all(&runs).unwrap(); + // Deterministic cleanup-validation failure, including for root users. + fs::write( + runs.join(format!("restore-{}", request.operation_id)), + b"incomplete restore artifact", + ) + .unwrap(); + Err("injected restore failure with persisted artifacts".to_string()) + } + + async fn cleanup_failed_restore( + &self, + _: &WorkerExecutionRestoreRequest, + ) -> Result<(), String> { + Err("injected restore artifact cleanup failure".to_string()) + } + } + + #[cfg(feature = "fs-store")] + #[test] + fn fs_restart_stop_cleans_prior_restore_artifacts_and_retains_journal_on_failure() { + let root = tempfile::tempdir().unwrap(); + let runtime_store = root.path().join("runtime"); + let options = crate::fs_store::FsRuntimeStoreOptions { + root: runtime_store.clone(), + runtime_id: "restart-artifact-runtime".to_string(), + display_name: None, + }; + let restore_id = Arc::new(Mutex::new(None)); + let backend = Arc::new( + WorkerRuntimeExecutionBackend::new(RestartArtifactFailureFactory { + inner: MockFactory { + client: MockClient::sequential(vec![]), + runtime_base: root.path().join("controllers"), + cwd: root.path().to_path_buf(), + store_dir: root.path().join("sessions"), + worker_metadata_dir: root.path().join("metadata"), + observed_cwds: Arc::new(Mutex::new(Vec::new())), + observed_workspace_clients: Arc::new(Mutex::new(Vec::new())), + }, + runtime_store: runtime_store.clone(), + restore_id: Arc::clone(&restore_id), + }) + .unwrap(), + ); + let runtime = + EmbeddedRuntime::with_fs_store_and_execution_backend(options.clone(), backend.clone()) + .unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(create_request("restart cleanup")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + assert!( + runtime + .restore_worker( + &worker.worker_ref, + runtime.test_restore_request(&worker.worker_ref) + ) + .is_err() + ); + assert!(runtime.stop_worker(&worker.worker_ref, None).is_err()); + let prior_restore_id = restore_id.lock().unwrap().unwrap(); + let aggregate = runtime_store + .join("workers") + .join(worker.worker_id.to_string()); + let journal_path = aggregate.join("worker.json"); + let journal_before = fs::read(&journal_path).unwrap(); + let journal: serde_json::Value = serde_json::from_slice(&journal_before).unwrap(); + assert_eq!(journal["execution_state"]["execution"]["operation"], "stop"); + assert_eq!( + journal["execution_state"]["execution"]["intent"]["pending_restore"]["operation_id"], + prior_restore_id.to_string() + ); + let artifact = aggregate + .join("runs") + .join(format!("restore-{prior_restore_id}")); + let create_run = aggregate + .join("runs") + .join(uuid::Uuid::now_v7().to_string()); + fs::create_dir_all(&create_run).unwrap(); + fs::write(create_run.join("keep.log"), "ordinary creation run").unwrap(); + let another_worker = runtime_store + .join("workers") + .join(WorkerId::now_v7().to_string()) + .join("runs") + .join(format!( + "restore-{}", + crate::execution::WorkerLifecycleOperationId::new() + )); + fs::create_dir_all(&another_worker).unwrap(); + fs::write(another_worker.join("keep.log"), "another Worker").unwrap(); + fs::create_dir_all(aggregate.join("session")).unwrap(); + fs::write( + aggregate.join("session").join("keep-authority"), + "saved Session authority", + ) + .unwrap(); + drop(runtime); + drop(backend); + + // All backend maps are empty after process restart. The nested journal + // restore id is not part of StopRequest; artifacts are discovered only + // below this exact Worker's aggregate, after excluding live resources. + let profile_base = root.path().join("must-not-launch-current-profile"); + let backend = Arc::new( + WorkerRuntimeExecutionBackend::new( + ProfileRuntimeWorkerFactory::new(&profile_base) + .with_runtime_store_dir(&runtime_store), + ) + .unwrap(), + ); + let restarted = + EmbeddedRuntime::with_fs_store_and_execution_backend(options.clone(), backend.clone()) + .unwrap(); + assert_eq!(fs::read(&journal_path).unwrap(), journal_before); + assert!(backend.workers.lock().unwrap().is_empty()); + assert!(backend.pending_restores.lock().unwrap().is_empty()); + assert!( + backend + .factory + .failed_restore_sessions + .lock() + .unwrap() + .is_empty() + ); + let failed = restarted.stop_worker(&worker.worker_ref, None).unwrap_err(); + assert!( + failed.to_string().contains("unsafe restore artifact"), + "{failed}" + ); + assert_eq!(fs::read(&journal_path).unwrap(), journal_before); + assert!(artifact.is_file()); + assert!(!profile_base.exists()); + + fs::remove_file(&artifact).unwrap(); + fs::create_dir(&artifact).unwrap(); + fs::write(artifact.join("worker.err.log"), "stale restore diagnostics").unwrap(); + restarted.stop_worker(&worker.worker_ref, None).unwrap(); + assert!(!artifact.exists()); + assert_eq!( + restarted.worker_detail(&worker.worker_ref).unwrap().status, + crate::catalog::WorkerStatus::Stopped + ); + let journal: serde_json::Value = + serde_json::from_slice(&fs::read(&journal_path).unwrap()).unwrap(); + assert_ne!( + journal["execution_state"]["state"], + "reconciliation_required" + ); + assert_eq!( + fs::read_to_string(aggregate.join("session").join("keep-authority")).unwrap(), + "saved Session authority" + ); + assert!(create_run.join("keep.log").is_file()); + assert!(another_worker.join("keep.log").is_file()); + assert!(!profile_base.exists()); + assert!(backend.workers.lock().unwrap().is_empty()); + restarted.stop_worker(&worker.worker_ref, None).unwrap(); + } + + #[tokio::test] + async fn profile_restore_artifact_failure_retains_failed_session_owner_for_retry() { + let root = tempfile::tempdir().unwrap(); + let factory = + ProfileRuntimeWorkerFactory::new(root.path()).with_runtime_store_dir(root.path()); + let worker_ref = WorkerRef::new(WorkerId::now_v7()); + let request = direct_restore_request(worker_ref.clone()); + let run = factory + .worker_restore_run_dir(&worker_ref, request.operation_id) + .unwrap(); + fs::create_dir_all(run.parent().unwrap()).unwrap(); + fs::write(&run, "invalid restore artifact").unwrap(); + let sessions = Arc::new(WorkdirSessionRouter::new()); + factory.failed_restore_sessions.lock().unwrap().insert( + worker_ref.clone(), + (request.operation_id, Arc::clone(&sessions)), + ); + assert!( + factory + .cleanup_failed_restore(&request) + .await + .unwrap_err() + .contains("unsafe restore artifact") + ); + assert!( + factory + .reconcile_stopped_worker(&worker_ref) + .await + .unwrap_err() + .contains("unsafe restore artifact") + ); + let retained = factory + .failed_restore_sessions + .lock() + .unwrap() + .get(&worker_ref) + .cloned() + .unwrap(); + assert_eq!(retained.0, request.operation_id); + assert!(Arc::ptr_eq(&retained.1, &sessions)); + fs::remove_file(&run).unwrap(); + fs::create_dir(&run).unwrap(); + fs::write(run.join("worker.out.log"), "stale restore artifact").unwrap(); + factory.reconcile_stopped_worker(&worker_ref).await.unwrap(); + assert!(!run.exists()); + assert!(factory.failed_restore_sessions.lock().unwrap().is_empty()); + } + + #[cfg(unix)] + #[test] + fn stopped_restore_artifact_cleanup_rejects_symlinks_and_noncanonical_names() { + use std::os::unix::fs::symlink; + let root = tempfile::tempdir().unwrap(); + let outside = tempfile::tempdir().unwrap(); + fs::write(outside.path().join("keep"), "outside authority").unwrap(); + let factory = + ProfileRuntimeWorkerFactory::new(root.path()).with_runtime_store_dir(root.path()); + let worker_ref = WorkerRef::new(WorkerId::now_v7()); + let run = factory + .worker_restore_run_dir( + &worker_ref, + crate::execution::WorkerLifecycleOperationId::new(), + ) + .unwrap(); + let runs = run.parent().unwrap(); + fs::create_dir_all(runs).unwrap(); + symlink(outside.path(), &run).unwrap(); + assert!( + factory + .stopped_restore_run_dirs(&worker_ref) + .unwrap_err() + .contains("unsafe restore artifact") + ); + fs::remove_file(&run).unwrap(); + fs::create_dir(&run).unwrap(); + symlink(outside.path(), run.join("escaped")).unwrap(); + assert!(factory.stopped_restore_run_dirs(&worker_ref).is_err()); + assert!(run.exists()); + fs::remove_file(run.join("escaped")).unwrap(); + let invalid = runs.join("restore-not-an-operation"); + fs::create_dir(&invalid).unwrap(); + assert!( + factory + .stopped_restore_run_dirs(&worker_ref) + .unwrap_err() + .contains("invalid restore artifact operation name") + ); + fs::remove_dir(&invalid).unwrap(); + let noncanonical = runs.join("restore-550E8400-E29B-41D4-A716-446655440000"); + fs::create_dir(&noncanonical).unwrap(); + assert!( + factory + .stopped_restore_run_dirs(&worker_ref) + .unwrap_err() + .contains("noncanonical restore artifact operation name") + ); + fs::remove_dir(&noncanonical).unwrap(); + fs::remove_dir(&run).unwrap(); + fs::remove_dir(runs).unwrap(); + symlink(outside.path(), runs).unwrap(); + assert!( + factory + .stopped_restore_run_dirs(&worker_ref) + .unwrap_err() + .contains("unsafe restore artifact directory") + ); + assert_eq!( + fs::read_to_string(outside.path().join("keep")).unwrap(), + "outside authority" + ); + } + + struct PendingFailureFactory { + restore_calls: Arc, + cleanup_calls: Arc, + cleanup_allowed: Arc, + } + + #[async_trait] + impl RuntimeWorkerFactory for PendingFailureFactory { + async fn spawn_controller( + &self, + _request: WorkerExecutionSpawnRequest, + ) -> Result { + panic!("stop must not spawn a Controller") + } + async fn restore_controller( + &self, + _request: WorkerExecutionRestoreRequest, + ) -> Result { + self.restore_calls.fetch_add(1, Ordering::SeqCst); + tokio::time::sleep(Duration::from_millis(50)).await; + Err("injected uncertain restore side effect".to_string()) + } + async fn cleanup_failed_restore( + &self, + _request: &WorkerExecutionRestoreRequest, + ) -> Result<(), String> { + self.cleanup_calls.fetch_add(1, Ordering::SeqCst); + if self.cleanup_allowed.load(Ordering::SeqCst) { + Ok(()) + } else { + Err("injected pending restore cleanup failure".to_string()) + } + } + } + + fn direct_restore_request(worker_ref: WorkerRef) -> WorkerExecutionRestoreRequest { + WorkerExecutionRestoreRequest { + operation_id: crate::execution::WorkerLifecycleOperationId::new(), + request: create_request("pending operation"), + workspace_scope: None, + context: test_execution_context(worker_ref.clone()), + worker_ref, + previous_workdir_attachments: Vec::new(), + logical_workdir_attachments: Vec::new(), + workdir_attachments: BTreeMap::new(), + config_bundle: None, + } + } + + #[test] + fn absent_execution_stop_reconciles_pending_restore_cleanup_without_launching() { + let restore_calls = Arc::new(AtomicUsize::new(0)); + let cleanup_calls = Arc::new(AtomicUsize::new(0)); + let cleanup_allowed = Arc::new(AtomicBool::new(false)); + let backend = WorkerRuntimeExecutionBackend::new(PendingFailureFactory { + restore_calls: Arc::clone(&restore_calls), + cleanup_calls: Arc::clone(&cleanup_calls), + cleanup_allowed: Arc::clone(&cleanup_allowed), + }) + .unwrap() + .with_spawn_restore_timeout(Duration::from_millis(1)); + let worker_ref = WorkerRef::new(WorkerId::now_v7()); + let request = direct_restore_request(worker_ref.clone()); + let operation_id = request.operation_id; + assert!(matches!( + backend.restore_worker(request.clone()), + WorkerExecutionSpawnResult::ReconciliationRequired { .. } + )); + assert!(backend.workers.lock().unwrap().is_empty()); + assert_eq!( + backend + .pending_restore(&worker_ref) + .unwrap() + .unwrap() + .request + .operation_id, + operation_id + ); + // Same operation must await/reuse the original future, never start again. + assert!(matches!( + backend.reconcile_restore(request.clone()), + WorkerExecutionSpawnResult::ReconciliationRequired { .. } + )); + let pending = backend.preflight_restore(&request).unwrap_err(); + assert_eq!( + pending.outcome, + crate::execution::WorkerExecutionOutcome::Busy + ); + assert!( + pending.worker_state.is_none(), + "pending resources must not attest AlreadyConnected" + ); + let stop = WorkerExecutionStopRequest { + operation_id: crate::execution::WorkerLifecycleOperationId::new(), + worker_ref: worker_ref.clone(), + }; + let failed = backend.stop_worker_operation(stop.clone()); + assert_eq!( + failed.outcome, + crate::execution::WorkerExecutionOutcome::Errored + ); + assert!( + failed + .message + .unwrap() + .contains("pending restore cleanup failure") + ); + assert_eq!( + backend + .pending_restore(&worker_ref) + .unwrap() + .unwrap() + .request + .operation_id, + operation_id + ); + cleanup_allowed.store(true, Ordering::SeqCst); + assert!(backend.stop_worker_operation(stop.clone()).is_accepted()); + assert!(backend.pending_restore(&worker_ref).unwrap().is_none()); + assert!( + backend.stop_worker_operation(stop).is_accepted(), + "cleanup then catalog-commit retry must converge" + ); + assert_eq!(restore_calls.load(Ordering::SeqCst), 1); + assert_eq!(cleanup_calls.load(Ordering::SeqCst), 2); + } + + struct LateControllerFactory { + inner: MockFactory, + restores: Arc, + release: Arc, + actual_handles: Arc>>, + } + + #[async_trait] + impl RuntimeWorkerFactory for LateControllerFactory { + async fn spawn_controller( + &self, + request: WorkerExecutionSpawnRequest, + ) -> Result { + self.inner.spawn_controller(request).await + } + async fn restore_controller( + &self, + request: WorkerExecutionRestoreRequest, + ) -> Result { + let first = self.restores.fetch_add(1, Ordering::SeqCst) == 0; + if first { + self.release.notified().await; + } + let controller = self.inner.restore_controller(request).await?; + self.actual_handles + .lock() + .unwrap() + .push(controller.handle.clone()); + Ok(controller) + } + } + + #[test] + fn stop_joins_late_restore_controller_and_old_methods_do_not_follow_new_execution() { + let root = tempfile::tempdir().unwrap(); + let restores = Arc::new(AtomicUsize::new(0)); + let release = Arc::new(tokio::sync::Notify::new()); + let actual_handles = Arc::new(Mutex::new(Vec::new())); + let backend = WorkerRuntimeExecutionBackend::new(LateControllerFactory { + inner: MockFactory { + client: MockClient::sequential(vec![]), + runtime_base: root.path().join("runtime"), + cwd: root.path().to_path_buf(), + store_dir: root.path().join("sessions"), + worker_metadata_dir: root.path().join("workers"), + observed_cwds: Arc::new(Mutex::new(Vec::new())), + observed_workspace_clients: Arc::new(Mutex::new(Vec::new())), + }, + restores: Arc::clone(&restores), + release: Arc::clone(&release), + actual_handles: Arc::clone(&actual_handles), + }) + .unwrap() + .with_spawn_restore_timeout(Duration::from_millis(1)); + let worker_ref = WorkerRef::new(WorkerId::now_v7()); + assert!(matches!( + backend.restore_worker(direct_restore_request(worker_ref.clone())), + WorkerExecutionSpawnResult::ReconciliationRequired { .. } + )); + assert!(backend.workers.lock().unwrap().is_empty()); + release.notify_one(); + assert!(backend.stop_worker(&worker_ref).is_accepted()); + assert_eq!( + restores.load(Ordering::SeqCst), + 1, + "stop must await, not relaunch" + ); + assert!(backend.workers.lock().unwrap().is_empty()); + assert!(backend.pending_restore(&worker_ref).unwrap().is_none()); + #[cfg(feature = "ws-server")] + assert!(backend.worker_snapshot(&worker_ref).is_none()); + let old_handle = actual_handles.lock().unwrap()[0].clone(); + // Restore the same identity, but long-lived consumers keep the actual + // old Controller transport rather than acquiring a WorkerRef token. + let mut backend = backend; + backend.spawn_restore_timeout = Duration::from_secs(5); + assert!(matches!( + backend.restore_worker(direct_restore_request(worker_ref.clone())), + WorkerExecutionSpawnResult::Connected { .. } + )); + let old_send = backend.run_on_adapter_runtime(async move { + old_handle + .send(Method::ListRewindTargets) + .await + .map_err(|error| error.to_string()) + }); + assert!( + old_send.is_err(), + "old protocol transport must stay closed after restore" + ); + assert_eq!(restores.load(Ordering::SeqCst), 2); + assert!(backend.stop_worker(&worker_ref).is_accepted()); + #[cfg(feature = "ws-server")] + assert!(backend.worker_snapshot(&worker_ref).is_none()); + } + + #[test] + fn absent_execution_has_typed_input_rejection_and_safe_stop() { + let backend = WorkerRuntimeExecutionBackend::new(FailingFactory).unwrap(); + let worker_ref = WorkerRef::new(WorkerId::now_v7()); + let rejected = backend.dispatch_input(&worker_ref, WorkerInput::user("input")); + assert_eq!(rejected.operation, WorkerExecutionOperation::Input); + assert_eq!( + rejected.outcome, + crate::execution::WorkerExecutionOutcome::Rejected + ); + assert!(rejected.message.unwrap().contains("live Worker execution")); + assert!(backend.stop_worker(&worker_ref).is_accepted()); + assert!(backend.workers.lock().unwrap().is_empty()); + } + + #[test] + fn poisoned_pending_resources_are_not_absence_evidence_for_stop() { + let backend = WorkerRuntimeExecutionBackend::new(FailingFactory).unwrap(); + let _ = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _guard = backend.pending_restores.lock().unwrap(); + panic!("injected resource registry poison"); + })); + let stopped = backend.stop_worker(&WorkerRef::new(WorkerId::now_v7())); + assert_eq!( + stopped.outcome, + crate::execution::WorkerExecutionOutcome::Errored + ); + assert!( + stopped + .message + .unwrap() + .contains("pending restore resources lock is poisoned") + ); + } + #[test] fn adapter_runtime_reports_task_panic() { let backend = WorkerRuntimeExecutionBackend::new(FailingFactory).unwrap(); @@ -4895,6 +8232,7 @@ mod tests { let worker_aggregate_dir = runtime_store_dir .join("workers") .join(worker_ref.worker_id.to_string()); + WorkerSessionStore::new(worker_aggregate_dir.join("session")).unwrap(); let worker_name = ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&worker_ref); let session_id = session_store::new_session_id(); WorkerAggregateStore::new(&worker_aggregate_dir, &worker_name) @@ -4965,6 +8303,7 @@ mod tests { let worker_aggregate_dir = runtime_store_dir .join("workers") .join(worker_ref.worker_id.to_string()); + WorkerSessionStore::new(worker_aggregate_dir.join("session")).unwrap(); let worker_name = ProfileRuntimeWorkerFactory::runtime_worker_name_for_ref(&worker_ref); let session_id = session_store::new_session_id(); let manifest = manifest::WorkerManifest::from_toml(&format!( @@ -5170,8 +8509,8 @@ mod tests { ); assert!(!first_run_socket.exists()); - let handle = WorkerExecutionHandle::new(worker.worker_ref.clone(), backend.backend_id()); - assert!(backend.stop_worker(&handle).is_accepted()); + let worker_ref = worker.worker_ref.clone(); + assert!(backend.stop_worker(&worker_ref).is_accepted()); drop(runtime); drop(backend); @@ -5253,8 +8592,8 @@ mod tests { }; let first = backend.restore_worker(restore_request.clone()); - let first_handle = match first { - WorkerExecutionSpawnResult::Connected { handle, .. } => handle, + match first { + WorkerExecutionSpawnResult::Connected { .. } => (), other => panic!("initial restore was not connected: {other:?}"), }; let factory_calls_after_restore = observed_workspace_clients.lock().unwrap().len(); @@ -5275,7 +8614,7 @@ mod tests { assert_eq!(workers.len(), 1); drop(workers); - assert!(backend.stop_worker(&first_handle).is_accepted()); + assert!(backend.stop_worker(&worker.worker_ref).is_accepted()); } #[test] @@ -5499,6 +8838,7 @@ mod tests { uuid::Uuid::parse_str(submission_id).expect("opaque submission id is a UUID"); } + #[cfg(feature = "ws-server")] #[test] fn adapter_dispatches_user_input_through_worker_run_lifecycle() { let client = MockClient::new(simple_text_events()); @@ -5650,7 +8990,12 @@ mod tests { .unwrap(); assert!(detached.workdir_attachments.is_empty()); runtime.stop_worker(&detail.worker_ref, None).unwrap(); - let restored_without_workdir = runtime.restore_worker(&detail.worker_ref).unwrap(); + let restored_without_workdir = runtime + .restore_worker( + &detail.worker_ref, + runtime.test_restore_request(&detail.worker_ref), + ) + .unwrap(); assert!(restored_without_workdir.workdir_attachments.is_empty()); assert!( !repo.path().join(".yoi").exists(), @@ -5924,36 +9269,6 @@ mod tests { let detail = runtime .create_worker(create_request("restore-after-stop")) .unwrap(); - let failed_task = backend - .spawn_on_adapter_runtime(async { panic!("injected owned task failure") }) - .unwrap(); - backend - .workers - .lock() - .unwrap() - .get(&detail.worker_ref) - .unwrap() - .tasks - .push("injected failure", failed_task, false); - - let first_stop = runtime.stop_worker(&detail.worker_ref, None).unwrap_err(); - assert!( - first_stop - .to_string() - .contains("injected failure task failed") - ); - assert_eq!( - runtime.worker_detail(&detail.worker_ref).unwrap().status, - crate::catalog::WorkerStatus::Idle - ); - assert!( - backend - .workers - .lock() - .unwrap() - .contains_key(&detail.worker_ref), - "failed cleanup must retain retry authority" - ); runtime.stop_worker(&detail.worker_ref, None).unwrap(); assert_eq!( runtime.worker_detail(&detail.worker_ref).unwrap().status, @@ -5967,7 +9282,12 @@ mod tests { .contains_key(&detail.worker_ref) ); - runtime.restore_worker(&detail.worker_ref).unwrap(); + runtime + .restore_worker( + &detail.worker_ref, + runtime.test_restore_request(&detail.worker_ref), + ) + .unwrap(); assert_eq!( runtime.worker_detail(&detail.worker_ref).unwrap().status, crate::catalog::WorkerStatus::Idle diff --git a/crates/worker-runtime/src/worker_source.rs b/crates/worker-runtime/src/worker_source.rs index 089ed9f8f..22f81aa79 100644 --- a/crates/worker-runtime/src/worker_source.rs +++ b/crates/worker-runtime/src/worker_source.rs @@ -555,9 +555,15 @@ impl WorkspaceClient for RuntimeOwnedWorkspaceClient { WorkspaceServerOperation::WorkerControlRestore { runtime_id, worker_id, + request, } => generated_workspace_response( client - .worker_control_restore(workspace_id, runtime_id, worker_id) + .worker_control_restore( + workspace_id, + runtime_id, + worker_id, + request, + ) .await, ), WorkspaceServerOperation::WorkerObservationSessions => { diff --git a/crates/worker/Cargo.toml b/crates/worker/Cargo.toml index 84dad3c7d..597cf460c 100644 --- a/crates/worker/Cargo.toml +++ b/crates/worker/Cargo.toml @@ -55,6 +55,8 @@ wip-http = { workspace = true } wip-protocol = { workspace = true } [dev-dependencies] +axum = { workspace = true } +workdir = { workspace = true, features = ["http-client"] } dotenv = "0.15.0" futures = { workspace = true } serial_test = "3.4.0" diff --git a/crates/worker/README.md b/crates/worker/README.md index 0c5015fdf..695b7460a 100644 --- a/crates/worker/README.md +++ b/crates/worker/README.md @@ -25,6 +25,80 @@ Does not own: A Worker is runtime authority, not UI state. It should commit model-visible events through history/session paths and keep current Worker-name state in Worker metadata rather than in transient runtime files. +## Workspace config WIP Feature + +Enable the opt-in surface in a Worker profile/manifest: + +```toml +[feature.workspace_config] +enabled = true +``` + +This setting accepts **only** `enabled` and defaults to false. Controller +registration additionally requires Workspace identity, an available injected +Workspace Client and WIP mode. It creates no grant. An owner must explicitly +create a Backend grant bound to Workspace/Runtime/Worker and read-only or +read-write access; standalone/Tools-mode config compatibility tools do not exist. + +`feature::builtin::workspace_config` registers `/workspace-config` through the +common structured invocation registry (T696). Its optional `access` argument is +`effective` (default), `read_only`, or `read_write`; explicit write never silently +downgrades. Feature preparation attaches before the request reaches the assistant +and reports the logical target and effective access. Native self-attach uses the +same `WorkspaceConfigFeature::attach` path. Backend alias attachment is idempotent +under the session lock; neither entry point blindly retries uncertain outcomes. + +The canonical alias is `workspace-config`; the content entrance is WIP +`/workspace-config`. Attachment identity and conditional detach use the existing +T697 Backend ledger. No OS mount, Workdir session, Bash capability, local config +copy or second attachment inventory is created. The registry source is logical, +not a Runtime filesystem target. + +The request-time provider maps `/workspace-config` to the Backend's empty root +path and descendants to canonical relative source paths, for example +`/workspace-config/main.dcdl` -> `main.dcdl`. Observations return structure, +Backend validators and current operations, **not bodies**. Files publish `read`; +read-write grants additionally publish `write`, unique-substring `edit`, and +`delete`. Directories and explicit absence observations publish `create` where +Backend metadata permits it. The root publishes `apply_changes` for atomic +create/update/delete/rename sets, allowing new imported +sources and their references to be saved together. Empty directory creation, +commands and generic POSIX operations are not exposed. The adapter uses the typed +`workers/self/workspace-config` GET/attach/observe/read/commit API; it never calls +an owner config API as a bypass. + +Client-managed PATH validators are compared against freshly resolved Host +metadata. Each mutation captures the ROOT validator, whole-tree revision/digest, +canonical entrypoints and node state from that same observation. Backend resolves +the live grant/link and validates canonical changes/CAS under its execution lock. +No operation accepts LLM-supplied validators or `expected_digest` fields. Returned +post-call path validators are refreshed from the exact committed revision, not +substituted with root validators or later unrelated state. UI/Worker edits and +same-bytes replacement revisions reject stale calls. Old descriptors cannot +bypass read-only restrictions, detach, revocation or Workspace boundaries. + +Bounds: depth 0..8, 256 observed nodes/changes, 512-byte relative paths, 256 KiB +aggregate changed text/read content, and 30-second retrieval/operation deadlines. +Path traversal and malformed parameters fail before content calls. Typed +`NotCommitted` failures remain distinct from `Unknown`; timeouts/lost mutation +responses are unknown and never automatically retried. Raw transport/provider +errors are not copied into model/UI output. Retrieval protocol failures are +reported as failures after recording their Client cache state, not as fresh +successful observations. Saving commits the canonical Workspace configuration +and existing projection/activation pipeline; it does **not** rewrite the resolved +manifests of already-running Workers. + +Public integration seams: `WorkspaceConfigFeature::for_workspace`, +`workspace_config::wip::mount_workspace_config_wip`, and +`WipRuntime::from_mounts` plus `discover`/`inspect`/`call`. Worker tests exercise the +real Feature registry, native mount and WIP Client/Host through a typed in-process +contract router: deep discovery, slash/native attach, read/edit/write, +create/delete, atomic multi-source updates, grant restrictions/revocation, +detach, cross-Workspace isolation, concurrent/same-bytes stale observations, +invalid/oversize/save/unknown failures and no retries. Those contract-router tests +are not a substitute for Backend canonical persistence, semantic validation, +projection activation or full Web-to-Backend acceptance tests. + ## See also - [`../../docs/design/worker-session-state.md`](../../docs/design/worker-session-state.md) diff --git a/crates/worker/src/backend_job.rs b/crates/worker/src/backend_job.rs new file mode 100644 index 000000000..e5b01b431 --- /dev/null +++ b/crates/worker/src/backend_job.rs @@ -0,0 +1,29 @@ +//! Host-authored Job authority, separate from reusable Profile policy. + +use serde::{Deserialize, Serialize}; + +/// Immutable execution capability delivered only by a trusted Backend host. +/// Names, instructions and Profiles never confer this capability. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct BackendJobExecutionBinding { + pub job_id: String, + pub attempt_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub input_revision: Option, + #[serde(default)] + pub subjektiv_consolidation: bool, +} + +impl BackendJobExecutionBinding { + pub fn validate(&self) -> Result<(), &'static str> { + if [&self.job_id, &self.attempt_id] + .into_iter() + .chain(self.input_revision.iter()) + .any(|value| value.trim().is_empty() || value.chars().any(char::is_control)) + { + return Err("Backend Job binding contains an invalid identity or input revision"); + } + Ok(()) + } +} diff --git a/crates/worker/src/checkout.rs b/crates/worker/src/checkout.rs new file mode 100644 index 000000000..dedc6e0d4 --- /dev/null +++ b/crates/worker/src/checkout.rs @@ -0,0 +1,757 @@ +//! Provider-owned request-time checkout projection. No filesystem path is opened +//! by the Worker: every observation and operation goes through the live router. + +use std::collections::BTreeMap; +use std::sync::Arc; +use std::time::Duration; + +use agen::tool::ToolError; +use async_trait::async_trait; +use manifest::{ToolPermissionAction, ToolPermissionConfig}; +use serde_json::{Value as Json, json}; +use sha2::{Digest, Sha256}; +use wip_protocol::{ + Documentation, INTERFACE_FORMAT_V1, InterfaceDescriptor, Object, OperationDeclaration, + ParameterDeclaration, ProtocolError, ProtocolErrorCode, ReturnDeclaration, TypeExpr, Value, +}; +use workdir::{ + CheckoutObservation, EntryKind, ListRequest, WorkdirPath, WorkdirSessionCapability as Cap, + WorkdirSessionRouter, +}; + +use crate::feature::builtin::manage_workdir::wip::{decode_identity, encode_identity}; +use crate::permission::permission_action_for; +use crate::wip::{ + WipCallContext, WipMountError, WipMountRegistry, WipOperationError, WipOperationHandler, + WipOperationOutput, WipProjection, WipProjectionKind, WipSubtreeMount, WipSubtreeProvider, + json_to_wip, wip_to_json, +}; + +const ROOT: &str = "/checkouts"; +const MAX_NODES: usize = 1024; +const DEADLINE: Duration = Duration::from_secs(30); + +/// Return a stable, injective attachment-local entrance, never a host path. +pub fn checkout_root(alias: &str) -> String { + format!("{ROOT}/{}", encode_identity(alias)) +} + +/// Register the native content namespace independently of inventory/lifecycle +/// Features. Its capabilities come from current attachments, not Feature flags. +pub fn mount_checkouts( + registry: &mut WipMountRegistry, + router: Arc, + tracker: tools::Tracker, + permissions: Option, +) -> Result<(), WipMountError> { + registry.allocate_namespace("checkout", "checkouts")?; + let provider = Arc::new(Provider { + router, + tracker, + permissions, + incarnation: uuid::Uuid::now_v7().to_string(), + permits: Arc::new(tokio::sync::Semaphore::new(16)), + deadline: DEADLINE, + }); + registry.mount(provider.collection())?; + registry.replace_compatibility_tools(ROOT, ["Read", "Edit", "Write", "Glob", "Grep"])?; + registry.mount_subtree(WipSubtreeMount { + root: ROOT.into(), + provider: Arc::new(provider), + }) +} + +struct Provider { + router: Arc, + tracker: tools::Tracker, + permissions: Option, + incarnation: String, + permits: Arc, + deadline: Duration, +} + +impl Provider { + fn allowed(&self, tool: &str, input: &Json) -> bool { + self.permissions + .as_ref() + .is_none_or(|p| permission_action_for(p, tool, input) == ToolPermissionAction::Allow) + } + fn may_allow(&self, tool: &str) -> bool { + let Some(p) = &self.permissions else { + return true; + }; + for rule in p.rules.iter().filter(|r| r.tool.eq_ignore_ascii_case(tool)) { + if rule.action == ToolPermissionAction::Allow { + return true; + } + if rule.pattern == "*" { + return false; + } + } + p.default_action == ToolPermissionAction::Allow + } + fn parse(&self, path: &str) -> Option<(String, WorkdirPath)> { + wip_protocol::validate_path(path).ok()?; + let rest = path.strip_prefix("/checkouts/")?; + let (slug, tail) = rest.split_once('/').unwrap_or((rest, "")); + if tail.contains('\0') { + return None; + } + Some((decode_identity(slug)?, WorkdirPath::new(tail).ok()?)) + } + fn wrap_validator(&self, alias: &str, generation: u64, workdir: &str, raw: &[u8]) -> Vec { + let mut hash = Sha256::new(); + for value in [ + self.incarnation.as_bytes(), + alias.as_bytes(), + &generation.to_be_bytes(), + workdir.as_bytes(), + raw, + ] { + hash.update(value.len().to_be_bytes()); + hash.update(value); + } + hash.finalize().to_vec() + } + fn collection(self: &Arc) -> WipProjection { + let mut state = Sha256::new(); + state.update(&self.incarnation); + for alias in self.router.aliases() { + if let Ok(s) = self.router.resolve(Some(alias.as_str())) { + state.update(alias.as_str()); + state.update(s.generation.to_be_bytes()); + state.update(s.session.workdir().id().as_str()); + state.update(format!("{:?}", s.session.capabilities())); + } + } + self.project( + ROOT, + descriptor(vec![operation( + "list", + "List currently accessible checkouts", + vec![], + )]), + state.finalize().to_vec(), + "collection", + Arc::new(Collection { + provider: self.clone(), + }), + ) + } + fn project( + &self, + path: &str, + descriptor: InterfaceDescriptor, + validator: Vec, + kind: &str, + handler: Arc, + ) -> WipProjection { + let encoded: String = path.as_bytes().iter().map(|b| format!("{b:02x}")).collect(); + let interface = format!("yoi.checkout/{}/{kind}/v1/@/{encoded}", self.incarnation); + let interface_validator = Sha256::digest(format!("{descriptor:?}")).to_vec(); + WipProjection { + route: path.into(), + capability: format!("checkout:{kind}"), + kind: WipProjectionKind::Native, + object: Object { + name: path.rsplit('/').next().unwrap_or_default().into(), + description: Some(format!("Attached Workdir {kind}")), + interfaces: vec![interface.clone()], + r#ref: None, + validator: Some(validator), + }, + interface, + descriptor, + interface_validator: Some(interface_validator), + handler, + } + } + async fn node(self: &Arc, path: &str) -> Result, ProtocolError> { + if path == ROOT { + return Ok(Some(self.collection())); + } + let Some((alias, target)) = self.parse(path) else { + return Ok(None); + }; + let Ok(selected) = self.router.resolve(Some(&alias)) else { + return Ok(None); + }; + let observation = match selected.session.checkout_observe(target.clone()).await { + Ok(o) => o, + Err( + workdir::WorkdirError::Denied(_) + | workdir::WorkdirError::OutOfScope(_) + | workdir::WorkdirError::NotFound(_) + | workdir::WorkdirError::SessionClosed + | workdir::WorkdirError::UnsupportedOperation(_) + | workdir::WorkdirError::Unsupported(_), + ) => return Ok(None), + Err(_) => { + return Err(error( + ProtocolErrorCode::Internal, + "checkout observation unavailable", + )); + } + }; + if !matches!(observation.kind, EntryKind::File | EntryKind::Directory) { + return Ok(None); + } + let permission_input = json!({"target_workdir":alias, "file_path":target.as_str()}); + if observation.kind == EntryKind::File && !self.allowed("Read", &permission_input) { + return Ok(None); + } + let ops = self.operations(&observation); + if ops.is_empty() { + return Ok(None); + } + let kind = if observation.kind == EntryKind::File { + "file" + } else { + "directory" + }; + let workdir = selected.session.workdir().id().as_str().to_owned(); + let validator = self.wrap_validator( + &alias, + selected.generation, + &workdir, + &observation.validator, + ); + let handler = Arc::new(FileHandler { + provider: self.clone(), + alias, + generation: selected.generation, + workdir, + target, + observation, + }); + let family = format!("{kind}-g{}", selected.generation); + Ok(Some(self.project( + path, + descriptor(ops), + validator, + &family, + handler, + ))) + } + fn operations(&self, o: &CheckoutObservation) -> Vec { + let mut ops = Vec::new(); + let has = |cap, name| o.capabilities.supports(cap) && self.may_allow(name); + if o.kind == EntryKind::File { + if has(Cap::Read, "Read") { + ops.push(operation( + "read", + "Read text with line offsets and numbered output", + crate::file_operation::parameters("read", true).unwrap(), + )); + } + if has(Cap::Edit, "Edit") { + ops.push(operation( + "edit", + "Replace a unique string (requires prior Read)", + crate::file_operation::parameters("edit", true).unwrap(), + )); + } + if has(Cap::Write, "Write") { + ops.push(operation( + "write", + "Overwrite this existing file (requires prior Read)", + crate::file_operation::parameters("write", true).unwrap(), + )); + } + } else { + if has(Cap::Glob, "Glob") { + ops.push(operation( + "glob", + "Glob within this directory", + vec![ + parameter("pattern", true, TypeExpr::String), + parameter("path", false, TypeExpr::String), + ], + )); + } + if has(Cap::Grep, "Grep") { + ops.push(operation( + "grep", + "Search within this directory with the existing Grep contract", + grep_parameters(), + )); + } + if has(Cap::Write, "Write") { + ops.push(operation("create_file", "Create a new relative file, creating permitted missing parents; never overwrite", vec![parameter("path",true,TypeExpr::String),parameter("content",true,TypeExpr::String)])); + } + } + ops + } +} + +#[async_trait] +impl WipSubtreeProvider for Arc { + async fn projection(&self, path: &str) -> Result, ProtocolError> { + let _permit = tokio::time::timeout(self.deadline, self.permits.acquire()) + .await + .map_err(|_| { + error( + ProtocolErrorCode::ResourceLimitExceeded, + "checkout concurrency deadline", + ) + })? + .map_err(|_| error(ProtocolErrorCode::Internal, "checkout unavailable"))?; + tokio::time::timeout(self.deadline, self.node(path)) + .await + .map_err(|_| { + error( + ProtocolErrorCode::ResourceLimitExceeded, + "checkout observation deadline", + ) + })? + } + async fn children(&self, path: &str) -> Result, ProtocolError> { + let _permit = tokio::time::timeout(self.deadline, self.permits.acquire()) + .await + .map_err(|_| { + error( + ProtocolErrorCode::ResourceLimitExceeded, + "checkout capacity deadline", + ) + })? + .map_err(|_| error(ProtocolErrorCode::Internal, "checkout unavailable"))?; + tokio::time::timeout(self.deadline, self.children_inner(path)) + .await + .map_err(|_| { + error( + ProtocolErrorCode::ResourceLimitExceeded, + "checkout enumeration deadline", + ) + })? + } +} + +impl Provider { + async fn children_inner(self: &Arc, path: &str) -> Result, ProtocolError> { + if path == ROOT { + let before = self.collection().object.validator.unwrap(); + if self.router.aliases().len() > MAX_NODES { + return Err(error( + ProtocolErrorCode::ResourceLimitExceeded, + "too many checkouts", + )); + } + let mut paths = Vec::new(); + for alias in self.router.aliases() { + let p = checkout_root(alias.as_str()); + if self.node(&p).await?.is_some() { + paths.push(p); + } + } + if before != self.collection().object.validator.unwrap() { + return Err(error( + ProtocolErrorCode::ValidatorMismatch, + "attachment collection changed", + )); + } + return Ok(paths); + } + let Some((alias, target)) = self.parse(path) else { + return Err(error(ProtocolErrorCode::NotFound, "checkout not published")); + }; + let selected = self + .router + .resolve(Some(&alias)) + .map_err(|_| error(ProtocolErrorCode::NotFound, "attachment expired"))?; + let o = selected + .session + .checkout_observe(target.clone()) + .await + .map_err(|_| error(ProtocolErrorCode::NotFound, "checkout not published"))?; + if o.kind != EntryKind::Directory { + return Ok(Vec::new()); + } + if !self.allowed( + "Glob", + &json!({"target_workdir":alias,"path":target.as_str(),"pattern":"*"}), + ) && !self.allowed( + "Grep", + &json!({"target_workdir":alias,"path":target.as_str(),"pattern":""}), + ) { + return Ok(Vec::new()); + } + let listing = selected + .session + .checkout_search(workdir::CheckoutSearchRequest::new( + workdir::CheckoutSearchOperation::List(ListRequest { + path: target, + limit: MAX_NODES + 1, + }), + )) + .await + .map_err(|_| error(ProtocolErrorCode::Internal, "checkout listing unavailable"))?; + let workdir::CheckoutSearchResult::List(listing) = listing else { + return Err(error( + ProtocolErrorCode::Internal, + "mismatched checkout listing", + )); + }; + if listing.truncated || listing.entries.len() > MAX_NODES { + return Err(error( + ProtocolErrorCode::ResourceLimitExceeded, + "directory observation exceeds node bound", + )); + } + let mut paths = Vec::new(); + for entry in listing.entries { + let p = object_path(&alias, &entry.path); + if self.node(&p).await?.is_some() { + paths.push(p); + } + } + if !self + .router + .resolve(Some(&alias)) + .is_ok_and(|current| current.generation == selected.generation) + { + return Err(error( + ProtocolErrorCode::ValidatorMismatch, + "attachment changed", + )); + } + Ok(paths) + } +} + +struct Collection { + provider: Arc, +} +#[async_trait] +impl WipOperationHandler for Collection { + async fn call( + &self, + operation: &str, + arguments: &BTreeMap, + _context: WipCallContext, + ) -> Result { + if operation != "list" || !arguments.is_empty() { + return Err(protocol( + ProtocolErrorCode::InvalidArguments, + "list accepts no arguments", + )); + } + let before = self.provider.collection().object.validator.unwrap(); + let paths = Arc::new(self.provider.clone()) + .children(ROOT) + .await + .map_err(WipOperationError::Protocol)?; + let mut items = Vec::new(); + for path in paths { + let (alias, _) = self.provider.parse(&path).expect("generated checkout path"); + let s = self.provider.router.resolve(Some(&alias)).map_err(|_| { + protocol(ProtocolErrorCode::ValidatorMismatch, "attachment changed") + })?; + items.push(json!({"path":path,"slug":encode_identity(&alias),"alias":alias, + "working_directory_id":s.session.workdir().id().as_str(),"generation":s.generation})); + } + let value = json_to_wip(&json!({"items":items})) + .map_err(|_| protocol(ProtocolErrorCode::Internal, "invalid checkout result"))?; + let after = self.provider.collection().object.validator.unwrap(); + if before != after { + return Err(protocol( + ProtocolErrorCode::ValidatorMismatch, + "attachment collection changed", + )); + } + Ok(WipOperationOutput::native_with_validator(value, after)) + } +} + +struct FileHandler { + provider: Arc, + alias: String, + generation: u64, + workdir: String, + target: WorkdirPath, + observation: CheckoutObservation, +} +impl FileHandler { + fn check_connection(&self) -> Result<(), WipOperationError> { + if self + .provider + .router + .resolve(Some(&self.alias)) + .is_ok_and(|s| { + s.generation == self.generation && s.session.workdir().id().as_str() == self.workdir + }) + { + Ok(()) + } else { + Err(protocol( + ProtocolErrorCode::ValidatorMismatch, + "attachment changed while publishing result", + )) + } + } +} +#[async_trait] +impl WipOperationHandler for FileHandler { + async fn call( + &self, + operation: &str, + arguments: &BTreeMap, + context: WipCallContext, + ) -> Result { + let tool = match operation { + "read" => "Read", + "edit" => "Edit", + "write" => "Write", + "create_file" => "Create", + "glob" => "Glob", + "grep" => "Grep", + _ => { + return Err(protocol( + ProtocolErrorCode::OperationNotFound, + "unknown checkout operation", + )); + } + }; + let mut args = serde_json::Map::new(); + for (k, v) in arguments { + args.insert( + k.clone(), + wip_to_json(v).map_err(|_| { + protocol( + ProtocolErrorCode::InvalidArguments, + "invalid checkout argument", + ) + })?, + ); + } + let mut permission_input = args.clone(); + permission_input.insert("target_workdir".into(), json!(self.alias)); + if tool == "Glob" || tool == "Grep" { + let relative = args.get("path").and_then(Json::as_str).unwrap_or(""); + permission_input.insert("path".into(), json!(join_under(&self.target, relative)?)); + } else if tool == "Create" { + let relative = args.get("path").and_then(Json::as_str).ok_or_else(|| { + protocol(ProtocolErrorCode::InvalidArguments, "create requires path") + })?; + permission_input.remove("path"); + permission_input.insert( + "file_path".into(), + json!(join_under(&self.target, relative)?), + ); + } else { + permission_input.insert("file_path".into(), json!(self.target.as_str())); + } + if !self.provider.allowed( + if tool == "Create" { "Write" } else { tool }, + &Json::Object(permission_input), + ) { + return Err(protocol( + ProtocolErrorCode::PermissionDenied, + "file operation denied; approval-required policy fails closed", + )); + } + let _permit = tokio::time::timeout(self.provider.deadline, self.provider.permits.acquire()) + .await + .map_err(|_| { + protocol( + ProtocolErrorCode::ResourceLimitExceeded, + "checkout operation capacity deadline", + ) + })? + .map_err(|_| protocol(ProtocolErrorCode::Internal, "checkout unavailable"))?; + let deadline = tokio::time::Instant::now() + self.provider.deadline; + let result = tokio::time::timeout_at( + deadline, + tools::execute_checkout_tool( + self.provider.router.clone(), + self.provider.tracker.clone(), + &self.alias, + self.generation, + self.target.clone(), + self.observation.validator.clone(), + tool, + Json::Object(args), + context.execution, + ), + ) + .await + .map_err(|_| { + if matches!(tool, "Write" | "Edit" | "Create") { + WipOperationError::OutcomeUnknown("file operation deadline after dispatch".into()) + } else { + protocol( + ProtocolErrorCode::ResourceLimitExceeded, + "file operation deadline", + ) + } + })? + .map_err(map_tool_error)?; + let links = tokio::time::timeout_at(deadline, async { + let mut links = Vec::new(); + self.check_connection()?; + for path in &result.paths { + let route = object_path(&self.alias, path); + // References are validated only against the captured live connection. + if self + .provider + .node(&route) + .await + .map_err(WipOperationError::Protocol)? + .is_some() + { + links.push(json!({"path":route})); + } + self.check_connection()?; + } + Ok::<_, WipOperationError>(links) + }) + .await + .map_err(|_| { + if matches!(tool, "Write" | "Edit" | "Create") { + WipOperationError::OutcomeUnknown("completed operation response deadline".into()) + } else { + protocol( + ProtocolErrorCode::ResourceLimitExceeded, + "checkout result deadline", + ) + } + })? + .map_err(|e| { + if matches!(tool, "Write" | "Edit" | "Create") { + WipOperationError::OutcomeUnknown("completed operation response unavailable".into()) + } else { + e + } + })?; + let value = json_to_wip( + &json!({"summary":result.output.summary,"content":result.output.content,"items":links}), + ) + .map_err(|_| { + WipOperationError::OutcomeUnknown("invalid completed file operation result".into()) + })?; + if let Some(raw) = result.validator { + Ok(WipOperationOutput::native_with_validator( + value, + self.provider + .wrap_validator(&self.alias, self.generation, &self.workdir, &raw), + )) + } else { + Ok(WipOperationOutput::native(value)) + } + } +} + +fn object_path(alias: &str, path: &WorkdirPath) -> String { + if path.is_root() { + checkout_root(alias) + } else { + format!("{}/{}", checkout_root(alias), path.as_str()) + } +} +fn join_under(parent: &WorkdirPath, relative: &str) -> Result { + let child = WorkdirPath::new(relative).map_err(|_| { + protocol( + ProtocolErrorCode::InvalidArguments, + "path must stay relative to this directory", + ) + })?; + if relative.starts_with('/') { + return Err(protocol( + ProtocolErrorCode::InvalidArguments, + "absolute paths are not permitted", + )); + } + Ok(if parent.is_root() { + child.as_str().into() + } else if child.is_root() { + parent.as_str().into() + } else { + format!("{}/{}", parent.as_str(), child.as_str()) + }) +} +fn parameter(name: &str, required: bool, r#type: TypeExpr) -> ParameterDeclaration { + ParameterDeclaration { + name: name.into(), + required, + documentation: None, + r#type, + } +} +fn operation( + name: &str, + summary: &str, + parameters: Vec, +) -> OperationDeclaration { + OperationDeclaration { + name: name.into(), + documentation: Some(Documentation { + summary: summary.into(), + details: None, + }), + parameters, + returns: ReturnDeclaration { + documentation: None, + r#type: TypeExpr::Json, + }, + } +} +fn descriptor(operations: Vec) -> InterfaceDescriptor { + InterfaceDescriptor{format:INTERFACE_FORMAT_V1.into(),documentation:Some(Documentation{summary:"AI-oriented attached Workdir operations".into(),details:Some("The route binds attachment and target. Line offsets are zero-based; text output numbers lines from one. Read-before-write and provider validators are both required. Paths are Worldspace references, never host paths.".into())}),types:Vec::new(),operations} +} +fn grep_parameters() -> Vec { + let mut p = vec![parameter("pattern", true, TypeExpr::String)]; + for name in ["path", "glob", "type", "output_mode"] { + p.push(parameter(name, false, TypeExpr::String)); + } + for name in ["-A", "-B", "-C", "head_limit", "offset"] { + p.push(parameter(name, false, TypeExpr::Integer)); + } + for name in ["case_insensitive", "multiline"] { + p.push(parameter(name, false, TypeExpr::Boolean)); + } + p +} +fn error(code: ProtocolErrorCode, message: &str) -> ProtocolError { + ProtocolError { + code, + message: message.into(), + } +} +fn protocol(code: ProtocolErrorCode, message: &str) -> WipOperationError { + WipOperationError::Protocol(error(code, message)) +} +fn map_tool_error(e: ToolError) -> WipOperationError { + match e { + ToolError::InvalidArgument(_) => protocol( + ProtocolErrorCode::InvalidArguments, + "invalid file arguments or read-before-write requirement", + ), + ToolError::StructuredConflict { code, .. } => match code.as_str() { + "checkout_denied" => protocol( + ProtocolErrorCode::PermissionDenied, + "file operation refused before effects", + ), + "checkout_invalid" => protocol( + ProtocolErrorCode::InvalidArguments, + "invalid checkout operation or target", + ), + "checkout_unavailable" => protocol( + ProtocolErrorCode::ResourceLimitExceeded, + "checkout provider declined before effects", + ), + "checkout_stale" => protocol( + ProtocolErrorCode::ValidatorMismatch, + "file or attachment observation is stale; rediscover and reread", + ), + _ => WipOperationError::OutcomeUnknown("unclassified file operation failure".into()), + }, + ToolError::Cancelled(o) => WipOperationError::Cancelled(o), + ToolError::Interrupted(o) => WipOperationError::Interrupted(o), + _ => WipOperationError::OutcomeUnknown( + "file operation failed; inspect current state before retry".into(), + ), + } +} + +#[cfg(test)] +#[path = "checkout_race_tests.rs"] +mod race_tests; diff --git a/crates/worker/src/checkout_http_tests.rs b/crates/worker/src/checkout_http_tests.rs new file mode 100644 index 000000000..169f7ee77 --- /dev/null +++ b/crates/worker/src/checkout_http_tests.rs @@ -0,0 +1,671 @@ +//! Real provider HTTP -> RemoteWorkdirSession -> router -> native WIP Client. +//! This deliberately does not exercise the production Backend grant network. +use super::*; +use axum::{ + Json as AxumJson, Router, + body::Body, + extract::{Path, State}, + http::{HeaderMap, StatusCode, Uri}, + response::{IntoResponse, Response as AxumResponse}, + routing::{delete, post}, +}; +use std::time::Duration; +use tempfile::TempDir; +use workdir::http::{ + OpenWorkdirSessionRequest, OpenWorkdirSessionResponse, RemoteWorkdirSession, WorkdirSessionId, + WorkdirSessionOperation, WorkdirSessionOperationRequest, WorkdirTransportError, + WorkdirTransportErrorCode, +}; +use workdir::{ + CheckoutOperation, CheckoutRequest, CheckoutSearchOperation, LocalWorkdirSession, + WorkdirAttachmentAlias, WorkdirError, WorkdirPath, WorkdirSession, WorkdirSessionCapabilities, + WorkdirSessionHandle, WorkdirSessionRouter, dispatch_workdir_session_operation, +}; + +const TOKEN: &str = "checkout-http-test-private-bearer"; +const SESSION: &str = "http-test-session"; + +#[derive(Clone, Copy, Debug)] +enum WriteResponseFault { + Malformed, + LostBody, +} + +struct ProviderState { + session: WorkdirSessionHandle, + requests: Mutex>, + operations: Mutex>, + write_fault: Mutex>, +} + +impl ProviderState { + fn authorize(&self, headers: &HeaderMap, method: &str, uri: &Uri) -> Result<(), AxumResponse> { + self.requests + .lock() + .unwrap() + .push((method.into(), uri.path().into())); + let expected = format!("Bearer {TOKEN}"); + if headers.get("authorization").and_then(|h| h.to_str().ok()) != Some(expected.as_str()) { + return Err(( + StatusCode::UNAUTHORIZED, + AxumJson(WorkdirTransportError { + code: WorkdirTransportErrorCode::Denied, + message: "provider authorization denied".into(), + }), + ) + .into_response()); + } + Ok(()) + } + + fn execute_calls(&self) -> usize { + self.operations + .lock() + .unwrap() + .iter() + .filter(|op| matches!(op, WorkdirSessionOperation::CheckoutExecute(_))) + .count() + } + + fn write_calls(&self) -> usize { + self.operations + .lock() + .unwrap() + .iter() + .filter(|op| { + matches!( + op, + WorkdirSessionOperation::CheckoutExecute(CheckoutRequest { + operation: CheckoutOperation::Write { .. }, + .. + }) + ) + }) + .count() + } +} + +fn provider_error(error: WorkdirError) -> AxumResponse { + let error = WorkdirTransportError::from_workdir_error(&error); + ( + StatusCode::from_u16(error.code.http_status()).unwrap(), + AxumJson(error), + ) + .into_response() +} + +async fn open_session( + State(state): State>, + Path(id): Path, + uri: Uri, + headers: HeaderMap, + AxumJson(_request): AxumJson, +) -> AxumResponse { + if let Err(response) = state.authorize(&headers, "POST", &uri) { + return response; + } + if id != state.session.workdir().id().as_str() { + return provider_error(WorkdirError::NotFound("unknown workdir".into())); + } + AxumJson(OpenWorkdirSessionResponse { + session_id: WorkdirSessionId::new(SESSION).unwrap(), + workdir_id: state.session.workdir().id().clone(), + capabilities: state.session.capabilities(), + }) + .into_response() +} + +async fn operate( + State(state): State>, + Path(id): Path, + uri: Uri, + headers: HeaderMap, + AxumJson(request): AxumJson, +) -> AxumResponse { + if let Err(response) = state.authorize(&headers, "POST", &uri) { + return response; + } + if id != SESSION { + return provider_error(WorkdirError::Unavailable("unknown session".into())); + } + let is_write = matches!( + &request.operation, + WorkdirSessionOperation::CheckoutExecute(CheckoutRequest { + operation: CheckoutOperation::Write { .. }, + .. + }) + ); + state + .operations + .lock() + .unwrap() + .push(request.operation.clone()); + let result = + dispatch_workdir_session_operation(state.session.as_ref(), request.operation).await; + match result { + // Inject only AFTER the real LocalWorkdirSession successfully committed. + Ok(result) => { + let fault = if is_write { + state.write_fault.lock().unwrap().take() + } else { + None + }; + match fault { + Some(WriteResponseFault::Malformed) => ( + StatusCode::OK, + [("content-type", "application/json")], + "{broken-response", + ) + .into_response(), + Some(WriteResponseFault::LostBody) => { + let body = Body::from_stream(futures::stream::once(async { + Err::, _>(std::io::Error::other("response lost after commit")) + })); + AxumResponse::builder() + .status(StatusCode::OK) + .header("content-type", "application/json") + .body(body) + .unwrap() + } + None => AxumJson(result).into_response(), + } + } + Err(error) => provider_error(error), + } +} + +async fn close_session( + State(state): State>, + Path(id): Path, + uri: Uri, + headers: HeaderMap, +) -> AxumResponse { + if let Err(response) = state.authorize(&headers, "DELETE", &uri) { + return response; + } + if id != SESSION { + return provider_error(WorkdirError::Unavailable("unknown session".into())); + } + match state.session.close().await { + Ok(()) => StatusCode::NO_CONTENT.into_response(), + Err(error) => provider_error(error), + } +} + +struct HttpFixture { + dir: TempDir, + state: Arc, + base_url: reqwest::Url, + server: tokio::task::JoinHandle<()>, +} + +impl Drop for HttpFixture { + fn drop(&mut self) { + self.server.abort(); + } +} + +impl HttpFixture { + async fn new(capabilities: WorkdirSessionCapabilities) -> Self { + let dir = TempDir::new().unwrap(); + std::fs::create_dir_all(dir.path().join("src/deep")).unwrap(); + std::fs::write( + dir.path().join("src/deep/a.txt"), + "first\nneedle needle\nlast\n", + ) + .unwrap(); + let session = Arc::new(LocalWorkdirSession::materialized( + dir.path().to_path_buf(), + dir.path().to_path_buf(), + manifest::SharedScope::new(manifest::Scope::writable(dir.path()).unwrap()), + capabilities, + )); + let state = Arc::new(ProviderState { + session, + requests: Mutex::new(Vec::new()), + operations: Mutex::new(Vec::new()), + write_fault: Mutex::new(None), + }); + let app = Router::new() + .route( + "/provider/v1/working-directories/{id}/sessions", + post(open_session), + ) + .route( + "/provider/v1/workdir-sessions/{id}/operations", + post(operate), + ) + .route("/provider/v1/workdir-sessions/{id}", delete(close_session)) + .with_state(state.clone()); + // Binding before spawn makes readiness deterministic; no sleeps or polling. + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let base_url = format!("http://{}/provider/", listener.local_addr().unwrap()) + .parse() + .unwrap(); + let server = tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + Self { + dir, + state, + base_url, + server, + } + } + + async fn open(&self, token: &str) -> Result { + RemoteWorkdirSession::open( + reqwest::Client::builder() + .no_proxy() + .timeout(Duration::from_secs(5)) + .build() + .unwrap(), + self.base_url.clone(), + token.to_owned(), + self.state.session.workdir().id().clone(), + OpenWorkdirSessionRequest::default(), + ) + .await + } + + fn public_output(&self, text: &str) { + assert!(!text.contains(TOKEN), "bearer token leaked"); + assert!( + !text.contains(self.dir.path().to_str().unwrap()), + "host path leaked" + ); + assert!( + !text.contains("/provider/"), + "transport prefix leaked into Worldspace" + ); + } + + fn assert_endpoints(&self) { + let open = format!( + "/provider/v1/working-directories/{}/sessions", + self.state.session.workdir().id() + ); + let operations = format!("/provider/v1/workdir-sessions/{SESSION}/operations"); + let close = format!("/provider/v1/workdir-sessions/{SESSION}"); + let requests = self.state.requests.lock().unwrap(); + assert!(requests.contains(&("POST".into(), open.clone()))); + assert!(requests.contains(&("POST".into(), operations.clone()))); + assert_eq!( + requests + .iter() + .filter(|(method, path)| method == "DELETE" && path == &close) + .count(), + 1 + ); + assert!(requests.iter().all(|(method, path)| (method == "POST" + && (path == &open || path == &operations)) + || (method == "DELETE" && path == &close))); + } +} + +fn native_runtime(remote: Arc) -> (WipRuntime, Arc) { + let router = Arc::new(WorkdirSessionRouter::new()); + router + .attach(WorkdirAttachmentAlias::new("main").unwrap(), remote) + .unwrap(); + let mut registry = WipMountRegistry::new(); + crate::checkout::mount_checkouts(&mut registry, router.clone(), tools::Tracker::new(), None) + .unwrap(); + let runtime = WipRuntime::new( + WipHost::new(registry), + SecurityContext::new("http-checkout-worker"), + 0, + ) + .unwrap(); + (runtime, router) +} + +async fn observe_interface(runtime: &WipRuntime, path: &str) -> String { + runtime.discover(path.into(), 0, true).await.unwrap(); + // Select from the real Client's Known Space, not a direct handler invocation. + let interface = { + let state = runtime.state.lock().unwrap(); + state + .client + .known_space(&state.session) + .unwrap() + .into_iter() + .find(|o| o.path == path) + .unwrap() + .object + .unwrap() + .interfaces[0] + .clone() + }; + runtime.inspect(interface.clone(), true).await.unwrap(); + interface +} + +async fn native_call(runtime: &WipRuntime, path: &str, operation: &str, args: Json) -> Value { + let interface = observe_interface(runtime, path).await; + runtime + .call( + path.into(), + interface, + operation.into(), + args, + Default::default(), + ) + .await + .unwrap(); + let state = runtime.state.lock().unwrap(); + match &state + .client + .call_history(&state.session) + .unwrap() + .back() + .unwrap() + .outcome + { + CallOutcome::Success(response) => response.result.clone(), + outcome => panic!("expected native Client success, got {outcome:?}"), + } +} + +fn field<'a>(value: &'a Value, name: &str) -> &'a Value { + let Value::Record(fields) = value else { + panic!("expected typed Record") + }; + fields.get(name).unwrap() +} + +fn first_link(value: &Value) -> &str { + let Value::List(items) = field(value, "items") else { + panic!("expected typed link List") + }; + assert_eq!(items.len(), 1); + let Value::String(path) = field(&items[0], "path") else { + panic!("expected typed link path") + }; + path +} + +async fn watchdog(future: impl std::future::Future) { + tokio::time::timeout(Duration::from_secs(20), future) + .await + .expect("HTTP checkout test watchdog"); +} + +#[tokio::test] +async fn checkout_http_native_search_links_read_edit_write_create_and_close() { + watchdog(async { + let fixture = HttpFixture::new(WorkdirSessionCapabilities::READ_WRITE).await; + let remote = Arc::new(fixture.open(TOKEN).await.unwrap()); + let (runtime, router) = native_runtime(remote); + let glob = native_call( + &runtime, + "/checkouts/main", + "glob", + json!({"pattern":"**/*.txt"}), + ) + .await; + let file = first_link(&glob).to_owned(); + assert_eq!(file, "/checkouts/main/src/deep/a.txt"); + let grep = native_call( + &runtime, + "/checkouts/main/src", + "grep", + json!({ + "pattern":"needle", "output_mode":"files_with_matches" + }), + ) + .await; + assert_eq!(first_link(&grep), file); + let read = native_call( + &runtime, + first_link(&grep), + "read", + json!({"offset":1,"limit":1}), + ) + .await; + let Value::String(content) = field(&read, "content") else { + panic!("expected line Read text") + }; + assert_eq!(content, " 2\tneedle needle\n"); + let edit = native_call( + &runtime, + &file, + "edit", + json!({ + "old_string":"needle", "new_string":"done", "replace_all":true + }), + ) + .await; + assert_eq!( + std::fs::read_to_string(fixture.dir.path().join("src/deep/a.txt")).unwrap(), + "first\ndone done\nlast\n" + ); + let write = native_call(&runtime, &file, "write", json!({"content":"saved\n"})).await; + assert_eq!( + std::fs::read_to_string(fixture.dir.path().join("src/deep/a.txt")).unwrap(), + "saved\n" + ); + let create = native_call( + &runtime, + "/checkouts/main/src", + "create_file", + json!({ + "path":"new/deeper.txt", "content":"created\n" + }), + ) + .await; + assert_eq!( + std::fs::read_to_string(fixture.dir.path().join("src/new/deeper.txt")).unwrap(), + "created\n" + ); + let created = native_call( + &runtime, + "/checkouts/main/src/new/deeper.txt", + "read", + json!({}), + ) + .await; + for value in [&glob, &grep, &read, &edit, &write, &create, &created] { + fixture.public_output(&serde_json::to_string(&wip_to_json(value).unwrap()).unwrap()); + } + let operations = fixture.state.operations.lock().unwrap(); + assert!( + operations + .iter() + .any(|op| matches!(op, WorkdirSessionOperation::CheckoutSearch(r) + if matches!(r.operation, CheckoutSearchOperation::Glob(_)))) + ); + assert!( + operations + .iter() + .any(|op| matches!(op, WorkdirSessionOperation::CheckoutSearch(r) + if matches!(r.operation, CheckoutSearchOperation::Grep(_)))) + ); + assert!(operations.iter().any( + |op| matches!(op, WorkdirSessionOperation::CheckoutExecute(r) + if matches!(r.operation, CheckoutOperation::Edit { .. })) + )); + assert!(operations.iter().any( + |op| matches!(op, WorkdirSessionOperation::CheckoutExecute(r) + if matches!(r.operation, CheckoutOperation::Create { .. })) + )); + drop(operations); + assert_eq!(fixture.state.write_calls(), 1); + router + .detach(&WorkdirAttachmentAlias::new("main").unwrap()) + .await + .unwrap(); + fixture.assert_endpoints(); + }) + .await; +} + +#[tokio::test] +async fn checkout_http_wrong_bearer_and_readonly_caps_fail_closed() { + watchdog(async { + let fixture = HttpFixture::new(WorkdirSessionCapabilities::READ_ONLY).await; + let error = fixture.open("wrong-bearer").await.unwrap_err(); + assert!(matches!(error, WorkdirError::Denied(_))); + fixture.public_output(&error.to_string()); + assert!(fixture.state.operations.lock().unwrap().is_empty()); + let remote = Arc::new(fixture.open(TOKEN).await.unwrap()); + assert_eq!(remote.capabilities(), WorkdirSessionCapabilities::READ_ONLY); + let (runtime, router) = native_runtime(remote.clone()); + let file = "/checkouts/main/src/deep/a.txt"; + native_call(&runtime, file, "read", json!({})).await; + let interface = observe_interface(&runtime, file).await; + { + let state = runtime.state.lock().unwrap(); + let observed = state.client.interface(&state.session, &interface).unwrap(); + let descriptor = observed.descriptor.as_ref().unwrap(); + assert_eq!( + descriptor + .operations + .iter() + .map(|op| op.name.as_str()) + .collect::>(), + ["read"] + ); + } + let directory_interface = observe_interface(&runtime, "/checkouts/main").await; + let before = fixture.state.execute_calls(); + for (path, interface, operation, args) in [ + ( + file, + interface.clone(), + "write", + json!({"content":"forbidden"}), + ), + ( + file, + interface, + "edit", + json!({"old_string":"needle", "new_string":"forbidden"}), + ), + ( + "/checkouts/main", + directory_interface, + "create_file", + json!({"path":"forbidden.txt","content":"bad"}), + ), + ] { + let error = runtime + .call( + path.into(), + interface, + operation.into(), + args, + Default::default(), + ) + .await + .unwrap_err(); + fixture.public_output(&error.to_string()); + } + // Preflight may observe objects, but no rejected mutation reaches execution. + assert_eq!(fixture.state.execute_calls(), before); + // A caller bypassing WIP publication still cannot mutate at the provider. + let parent = remote.checkout_observe(WorkdirPath::root()).await.unwrap(); + let error = remote + .checkout_execute(CheckoutRequest { + target: parent.path, + validator: parent.validator, + operation: CheckoutOperation::Create { + path: WorkdirPath::new("forbidden.txt").unwrap(), + content: b"bad".to_vec(), + }, + }) + .await + .unwrap_err(); + assert!(matches!( + error, + WorkdirError::Unsupported(_) + | WorkdirError::UnsupportedOperation(_) + | WorkdirError::ReadOnly(_) + | WorkdirError::Denied(_) + )); + fixture.public_output(&error.to_string()); + assert_eq!( + std::fs::read_to_string(fixture.dir.path().join("src/deep/a.txt")).unwrap(), + "first\nneedle needle\nlast\n" + ); + assert!(!fixture.dir.path().join("forbidden.txt").exists()); + router + .detach(&WorkdirAttachmentAlias::new("main").unwrap()) + .await + .unwrap(); + fixture.assert_endpoints(); + }) + .await; +} + +async fn committed_write_unknown(fault: WriteResponseFault) { + let fixture = HttpFixture::new(WorkdirSessionCapabilities::READ_WRITE).await; + let remote = Arc::new(fixture.open(TOKEN).await.unwrap()); + let (runtime, router) = native_runtime(remote); + let file = "/checkouts/main/src/deep/a.txt"; + native_call(&runtime, file, "read", json!({})).await; + let interface = observe_interface(&runtime, file).await; + *fixture.state.write_fault.lock().unwrap() = Some(fault); + let error = runtime + .call( + file.into(), + interface, + "write".into(), + json!({"content":"committed\n"}), + Default::default(), + ) + .await + .unwrap_err(); + assert!(error.to_string().contains("outcome unknown")); + assert!(error.to_string().contains("do not retry automatically")); + fixture.public_output(&error.to_string()); + assert_eq!( + runtime.audit().last().unwrap().outcome, + WipAuditOutcome::OutcomeUnknown + ); + { + let state = runtime.state.lock().unwrap(); + assert!(matches!( + state + .client + .call_history(&state.session) + .unwrap() + .back() + .unwrap() + .outcome, + CallOutcome::Unknown { .. } + )); + } + assert_eq!( + std::fs::read_to_string(fixture.dir.path().join("src/deep/a.txt")).unwrap(), + "committed\n" + ); + assert_eq!( + fixture.state.write_calls(), + 1, + "committed Write must not be automatically retried" + ); + // Explicit inspection of the effect is safe; it must not replay the mutation. + let read = native_call(&runtime, file, "read", json!({})).await; + fixture.public_output(&serde_json::to_string(&wip_to_json(&read).unwrap()).unwrap()); + let Value::String(content) = field(&read, "content") else { + panic!("expected Read text") + }; + assert!(content.contains("committed")); + assert_eq!(fixture.state.write_calls(), 1); + router + .detach(&WorkdirAttachmentAlias::new("main").unwrap()) + .await + .unwrap(); + fixture.assert_endpoints(); +} + +#[tokio::test] +async fn checkout_http_malformed_response_after_write_is_unknown_without_retry() { + watchdog(committed_write_unknown(WriteResponseFault::Malformed)).await; +} + +#[tokio::test] +async fn checkout_http_lost_response_after_write_is_unknown_without_retry() { + watchdog(committed_write_unknown(WriteResponseFault::LostBody)).await; +} diff --git a/crates/worker/src/checkout_race_tests.rs b/crates/worker/src/checkout_race_tests.rs new file mode 100644 index 000000000..954ce0977 --- /dev/null +++ b/crates/worker/src/checkout_race_tests.rs @@ -0,0 +1,538 @@ +//! Deterministic attachment races at provider observation boundaries. +//! +//! These tests deliberately avoid spawned tasks, sleeps, and detach attempts +//! inside a routed operation (which is busy and cannot be detached). Deadline +//! tests use the provider's configurable deadline and a permanently pending +//! observation, without Tokio's test-util clock or the production 30-second wait. +use super::*; + +use std::future::pending; +use std::sync::Mutex; +use std::sync::atomic::{AtomicUsize, Ordering}; +use tempfile::TempDir; +use workdir::{ + CheckoutRequest, CheckoutResult, CommandHandle, CommandOutput, CommandOutputRequest, + CommandRequest, CommandStatus, EditRequest, EditResult, GlobRequest, GlobResult, GrepRequest, + GrepResult, ListResult, LocalWorkdirSession, ReadBytesRequest, ReadBytesResult, ReadRequest, + ReadResult, StatRequest, StatResult, Workdir, WorkdirAttachmentAlias, WorkdirError, + WorkdirScopeAuthorizationRequest, WorkdirScopeOverlapRequest, WorkdirSession, + WorkdirSessionCapabilities, WorkdirSessionHandle, WriteRequest, WriteResult, +}; + +type ObserveHook = Box; + +/// Shared local provider with a one-shot, synchronous observation boundary. +/// Take the callback out of the lock before invoking it: it may call the router. +struct HookedSession { + local: LocalWorkdirSession, + observe_hook: Mutex>, + stall_path: Mutex>, + stalled_observations: AtomicUsize, +} + +impl std::fmt::Debug for HookedSession { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("HookedSession") + .field("local", &self.local) + .field("stall_path", &self.stall_path) + .field("stalled_observations", &self.stalled_observations) + .finish_non_exhaustive() + } +} + +impl HookedSession { + fn new(dir: &TempDir) -> Arc { + Arc::new(Self { + local: LocalWorkdirSession::materialized( + dir.path().to_path_buf(), + dir.path().to_path_buf(), + manifest::SharedScope::new(manifest::Scope::writable(dir.path()).unwrap()), + WorkdirSessionCapabilities::READ_WRITE, + ), + observe_hook: Mutex::new(None), + stall_path: Mutex::new(None), + stalled_observations: AtomicUsize::new(0), + }) + } + + fn on_next_observe(&self, hook: impl FnOnce(&WorkdirPath) + Send + 'static) { + let mut slot = self.observe_hook.lock().unwrap(); + assert!(slot.is_none(), "an observation hook is already armed"); + *slot = Some(Box::new(hook)); + } + + fn stall_observation(&self, path: Option) { + *self.stall_path.lock().unwrap() = path; + } +} + +#[async_trait] +impl WorkdirSession for HookedSession { + fn workdir(&self) -> &Workdir { + self.local.workdir() + } + + fn capabilities(&self) -> WorkdirSessionCapabilities { + self.local.capabilities() + } + + async fn authorize_scope_path( + &self, + request: WorkdirScopeAuthorizationRequest, + ) -> Result<(), WorkdirError> { + self.local.authorize_scope_path(request).await + } + + async fn scope_rules_overlap( + &self, + request: WorkdirScopeOverlapRequest, + ) -> Result { + self.local.scope_rules_overlap(request).await + } + + async fn checkout_observe( + &self, + path: WorkdirPath, + ) -> Result { + let hook = { self.observe_hook.lock().unwrap().take() }; + if let Some(hook) = hook { + hook(&path); + } + let should_stall = { + self.stall_path + .lock() + .unwrap() + .as_ref() + .is_some_and(|stall| stall.as_str() == path.as_str()) + }; + if should_stall { + self.stalled_observations.fetch_add(1, Ordering::SeqCst); + return pending().await; + } + self.local.checkout_observe(path).await + } + + async fn checkout_execute( + &self, + request: CheckoutRequest, + ) -> Result { + self.local.checkout_execute(request).await + } + + async fn checkout_search( + &self, + request: workdir::CheckoutSearchRequest, + ) -> Result { + self.local.checkout_search(request).await + } + + async fn stat(&self, request: StatRequest) -> Result { + self.local.stat(request).await + } + + async fn read(&self, request: ReadRequest) -> Result { + self.local.read(request).await + } + + async fn read_bytes(&self, request: ReadBytesRequest) -> Result { + self.local.read_bytes(request).await + } + + async fn write(&self, request: WriteRequest) -> Result { + self.local.write(request).await + } + + async fn edit(&self, request: EditRequest) -> Result { + self.local.edit(request).await + } + + async fn list(&self, request: ListRequest) -> Result { + self.local.list(request).await + } + + async fn glob(&self, request: GlobRequest) -> Result { + self.local.glob(request).await + } + + async fn grep(&self, request: GrepRequest) -> Result { + self.local.grep(request).await + } + + async fn start_command(&self, request: CommandRequest) -> Result { + self.local.start_command(request).await + } + + async fn command_status(&self, handle: CommandHandle) -> Result { + self.local.command_status(handle).await + } + + async fn command_output( + &self, + request: CommandOutputRequest, + ) -> Result { + self.local.command_output(request).await + } + + async fn cancel_command(&self, handle: CommandHandle) -> Result<(), WorkdirError> { + self.local.cancel_command(handle).await + } + + async fn close(&self) -> Result<(), WorkdirError> { + self.local.close().await + } +} + +fn provider(router: Arc) -> Arc { + Arc::new(Provider { + router, + tracker: tools::Tracker::new(), + permissions: None, + incarnation: "checkout-race-tests".into(), + permits: Arc::new(tokio::sync::Semaphore::new(16)), + deadline: DEADLINE, + }) +} + +#[tokio::test] +async fn checkout_wip_file_operations_share_core_and_keep_prior_read_boundary() { + let dir = TempDir::new().unwrap(); + std::fs::write(dir.path().join("text.txt"), "旧 旧\n").unwrap(); + let router = Arc::new(WorkdirSessionRouter::new()); + router + .attach( + WorkdirAttachmentAlias::new("main").unwrap(), + HookedSession::new(&dir), + ) + .unwrap(); + let provider = provider(router); + let route = format!("{}/text.txt", checkout_root("main")); + let projection = provider.node(&route).await.unwrap().unwrap(); + let invalid = BTreeMap::from([ + ("old_string".into(), Value::String("旧".into())), + ("new_string".into(), Value::String("旧".into())), + ]); + assert!( + matches!(projection.handler.call("edit", &invalid, context()).await, + Err(WipOperationError::Protocol(error)) if error.code == ProtocolErrorCode::InvalidArguments) + ); + let edit = BTreeMap::from([ + ("old_string".into(), Value::String("旧".into())), + ("new_string".into(), Value::String("新".into())), + ("replace_all".into(), Value::Boolean(true)), + ]); + // Valid shared arguments do not bypass the provider's Read requirement. + assert!( + projection + .handler + .call("edit", &edit, context()) + .await + .is_err() + ); + assert_eq!( + std::fs::read_to_string(dir.path().join("text.txt")).unwrap(), + "旧 旧\n" + ); + projection + .handler + .call("read", &BTreeMap::new(), context()) + .await + .unwrap_or_else(|_| panic!("shared read failed")); + projection + .handler + .call("edit", &edit, context()) + .await + .unwrap_or_else(|_| panic!("shared edit failed after read")); + assert_eq!( + std::fs::read_to_string(dir.path().join("text.txt")).unwrap(), + "新 新\n" + ); + let projection = provider.node(&route).await.unwrap().unwrap(); + projection + .handler + .call( + "write", + &BTreeMap::from([("content".into(), Value::String("written\n".into()))]), + context(), + ) + .await + .unwrap_or_else(|_| panic!("shared write failed after read/edit")); + assert_eq!( + std::fs::read_to_string(dir.path().join("text.txt")).unwrap(), + "written\n" + ); +} + +fn deadline_provider(router: Arc) -> Arc { + let mut p = provider(router); + let settings = Arc::get_mut(&mut p).unwrap(); + // Allow ample time for the tiny local search before stalling post-processing. + // The outer test watchdog is intentionally much more generous than this. + settings.deadline = Duration::from_millis(100); + settings.permits = Arc::new(tokio::sync::Semaphore::new(1)); + p +} + +fn context() -> WipCallContext { + WipCallContext { + execution: Default::default(), + security_context: "checkout-race-tests".into(), + } +} + +fn assert_validator_mismatch(result: Result) { + match result { + Err(WipOperationError::Protocol(e)) => { + assert_eq!(e.code, ProtocolErrorCode::ValidatorMismatch); + } + Err(_) => panic!("expected ValidatorMismatch, received a non-protocol error"), + Ok(_) => panic!("a changed attachment snapshot must not be published"), + } +} + +#[tokio::test] +async fn collection_list_rejects_alias_attached_during_observation_and_retries_coherently() { + let main_dir = TempDir::new().unwrap(); + let extra_dir = TempDir::new().unwrap(); + let main = HookedSession::new(&main_dir); + let extra: WorkdirSessionHandle = HookedSession::new(&extra_dir); + let router = Arc::new(WorkdirSessionRouter::new()); + router + .attach(WorkdirAttachmentAlias::new("main").unwrap(), main.clone()) + .unwrap(); + let p = provider(router.clone()); + let before = p.collection().object.validator; + let hook_router = router.clone(); + main.on_next_observe(move |path| { + assert!(path.is_root(), "collection enumeration observes the root"); + hook_router + .attach(WorkdirAttachmentAlias::new("extra").unwrap(), extra) + .unwrap(); + }); + + let collection = Collection { + provider: p.clone(), + }; + assert_validator_mismatch(collection.call("list", &BTreeMap::new(), context()).await); + assert_ne!(before, p.collection().object.validator); + assert!(router.resolve(Some("extra")).is_ok(), "the hook must run"); + + // The callback was consumed. A fresh snapshot includes both aliases, with + // identity metadata from exactly the router state being enumerated. + let output = match collection.call("list", &BTreeMap::new(), context()).await { + Ok(output) => output, + Err(_) => panic!("the stable retry must succeed"), + }; + let value = wip_to_json(&output.value).unwrap(); + let items = value["items"].as_array().unwrap(); + assert_eq!(items.len(), 2); + for alias in ["extra", "main"] { + let item = items + .iter() + .find(|item| item["alias"] == json!(alias)) + .expect("each attached alias must appear in the stable snapshot"); + let selected = router.resolve(Some(alias)).unwrap(); + assert_eq!(item["alias"], json!(alias)); + assert_eq!(item["path"], json!(checkout_root(alias))); + assert_eq!(item["slug"], json!(encode_identity(alias))); + assert_eq!(item["generation"], json!(selected.generation)); + assert_eq!( + item["working_directory_id"], + json!(selected.session.workdir().id().as_str()) + ); + } +} + +#[tokio::test] +async fn root_enumeration_rejects_alias_attached_during_observation() { + let main_dir = TempDir::new().unwrap(); + let extra_dir = TempDir::new().unwrap(); + let main = HookedSession::new(&main_dir); + let extra: WorkdirSessionHandle = HookedSession::new(&extra_dir); + let router = Arc::new(WorkdirSessionRouter::new()); + router + .attach(WorkdirAttachmentAlias::new("main").unwrap(), main.clone()) + .unwrap(); + let hook_router = router.clone(); + main.on_next_observe(move |_| { + hook_router + .attach(WorkdirAttachmentAlias::new("extra").unwrap(), extra) + .unwrap(); + }); + let subtree = Arc::new(provider(router.clone())); + + match subtree.children(ROOT).await { + Err(e) => assert_eq!(e.code, ProtocolErrorCode::ValidatorMismatch), + Ok(_) => panic!("enumeration must reject a mixed attachment snapshot"), + } + assert!(router.resolve(Some("extra")).is_ok(), "the hook must run"); + let mut paths = subtree.children(ROOT).await.unwrap(); + paths.sort(); + assert_eq!(paths, vec![checkout_root("extra"), checkout_root("main")]); +} + +#[tokio::test] +async fn file_handler_connection_fences_detach_and_same_session_alias_reuse() { + let dir = TempDir::new().unwrap(); + std::fs::write(dir.path().join("file.txt"), "unchanged\n").unwrap(); + let session: WorkdirSessionHandle = HookedSession::new(&dir); + let router = Arc::new(WorkdirSessionRouter::new()); + let alias = WorkdirAttachmentAlias::new("main").unwrap(); + router.attach(alias.clone(), session.clone()).unwrap(); + let selected = router.resolve(Some("main")).unwrap(); + let target = WorkdirPath::new("file.txt").unwrap(); + let observation = selected + .session + .checkout_observe(target.clone()) + .await + .unwrap(); + let mut handler = FileHandler { + provider: provider(router.clone()), + alias: "main".into(), + generation: selected.generation, + workdir: selected.session.workdir().id().as_str().into(), + target, + observation, + }; + drop(selected); + assert!(handler.check_connection().is_ok()); + + // No routed operation is active here. Reuse the exact same provider identity + // to ensure generation, not changed workdir ID/content, causes rejection. + // Detach may close the local session; no subsequent filesystem operation is + // made on it. This test only exercises the router's connection identity. + router.detach(&alias).await.unwrap(); + assert_validator_mismatch(handler.check_connection()); + router.attach(alias, session).unwrap(); + let replacement = router.resolve(Some("main")).unwrap(); + assert_ne!(replacement.generation, handler.generation); + assert_eq!(replacement.session.workdir().id().as_str(), handler.workdir); + assert_validator_mismatch(handler.check_connection()); + + // Positive control: the new generation is accepted with the same identity. + handler.generation = replacement.generation; + assert!(handler.check_connection().is_ok()); +} + +fn assert_deadline(result: Result, message: &str) { + match result { + Err(WipOperationError::Protocol(e)) => { + assert_eq!(e.code, ProtocolErrorCode::ResourceLimitExceeded); + assert_eq!(e.message, message); + } + Err(_) => panic!("expected a protocol deadline error"), + Ok(_) => panic!("a stalled provider observation must time out"), + } +} + +fn assert_permit_released(p: &Provider) { + assert_eq!(p.permits.available_permits(), 1); + let permit = p + .permits + .try_acquire() + .expect("the timed-out operation must release its permit"); + drop(permit); +} + +#[tokio::test] +async fn collection_list_and_root_enumeration_bound_stalled_observation_and_release_permit() { + let dir = TempDir::new().unwrap(); + let session = HookedSession::new(&dir); + let router = Arc::new(WorkdirSessionRouter::new()); + router + .attach( + WorkdirAttachmentAlias::new("main").unwrap(), + session.clone(), + ) + .unwrap(); + session.stall_observation(Some(WorkdirPath::new("").unwrap())); + let p = deadline_provider(router); + let collection = Collection { + provider: p.clone(), + }; + + // The outer watchdog prevents a missing production timeout from hanging the + // suite. The exact protocol message proves the inner enumeration timed out. + let result = tokio::time::timeout( + Duration::from_secs(2), + collection.call("list", &BTreeMap::new(), context()), + ) + .await + .expect("Collection.list must be bounded by the provider deadline"); + assert_deadline(result, "checkout enumeration deadline"); + assert_eq!(session.stalled_observations.load(Ordering::SeqCst), 1); + assert_permit_released(&p); + + // Exercise the subtree entry point independently with the stall still armed. + let subtree = Arc::new(p.clone()); + let result = tokio::time::timeout(Duration::from_secs(2), subtree.children(ROOT)) + .await + .expect("root enumeration must be bounded by the provider deadline") + .map_err(WipOperationError::Protocol); + assert_deadline(result, "checkout enumeration deadline"); + assert_eq!(session.stalled_observations.load(Ordering::SeqCst), 2); + assert_permit_released(&p); + + // Removing the stall makes the same provider usable immediately, without + // resetting its semaphore or waiting for any background task to finish. + session.stall_observation(None); + let paths = tokio::time::timeout(Duration::from_secs(2), subtree.children(ROOT)) + .await + .expect("enumeration must remain usable after cancellation") + .unwrap(); + assert_eq!(paths, vec![checkout_root("main")]); + assert_permit_released(&p); +} + +#[tokio::test] +async fn grep_post_search_link_observation_is_deadline_bounded_and_releases_permit() { + let dir = TempDir::new().unwrap(); + std::fs::write(dir.path().join("a.txt"), "needle\n").unwrap(); + let session = HookedSession::new(&dir); + let router = Arc::new(WorkdirSessionRouter::new()); + router + .attach( + WorkdirAttachmentAlias::new("main").unwrap(), + session.clone(), + ) + .unwrap(); + let p = deadline_provider(router); + let root = checkout_root("main"); + let projection = p.node(&root).await.unwrap().unwrap(); + + // Directory observations and the normal local grep/checkout execution can + // complete. Only publishing the typed search result path gets stuck. + session.stall_observation(Some(WorkdirPath::new("a.txt").unwrap())); + let arguments = BTreeMap::from([ + ("pattern".into(), Value::String("needle".into())), + ( + "output_mode".into(), + Value::String("files_with_matches".into()), + ), + ]); + let result = tokio::time::timeout( + Duration::from_secs(2), + projection.handler.call("grep", &arguments, context()), + ) + .await + .expect("post-search link observations must share the operation deadline"); + assert_deadline(result, "checkout result deadline"); + assert_eq!(session.stalled_observations.load(Ordering::SeqCst), 1); + assert_permit_released(&p); + + // A successful retry both checks cancellation cleanup and proves that the + // delegated search produces the typed path consumed by post-processing. + session.stall_observation(None); + let result = tokio::time::timeout( + Duration::from_secs(2), + projection.handler.call("grep", &arguments, context()), + ) + .await + .expect("grep must remain usable after cancelling result publication"); + let output = match result { + Ok(output) => output, + Err(_) => panic!("grep must succeed once the link observation is unstalled"), + }; + let value = wip_to_json(&output.value).unwrap(); + assert_eq!(value["items"], json!([{"path": format!("{root}/a.txt")}])); + assert_permit_released(&p); +} diff --git a/crates/worker/src/checkout_wip_tests.rs b/crates/worker/src/checkout_wip_tests.rs new file mode 100644 index 000000000..389f9df50 --- /dev/null +++ b/crates/worker/src/checkout_wip_tests.rs @@ -0,0 +1,656 @@ +//! Exercises native checkout interactions through the real WIP Client/HTTP codec. +use super::*; +use tempfile::TempDir; +use workdir::{ + LocalWorkdirSession, WorkdirAttachmentAlias, WorkdirSessionCapabilities, WorkdirSessionHandle, + WorkdirSessionRouter, +}; + +fn session(dir: &TempDir, read_only: bool) -> WorkdirSessionHandle { + let caps = if read_only { + WorkdirSessionCapabilities::READ_ONLY + } else { + WorkdirSessionCapabilities::READ_WRITE + }; + Arc::new(LocalWorkdirSession::materialized( + dir.path().to_path_buf(), + dir.path().to_path_buf(), + manifest::SharedScope::new(manifest::Scope::writable(dir.path()).unwrap()), + caps, + )) +} +fn runtime( + router: Arc, + permissions: Option, +) -> WipRuntime { + let mut registry = WipMountRegistry::new(); + crate::checkout::mount_checkouts(&mut registry, router, tools::Tracker::new(), permissions) + .unwrap(); + WipRuntime::new( + WipHost::new(registry), + SecurityContext::new("checkout-test-worker"), + 0, + ) + .unwrap() +} +async fn interface(r: &WipRuntime, path: &str) -> String { + r.discover(path.into(), 0, true).await.unwrap(); + let p = r.host.projection_live(path).await.unwrap().unwrap(); + r.inspect(p.interface.clone(), true).await.unwrap(); + p.interface +} +async fn call( + r: &WipRuntime, + path: &str, + operation: &str, + args: Json, +) -> Result { + let i = interface(r, path).await; + r.call(path.into(), i, operation.into(), args, Default::default()) + .await +} + +#[tokio::test] +async fn checkout_wip_search_preserves_root_and_nested_ignore_rules_and_links() { + let dir = TempDir::new().unwrap(); + std::fs::create_dir(dir.path().join(".git")).unwrap(); + std::fs::create_dir(dir.path().join("nested")).unwrap(); + for (name, content) in [ + (".ignore", "root-denied.txt\npriority.txt\n"), + (".gitignore", "git-denied.txt\n"), + ("nested/.gitignore", "!priority.txt\nnested-denied.txt\n"), + ] { + std::fs::write(dir.path().join(name), content).unwrap(); + } + for name in [ + "visible.txt", + "root-denied.txt", + "git-denied.txt", + "nested/visible.txt", + "nested/root-denied.txt", + "nested/git-denied.txt", + "nested/priority.txt", + "nested/nested-denied.txt", + ] { + std::fs::write(dir.path().join(name), "needle fixture\n").unwrap(); + } + let source = session(&dir, false); + let router = Arc::new(WorkdirSessionRouter::new()); + router + .attach(WorkdirAttachmentAlias::new("main").unwrap(), source.clone()) + .unwrap(); + let r = runtime(router, None); + for base in ["", "nested"] { + let ordinary = source + .glob(workdir::GlobRequest { + path: workdir::WorkdirPath::new(base).unwrap(), + pattern: "**/*.txt".into(), + limit: 100, + }) + .await + .unwrap(); + let expected = ordinary + .paths + .iter() + .map(|path| format!("/checkouts/main/{}", path.as_str())) + .collect::>(); + assert_eq!(expected.len(), if base.is_empty() { 2 } else { 1 }); + let route = if base.is_empty() { + "/checkouts/main" + } else { + "/checkouts/main/nested" + }; + for (operation, args) in [ + ("glob", json!({"pattern":"**/*.txt"})), + ( + "grep", + json!({"pattern":"needle","output_mode":"files_with_matches"}), + ), + ] { + let output = call(&r, route, operation, args).await.unwrap(); + let result: Json = serde_json::from_str(output.content.as_deref().unwrap()).unwrap(); + let mut paths = result["items"] + .as_array() + .unwrap() + .iter() + .map(|item| item["path"].as_str().unwrap().to_string()) + .collect::>(); + // Grep retains traversal order; Glob sorts its result paths. + paths.sort(); + assert_eq!(paths, expected, "{operation} at {route}"); + for path in paths { + assert!(r.host.projection_live(&path).await.unwrap().is_some()); + } + assert!(!output.content.unwrap().contains("denied.txt")); + } + } +} + +#[tokio::test] +async fn checkout_wip_native_roundtrip_search_read_edit_write_create() { + let dir = TempDir::new().unwrap(); + std::fs::create_dir_all(dir.path().join("src/deep")).unwrap(); + std::fs::write( + dir.path().join("src/deep/a.txt"), + "first\nneedle needle\nlast\n", + ) + .unwrap(); + let router = Arc::new(WorkdirSessionRouter::new()); + router + .attach( + WorkdirAttachmentAlias::new("main").unwrap(), + session(&dir, false), + ) + .unwrap(); + let r = runtime(router, None); + let root = r.host.observe_live("/", 2).await.unwrap(); + assert_eq!(root.children.unwrap()[0].object.name, "checkouts"); + let file = "/checkouts/main/src/deep/a.txt"; + let glob = call(&r, "/checkouts/main", "glob", json!({"pattern":"**/*.txt"})) + .await + .unwrap(); + assert!(glob.content.unwrap().contains(file)); + let grep = call( + &r, + "/checkouts/main/src", + "grep", + json!({"pattern":"needle","output_mode":"files_with_matches"}), + ) + .await + .unwrap(); + let grep_result: Json = serde_json::from_str(grep.content.as_deref().unwrap()).unwrap(); + assert_eq!(grep_result["items"][0]["path"], file); + let followed = grep_result["items"][0]["path"].as_str().unwrap(); + assert_eq!(followed, file); + assert!( + call( + &r, + file, + "edit", + json!({"old_string":"needle","new_string":"done"}) + ) + .await + .is_err() + ); + let read = call(&r, file, "read", json!({"offset":1,"limit":1})) + .await + .unwrap(); + assert!(read.content.unwrap().contains("needle needle")); + assert!( + call( + &r, + file, + "edit", + json!({"old_string":"needle","new_string":"done"}) + ) + .await + .is_err() + ); + call( + &r, + file, + "edit", + json!({"old_string":"needle","new_string":"done","replace_all":true}), + ) + .await + .unwrap(); + assert_eq!( + std::fs::read_to_string(dir.path().join("src/deep/a.txt")).unwrap(), + "first\ndone done\nlast\n" + ); + call(&r, file, "write", json!({"content":"saved\n"})) + .await + .unwrap(); + let reread = call(&r, file, "read", json!({})).await.unwrap(); + assert!(reread.content.unwrap().contains("saved")); + call( + &r, + "/checkouts/main/src", + "create_file", + json!({"path":"new/deeper.txt","content":"created\n"}), + ) + .await + .unwrap(); + assert_eq!( + std::fs::read_to_string(dir.path().join("src/new/deeper.txt")).unwrap(), + "created\n" + ); + assert!( + call( + &r, + "/checkouts/main/src", + "create_file", + json!({"path":"new/deeper.txt","content":"overwrite"}) + ) + .await + .is_err() + ); + call(&r, "/checkouts/main/src/new/deeper.txt", "read", json!({})) + .await + .unwrap(); +} + +#[tokio::test] +async fn checkout_wip_multiple_aliases_read_only_and_escape_rejection() { + let a = TempDir::new().unwrap(); + let b = TempDir::new().unwrap(); + std::fs::write(a.path().join("a.txt"), "A").unwrap(); + std::fs::write(b.path().join("b.txt"), "B").unwrap(); + let router = Arc::new(WorkdirSessionRouter::new()); + router + .attach( + WorkdirAttachmentAlias::new("a").unwrap(), + session(&a, false), + ) + .unwrap(); + router + .attach(WorkdirAttachmentAlias::new("b").unwrap(), session(&b, true)) + .unwrap(); + let r = runtime(router, None); + let observed = r.host.observe_live("/checkouts", 2).await.unwrap(); + assert_eq!(observed.children.unwrap().len(), 2); + let bfile = r + .host + .projection_live("/checkouts/b/b.txt") + .await + .unwrap() + .unwrap(); + assert_eq!( + bfile + .descriptor + .operations + .iter() + .map(|o| o.name.as_str()) + .collect::>(), + vec!["read"] + ); + assert!( + r.host + .projection_live("/checkouts/b/a.txt") + .await + .unwrap() + .is_none() + ); + assert!( + r.host + .projection_live("/checkouts/a/../b/b.txt") + .await + .is_err() + ); + assert!( + call( + &r, + "/checkouts/a", + "glob", + json!({"pattern":"*","path":"../b"}) + ) + .await + .is_err() + ); + assert!( + call( + &r, + "/checkouts/a", + "create_file", + json!({"path":"/outside.txt","content":"bad"}) + ) + .await + .is_err() + ); + assert!( + call( + &r, + "/checkouts/a/a.txt", + "read", + json!({"target_workdir":"b"}) + ) + .await + .is_err() + ); + assert!( + call( + &r, + "/checkouts/a/a.txt", + "read", + json!({"file_path":"b.txt"}) + ) + .await + .is_err() + ); +} + +#[tokio::test] +async fn checkout_wip_detach_alias_reuse_and_restore_fence_old_references() { + let a = TempDir::new().unwrap(); + let b = TempDir::new().unwrap(); + std::fs::write(a.path().join("file.txt"), "same").unwrap(); + std::fs::write(b.path().join("file.txt"), "same").unwrap(); + let router = Arc::new(WorkdirSessionRouter::new()); + let alias = WorkdirAttachmentAlias::new("main").unwrap(); + router.attach(alias.clone(), session(&a, false)).unwrap(); + let r = runtime(router.clone(), None); + let path = "/checkouts/main/file.txt"; + call(&r, path, "read", json!({})).await.unwrap(); + let old = r.host.projection_live(path).await.unwrap().unwrap(); + router.detach(&alias).await.unwrap(); + assert!(r.host.projection_live(path).await.unwrap().is_none()); + assert!(r.host.fetch_interface_live(&old.interface).await.is_err()); + router.attach(alias, session(&b, false)).unwrap(); + let new = r.host.projection_live(path).await.unwrap().unwrap(); + assert_ne!(old.interface, new.interface); + assert_ne!(old.object.validator, new.object.validator); + assert!(r.host.fetch_interface_live(&old.interface).await.is_err()); + assert!( + call( + &r, + path, + "write", + json!({"content":"must read new attachment"}) + ) + .await + .is_err() + ); + let restored = runtime(router, None); + let next = restored.host.projection_live(path).await.unwrap().unwrap(); + assert_ne!(new.interface, next.interface); + assert_ne!(new.object.validator, next.object.validator); + assert!( + restored + .host + .fetch_interface_live(&new.interface) + .await + .is_err() + ); + assert_eq!( + std::fs::read_to_string(b.path().join("file.txt")).unwrap(), + "same" + ); +} + +#[tokio::test] +async fn checkout_wip_provider_validator_rejects_same_content_inode_substitution() { + let dir = TempDir::new().unwrap(); + std::fs::write(dir.path().join("file.txt"), "same").unwrap(); + let router = Arc::new(WorkdirSessionRouter::new()); + router + .attach( + WorkdirAttachmentAlias::new("main").unwrap(), + session(&dir, false), + ) + .unwrap(); + let r = runtime(router, None); + let path = "/checkouts/main/file.txt"; + call(&r, path, "read", json!({})).await.unwrap(); + let p = r.host.projection_live(path).await.unwrap().unwrap(); + std::fs::write(dir.path().join("replacement"), "same").unwrap(); + std::fs::rename(dir.path().join("replacement"), dir.path().join("file.txt")).unwrap(); + // Call the captured handler directly: provider must reject even if Host validation was earlier. + let out = p + .handler + .call( + "write", + &BTreeMap::from([("content".into(), Value::String("bad".into()))]), + WipCallContext { + execution: Default::default(), + security_context: "checkout-test-worker".into(), + }, + ) + .await; + assert!(matches!( + out, + Err(WipOperationError::Protocol(ProtocolError { + code: ProtocolErrorCode::ValidatorMismatch, + .. + })) + )); + assert_eq!( + std::fs::read_to_string(dir.path().join("file.txt")).unwrap(), + "same" + ); +} + +#[tokio::test] +async fn checkout_wip_policy_is_not_feature_authority_and_compatibility_is_replaced() { + let dir = TempDir::new().unwrap(); + std::fs::write(dir.path().join("file.txt"), "secret").unwrap(); + let router = Arc::new(WorkdirSessionRouter::new()); + router + .attach( + WorkdirAttachmentAlias::new("main").unwrap(), + session(&dir, false), + ) + .unwrap(); + let permissions = ToolPermissionConfig { + default_action: ToolPermissionAction::Deny, + rules: Vec::new(), + }; + let r = runtime(router.clone(), Some(permissions)); + assert!( + r.host + .projection_live("/checkouts/main/file.txt") + .await + .unwrap() + .is_none() + ); + assert!( + r.host + .observe_live("/checkouts", 1) + .await + .unwrap() + .children + .unwrap() + .is_empty() + ); + let mut registry = WipMountRegistry::new(); + crate::checkout::mount_checkouts(&mut registry, router.clone(), tools::Tracker::new(), None) + .unwrap(); + for def in tools::routed_builtin_tools(router, tools::Tracker::new(), dir.path().join("bash")) { + let (meta, tool) = def(); + let p = compatibility_projection(meta, tool, None).unwrap(); + registry.mount(p).unwrap(); + } + for tool in ["Read", "Edit", "Write", "Glob", "Grep"] { + assert!(!registry.routes().any(|p| p == format!("/tools/{tool}"))); + } + assert!(registry.routes().any(|p| p == "/tools/Bash")); +} + +#[tokio::test] +async fn checkout_wip_bounded_observation_and_deep_subtree_collision() { + let dir = TempDir::new().unwrap(); + let router = Arc::new(WorkdirSessionRouter::new()); + router + .attach( + WorkdirAttachmentAlias::new("main").unwrap(), + session(&dir, false), + ) + .unwrap(); + let mut registry = WipMountRegistry::new(); + crate::checkout::mount_checkouts(&mut registry, router, tools::Tracker::new(), None).unwrap(); + let placeholder = registry.mounts["/checkouts"].resolved(); + let mut collision = placeholder.clone(); + collision.route = "/checkouts/main/a".into(); + collision.object.name = "a".into(); + assert!(matches!( + registry.mount(collision), + Err(WipMountError::RouteCollision { .. }) + )); + let host = WipHost::new(registry); + assert_eq!( + host.observe_live("/checkouts", 33).await.unwrap_err().code, + ProtocolErrorCode::ResourceLimitExceeded + ); +} + +#[tokio::test] +async fn checkout_wip_post_operation_validators_support_cached_chains() { + let dir = TempDir::new().unwrap(); + std::fs::write(dir.path().join("file.txt"), "old\n").unwrap(); + let router = Arc::new(WorkdirSessionRouter::new()); + router + .attach( + WorkdirAttachmentAlias::new("main").unwrap(), + session(&dir, false), + ) + .unwrap(); + let r = runtime(router, None); + let path = "/checkouts/main/file.txt"; + let i = interface(&r, path).await; + // No rediscovery or Interface refresh between any of these operations. + for (op, args) in [ + ("read", json!({})), + ("edit", json!({"old_string":"old","new_string":"edited"})), + ("write", json!({"content":"saved\n"})), + ("read", json!({})), + ] { + r.call(path.into(), i.clone(), op.into(), args, Default::default()) + .await + .unwrap(); + } + assert_eq!( + std::fs::read_to_string(dir.path().join("file.txt")).unwrap(), + "saved\n" + ); + let parent = "/checkouts/main"; + let i = interface(&r, parent).await; + for name in ["one.txt", "nested/two.txt"] { + r.call( + parent.into(), + i.clone(), + "create_file".into(), + json!({"path":name,"content":"new"}), + Default::default(), + ) + .await + .unwrap(); + } + assert_eq!( + std::fs::read_to_string(dir.path().join("nested/two.txt")).unwrap(), + "new" + ); +} + +#[tokio::test] +async fn checkout_wip_scoped_child_search_and_revoke_cannot_expose_siblings() { + use workdir::{ + WorkdirPath, WorkdirToolBroker, WorkdirToolScope, WorkdirToolScopePermission as Permission, + WorkdirToolScopeRule, + }; + let dir = TempDir::new().unwrap(); + std::fs::create_dir(dir.path().join("child")).unwrap(); + std::fs::write(dir.path().join("child/a.txt"), "visible needle").unwrap(); + std::fs::write(dir.path().join("private-sibling-name.txt"), "hidden needle").unwrap(); + let broker = Arc::new(WorkdirToolBroker::new(session(&dir, false))); + let lease = broker + .scope(WorkdirToolScope { + rules: vec![WorkdirToolScopeRule { + target: WorkdirPath::new("child").unwrap(), + permission: Permission::Write, + recursive: true, + symlink_policy: Default::default(), + }], + cwd: WorkdirPath::new("child").unwrap(), + command: false, + }) + .await + .unwrap(); + let parent_router = Arc::new(WorkdirSessionRouter::new()); + parent_router + .attach( + WorkdirAttachmentAlias::new("parent").unwrap(), + broker.tool_session(), + ) + .unwrap(); + let parent = runtime(parent_router, None); + let parent_file = parent + .host + .projection_live("/checkouts/parent/child/a.txt") + .await + .unwrap() + .unwrap(); + assert!( + !parent_file + .descriptor + .operations + .iter() + .any(|op| op.name == "write" || op.name == "edit") + ); + let child_router = Arc::new(WorkdirSessionRouter::new()); + child_router + .attach( + WorkdirAttachmentAlias::new("child").unwrap(), + lease.tool_session(), + ) + .unwrap(); + let child = runtime(child_router, None); + let observed = child + .host + .observe_live("/checkouts/child", 2) + .await + .unwrap(); + assert_eq!(observed.children.unwrap().len(), 1); + for (op, args) in [ + ("glob", json!({"pattern":"**/*"})), + ( + "grep", + json!({"pattern":"needle","output_mode":"files_with_matches"}), + ), + ] { + let output = call(&child, "/checkouts/child", op, args) + .await + .unwrap() + .content + .unwrap(); + assert!(output.contains("/checkouts/child/a.txt")); + assert!(!output.contains("private-sibling-name")); + assert!(!output.contains("hidden needle")); + } + assert!( + child + .host + .projection_live("/checkouts/child/private-sibling-name.txt") + .await + .unwrap() + .is_none() + ); + let file = "/checkouts/child/a.txt"; + let i = interface(&child, file).await; + child + .call( + file.into(), + i.clone(), + "read".into(), + json!({}), + Default::default(), + ) + .await + .unwrap(); + child + .call( + file.into(), + i.clone(), + "write".into(), + json!({"content":"changed"}), + Default::default(), + ) + .await + .unwrap(); + lease.close().await.unwrap(); + assert!(child.host.projection_live(file).await.unwrap().is_none()); + assert!( + child + .call( + file.into(), + i, + "write".into(), + json!({"content":"after revoke"}), + Default::default() + ) + .await + .is_err() + ); + assert_eq!( + std::fs::read_to_string(dir.path().join("child/a.txt")).unwrap(), + "changed" + ); +} diff --git a/crates/worker/src/controller.rs b/crates/worker/src/controller.rs index 950a81a55..509c2a85e 100644 --- a/crates/worker/src/controller.rs +++ b/crates/worker/src/controller.rs @@ -64,6 +64,20 @@ pub struct WorkerHandle { } impl WorkerHandle { + /// Transport closure belongs to this Controller endpoint, not a WorkerRef. + pub fn protocol_is_closed(&self) -> bool { + self.method_tx.is_closed() + } + + pub async fn protocol_closed(&self) { + self.method_tx.closed().await; + } + + /// Compare actual endpoints when an execution-bound transport is dispatched. + pub fn same_controller(&self, other: &Self) -> bool { + self.method_tx.same_channel(&other.method_tx) + } + pub async fn send(&self, method: Method) -> Result<(), mpsc::error::SendError> { // Reject known-busy sends before channel dispatch, including maintenance // that cannot receive methods until it finishes. The controller checks @@ -184,6 +198,7 @@ impl WorkerHandle { &self, kind: protocol::CompletionKind, prefix: &str, + context: Option<&protocol::CompletionContext>, ) -> Vec { match kind { protocol::CompletionKind::File => { @@ -196,9 +211,62 @@ impl WorkerHandle { .map(|candidate| protocol::CompletionEntry { value: candidate.path, is_dir: candidate.is_dir, + description: None, + usage: None, + invocation: None, }) .collect() } + protocol::CompletionKind::Feature => self + .shared_state + .feature_invocations() + .feature_completions(prefix), + protocol::CompletionKind::FeatureArgument => { + let Some(context) = context else { + return Vec::new(); + }; + let registry = self.shared_state.feature_invocations(); + let Some(argument) = context.argument.as_deref() else { + return registry.argument_name_completions(&context.invocation, prefix); + }; + let worker_file = registry + .descriptor(&context.invocation) + .and_then(|descriptor| { + descriptor + .arguments + .iter() + .find(|candidate| candidate.name == argument) + }) + .is_some_and(|descriptor| { + matches!( + descriptor.completion, + protocol::InvocationCompletion::WorkerFile + ) + }); + if worker_file { + let Some(view) = self.shared_state.fs_view() else { + return Vec::new(); + }; + let candidates = view.list_file_completions(prefix).await; + if registry.descriptor(&context.invocation).is_none() { + return Vec::new(); + } + return crate::feature::invocation::bounded_completion_entries( + candidates + .into_iter() + .map(|candidate| protocol::CompletionEntry { + value: candidate.path, + is_dir: candidate.is_dir, + description: None, + usage: None, + invocation: None, + }), + ); + } + registry + .argument_completions(&context.invocation, argument, prefix) + .await + } } } @@ -435,11 +503,29 @@ fn stage_pending_notifications( count } +/// The durable input append is the activation boundary. Clear live state in +/// the same poll, before invocation preparation can checkpoint it again; the +/// oneshot only publishes the already-completed transition to the Controller. +pub(crate) fn input_committed_hook( + pending: crate::worker::PendingSubmissionHandle, + submission_id: String, + sender: oneshot::Sender<()>, +) -> impl FnOnce() { + move || { + pending.finish_activation(&submission_id); + let _ = sender.send(()); + } +} + fn notification_coalesce_remaining( pending_submissions: &crate::worker::PendingSubmissionHandle, notify_buffer: &NotifyBuffer, delay: Duration, + can_schedule_run: bool, ) -> Option { + if !can_schedule_run { + return None; + } let Some(accepted_at_ms) = pending_submissions.oldest_pending_notification_accepted_at_ms() else { return notify_buffer @@ -801,9 +887,6 @@ impl WorkerController { None, ) .await?; - let command_observer = fs_for_view - .as_ref() - .and_then(|session| wire_workdir_command_events(session, &in_flight)); // Intake role Workers self-terminate only after a successful // TicketIntakeReady turn has fully settled back to Idle. The request @@ -846,6 +929,7 @@ impl WorkerController { manifest_toml.clone(), greeting, )); + shared_state.set_feature_invocations(worker.feature_invocations()); let usage_state = shared_state.clone(); let usage_events = working_event_tx.clone(); worker.engine_mut().on_usage(move |event| { @@ -856,8 +940,8 @@ impl WorkerController { protocol::ContextTokenSource::Measured, ); }); - if let Some(fs_for_view) = fs_for_view { - shared_state.set_fs_view(crate::fs_view::WorkerFsView::new(fs_for_view)); + if let Some(fs_for_view) = fs_for_view.as_ref() { + shared_state.set_fs_view(crate::fs_view::WorkerFsView::new(fs_for_view.clone())); } runtime_dir.write_manifest(&manifest_toml).await?; runtime_dir.write_status(&shared_state).await?; @@ -879,8 +963,23 @@ impl WorkerController { pending_activations, }; + // Do not start an observer before fallible head/recovery/runtime-file + // initialization. The only remaining fallible step is socket startup, + // whose error path must abort AND join this task before returning. + let command_observer = fs_for_view + .as_ref() + .and_then(|session| wire_workdir_command_events(session, &in_flight)); let socket_server = match transport { - WorkerControllerTransport::UnixSocket => Some(SocketServer::start(&handle).await?), + WorkerControllerTransport::UnixSocket => match SocketServer::start(&handle).await { + Ok(server) => Some(server), + Err(error) => { + if let Some(observer) = command_observer { + observer.abort(); + let _ = observer.await; + } + return Err(error); + } + }, WorkerControllerTransport::InProcess => None, }; @@ -1277,10 +1376,12 @@ where ), None => durable_parent_notifications, }; - let backend_job_profile = worker.manifest().engine.instruction == "internal.backend_job_system"; + let backend_job = worker.backend_job().cloned(); + // Bounded Jobs complete their lifecycle synchronously, never by post-run background work. + let lifecycle_enabled = worker.manifest_lifecycle_features_enabled() && backend_job.is_none(); let prompts = worker.prompts().clone(); let tracker = tools::Tracker::new(); - if !backend_job_profile { + { let paste_store = worker.store().clone(); let paste_session_id = worker.session_id(); worker @@ -1327,7 +1428,9 @@ where let worker_enabled = feature_config.worker.enabled; let sub_worker_enabled = feature_config.sub_worker.enabled; let mut feature_registry = FeatureRegistryBuilder::new(); - if backend_job_profile { + feature_registry + .add_module(crate::feature::builtin::chat_invocation::AttachmentInvocationFeature); + if let Some(binding) = backend_job.as_ref() { let workspace_client = worker.workspace_client_handle(); if !workspace_client.is_available() || workspace_client.workspace_id().is_none() { return Err(std::io::Error::new( @@ -1335,11 +1438,18 @@ where "Backend Job result capability requires Backend Workspace API authority", )); } - feature_registry.add_module( + let mut result_feature = crate::feature::builtin::backend_job_result::BackendJobResultFeature::new( - workspace_client, - ), - ); + workspace_client.clone(), + ); + if binding.subjektiv_consolidation { + let surface = crate::feature::builtin::memory_surface_lifecycle::SubjektivSurfaceLifecycleFeature::for_job( + workspace_client, spawner_manifest.clone(), worker.llm_client_handle(), + prompts.clone(), spawner_workspace_context.clone(), + )?; + result_feature = result_feature.with_surface(surface); + } + feature_registry.add_module(result_feature); } let memory_profile = &worker.manifest().feature.memory.profile; let subjektiv_profile = &worker.manifest().feature.subjektiv.profile; @@ -1352,7 +1462,11 @@ where let mut feature_prompt_contribution = memory_install_plan.as_ref().map(|plan| { ( plan.resident_summary_source.clone(), - plan.system_prompt_override.clone(), + if backend_job.is_none() { + plan.system_prompt_override.clone() + } else { + None + }, ) }); let memory_lifecycle_config = worker.manifest().feature.memory.clone(); @@ -1366,7 +1480,9 @@ where worker.prompts().load_full(), )?; if let Some(plan) = subjektiv_consolidation_plan { - feature_prompt_contribution = Some((None, Some(plan.system_prompt_override))); + if backend_job.is_none() { + feature_prompt_contribution = Some((None, Some(plan.system_prompt_override))); + } feature_registry.add_module(plan.module); } let ordinary_subjektiv_features_enabled = @@ -1375,6 +1491,7 @@ where crate::feature::builtin::memory::ordinary_subjektiv_resident_summary_source( worker.manifest(), worker.workspace_client_handle(), + worker.prompts(), )? { let workspace_id = worker @@ -1384,7 +1501,7 @@ where .to_string(); if let Some(refresh_feature) = crate::feature::builtin::memory::SubjektivResidentRestoreRefreshFeature::for_host( - worker.manifest_lifecycle_features_enabled(), + lifecycle_enabled, Arc::clone(&resident_summary_source), worker.prompts(), workspace_id, @@ -1415,7 +1532,7 @@ where } if let Some(memory_lifecycle) = crate::feature::builtin::memory_lifecycle::MemoryLifecycleFeature::from_resolved_config( - worker.manifest_lifecycle_features_enabled(), + lifecycle_enabled, memory_lifecycle_config, worker.committed_session_capture_handle(), worker.session_extension_handle(), @@ -1429,8 +1546,8 @@ where { feature_registry.add_module(memory_lifecycle); } - if let Some(surface_lifecycle) = crate::feature::builtin::memory_surface_lifecycle::SubjektivSurfaceLifecycleFeature::from_manifest( - worker.manifest_lifecycle_features_enabled(), + if backend_job.is_none() && let Some(surface_lifecycle) = crate::feature::builtin::memory_surface_lifecycle::SubjektivSurfaceLifecycleFeature::from_manifest( + lifecycle_enabled, worker.workspace_client_handle(), spawner_manifest.clone(), worker.llm_client_handle(), @@ -1442,7 +1559,7 @@ where if ordinary_subjektiv_features_enabled && let Some(subjektiv_lifecycle) = crate::feature::builtin::memory_lifecycle::SubjektivLifecycleFeature::from_resolved_config( - worker.manifest_lifecycle_features_enabled(), + lifecycle_enabled, worker.manifest().feature.subjektiv.clone(), worker.committed_session_capture_handle(), worker.session_extension_handle(), @@ -1504,7 +1621,22 @@ where ), ); } - if feature_config.manage_workdir.enabled { + let wip_mode = worker.manifest().worker.mode == manifest::WorkerMode::Wip; + let workspace_config_feature = + crate::feature::builtin::workspace_config::WorkspaceConfigFeature::configured( + worker.workspace_client_handle(), + feature_config.workspace_config.enabled, + wip_mode, + ); + if let Some(module) = &workspace_config_feature { + feature_registry.add_module(module.clone()); + } + let mut workdir_wip_feature = None; + if feature_config.manage_workdir.enabled + || (wip_mode + && feature_config.workdir_catalog.enabled + && worker.workspace_client_handle().is_available()) + { // Workdir lifecycle is Workspace control-plane authority. The Worker // receives only the injected WorkspaceClient and never Runtime URLs, // repository paths, materializer handles, or cleanup sessions. @@ -1520,7 +1652,7 @@ where } let shutdown_registry = spawned_registry.clone(); let reopen_registry = spawned_registry.clone(); - feature_registry.add_module( + let module = crate::feature::builtin::manage_workdir::ManageWorkdirFeature::with_child_lifecycle( workspace_client, worker.workdir_sessions(), @@ -1529,8 +1661,11 @@ where Box::pin(async move { child_registry.shutdown_internal().await }) }), Arc::new(move || reopen_registry.reopen_internal()), - ), - ); + ); + if feature_config.manage_workdir.enabled { + feature_registry.add_module(module.clone()); + } + workdir_wip_feature = Some(module); } if feature_config.workspace_worker_discovery.enabled { let workspace_client = worker.workspace_client_handle(); @@ -1585,20 +1720,60 @@ where } let host_worker_observation_provider = worker.worker_observation_provider(); - let wip_mode = worker.manifest().worker.mode == manifest::WorkerMode::Wip; let wip_permissions = worker.manifest().permissions.clone(); let mut wip_mount_registry = crate::wip::WipMountRegistry::new(); + if let Some(feature) = &workspace_config_feature { + crate::feature::builtin::workspace_config::wip::mount_workspace_config_wip( + &mut wip_mount_registry, + feature, + ) + .map_err(|error| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("mount Workspace config WIP projection: {error}"), + ) + })?; + } + if wip_mode { + crate::checkout::mount_checkouts( + &mut wip_mount_registry, + worker.workdir_sessions(), + tracker.clone(), + wip_permissions.clone(), + ) + .map_err(|error| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("mount checkout WIP projection: {error}"), + ) + })?; + } + if wip_mode && let Some(feature) = &workdir_wip_feature { + crate::feature::builtin::manage_workdir::wip::mount_workspace_workdir_wip( + &mut wip_mount_registry, + feature, + feature_config.workdir_catalog.enabled, + feature_config.manage_workdir.enabled, + wip_permissions.clone(), + ) + .map_err(|error| { + std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("mount Workdir WIP projection: {error}"), + ) + })?; + } { let workspace_client = worker.workspace_client_handle(); let engine = worker.engine_mut(); if feature_config.merge_request.any() && wip_mode { - let feature_route = wip_mount_registry - .allocate_feature_route("merge-request") + let namespace_route = wip_mount_registry + .allocate_namespace("merge-request", "merge-requests") .map_err(|error| { std::io::Error::new( std::io::ErrorKind::InvalidInput, - format!("allocate Merge Request WIP route: {error}"), + format!("allocate Merge Request WIP namespace: {error}"), ) })?; crate::feature::builtin::merge_request::mount_workspace_http_merge_request_wip( @@ -1606,7 +1781,7 @@ where workspace_client.clone(), feature_config.merge_request, wip_permissions.clone(), - &feature_route, + &namespace_route, ) .map_err(|error| { std::io::Error::new( @@ -1623,12 +1798,12 @@ where intake: feature_config.ticket.intake, workflow: feature_config.ticket.workflow, }; - let feature_route = wip_mount_registry - .allocate_feature_route("ticket") + let namespace_route = wip_mount_registry + .allocate_namespace("ticket", "tickets") .map_err(|error| { std::io::Error::new( std::io::ErrorKind::InvalidInput, - format!("allocate Ticket WIP route: {error}"), + format!("allocate Ticket WIP namespace: {error}"), ) })?; crate::feature::builtin::ticket::mount_workspace_http_ticket_wip( @@ -1636,7 +1811,7 @@ where workspace_client.clone(), ticket_access, wip_permissions.clone(), - &feature_route, + &namespace_route, ) .map_err(|error| { std::io::Error::new( @@ -1657,19 +1832,19 @@ where engine.register_tool(definition); } if wip_mode { - let feature_route = wip_mount_registry - .allocate_feature_route("objective") + let namespace_route = wip_mount_registry + .allocate_namespace("objective", "objectives") .map_err(|error| { std::io::Error::new( std::io::ErrorKind::InvalidInput, - format!("allocate Objective WIP route: {error}"), + format!("allocate Objective WIP namespace: {error}"), ) })?; crate::feature::builtin::objective::mount_workspace_http_objective_wip( &mut wip_mount_registry, workspace_client.clone(), wip_permissions.clone(), - &feature_route, + &namespace_route, ) .map_err(|error| { std::io::Error::new( @@ -2081,9 +2256,11 @@ async fn controller_loop( submission.input, vec![extension], submission.provenance, - move || { - let _ = input_commit_tx.send(()); - }, + input_committed_hook( + pending_submissions.clone(), + committed_submission.submission_id.clone(), + input_commit_tx, + ), ), &mut method_rx, &working_event_tx, @@ -2140,10 +2317,6 @@ async fn controller_loop( .await } }; - if notify_buffer.is_empty() { - pending_submissions.finish_notification_batch(); - } - if !shutdown && may_drain_pending && new_status == WorkerStatus::Idle { match prepare_pending_run(&pending_submissions, ¬ify_buffer, None, false) { Ok(Some(next)) => { @@ -2192,12 +2365,16 @@ async fn controller_loop( continue; } + // An expired Notify deadline must not repeatedly schedule a run that + // cannot cross the invocation fence. Keep the accepted notification for + // a fresh input/recovery boundary instead of retrying business effects. let notification_delay = (shared_state.catalog_status() == WorkerStatus::Idle) .then(|| { notification_coalesce_remaining( &pending_submissions, ¬ify_buffer, notification_coalesce_delay, + worker.can_schedule_notification_run(), ) }) .flatten(); @@ -3754,6 +3931,21 @@ mod tests { assert!(feature_install < materialize_gate); assert!(materialize_gate < materialize); assert!(materialize < exposure); + let observer = startup.find("let command_observer = fs_for_view").unwrap(); + let recover = startup.find(".recover_unfinished_compaction()").unwrap(); + let files = startup + .find("runtime_dir.write_status(&shared_state).await?") + .unwrap(); + let loop_start = startup.find("tokio::spawn(controller_loop(").unwrap(); + assert!(recover < observer && files < observer && observer < loop_start); + let socket_startup = &startup[observer..loop_start]; + let abort = socket_startup.find("observer.abort()").unwrap(); + let join = socket_startup.find("observer.await").unwrap(); + let failure = socket_startup.find("return Err(error)").unwrap(); + assert!( + abort < join && join < failure, + "startup cannot detach its observer on error" + ); assert!( startup[materialize_gate..materialize] .contains("needs_initial_session_head_materialization") @@ -3871,6 +4063,40 @@ mod tests { assert!(notify_buffer.has_notification_pending()); } + #[test] + fn invocation_fence_suspends_expired_notification_deadline_without_dropping_input() { + let temp = TempDir::new().unwrap(); + let pending = crate::worker::PendingSubmissionHandle::for_test(temp.path()); + let buffer = NotifyBuffer::new(); + pending + .accept_notification("held".into(), "held".into()) + .unwrap(); + let before = pending.persisted_entries_for_test().len(); + assert_eq!( + notification_coalesce_remaining(&pending, &buffer, Duration::ZERO, true), + Some(Duration::ZERO) + ); + for _ in 0..4 { + assert_eq!( + notification_coalesce_remaining(&pending, &buffer, Duration::ZERO, false), + None + ); + } + assert_eq!(pending.persisted_entries_for_test().len(), before); + assert_eq!(pending.snapshot().notification_count, 1); + // A busy-time staged notification is equally retained behind the fence. + stage_pending_notifications(&pending, &buffer); + assert_eq!( + notification_coalesce_remaining(&pending, &buffer, Duration::ZERO, false), + None + ); + assert!(buffer.has_notification_pending()); + assert_eq!( + notification_coalesce_remaining(&pending, &buffer, Duration::ZERO, true), + Some(Duration::ZERO) + ); + } + #[test] fn restored_mixed_activations_preserve_global_fifo_order() { let temp = TempDir::new().unwrap(); diff --git a/crates/worker/src/feature.rs b/crates/worker/src/feature.rs index 41dfdcc56..3d3ea8983 100644 --- a/crates/worker/src/feature.rs +++ b/crates/worker/src/feature.rs @@ -20,6 +20,7 @@ use agen::Engine; use agen::llm_client::client::LlmClient; use agen::state::Mutable; use agen::tool::ToolDefinition; +use protocol::FeatureInvocationDescriptor; use serde::{Deserialize, Serialize}; use thiserror::Error; @@ -623,6 +624,9 @@ pub struct FeatureDescriptor { pub provides_services: Vec, pub requires_services: Vec, pub protocol_providers: Vec, + /// Declarative, user-selected chat invocations contributed by this Feature. + /// Metadata is public; handlers are registered separately during install. + pub chat_invocations: Vec, } impl FeatureDescriptor { @@ -640,6 +644,7 @@ impl FeatureDescriptor { provides_services: Vec::new(), requires_services: Vec::new(), protocol_providers: Vec::new(), + chat_invocations: Vec::new(), } } @@ -682,6 +687,11 @@ impl FeatureDescriptor { self.protocol_providers.push(provider); self } + + pub fn with_chat_invocation(mut self, invocation: FeatureInvocationDescriptor) -> Self { + self.chat_invocations.push(invocation); + self + } } /// Feature module contribution boundary. @@ -739,6 +749,7 @@ pub enum FeatureContributionKind { BackgroundTask, Service, ProtocolProvider, + ChatInvocation, Notification, Alert, Diagnostic, @@ -765,6 +776,7 @@ pub struct FeatureInstallReport { pub provided_services: Vec, pub resolved_service_requirements: Vec, pub protocol_providers: Vec, + pub installed_chat_invocations: Vec, pub skipped: Vec, pub diagnostics: Vec, } @@ -782,6 +794,7 @@ impl FeatureInstallReport { provided_services: Vec::new(), resolved_service_requirements: Vec::new(), protocol_providers: Vec::new(), + installed_chat_invocations: Vec::new(), skipped: Vec::new(), diagnostics: Vec::new(), } @@ -794,6 +807,7 @@ impl FeatureInstallReport { self.installed_instructions.clear(); self.declared_background_tasks.clear(); self.provided_services.clear(); + self.installed_chat_invocations.clear(); } fn mark_skipped( @@ -819,6 +833,7 @@ struct FeatureContributionDeclarations { provided_services: HashSet<(ServiceId, String)>, required_services: HashSet, protocol_providers: HashSet, + chat_invocations: Vec, } impl FeatureContributionDeclarations { @@ -859,6 +874,7 @@ impl FeatureContributionDeclarations { .iter() .map(|provider| provider.id.clone()) .collect(), + chat_invocations: descriptor.chat_invocations.clone(), } } @@ -1527,6 +1543,7 @@ pub struct FeatureInstallContext<'a> { hook_builder: &'a mut HookRegistryBuilder, background_task_builder: &'a mut FeatureBackgroundTaskRegistryBuilder, service_registry: &'a mut FeatureServiceRegistry, + invocation_registry: &'a mut FeatureInvocationRegistry, report: &'a mut FeatureInstallReport, } @@ -1591,6 +1608,15 @@ impl FeatureInstallContext<'_> { } } + pub fn chat_invocations(&mut self) -> InvocationContributionRegistrar<'_> { + InvocationContributionRegistrar { + feature_id: self.feature_id, + declarations: &self.declarations.chat_invocations, + registry: self.invocation_registry, + installed: &mut self.report.installed_chat_invocations, + } + } + pub fn diagnostics(&mut self) -> FeatureDiagnosticSink<'_> { FeatureDiagnosticSink { report: self.report, @@ -1604,6 +1630,7 @@ pub struct FeatureRegistryInstallReport { pub reports: Vec, pub services: FeatureServiceRegistry, pub background_tasks: FeatureBackgroundTaskRegistry, + pub chat_invocations: FeatureInvocationRegistry, pub plan_error: Option, } @@ -2026,11 +2053,13 @@ impl FeatureRegistryBuilder { reports, services: FeatureServiceRegistry::default(), background_tasks: FeatureBackgroundTaskRegistry::default(), + chat_invocations: FeatureInvocationRegistry::default(), plan_error: Some(error), }; } }; let mut service_registry = FeatureServiceRegistry::default(); + let mut invocation_registry = FeatureInvocationRegistry::default(); let mut background_task_builder = FeatureBackgroundTaskRegistryBuilder::default(); let mut reports = Vec::with_capacity(plan.ordered_indices.len()); let install_hook_checkpoint = hook_builder.checkpoint(); @@ -2055,6 +2084,7 @@ impl FeatureRegistryBuilder { let hook_checkpoint = hook_builder.checkpoint(); let background_checkpoint = background_task_builder.checkpoint(); let service_checkpoint = service_registry.clone(); + let invocation_checkpoint = invocation_registry.clone(); let tool_checkpoint = pending_tools.len(); let installed_tool_checkpoint = installed_tool_names.clone(); @@ -2102,6 +2132,7 @@ impl FeatureRegistryBuilder { hook_builder, background_task_builder: &mut background_task_builder, service_registry: &mut service_registry, + invocation_registry: &mut invocation_registry, report: &mut report, }; module.install(&mut context) @@ -2113,6 +2144,7 @@ impl FeatureRegistryBuilder { hook_builder.rollback_to(hook_checkpoint); background_task_builder.rollback_to(&background_checkpoint); service_registry = service_checkpoint.clone(); + invocation_registry = invocation_checkpoint.clone(); pending_tools.truncate(tool_checkpoint); installed_tool_names = installed_tool_checkpoint.clone(); report.clear_installed_contributions(); @@ -2138,6 +2170,14 @@ impl FeatureRegistryBuilder { ))); } } + for invocation in &descriptor.chat_invocations { + if !report.installed_chat_invocations.contains(invocation) { + report.diagnostics.push(FeatureDiagnostic::error(format!( + "feature `{}` declared chat invocation `{}` but did not register an executable handler", + descriptor.id, invocation.identity + ))); + } + } if report .diagnostics .iter() @@ -2146,6 +2186,7 @@ impl FeatureRegistryBuilder { hook_builder.rollback_to(hook_checkpoint); background_task_builder.rollback_to(&background_checkpoint); service_registry = service_checkpoint.clone(); + invocation_registry = invocation_checkpoint.clone(); pending_tools.truncate(tool_checkpoint); installed_tool_names = installed_tool_checkpoint.clone(); report.clear_installed_contributions(); @@ -2176,6 +2217,7 @@ impl FeatureRegistryBuilder { reports, services: FeatureServiceRegistry::default(), background_tasks: FeatureBackgroundTaskRegistry::default(), + chat_invocations: FeatureInvocationRegistry::default(), plan_error: None, } } else { @@ -2183,6 +2225,7 @@ impl FeatureRegistryBuilder { reports, services: service_registry, background_tasks: background_task_builder.build(), + chat_invocations: invocation_registry, plan_error: None, } } @@ -2225,11 +2268,24 @@ pub enum FeatureInstallError { first_feature: String, duplicate_feature: String, }, + #[error("duplicate invocation identity `{identity}`")] + DuplicateInvocationIdentity { identity: String }, + #[error("duplicate invocation name `{name}` from `{duplicate}`; first registered by `{first}`")] + DuplicateInvocationName { + name: String, + first: String, + duplicate: String, + }, #[error("feature install failed: {0}")] Install(String), } pub mod background; +pub mod invocation; +pub use invocation::{ + FeatureInvocationCompletionProvider, FeatureInvocationContext, FeatureInvocationHandler, + FeatureInvocationHandlerError, FeatureInvocationRegistry, InvocationContributionRegistrar, +}; pub mod builtin; pub mod mcp; pub(crate) mod session; diff --git a/crates/worker/src/feature/background.rs b/crates/worker/src/feature/background.rs index 976734b88..e83c8f744 100644 --- a/crates/worker/src/feature/background.rs +++ b/crates/worker/src/feature/background.rs @@ -190,7 +190,7 @@ impl BackgroundTaskCancellation { } } - fn cancel(&self) { + pub(crate) fn cancel(&self) { self.sender.send_replace(true); } } diff --git a/crates/worker/src/feature/builtin.rs b/crates/worker/src/feature/builtin.rs index b1206d1bc..e4532e63c 100644 --- a/crates/worker/src/feature/builtin.rs +++ b/crates/worker/src/feature/builtin.rs @@ -5,6 +5,7 @@ //! an external plugin-loading surface. pub mod backend_job_result; +pub mod chat_invocation; pub mod flow_transition; pub mod manage_workdir; pub mod manage_worker; @@ -23,6 +24,7 @@ pub(crate) mod subjektiv_session; pub mod task; pub mod ticket; pub mod worker_observation; +pub mod workspace_config; pub mod workspace_worker_discovery; pub use task::{TaskFeature, task_tools_feature}; diff --git a/crates/worker/src/feature/builtin/backend_job_result.rs b/crates/worker/src/feature/builtin/backend_job_result.rs index 93808162a..bf398dcc5 100644 --- a/crates/worker/src/feature/builtin/backend_job_result.rs +++ b/crates/worker/src/feature/builtin/backend_job_result.rs @@ -4,7 +4,9 @@ //! identity come from the authenticated Workspace client, while Job, attempt, //! and input revision are resolved from the Backend-owned binding. -use std::sync::Arc; +use crate::feature::background::BackgroundTaskCancellation; +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; use agen::tool::{Tool, ToolError, ToolExecutionContext, ToolMeta, ToolOutput}; use async_trait::async_trait; @@ -12,6 +14,7 @@ use schemars::JsonSchema; use serde::Deserialize; use serde_json::Value; +use crate::feature::builtin::memory_surface_lifecycle::SubjektivSurfaceLifecycleFeature; use crate::feature::{ FeatureDescriptor, FeatureInstallContext, FeatureInstallError, FeatureModule, ToolContribution, ToolDeclaration, ToolDefinition, @@ -27,9 +30,61 @@ struct SubmitResultInput { result: Value, } +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct ConsolidationResult { + subject_id: String, + candidate_ids: Vec, +} + +#[derive(Debug, Deserialize, JsonSchema)] +#[serde(deny_unknown_fields)] +struct SubmitConsolidationResultInput { + /// Closed candidate ids only. Host awaits and adds the actual surface outcome. + result: ConsolidationResult, +} + +#[derive(Default)] +struct ResultSubmissionState { + sealed: Option, + reusable_surface: Option, +} + +struct SealedResult { + original: Value, + completed: Value, +} + #[derive(Clone)] struct SubmitResultTool { client: Arc, + surface: Option, + submission: Arc>, + active: Arc>>, +} + +struct ActiveResultGuard { + id: String, + active: Arc>>, + cancellation: BackgroundTaskCancellation, +} +impl Drop for ActiveResultGuard { + fn drop(&mut self) { + self.cancellation.cancel(); + self.active + .lock() + .expect("Job result cancellation state poisoned") + .remove(&self.id); + } +} + +fn cancelled_output() -> ToolError { + ToolError::Cancelled(ToolOutput { + summary: "Job result submission cancelled; committed candidate decisions are preserved" + .into(), + content: None, + attachments: Vec::new(), + }) } #[async_trait] @@ -37,27 +92,128 @@ impl Tool for SubmitResultTool { async fn execute( &self, input_json: &str, - _context: ToolExecutionContext, + context: ToolExecutionContext, ) -> Result { let input: SubmitResultInput = serde_json::from_str(input_json) .map_err(|error| ToolError::InvalidArgument(error.to_string()))?; + let id = context.execution_id(); + let cancellation = self + .active + .lock() + .expect("Job result cancellation state poisoned") + .entry(id.clone()) + .or_default() + .clone(); + let _active = ActiveResultGuard { + id, + active: self.active.clone(), + cancellation: cancellation.clone(), + }; + // Serialize result calls and retain the exact lifecycle outcome for ambiguous retries. + let mut submission = tokio::select! { + guard = self.submission.lock() => guard, + _ = cancellation.cancelled() => return Err(cancelled_output()), + }; + // An existing seal may already have had an effect, even if this replay + // is now definitively rejected. It must never be reopened. + let was_previously_sealed = submission.sealed.is_some(); + let result = if let Some(sealed) = submission.sealed.as_ref() { + if sealed.original != input.result { + return Err(ToolError::InvalidArgument( + "Job result is already sealed; retry the identical result".into(), + )); + } + sealed.completed.clone() + } else { + let mut result = input.result.clone(); + if let Some(surface) = &self.surface { + let checked = serde_json::from_str::(input_json) + .map_err(|error| { + ToolError::InvalidArgument(format!( + "invalid consolidation result (surface is Host-generated): {error}" + )) + })? + .result; + if checked.subject_id.trim().is_empty() + || checked.candidate_ids.iter().any(|id| id.trim().is_empty()) + { + return Err(ToolError::InvalidArgument( + "subject_id and candidate ids must not be empty".into(), + )); + } + let object = result + .as_object_mut() + .expect("typed consolidation result is an object"); + let outcome = surface + .complete_for_job(cancellation.clone(), submission.reusable_surface.as_ref()) + .await + .map_err(|error| { + if cancellation.is_cancelled() { + cancelled_output() + } else { + ToolError::ExecutionFailed(error.to_string()) + } + })?; + submission.reusable_surface = Some(outcome.clone()); + object.insert("surface".into(), outcome); + } + result + }; + if cancellation.is_cancelled() { + return Err(cancelled_output()); + } let workspace_id = self.client.workspace_id().ok_or_else(|| { ToolError::ExecutionFailed( "Backend Job result submission requires Workspace identity".to_string(), ) })?; - let response = self - .client - .execute(WorkspaceRequest::json( - WorkspaceRequestMethod::Post, - format!("/api/w/{workspace_id}/workers/self/backend-job-result"), - serde_json::to_string(&serde_json::json!({ "result": input.result })) - .map_err(|error| ToolError::Internal(error.to_string()))?, - )) - .map_err(|error| ToolError::ExecutionFailed(error.to_string()))?; + let request = WorkspaceRequest::json( + WorkspaceRequestMethod::Post, + format!("/api/w/{workspace_id}/workers/self/backend-job-result"), + serde_json::to_string(&serde_json::json!({ "result": result })) + .map_err(|error| ToolError::Internal(error.to_string()))?, + ); + // Seal before dispatch: a transport failure may follow durable acceptance. + if submission.sealed.is_none() { + submission.sealed = Some(SealedResult { + original: input.result, + completed: result, + }); + submission.reusable_surface = None; + } + let response = match self.client.execute(request) { + Ok(response) => response, + Err(error) => { + return Err(ToolError::ExecutionFailed(format!( + "Backend Job result acceptance is unknown; retry the identical result: {error}" + ))); + } + }; + let definitive_rejection = matches!(response.status, 400 | 403 | 409 | 422); if !response.is_success() { + let sealed = submission.sealed.as_mut().expect("sealed before dispatch"); + if definitive_rejection && !was_previously_sealed { + // Only a definitively unaccepted request can be corrected. Reuse + // a ready snapshot after a new preparation confirms its revision; + // a conflict invalidates even that snapshot. Failed outcomes are + // rebuilt, since another generation can replace their marker. + submission.reusable_surface = if response.status != 409 { + sealed + .completed + .get("surface") + .filter(|surface| surface["availability"] == "ready") + .cloned() + } else { + None + }; + submission.sealed = None; + return Err(ToolError::ExecutionFailed(format!( + "Backend Job result was definitively rejected with HTTP {}; correct result fields or retry to rebuild a stale surface: {}", + response.status, response.body + ))); + } return Err(ToolError::ExecutionFailed(format!( - "Backend Job result submission was rejected with HTTP {}: {}", + "Backend Job result acceptance remains sealed; retry the identical result (HTTP {}): {}", response.status, response.body ))); } @@ -67,11 +223,30 @@ impl Tool for SubmitResultTool { attachments: Vec::new(), }) } + + async fn cancel_execution(&self, context: &ToolExecutionContext) -> Result<(), ToolError> { + self.active + .lock() + .expect("Job result cancellation state poisoned") + .entry(context.execution_id()) + .or_default() + .cancel(); + Ok(()) + } } -fn definition(client: Arc) -> ToolDefinition { - let schema = serde_json::to_value(schemars::schema_for!(SubmitResultInput)) - .expect("Backend Job result schema must serialize"); +fn definition( + client: Arc, + surface: Option, +) -> ToolDefinition { + let submission = Arc::new(tokio::sync::Mutex::new(ResultSubmissionState::default())); + let active = Arc::new(Mutex::new(HashMap::new())); + let schema = if surface.is_some() { + serde_json::to_value(schemars::schema_for!(SubmitConsolidationResultInput)) + } else { + serde_json::to_value(schemars::schema_for!(SubmitResultInput)) + } + .expect("Backend Job result schema must serialize"); Arc::new(move || { ( ToolMeta::new(TOOL_NAME) @@ -79,6 +254,9 @@ fn definition(client: Arc) -> ToolDefinition { .input_schema(schema.clone()), Arc::new(SubmitResultTool { client: client.clone(), + surface: surface.clone(), + submission: submission.clone(), + active: active.clone(), }) as Arc, ) }) @@ -87,11 +265,22 @@ fn definition(client: Arc) -> ToolDefinition { #[derive(Clone)] pub struct BackendJobResultFeature { client: Arc, + surface: Option, } impl BackendJobResultFeature { pub fn new(client: Arc) -> Self { - Self { client } + Self { + client, + surface: None, + } + } +} + +impl BackendJobResultFeature { + pub(crate) fn with_surface(mut self, surface: SubjektivSurfaceLifecycleFeature) -> Self { + self.surface = Some(surface); + self } } @@ -105,7 +294,7 @@ impl FeatureModule for BackendJobResultFeature { fn install(&self, context: &mut FeatureInstallContext<'_>) -> Result<(), FeatureInstallError> { context.tools().register(ToolContribution::new( TOOL_NAME, - definition(self.client.clone()), + definition(self.client.clone(), self.surface.clone()), )) } } @@ -172,4 +361,14 @@ mod tests { .ends_with("/workers/self/backend-job-result") ); } + #[tokio::test] + async fn cancellation_before_execution_never_submits_a_result() { + let client = Arc::new(RecordingClient::default()); + let (_, tool) = definition(client.clone(), None)(); + let context = ToolExecutionContext::new("result-call", "batch", 0); + tool.cancel_execution(&context).await.unwrap(); + let outcome = tool.execute(r#"{"result":{"valid":true}}"#, context).await; + assert!(matches!(outcome, Err(ToolError::Cancelled(_)))); + assert!(client.requests.lock().unwrap().is_empty()); + } } diff --git a/crates/worker/src/feature/builtin/chat_invocation.rs b/crates/worker/src/feature/builtin/chat_invocation.rs new file mode 100644 index 000000000..576ce09c9 --- /dev/null +++ b/crates/worker/src/feature/builtin/chat_invocation.rs @@ -0,0 +1,72 @@ +//! Builtin metadata for client-local attachment staging. +//! +//! `/attach` uses the common invocation declaration and parser, but its path is +//! intentionally consumed by the client adapter. The Worker accepts only the +//! resulting immutable `UploadedFile` segment and never resolves a client path. + +use async_trait::async_trait; +use protocol::{ + FeatureInvocation, FeatureInvocationDescriptor, FeatureInvocationIdentity, + FeatureInvocationResult, FeatureInvocationSyntax, InvocationArgumentDescriptor, + InvocationArgumentType, InvocationClientAdapter, InvocationCompletion, +}; + +use crate::feature::{ + FeatureDescriptor, FeatureInstallContext, FeatureInstallError, FeatureInvocationContext, + FeatureInvocationHandler, FeatureInvocationHandlerError, FeatureModule, +}; + +pub const ATTACH_FEATURE_ID: &str = "attachments"; +pub const ATTACH_INVOCATION_ID: &str = "builtin:attachments/attach"; + +pub fn attach_invocation_descriptor() -> FeatureInvocationDescriptor { + FeatureInvocationDescriptor { + identity: FeatureInvocationIdentity(ATTACH_INVOCATION_ID.into()), + name: "attach".into(), + aliases: Vec::new(), + display_name: "Attach file".into(), + description: "Stage a client-local file as an editable attachment chip.".into(), + syntax: FeatureInvocationSyntax::Parenthesized, + arguments: vec![InvocationArgumentDescriptor { + name: "file".into(), + position: Some(0), + required: true, + value_type: InvocationArgumentType::ClientFile, + completion: InvocationCompletion::ClientFile, + description: Some("File available to this client".into()), + }], + client_adapter: Some(InvocationClientAdapter::Attachment), + } +} + +pub struct AttachmentInvocationFeature; + +impl FeatureModule for AttachmentInvocationFeature { + fn descriptor(&self) -> FeatureDescriptor { + FeatureDescriptor::builtin(ATTACH_FEATURE_ID, "Attachments") + .with_description("Client-local attachment staging and typed uploaded-file input") + .with_chat_invocation(attach_invocation_descriptor()) + } + + fn install(&self, context: &mut FeatureInstallContext<'_>) -> Result<(), FeatureInstallError> { + context.chat_invocations().register( + attach_invocation_descriptor(), + AttachmentMustBeStagedByClient, + ) + } +} + +struct AttachmentMustBeStagedByClient; + +#[async_trait] +impl FeatureInvocationHandler for AttachmentMustBeStagedByClient { + async fn invoke( + &self, + _context: FeatureInvocationContext, + _invocation: &FeatureInvocation, + ) -> Result { + Err(FeatureInvocationHandlerError::failed( + "attachment invocation reached the Worker without client-local upload staging", + )) + } +} diff --git a/crates/worker/src/feature/builtin/manage_workdir.rs b/crates/worker/src/feature/builtin/manage_workdir.rs index 54514dd75..28867c0ca 100644 --- a/crates/worker/src/feature/builtin/manage_workdir.rs +++ b/crates/worker/src/feature/builtin/manage_workdir.rs @@ -5,6 +5,8 @@ //! endpoints, credentials, materializer handles, and operation sessions stay //! behind [`WorkspaceClient`]. +pub mod wip; + use std::future::Future; use std::pin::Pin; use std::sync::Arc; @@ -67,6 +69,7 @@ pub struct ManageWorkdirFeature { session_router: Arc, before_workdir_release: Option, after_workdir_attach: Option, + attachment_mutation_lock: Arc>, } impl std::fmt::Debug for ManageWorkdirFeature { @@ -86,6 +89,7 @@ impl ManageWorkdirFeature { session_router: Arc::new(workdir::WorkdirSessionRouter::new()), before_workdir_release: None, after_workdir_attach: None, + attachment_mutation_lock: Arc::new(tokio::sync::Mutex::new(())), } } @@ -100,8 +104,19 @@ impl ManageWorkdirFeature { session_router, before_workdir_release: Some(before_workdir_release), after_workdir_attach: Some(after_workdir_attach), + attachment_mutation_lock: Arc::new(tokio::sync::Mutex::new(())), } } + fn backend(&self) -> WorkspaceHttpWorkdirBackend { + let mut backend = WorkspaceHttpWorkdirBackend::new(self.client.clone()) + .with_session_router(self.session_router.clone()) + .with_child_lifecycle( + self.before_workdir_release.clone(), + self.after_workdir_attach.clone(), + ); + backend.attachment_mutation_lock = self.attachment_mutation_lock.clone(); + backend + } } pub fn manage_workdir_feature(client: Arc) -> ManageWorkdirFeature { @@ -120,12 +135,7 @@ impl FeatureModule for ManageWorkdirFeature { } fn install(&self, context: &mut FeatureInstallContext<'_>) -> Result<(), FeatureInstallError> { - let backend = WorkspaceHttpWorkdirBackend::new(self.client.clone()) - .with_session_router(self.session_router.clone()) - .with_child_lifecycle( - self.before_workdir_release.clone(), - self.after_workdir_attach.clone(), - ); + let backend = self.backend(); for (name, definition) in [ ( LIST_TOOL, @@ -343,6 +353,102 @@ impl WorkdirSession for WorkspaceAttachedWorkdirSession { } } + async fn checkout_observe( + &self, + path: workdir::WorkdirPath, + ) -> Result { + match self.operate(WorkdirSessionOperation::CheckoutObserve(path.clone()))? { + WorkdirSessionOperationResult::CheckoutObserve(mut result) + if result.path == path + && !result.validator.is_empty() + && result.validator.len() <= 256 => + { + result.capabilities = result.capabilities.intersection(self.capabilities); + Ok(result) + } + _ => Err(Self::mismatch("checkout_observe")), + } + } + + async fn checkout_execute( + &self, + request: workdir::CheckoutRequest, + ) -> Result { + let cap = request.operation.capability(); + if !self.capabilities.supports(cap) { + return Err(WorkdirError::Unsupported(cap)); + } + let target = request.target.clone(); + let mutation = cap != workdir::WorkdirSessionCapability::Read; + match self.operate(WorkdirSessionOperation::CheckoutExecute(request)) { + Ok(WorkdirSessionOperationResult::CheckoutExecute(mut result)) + if result.observation.path == target + && !result.observation.validator.is_empty() + && result.observation.validator.len() <= 256 + && matches!( + (&result.output, cap), + ( + workdir::CheckoutOutput::Read(_), + workdir::WorkdirSessionCapability::Read + ) | ( + workdir::CheckoutOutput::Write(_), + workdir::WorkdirSessionCapability::Write + ) | ( + workdir::CheckoutOutput::Edit(_), + workdir::WorkdirSessionCapability::Edit + ) + ) => + { + result.observation.capabilities = result + .observation + .capabilities + .intersection(self.capabilities); + Ok(result) + } + Ok(_) if mutation => Err(WorkdirError::OutcomeUnknown( + "mismatched checkout mutation response".into(), + )), + Ok(_) => Err(Self::mismatch("checkout_execute")), + Err(WorkdirError::Transport(_) | WorkdirError::Unavailable(_)) if mutation => { + Err(WorkdirError::OutcomeUnknown( + "checkout mutation response lost; inspect effects before retry".into(), + )) + } + Err(error) => Err(error), + } + } + + async fn checkout_search( + &self, + request: workdir::CheckoutSearchRequest, + ) -> Result { + use workdir::{ + CheckoutSearchOperation as Op, CheckoutSearchResult as SearchResult, + WorkdirSessionCapability as Cap, + }; + let cap = match &request.operation { + Op::List(_) => Cap::Read, + Op::Glob(_) => Cap::Glob, + Op::Grep(_) => Cap::Grep, + }; + if !self.capabilities.supports(cap) { + return Err(WorkdirError::Unsupported(cap)); + } + match self.operate(WorkdirSessionOperation::CheckoutSearch(request))? { + WorkdirSessionOperationResult::CheckoutSearch(result) + if matches!( + (&result, cap), + (SearchResult::List(_), Cap::Read) + | (SearchResult::Glob(_), Cap::Glob) + | (SearchResult::Grep(_), Cap::Grep) + ) => + { + Ok(result) + } + _ => Err(Self::mismatch("checkout_search")), + } + } + async fn stat(&self, request: StatRequest) -> Result { match self.operate(WorkdirSessionOperation::Stat(request))? { WorkdirSessionOperationResult::Stat(result) => Ok(result), @@ -581,11 +687,24 @@ impl WorkspaceHttpWorkdirBackend { } fn detach(&self, alias: &str) -> Result { + self.detach_expected(alias, None) + } + + fn detach_expected( + &self, + alias: &str, + expected_connection_id: Option<&str>, + ) -> Result { let workspace_id = encode_path_segment(self.workspace_id()?); let alias_path = encode_path_segment(alias); + let suffix = expected_connection_id + .map(|id| format!("?expected_connection_id={}", encode_path_segment(id))) + .unwrap_or_default(); let response = self.execute_json::(WorkspaceRequest { method: WorkspaceRequestMethod::Delete, - path: format!("/api/w/{workspace_id}/workers/self/workdir-attachments/{alias_path}"), + path: format!( + "/api/w/{workspace_id}/workers/self/workdir-attachments/{alias_path}{suffix}" + ), body: None, })?; workdir_output( @@ -622,6 +741,72 @@ impl WorkspaceHttpWorkdirBackend { ) } + async fn detach_managed( + &self, + alias: &str, + expected_connection_id: Option<&str>, + ) -> Result { + let _mutation_guard = self.attachment_mutation_lock.lock().await; + let alias = validate_identity(alias, DETACH_TOOL, "alias")?; + if let Some(expected) = expected_connection_id { + // Under the shared local mutation lock, check lifetime before any + // local route exclusion or child cleanup. Backend DELETE still performs + // the authoritative conditional mutation under its lifecycle lock. + let workspace = encode_path_segment(self.workspace_id()?); + let page: serde_json::Value = self.execute_json(WorkspaceRequest::get(format!( + "/api/w/{workspace}/workers/self/workdir-attachments?limit=1&connection_id={}", + encode_path_segment(expected) + )))?; + let current = page + .get("items") + .and_then(serde_json::Value::as_array) + .is_some_and(|items| { + items.iter().any(|item| { + item.get("connection_id") + .and_then(serde_json::Value::as_str) + == Some(expected) + && item.get("alias").and_then(serde_json::Value::as_str) == Some(alias) + }) + }); + if !current { + return Err(ToolError::StructuredConflict { + code: "workspace_http_409".into(), + message: "attachment lifetime changed before local cleanup".into(), + }); + } + } + let alias_key = workdir::WorkdirAttachmentAlias::new(alias) + .map_err(|error| ToolError::InvalidArgument(error.to_string()))?; + self.session_router + .begin_detach(&alias_key) + .await + .map_err(|error| ToolError::ExecutionFailed(error.to_string()))?; + if let Some(before_release) = &self.before_workdir_release + && let Err(error) = before_release().await + { + self.session_router.cancel_detach(&alias_key); + return Err(ToolError::ExecutionFailed(format!( + "stop Internal SubWorkers before Workdir detach: {error}" + ))); + } + let result = match self.detach_expected(alias, expected_connection_id) { + Ok(result) => result, + Err(error) => { + self.session_router.cancel_detach(&alias_key); + return Err(error); + } + }; + self.session_router + .finish_detach(&alias_key) + .await + .map_err(|error| { + ToolError::ExecutionFailed(format!( + "detach Workdir session route after Backend release: {error}" + )) + })?; + Ok(result) + } + fn execute_json Deserialize<'de>>( &self, request: WorkspaceRequest, @@ -697,42 +882,10 @@ impl Tool for WorkspaceHttpWorkdirTool { result } WorkdirOperation::Detach => { - let _mutation_guard = self.backend.attachment_mutation_lock.lock().await; let input = parse_input::(input_json)?; - let alias = validate_identity(&input.alias, DETACH_TOOL, "alias")?; - let alias_key = workdir::WorkdirAttachmentAlias::new(alias) - .map_err(|error| ToolError::InvalidArgument(error.to_string()))?; - self.backend - .session_router - .begin_detach(&alias_key) - .await - .map_err(|error| ToolError::ExecutionFailed(error.to_string()))?; - if let Some(before_release) = &self.backend.before_workdir_release - && let Err(error) = before_release().await - { - self.backend.session_router.cancel_detach(&alias_key); - return Err(ToolError::ExecutionFailed(format!( - "stop Internal SubWorkers before Workdir detach: {error}" - ))); - } - let result = match self.backend.detach(alias) { - Ok(result) => result, - Err(error) => { - self.backend.session_router.cancel_detach(&alias_key); - return Err(error); - } - }; - self.backend - .session_router - .finish_detach(&alias_key) - .await - .map_err(|error| { - ToolError::ExecutionFailed(format!( - "detach Workdir session route after Backend release: {error}" - )) - })?; - Ok(result) + self.backend.detach_managed(&input.alias, None).await } + WorkdirOperation::Delete => self .backend .delete(parse_input::(input_json)?), @@ -747,6 +900,12 @@ fn parse_input Deserialize<'de>>(input: &str) -> Result Deserialize<'de>>( response: WorkspaceResponse, ) -> Result { + if matches!(response.status, 401 | 403 | 404 | 409) { + return Err(ToolError::StructuredConflict { + code: format!("workspace_http_{}", response.status), + message: "Workspace lifecycle authority rejected the request before execution".into(), + }); + } if !response.is_success() { return Err(ToolError::ExecutionFailed(format!( "Workspace Workdir API returned HTTP {}: {}", @@ -1351,6 +1510,56 @@ mod tests { assert!(body.get("session_id").is_none()); } + #[tokio::test] + async fn attached_checkout_mutations_preserve_known_rejection_and_lost_outcomes() { + let client = Arc::new(RecordingWorkspaceClient::new(vec![ + response(json!({"operation":"write","result":{"bytes_written":3,"created":false}})), + WorkspaceResponse { + status: 200, + body: "invalid payload".into(), + }, + error_response( + 409, + workdir::http::WorkdirTransportErrorCode::Conflict, + "stale", + ), + ])); + let session = WorkspaceAttachedWorkdirSession::handle(client.clone(), "checkout"); + let request = workdir::CheckoutRequest { + target: workdir::WorkdirPath::new("file.txt").unwrap(), + validator: vec![1], + operation: workdir::CheckoutOperation::Write { + content: b"new".to_vec(), + expected_hash: [0; 32], + }, + }; + for _ in 0..2 { + assert!(matches!( + session.checkout_execute(request.clone()).await, + Err(WorkdirError::OutcomeUnknown(_)) + )); + } + assert!(matches!( + session.checkout_execute(request.clone()).await, + Err(WorkdirError::Conflict(_)) + )); + // Missing response simulates the Workspace client losing the provider result. + assert!(matches!( + session.checkout_execute(request).await, + Err(WorkdirError::OutcomeUnknown(_)) + )); + let calls = client.requests(); + assert_eq!(calls.len(), 4); + for call in calls { + let body: serde_json::Value = + serde_json::from_str(call.body.as_deref().unwrap()).unwrap(); + assert_eq!(body["target_workdir"], "checkout"); + assert_eq!(body["operation"]["operation"], "checkout_execute"); + assert_eq!(body["operation"]["request"]["target"], "file.txt"); + assert!(body.get("runtime_id").is_none()); + } + } + #[tokio::test] async fn attached_session_does_not_send_worker_local_bash_spill_path() { let client = Arc::new(RecordingWorkspaceClient::new(vec![response(json!({ diff --git a/crates/worker/src/feature/builtin/manage_workdir/wip.rs b/crates/worker/src/feature/builtin/manage_workdir/wip.rs new file mode 100644 index 000000000..149a4b684 --- /dev/null +++ b/crates/worker/src/feature/builtin/manage_workdir/wip.rs @@ -0,0 +1,1822 @@ +//! Host-owned Repository, Workdir and Worker attachment Objects. +//! Inventory is resolved from Workspace authority on demand, not a second catalog. + +use std::collections::BTreeMap; +use std::sync::Arc; + +use async_trait::async_trait; +use manifest::{ToolPermissionAction, ToolPermissionConfig}; +use serde_json::{Value as Json, json}; +use sha2::{Digest, Sha256}; +use wip_protocol::{ + Documentation, INTERFACE_FORMAT_V1, InterfaceDescriptor, Object, OperationDeclaration, + ParameterDeclaration, ProtocolError, ProtocolErrorCode, ReturnDeclaration, TypeExpr, Value, +}; + +use super::*; +use crate::permission::permission_action_for; +use crate::wip::{ + WipCallContext, WipDynamicItem, WipDynamicItemResolver, WipDynamicMount, + WipDynamicOperationContribution, WipDynamicOperationResolver, WipMountError, WipMountRegistry, + WipOperationContribution, WipOperationError, WipOperationHandler, WipOperationOutput, + WipProjection, WipProjectionKind, json_to_wip, wip_to_json, +}; + +const REPOSITORIES: &str = "/repositories"; +const WORKDIRS: &str = "/workdirs"; +const ATTACHMENTS: &str = "/workdir-attachments"; + +#[derive(Clone, Copy, Debug)] +enum Domain { + Repository, + Workdir, + Attachment, +} + +impl Domain { + fn route(self) -> &'static str { + match self { + Self::Repository => REPOSITORIES, + Self::Workdir => WORKDIRS, + Self::Attachment => ATTACHMENTS, + } + } + fn owner(self) -> &'static str { + match self { + Self::Repository => "repository", + Self::Workdir => "workdir", + Self::Attachment => "workdir-attachment", + } + } + fn list_permission(self) -> &'static str { + match self { + Self::Repository => "RepositoryList", + Self::Workdir => LIST_TOOL, + Self::Attachment => "WorkdirAttachmentList", + } + } + fn read_permission(self) -> &'static str { + match self { + Self::Repository => "RepositoryRead", + Self::Workdir => "WorkdirRead", + Self::Attachment => "WorkdirAttachmentRead", + } + } + fn identity_field(self) -> &'static str { + match self { + Self::Repository => "repository_key", + Self::Workdir => "working_directory_id", + Self::Attachment => "connection_id", + } + } +} + +#[derive(Clone)] +struct Provider { + backend: WorkspaceHttpWorkdirBackend, + permissions: Option, + catalog: bool, + manage: bool, + config_content: bool, +} + +/// Register Object ownership once, then contribute independent management operations. +/// The passed Feature shares its session router, mutation lock and child lifecycle with Tools. +pub fn mount_workspace_workdir_wip( + registry: &mut WipMountRegistry, + feature: &ManageWorkdirFeature, + catalog: bool, + manage: bool, + permissions: Option, +) -> Result<(), WipMountError> { + let provider = Provider { + backend: feature.backend(), + permissions, + catalog, + manage, + config_content: registry + .routes() + .any(|route| route == crate::feature::builtin::workspace_config::CONTENT_ROOT), + }; + for domain in [Domain::Repository, Domain::Workdir, Domain::Attachment] { + let namespace = + registry.allocate_namespace(domain.owner(), domain.route().trim_start_matches('/'))?; + let interface = format!("yoi.{}/collection/v1", domain.owner()); + let handler = Arc::new(CollectionHandler { + provider: provider.clone(), + domain, + }); + registry.mount(WipProjection { + route: namespace.root().into(), + capability: format!("{}:collection", domain.owner()), + kind: WipProjectionKind::Native, + object: object( + domain.route(), + domain.route(), + &interface, + &json!({"kind": "collection"}), + ), + interface: interface.clone(), + descriptor: collection_descriptor(domain, false), + interface_validator: None, + handler: handler.clone(), + })?; + if manage && matches!(domain, Domain::Workdir) { + registry.contribute_operations(WipOperationContribution { + route: domain.route().into(), + contributor: FEATURE_ID.into(), + interface, + descriptor: collection_descriptor(domain, true), + handler, + })?; + } + let item_interface = format!("yoi.{}/item/v1", domain.owner()); + let resolver = Arc::new(ItemResolver { + provider: provider.clone(), + domain, + }); + registry.mount_dynamic(WipDynamicMount { + collection_route: domain.route().into(), + capability: format!("{}:item", domain.owner()), + interface: item_interface.clone(), + descriptor: item_descriptor(domain, false), + interface_validator: None, + resolver: resolver.clone(), + })?; + if manage && !matches!(domain, Domain::Repository) { + registry.contribute_dynamic_operations(WipDynamicOperationContribution { + collection_route: domain.route().into(), + contributor: FEATURE_ID.into(), + interface: item_interface, + descriptor: item_descriptor(domain, true), + resolver, + })?; + } + } + registry.replace_compatibility_tools( + WORKDIRS, + [LIST_TOOL, CREATE_TOOL, ATTACH_TOOL, DELETE_TOOL], + )?; + registry.replace_compatibility_tools(ATTACHMENTS, [DETACH_TOOL])?; + Ok(()) +} + +impl Provider { + fn allowed(&self, name: &str, input: &Json) -> bool { + self.permissions + .as_ref() + .is_none_or(|p| permission_action_for(p, name, input) == ToolPermissionAction::Allow) + } + // An argument-dependent permission cannot be decided until call. Advertise only + // if the policy can allow this tool, then recheck the exact route-bound input. + fn may_allow(&self, name: &str) -> bool { + let Some(p) = &self.permissions else { + return true; + }; + for rule in p.rules.iter().filter(|r| r.tool.eq_ignore_ascii_case(name)) { + if rule.action == ToolPermissionAction::Allow { + return true; + } + if rule.pattern == "*" { + return false; + } + } + p.default_action == ToolPermissionAction::Allow + } + fn authorize(&self, name: &str, input: &Json) -> Result<(), WipOperationError> { + if self.allowed(name, input) { + Ok(()) + } else { + Err(failure( + ProtocolErrorCode::PermissionDenied, + format!("permission denied for `{name}`; approval-required policies fail closed"), + )) + } + } + fn request(&self, path: &str) -> Result { + let workspace = self.backend.workspace_id().map_err(map_tool_error)?; + let response = self + .backend + .client + .execute(WorkspaceRequest::get(format!( + "/api/w/{}/{path}", + encode_path_segment(workspace) + ))) + .map_err(|_| { + WipOperationError::OutcomeUnknown("Workspace inventory request unavailable".into()) + })?; + if !response.is_success() { + return Err(match response.status { + 401 | 403 => failure( + ProtocolErrorCode::PermissionDenied, + "Workspace authority denied this inventory request", + ), + 404 => failure( + ProtocolErrorCode::NotFound, + "inventory Object no longer exists", + ), + 409 => failure( + ProtocolErrorCode::ValidatorMismatch, + "inventory observation is stale; rediscover", + ), + _ => WipOperationError::OutcomeUnknown("Workspace inventory request failed".into()), + }); + } + serde_json::from_str(&response.body).map_err(|_| { + WipOperationError::OutcomeUnknown("invalid Workspace inventory response".into()) + }) + } + fn inventory(&self, domain: Domain) -> Result, WipOperationError> { + let response = self.request(match domain { + Domain::Repository => "repositories", + Domain::Workdir => "workers/self/workdir-catalog?limit=1", + Domain::Attachment => "workers/self/workdir-attachments?limit=1", + })?; + response + .get("items") + .and_then(Json::as_array) + .cloned() + .ok_or_else(|| { + WipOperationError::OutcomeUnknown("invalid Workspace inventory collection".into()) + }) + } + fn raw_item(&self, domain: Domain, id: &str) -> Result { + match domain { + Domain::Attachment => self + .request(&format!( + "workers/self/workdir-attachments?limit=1&connection_id={}", + encode_path_segment(id) + ))? + .get("items") + .and_then(Json::as_array) + .cloned() + .ok_or_else(|| { + WipOperationError::OutcomeUnknown("invalid attachment lookup response".into()) + })? + .into_iter() + .find(|v| v.get("connection_id").and_then(Json::as_str) == Some(id)) + .ok_or_else(|| { + failure( + ProtocolErrorCode::NotFound, + "attachment lifetime no longer exists; list current attachments", + ) + }), + Domain::Repository | Domain::Workdir => { + let prefix = if matches!(domain, Domain::Repository) { + "repositories" + } else { + "working-directories" + }; + self.request(&format!("{prefix}/{}", encode_path_segment(id)))? + .get("item") + .cloned() + .ok_or_else(|| { + WipOperationError::OutcomeUnknown("invalid Workspace inventory item".into()) + }) + } + } + } + fn read_input(&self, domain: Domain, id: &str) -> Json { + json!({domain.identity_field(): id}) + } + fn projected(&self, domain: Domain, raw: &Json) -> Result { + let id = raw + .get(domain.identity_field()) + .and_then(Json::as_str) + .ok_or_else(|| { + WipOperationError::OutcomeUnknown("inventory identity missing".into()) + })?; + let mut result = serde_json::Map::new(); + let fields: &[&str] = match domain { + Domain::Repository => &[ + "repository_key", + "kind", + "provider", + "display_name", + "description", + "default_selector", + ], + Domain::Workdir => &[ + "working_directory_id", + "display_name", + "creation_selector", + "creation_ref", + "creation_tree", + "current_selector", + "current_ref", + "current_tree", + "status", + "cleanliness", + "occupied_by", + ], + Domain::Attachment => &[ + "connection_id", + "alias", + "working_directory_id", + "capabilities", + ], + }; + for field in fields { + if let Some(value) = raw.get(*field) { + result.insert((*field).into(), value.clone()); + } + } + if matches!(domain, Domain::Repository) { + // A missing default is explicit. No guessed main/ref, source URI or credentials. + result.entry("default_selector").or_insert(Json::Null); + } + if matches!(domain, Domain::Workdir) { + if let Some(source) = raw.get("source") { + let kind = source + .get("kind") + .and_then(Json::as_str) + .unwrap_or("unknown"); + let mut projected = json!({"kind": kind}); + if kind == "repository" + && let Some(key) = source.get("repository_key") + { + projected["repository_key"] = key.clone(); + projected["repository_path"] = json!(item_path( + Domain::Repository, + key.as_str().unwrap_or_default() + )); + } + if kind == "workspace_config" + && source.get("content_path").and_then(Json::as_str) + == Some(crate::feature::builtin::workspace_config::CONTENT_ROOT) + && matches!( + source.get("access").and_then(Json::as_str), + Some("read_only" | "read_write") + ) + { + for field in ["access", "content_path", "purpose"] { + if let Some(value) = source.get(field) { + projected[field] = value.clone(); + } + } + } + result.insert("source".into(), projected); + } + } + if matches!(domain, Domain::Attachment) { + if let Some(id) = raw.get("working_directory_id").and_then(Json::as_str) { + result.insert("workdir_path".into(), json!(item_path(Domain::Workdir, id))); + } + // Content references are resolved from current Backend authority, + // never inferred from the alias or an OS session. An old lifetime + // cannot refer to a replacement connection at the same root. + if self.config_content + && raw.get("alias").and_then(Json::as_str) + == Some(crate::feature::builtin::workspace_config::ATTACHMENT_ALIAS) + && let Ok(Some(current)) = + crate::feature::builtin::workspace_config::backend::WorkspaceConfigBackend::new( + self.backend.client.clone(), + ) + .current() + && current.connection_id == id + && raw.get("working_directory_id").and_then(Json::as_str) + == Some(current.working_directory_id.as_str()) + && raw.get("alias").and_then(Json::as_str) == Some(current.alias.as_str()) + { + result.insert("content_path".into(), json!(current.content_path)); + result.insert("name".into(), json!(current.name)); + result.insert("purpose".into(), json!(current.purpose)); + result.insert("access".into(), json!(current.access)); + } + if !result.contains_key("content_path") + && let Some(alias) = raw.get("alias").and_then(Json::as_str) + && let Ok(selected) = self.backend.session_router.resolve(Some(alias)) + && raw.get("working_directory_id").and_then(Json::as_str) + == Some(selected.session.workdir().id().as_str()) + && selected + .session + .capabilities() + .supports(workdir::WorkdirSessionCapability::Read) + { + result.insert( + "checkout_path".into(), + json!(crate::checkout::checkout_root(alias)), + ); + result.insert("checkout_slug".into(), json!(encode_identity(alias))); + } + } + result.insert("path".into(), json!(item_path(domain, id))); + Ok(Json::Object(result)) + } +} + +struct CollectionHandler { + provider: Provider, + domain: Domain, +} +#[async_trait] +impl WipOperationHandler for CollectionHandler { + fn contextual_interface(&self) -> bool { + true + } + fn object_validator(&self) -> Option> { + let mut digest = Sha256::new(); + digest.update(self.domain.route()); + if matches!(self.domain, Domain::Workdir | Domain::Attachment) { + // The complete-set revision is computed by the existing Backend + // registry/ledger in the same snapshot as a bounded page. Never infer + // collection freshness from just the first page's items. + let path = if matches!(self.domain, Domain::Workdir) { + "workers/self/workdir-catalog?limit=1" + } else { + "workers/self/workdir-attachments?limit=1" + }; + match self + .provider + .request(path) + .ok() + .and_then(|p| p.get("revision").and_then(Json::as_str).map(str::to_owned)) + { + Some(revision) => { + digest.update([0]); + digest.update(revision); + } + None => digest.update(b"inventory-revision-unavailable"), + } + return Some(digest.finalize().to_vec()); + } + match self.provider.inventory(self.domain) { + Ok(items) => { + for raw in items { + if let Some(id) = raw.get(self.domain.identity_field()).and_then(Json::as_str) + && self.provider.allowed( + self.domain.read_permission(), + &self.provider.read_input(self.domain, id), + ) + && let Ok(value) = self.provider.projected(self.domain, &raw) + { + digest.update(value.to_string()); + digest.update([0]); + } + } + } + Err(_) => digest.update(b"inventory-unavailable"), + } + Some(digest.finalize().to_vec()) + } + fn is_visible(&self) -> bool { + (self.provider.catalog + && (self.provider.may_allow(self.domain.list_permission()) + || self.provider.may_allow(self.domain.read_permission()))) + || (self.provider.manage + && matches!(self.domain, Domain::Workdir) + && self.provider.may_allow(CREATE_TOOL)) + } + fn operation_available(&self, operation: &str) -> bool { + match operation { + "list" => { + self.provider.catalog && self.provider.may_allow(self.domain.list_permission()) + } + "create" => { + self.provider.manage + && matches!(self.domain, Domain::Workdir) + && self.provider.may_allow(CREATE_TOOL) + } + _ => false, + } + } + async fn call( + &self, + operation: &str, + arguments: &BTreeMap, + context: WipCallContext, + ) -> Result { + let input = argument_json(arguments)?; + if !self.operation_available(operation) { + return Err(failure( + ProtocolErrorCode::PermissionDenied, + "operation is not available", + )); + } + let value = match operation { + "list" => { + self.provider + .authorize(self.domain.list_permission(), &input)?; + let limit = page_limit(&input)?; + let cursor = input.get("cursor").and_then(Json::as_str); + if matches!(self.domain, Domain::Workdir | Domain::Attachment) { + let attachment = matches!(self.domain, Domain::Attachment); + let path = if attachment { + let offset = cursor + .map(|c| { + c.strip_prefix("offset:") + .and_then(|s| s.parse::().ok()) + .ok_or_else(|| { + failure( + ProtocolErrorCode::InvalidArguments, + "invalid attachment page cursor", + ) + }) + }) + .transpose()? + .unwrap_or(0); + format!("workers/self/workdir-attachments?limit={limit}&offset={offset}") + } else { + let suffix = cursor + .map(|c| format!("&cursor={}", encode_path_segment(c))) + .unwrap_or_default(); + format!("workers/self/workdir-catalog?limit={limit}{suffix}") + }; + let page = self.provider.request(&path)?; + let raws = page + .get("items") + .and_then(Json::as_array) + .filter(|v| v.len() <= limit) + .ok_or_else(|| { + WipOperationError::OutcomeUnknown( + "invalid bounded inventory page".into(), + ) + })?; + if page.get("revision").and_then(Json::as_str).is_none() { + return Err(WipOperationError::OutcomeUnknown( + "inventory page revision missing".into(), + )); + } + let mut items = Vec::new(); + for raw in raws { + let id = raw + .get(self.domain.identity_field()) + .and_then(Json::as_str) + .ok_or_else(|| { + WipOperationError::OutcomeUnknown( + "inventory identity missing".into(), + ) + })?; + if self.provider.allowed( + self.domain.read_permission(), + &self.provider.read_input(self.domain, id), + ) { + items.push(self.provider.projected(self.domain, raw)?); + } + } + let next = if attachment { + match page.get("next_offset") { + Some(Json::Null) => None, + Some(v) if v.as_u64().is_some_and(|n| n <= u32::MAX as u64) => { + Some(format!("offset:{}", v.as_u64().unwrap())) + } + _ => { + return Err(WipOperationError::OutcomeUnknown( + "invalid attachment next page".into(), + )); + } + } + } else { + match page.get("next_cursor") { + Some(Json::Null) => None, + Some(Json::String(v)) if cursor.is_none_or(|old| v.as_str() > old) => { + Some(v.clone()) + } + _ => { + return Err(WipOperationError::OutcomeUnknown( + "invalid Workdir next page".into(), + )); + } + } + }; + // Empty describes this permission-filtered page, not the + // entire catalog. A sparse page still advances the Backend cursor. + return native( + json!({"empty":items.is_empty(), "items":items, "has_more":next.is_some(), "next_cursor":next, "limit":limit}), + ); + } + let mut items = Vec::new(); + for raw in self.provider.inventory(self.domain)? { + let id = raw + .get(self.domain.identity_field()) + .and_then(Json::as_str) + .ok_or_else(|| { + WipOperationError::OutcomeUnknown("inventory identity missing".into()) + })?; + if self.provider.allowed( + self.domain.read_permission(), + &self.provider.read_input(self.domain, id), + ) { + items.push(self.provider.projected(self.domain, &raw)?); + } + } + items.sort_by(|a, b| { + a[self.domain.identity_field()] + .as_str() + .cmp(&b[self.domain.identity_field()].as_str()) + }); + let total_visible = items.len(); + let remaining: Vec<_> = items + .into_iter() + .filter(|v| { + cursor.is_none_or(|c| { + v[self.domain.identity_field()] + .as_str() + .is_some_and(|id| id > c) + }) + }) + .collect(); + let has_more = remaining.len() > limit; + let items: Vec<_> = remaining.into_iter().take(limit).collect(); + let next_cursor = if has_more { + items + .last() + .map(|v| v[self.domain.identity_field()].clone()) + } else { + None + }; + json!({"items": items, "empty": total_visible == 0, "has_more": has_more, "next_cursor": next_cursor, "limit": limit}) + } + "create" => { + self.provider.authorize(CREATE_TOOL, &input)?; + let tool = WorkspaceHttpWorkdirTool { + backend: self.provider.backend.clone(), + operation: WorkdirOperation::Create, + }; + let output = tool + .execute(&input.to_string(), context.execution) + .await + .map_err(map_tool_error)?; + let raw = output_json(output)?; + json!({"item": self.provider.projected(Domain::Workdir, &raw["item"])?, "runtime_id":raw["runtime_id"]}) + } + _ => { + return Err(failure( + ProtocolErrorCode::OperationNotFound, + "operation not published", + )); + } + }; + native(value) + } +} + +struct ItemResolver { + provider: Provider, + domain: Domain, +} +impl WipDynamicItemResolver for ItemResolver { + fn resolve(&self, reference: &str) -> Option { + let id = decode_identity(reference)?; + if !self.provider.catalog + || !self.provider.allowed( + self.domain.read_permission(), + &self.provider.read_input(self.domain, &id), + ) + { + return None; + } + let raw = self.provider.raw_item(self.domain, &id).ok()?; + let projected = self.provider.projected(self.domain, &raw).ok()?; + let interface = format!("yoi.{}/item/v1", self.domain.owner()); + Some(WipDynamicItem { + object: object(&item_path(self.domain, &id), &id, &interface, &projected), + handler: Arc::new(ItemHandler { + provider: self.provider.clone(), + domain: self.domain, + id, + }), + }) + } +} +impl WipDynamicOperationResolver for ItemResolver { + fn handler(&self, reference: &str) -> Arc { + Arc::new(ItemHandler { + provider: self.provider.clone(), + domain: self.domain, + id: decode_identity(reference).unwrap_or_default(), + }) + } +} +struct ItemHandler { + provider: Provider, + domain: Domain, + id: String, +} +impl ItemHandler { + fn input(&self, arguments: &BTreeMap) -> Result { + let mut input = argument_json(arguments)?; + // Defense in depth in addition to descriptor validation, including direct handler calls. + if input.get(self.domain.identity_field()).is_some() + || input.get("alias").is_some() && matches!(self.domain, Domain::Attachment) + { + return Err(failure( + ProtocolErrorCode::InvalidArguments, + "route-bound identity cannot be overridden", + )); + } + input[self.domain.identity_field()] = json!(self.id); + Ok(input) + } + fn supports(&self, operation: &str) -> bool { + let Ok(raw) = self.provider.raw_item(self.domain, &self.id) else { + return false; + }; + match operation { + "read" => true, + "attach" => { + matches!(self.domain, Domain::Workdir) + && matches!( + raw.get("source") + .and_then(|v| v.get("kind")) + .and_then(Json::as_str), + Some("repository" | "external_grant") + ) + && raw.get("status").and_then(Json::as_str) == Some("active") + } + "delete" => { + matches!(self.domain, Domain::Workdir) + && raw.get("cleanup_target").is_some_and(|v| !v.is_null()) + } + "detach" => matches!(self.domain, Domain::Attachment), + _ => false, + } + } +} +#[async_trait] +impl WipOperationHandler for ItemHandler { + fn contextual_interface(&self) -> bool { + true + } + fn operation_available(&self, operation: &str) -> bool { + let (enabled, permission) = match operation { + "read" => (self.provider.catalog, self.domain.read_permission()), + "attach" => (self.provider.manage, ATTACH_TOOL), + "delete" => (self.provider.manage, DELETE_TOOL), + "detach" => (self.provider.manage, DETACH_TOOL), + _ => return false, + }; + enabled && self.provider.may_allow(permission) && self.supports(operation) + } + async fn call( + &self, + operation: &str, + arguments: &BTreeMap, + context: WipCallContext, + ) -> Result { + let mut input = self.input(arguments)?; + // Recheck reference permission as well as mutation permission on every call. + self.provider.authorize( + self.domain.read_permission(), + &self.provider.read_input(self.domain, &self.id), + )?; + if !self.operation_available(operation) { + return Err(failure( + ProtocolErrorCode::OperationNotFound, + "target or subject does not support this operation", + )); + } + let value = match operation { + "read" => self + .provider + .projected(self.domain, &self.provider.raw_item(self.domain, &self.id)?)?, + "attach" | "delete" => { + let (permission, op) = if operation == "attach" { + (ATTACH_TOOL, WorkdirOperation::Attach) + } else { + (DELETE_TOOL, WorkdirOperation::Delete) + }; + self.provider.authorize(permission, &input)?; + let output = WorkspaceHttpWorkdirTool { + backend: self.provider.backend.clone(), + operation: op, + } + .execute(&input.to_string(), context.execution) + .await + .map_err(map_tool_error)?; + let raw = output_json(output)?; + if operation == "attach" { + let mut result = raw; + result["workdir_path"] = json!(item_path(Domain::Workdir, &self.id)); + result["attachments_path"] = json!(ATTACHMENTS); + if let Some(alias) = input.get("alias").and_then(Json::as_str) + && self + .provider + .backend + .session_router + .resolve(Some(alias)) + .is_ok() + { + result["checkout_path"] = json!(crate::checkout::checkout_root(alias)); + result["checkout_slug"] = json!(encode_identity(alias)); + } + // Backend attachment list supplies the durable lifetime identity. + result + } else { + raw + } + } + "detach" => { + let raw = self.provider.raw_item(Domain::Attachment, &self.id)?; + let alias = raw.get("alias").and_then(Json::as_str).ok_or_else(|| { + WipOperationError::OutcomeUnknown("attachment alias missing".into()) + })?; + input + .as_object_mut() + .expect("route input is an object") + .remove("connection_id"); + input["alias"] = json!(alias); + self.provider.authorize(DETACH_TOOL, &input)?; + output_json( + self.provider + .backend + .detach_managed(alias, Some(&self.id)) + .await + .map_err(map_tool_error)?, + )? + } + _ => { + return Err(failure( + ProtocolErrorCode::OperationNotFound, + "operation not published", + )); + } + }; + native(value) + } +} + +fn object(path: &str, identity: &str, interface: &str, value: &Json) -> Object { + let mut digest = Sha256::new(); + digest.update(path); + digest.update([0]); + digest.update(value.to_string()); + Object { + name: path.rsplit('/').next().unwrap_or_default().into(), + description: Some(format!("Workspace authority Object: {identity}")), + interfaces: vec![interface.into()], + r#ref: Some(identity.into()), + validator: Some(digest.finalize().to_vec()), + } +} +fn native(value: Json) -> Result { + Ok(WipOperationOutput::native( + json_to_wip(&value).map_err(WipOperationError::OutcomeUnknown)?, + )) +} +fn failure(code: ProtocolErrorCode, message: impl Into) -> WipOperationError { + WipOperationError::Protocol(ProtocolError { + code, + message: message.into(), + }) +} +fn map_tool_error(error: ToolError) -> WipOperationError { + match error { + ToolError::InvalidArgument(message) => { + failure(ProtocolErrorCode::InvalidArguments, message) + } + ToolError::Cancelled(output) => WipOperationError::Cancelled(output), + ToolError::Interrupted(output) => WipOperationError::Interrupted(output), + ToolError::StructuredConflict { code, .. } => { + let (code, message) = match code.as_str() { + "workspace_http_401" | "workspace_http_403" => ( + ProtocolErrorCode::PermissionDenied, + "Workspace authority denied the lifecycle request", + ), + "workspace_http_404" => (ProtocolErrorCode::NotFound, "Object no longer exists"), + "workspace_http_409" => ( + ProtocolErrorCode::ValidatorMismatch, + "connection or lifecycle observation is stale; rediscover", + ), + _ => { + return WipOperationError::OutcomeUnknown( + "Workspace lifecycle conflict with unknown execution outcome".into(), + ); + } + }; + failure(code, message) + } + // Never expose raw backend errors (which may include host/provider details). + _ => WipOperationError::OutcomeUnknown( + "Workspace lifecycle operation failed; outcome unknown; do not automatically retry" + .into(), + ), + } +} +fn output_json(output: ToolOutput) -> Result { + serde_json::from_str(output.content.as_deref().unwrap_or_default()) + .map_err(|_| WipOperationError::OutcomeUnknown("invalid lifecycle result".into())) +} +fn argument_json(arguments: &BTreeMap) -> Result { + let mut values = serde_json::Map::new(); + for (name, value) in arguments { + if !matches!(value, Value::Unit) { + values.insert( + name.clone(), + wip_to_json(value).map_err(|e| failure(ProtocolErrorCode::InvalidArguments, e))?, + ); + } + } + Ok(Json::Object(values)) +} +fn page_limit(input: &Json) -> Result { + let limit = match input.get("limit") { + None | Some(Json::Null) => 50, + Some(value) => value.as_u64().ok_or_else(|| { + failure( + ProtocolErrorCode::InvalidArguments, + "limit must be an integer in 1..=100", + ) + })?, + }; + if !(1..=100).contains(&limit) { + return Err(failure( + ProtocolErrorCode::InvalidArguments, + "limit must be 1..=100", + )); + } + Ok(limit as usize) +} + +// Readable ASCII identities remain unchanged. Every other UTF-8 identity uses a +// canonical ~hex segment; the escape marker is itself always escaped. This is +// injective, does not exclude existing keys, and never decodes a second route. +pub(crate) fn encode_identity(id: &str) -> String { + if !id.is_empty() + && id + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_')) + { + id.into() + } else { + format!( + "~{}", + id.as_bytes() + .iter() + .map(|b| format!("{b:02x}")) + .collect::() + ) + } +} +pub(crate) fn decode_identity(segment: &str) -> Option { + let id = if let Some(hex) = segment.strip_prefix('~') { + if hex.len() % 2 != 0 || hex.is_empty() || !hex.bytes().all(|b| b.is_ascii_hexdigit()) { + return None; + } + let bytes = (0..hex.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&hex[i..i + 2], 16)) + .collect::, _>>() + .ok()?; + String::from_utf8(bytes).ok()? + } else { + segment.into() + }; + (encode_identity(&id) == segment && !id.is_empty()).then_some(id) +} +fn item_path(domain: Domain, id: &str) -> String { + format!("{}/{}", domain.route(), encode_identity(id)) +} +fn parameter(name: &str, required: bool, r#type: TypeExpr) -> ParameterDeclaration { + ParameterDeclaration { + name: name.into(), + required, + documentation: None, + r#type, + } +} +fn operation( + name: &str, + summary: &str, + parameters: Vec, +) -> OperationDeclaration { + OperationDeclaration { + name: name.into(), + documentation: Some(Documentation { + summary: summary.into(), + details: None, + }), + parameters, + returns: ReturnDeclaration { + documentation: None, + r#type: TypeExpr::Json, + }, + } +} +fn descriptor( + domain: Domain, + item: bool, + operations: Vec, +) -> InterfaceDescriptor { + InterfaceDescriptor { format: INTERFACE_FORMAT_V1.into(), documentation: Some(Documentation { + summary: format!("Workspace {} {}", domain.owner(), if item { "item" } else { "collection" }), + details: Some("Object identity is route-bound. Feature implementation, target capability and subject permission are independent. Inventory does not grant content or command access.".into()), + }), types: Vec::new(), operations } +} +fn collection_descriptor(domain: Domain, management: bool) -> InterfaceDescriptor { + descriptor( + domain, + false, + if management { + vec![operation( + "create", + CREATE_DESCRIPTION, + vec![ + parameter("repository_key", true, TypeExpr::String), + parameter("selector", false, TypeExpr::String), + parameter("runtime_id", false, TypeExpr::String), + parameter("display_name", false, TypeExpr::String), + ], + )] + } else { + vec![operation( + "list", + "List a bounded page of visible Objects with item paths", + vec![ + parameter("limit", false, TypeExpr::Integer), + parameter("cursor", false, TypeExpr::String), + ], + )] + }, + ) +} +fn item_descriptor(domain: Domain, management: bool) -> InterfaceDescriptor { + descriptor( + domain, + true, + if management { + match domain { + Domain::Workdir => vec![ + operation( + "attach", + ATTACH_DESCRIPTION, + vec![parameter("alias", true, TypeExpr::String)], + ), + operation( + "delete", + DELETE_DESCRIPTION, + vec![parameter("reason", true, TypeExpr::String)], + ), + ], + Domain::Attachment => vec![operation("detach", DETACH_DESCRIPTION, vec![])], + Domain::Repository => vec![], + } + } else { + vec![operation( + "read", + "Read this Object through current Workspace authority", + vec![], + )] + }, + ) +} + +#[cfg(test)] +pub(crate) mod tests { + use super::*; + use manifest::ToolPermissionRule; + use std::sync::Mutex; + + #[derive(Debug)] + pub(crate) struct CatalogClient { + pub(crate) state: Mutex, + } + #[derive(Debug)] + pub(crate) struct CatalogState { + pub(crate) repositories: Vec, + pub(crate) workdirs: Vec, + pub(crate) attachments: Vec, + pub(crate) requests: Vec, + pub(crate) denied: bool, + config_current: Option, + generation: usize, + } + impl Default for CatalogClient { + fn default() -> Self { + Self { + state: Mutex::new(CatalogState { + repositories: vec![ + json!({"repository_key":"registered-key", "kind":"git", "provider":"git", "source":{"path":"/secret/host", "credential":"SECRET"}, "git":{"remotes":["secret-uri"]}}), + ], + workdirs: vec![], + attachments: vec![], + requests: vec![], + denied: false, + config_current: None, + generation: 0, + }), + } + } + } + fn directory(id: &str, repository: &str) -> Json { + json!({"working_directory_id":id, "source":{"kind":"repository", "repository_key":repository}, "display_name":"Review checkout", "status":"active", "cleanliness":"clean", "materializer_kind":"runtime_git_clone", "cleanup_target":{"kind":"runtime_git_clone", "repository_key":repository, "working_directory_id":id}}) + } + impl WorkspaceClient for CatalogClient { + fn workspace_id(&self) -> Option<&str> { + Some("test-workspace") + } + fn kind(&self) -> &str { + "catalog-test" + } + fn is_available(&self) -> bool { + true + } + fn execute( + &self, + request: WorkspaceRequest, + ) -> Result { + let mut state = self.state.lock().unwrap(); + state.requests.push(request.clone()); + if state.denied { + return Ok(WorkspaceResponse { + status: 403, + body: "SECRET /host/path".into(), + }); + } + let path = request.path.strip_prefix("/api/w/test-workspace/").unwrap(); + let body: Json = request + .body + .as_deref() + .map(|b| serde_json::from_str(b).unwrap()) + .unwrap_or(Json::Null); + let value = match (request.method, path.split('?').next().unwrap()) { + (WorkspaceRequestMethod::Get, "workers/self/workspace-config") => { + state.config_current.clone().unwrap_or(Json::Null) + } + (WorkspaceRequestMethod::Get, "repositories") => { + json!({"items":state.repositories}) + } + (WorkspaceRequestMethod::Get, "working-directories") => { + // The legacy browser/Tool endpoint is a latest-200 snapshot, + // not a complete WIP catalog; regressions must not use it. + json!({"items":state.workdirs.iter().rev().take(200).collect::>()}) + } + (WorkspaceRequestMethod::Get, "workers/self/workdir-catalog") => { + let query: BTreeMap<_, _> = path + .split('?') + .nth(1) + .unwrap_or_default() + .split('&') + .filter_map(|p| p.split_once('=')) + .collect(); + let limit = query + .get("limit") + .and_then(|s| s.parse::().ok()) + .unwrap_or(50); + let cursor = query.get("cursor").copied(); + let mut all: Vec<_> = state.workdirs.iter().collect(); + all.sort_by_key(|v| v["working_directory_id"].as_str().unwrap()); + let revision = Sha256::digest(serde_json::to_vec(&all).unwrap()) + .iter() + .map(|b| format!("{b:02x}")) + .collect::(); + let remaining: Vec<_> = all + .into_iter() + .filter(|v| { + cursor.is_none_or(|c| v["working_directory_id"].as_str().unwrap() > c) + }) + .collect(); + let has_more = remaining.len() > limit; + let items: Vec<_> = remaining.into_iter().take(limit).collect(); + let next = if has_more { + items.last().map(|v| v["working_directory_id"].clone()) + } else { + None + }; + json!({"items":items, "next_cursor":next, "revision":revision}) + } + (WorkspaceRequestMethod::Get, "workers/self/workdir-attachments") => { + let query: BTreeMap<_, _> = path + .split('?') + .nth(1) + .unwrap_or_default() + .split('&') + .filter_map(|p| p.split_once('=')) + .collect(); + let mut all: Vec<_> = state.attachments.iter().collect(); + all.sort_by_key(|v| v["alias"].as_str().unwrap()); + let revision = Sha256::digest(serde_json::to_vec(&all).unwrap()) + .iter() + .map(|b| format!("{b:02x}")) + .collect::(); + if let Some(expected) = query.get("connection_id") { + json!({"items":all.into_iter().filter(|a| a["connection_id"] == *expected).collect::>(), "next_offset":null, "revision":revision}) + } else { + let limit = query + .get("limit") + .and_then(|s| s.parse::().ok()) + .unwrap_or(50); + let offset = query + .get("offset") + .and_then(|s| s.parse::().ok()) + .unwrap_or(0); + let count = all.len(); + let items: Vec<_> = all.into_iter().skip(offset).take(limit).collect(); + let next = (offset + limit < count).then_some(offset + limit); + json!({"items":items, "next_offset":next, "revision":revision}) + } + } + (WorkspaceRequestMethod::Get, p) if p.starts_with("repositories/") => { + let id = p.trim_start_matches("repositories/"); + match state + .repositories + .iter() + .find(|r| encode_path_segment(r["repository_key"].as_str().unwrap()) == id) + { + Some(r) => json!({"item":r}), + None => { + return Ok(WorkspaceResponse { + status: 404, + body: String::new(), + }); + } + } + } + (WorkspaceRequestMethod::Get, p) if p.starts_with("working-directories/") => { + let id = p.trim_start_matches("working-directories/"); + match state + .workdirs + .iter() + .find(|r| r["working_directory_id"] == id) + { + Some(r) => json!({"item":r}), + None => { + return Ok(WorkspaceResponse { + status: 404, + body: String::new(), + }); + } + } + } + (WorkspaceRequestMethod::Post, "working-directories") => { + let raw = directory("wd-created", body["repository_key"].as_str().unwrap()); + state.workdirs.push(raw.clone()); + json!({"workspace_id":"test-workspace", "runtime_id":"runtime", "item":raw, "diagnostics":[]}) + } + (WorkspaceRequestMethod::Post, "workers/self/workdir-attachments") => { + state.generation += 1; + let raw = json!({"alias":body["alias"], "working_directory_id":body["working_directory_id"], "connection_id":format!("connection-{}", state.generation), "capabilities":{"bits":63}}); + state.attachments.push(raw); + json!({"workspace_id":"test-workspace", "alias":body["alias"], "working_directory_id":body["working_directory_id"], "capabilities":{"bits":63}, "attached":true}) + } + (WorkspaceRequestMethod::Delete, p) + if p.starts_with("workers/self/workdir-attachments/") => + { + let alias = p.trim_start_matches("workers/self/workdir-attachments/"); + let expected = path + .split("expected_connection_id=") + .nth(1) + .unwrap_or_default(); + let Some(index) = state + .attachments + .iter() + .position(|r| r["alias"] == alias && r["connection_id"] == expected) + else { + return Ok(WorkspaceResponse { + status: 409, + body: "stale".into(), + }); + }; + let raw = state.attachments.remove(index); + json!({"workspace_id":"test-workspace", "alias":raw["alias"], "working_directory_id":raw["working_directory_id"], "capabilities":raw["capabilities"], "attached":false}) + } + (WorkspaceRequestMethod::Delete, p) if p.starts_with("working-directories/") => { + json!({"working_directory_id":p.trim_start_matches("working-directories/"), "disposition":"retained", "retryable":false}) + } + other => panic!("unexpected catalog request: {other:?}"), + }; + Ok(WorkspaceResponse { + status: 200, + body: value.to_string(), + }) + } + } + #[test] + fn config_content_reference_requires_enabled_mount_and_exact_current_lifetime() { + let client = Arc::new(CatalogClient::default()); + let current = json!({"workspace_id":"test-workspace", "alias":"workspace-config", "connection_id":"config-lifetime", "working_directory_id":"config-workdir", "access":"read_only", "name":"Workspace configuration", "purpose":"Configuration editing", "content_path":"/workspace-config", "already_attached":true}); + client.state.lock().unwrap().config_current = Some(current); + let raw = json!({"alias":"workspace-config", "connection_id":"config-lifetime", "working_directory_id":"config-workdir", "capabilities":{"bits":25}, "host_path":"/SECRET"}); + let mut p = provider(client.clone(), true, true, None); + assert!( + p.projected(Domain::Attachment, &raw) + .ok() + .unwrap() + .get("content_path") + .is_none() + ); + p.config_content = true; + let mut physical = raw.clone(); + physical["alias"] = json!("checkout"); + assert!( + p.projected(Domain::Attachment, &physical) + .ok() + .unwrap() + .get("content_path") + .is_none() + ); + assert!( + client.state.lock().unwrap().requests.is_empty(), + "physical inventories must not perform a config request per attachment" + ); + let projected = p.projected(Domain::Attachment, &raw).ok().unwrap(); + assert_eq!(projected["content_path"], "/workspace-config"); + assert_eq!(projected["access"], "read_only"); + assert_eq!(projected["purpose"], "Configuration editing"); + assert!(projected.get("checkout_path").is_none()); + assert!(!projected.to_string().contains("SECRET")); + let mut old = raw.clone(); + old["connection_id"] = json!("previous-lifetime"); + assert!( + p.projected(Domain::Attachment, &old) + .ok() + .unwrap() + .get("content_path") + .is_none() + ); + client.state.lock().unwrap().denied = true; + assert!( + p.projected(Domain::Attachment, &raw) + .ok() + .unwrap() + .get("content_path") + .is_none() + ); + } + + #[test] + fn config_workdir_source_preserves_safe_mapping_not_private_management_data() { + let p = provider(Arc::new(CatalogClient::default()), true, false, None); + let raw = json!({"working_directory_id":"config-workdir", "source":{"kind":"workspace_config", "access":"read_write", "content_path":"/workspace-config", "purpose":"Configuration editing", "grant_id":"PRIVATE", "host_path":"/SECRET"}}); + let projected = p.projected(Domain::Workdir, &raw).ok().unwrap(); + assert_eq!( + projected["source"], + json!({"kind":"workspace_config", "access":"read_write", "content_path":"/workspace-config", "purpose":"Configuration editing"}) + ); + assert!(!projected.to_string().contains("PRIVATE")); + assert!(!projected.to_string().contains("SECRET")); + } + + fn provider( + client: Arc, + catalog: bool, + manage: bool, + permissions: Option, + ) -> Provider { + Provider { + backend: ManageWorkdirFeature::new(client).backend(), + catalog, + manage, + config_content: false, + permissions, + } + } + fn arguments(input: Json) -> BTreeMap { + input + .as_object() + .unwrap() + .iter() + .map(|(k, v)| (k.clone(), json_to_wip(v).unwrap())) + .collect() + } + fn context() -> WipCallContext { + WipCallContext { + execution: ToolExecutionContext::default(), + security_context: "test".into(), + } + } + async fn call(handler: &dyn WipOperationHandler, operation: &str, input: Json) -> Json { + let output = handler + .call(operation, &arguments(input), context()) + .await + .unwrap_or_else(|_| panic!("{operation} failed")); + wip_to_json(&output.value).unwrap() + } + fn policy(allowed: &[&str]) -> ToolPermissionConfig { + ToolPermissionConfig { + default_action: ToolPermissionAction::Deny, + rules: allowed + .iter() + .map(|name| ToolPermissionRule { + tool: (*name).into(), + pattern: "*".into(), + action: ToolPermissionAction::Allow, + }) + .collect(), + } + } + + #[test] + fn catalog_route_encoding_is_injective_and_canonical_for_existing_keys() { + let keys = [ + "main", + "Main", + "repo/name", + "../main", + ".", + "日本語", + "~2e", + "a.b", + "%2f", + "with space", + ]; + let mut paths = std::collections::HashSet::new(); + for key in keys { + let encoded = encode_identity(key); + assert_eq!(decode_identity(&encoded).as_deref(), Some(key)); + assert!(paths.insert(encoded.clone())); + wip_protocol::validate_path(&format!("/repositories/{encoded}")).unwrap(); + } + for invalid in ["~", "~xx", "~é", "~6162", "a.b", "..", "~C3A9"] { + assert!(decode_identity(invalid).is_none(), "{invalid}"); + } + } + + #[tokio::test] + async fn catalog_fresh_workspace_find_key_create_attach_read_detach_preserves_workdir() { + let client = Arc::new(CatalogClient::default()); + let p = provider(client.clone(), true, true, None); + let repositories = CollectionHandler { + provider: p.clone(), + domain: Domain::Repository, + }; + let listed = call(&repositories, "list", json!({})).await; + assert_eq!(listed["items"][0]["repository_key"], "registered-key"); + assert_eq!(listed["items"][0]["path"], "/repositories/registered-key"); + assert!(listed["items"][0]["default_selector"].is_null()); + assert!(!listed.to_string().contains("SECRET")); + assert!(!listed.to_string().contains("/secret/host")); + let repository = ItemHandler { + provider: p.clone(), + domain: Domain::Repository, + id: "registered-key".into(), + }; + assert!(!repository.operation_available("create")); + let read = call(&repository, "read", json!({})).await; + let workdirs = CollectionHandler { + provider: p.clone(), + domain: Domain::Workdir, + }; + let created = call( + &workdirs, + "create", + json!({"repository_key":read["repository_key"]}), + ) + .await; + assert_eq!(created["item"]["path"], "/workdirs/wd-created"); + assert!( + client.state.lock().unwrap().attachments.is_empty(), + "create is not attach" + ); + let workdir = ItemHandler { + provider: p.clone(), + domain: Domain::Workdir, + id: "wd-created".into(), + }; + let attached = call(&workdir, "attach", json!({"alias":"checkout"})).await; + assert_eq!(attached["attachments_path"], ATTACHMENTS); + let attachments = CollectionHandler { + provider: p.clone(), + domain: Domain::Attachment, + }; + let listed = call(&attachments, "list", json!({})).await; + assert_eq!(listed["items"][0]["alias"], "checkout"); + assert_eq!(listed["items"][0]["workdir_path"], "/workdirs/wd-created"); + let connection = ItemHandler { + provider: p.clone(), + domain: Domain::Attachment, + id: listed["items"][0]["connection_id"].as_str().unwrap().into(), + }; + call(&connection, "detach", json!({})).await; + assert!(client.state.lock().unwrap().attachments.is_empty()); + assert_eq!( + client.state.lock().unwrap().workdirs.len(), + 1, + "detach is not delete" + ); + assert!(p.backend.session_router.aliases().is_empty()); + let retained = call(&workdir, "delete", json!({"reason":"done"})).await; + assert_eq!(retained["disposition"], "retained"); + } + + #[tokio::test] + async fn catalog_feature_permission_and_target_capability_are_independent() { + let client = Arc::new(CatalogClient::default()); + client + .state + .lock() + .unwrap() + .workdirs + .push(directory("wd", "registered-key")); + for manage in [false, true] { + let p = provider( + client.clone(), + true, + manage, + Some(policy(&[ + "RepositoryList", + "RepositoryRead", + LIST_TOOL, + "WorkdirRead", + ])), + ); + let handler = ItemHandler { + provider: p.clone(), + domain: Domain::Workdir, + id: "wd".into(), + }; + assert!(handler.operation_available("read")); + assert!(!handler.operation_available("attach")); + assert!(matches!( + handler + .call("attach", &arguments(json!({"alias":"x"})), context()) + .await, + Err(WipOperationError::Protocol(_)) + )); + let resolver = ItemResolver { + provider: p, + domain: Domain::Workdir, + }; + assert!(resolver.resolve("wd").is_some()); + } + let denied = provider(client.clone(), true, true, Some(policy(&[]))); + assert!( + ItemResolver { + provider: denied.clone(), + domain: Domain::Workdir + } + .resolve("wd") + .is_none() + ); + assert!( + !CollectionHandler { + provider: denied, + domain: Domain::Repository + } + .is_visible() + ); + let p = provider(client.clone(), true, true, None); + client.state.lock().unwrap().workdirs.push(json!({"working_directory_id":"external", "status":"active", "source":{"kind":"external_grant", "grant_id":"opaque", "grant_permissions":{"read":true}}})); + let external = ItemHandler { + provider: p.clone(), + domain: Domain::Workdir, + id: "external".into(), + }; + assert!(external.operation_available("attach")); + assert!(!external.operation_available("delete")); + let raw = call(&external, "read", json!({})).await; + assert!(raw["source"].get("grant_id").is_none()); + client.state.lock().unwrap().denied = true; + assert!(!external.operation_available("read")); + assert!( + ItemResolver { + provider: p, + domain: Domain::Workdir + } + .resolve("external") + .is_none() + ); + assert!( + client + .state + .lock() + .unwrap() + .requests + .iter() + .all(|r| r.method == WorkspaceRequestMethod::Get) + ); + } + + #[tokio::test] + async fn catalog_empty_paging_route_bound_identity_and_failure_classification() { + let client = Arc::new(CatalogClient::default()); + let p = provider(client.clone(), true, true, None); + let handler = CollectionHandler { + provider: p.clone(), + domain: Domain::Repository, + }; + client.state.lock().unwrap().repositories.clear(); + let empty = call(&handler, "list", json!({})).await; + assert_eq!(empty["empty"], true); + client.state.lock().unwrap().repositories = (0..3) + .map(|i| json!({"repository_key":format!("key-{i}")})) + .collect(); + let first = call(&handler, "list", json!({"limit":2})).await; + assert_eq!(first["items"].as_array().unwrap().len(), 2); + let second = call( + &handler, + "list", + json!({"limit":2, "cursor":first["next_cursor"]}), + ) + .await; + assert_eq!(second["items"][0]["repository_key"], "key-2"); + assert_eq!(second["has_more"], false); + assert!( + handler + .call("list", &arguments(json!({"limit":101})), context()) + .await + .is_err() + ); + let item = ItemHandler { + provider: p, + domain: Domain::Workdir, + id: "wd-bound".into(), + }; + let requests_before = client.state.lock().unwrap().requests.len(); + assert!( + item.call( + "delete", + &arguments(json!({"working_directory_id":"different", "reason":"done"})), + context() + ) + .await + .is_err() + ); + assert_eq!(client.state.lock().unwrap().requests.len(), requests_before); + assert!(matches!( + map_tool_error(ToolError::StructuredConflict { + code: "workspace_http_409".into(), + message: "SECRET".into() + }), + WipOperationError::Protocol(ProtocolError { + code: ProtocolErrorCode::ValidatorMismatch, + .. + }) + )); + assert!(matches!( + map_tool_error(ToolError::ExecutionFailed( + "transport uncertain SECRET".into() + )), + WipOperationError::OutcomeUnknown(_) + )); + } + + #[test] + fn catalog_registration_uses_host_namespace_and_rejects_conflicts() { + let feature = ManageWorkdirFeature::new(Arc::new(CatalogClient::default())); + let mut registry = WipMountRegistry::new(); + mount_workspace_workdir_wip(&mut registry, &feature, true, true, None).unwrap(); + assert_eq!( + registry.routes().collect::>(), + [REPOSITORIES, ATTACHMENTS, WORKDIRS] + ); + assert!(matches!( + mount_workspace_workdir_wip(&mut registry, &feature, true, true, None), + Err(WipMountError::RouteCollision { .. }) + )); + assert!(matches!( + registry.contribute_operations(WipOperationContribution { + route: WORKDIRS.into(), + contributor: "another-feature".into(), + interface: "yoi.workdir/collection/v1".into(), + descriptor: collection_descriptor(Domain::Workdir, true), + handler: Arc::new(CollectionHandler { + provider: provider(Arc::new(CatalogClient::default()), true, true, None), + domain: Domain::Workdir + }) + }), + Err(WipMountError::OperationCollision { .. }) + )); + } +} + +#[cfg(test)] +mod lifetime_tests { + use super::tests::CatalogClient; + use super::*; + use std::sync::atomic::{AtomicUsize, Ordering}; + + #[tokio::test] + async fn attachment_pages_and_direct_lookup_are_not_limited_to_first_page() { + let client = Arc::new(CatalogClient::default()); + client.state.lock().unwrap().attachments = (0..65).map(|n| json!({"connection_id":format!("lifetime-{n:03}"), "alias":format!("alias-{n:03}"), "working_directory_id":"wd", "capabilities":{"bits":25}})).collect(); + let p = Provider { + backend: ManageWorkdirFeature::new(client).backend(), + permissions: None, + catalog: true, + manage: true, + config_content: false, + }; + let handler = CollectionHandler { + provider: p.clone(), + domain: Domain::Attachment, + }; + let args = BTreeMap::from([("limit".into(), Value::Integer(50))]); + let context = || WipCallContext { + execution: ToolExecutionContext::default(), + security_context: "test".into(), + }; + let first = handler + .call("list", &args, context()) + .await + .unwrap_or_else(|_| panic!("list failed")); + let first = wip_to_json(&first.value).unwrap(); + assert_eq!(first["items"].as_array().unwrap().len(), 50); + assert_eq!(first["next_cursor"], "offset:50"); + let args = BTreeMap::from([("cursor".into(), Value::String("offset:50".into()))]); + let second = handler + .call("list", &args, context()) + .await + .unwrap_or_else(|_| panic!("list failed")); + let second = wip_to_json(&second.value).unwrap(); + assert_eq!(second["items"].as_array().unwrap().len(), 15); + assert_eq!(second["has_more"], false); + let resolver = ItemResolver { + provider: p, + domain: Domain::Attachment, + }; + assert!(resolver.resolve("lifetime-064").is_some()); + assert!(resolver.resolve("lifetime-unknown").is_none()); + } + + #[tokio::test] + async fn stale_lifetime_preflight_never_closes_new_alias_or_stops_children() { + let client = Arc::new(CatalogClient::default()); + client.state.lock().unwrap().attachments.push(json!({"connection_id":"replacement", "alias":"checkout", "working_directory_id":"same-workdir", "capabilities":{"bits":63}})); + let hooks = Arc::new(AtomicUsize::new(0)); + let hook_calls = hooks.clone(); + let backend = WorkspaceHttpWorkdirBackend::new(client.clone()).with_child_lifecycle( + Some(Arc::new(move || { + let hook_calls = hook_calls.clone(); + Box::pin(async move { + hook_calls.fetch_add(1, Ordering::SeqCst); + Ok(()) + }) + })), + None, + ); + let alias = workdir::WorkdirAttachmentAlias::new("checkout").unwrap(); + backend + .session_router + .attach( + alias.clone(), + WorkspaceAttachedWorkdirSession::handle_for_workdir( + client.clone(), + "checkout", + "same-workdir", + ), + ) + .unwrap(); + let error = backend + .detach_managed("checkout", Some("old-lifetime")) + .await + .unwrap_err(); + assert!(matches!(error, ToolError::StructuredConflict { .. })); + assert_eq!(hooks.load(Ordering::SeqCst), 0); + assert!(backend.session_router.session(&alias).is_some()); + assert_eq!( + client.state.lock().unwrap().attachments[0]["connection_id"], + "replacement" + ); + assert!( + client + .state + .lock() + .unwrap() + .requests + .iter() + .all(|r| r.method == WorkspaceRequestMethod::Get) + ); + } +} + +#[cfg(test)] +mod logical_capability_tests { + use super::*; + #[test] + fn logical_reference_does_not_gain_physical_attach_delete_or_command() { + let client = Arc::new(super::tests::CatalogClient::default()); + client.state.lock().unwrap().workdirs.push(json!({"working_directory_id":"logical-settings", "source":{"kind":"workspace_config"}, "status":"active"})); + let provider = Provider { + backend: ManageWorkdirFeature::new(client).backend(), + catalog: true, + manage: true, + config_content: false, + permissions: None, + }; + let item = ItemHandler { + provider, + domain: Domain::Workdir, + id: "logical-settings".into(), + }; + assert!(item.operation_available("read")); + for operation in ["attach", "delete", "command"] { + assert!(!item.operation_available(operation)); + } + } +} + +#[cfg(test)] +mod page_contract_tests { + use super::*; + #[test] + fn bounded_page_limit_rejects_negative_and_noninteger_inputs() { + assert_eq!(page_limit(&json!({})).unwrap_or_default(), 50); + assert_eq!(page_limit(&json!({"limit":100})).unwrap_or_default(), 100); + for input in [ + json!({"limit":-1}), + json!({"limit":0}), + json!({"limit":101}), + json!({"limit":"50"}), + json!({"limit":1.5}), + ] { + assert!(matches!( + page_limit(&input), + Err(WipOperationError::Protocol(ProtocolError { + code: ProtocolErrorCode::InvalidArguments, + .. + })) + )); + } + } +} diff --git a/crates/worker/src/feature/builtin/manage_worker.rs b/crates/worker/src/feature/builtin/manage_worker.rs index 5c746f9b6..789229581 100644 --- a/crates/worker/src/feature/builtin/manage_worker.rs +++ b/crates/worker/src/feature/builtin/manage_worker.rs @@ -77,8 +77,9 @@ pub trait WorkerControlService: Send + Sync { &self, runtime_id: String, worker_id: String, + request: server_api::WorkerRestoreRequest, ) -> Result { - let _ = (runtime_id, worker_id); + let _ = (runtime_id, worker_id, request); Err(WorkspaceClientError::Unavailable( "Runtime Worker restore is unavailable".to_string(), )) @@ -347,6 +348,7 @@ impl WorkerControlService for WorkspaceWorkerControlService { &self, runtime_id: String, worker_id: String, + request: server_api::WorkerRestoreRequest, ) -> Result { if !self.runtime_worker_control { return Err(WorkspaceClientError::Unavailable( @@ -357,6 +359,7 @@ impl WorkerControlService for WorkspaceWorkerControlService { crate::worker::WorkspaceServerOperation::WorkerControlRestore { runtime_id, worker_id, + request, }, ) } @@ -750,8 +753,13 @@ enum WorkerSubjectInput { #[derive(Debug, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] -struct WorkerTargetInput { +struct WorkerRestoreInput { subject: WorkerSubjectInput, + expected_observation_token: String, + /// Reuse the reported ID for explicit unknown-outcome recovery. A new intent + /// defaults to the durable Tool call identity. + #[serde(default)] + request_id: Option, } #[derive(Debug, Deserialize, JsonSchema)] @@ -864,7 +872,7 @@ impl WorkerOperation { Self::Cancel => "Cancel the current turn of a known Runtime Worker when allowed.", Self::Stop => "Stop a known Runtime Worker when allowed.", Self::Restore => { - "Restore a stopped Backend/Runtime Worker session in the current Workspace." + "Restore a stopped Backend/Runtime Worker session in the current Workspace. Supply expected_observation_token from WorkerList; a conflict requires refreshing and deliberate new intent. New requests default to the durable Tool call ID. For explicit recovery of an unknown/pending outcome, supply the reported request_id and the original token unchanged; never create a new intent or automatically retry." } Self::Remove => { "Remove an eligible stopped, unassigned, non-internal Worker. Supply a bounded reason; Backend validation and retention are authoritative." @@ -981,12 +989,41 @@ impl Tool for WorkspaceWorkerTool { .map_err(control_tool_error)? } WorkerOperation::Restore => { - let input = parse::(input_json, "WorkerRestore")?; + let input = parse::(input_json, "WorkerRestore")?; let (runtime_id, worker_id) = runtime_subject_ids(&input.subject, self.operation)?; - self.control - .restore_worker(runtime_id, worker_id) + let request_id = match input.request_id { + Some(id) => non_empty(id, "request_id")?, + None => format!( + "worker-restore:{}", + non_empty(ctx.call_id.clone(), "tool call_id")? + ), + }; + if request_id.len() > 512 { + return Err(ToolError::InvalidArgument( + "request_id must contain at most 512 bytes".into(), + )); + } + let recoverable_error = |error| { + ToolError::ExecutionFailed(format!( + "WorkerRestore request_id={request_id}; reuse this ID with the original observation token only for explicit unknown/pending outcome recovery, not after a stale-intent conflict: {error}" + )) + }; + let response = self + .control + .restore_worker( + runtime_id, + worker_id, + server_api::WorkerRestoreRequest { + expected_observation_token: non_empty( + input.expected_observation_token, + "expected_observation_token", + )?, + request_id: request_id.clone(), + }, + ) .await - .map_err(control_tool_error)? + .map_err(|error| recoverable_error(control_tool_error(error)))?; + return tool_output(self.operation, response).map_err(recoverable_error); } WorkerOperation::Remove => { let input = parse::(input_json, "WorkerRemove")?; @@ -1058,6 +1095,18 @@ fn tool_output( response.status, response.body ))); } + if operation == WorkerOperation::Restore { + let restored: server_api::WorkerRestoreResponse = serde_json::from_str(&response.body) + .map_err(|error| { + ToolError::ExecutionFailed(format!("invalid WorkerRestore response: {error}")) + })?; + if restored.result.state != server_api::WorkerRestoreState::Accepted { + return Err(ToolError::ExecutionFailed(format!( + "WorkerRestore returned {:?}: {}", + restored.result.state, response.body + ))); + } + } if operation == WorkerOperation::Stop && let Ok(summary) = serde_json::from_str::(&response.body) { @@ -1150,7 +1199,7 @@ fn worker_tool_contribution( } } WorkerOperation::Restore => { - definition::(operation, control, runtime_worker_control) + definition::(operation, control, runtime_worker_control) } WorkerOperation::Remove => { definition::(operation, control, runtime_worker_control) @@ -1410,6 +1459,95 @@ mod tests { }) } + #[tokio::test] + async fn worker_restore_forwards_exact_observation_and_request_identity() { + let client = Arc::new(RecordingWorkspaceClient::default()); + let control = test_control(client.clone()); + for _ in 0..2 { + control + .restore_worker( + "runtime/a".into(), + "worker/b".into(), + server_api::WorkerRestoreRequest { + expected_observation_token: "same-observation".into(), + request_id: "same-request".into(), + }, + ) + .await + .unwrap(); + } + let requests = client.requests.lock().unwrap(); + assert_eq!(requests.len(), 2); + assert_eq!( + requests[0].path, + "/api/w/workspace%2Ftest/worker-control/workers/runtime%2Fa/worker%2Fb/restore" + ); + assert_eq!(requests[0].body, requests[1].body); + let body: serde_json::Value = + serde_json::from_str(requests[0].body.as_deref().unwrap()).unwrap(); + assert_eq!( + body, + serde_json::json!({"expected_observation_token":"same-observation","request_id":"same-request"}) + ); + assert!(serde_json::from_str::(r#"{"subject":{"kind":"runtime_worker","runtime_id":"runtime-a","worker_id":"worker-a"}}"#).is_err()); + } + + #[tokio::test] + async fn worker_restore_reports_retry_identity_and_reuses_it_across_explicit_tool_calls() { + let client = Arc::new(RecordingWorkspaceClient::default()); + let tool = WorkspaceWorkerTool { + operation: WorkerOperation::Restore, + control: test_control(client.clone()), + runtime_worker_control: true, + }; + let mut input = serde_json::json!({ + "subject": {"kind":"runtime_worker", "runtime_id":"runtime-a", "worker_id":"worker-b"}, + "expected_observation_token":"original-observation" + }); + // The fixture deliberately returns malformed JSON shape: delivery may + // have succeeded, so retain the exact tuple rather than issuing new intent. + let first = tool + .execute( + &input.to_string(), + ToolExecutionContext::new("original-call", "batch", 0), + ) + .await + .unwrap_err(); + assert!( + first + .to_string() + .contains("request_id=worker-restore:original-call") + ); + input["request_id"] = "worker-restore:original-call".into(); + let second = tool + .execute( + &input.to_string(), + ToolExecutionContext::new("explicit-recovery-call", "batch", 1), + ) + .await + .unwrap_err(); + assert!( + second + .to_string() + .contains("request_id=worker-restore:original-call") + ); + { + let requests = client.requests.lock().unwrap(); + assert_eq!(requests.len(), 2); + assert_eq!(requests[0].body, requests[1].body); + } + input["request_id"] = "x".repeat(513).into(); + assert!( + tool.execute( + &input.to_string(), + ToolExecutionContext::new("invalid", "batch", 2) + ) + .await + .is_err() + ); + assert_eq!(client.requests.lock().unwrap().len(), 2); + } + #[tokio::test] async fn worker_spawn_forwards_typed_initial_submit_to_workspace_api() { let client = Arc::new(RecordingWorkspaceClient::default()); diff --git a/crates/worker/src/feature/builtin/memory.rs b/crates/worker/src/feature/builtin/memory.rs index d1ed3689d..d25581cf5 100644 --- a/crates/worker/src/feature/builtin/memory.rs +++ b/crates/worker/src/feature/builtin/memory.rs @@ -4,7 +4,8 @@ //! authority. In that case model-visible Memory tools must go through the //! workspace backend instead of resolving `.yoi/memory` from a Worker workdir. -use std::sync::Arc; +use std::collections::HashMap; +use std::sync::{Arc, Mutex}; use std::time::Duration; use agen::tool::{Tool, ToolDefinition, ToolError, ToolExecutionContext, ToolMeta, ToolOutput}; @@ -24,11 +25,15 @@ use crate::feature::{ FeatureDescriptor, FeatureHookPoint, FeatureInstallContext, FeatureInstallError, FeatureModule, HookDeclaration, ToolContribution, ToolDeclaration, }; -use crate::hook::{Hook, HookError, HookErrorCategory, HookExecutionPolicy, WorkerRestored}; +use crate::hook::{ + BeforeSessionRewrite, BeforeSessionRewriteAction, BeforeSessionRewriteContext, Hook, HookError, + HookErrorCategory, HookExecutionPolicy, HookPreRequestAction, PreLlmRequest, PreRequestContext, + WorkerRestored, +}; use crate::prompt::catalog::{PromptCatalog, WorkerPrompt}; use crate::worker::{ - SystemPromptContribution, SystemPromptContributionSource, WorkspaceClient, - WorkspaceClientError, WorkspaceRequest, WorkspaceRequestMethod, + ResidentContextRefresh, SystemPromptContribution, SystemPromptContributionSource, + WorkspaceClient, WorkspaceClientError, WorkspaceRequest, WorkspaceRequestMethod, }; #[derive(Clone, Debug)] @@ -74,6 +79,8 @@ pub enum WorkspaceMemoryBackendError { Decode(#[from] serde_json::Error), #[error("workspace memory backend rejected operation: {0}")] Backend(String), + #[error("workspace memory backend returned an invalid response: {0}")] + InvalidResponse(String), } impl dyn WorkspaceClient + '_ { @@ -672,12 +679,17 @@ struct WorkspaceResidentSummarySource { client: Arc, } -struct WorkspaceSubjektivResidentSummarySource { - client: Arc, +#[derive(Default)] +struct SubjectBehaviorRefreshState { + applied: Option<(u64, String)>, + pending: HashMap, } -const READY_EMPTY_RESIDENT_SURFACE: &str = - "The current ready resident memory surface is intentionally empty."; +struct WorkspaceSubjektivResidentSummarySource { + client: Arc, + prompts: Arc>, + behavior_state: Mutex, +} fn resident_summary_contribution( output: memory::backend::MemoryResidentSummaryOutput, @@ -715,37 +727,133 @@ impl SystemPromptContributionSource for WorkspaceResidentSummarySource { } } -#[async_trait] -impl SystemPromptContributionSource for WorkspaceSubjektivResidentSummarySource { - async fn load(&self) -> SystemPromptContribution { +impl WorkspaceSubjektivResidentSummarySource { + async fn fetch( + &self, + ) -> Result { match execute_subjektiv_memory_operation_with( self.client.as_ref(), - server_api::SubjektivMemoryBackendOperation::ResidentSummary(Default::default()), + server_api::SubjektivMemoryBackendOperation::ResidentContext(Default::default()), |client, request| { client.execute_with_timeout(request, RESIDENT_SUMMARY_REQUEST_TIMEOUT) }, - ) { - Ok(server_api::SubjektivMemoryBackendResponse::ResidentSummary(output)) => { - match resident_summary_contribution(output) { - SystemPromptContribution::Ready(content) if content.trim().is_empty() => { - SystemPromptContribution::Ready(READY_EMPTY_RESIDENT_SURFACE.to_string()) - } - contribution => contribution, + )? { + server_api::SubjektivMemoryBackendResponse::ResidentContext(output) => Ok(output), + other => Err(WorkspaceMemoryBackendError::InvalidResponse(format!( + "unexpected subject resident context response: {other:?}" + ))), + } + } + + fn render( + &self, + output: &server_api::SubjektivResidentContextOutput, + ) -> Result { + use memory::backend::MemoryResidentSummaryAvailability as Availability; + let availability = match output.memory_surface.availability { + Availability::Ready => "ready", + Availability::Ungenerated => "ungenerated", + Availability::Stale => "stale", + Availability::Failed => "failed", + Availability::Unavailable => "unavailable", + }; + self.prompts + .load() + .subjektiv_resident_context( + &output.behavior_md, + availability, + output.memory_surface.content.as_deref(), + ) + .map_err(|error| error.to_string()) + } + + fn remember_behavior(&self, output: &server_api::SubjektivResidentContextOutput) { + let mut state = self + .behavior_state + .lock() + .expect("subject behavior state poisoned"); + state.applied = Some((output.behavior_revision, output.behavior_md.clone())); + state.pending.remove(&output.behavior_revision); + } +} + +#[async_trait] +impl SystemPromptContributionSource for WorkspaceSubjektivResidentSummarySource { + async fn load(&self) -> SystemPromptContribution { + match self.fetch().await { + Ok(output) => match self.render(&output) { + Ok(rendered) => { + self.remember_behavior(&output); + SystemPromptContribution::Ready(rendered) } - } - Ok(other) => { - tracing::debug!(?other, "unexpected subject resident Memory response"); - SystemPromptContribution::Unavailable - } + Err(error) => { + tracing::error!(%error, "subject resident context render failed"); + SystemPromptContribution::Ready( + "The Host could not render the Subject resident context. Do not treat this as unset behavior or infer replacement behavior from Memory." + .to_string(), + ) + } + }, Err(error) => { - tracing::debug!(%error, "subject resident Memory unavailable"); - SystemPromptContribution::Unavailable + tracing::error!(%error, "subject resident context fetch failed"); + SystemPromptContribution::Ready( + "The Host could not fetch the Subject resident context. Do not treat this as unset behavior or infer replacement behavior from Memory." + .to_string(), + ) } } } + + async fn load_changed_resident_context( + &self, + ) -> Result, String> { + let output = self.fetch().await.map_err(|error| error.to_string())?; + let changed = self + .behavior_state + .lock() + .expect("subject behavior state poisoned") + .applied + .as_ref() + .is_none_or(|(revision, body)| { + *revision != output.behavior_revision || body != &output.behavior_md + }); + if !changed { + return Ok(None); + } + let rendered = self.render(&output)?; + let revision = output.behavior_revision; + self.behavior_state + .lock() + .expect("subject behavior state poisoned") + .pending + .insert(revision, output.behavior_md); + Ok(Some(ResidentContextRefresh { + body: rendered, + revision, + })) + } + + fn confirm_resident_context_revision(&self, revision: u64) { + let mut state = self + .behavior_state + .lock() + .expect("subject behavior state poisoned"); + if let Some(body) = state.pending.remove(&revision) { + state.applied = Some((revision, body)); + } + } + + fn invalidate_resident_context_representation(&self) { + self.behavior_state + .lock() + .expect("subject behavior state poisoned") + .applied = None; + } } const SUBJEKTIV_RESTORE_REFRESH_HOOK: &str = "refresh-resident-surface-after-restore"; +const SUBJEKTIV_BEHAVIOR_REFRESH_HOOK: &str = "refresh-subject-behavior-before-request"; +const SUBJEKTIV_BEHAVIOR_REWRITE_HOOK: &str = "invalidate-subject-behavior-before-rewrite"; /// subjektiv-owned restore behavior. Initial prompt contribution remains a /// shared host facility, while loading and rendering a post-restore refresh is @@ -793,6 +901,14 @@ impl FeatureModule for SubjektivResidentRestoreRefreshFeature { SUBJEKTIV_RESTORE_REFRESH_HOOK, FeatureHookPoint::WorkerRestored, )) + .with_hook(HookDeclaration::new( + SUBJEKTIV_BEHAVIOR_REFRESH_HOOK, + FeatureHookPoint::PreLlmRequest, + )) + .with_hook(HookDeclaration::new( + SUBJEKTIV_BEHAVIOR_REWRITE_HOOK, + FeatureHookPoint::BeforeSessionRewrite, + )) } fn install(&self, context: &mut FeatureInstallContext<'_>) -> Result<(), FeatureInstallError> { @@ -804,10 +920,64 @@ impl FeatureModule for SubjektivResidentRestoreRefreshFeature { prompts: Arc::clone(&self.prompts), workspace_id: self.workspace_id.clone(), }, + )?; + context.hooks().add_pre_llm_request( + SUBJEKTIV_BEHAVIOR_REFRESH_HOOK, + HookExecutionPolicy::fail_closed(), + SubjektivBehaviorRefreshHook { + source: Arc::clone(&self.source), + }, + )?; + context.hooks().add_before_session_rewrite( + SUBJEKTIV_BEHAVIOR_REWRITE_HOOK, + HookExecutionPolicy::fail_closed(), + SubjektivBehaviorRewriteHook { + source: Arc::clone(&self.source), + }, ) } } +struct SubjektivBehaviorRewriteHook { + source: Arc, +} + +#[async_trait] +impl Hook for SubjektivBehaviorRewriteHook { + async fn call( + &self, + _context: &BeforeSessionRewriteContext, + ) -> Result { + self.source.invalidate_resident_context_representation(); + Ok(BeforeSessionRewriteAction::Continue) + } +} + +struct SubjektivBehaviorRefreshHook { + source: Arc, +} + +#[async_trait] +impl Hook for SubjektivBehaviorRefreshHook { + async fn call(&self, context: &PreRequestContext) -> Result { + let refresh = self + .source + .load_changed_resident_context() + .await + .map_err(|error| HookError::new(HookErrorCategory::Dependency, error))?; + if let Some(refresh) = refresh { + let system_items = context.system_items().ok_or_else(|| { + HookError::new( + HookErrorCategory::Dependency, + "Subject behavior refresh requires durable session append authority", + ) + })?; + system_items.append_subject_behavior_refresh(refresh.body, refresh.revision); + } + Ok(HookPreRequestAction::Continue) + } +} + struct SubjektivResidentRestoreRefreshHook { source: Arc, prompts: Arc>, @@ -945,12 +1115,13 @@ pub(crate) fn is_builtin_subjektiv_consolidation_profile( pub(crate) fn ordinary_subjektiv_features_enabled(manifest: &manifest::WorkerManifest) -> bool { manifest.feature.subjektiv.execution_enabled() - && !is_builtin_subjektiv_consolidation_profile(manifest) + && !manifest.feature.subjektiv.profile.consolidation_tools } pub(crate) fn ordinary_subjektiv_resident_summary_source( manifest: &manifest::WorkerManifest, client: Arc, + prompts: Arc>, ) -> std::io::Result>> { if !ordinary_subjektiv_features_enabled(manifest) { return Ok(None); @@ -988,6 +1159,8 @@ pub(crate) fn ordinary_subjektiv_resident_summary_source( } Ok(Some(Arc::new(WorkspaceSubjektivResidentSummarySource { client, + prompts, + behavior_state: Mutex::new(SubjectBehaviorRefreshState::default()), }))) } @@ -1002,7 +1175,7 @@ impl SubjektivConsolidationFeatureInstallPlan { client: Arc, prompts: Arc, ) -> std::io::Result> { - if !is_builtin_subjektiv_consolidation_profile(manifest) { + if !manifest.feature.subjektiv.profile.consolidation_tools { return Ok(None); } let config = &manifest.feature.subjektiv; @@ -1120,6 +1293,10 @@ mod tests { use agen::tool::ToolDefinition; use std::sync::Mutex; + fn test_prompts() -> Arc> { + Arc::new(ArcSwap::from(PromptCatalog::builtins_only().unwrap())) + } + fn test_client() -> Arc { Arc::new(crate::worker::TestWorkspaceHttpClient::new( "workspace", @@ -1132,6 +1309,7 @@ mod tests { availability: memory::backend::MemoryResidentSummaryAvailability, content: Option, scope_allowed: bool, + behavior: Mutex<(u64, String)>, paths: Mutex>, timeouts: Mutex>, } @@ -1145,6 +1323,7 @@ mod tests { availability, content: content.map(str::to_string), scope_allowed: true, + behavior: Mutex::new((3, "Be deliberate.".to_string())), paths: Mutex::new(Vec::new()), timeouts: Mutex::new(Vec::new()), } @@ -1155,6 +1334,7 @@ mod tests { availability: memory::backend::MemoryResidentSummaryAvailability::Ready, content: Some("must not be injected".into()), scope_allowed: false, + behavior: Mutex::new((3, "must not be injected".to_string())), paths: Mutex::new(Vec::new()), timeouts: Mutex::new(Vec::new()), } @@ -1184,7 +1364,7 @@ mod tests { serde_json::from_str(request.body.as_deref().unwrap_or_default()).unwrap(); assert!(matches!( request.operation, - server_api::SubjektivMemoryBackendOperation::ResidentSummary(_) + server_api::SubjektivMemoryBackendOperation::ResidentContext(_) )); if !self.scope_allowed { return Ok(WorkspaceResponse { @@ -1192,13 +1372,18 @@ mod tests { body: "subject scope unavailable".into(), }); } + let (behavior_revision, behavior_md) = self.behavior.lock().unwrap().clone(); Ok(WorkspaceResponse { status: 200, body: serde_json::to_string( - &server_api::SubjektivMemoryBackendResponse::ResidentSummary( - memory::backend::MemoryResidentSummaryOutput { - availability: self.availability, - content: self.content.clone(), + &server_api::SubjektivMemoryBackendResponse::ResidentContext( + server_api::SubjektivResidentContextOutput { + behavior_md, + behavior_revision, + memory_surface: memory::backend::MemoryResidentSummaryOutput { + availability: self.availability, + content: self.content.clone(), + }, }, ), ) @@ -1511,13 +1696,19 @@ permission = "write" memory::backend::MemoryResidentSummaryAvailability::Ready, Some("current subject surface"), )); - let ready = ordinary_subjektiv_resident_summary_source(&manifest, ready_client.clone()) - .unwrap() - .unwrap(); - assert_eq!( - ready.load().await, - SystemPromptContribution::Ready("current subject surface".into()) - ); + let ready = ordinary_subjektiv_resident_summary_source( + &manifest, + ready_client.clone(), + test_prompts(), + ) + .unwrap() + .unwrap(); + let loaded = ready.load().await; + let SystemPromptContribution::Ready(loaded) = loaded else { + panic!("subject resident context must be present"); + }; + assert!(loaded.contains("Be deliberate.")); + assert!(loaded.contains("current subject surface")); assert_eq!( ready_client.paths.lock().unwrap().clone(), vec!["/api/w/workspace/subjektiv/memory".to_string()] @@ -1531,13 +1722,18 @@ permission = "write" memory::backend::MemoryResidentSummaryAvailability::Ready, None, )); - let ready_empty = ordinary_subjektiv_resident_summary_source(&manifest, ready_empty_client) - .unwrap() - .unwrap(); - assert_eq!( - ready_empty.load().await, - SystemPromptContribution::Ready(READY_EMPTY_RESIDENT_SURFACE.into()) - ); + let ready_empty = ordinary_subjektiv_resident_summary_source( + &manifest, + ready_empty_client, + test_prompts(), + ) + .unwrap() + .unwrap(); + let SystemPromptContribution::Ready(ready_empty) = ready_empty.load().await else { + panic!("empty surface must retain Subject behavior"); + }; + assert!(ready_empty.contains("Be deliberate.")); + assert!(ready_empty.contains("intentionally empty")); for availability in [ memory::backend::MemoryResidentSummaryAvailability::Ungenerated, @@ -1550,22 +1746,107 @@ permission = "write" availability, Some("must not be injected"), )), + test_prompts(), ) .unwrap() .unwrap(); - assert_eq!(source.load().await, SystemPromptContribution::Unavailable); + let SystemPromptContribution::Ready(loaded) = source.load().await else { + panic!("Subject behavior must survive unavailable surfaces"); + }; + assert!(loaded.contains("Be deliberate.")); + assert!(!loaded.contains("must not be injected")); } } + #[tokio::test] + async fn subject_resident_source_emits_only_changed_behavior_revisions() { + let manifest = subject_manifest("workspace"); + let client = Arc::new(SubjectResidentClient::new( + memory::backend::MemoryResidentSummaryAvailability::Ungenerated, + None, + )); + let source = + ordinary_subjektiv_resident_summary_source(&manifest, client.clone(), test_prompts()) + .unwrap() + .unwrap(); + + let initial = source.load().await; + assert!(matches!(initial, SystemPromptContribution::Ready(_))); + assert!( + source + .load_changed_resident_context() + .await + .unwrap() + .is_none() + ); + + *client.behavior.lock().unwrap() = (4, "Ask when uncertain.".to_string()); + let refresh = source + .load_changed_resident_context() + .await + .unwrap() + .expect("changed behavior must produce one refresh"); + assert_eq!(refresh.revision, 4); + assert!(refresh.body.contains("Ask when uncertain.")); + source.confirm_resident_context_revision(refresh.revision); + assert!( + source + .load_changed_resident_context() + .await + .unwrap() + .is_none() + ); + + *client.behavior.lock().unwrap() = (5, String::new()); + let cleared = source + .load_changed_resident_context() + .await + .unwrap() + .expect("cleared behavior must produce one refresh"); + assert_eq!(cleared.revision, 5); + assert!(cleared.body.contains("No user-managed behavior is set")); + source.confirm_resident_context_revision(cleared.revision); + assert!( + source + .load_changed_resident_context() + .await + .unwrap() + .is_none() + ); + + source.invalidate_resident_context_representation(); + let replay_after_rewrite = source + .load_changed_resident_context() + .await + .unwrap() + .expect("history rewrite must require a durable current representation"); + assert_eq!(replay_after_rewrite.revision, 5); + assert!( + replay_after_rewrite + .body + .contains("No user-managed behavior is set") + ); + // Initial load and every live update/clear/replay probe retain the + // target-side bounded resident request contract. + assert_eq!( + client.timeouts.lock().unwrap().as_slice(), + &[RESIDENT_SUMMARY_REQUEST_TIMEOUT; 7] + ); + } + #[tokio::test] async fn subject_resident_source_fails_closed_without_trusted_subject_scope() { let manifest = subject_manifest("workspace"); let client = Arc::new(SubjectResidentClient::denied()); - let source = ordinary_subjektiv_resident_summary_source(&manifest, client.clone()) - .unwrap() - .unwrap(); + let source = + ordinary_subjektiv_resident_summary_source(&manifest, client.clone(), test_prompts()) + .unwrap() + .unwrap(); - assert_eq!(source.load().await, SystemPromptContribution::Unavailable); + let SystemPromptContribution::Ready(failure) = source.load().await else { + panic!("fetch failure must be explicit in resident context"); + }; + assert!(failure.contains("could not fetch")); assert_eq!( client.paths.lock().unwrap().clone(), vec!["/api/w/workspace/subjektiv/memory".to_string()] @@ -1593,7 +1874,7 @@ permission = "write" ) .unwrap(); assert!( - ordinary_subjektiv_resident_summary_source(&disabled, test_client()) + ordinary_subjektiv_resident_summary_source(&disabled, test_client(), test_prompts()) .unwrap() .is_none() ); @@ -1605,22 +1886,30 @@ permission = "write" Some("policy-only surface must not load"), )); assert!( - ordinary_subjektiv_resident_summary_source(&policy_only, unattached_client.clone()) - .unwrap() - .is_none() + ordinary_subjektiv_resident_summary_source( + &policy_only, + unattached_client.clone(), + test_prompts() + ) + .unwrap() + .is_none() ); assert!(unattached_client.paths.lock().unwrap().is_empty()); let foreign = subject_manifest("other-workspace"); - assert!(ordinary_subjektiv_resident_summary_source(&foreign, test_client()).is_err()); + assert!( + ordinary_subjektiv_resident_summary_source(&foreign, test_client(), test_prompts()) + .is_err() + ); } #[test] fn subject_restore_refresh_is_suppressed_for_internal_hosts() { let manifest = subject_manifest("workspace"); - let source = ordinary_subjektiv_resident_summary_source(&manifest, test_client()) - .unwrap() - .unwrap(); + let source = + ordinary_subjektiv_resident_summary_source(&manifest, test_client(), test_prompts()) + .unwrap() + .unwrap(); let prompts = Arc::new(ArcSwap::from( crate::prompt::catalog::PromptCatalog::builtins_only().unwrap(), )); @@ -1634,9 +1923,10 @@ permission = "write" #[test] fn subject_restore_refresh_feature_declares_only_restore_lifecycle() { let manifest = subject_manifest("workspace"); - let source = ordinary_subjektiv_resident_summary_source(&manifest, test_client()) - .unwrap() - .unwrap(); + let source = + ordinary_subjektiv_resident_summary_source(&manifest, test_client(), test_prompts()) + .unwrap() + .unwrap(); let feature = SubjektivResidentRestoreRefreshFeature::new( source, Arc::new(ArcSwap::from( @@ -1647,8 +1937,13 @@ permission = "write" let descriptor = feature.descriptor(); assert!(descriptor.tools.is_empty()); - assert_eq!(descriptor.hooks.len(), 1); + assert_eq!(descriptor.hooks.len(), 3); assert_eq!(descriptor.hooks[0].point, FeatureHookPoint::WorkerRestored); + assert_eq!(descriptor.hooks[1].point, FeatureHookPoint::PreLlmRequest); + assert_eq!( + descriptor.hooks[2].point, + FeatureHookPoint::BeforeSessionRewrite + ); } #[test] diff --git a/crates/worker/src/feature/builtin/memory_surface_lifecycle.rs b/crates/worker/src/feature/builtin/memory_surface_lifecycle.rs index 7e11ac8ee..657d9b96e 100644 --- a/crates/worker/src/feature/builtin/memory_surface_lifecycle.rs +++ b/crates/worker/src/feature/builtin/memory_surface_lifecycle.rs @@ -104,6 +104,23 @@ impl SubjektivSurfaceLifecycleFeature { if !lifecycle_enabled || !dedicated { return Ok(None); } + Self::for_job( + workspace_client, + manifest, + client, + prompts, + workspace_context, + ) + .map(Some) + } + + pub(crate) fn for_job( + workspace_client: Arc, + manifest: WorkerManifest, + client: Box, + prompts: Arc>, + workspace_context: WorkerWorkspaceContext, + ) -> std::io::Result { manifest .feature .subjektiv @@ -121,7 +138,7 @@ impl SubjektivSurfaceLifecycleFeature { "subjektiv surface generation requires Backend Workspace API authority", )); } - Ok(Some(Self { + Ok(Self { task: SubjektivSurfaceLifecycleTask { workspace_client, manifest, @@ -129,7 +146,18 @@ impl SubjektivSurfaceLifecycleFeature { prompts, workspace_context, }, - })) + }) + } + + /// The Job tool awaits this clean-context lifecycle before submitting its result. + pub(crate) async fn complete_for_job( + &self, + cancellation: BackgroundTaskCancellation, + reusable_surface: Option<&serde_json::Value>, + ) -> Result { + self.task + .generate(None, cancellation, reusable_surface) + .await } } @@ -182,80 +210,125 @@ impl FeatureBackgroundTask for SubjektivSurfaceLifecycleTask { context: BackgroundTaskContext, cancellation: BackgroundTaskCancellation, ) -> Result<(), HookError> { + self.generate(Some(&context), cancellation, None) + .await + .map(|_| ()) + } +} + +impl SubjektivSurfaceLifecycleTask { + async fn generate( + &self, + context: Option<&BackgroundTaskContext>, + cancellation: BackgroundTaskCancellation, + reusable_surface: Option<&serde_json::Value>, + ) -> Result { for attempt in 0..MAX_GENERATION_ATTEMPTS { - context.generation_fence.ensure_current()?; + if let Some(context) = context { + context.generation_fence.ensure_current()?; + } if cancellation.is_cancelled() { - return Ok(()); + return Err(HookError::new( + HookErrorCategory::Cancelled, + "surface generation cancelled", + )); } let generation = self .workspace_client .prepare_subjektiv_memory_surface() .await .map_err(surface_hook_error)?; + if context.is_none() { + if let Some(snapshot_id) = &generation.current_snapshot_id { + return Ok(serde_json::json!({ + "availability": "ready", "generation_id": generation.generation_id, + "store_revision": generation.store_revision, "snapshot_id": snapshot_id, + })); + } + } + // Preparation is Backend-fenced to the live attempt and immutable batch. + // Only a ready snapshot is immutable within its store revision. Never + // reuse a failed marker, which a later generation can replace. + if let Some(surface) = reusable_surface.filter(|surface| { + surface["availability"] == "ready" + && surface["store_revision"].as_u64() == Some(generation.store_revision) + }) { + return Ok(surface.clone()); + } if generation.materials.is_empty() && generation.active_memory_count > 0 { - self.record_failure(&generation.generation_id, "input_budget_exhausted") + return self + .record_failure(&generation, "input_budget_exhausted", context.is_none()) .await; - tracing::warn!( - active_memory_count = generation.active_memory_count, - "subject Memory surface materials could not fit the bounded input" - ); - return Ok(()); } let points = if generation.materials.is_empty() { Vec::new() } else { - match self.edit_surface(&generation, cancellation.clone()).await { - Ok(points) => points, - Err(failure) => { - self.record_failure(&generation.generation_id, failure.reason_code) - .await; + match tokio::time::timeout( + TASK_TIMEOUT, + self.edit_surface(&generation, cancellation.clone()), + ) + .await + { + Ok(Ok(points)) => points, + Ok(Err(failure)) => { + if cancellation.is_cancelled() { + return Err(HookError::new( + HookErrorCategory::Cancelled, + "surface generation cancelled", + )); + } tracing::warn!(error = %failure.error, reason_code = failure.reason_code, "subject Memory surface editor failed"); - return Ok(()); + return self + .record_failure(&generation, failure.reason_code, context.is_none()) + .await; + } + Err(_) => { + return self + .record_failure(&generation, "editor_timeout", context.is_none()) + .await; } } }; - context.generation_fence.ensure_current()?; - if cancellation.is_cancelled() { - return Ok(()); + if let Some(context) = context { + context.generation_fence.ensure_current()?; } - let publish = self + if cancellation.is_cancelled() { + return Err(HookError::new( + HookErrorCategory::Cancelled, + "surface generation cancelled", + )); + } + match self .workspace_client .publish_subjektiv_memory_surface(server_api::SubjektivSurfacePublishRequest { generation_id: generation.generation_id.clone(), points, }) - .await; - match publish { + .await + { Ok(output) => { - tracing::debug!( - snapshot_id = output.snapshot_id, - store_revision = output.built_from_store_revision, - empty = output.empty, - "published subject Memory surface" - ); - return Ok(()); + return Ok(serde_json::json!({ + "availability": "ready", "generation_id": generation.generation_id, + "store_revision": output.built_from_store_revision, "snapshot_id": output.snapshot_id, + })); } Err(WorkspaceMemoryBackendError::Http { status, .. }) if status == reqwest::StatusCode::CONFLICT && attempt + 1 < MAX_GENERATION_ATTEMPTS => { - // Confirmed Memory moved while editing. Discard the stale - // output and rebuild once from a fresh bounded generation. continue; } Err(error) => { - self.record_failure(&generation.generation_id, "publish_failed") - .await; tracing::warn!(%error, "subject Memory surface publication failed"); - return Ok(()); + return self + .record_failure(&generation, "publish_failed", context.is_none()) + .await; } } } - Ok(()) + unreachable!("bounded generation loop always returns on its last attempt") } -} -impl SubjektivSurfaceLifecycleTask { async fn edit_surface( &self, generation: &server_api::SubjektivSurfacePrepareResponse, @@ -344,17 +417,39 @@ impl SubjektivSurfaceLifecycleTask { .map_err(SurfaceEditorFailure::editor) } - async fn record_failure(&self, generation_id: &str, reason_code: &str) { - if let Err(error) = self + async fn record_failure( + &self, + generation: &server_api::SubjektivSurfacePrepareResponse, + reason_code: &str, + job_confirmation_required: bool, + ) -> Result { + let failure = self .workspace_client .fail_subjektiv_memory_surface(server_api::SubjektivSurfaceFailureRequest { - generation_id: generation_id.to_string(), + generation_id: generation.generation_id.clone(), reason_code: reason_code.to_string(), }) .await + .map_err(surface_hook_error)?; + // Success must attest the actual persisted marker, not merely receipt of + // a failure notification. A protected ready surface or moved revision + // must yield a Backend rejection/non-failed status, never a failed claim. + let confirmed_status = if job_confirmation_required { + "failed_confirmed" + } else { + "failed" + }; + if failure.status != confirmed_status || failure.store_revision != generation.store_revision { - tracing::debug!(%error, "could not record subject Memory surface failure"); + return Err(HookError::new( + HookErrorCategory::Internal, + "Backend did not confirm a failed surface for this generation's revision; no Job result was submitted", + )); } + Ok(serde_json::json!({ + "availability": "failed", "generation_id": generation.generation_id, + "store_revision": failure.store_revision, "reason_code": reason_code, + })) } } @@ -537,6 +632,7 @@ mod tests { ) -> server_api::SubjektivSurfacePrepareResponse { server_api::SubjektivSurfacePrepareResponse { generation_id: "generation-1".into(), + current_snapshot_id: None, store_revision: 1, active_memory_count: materials.len(), materials, @@ -561,6 +657,13 @@ mod tests { prepare_calls: AtomicUsize, publish_calls: AtomicUsize, failure_reasons: Mutex>, + job_results: Mutex>, + result_responses: + Mutex>>, + current_revision: AtomicUsize, + failure_status: Mutex, + failure_revision: AtomicUsize, + ready_snapshot: Mutex>, } impl SurfaceWorkspaceClient { @@ -571,6 +674,12 @@ mod tests { prepare_calls: AtomicUsize::new(0), publish_calls: AtomicUsize::new(0), failure_reasons: Mutex::new(Vec::new()), + job_results: Mutex::new(Vec::new()), + result_responses: Mutex::new(Default::default()), + current_revision: AtomicUsize::new(0), + failure_status: Mutex::new("failed".into()), + failure_revision: AtomicUsize::new(0), + ready_snapshot: Mutex::new(None), } } @@ -581,6 +690,15 @@ mod tests { } } + fn revision(&self, fallback: usize) -> u64 { + let current = self.current_revision.load(Ordering::SeqCst); + if current == 0 { + fallback as u64 + } else { + current as u64 + } + } + fn response(value: server_api::SubjektivMemoryBackendResponse) -> WorkspaceResponse { WorkspaceResponse { status: 200, @@ -606,6 +724,28 @@ mod tests { &self, request: WorkspaceRequest, ) -> Result { + if request.path.ends_with("/workers/self/backend-job-result") { + assert!( + self.publish_calls.load(Ordering::SeqCst) > 0 + || !self.failure_reasons.lock().unwrap().is_empty(), + "result must follow publication or recorded failure" + ); + self.job_results + .lock() + .unwrap() + .push(serde_json::from_str(request.body.as_deref().unwrap()).unwrap()); + return self + .result_responses + .lock() + .unwrap() + .pop_front() + .unwrap_or_else(|| { + Ok(WorkspaceResponse { + status: 200, + body: "{\"replayed\":false}".into(), + }) + }); + } let request: server_api::SubjektivMemoryBackendRequest = serde_json::from_str(request.body.as_deref().unwrap_or_default()) .map_err(|error| WorkspaceClientError::Request(error.to_string()))?; @@ -615,7 +755,8 @@ mod tests { let mut prepared = generation(self.input_token_budget, vec![material("memory-1", 32)]); prepared.generation_id = format!("generation-{call}"); - prepared.store_revision = call as u64; + prepared.store_revision = self.revision(call); + prepared.current_snapshot_id = self.ready_snapshot.lock().unwrap().clone(); Ok(Self::response( server_api::SubjektivMemoryBackendResponse::SurfacePrepared(prepared), )) @@ -633,7 +774,7 @@ mod tests { server_api::SubjektivMemoryBackendResponse::SurfacePublished( server_api::SubjektivSurfacePublishResponse { snapshot_id: format!("surface-{call}"), - built_from_store_revision: call as u64, + built_from_store_revision: self.revision(call), empty: false, }, ), @@ -644,8 +785,13 @@ mod tests { Ok(Self::response( server_api::SubjektivMemoryBackendResponse::SurfaceFailed( server_api::SubjektivSurfaceFailureResponse { - store_revision: self.prepare_calls.load(Ordering::SeqCst) as u64, - status: "failed".into(), + store_revision: if self.failure_revision.load(Ordering::SeqCst) > 0 + { + self.failure_revision.load(Ordering::SeqCst) as u64 + } else { + self.revision(self.prepare_calls.load(Ordering::SeqCst)) + }, + status: self.failure_status.lock().unwrap().clone(), }, ), )) @@ -974,4 +1120,340 @@ permission = "write" let error = bounded_editor_input("system policy", &generation).unwrap_err(); assert!(error.to_string().contains("input budget")); } + #[tokio::test] + async fn job_result_waits_for_surface_and_retries_the_exact_outcome() { + for failed in [false, true] { + let workspace = Arc::new(if failed { + SurfaceWorkspaceClient::with_input_token_budget(1) + } else { + SurfaceWorkspaceClient::new(0) + }); + *workspace.failure_status.lock().unwrap() = "failed_confirmed".into(); + let lifecycle = SubjektivSurfaceLifecycleFeature { + task: test_task(workspace.clone(), Arc::new(AtomicUsize::new(0))), + }; + let mut pending = Vec::new(); + let mut hooks = crate::hook::HookRegistryBuilder::default(); + FeatureRegistryBuilder::new() + .with_module( + crate::feature::builtin::backend_job_result::BackendJobResultFeature::new( + workspace.clone(), + ) + .with_surface(lifecycle), + ) + .install_into_pending(&mut pending, &mut hooks); + let (_, tool) = pending.remove(0)(); + let input = r#"{"result":{"subject_id":"subject-1","candidate_ids":["candidate-1"]}}"#; + for _ in 0..2 { + tool.execute(input, agen::tool::ToolExecutionContext::direct()) + .await + .unwrap(); + } + let results = workspace.job_results.lock().unwrap(); + assert_eq!(results.len(), 2); + assert_eq!(results[0], results[1]); + assert_eq!( + results[0]["result"]["surface"]["availability"], + if failed { "failed" } else { "ready" } + ); + assert_eq!( + results[0]["result"]["surface"]["generation_id"], + "generation-1" + ); + assert_eq!(results[0]["result"]["surface"]["store_revision"], 1); + assert_eq!(workspace.prepare_calls.load(Ordering::SeqCst), 1); + assert!( + tool.execute( + r#"{"result":{"subject_id":"subject-1","candidate_ids":[]}}"#, + agen::tool::ToolExecutionContext::direct() + ) + .await + .is_err() + ); + } + } + + #[tokio::test] + async fn job_uses_backend_confirmed_current_surface_without_regeneration() { + let workspace = Arc::new(SurfaceWorkspaceClient::new(0)); + *workspace.ready_snapshot.lock().unwrap() = Some("already-current".into()); + let calls = Arc::new(AtomicUsize::new(0)); + let task = test_task(workspace.clone(), calls.clone()); + let outcome = task + .generate(None, BackgroundTaskCancellation::default(), None) + .await + .unwrap(); + assert_eq!(outcome["availability"], "ready"); + assert_eq!(outcome["generation_id"], "generation-1"); + assert_eq!(outcome["snapshot_id"], "already-current"); + assert_eq!(calls.load(Ordering::SeqCst), 0); + assert_eq!(workspace.publish_calls.load(Ordering::SeqCst), 0); + assert!(workspace.failure_reasons.lock().unwrap().is_empty()); + } + + #[tokio::test] + async fn job_result_rejects_model_surface_claims_before_generation() { + let workspace = Arc::new(SurfaceWorkspaceClient::new(0)); + let lifecycle = SubjektivSurfaceLifecycleFeature { + task: test_task(workspace.clone(), Arc::new(AtomicUsize::new(0))), + }; + let mut pending = Vec::new(); + let mut hooks = crate::hook::HookRegistryBuilder::default(); + FeatureRegistryBuilder::new() + .with_module( + crate::feature::builtin::backend_job_result::BackendJobResultFeature::new( + workspace.clone(), + ) + .with_surface(lifecycle), + ) + .install_into_pending(&mut pending, &mut hooks); + let (_, tool) = pending.remove(0)(); + assert!(tool.execute(r#"{"result":{"subject_id":"subject-1","candidate_ids":[],"surface":{"availability":"ready"}}}"#, agen::tool::ToolExecutionContext::direct()).await.is_err()); + assert_eq!(workspace.prepare_calls.load(Ordering::SeqCst), 0); + assert!(workspace.job_results.lock().unwrap().is_empty()); + } + + #[derive(Clone)] + struct PendingSurfaceEditorClient(Arc); + #[async_trait] + impl LlmClient for PendingSurfaceEditorClient { + fn clone_boxed(&self) -> Box { + Box::new(self.clone()) + } + async fn stream( + &self, + _request: Request, + ) -> Result> + Send>>, ClientError> + { + self.0.notify_one(); + Ok(Box::pin(futures::stream::pending())) + } + } + + #[tokio::test] + async fn cancelling_job_surface_waits_for_editor_and_never_submits_result() { + let workspace = Arc::new(SurfaceWorkspaceClient::new(0)); + let entered = Arc::new(tokio::sync::Notify::new()); + let mut task = test_task(workspace.clone(), Arc::new(AtomicUsize::new(0))); + task.client = Box::new(PendingSurfaceEditorClient(entered.clone())); + let mut pending = Vec::new(); + let mut hooks = crate::hook::HookRegistryBuilder::default(); + FeatureRegistryBuilder::new() + .with_module( + crate::feature::builtin::backend_job_result::BackendJobResultFeature::new( + workspace.clone(), + ) + .with_surface(SubjektivSurfaceLifecycleFeature { task }), + ) + .install_into_pending(&mut pending, &mut hooks); + let (_, tool) = pending.remove(0)(); + let running_tool = tool.clone(); + let context = agen::tool::ToolExecutionContext::new("result-call", "batch", 0); + let running_context = context.clone(); + let running = tokio::spawn(async move { + running_tool + .execute( + r#"{"result":{"subject_id":"subject-1","candidate_ids":[]}}"#, + running_context, + ) + .await + }); + tokio::time::timeout(Duration::from_secs(5), entered.notified()) + .await + .unwrap(); + tool.cancel_execution(&context).await.unwrap(); + let outcome = tokio::time::timeout(Duration::from_secs(5), running) + .await + .unwrap() + .unwrap(); + assert!(matches!(outcome, Err(agen::tool::ToolError::Cancelled(_)))); + assert_eq!(workspace.publish_calls.load(Ordering::SeqCst), 0); + assert!(workspace.job_results.lock().unwrap().is_empty()); + assert!(workspace.failure_reasons.lock().unwrap().is_empty()); + } + fn job_result_tool(workspace: Arc) -> Arc { + *workspace.failure_status.lock().unwrap() = "failed_confirmed".into(); + let mut pending = Vec::new(); + let mut hooks = crate::hook::HookRegistryBuilder::default(); + FeatureRegistryBuilder::new() + .with_module( + crate::feature::builtin::backend_job_result::BackendJobResultFeature::new( + workspace.clone(), + ) + .with_surface(SubjektivSurfaceLifecycleFeature { + task: test_task(workspace, Arc::new(AtomicUsize::new(0))), + }), + ) + .install_into_pending(&mut pending, &mut hooks); + pending.remove(0)().1 + } + + const CORRECT_RESULT: &str = + r#"{"result":{"subject_id":"subject-1","candidate_ids":["candidate-1"]}}"#; + const WRONG_RESULT: &str = r#"{"result":{"subject_id":"wrong-subject","candidate_ids":[]}}"#; + + fn rejection(status: u16) -> Result { + Ok(WorkspaceResponse { + status, + body: "explicit rejection before acceptance".into(), + }) + } + + #[tokio::test] + async fn job_result_definitive_rejection_allows_correction_and_reuses_current_ready_surface() { + for status in [400, 403, 422] { + let workspace = Arc::new(SurfaceWorkspaceClient::new(0)); + workspace.current_revision.store(1, Ordering::SeqCst); + workspace + .result_responses + .lock() + .unwrap() + .push_back(rejection(status)); + let tool = job_result_tool(workspace.clone()); + assert!( + tool.execute(WRONG_RESULT, agen::tool::ToolExecutionContext::direct()) + .await + .is_err() + ); + tool.execute(CORRECT_RESULT, agen::tool::ToolExecutionContext::direct()) + .await + .unwrap(); + let results = workspace.job_results.lock().unwrap(); + assert_eq!(results.len(), 2); + assert_ne!( + results[0]["result"]["subject_id"], + results[1]["result"]["subject_id"] + ); + assert_eq!( + results[0]["result"]["surface"], + results[1]["result"]["surface"] + ); + assert_eq!( + workspace.prepare_calls.load(Ordering::SeqCst), + 2, + "reuse must verify current revision" + ); + assert_eq!( + workspace.publish_calls.load(Ordering::SeqCst), + 1, + "correction must not duplicate publication" + ); + drop(results); + assert!( + tool.execute(WRONG_RESULT, agen::tool::ToolExecutionContext::direct()) + .await + .is_err(), + "accepted results stay sealed" + ); + } + } + + #[tokio::test] + async fn job_result_rejection_rebuilds_conflicting_or_stale_surfaces() { + for status in [400, 409] { + let workspace = Arc::new(SurfaceWorkspaceClient::new(0)); + workspace.current_revision.store(1, Ordering::SeqCst); + workspace + .result_responses + .lock() + .unwrap() + .push_back(rejection(status)); + let tool = job_result_tool(workspace.clone()); + assert!( + tool.execute(CORRECT_RESULT, agen::tool::ToolExecutionContext::direct()) + .await + .is_err() + ); + workspace.current_revision.store(2, Ordering::SeqCst); + tool.execute(CORRECT_RESULT, agen::tool::ToolExecutionContext::direct()) + .await + .unwrap(); + let results = workspace.job_results.lock().unwrap(); + assert_eq!(results.len(), 2); + assert_eq!(results[0]["result"]["surface"]["store_revision"], 1); + assert_eq!(results[1]["result"]["surface"]["store_revision"], 2); + assert_ne!( + results[0]["result"]["surface"]["generation_id"], + results[1]["result"]["surface"]["generation_id"] + ); + assert_eq!(workspace.prepare_calls.load(Ordering::SeqCst), 2); + assert_eq!(workspace.publish_calls.load(Ordering::SeqCst), 2); + } + } + + #[tokio::test] + async fn job_result_ambiguous_acceptance_never_reopens_even_after_later_rejection() { + for first in [ + Err(WorkspaceClientError::Request( + "accepted but response lost".into(), + )), + rejection(500), + ] { + let workspace = Arc::new(SurfaceWorkspaceClient::new(0)); + workspace + .result_responses + .lock() + .unwrap() + .extend([first, rejection(403)]); + let tool = job_result_tool(workspace.clone()); + assert!( + tool.execute(CORRECT_RESULT, agen::tool::ToolExecutionContext::direct()) + .await + .is_err() + ); + assert!( + tool.execute(WRONG_RESULT, agen::tool::ToolExecutionContext::direct()) + .await + .is_err() + ); + assert!( + tool.execute(CORRECT_RESULT, agen::tool::ToolExecutionContext::direct()) + .await + .is_err() + ); + assert!( + tool.execute(WRONG_RESULT, agen::tool::ToolExecutionContext::direct()) + .await + .is_err() + ); + tool.execute(CORRECT_RESULT, agen::tool::ToolExecutionContext::direct()) + .await + .unwrap(); + let results = workspace.job_results.lock().unwrap(); + assert_eq!(results.len(), 3); + assert_eq!(results[0], results[1]); + assert_eq!(results[1], results[2]); + assert_eq!(workspace.prepare_calls.load(Ordering::SeqCst), 1); + assert_eq!(workspace.publish_calls.load(Ordering::SeqCst), 1); + } + } + + #[tokio::test] + async fn job_result_never_submits_unconfirmed_failure_or_wrong_revision() { + for (status, revision) in [ + ("failed", 0), + ("ready", 0), + ("stale", 0), + ("failed_confirmed", 2), + ] { + let workspace = Arc::new(SurfaceWorkspaceClient::with_input_token_budget(1)); + workspace.current_revision.store(1, Ordering::SeqCst); + let tool = job_result_tool(workspace.clone()); + *workspace.failure_status.lock().unwrap() = status.into(); + workspace.failure_revision.store(revision, Ordering::SeqCst); + assert!( + tool.execute(CORRECT_RESULT, agen::tool::ToolExecutionContext::direct()) + .await + .is_err() + ); + assert!(workspace.job_results.lock().unwrap().is_empty()); + // Nothing was dispatched/sealed: a real confirmed outcome remains retryable. + *workspace.failure_status.lock().unwrap() = "failed_confirmed".into(); + workspace.failure_revision.store(0, Ordering::SeqCst); + tool.execute(CORRECT_RESULT, agen::tool::ToolExecutionContext::direct()) + .await + .unwrap(); + assert_eq!(workspace.job_results.lock().unwrap().len(), 1); + } + } } diff --git a/crates/worker/src/feature/builtin/merge_request.rs b/crates/worker/src/feature/builtin/merge_request.rs index afc5e08f7..88460b2d7 100644 --- a/crates/worker/src/feature/builtin/merge_request.rs +++ b/crates/worker/src/feature/builtin/merge_request.rs @@ -5,9 +5,9 @@ use crate::feature::{ }; use crate::permission::permission_action_for; use crate::wip::{ - WipCallContext, WipDynamicItem, WipDynamicItemResolver, WipDynamicMount, WipFeatureRoute, - WipMountError, WipMountRegistry, WipOperationError, WipOperationHandler, WipOperationOutput, - WipProjection, WipProjectionKind, json_to_wip, wip_to_json, + WipCallContext, WipDynamicItem, WipDynamicItemResolver, WipDynamicMount, WipMountError, + WipMountRegistry, WipNamespaceRoute, WipOperationError, WipOperationHandler, + WipOperationOutput, WipProjection, WipProjectionKind, json_to_wip, wip_to_json, }; use crate::worker::{WorkspaceClient, WorkspaceRequest, WorkspaceRequestMethod}; use agen::tool::{Tool, ToolError, ToolExecutionContext, ToolMeta, ToolOutput}; @@ -529,7 +529,7 @@ fn native_merge_request_tools( let (meta, tool) = definition(client.clone(), kind)(); jsonschema::validator_for(&meta.input_schema).map_err(|error| { WipMountError::InvalidProjection { - route: "/features/merge-request/merge-requests".into(), + route: "/merge-requests".into(), message: format!("Merge Request tool schema cannot be retained: {error}"), } })?; @@ -562,9 +562,9 @@ pub fn mount_workspace_http_merge_request_wip( client: Arc, config: MergeRequestFeatureConfig, permissions: Option, - feature_route: &WipFeatureRoute, + namespace_route: &WipNamespaceRoute, ) -> Result<(), WipMountError> { - let collection_route = feature_route.child("merge-requests")?; + let collection_route = namespace_route.root().to_string(); let tools = native_merge_request_tools(client, config)?; let claimed_tools = tools .iter() @@ -1049,7 +1049,7 @@ fn merge_request_operation_declaration( .get("properties") .and_then(Value::as_object) .ok_or_else(|| WipMountError::InvalidProjection { - route: "/features/merge-request/merge-requests".into(), + route: "/merge-requests".into(), message: format!( "Merge Request tool `{}` schema has no properties", tool.name @@ -1474,7 +1474,9 @@ mod tests { #[test] fn native_mount_uses_host_route_and_accepts_only_one_canonical_item_segment() { let mut registry = WipMountRegistry::new(); - let feature_route = registry.allocate_feature_route("merge-request").unwrap(); + let namespace_route = registry + .allocate_namespace("merge-request", "merge-requests") + .unwrap(); mount_workspace_http_merge_request_wip( &mut registry, Arc::new(TestWorkspaceHttpClient::new("workspace", "http://unused")), @@ -1484,17 +1486,14 @@ mod tests { ..Default::default() }, None, - &feature_route, + &namespace_route, ) .unwrap(); - assert_eq!( - registry.routes().collect::>(), - ["/features/merge-request/merge-requests"] - ); + assert_eq!(registry.routes().collect::>(), ["/merge-requests"]); let resolver = MergeRequestItemResolver { tools: HashMap::new(), permissions: None, - collection_route: "/features/merge-request/merge-requests".into(), + collection_route: "/merge-requests".into(), revisions: Arc::new(Mutex::new(MergeRequestRevisionState::default())), }; assert!( @@ -1645,7 +1644,7 @@ mod tests { let resolver = MergeRequestItemResolver { tools: HashMap::new(), permissions: None, - collection_route: "/features/merge-request/merge-requests".into(), + collection_route: "/merge-requests".into(), revisions: Arc::clone(&revisions), }; let first = resolver.resolve("MR-1").unwrap().object.validator; diff --git a/crates/worker/src/feature/builtin/objective.rs b/crates/worker/src/feature/builtin/objective.rs index 9f30111c6..3b86d0935 100644 --- a/crates/worker/src/feature/builtin/objective.rs +++ b/crates/worker/src/feature/builtin/objective.rs @@ -21,9 +21,9 @@ use wip_protocol::{ use crate::permission::permission_action_for; use crate::wip::{ - WipCallContext, WipDynamicItem, WipDynamicItemResolver, WipDynamicMount, WipFeatureRoute, - WipMountError, WipMountRegistry, WipOperationError, WipOperationHandler, WipOperationOutput, - WipProjection, WipProjectionKind, json_to_wip, wip_to_json, + WipCallContext, WipDynamicItem, WipDynamicItemResolver, WipDynamicMount, WipMountError, + WipMountRegistry, WipNamespaceRoute, WipOperationError, WipOperationHandler, + WipOperationOutput, WipProjection, WipProjectionKind, json_to_wip, wip_to_json, }; use crate::worker::{WorkspaceClient, WorkspaceRequest, WorkspaceRequestMethod}; @@ -551,9 +551,9 @@ pub fn mount_workspace_http_objective_wip( registry: &mut WipMountRegistry, client: Arc, permissions: Option, - feature_route: &WipFeatureRoute, + namespace_route: &WipNamespaceRoute, ) -> Result<(), WipMountError> { - let collection_route = feature_route.child("objectives")?; + let collection_route = namespace_route.root().to_string(); let backend = WorkspaceHttpObjectiveBackend::new(client); let revisions = Arc::new(Mutex::new(ObjectiveRevisionState::default())); let collection_descriptor = objective_collection_descriptor(); @@ -1775,7 +1775,9 @@ mod tests { })); let mut registry = WipMountRegistry::new(); - let feature_route = registry.allocate_feature_route("objective").unwrap(); + let namespace_route = registry + .allocate_namespace("objective", "objectives") + .unwrap(); mount_workspace_http_objective_wip( &mut registry, Arc::new(crate::worker::TestWorkspaceHttpClient::new( @@ -1783,18 +1785,16 @@ mod tests { "http://backend", )), None, - &feature_route, + &namespace_route, ) .unwrap(); - assert_eq!( - registry.routes().collect::>(), - ["/features/objective/objectives"] + assert_eq!(registry.routes().collect::>(), ["/objectives"]); + assert_eq!(namespace_route.root(), "/objectives"); + assert!( + registry + .allocate_namespace("objective", "../tickets") + .is_err() ); - assert_eq!( - feature_route.child("objectives").unwrap(), - "/features/objective/objectives" - ); - assert!(feature_route.child("../tickets").is_err()); } #[test] @@ -1805,7 +1805,7 @@ mod tests { crate::worker::TestWorkspaceHttpClient::new("workspace", "http://backend"), )), permissions: None, - collection_route: "/features/objective/objectives".into(), + collection_route: "/objectives".into(), revisions: Arc::clone(&revisions), }; let initial = resolver.resolve("O-3").unwrap(); @@ -1971,7 +1971,7 @@ mod tests { let collection = ObjectiveCollectionWipHandler { backend, permissions: None, - collection_route: "/features/objective/objectives".into(), + collection_route: "/objectives".into(), revisions: Arc::new(Mutex::new(ObjectiveRevisionState::default())), }; let result = collection @@ -2013,7 +2013,7 @@ mod tests { let collection = ObjectiveCollectionWipHandler { backend: backend.clone(), permissions: None, - collection_route: "/features/objective/objectives".into(), + collection_route: "/objectives".into(), revisions: Arc::clone(&revisions), }; let context = || WipCallContext { @@ -2170,7 +2170,7 @@ mod tests { crate::worker::TestWorkspaceHttpClient::new("workspace", base_url), )), permissions: None, - collection_route: "/features/objective/objectives".into(), + collection_route: "/objectives".into(), revisions: Arc::new(Mutex::new(ObjectiveRevisionState::default())), }; let output = match handler @@ -2191,10 +2191,7 @@ mod tests { let output = crate::wip::wip_to_json(&output.value).unwrap(); assert_eq!(output["next_cursor"], "next-page"); assert_eq!(output["has_more"], true); - assert_eq!( - output["objectives"][0]["path"], - "/features/objective/objectives/O-3" - ); + assert_eq!(output["objectives"][0]["path"], "/objectives/O-3"); assert_eq!(output["objectives"][0]["linked_tickets"][0], "T-7"); assert!(!output.to_string().contains("00001OBJECTIVE")); assert!(!output.to_string().contains("00001TICKET")); @@ -2243,7 +2240,7 @@ mod tests { let resolver = ObjectiveItemResolver { backend: backend.clone(), permissions: None, - collection_route: "/features/objective/objectives".into(), + collection_route: "/objectives".into(), revisions: Arc::clone(&revisions), }; let before = resolver.resolve("O-3").unwrap().object.validator; diff --git a/crates/worker/src/feature/builtin/ticket.rs b/crates/worker/src/feature/builtin/ticket.rs index 8a82ab6f3..f740ae6d0 100644 --- a/crates/worker/src/feature/builtin/ticket.rs +++ b/crates/worker/src/feature/builtin/ticket.rs @@ -34,9 +34,9 @@ use crate::feature::{ }; use crate::permission::permission_action_for; use crate::wip::{ - WipCallContext, WipDynamicItem, WipDynamicItemResolver, WipDynamicMount, WipFeatureRoute, - WipMountError, WipMountRegistry, WipOperationError, WipOperationHandler, WipOperationOutput, - WipProjection, WipProjectionKind, json_to_wip, wip_to_json, + WipCallContext, WipDynamicItem, WipDynamicItemResolver, WipDynamicMount, WipMountError, + WipMountRegistry, WipNamespaceRoute, WipOperationError, WipOperationHandler, + WipOperationOutput, WipProjection, WipProjectionKind, json_to_wip, wip_to_json, }; use crate::worker::{WorkspaceClient, WorkspaceRequest, WorkspaceRequestMethod}; use agen::tool::{Tool, ToolError, ToolExecutionContext, ToolMeta, ToolOutput}; @@ -1377,7 +1377,7 @@ fn native_ticket_tools( let (meta, tool) = definition(); let projection = native_ticket_operation(&meta.name).ok_or_else(|| { WipMountError::InvalidProjection { - route: "/features/ticket/tickets".into(), + route: "/tickets".into(), message: format!( "enabled Ticket tool `{}` has no native projection", meta.name @@ -1386,7 +1386,7 @@ fn native_ticket_tools( })?; jsonschema::validator_for(&meta.input_schema).map_err(|error| { WipMountError::InvalidProjection { - route: "/features/ticket/tickets".into(), + route: "/tickets".into(), message: format!("Ticket tool schema cannot be retained: {error}"), } })?; @@ -1418,9 +1418,9 @@ pub fn mount_workspace_http_ticket_wip( client: Arc, access: TicketFeatureAccess, permissions: Option, - feature_route: &WipFeatureRoute, + namespace_route: &WipNamespaceRoute, ) -> Result<(), WipMountError> { - let collection_route = feature_route.child("tickets")?; + let collection_route = namespace_route.root().to_string(); let tools = native_ticket_tools(client, access)?; let claimed_tools = tools .iter() @@ -1920,7 +1920,7 @@ fn ticket_operation_declaration( .get("properties") .and_then(Value::as_object) .ok_or_else(|| WipMountError::InvalidProjection { - route: "/features/ticket/tickets".into(), + route: "/tickets".into(), message: format!("Ticket tool `{}` schema has no properties", tool.name), })?; let required = tool @@ -2103,7 +2103,7 @@ mod tests { #[test] fn native_ticket_mount_uses_host_route_and_rejects_non_ticket_children() { let mut registry = WipMountRegistry::new(); - let feature_route = registry.allocate_feature_route("ticket").unwrap(); + let namespace_route = registry.allocate_namespace("ticket", "tickets").unwrap(); mount_workspace_http_ticket_wip( &mut registry, Arc::new(crate::worker::TestWorkspaceHttpClient::new( @@ -2112,20 +2112,21 @@ mod tests { )), TicketFeatureAccess::review(), None, - &feature_route, + &namespace_route, ) .unwrap(); - assert_eq!( - registry.routes().collect::>(), - ["/features/ticket/tickets"] + assert_eq!(registry.routes().collect::>(), ["/tickets"]); + assert!( + registry + .allocate_namespace("ticket", "../objectives") + .is_err() ); - assert!(feature_route.child("../objectives").is_err()); let revisions = Arc::new(Mutex::new(TicketRevisionState::default())); let resolver = TicketItemResolver { tools: HashMap::new(), permissions: None, - collection_route: "/features/ticket/tickets".into(), + collection_route: "/tickets".into(), revisions, }; assert!(resolver.resolve("T-42").is_some()); @@ -2189,7 +2190,7 @@ mod tests { let resolver = TicketItemResolver { tools: HashMap::new(), permissions: None, - collection_route: "/features/ticket/tickets".into(), + collection_route: "/tickets".into(), revisions: Arc::clone(&revisions), }; let unobserved_internal = resolver.resolve("00001TICKET").unwrap().object.validator; @@ -2243,7 +2244,7 @@ mod tests { let resolver = TicketItemResolver { tools: HashMap::new(), permissions: None, - collection_route: "/features/ticket/tickets".into(), + collection_route: "/tickets".into(), revisions: Arc::clone(&revisions), }; let target_v1 = resolver.resolve("T-43").unwrap().object.validator; diff --git a/crates/worker/src/feature/builtin/workspace_config.rs b/crates/worker/src/feature/builtin/workspace_config.rs new file mode 100644 index 000000000..e847266b5 --- /dev/null +++ b/crates/worker/src/feature/builtin/workspace_config.rs @@ -0,0 +1,333 @@ +//! WIP-only, grant-bound Workspace configuration Feature. No local config store +//! or OS Workdir/session is created by this logical attachment. + +pub mod backend; +mod content; +pub mod wip; + +use std::sync::Arc; +use std::time::Duration; + +use async_trait::async_trait; +use protocol::{ + FeatureInvocation, FeatureInvocationDescriptor, FeatureInvocationIdentity, + FeatureInvocationResult, FeatureInvocationStatus, FeatureInvocationSyntax, + InvocationArgumentDescriptor, InvocationArgumentType, InvocationCompletion, InvocationValue, +}; + +use crate::feature::{ + FeatureDescriptor, FeatureInstallContext, FeatureInstallError, FeatureInvocationContext, + FeatureInvocationHandler, FeatureInvocationHandlerError, FeatureModule, +}; +use crate::worker::WorkspaceClient; + +pub const FEATURE_ID: &str = "workspace-config"; +pub const INVOCATION_ID: &str = "builtin:workspace-config/attach"; +pub const ATTACHMENT_ALIAS: &str = "workspace-config"; +pub const CONTENT_ROOT: &str = "/workspace-config"; +const DEADLINE: Duration = Duration::from_secs(30); + +/// Requested restriction, not an authorization grant. `None` selects the +/// effective Backend grant. Explicit read_write must never silently downgrade. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ConfigAccess { + ReadOnly, + ReadWrite, +} + +impl ConfigAccess { + pub fn as_str(self) -> &'static str { + match self { + Self::ReadOnly => "read_only", + Self::ReadWrite => "read_write", + } + } +} + +fn requested_access(value: Option<&str>) -> Result, ConfigAttachError> { + match value { + None | Some("effective") => Ok(None), + Some("read_only") => Ok(Some(ConfigAccess::ReadOnly)), + Some("read_write") => Ok(Some(ConfigAccess::ReadWrite)), + _ => Err(ConfigAttachError::InvalidRequest), + } +} + +/// Adapter result from the *existing Backend attachment ledger*. No Worker-local +/// attachment registry or authorization state is created from this result. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct ConfigAttachment { + pub connection_id: String, + pub access: ConfigAccess, + pub already_attached: bool, +} + +/// Safe diagnostics only. Raw Backend/transport errors may contain host paths or +/// secrets and must be classified by the adapter, never copied into these results. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum ConfigAttachError { + InvalidRequest, + Denied, + Unavailable, + OutcomeUnknown, +} + +impl ConfigAttachError { + fn message(self) -> &'static str { + match self { + Self::InvalidRequest => "Invalid Workspace config attachment request", + Self::Denied => "Workspace config access denied; explicit access is not downgraded", + Self::Unavailable => "Workspace config attachment unavailable before effects", + Self::OutcomeUnknown => { + "Workspace config attachment outcome unknown; inspect current attachments before retrying" + } + } + } + + fn invocation_error(self) -> FeatureInvocationHandlerError { + if self == Self::OutcomeUnknown { + FeatureInvocationHandlerError::outcome_unknown(self.message()) + } else { + FeatureInvocationHandlerError::failed(self.message()) + } + } +} + +/// Narrow application adapter seam, not a config compatibility Tool/API. +/// +/// The production implementation captures an identity-bound WorkspaceClient and +/// uses the shared typed Backend attach operation. Both validation (including +/// T696 replay/resume) and attach must consult current authority. Attach rechecks +/// authorization atomically with ledger mutation, enforces explicit access and +/// alias identity. The execution key identifies the host receipt, not a grant; +/// Backend alias attach is idempotent under its session lock. Never blind retry. +#[async_trait] +pub trait WorkspaceConfigAttachmentBackend: Send + Sync { + fn is_available(&self) -> bool; + fn validate_access(&self, access: Option) -> Result<(), ConfigAttachError>; + async fn attach( + &self, + idempotency_key: &str, + access: Option, + ) -> Result; +} + +/// Shared slash/self-attach handler. Possession of Workdir tools is not a grant. +#[derive(Clone)] +pub struct WorkspaceConfigFeature { + client: Arc, + backend: Arc, + content_backend: Option>, + wip_mode: bool, + permits: Arc, +} + +impl WorkspaceConfigFeature { + /// Host supplies the actual mode; a user invocation cannot enable WIP or + /// change the Workspace/security context captured by the adapter. + pub fn new( + client: Arc, + backend: Arc, + wip_mode: bool, + ) -> Self { + Self { + client, + backend, + content_backend: None, + wip_mode, + permits: Arc::new(tokio::sync::Semaphore::new(16)), + } + } + + /// Controller registration gate: a flag cannot create Backend authority or + /// turn a Tools/standalone Worker into a Workspace WIP Worker. + pub fn configured( + client: Arc, + enabled: bool, + wip_mode: bool, + ) -> Option { + (enabled + && wip_mode + && client.is_available() + && client + .workspace_id() + .is_some_and(|id| !id.is_empty() && !id.chars().any(char::is_control))) + .then(|| Self::for_workspace(client, true)) + } + + /// Production adapter captures the injected identity-bound Workspace client. + pub fn for_workspace(client: Arc, wip_mode: bool) -> Self { + let backend = Arc::new(backend::WorkspaceConfigBackend::new(client.clone())); + let mut feature = Self::new(client, backend.clone(), wip_mode); + feature.content_backend = Some(backend); + feature + } + + pub fn is_available(&self) -> bool { + self.wip_mode + && self.client.is_available() + && self.client.workspace_id().is_some_and(|id| !id.is_empty()) + && self.backend.is_available() + } + + fn validate_access(&self, access: Option) -> Result<(), ConfigAttachError> { + if !self.is_available() { + return Err(ConfigAttachError::Denied); + } + self.backend.validate_access(access) + } + + /// The only attach execution path used by T696 and native WIP. No router + /// session, command capability, or attachment lifetime is synthesized here. + pub async fn attach( + &self, + idempotency_key: &str, + access: Option, + ) -> Result { + if idempotency_key.is_empty() || idempotency_key.len() > 256 { + return Err(ConfigAttachError::InvalidRequest); + } + self.validate_access(access)?; + let _permit = tokio::time::timeout(DEADLINE, self.permits.acquire()) + .await + .map_err(|_| ConfigAttachError::Unavailable)? + .map_err(|_| ConfigAttachError::Unavailable)?; + // Authority can change while queued. Backend still rechecks on execution. + self.validate_access(access)?; + let result = tokio::time::timeout(DEADLINE, self.backend.attach(idempotency_key, access)) + .await + .map_err(|_| ConfigAttachError::OutcomeUnknown)??; + // The adapter must map only a safe lifetime identity from a validated + // Backend result. Invalid completion metadata cannot imply no effects. + if result.connection_id.is_empty() + || result.connection_id.len() > 256 + || !result + .connection_id + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_')) + || access.is_some_and(|requested| requested != result.access) + { + return Err(ConfigAttachError::OutcomeUnknown); + } + Ok(result) + } +} + +pub fn workspace_config_invocation_descriptor() -> FeatureInvocationDescriptor { + FeatureInvocationDescriptor { + identity: FeatureInvocationIdentity(INVOCATION_ID.into()), + name: FEATURE_ID.into(), + aliases: Vec::new(), + display_name: "Workspace config".into(), + description: "Select this Workspace's logical config attachment before the request reaches the assistant. Backend grants determine access; no host filesystem or command access is added.".into(), + syntax: FeatureInvocationSyntax::Parenthesized, + arguments: vec![InvocationArgumentDescriptor { + name: "access".into(), + position: Some(0), + required: false, + value_type: InvocationArgumentType::Enum { + values: vec!["effective".into(), "read_only".into(), "read_write".into()], + }, + completion: InvocationCompletion::None, + description: Some("Default: effective Backend grant. read_only restricts the connection; explicit read_write fails if unavailable.".into()), + }], + client_adapter: None, + } +} + +impl FeatureModule for WorkspaceConfigFeature { + fn descriptor(&self) -> FeatureDescriptor { + let descriptor = FeatureDescriptor::builtin(FEATURE_ID, "Workspace config") + .with_description("WIP-only logical Workspace config attachment preparation"); + if self.wip_mode && self.client.workspace_id().is_some_and(|id| !id.is_empty()) { + descriptor.with_chat_invocation(workspace_config_invocation_descriptor()) + } else { + descriptor + } + } + + fn install(&self, context: &mut FeatureInstallContext<'_>) -> Result<(), FeatureInstallError> { + if self.wip_mode && self.client.workspace_id().is_some_and(|id| !id.is_empty()) { + context + .chat_invocations() + .register(workspace_config_invocation_descriptor(), self.clone())?; + } + // No tools are declared/installed, including in non-WIP mode. + Ok(()) + } +} + +fn invocation_access( + invocation: &FeatureInvocation, +) -> Result, ConfigAttachError> { + if invocation.identity.0 != INVOCATION_ID + || invocation.name != FEATURE_ID + || invocation.arguments.len() > 1 + { + return Err(ConfigAttachError::InvalidRequest); + } + match invocation.arguments.first() { + None => requested_access(None), + Some(argument) if argument.name == "access" => match &argument.value { + InvocationValue::String(value) => requested_access(Some(value)), + _ => Err(ConfigAttachError::InvalidRequest), + }, + _ => Err(ConfigAttachError::InvalidRequest), + } +} + +#[async_trait] +impl FeatureInvocationHandler for WorkspaceConfigFeature { + fn is_available(&self) -> bool { + WorkspaceConfigFeature::is_available(self) + } + + fn validate( + &self, + invocation: &FeatureInvocation, + ) -> Result<(), FeatureInvocationHandlerError> { + let access = invocation_access(invocation).map_err(ConfigAttachError::invocation_error)?; + self.validate_access(access) + .map_err(ConfigAttachError::invocation_error) + } + + async fn invoke( + &self, + context: FeatureInvocationContext, + invocation: &FeatureInvocation, + ) -> Result { + if context.invocation_id != invocation.invocation_id { + return Err(ConfigAttachError::InvalidRequest.invocation_error()); + } + let access = invocation_access(invocation).map_err(ConfigAttachError::invocation_error)?; + let result = self + .attach(&context.invocation_id, access) + .await + .map_err(ConfigAttachError::invocation_error)?; + let action = if result.already_attached { + "Already attached" + } else { + "Attached" + }; + Ok(FeatureInvocationResult { + invocation_id: context.invocation_id, + identity: invocation.identity.clone(), + status: FeatureInvocationStatus::Succeeded, + message: format!( + "{action}: {ATTACHMENT_ALIAS} (Workspace configuration; {}).", + result.access.as_str() + ), + context: Some(format!( + "Workspace configuration is available as logical attachment `{ATTACHMENT_ALIAS}` at WIP `{CONTENT_ROOT}` with {} access. Discover structure through WIP; read and change bodies through published Operations. This attachment grants no OS paths or command execution. Backend validation, concurrency checks and activation rules apply; saving does not imply running Workers adopted new configuration.", + result.access.as_str() + )), + }) + } +} + +#[cfg(test)] +#[path = "workspace_config_tests.rs"] +mod tests; + +#[cfg(test)] +mod production_tests; diff --git a/crates/worker/src/feature/builtin/workspace_config/backend.rs b/crates/worker/src/feature/builtin/workspace_config/backend.rs new file mode 100644 index 000000000..97326eb72 --- /dev/null +++ b/crates/worker/src/feature/builtin/workspace_config/backend.rs @@ -0,0 +1,302 @@ +//! Identity-bound typed Workspace transport. Never retries a dispatched request. +use std::sync::Arc; + +use async_trait::async_trait; +use serde::{Serialize, de::DeserializeOwned}; +use server_api::{ + WorkspaceConfigAccess, WorkspaceConfigApiError, WorkspaceConfigAttachRequest, + WorkspaceConfigAttachment, WorkspaceConfigFailureClassification, +}; +use wip_protocol::{ProtocolError, ProtocolErrorCode}; + +use super::{ + ATTACHMENT_ALIAS, CONTENT_ROOT, ConfigAccess, ConfigAttachError, ConfigAttachment, DEADLINE, + WorkspaceConfigAttachmentBackend, +}; +use crate::wip::WipOperationError; +use crate::worker::{ + WorkspaceClient, WorkspaceClientError, WorkspaceRequest, WorkspaceRequestMethod, +}; + +pub const MAX_TEXT_BYTES: usize = 256 * 1024; +pub const MAX_PATH_BYTES: usize = 512; +pub const MAX_NODES: usize = 256; +pub const MAX_DEPTH: u32 = 8; +const MAX_RESPONSE_BYTES: usize = 2 * 1024 * 1024; + +#[derive(Clone)] +pub struct WorkspaceConfigBackend { + client: Arc, +} + +impl WorkspaceConfigBackend { + pub fn new(client: Arc) -> Self { + Self { client } + } + + fn base(&self) -> Result { + let workspace = self + .client + .workspace_id() + .filter(|id| !id.is_empty()) + .ok_or_else(|| { + failure( + ProtocolErrorCode::PermissionDenied, + "Workspace config authority unavailable", + ) + })?; + let encoded: String = workspace + .bytes() + .map(|byte| { + if byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_') { + (byte as char).to_string() + } else { + format!("%{byte:02X}") + } + }) + .collect(); + Ok(format!("/api/w/{encoded}/workers/self/workspace-config")) + } + + fn request( + &self, + request: WorkspaceRequest, + mutation: bool, + ) -> Result { + let response = self + .client + .execute_with_timeout(request, DEADLINE) + .map_err(|error| { + if mutation && !matches!(error, WorkspaceClientError::InvalidPath(_)) { + unknown() + } else { + failure( + ProtocolErrorCode::Internal, + "Workspace config transport unavailable", + ) + } + })?; + if response.body.len() > MAX_RESPONSE_BYTES { + return Err(if mutation { + unknown() + } else { + failure( + ProtocolErrorCode::ResourceLimitExceeded, + "Workspace config response exceeds limit", + ) + }); + } + if !response.is_success() { + if let Ok(error) = serde_json::from_str::(&response.body) { + if error.classification == WorkspaceConfigFailureClassification::Unknown { + return Err(unknown()); + } + // Do not reflect arbitrary transport/provider text into model output. + let code = match error.status { + 401 | 403 | 404 => ProtocolErrorCode::PermissionDenied, + 409 => ProtocolErrorCode::ValidatorMismatch, + 413 | 429 => ProtocolErrorCode::ResourceLimitExceeded, + 400 | 422 => ProtocolErrorCode::InvalidArguments, + _ => ProtocolErrorCode::Internal, + }; + return Err(failure( + code, + "Workspace config request rejected without commit", + )); + } + return Err(if mutation { + unknown() + } else { + failure( + if matches!(response.status, 401 | 403 | 404) { + ProtocolErrorCode::PermissionDenied + } else { + ProtocolErrorCode::Internal + }, + "Workspace config request denied or unavailable", + ) + }); + } + serde_json::from_str(&response.body).map_err(|_| { + if mutation { + unknown() + } else { + failure( + ProtocolErrorCode::Internal, + "Invalid Workspace config response", + ) + } + }) + } + + /// GET checks the current grant even when the Worker has not attached yet. + pub fn current(&self) -> Result, WipOperationError> { + let result: Option = + self.request(WorkspaceRequest::get(self.base()?), false)?; + if let Some(attachment) = &result { + self.validate_attachment(attachment)?; + } + Ok(result) + } + + fn validate_attachment( + &self, + attachment: &WorkspaceConfigAttachment, + ) -> Result<(), WipOperationError> { + if Some(attachment.workspace_id.as_str()) != self.client.workspace_id() + || attachment.alias != ATTACHMENT_ALIAS + || attachment.content_path != CONTENT_ROOT + || attachment.connection_id.is_empty() + || attachment.connection_id.len() > 256 + || !attachment + .connection_id + .bytes() + .all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_')) + { + return Err(failure( + ProtocolErrorCode::PermissionDenied, + "Invalid Workspace config attachment identity", + )); + } + Ok(()) + } + + pub async fn post( + &self, + suffix: &str, + request: &R, + mutation: bool, + ) -> Result { + let body = serde_json::to_string(request).map_err(|_| { + failure( + ProtocolErrorCode::InvalidArguments, + "Invalid Workspace config request", + ) + })?; + if body.len() > MAX_RESPONSE_BYTES { + return Err(failure( + ProtocolErrorCode::ResourceLimitExceeded, + "Workspace config request exceeds limit", + )); + } + let request = WorkspaceRequest::json( + WorkspaceRequestMethod::Post, + format!("{}{suffix}", self.base()?), + body, + ); + let this = self.clone(); + tokio::time::timeout( + DEADLINE, + tokio::task::spawn_blocking(move || this.request(request, mutation)), + ) + .await + .map_err(|_| { + if mutation { + unknown() + } else { + failure( + ProtocolErrorCode::ResourceLimitExceeded, + "Workspace config request deadline exceeded", + ) + } + })? + .map_err(|_| { + if mutation { + unknown() + } else { + failure( + ProtocolErrorCode::Internal, + "Workspace config transport unavailable", + ) + } + })? + } + + pub async fn current_async( + &self, + ) -> Result, WipOperationError> { + let this = self.clone(); + tokio::time::timeout( + DEADLINE, + tokio::task::spawn_blocking(move || this.current()), + ) + .await + .map_err(|_| { + failure( + ProtocolErrorCode::ResourceLimitExceeded, + "Workspace config request deadline exceeded", + ) + })? + .map_err(|_| { + failure( + ProtocolErrorCode::Internal, + "Workspace config transport unavailable", + ) + })? + } +} + +#[async_trait] +impl WorkspaceConfigAttachmentBackend for WorkspaceConfigBackend { + fn is_available(&self) -> bool { + self.client.is_available() + } + fn validate_access(&self, access: Option) -> Result<(), ConfigAttachError> { + let current = self.current().map_err(attach_error)?; + if access == Some(ConfigAccess::ReadWrite) + && current.is_some_and(|a| a.access == WorkspaceConfigAccess::ReadOnly) + { + return Err(ConfigAttachError::Denied); + } + Ok(()) + } + async fn attach( + &self, + _execution_key: &str, + access: Option, + ) -> Result { + // Alias attach is idempotent under the Backend session lock. The T696 + // fenced receipt owns replay; no new idempotency protocol or blind retry. + let request = WorkspaceConfigAttachRequest { + alias: Some(ATTACHMENT_ALIAS.into()), + access: access.map(|a| match a { + ConfigAccess::ReadOnly => WorkspaceConfigAccess::ReadOnly, + ConfigAccess::ReadWrite => WorkspaceConfigAccess::ReadWrite, + }), + }; + let attachment: WorkspaceConfigAttachment = + self.post("", &request, true).await.map_err(attach_error)?; + self.validate_attachment(&attachment) + .map_err(|_| ConfigAttachError::OutcomeUnknown)?; + Ok(ConfigAttachment { + connection_id: attachment.connection_id, + access: match attachment.access { + WorkspaceConfigAccess::ReadOnly => ConfigAccess::ReadOnly, + WorkspaceConfigAccess::ReadWrite => ConfigAccess::ReadWrite, + }, + already_attached: attachment.already_attached, + }) + } +} + +fn attach_error(error: WipOperationError) -> ConfigAttachError { + match error { + WipOperationError::OutcomeUnknown(_) => ConfigAttachError::OutcomeUnknown, + WipOperationError::Protocol(error) => match error.code { + ProtocolErrorCode::PermissionDenied => ConfigAttachError::Denied, + ProtocolErrorCode::InvalidArguments => ConfigAttachError::InvalidRequest, + _ => ConfigAttachError::Unavailable, + }, + _ => ConfigAttachError::Unavailable, + } +} + +pub(super) fn failure(code: ProtocolErrorCode, message: &str) -> WipOperationError { + WipOperationError::Protocol(ProtocolError { + code, + message: message.into(), + }) +} +pub(super) fn unknown() -> WipOperationError { + WipOperationError::OutcomeUnknown("Workspace config outcome unknown; refresh before any subsequent operation; no automatic retry".into()) +} diff --git a/crates/worker/src/feature/builtin/workspace_config/content.rs b/crates/worker/src/feature/builtin/workspace_config/content.rs new file mode 100644 index 000000000..3fccc35f8 --- /dev/null +++ b/crates/worker/src/feature/builtin/workspace_config/content.rs @@ -0,0 +1,744 @@ +//! Request-time logical subtree. Metadata is captured per resolution, never +//! cached as authority; authorization and canonical tree CAS stay in Backend. +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::Arc; + +use async_trait::async_trait; +use serde::Deserialize; +use serde_json::json; +use server_api::{ + ConfigCommitRequest, ConfigContentType, ConfigTreeChange, WorkspaceConfigAccess, + WorkspaceConfigAttachment, WorkspaceConfigCommitRequest, WorkspaceConfigCommitResponse, + WorkspaceConfigNode, WorkspaceConfigNodeKind, WorkspaceConfigObserveRequest, + WorkspaceConfigObserveResponse, WorkspaceConfigReadRequest, WorkspaceConfigReadResponse, +}; +use sha2::{Digest, Sha256}; +use wip_protocol::{ + Documentation, INTERFACE_FORMAT_V1, InterfaceDescriptor, Object, OperationDeclaration, + ParameterDeclaration, ProtocolError, ProtocolErrorCode, ReturnDeclaration, TypeExpr, Value, +}; + +use super::backend::{ + MAX_DEPTH, MAX_NODES, MAX_PATH_BYTES, MAX_TEXT_BYTES, WorkspaceConfigBackend, failure, unknown, +}; +use super::{CONTENT_ROOT, WorkspaceConfigFeature}; +use crate::wip::{ + WipCallContext, WipOperationError, WipOperationHandler, WipOperationOutput, WipProjection, + WipProjectionKind, WipSubtreeProvider, contextual_reference, json_to_wip, +}; + +pub(super) struct ConfigProvider { + pub feature: WorkspaceConfigFeature, + pub backend: Arc, +} + +/// Backend tree root is empty; source names are canonical relative config paths. +fn source_path(route: &str) -> Result { + if route == CONTENT_ROOT { + return Ok(String::new()); + } + let path = route + .strip_prefix(&format!("{CONTENT_ROOT}/")) + .ok_or_else(invalid)?; + validate_source(path)?; + Ok(path.into()) +} +fn validate_source(path: &str) -> Result<(), WipOperationError> { + if path.is_empty() + || path.len() > MAX_PATH_BYTES + || path.starts_with('/') + || path.contains('\\') + || path.chars().any(char::is_control) + || path + .split('/') + .any(|s| s.is_empty() || s == "." || s == "..") + { + return Err(invalid()); + } + Ok(()) +} +fn route(path: &str) -> String { + if path.is_empty() { + CONTENT_ROOT.into() + } else { + format!("{CONTENT_ROOT}/{path}") + } +} +fn invalid() -> WipOperationError { + failure( + ProtocolErrorCode::InvalidArguments, + "Invalid Workspace config operation arguments", + ) +} +fn protocol(error: WipOperationError) -> ProtocolError { + match error { + WipOperationError::Protocol(error) => error, + _ => ProtocolError { + code: ProtocolErrorCode::Internal, + message: "Workspace config metadata unavailable".into(), + }, + } +} +/// A denied namespace is absent from observation, not a failure of the whole +/// Worldspace. Do not mask transport, validation or Backend failures as absence. +fn visible_metadata(result: Result) -> Result, ProtocolError> { + match result { + Ok(value) => Ok(Some(value)), + Err(WipOperationError::Protocol(error)) + if error.code == ProtocolErrorCode::PermissionDenied => + { + Ok(None) + } + Err(error) => Err(protocol(error)), + } +} + +fn same_snapshot( + snapshot: &WorkspaceConfigObserveResponse, + other: &WorkspaceConfigObserveResponse, +) -> Result<(), WipOperationError> { + if snapshot.connection_id != other.connection_id + || snapshot.validator != other.validator + || snapshot.revision != other.revision + || snapshot.digest != other.digest + || snapshot.entrypoints != other.entrypoints + { + return Err(failure( + ProtocolErrorCode::ValidatorMismatch, + "Workspace config observation is stale", + )); + } + Ok(()) +} + +async fn observe( + backend: &WorkspaceConfigBackend, + connection: &str, + paths: Vec, + depth: u32, +) -> Result { + if paths.len() > MAX_NODES || depth > MAX_DEPTH { + return Err(invalid()); + } + for path in &paths { + if !path.is_empty() { + validate_source(path)?; + } + } + let result: WorkspaceConfigObserveResponse = backend + .post( + "/observe", + &WorkspaceConfigObserveRequest { + connection_id: connection.into(), + paths: paths.clone(), + depth, + }, + false, + ) + .await?; + if result.connection_id != connection + || result.nodes.len() > MAX_NODES + || result.validator.is_empty() + || result.validator.len() > 4096 + || result.digest.is_empty() + { + return Err(failure( + ProtocolErrorCode::Internal, + "Invalid Workspace config metadata", + )); + } + if result.entrypoints.is_empty() || result.entrypoints.len() > MAX_NODES { + return Err(failure( + ProtocolErrorCode::Internal, + "Invalid Workspace config entrypoints", + )); + } + for path in &result.entrypoints { + validate_source(path)?; + } + let mut seen = BTreeSet::new(); + for node in &result.nodes { + if !node.path.is_empty() { + validate_source(&node.path)?; + } + if !seen.insert(&node.path) + || node.validator.is_empty() + || node.validator.len() > 4096 + || !paths.iter().any(|parent| { + if node.path == *parent { + return true; + } + let suffix = if parent.is_empty() { + Some(node.path.as_str()) + } else { + node.path.strip_prefix(&format!("{parent}/")) + }; + suffix.is_some_and(|s| s.split('/').count() <= depth as usize) + }) + { + return Err(failure( + ProtocolErrorCode::Internal, + "Invalid Workspace config metadata scope", + )); + } + } + Ok(result) +} + +#[async_trait] +impl WipSubtreeProvider for ConfigProvider { + fn max_depth(&self) -> u32 { + MAX_DEPTH + } + fn max_nodes(&self) -> usize { + MAX_NODES + } + + async fn projection(&self, path: &str) -> Result, ProtocolError> { + let source = source_path(path).map_err(protocol)?; + let Some(attachment) = visible_metadata(self.backend.current_async().await)? else { + return Ok(None); + }; + let Some(attachment) = attachment else { + // An authorized but unattached Worker sees only the self-attach entrance. + if path != CONTENT_ROOT { + return Ok(None); + } + let mut projection = super::wip::attach_projection(&self.feature); + projection.interface = contextual_reference(&projection.interface, path); + projection.object.interfaces = vec![projection.interface.clone()]; + return Ok(Some(projection)); + }; + let Some(snapshot) = visible_metadata( + observe( + &self.backend, + &attachment.connection_id, + vec![source.clone()], + 0, + ) + .await, + )? + else { + return Ok(None); + }; + let Some(node) = snapshot.nodes.iter().find(|n| n.path == source).cloned() else { + return Ok(None); + }; + let operations = operations(&node, attachment.access, path == CONTENT_ROOT); + // Missing nodes are explicit absence observations for safe create, not files. + if operations.is_empty() && node.kind == WorkspaceConfigNodeKind::Missing { + return Ok(None); + } + let descriptor = descriptor(&operations); + let interface = contextual_reference("yoi.workspace-config/node/v1", path); + let interface_validator = Some(Sha256::digest(format!("{descriptor:?}")).to_vec()); + Ok(Some(WipProjection { + route: path.into(), + capability: "workspace-config:content".into(), + kind: WipProjectionKind::Native, + object: Object { + name: path.rsplit('/').next().unwrap_or("workspace-config").into(), + description: Some(format!( + "Logical Workspace config {:?}; bodies are available only through Operations", + node.kind + )), + interfaces: vec![interface.clone()], + r#ref: None, + validator: Some(node.validator.as_bytes().to_vec()), + }, + interface, + descriptor, + interface_validator, + handler: Arc::new(ConfigHandler { + feature: self.feature.clone(), + backend: self.backend.clone(), + attachment, + snapshot, + node, + operations, + }), + })) + } + + async fn children(&self, path: &str) -> Result, ProtocolError> { + let source = source_path(path).map_err(protocol)?; + let Some(Some(attachment)) = visible_metadata(self.backend.current_async().await)? else { + return Ok(Vec::new()); + }; + let Some(snapshot) = visible_metadata( + observe( + &self.backend, + &attachment.connection_id, + vec![source.clone()], + 1, + ) + .await, + )? + else { + return Ok(Vec::new()); + }; + Ok(snapshot + .nodes + .iter() + .filter(|n| n.path != source && n.kind != WorkspaceConfigNodeKind::Missing) + .map(|n| route(&n.path)) + .collect()) + } +} + +fn operations( + node: &WorkspaceConfigNode, + access: WorkspaceConfigAccess, + root: bool, +) -> Vec { + let mut names = Vec::new(); + // Backend operations are a publication ceiling; never publish a write on RO. + for name in ["read", "write", "edit", "create", "delete", "apply_changes"] { + let kind_ok = match name { + "read" | "write" | "edit" | "delete" => node.kind == WorkspaceConfigNodeKind::File, + "apply_changes" => root, + "create" => node.kind != WorkspaceConfigNodeKind::File, + _ => false, + }; + if kind_ok + && (name == "read" || access == WorkspaceConfigAccess::ReadWrite) + && node.operations.iter().any(|n| n == name) + { + names.push(name.into()); + } + } + if root { + names.push("attach".into()); + } + names +} + +fn parameter(name: &str, required: bool, r#type: TypeExpr) -> ParameterDeclaration { + ParameterDeclaration { + name: name.into(), + required, + documentation: None, + r#type, + } +} +fn descriptor(names: &[String]) -> InterfaceDescriptor { + InterfaceDescriptor { + format: INTERFACE_FORMAT_V1.into(), types: Vec::new(), + documentation: Some(Documentation { + summary: "Logical Workspace configuration file operations".into(), + details: Some("Backend grants, semantic validation, atomic whole-tree CAS and activation apply. Validators and change digests are managed automatically. apply_changes atomically accepts create/update/delete/rename changes without validators or expected_digest fields. Saving updates configuration for future operation boundaries, not already-running Worker manifests. No OS or command access.".into()), + }), + operations: names.iter().map(|name| OperationDeclaration { + name: name.clone(), documentation: None, + parameters: match name.as_str() { + "attach" => vec![parameter("access", false, TypeExpr::String)], + "read" | "write" | "edit" => crate::file_operation::parameters(name, false).unwrap(), + "delete" => vec![], + "create" => vec![parameter("path", false, TypeExpr::String), parameter("content", true, TypeExpr::String), parameter("content_type", false, TypeExpr::String)], + "apply_changes" => vec![parameter("changes", true, TypeExpr::Json)], + _ => vec![], + }, + returns: ReturnDeclaration { documentation: None, r#type: TypeExpr::Json }, + }).collect(), + } +} + +struct ConfigHandler { + feature: WorkspaceConfigFeature, + backend: Arc, + attachment: WorkspaceConfigAttachment, + snapshot: WorkspaceConfigObserveResponse, + node: WorkspaceConfigNode, + operations: Vec, +} + +#[derive(Debug, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +enum Change { + Create { + path: String, + content: String, + #[serde(default)] + content_type: Option, + }, + Update { + path: String, + content: String, + }, + Delete { + path: String, + }, + Rename { + from: String, + to: String, + }, +} +impl Change { + fn target(&self) -> &str { + match self { + Self::Create { path, .. } | Self::Update { path, .. } | Self::Delete { path } => path, + Self::Rename { from, .. } => from, + } + } +} +fn text(arguments: &BTreeMap, key: &str) -> Result { + match arguments.get(key) { + Some(Value::String(s)) if s.len() <= MAX_TEXT_BYTES => Ok(s.clone()), + _ => Err(invalid()), + } +} +fn text_limits() -> fs_operation::text::TextLimits { + fs_operation::text::TextLimits { + max_input_bytes: Some(MAX_TEXT_BYTES), + max_output_bytes: Some(MAX_TEXT_BYTES), + max_replacements: None, + } +} +fn decode_text( + arguments: &BTreeMap, +) -> Result { + let value = Value::Record(arguments.clone()); + fs_operation::text::decode(crate::wip::wip_to_json(&value).map_err(|_| invalid())?) + .map_err(text_error) +} +fn text_error(error: fs_operation::text::TextError) -> WipOperationError { + use fs_operation::text::TextError; + match error { + TextError::OutputTooLarge { .. } + | TextError::Overflow + | TextError::ReplacementLimitExceeded { .. } => failure( + ProtocolErrorCode::ResourceLimitExceeded, + "Workspace config edit exceeds text limit", + ), + TextError::IdenticalStrings => failure( + ProtocolErrorCode::InvalidArguments, + "old_string and new_string are identical", + ), + TextError::Decode(_) + | TextError::EmptyOldString + | TextError::NotFound + | TextError::MultipleMatches { .. } + | TextError::InputTooLarge { .. } => invalid(), + } +} +fn content_type(value: Option<&Value>) -> Result { + match value { + None => Ok(ConfigContentType::Decodal), + Some(Value::String(value)) if value == "decodal" => Ok(ConfigContentType::Decodal), + Some(Value::String(value)) if value == "text" => Ok(ConfigContentType::Text), + _ => Err(invalid()), + } +} + +impl ConfigHandler { + async fn read(&self) -> Result { + let response: WorkspaceConfigReadResponse = self + .backend + .post( + "/read", + &WorkspaceConfigReadRequest { + connection_id: self.attachment.connection_id.clone(), + path: self.node.path.clone(), + validator: self.node.validator.clone(), + }, + false, + ) + .await?; + if response.path != self.node.path + || response.validator != self.node.validator + || Some(&response.digest) != self.node.digest.as_ref() + || Some(&response.content_type) != self.node.content_type.as_ref() + || response.content.len() > MAX_TEXT_BYTES + { + return Err(failure( + ProtocolErrorCode::Internal, + "Invalid Workspace config read response", + )); + } + let mut response = response; + response.content = fs_operation::text::read(response.content, text_limits()) + .map_err(text_error)? + .content; + Ok(response) + } + + async fn canonical_changes( + &self, + changes: Vec, + ) -> Result, WipOperationError> { + if changes.is_empty() || changes.len() > MAX_NODES { + return Err(invalid()); + } + let mut paths = BTreeSet::new(); + let mut text_bytes = 0usize; + for change in &changes { + validate_source(change.target())?; + if !paths.insert(change.target().to_owned()) { + return Err(invalid()); + } + match change { + Change::Create { content, .. } | Change::Update { content, .. } => { + text_bytes = text_bytes.saturating_add(content.len()); + } + Change::Rename { to, .. } => { + validate_source(to)?; + } + _ => {} + } + } + if text_bytes > MAX_TEXT_BYTES { + return Err(failure( + ProtocolErrorCode::ResourceLimitExceeded, + "Workspace config text exceeds limit", + )); + } + // Resolve canonical per-file preconditions at the SAME captured tree revision. + let metadata = observe( + &self.backend, + &self.attachment.connection_id, + paths.into_iter().collect(), + 0, + ) + .await?; + same_snapshot(&self.snapshot, &metadata)?; + changes + .into_iter() + .map(|change| { + let node = metadata + .nodes + .iter() + .find(|n| n.path == change.target()) + .ok_or_else(invalid)?; + let digest = || { + node.digest + .clone() + .filter(|_| node.kind == WorkspaceConfigNodeKind::File) + .ok_or_else(|| { + failure( + ProtocolErrorCode::ValidatorMismatch, + "Workspace config file no longer exists", + ) + }) + }; + Ok(match change { + Change::Create { + path, + content, + content_type, + } => { + if node.kind != WorkspaceConfigNodeKind::Missing { + return Err(failure( + ProtocolErrorCode::ValidatorMismatch, + "Workspace config create target already exists", + )); + } + ConfigTreeChange::Create { + path, + content, + content_type: content_type.unwrap_or(ConfigContentType::Decodal), + } + } + Change::Update { path, content } => ConfigTreeChange::Update { + path, + content, + expected_digest: digest()?, + }, + Change::Delete { path } => ConfigTreeChange::Delete { + path, + expected_digest: digest()?, + }, + Change::Rename { from, to } => ConfigTreeChange::Rename { + from, + to, + expected_digest: digest()?, + }, + }) + }) + .collect() + } +} + +#[async_trait] +impl WipOperationHandler for ConfigHandler { + async fn call( + &self, + operation: &str, + arguments: &BTreeMap, + context: WipCallContext, + ) -> Result { + let dispatched = std::sync::atomic::AtomicBool::new(false); + tokio::time::timeout( + super::DEADLINE, + self.execute(operation, arguments, context, &dispatched), + ) + .await + .map_err(|_| { + if dispatched.load(std::sync::atomic::Ordering::Acquire) { + unknown() + } else { + failure( + ProtocolErrorCode::ResourceLimitExceeded, + "Workspace config operation deadline exceeded before effects", + ) + } + })? + } +} + +impl ConfigHandler { + async fn execute( + &self, + operation: &str, + arguments: &BTreeMap, + context: WipCallContext, + dispatched: &std::sync::atomic::AtomicBool, + ) -> Result { + if !self.operations.iter().any(|name| name == operation) { + return Err(failure( + ProtocolErrorCode::OperationNotFound, + "Workspace config operation is not published", + )); + } + // Direct handler calls also enforce parameter shape; no model validators accepted. + let declaration = descriptor(&self.operations) + .operations + .into_iter() + .find(|op| op.name == operation) + .ok_or_else(invalid)?; + if arguments + .keys() + .any(|name| !declaration.parameters.iter().any(|p| p.name == *name)) + { + return Err(invalid()); + } + if operation == "attach" { + dispatched.store(true, std::sync::atomic::Ordering::Release); + return super::wip::attach_projection(&self.feature) + .handler + .call(operation, arguments, context) + .await; + } + if operation == "read" { + let _: fs_operation::text::ReadArgs = decode_text(arguments)?; + let result = self.read().await?; + // Validator goes to Client state, not the model operation result. + let value = json_to_wip(&json!({ + "path": result.path, "content": result.content, + "content_type": result.content_type, "digest": result.digest, + })) + .map_err(|_| invalid())?; + return Ok(WipOperationOutput::native_with_validator( + value, + self.node.validator.as_bytes().to_vec(), + )); + } + if self.attachment.access != WorkspaceConfigAccess::ReadWrite { + return Err(failure( + ProtocolErrorCode::PermissionDenied, + "Workspace config write denied", + )); + } + let changes = match operation { + "write" => { + let args: fs_operation::text::WriteArgs = decode_text(arguments)?; + let result = + fs_operation::text::write(args.content, text_limits()).map_err(text_error)?; + vec![Change::Update { + path: self.node.path.clone(), + content: result.content, + }] + } + "edit" => { + let args: fs_operation::text::EditArgs = decode_text(arguments)?; + // Reject basic invalid input before reading or entering the commit + // path. The captured read and whole-tree CAS remain bound here. + args.validate(text_limits()).map_err(text_error)?; + let original = self.read().await?.content; + let result = fs_operation::text::edit(&original, &args, text_limits()) + .map_err(text_error)?; + vec![Change::Update { + path: self.node.path.clone(), + content: result.content, + }] + } + "delete" => vec![Change::Delete { + path: self.node.path.clone(), + }], + "create" => { + let path = match (self.node.kind, arguments.get("path")) { + (WorkspaceConfigNodeKind::Missing, None) => self.node.path.clone(), + (WorkspaceConfigNodeKind::Directory, Some(Value::String(path))) => { + validate_source(path)?; + if self.node.path.is_empty() { + path.clone() + } else { + format!("{}/{path}", self.node.path) + } + } + _ => return Err(invalid()), + }; + vec![Change::Create { + path, + content: text(arguments, "content")?, + content_type: Some(content_type(arguments.get("content_type"))?), + }] + } + "apply_changes" => { + let Some(value) = arguments.get("changes") else { + return Err(invalid()); + }; + serde_json::from_value::>( + crate::wip::wip_to_json(value).map_err(|_| invalid())?, + ) + .map_err(|_| invalid())? + } + _ => return Err(invalid()), + }; + let changes = self.canonical_changes(changes).await?; + dispatched.store(true, std::sync::atomic::Ordering::Release); + let result: WorkspaceConfigCommitResponse = self + .backend + .post( + "/commit", + &WorkspaceConfigCommitRequest { + connection_id: self.attachment.connection_id.clone(), + validator: self.snapshot.validator.clone(), + request: ConfigCommitRequest { + base_revision: self.snapshot.revision, + base_digest: self.snapshot.digest.clone(), + changes, + entrypoints: self.snapshot.entrypoints.clone(), + }, + }, + true, + ) + .await?; + // Commit response supplies ROOT state. Obtain the exact post-commit PATH + // validator, never substitute the root validator for a file validator. + let updated = observe( + &self.backend, + &self.attachment.connection_id, + vec![self.node.path.clone()], + 0, + ) + .await + .map_err(|_| unknown())?; + if updated.revision != result.revision + || updated.digest != result.digest + || updated.validator != result.validator + { + return Err(unknown()); + } + let node = updated + .nodes + .iter() + .find(|n| n.path == self.node.path) + .ok_or_else(unknown)?; + let value = json_to_wip(&json!({"revision": result.revision, "digest": result.digest, + "activation": "Saved to canonical Workspace configuration; existing Worker manifests are not rewritten"})).map_err(|_| unknown())?; + Ok(WipOperationOutput::native_with_validator( + value, + node.validator.as_bytes().to_vec(), + )) + } +} diff --git a/crates/worker/src/feature/builtin/workspace_config/production_tests.rs b/crates/worker/src/feature/builtin/workspace_config/production_tests.rs new file mode 100644 index 000000000..4c6107af2 --- /dev/null +++ b/crates/worker/src/feature/builtin/workspace_config/production_tests.rs @@ -0,0 +1,929 @@ +//! Worker-side E2E across real Feature invocation, WIP Client/Host and typed +//! Workspace transport. Canonical persistence/projection tests live in Backend; +//! this deterministic router models its published auth/CAS/response contract. +use super::*; +use crate::feature::FeatureRegistryBuilder; +use crate::hook::HookRegistryBuilder; +use crate::wip::{WipMountRegistry, WipRuntime}; +use crate::worker::{ + WorkspaceClient, WorkspaceClientError, WorkspaceRequest, WorkspaceRequestMethod, + WorkspaceResponse, +}; +use agen::tool::ToolExecutionContext; +use protocol::{FeatureInvocation, FeatureInvocationIdentity}; +use serde::Serialize; +use serde_json::{Value as Json, json}; +use server_api::*; +use sha2::{Digest, Sha256}; +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::{Arc, Mutex}; + +#[derive(Debug)] +struct State { + granted: bool, + read_only: bool, + active: Option, + next_connection: u64, + revision: u64, + entries: BTreeMap, + commits: Vec, + requests: Vec, + fail_commit: bool, + fail_current: bool, + lose_commit_reply: bool, + race_commit: bool, +} +#[derive(Debug)] +struct Router { + workspace: &'static str, + state: Arc>, +} +impl Router { + fn new() -> Arc { + Arc::new(Self { + workspace: "workspace-1", + state: Arc::new(Mutex::new(State { + granted: true, + read_only: false, + active: None, + next_connection: 0, + revision: 1, + entries: BTreeMap::from([ + ( + "main.dcdl".into(), + (ConfigContentType::Decodal, "model = old".into()), + ), + ( + "profiles/team/deep/model.dcdl".into(), + (ConfigContentType::Decodal, "model = deep".into()), + ), + ]), + commits: Vec::new(), + requests: Vec::new(), + fail_commit: false, + fail_current: false, + lose_commit_reply: false, + race_commit: false, + })), + }) + } +} +fn response(value: &T) -> WorkspaceResponse { + WorkspaceResponse { + status: 200, + body: serde_json::to_string(value).unwrap(), + } +} +fn denied(status: u16, classification: WorkspaceConfigFailureClassification) -> WorkspaceResponse { + // Deliberately unsafe provider text must never be propagated by the adapter. + WorkspaceResponse { + status, + body: serde_json::to_string(&WorkspaceConfigApiError { + status, + code: "test".into(), + message: "secret-token /private/host/config".into(), + classification, + }) + .unwrap(), + } +} +fn digest(content: &str) -> String { + Sha256::digest(content.as_bytes()) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} +impl State { + fn tree_digest(&self) -> String { + digest(&serde_json::to_string(&self.entries).unwrap()) + } + fn validator(&self, path: &str) -> String { + format!( + "wc:{}:{}:{}:{}", + self.active.as_ref().unwrap().connection_id, + self.revision, + self.tree_digest(), + path + ) + } + fn node(&self, path: &str, kind: WorkspaceConfigNodeKind) -> WorkspaceConfigNode { + let mut operations = match kind { + WorkspaceConfigNodeKind::File => vec!["read".into()], + _ => vec![], + }; + if !self.read_only { + match kind { + WorkspaceConfigNodeKind::File => { + operations.extend(["write", "edit", "delete"].map(str::to_string)) + } + WorkspaceConfigNodeKind::Missing => operations.push("create".into()), + WorkspaceConfigNodeKind::Directory => { + operations.push("create".into()); + if path.is_empty() { + operations.push("apply_changes".into()); + } + } + } + } + WorkspaceConfigNode { + path: path.into(), + kind, + validator: self.validator(path), + digest: self.entries.get(path).map(|(_, s)| digest(s)), + content_type: self.entries.get(path).map(|(t, _)| *t), + operations, + } + } +} +impl WorkspaceClient for Router { + fn workspace_id(&self) -> Option<&str> { + Some(self.workspace) + } + fn kind(&self) -> &str { + "typed-config-router" + } + fn is_available(&self) -> bool { + true + } + fn execute( + &self, + request: WorkspaceRequest, + ) -> Result { + let mut state = self.state.lock().unwrap(); + state.requests.push(request.clone()); + let prefix = "/api/w/workspace-1/workers/self/workspace-config"; + if self.workspace != "workspace-1" || !request.path.starts_with(prefix) || !state.granted { + return Ok(denied( + 403, + WorkspaceConfigFailureClassification::NotCommitted, + )); + } + let suffix = request.path.strip_prefix(prefix).unwrap(); + if suffix.is_empty() { + if request.method == WorkspaceRequestMethod::Get { + if state.fail_current { + return Ok(WorkspaceResponse { + status: 503, + body: "unsafe provider diagnostic /private/host/config".into(), + }); + } + return Ok(response(&state.active)); + } + let attach: WorkspaceConfigAttachRequest = + serde_json::from_str(request.body.as_ref().unwrap()).unwrap(); + assert_eq!(attach.alias.as_deref(), Some(ATTACHMENT_ALIAS)); + if attach.access == Some(WorkspaceConfigAccess::ReadWrite) && state.read_only { + return Ok(denied( + 403, + WorkspaceConfigFailureClassification::NotCommitted, + )); + } + let already_attached = state.active.is_some(); + if !already_attached { + state.next_connection += 1; + } + let access = attach.access.unwrap_or(if state.read_only { + WorkspaceConfigAccess::ReadOnly + } else { + WorkspaceConfigAccess::ReadWrite + }); + state.read_only |= access == WorkspaceConfigAccess::ReadOnly; + let attachment = WorkspaceConfigAttachment { + workspace_id: "workspace-1".into(), + connection_id: format!("connection-{}", state.next_connection), + alias: ATTACHMENT_ALIAS.into(), + working_directory_id: "logical-grant-1".into(), + access, + name: "Workspace configuration".into(), + purpose: "Canonical config".into(), + content_path: CONTENT_ROOT.into(), + already_attached, + }; + state.active = Some(attachment.clone()); + return Ok(response(&attachment)); + } + let input: Json = serde_json::from_str(request.body.as_ref().unwrap()).unwrap(); + if state.active.as_ref().map(|a| a.connection_id.as_str()) + != input["connection_id"].as_str() + { + return Ok(denied( + 409, + WorkspaceConfigFailureClassification::NotCommitted, + )); + } + match suffix { + "/observe" => { + let request: WorkspaceConfigObserveRequest = serde_json::from_value(input).unwrap(); + assert!(request.depth <= 8 && request.paths.len() <= 256); + let mut kinds = + BTreeMap::from([(String::new(), WorkspaceConfigNodeKind::Directory)]); + for path in state.entries.keys() { + kinds.insert(path.clone(), WorkspaceConfigNodeKind::File); + let mut path = path.as_str(); + while let Some((parent, _)) = path.rsplit_once('/') { + kinds + .entry(parent.into()) + .or_insert(WorkspaceConfigNodeKind::Directory); + path = parent; + } + } + let mut selected = BTreeSet::new(); + for path in &request.paths { + selected.insert(path.clone()); + let prefix = if path.is_empty() { + String::new() + } else { + format!("{path}/") + }; + for child in kinds.keys() { + if let Some(suffix) = child.strip_prefix(&prefix) + && child != path + && suffix.split('/').count() <= request.depth as usize + { + selected.insert(child.clone()); + } + } + } + if selected.len() > 256 { + return Ok(denied( + 413, + WorkspaceConfigFailureClassification::NotCommitted, + )); + } + let nodes = selected + .into_iter() + .map(|p| { + state.node( + &p, + *kinds.get(&p).unwrap_or(&WorkspaceConfigNodeKind::Missing), + ) + }) + .collect(); + Ok(response(&WorkspaceConfigObserveResponse { + connection_id: state.active.as_ref().unwrap().connection_id.clone(), + validator: state.validator(""), + revision: state.revision, + digest: state.tree_digest(), + entrypoints: vec!["main.dcdl".into()], + nodes, + })) + } + "/read" => { + let request: WorkspaceConfigReadRequest = serde_json::from_value(input).unwrap(); + if request.validator != state.validator(&request.path) { + return Ok(denied( + 409, + WorkspaceConfigFailureClassification::NotCommitted, + )); + } + let Some((content_type, content)) = state.entries.get(&request.path) else { + return Ok(denied( + 404, + WorkspaceConfigFailureClassification::NotCommitted, + )); + }; + Ok(response(&WorkspaceConfigReadResponse { + path: request.path.clone(), + content: content.clone(), + content_type: *content_type, + digest: digest(content), + validator: state.validator(&request.path), + })) + } + "/commit" => { + let request: WorkspaceConfigCommitRequest = serde_json::from_value(input).unwrap(); + state.commits.push(request.clone()); + if state.race_commit { + state.revision += 1; + state.race_commit = false; + } + if state.read_only { + return Ok(denied( + 403, + WorkspaceConfigFailureClassification::NotCommitted, + )); + } + assert_eq!(request.request.entrypoints, ["main.dcdl"]); + if request.validator != state.validator("") + || request.request.base_revision != state.revision + || request.request.base_digest != state.tree_digest() + { + return Ok(denied( + 409, + WorkspaceConfigFailureClassification::NotCommitted, + )); + } + if state.fail_commit { + return Ok(denied( + 500, + WorkspaceConfigFailureClassification::NotCommitted, + )); + } + let mut entries = state.entries.clone(); + for change in request.request.changes { + match change { + ConfigTreeChange::Create { + path, + content_type, + content, + } => { + if entries.contains_key(&path) { + return Ok(denied( + 409, + WorkspaceConfigFailureClassification::NotCommitted, + )); + } + entries.insert(path, (content_type, content)); + } + ConfigTreeChange::Update { + path, + expected_digest, + content, + } => { + let Some((kind, old)) = entries.get(&path) else { + return Ok(denied( + 409, + WorkspaceConfigFailureClassification::NotCommitted, + )); + }; + if digest(old) != expected_digest { + return Ok(denied( + 409, + WorkspaceConfigFailureClassification::NotCommitted, + )); + } + entries.insert(path, (*kind, content)); + } + ConfigTreeChange::Delete { + path, + expected_digest, + } => { + if entries + .get(&path) + .is_none_or(|(_, s)| digest(s) != expected_digest) + { + return Ok(denied( + 409, + WorkspaceConfigFailureClassification::NotCommitted, + )); + } + entries.remove(&path); + } + ConfigTreeChange::Rename { + from, + to, + expected_digest, + } => { + if entries + .get(&from) + .is_none_or(|(_, s)| digest(s) != expected_digest) + || entries.contains_key(&to) + { + return Ok(denied( + 409, + WorkspaceConfigFailureClassification::NotCommitted, + )); + } + let entry = entries.remove(&from).unwrap(); + entries.insert(to, entry); + } + } + } + if entries.values().any(|(_, s)| s == "INVALID") { + return Ok(denied( + 400, + WorkspaceConfigFailureClassification::NotCommitted, + )); + } + state.entries = entries; + state.revision += 1; + if state.lose_commit_reply { + state.lose_commit_reply = false; + return Err(WorkspaceClientError::Request( + "secret-token /private/host/config".into(), + )); + } + Ok(response(&WorkspaceConfigCommitResponse { + validator: state.validator(""), + revision: state.revision, + digest: state.tree_digest(), + })) + } + _ => panic!("unregistered endpoint: {suffix}"), + } + } +} +fn invocation(key: &str) -> FeatureInvocation { + FeatureInvocation { + invocation_id: key.into(), + identity: FeatureInvocationIdentity(INVOCATION_ID.into()), + name: FEATURE_ID.into(), + arguments: vec![], + } +} +fn runtime(client: Arc) -> (WorkspaceConfigFeature, WipRuntime) { + let feature = WorkspaceConfigFeature::for_workspace(client, true); + let mut mounts = WipMountRegistry::new(); + wip::mount_workspace_config_wip(&mut mounts, &feature).unwrap(); + let runtime = WipRuntime::from_mounts(mounts, "worker-1@workspace-1".into()).unwrap(); + (feature, runtime) +} +fn interface(path: &str, attached: bool) -> String { + crate::wip::contextual_reference( + if attached { + "yoi.workspace-config/node/v1" + } else { + "yoi.workspace-config/attachment/v1" + }, + path, + ) +} +async fn observe_interface(runtime: &WipRuntime, path: &str, attached: bool) { + runtime.discover(path.into(), 0, true).await.unwrap(); + runtime + .inspect(interface(path, attached), true) + .await + .unwrap(); +} +async fn call( + runtime: &WipRuntime, + path: &str, + op: &str, + args: Json, +) -> Result { + runtime + .call( + path.into(), + interface(path, true), + op.into(), + args, + ToolExecutionContext::direct(), + ) + .await +} + +#[tokio::test] +async fn workspace_config_production_slash_deep_discover_read_edit_and_atomic_import_changes() { + let client = Router::new(); + let (feature, runtime) = runtime(client.clone()); + let mut tools = Vec::new(); + let report = FeatureRegistryBuilder::new() + .with_module(feature) + .install_into_pending(&mut tools, &mut HookRegistryBuilder::default()); + assert!(tools.is_empty() && !report.has_errors()); + assert_eq!( + report + .chat_invocations + .feature_completions("workspace-c") + .len(), + 1 + ); + assert!( + report + .chat_invocations + .invoke(&invocation("slash-1")) + .await + .unwrap() + .message + .starts_with("Attached") + ); + assert!( + report + .chat_invocations + .invoke(&invocation("slash-2")) + .await + .unwrap() + .message + .starts_with("Already attached") + ); + let discovered = runtime + .discover(CONTENT_ROOT.into(), 5, true) + .await + .unwrap() + .content + .unwrap(); + assert!(discovered.contains("profiles/team/deep/model.dcdl")); + assert!(!discovered.contains("model = deep")); + let path = "/workspace-config/profiles/team/deep/model.dcdl"; + observe_interface(&runtime, path, true).await; + assert!( + call(&runtime, path, "read", json!({})) + .await + .unwrap() + .content + .unwrap() + .contains("model = deep") + ); + call( + &runtime, + path, + "edit", + json!({"old_string":"deep","new_string":"changed"}), + ) + .await + .unwrap(); + assert_eq!( + client.state.lock().unwrap().entries["profiles/team/deep/model.dcdl"].1, + "model = changed" + ); + // Client receives the new path validator automatically, no model tracking. + call(&runtime, path, "write", json!({"content":"model = second"})) + .await + .unwrap(); + observe_interface(&runtime, CONTENT_ROOT, true).await; + call( + &runtime, + CONTENT_ROOT, + "apply_changes", + json!({"changes":[ + {"kind":"create","path":"profiles/new.dcdl","content":"model = imported"}, + {"kind":"update","path":"main.dcdl","content":"import profiles/new.dcdl"} + ]}), + ) + .await + .unwrap(); + let state = client.state.lock().unwrap(); + assert_eq!(state.entries["main.dcdl"].1, "import profiles/new.dcdl"); + assert_eq!(state.entries["profiles/new.dcdl"].1, "model = imported"); + assert_eq!(state.commits.last().unwrap().request.changes.len(), 2); + assert!( + state + .requests + .iter() + .all(|r| !r.path.contains("session") && !r.path.contains("bash")) + ); +} + +#[tokio::test] +async fn workspace_config_production_native_attach_create_delete_and_detach_end_use() { + let client = Router::new(); + let (_, runtime) = runtime(client.clone()); + observe_interface(&runtime, CONTENT_ROOT, false).await; + runtime + .call( + CONTENT_ROOT.into(), + interface(CONTENT_ROOT, false), + "attach".into(), + json!({}), + ToolExecutionContext::direct(), + ) + .await + .unwrap(); + let directory = "/workspace-config/profiles"; + observe_interface(&runtime, directory, true).await; + call( + &runtime, + directory, + "create", + json!({"path":"notes.txt", "content_type":"text", "content":"configuration notes"}), + ) + .await + .unwrap(); + assert_eq!( + client.state.lock().unwrap().entries["profiles/notes.txt"].0, + ConfigContentType::Text + ); + let path = "/workspace-config/new/nested/file.dcdl"; + observe_interface(&runtime, path, true).await; + call(&runtime, path, "create", json!({"content":"new = true"})) + .await + .unwrap(); + observe_interface(&runtime, path, true).await; + call(&runtime, path, "delete", json!({})).await.unwrap(); + assert!( + !client + .state + .lock() + .unwrap() + .entries + .contains_key("new/nested/file.dcdl") + ); + client.state.lock().unwrap().active = None; + assert!( + call(&runtime, "/workspace-config/main.dcdl", "read", json!({})) + .await + .is_err() + ); + assert!( + runtime + .discover("/workspace-config/main.dcdl".into(), 0, true) + .await + .is_err() + ); +} + +#[tokio::test] +async fn workspace_config_production_revocation_ro_cross_workspace_and_cache_isolation() { + let client = Router::new(); + let (feature, runtime) = runtime(client.clone()); + feature.attach("grant", None).await.unwrap(); + let path = "/workspace-config/main.dcdl"; + observe_interface(&runtime, path, true).await; + client.state.lock().unwrap().granted = false; + assert!(call(&runtime, path, "read", json!({})).await.is_err()); + assert!(feature.attach("revoked", None).await.is_err()); + client.state.lock().unwrap().granted = true; + client.state.lock().unwrap().read_only = true; + client.state.lock().unwrap().active.as_mut().unwrap().access = WorkspaceConfigAccess::ReadOnly; + assert!( + feature + .attach("explicit-write", Some(ConfigAccess::ReadWrite)) + .await + .is_err() + ); + observe_interface(&runtime, path, true).await; + let descriptor = runtime + .inspect(interface(path, true), true) + .await + .unwrap() + .content + .unwrap(); + let descriptor: Json = serde_json::from_str(&descriptor).unwrap(); + let names = descriptor["descriptor"]["operations"] + .as_array() + .unwrap() + .iter() + .map(|op| op["name"].as_str().unwrap()) + .collect::>(); + assert_eq!(names, ["read"]); + assert!( + call(&runtime, path, "write", json!({"content":"DENIED"})) + .await + .is_err() + ); + let cross = Arc::new(Router { + workspace: "other-workspace", + state: client.state.clone(), + }); + let (feature, other) = runtime_for_cross(cross); + assert!(feature.attach("cross", None).await.is_err()); + assert!(other.discover(path.into(), 0, false).await.is_err()); + assert!(client.state.lock().unwrap().commits.is_empty()); +} +fn runtime_for_cross(client: Arc) -> (WorkspaceConfigFeature, WipRuntime) { + runtime(client) +} + +#[tokio::test] +async fn workspace_config_production_stale_same_bytes_and_commit_race_never_overwrite() { + let client = Router::new(); + let (feature, runtime) = runtime(client.clone()); + feature.attach("initial", None).await.unwrap(); + let path = "/workspace-config/main.dcdl"; + observe_interface(&runtime, path, true).await; + client.state.lock().unwrap().revision += 1; // Same bytes, different identity revision. + assert!( + call(&runtime, path, "write", json!({"content":"overwritten"})) + .await + .is_err() + ); + assert!(client.state.lock().unwrap().commits.is_empty()); + observe_interface(&runtime, path, true).await; + client.state.lock().unwrap().race_commit = true; + assert!( + call(&runtime, path, "write", json!({"content":"raced"})) + .await + .is_err() + ); + assert_eq!( + client.state.lock().unwrap().entries["main.dcdl"].1, + "model = old" + ); + assert_eq!(client.state.lock().unwrap().commits.len(), 1); +} + +#[tokio::test] +async fn workspace_config_production_invalid_size_save_failure_unknown_are_safe_and_not_retried() { + let client = Router::new(); + let (feature, runtime) = runtime(client.clone()); + feature.attach("initial", None).await.unwrap(); + let path = "/workspace-config/main.dcdl"; + for content in ["INVALID".into(), "x".repeat(256 * 1024 + 1)] { + observe_interface(&runtime, path, true).await; + let error = call(&runtime, path, "write", json!({"content":content})) + .await + .unwrap_err() + .to_string(); + assert!(!error.contains("secret-token") && !error.contains("/private/host")); + assert_eq!( + client.state.lock().unwrap().entries["main.dcdl"].1, + "model = old" + ); + } + observe_interface(&runtime, path, true).await; + let before = client.state.lock().unwrap().commits.len(); + assert!( + call( + &runtime, + path, + "write", + json!({"content":"valid","validator":"model-supplied"}) + ) + .await + .is_err() + ); + assert_eq!(before, client.state.lock().unwrap().commits.len()); + client.state.lock().unwrap().fail_commit = true; + assert!( + call(&runtime, path, "write", json!({"content":"valid"})) + .await + .is_err() + ); + assert_eq!( + client.state.lock().unwrap().entries["main.dcdl"].1, + "model = old" + ); + client.state.lock().unwrap().fail_commit = false; + observe_interface(&runtime, path, true).await; + client.state.lock().unwrap().lose_commit_reply = true; + let count = client.state.lock().unwrap().commits.len(); + let error = call( + &runtime, + path, + "write", + json!({"content":"saved-but-reply-lost"}), + ) + .await + .unwrap_err() + .to_string(); + assert!(error.contains("unknown")); + assert!(!error.contains("secret-token") && !error.contains("/private/host")); + assert_eq!(client.state.lock().unwrap().commits.len(), count + 1); + assert_eq!( + client.state.lock().unwrap().entries["main.dcdl"].1, + "saved-but-reply-lost" + ); +} + +#[tokio::test] +async fn workspace_config_production_bounds_and_traversal_reject_before_content_calls() { + let client = Router::new(); + let (feature, runtime) = runtime(client.clone()); + feature.attach("initial", None).await.unwrap(); + assert!( + runtime + .discover(CONTENT_ROOT.into(), 9, true) + .await + .is_err() + ); + for path in [ + "/workspace-config/../secret", + "/workspace-config//main.dcdl", + "/workspace-config/./main.dcdl", + ] { + assert!(runtime.discover(path.into(), 0, true).await.is_err()); + } + let path = format!("/workspace-config/{}", "a".repeat(513)); + assert!(runtime.discover(path, 0, true).await.is_err()); + assert!(client.state.lock().unwrap().commits.is_empty()); +} + +#[tokio::test] +async fn workspace_config_production_reattach_new_lifetime_rejects_old_same_bytes_cache() { + let client = Router::new(); + let (feature, runtime) = runtime(client.clone()); + let first = feature.attach("first", None).await.unwrap(); + let path = "/workspace-config/main.dcdl"; + observe_interface(&runtime, path, true).await; + client.state.lock().unwrap().active = None; + let second = feature.attach("second", None).await.unwrap(); + assert_ne!(first.connection_id, second.connection_id); + assert!( + call(&runtime, path, "write", json!({"content":"bad"})) + .await + .is_err() + ); + assert!(client.state.lock().unwrap().commits.is_empty()); + observe_interface(&runtime, path, true).await; + call(&runtime, path, "write", json!({"content":"new-lifetime"})) + .await + .unwrap(); + assert_eq!( + client.state.lock().unwrap().entries["main.dcdl"].1, + "new-lifetime" + ); +} + +#[tokio::test] +async fn workspace_config_production_cumulative_node_limit_and_edit_growth_are_bounded() { + let client = Router::new(); + let (feature, runtime) = runtime(client.clone()); + feature.attach("initial", None).await.unwrap(); + let path = "/workspace-config/main.dcdl"; + observe_interface(&runtime, path, true).await; + let before = client.state.lock().unwrap().commits.len(); + assert!( + call( + &runtime, + path, + "edit", + json!({ + "old_string":"o", "new_string":"x".repeat(256*1024), "replace_all":true + }) + ) + .await + .is_err() + ); + assert_eq!(client.state.lock().unwrap().commits.len(), before); + { + let mut state = client.state.lock().unwrap(); + for i in 0..90 { + state.entries.insert( + format!("groups/{i}/nested/f.dcdl"), + (ConfigContentType::Decodal, "x = true".into()), + ); + } + } + // No individual depth-1 response exceeds the cap; the whole Host traversal + // must nevertheless stop before exceeding 256 Objects. + let error = runtime + .discover(CONTENT_ROOT.into(), 4, true) + .await + .unwrap_err(); + assert!(error.to_string().contains("ResourceLimitExceeded")); +} + +#[tokio::test] +async fn workspace_config_production_backend_failure_is_not_hidden_as_unauthorized_absence() { + let client = Router::new(); + client.state.lock().unwrap().fail_current = true; + let (_, runtime) = runtime(client); + let error = runtime + .discover(CONTENT_ROOT.into(), 0, true) + .await + .unwrap_err(); + let message = error.to_string(); + assert!(message.contains("Internal"), "{message}"); + assert!(!message.contains("/private/host/config")); +} + +#[tokio::test] +async fn workspace_config_edit_shared_argument_rejection_never_commits() { + let client = Router::new(); + let (feature, runtime) = runtime(client.clone()); + feature.attach("initial", None).await.unwrap(); + let path = "/workspace-config/main.dcdl"; + observe_interface(&runtime, path, true).await; + for (args, expected) in [ + ( + json!({"old_string":"old","new_string":"old"}), + "old_string and new_string are identical", + ), + ( + json!({"old_string":"old","new_string":"new","replace_all":"true"}), + "expected boolean", + ), + ] { + let error = call(&runtime, path, "edit", args).await.unwrap_err(); + assert!(error.to_string().contains(expected), "{error:?}"); + let state = client.state.lock().unwrap(); + assert!(state.commits.is_empty()); + assert_eq!(state.revision, 1); + assert_eq!(state.entries["main.dcdl"].1, "model = old"); + // Argument-only rejection must not even fetch a preimage. + assert!( + state + .requests + .iter() + .all(|request| !request.path.ends_with("/read")) + ); + } + // Rejection has not made the observation stale or poisoned the commit path. + call( + &runtime, + path, + "edit", + json!({"old_string":"old","new_string":"新"}), + ) + .await + .unwrap(); + let state = client.state.lock().unwrap(); + assert_eq!(state.commits.len(), 1); + assert_eq!(state.entries["main.dcdl"].1, "model = 新"); +} + +#[tokio::test] +async fn workspace_config_production_ungranted_root_does_not_poison_worldspace_discovery() { + let client = Router::new(); + client.state.lock().unwrap().granted = false; + let (_, runtime) = runtime(client.clone()); + let discovery = runtime.discover("/".into(), 2, true).await.unwrap(); + let discovery: Json = serde_json::from_str(discovery.content.as_deref().unwrap()).unwrap(); + assert_eq!(discovery["known_space"].as_array().unwrap().len(), 1); + assert_eq!(discovery["known_space"][0]["path"], "/"); + assert!( + runtime + .discover(CONTENT_ROOT.into(), 0, true) + .await + .is_err() + ); + assert!( + client + .state + .lock() + .unwrap() + .requests + .iter() + .all(|r| r.method == WorkspaceRequestMethod::Get) + ); +} diff --git a/crates/worker/src/feature/builtin/workspace_config/wip.rs b/crates/worker/src/feature/builtin/workspace_config/wip.rs new file mode 100644 index 000000000..946c89eee --- /dev/null +++ b/crates/worker/src/feature/builtin/workspace_config/wip.rs @@ -0,0 +1,182 @@ +//! Shared native self-attach entrance and production request-time config subtree. + +use std::collections::BTreeMap; +use std::sync::Arc; + +use async_trait::async_trait; +use serde_json::json; +use sha2::{Digest, Sha256}; +use wip_protocol::{ + Documentation, INTERFACE_FORMAT_V1, InterfaceDescriptor, Object, OperationDeclaration, + ParameterDeclaration, ProtocolError, ProtocolErrorCode, ReturnDeclaration, TypeExpr, Value, +}; + +use super::{ + ATTACHMENT_ALIAS, CONTENT_ROOT, ConfigAttachError, WorkspaceConfigFeature, requested_access, +}; +use crate::feature::builtin::manage_workdir::wip::encode_identity; +use crate::wip::{ + WipCallContext, WipMountError, WipMountRegistry, WipOperationError, WipOperationHandler, + WipOperationOutput, WipProjection, WipProjectionKind, json_to_wip, +}; + +/// Mount the Feature's native attachment entrance and, for the production +/// adapter, the grant-bound config subtree. No static file inventory is created. +pub fn mount_workspace_config_wip( + registry: &mut WipMountRegistry, + feature: &WorkspaceConfigFeature, +) -> Result<(), WipMountError> { + mount_workspace_config_attach_wip(registry, feature)?; + if feature.wip_mode + && feature + .client + .workspace_id() + .is_some_and(|id| !id.is_empty()) + && let Some(backend) = &feature.content_backend + { + registry.mount_subtree(crate::wip::WipSubtreeMount { + root: CONTENT_ROOT.into(), + provider: Arc::new(super::content::ConfigProvider { + feature: feature.clone(), + backend: backend.clone(), + }), + })?; + } + Ok(()) +} + +/// Shared root registration; no OS session or duplicate attachment inventory. +pub fn mount_workspace_config_attach_wip( + registry: &mut WipMountRegistry, + feature: &WorkspaceConfigFeature, +) -> Result<(), WipMountError> { + if !feature.wip_mode || feature.client.workspace_id().is_none_or(str::is_empty) { + return Ok(()); + } + registry.allocate_namespace("workspace-config", "workspace-config")?; + registry.mount(attach_projection(feature))?; + Ok(()) +} + +pub(super) fn attach_projection(feature: &WorkspaceConfigFeature) -> WipProjection { + let interface = "yoi.workspace-config/attachment/v1"; + let descriptor = InterfaceDescriptor { + format: INTERFACE_FORMAT_V1.into(), + documentation: Some(Documentation { + summary: "Attach this Workspace's logical configuration".into(), + details: Some("Uses the same Backend ledger operation as /workspace-config. Default access is the effective grant; explicit read_write is never downgraded. No host path, Workdir session or Bash capability is created.".into()), + }), + types: Vec::new(), + operations: vec![OperationDeclaration { + name: "attach".into(), + documentation: Some(Documentation { + summary: "Attach or resolve the existing workspace-config connection".into(), + details: None, + }), + parameters: vec![ParameterDeclaration { + name: "access".into(), + required: false, + documentation: None, + r#type: TypeExpr::String, + }], + returns: ReturnDeclaration { + documentation: None, + r#type: TypeExpr::Json, + }, + }], + }; + let interface_validator = Some(Sha256::digest(format!("{descriptor:?}")).to_vec()); + WipProjection { + route: CONTENT_ROOT.into(), + capability: "workspace-config:attachment".into(), + kind: WipProjectionKind::Native, + object: Object { + name: ATTACHMENT_ALIAS.into(), + description: Some("Logical Workspace configuration attachment entrance".into()), + interfaces: vec![interface.into()], + r#ref: None, + // Attaching does not modify config content. The content subtree will + // supply its own Backend validators for file/directory Operations. + validator: None, + }, + interface: interface.into(), + descriptor, + interface_validator, + handler: Arc::new(AttachHandler(feature.clone())), + } +} + +struct AttachHandler(WorkspaceConfigFeature); + +#[async_trait] +impl WipOperationHandler for AttachHandler { + fn is_visible(&self) -> bool { + self.0.is_available() && self.0.validate_access(None).is_ok() + } + + fn operation_available(&self, operation: &str) -> bool { + operation == "attach" && self.0.validate_access(None).is_ok() + } + + fn contextual_interface(&self) -> bool { + true + } + + async fn call( + &self, + operation: &str, + arguments: &BTreeMap, + context: WipCallContext, + ) -> Result { + if operation != "attach" { + return Err(failure( + ProtocolErrorCode::OperationNotFound, + "Unknown config attachment operation", + )); + } + if arguments.keys().any(|key| key != "access") { + return Err(map_error(ConfigAttachError::InvalidRequest)); + } + let access = match arguments.get("access") { + None => requested_access(None), + Some(Value::String(value)) => requested_access(Some(value)), + _ => Err(ConfigAttachError::InvalidRequest), + } + .map_err(map_error)?; + // Host execution identity, never a model-supplied source or grant. + let key = format!("wip:{}", context.execution.execution_id()); + let result = self.0.attach(&key, access).await.map_err(map_error)?; + let value = json_to_wip(&json!({ + "alias": ATTACHMENT_ALIAS, + "purpose": "Workspace configuration", + "access": result.access.as_str(), + "connection_id": result.connection_id, + "attachment_path": format!("/workdir-attachments/{}", encode_identity(&result.connection_id)), + "content_path": CONTENT_ROOT, + "already_attached": result.already_attached, + })).map_err(|_| WipOperationError::OutcomeUnknown("Completed config attachment response unavailable".into()))?; + Ok(WipOperationOutput::native(value)) + } +} + +fn failure(code: ProtocolErrorCode, message: &str) -> WipOperationError { + WipOperationError::Protocol(ProtocolError { + code, + message: message.into(), + }) +} + +fn map_error(error: ConfigAttachError) -> WipOperationError { + match error { + ConfigAttachError::InvalidRequest => { + failure(ProtocolErrorCode::InvalidArguments, error.message()) + } + ConfigAttachError::Denied => failure(ProtocolErrorCode::PermissionDenied, error.message()), + ConfigAttachError::Unavailable => { + failure(ProtocolErrorCode::ResourceLimitExceeded, error.message()) + } + ConfigAttachError::OutcomeUnknown => { + WipOperationError::OutcomeUnknown(error.message().into()) + } + } +} diff --git a/crates/worker/src/feature/builtin/workspace_config_tests.rs b/crates/worker/src/feature/builtin/workspace_config_tests.rs new file mode 100644 index 000000000..1f5a1763f --- /dev/null +++ b/crates/worker/src/feature/builtin/workspace_config_tests.rs @@ -0,0 +1,334 @@ +//! Backend-independent preparation tests; not config persistence/E2E evidence. +use super::*; +use std::collections::BTreeMap; +use std::sync::Mutex; +use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; + +use agen::tool::ToolExecutionContext; +use protocol::InvocationArgumentValue; +use wip_protocol::{ProtocolErrorCode, Value}; + +use crate::feature::{FeatureInvocationRegistry, FeatureRegistryBuilder}; +use crate::hook::HookRegistryBuilder; +use crate::wip::{WipCallContext, WipMountRegistry, WipOperationError}; +use crate::worker::{WorkspaceClientError, WorkspaceRequest, WorkspaceResponse}; + +#[derive(Debug)] +struct Client { + workspace: Option<&'static str>, + available: bool, +} +impl WorkspaceClient for Client { + fn workspace_id(&self) -> Option<&str> { + self.workspace + } + fn kind(&self) -> &str { + "test" + } + fn is_available(&self) -> bool { + self.available + } + fn execute(&self, _: WorkspaceRequest) -> Result { + panic!("scaffolding must not invent Backend endpoints"); + } +} + +struct Backend { + granted: AtomicBool, + read_only: AtomicBool, + unknown: AtomicBool, + calls: AtomicUsize, + keys: Mutex>, + active: Mutex>, +} +impl Backend { + fn new() -> Self { + Self { + granted: AtomicBool::new(true), + read_only: AtomicBool::new(false), + unknown: AtomicBool::new(false), + calls: AtomicUsize::new(0), + keys: Mutex::new(Vec::new()), + active: Mutex::new(None), + } + } +} +#[async_trait] +impl WorkspaceConfigAttachmentBackend for Backend { + fn is_available(&self) -> bool { + self.granted.load(Ordering::SeqCst) + } + fn validate_access(&self, access: Option) -> Result<(), ConfigAttachError> { + if !self.granted.load(Ordering::SeqCst) + || (self.read_only.load(Ordering::SeqCst) && access == Some(ConfigAccess::ReadWrite)) + { + return Err(ConfigAttachError::Denied); + } + Ok(()) + } + async fn attach( + &self, + key: &str, + access: Option, + ) -> Result { + self.validate_access(access)?; + self.calls.fetch_add(1, Ordering::SeqCst); + self.keys.lock().unwrap().push(key.into()); + if self.unknown.load(Ordering::SeqCst) { + return Err(ConfigAttachError::OutcomeUnknown); + } + let mut active = self.active.lock().unwrap(); + let already_attached = active.is_some(); + let access = access.unwrap_or(if self.read_only.load(Ordering::SeqCst) { + ConfigAccess::ReadOnly + } else { + ConfigAccess::ReadWrite + }); + let result = ConfigAttachment { + connection_id: "connection-1".into(), + access, + already_attached, + }; + *active = Some(result.clone()); + Ok(result) + } +} + +fn feature( + backend: Arc, + wip_mode: bool, + workspace: Option<&'static str>, +) -> WorkspaceConfigFeature { + WorkspaceConfigFeature::new( + Arc::new(Client { + workspace, + available: true, + }), + backend, + wip_mode, + ) +} +fn registry(feature: WorkspaceConfigFeature) -> FeatureInvocationRegistry { + let mut tools = Vec::new(); + let report = FeatureRegistryBuilder::new() + .with_module(feature) + .install_into_pending(&mut tools, &mut HookRegistryBuilder::default()); + assert!(!report.has_errors()); + assert!(tools.is_empty(), "no config compatibility tools"); + report.chat_invocations +} +fn invocation(key: &str, access: Option<&str>) -> FeatureInvocation { + FeatureInvocation { + invocation_id: key.into(), + identity: FeatureInvocationIdentity(INVOCATION_ID.into()), + name: FEATURE_ID.into(), + arguments: access + .into_iter() + .map(|value| InvocationArgumentValue { + name: "access".into(), + value: InvocationValue::String(value.into()), + }) + .collect(), + } +} +fn context() -> WipCallContext { + WipCallContext { + execution: ToolExecutionContext::direct(), + security_context: "test-worker".into(), + } +} + +#[tokio::test] +async fn workspace_config_common_completion_selection_and_preparation_use_backend_ledger() { + let backend = Arc::new(Backend::new()); + let registry = registry(feature(backend.clone(), true, Some("workspace-1"))); + let completion = registry.feature_completions("workspace-c"); + assert_eq!(completion.len(), 1); + let descriptor = completion[0].invocation.as_ref().unwrap(); + assert_eq!(descriptor.identity.0, INVOCATION_ID); + assert!(descriptor.client_adapter.is_none()); + assert_eq!( + registry + .argument_completions(&descriptor.identity, "access", "read_") + .await + .into_iter() + .map(|entry| entry.value) + .collect::>(), + ["read_only", "read_write"] + ); + + let first = registry + .invoke(&invocation("selected-1", None)) + .await + .unwrap(); + assert_eq!(first.status, FeatureInvocationStatus::Succeeded); + assert!(first.message.contains("read_write")); + assert!(first.context.unwrap().contains("WIP `/workspace-config`")); + let second = registry + .invoke(&invocation("selected-2", None)) + .await + .unwrap(); + assert!(second.message.starts_with("Already attached")); + assert_eq!(*backend.keys.lock().unwrap(), ["selected-1", "selected-2"]); +} + +#[tokio::test] +async fn workspace_config_slash_and_wip_self_attach_share_handler_and_identity() { + let backend = Arc::new(Backend::new()); + let feature = feature(backend.clone(), true, Some("workspace-1")); + let slash = registry(feature.clone()); + slash.invoke(&invocation("slash-1", None)).await.unwrap(); + let projection = wip::attach_projection(&feature); + assert_eq!(projection.route, CONTENT_ROOT); + assert_eq!(projection.descriptor.operations.len(), 1); + let result = projection + .handler + .call("attach", &BTreeMap::new(), context()) + .await; + assert!(result.is_ok()); + assert_eq!(backend.calls.load(Ordering::SeqCst), 2); + assert_eq!( + backend + .active + .lock() + .unwrap() + .as_ref() + .unwrap() + .connection_id, + "connection-1" + ); + assert!(backend.keys.lock().unwrap()[1].starts_with("wip:")); +} + +#[tokio::test] +async fn workspace_config_effective_access_and_explicit_read_write_do_not_silently_downgrade() { + let backend = Arc::new(Backend::new()); + backend.read_only.store(true, Ordering::SeqCst); + let feature = feature(backend.clone(), true, Some("workspace-1")); + let registry = registry(feature.clone()); + let result = registry + .invoke(&invocation("effective-1", None)) + .await + .unwrap(); + assert!(result.message.contains("read_only")); + assert!( + registry + .invoke(&invocation("write-1", Some("read_write"))) + .await + .is_err() + ); + assert_eq!(backend.calls.load(Ordering::SeqCst), 1); + let args = BTreeMap::from([("access".into(), Value::String("read_write".into()))]); + assert!( + matches!(wip::attach_projection(&feature).handler.call("attach", &args, context()).await, + Err(WipOperationError::Protocol(error)) if error.code == ProtocolErrorCode::PermissionDenied) + ); + assert_eq!(backend.calls.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn workspace_config_old_handler_and_replay_validation_recheck_revocation() { + let backend = Arc::new(Backend::new()); + let feature = feature(backend.clone(), true, Some("workspace-1")); + let old = wip::attach_projection(&feature); + let registry = registry(feature); + let request = invocation("replay-1", None); + registry.invoke(&request).await.unwrap(); + backend.granted.store(false, Ordering::SeqCst); + assert!(registry.feature_completions("").is_empty()); + assert!(registry.validate(&request).is_err()); + assert!(!old.handler.is_visible()); + assert!( + matches!(old.handler.call("attach", &BTreeMap::new(), context()).await, + Err(WipOperationError::Protocol(error)) if error.code == ProtocolErrorCode::PermissionDenied) + ); + assert_eq!(backend.calls.load(Ordering::SeqCst), 1); +} + +#[tokio::test] +async fn workspace_config_unknown_attach_is_not_retried_and_never_reports_success() { + let backend = Arc::new(Backend::new()); + backend.unknown.store(true, Ordering::SeqCst); + let feature = feature(backend.clone(), true, Some("workspace-1")); + let registry = registry(feature.clone()); + let error = registry + .invoke(&invocation("unknown-1", None)) + .await + .unwrap_err(); + assert!(error.outcome_unknown); + assert_eq!(backend.calls.load(Ordering::SeqCst), 1); + assert!(matches!( + wip::attach_projection(&feature) + .handler + .call("attach", &BTreeMap::new(), context()) + .await, + Err(WipOperationError::OutcomeUnknown(_)) + )); + assert_eq!(backend.calls.load(Ordering::SeqCst), 2); +} + +#[tokio::test] +async fn workspace_config_non_wip_and_missing_workspace_have_no_fallback() { + for (wip_mode, workspace) in [(false, Some("workspace-1")), (true, None), (true, Some(""))] { + let backend = Arc::new(Backend::new()); + let feature = feature(backend.clone(), wip_mode, workspace); + assert!(registry(feature.clone()).descriptors().is_empty()); + let mut mounts = WipMountRegistry::new(); + wip::mount_workspace_config_attach_wip(&mut mounts, &feature).unwrap(); + assert_eq!(mounts.routes().count(), 0); + assert_eq!( + feature.attach("not-enabled", None).await, + Err(ConfigAttachError::Denied) + ); + assert_eq!(backend.calls.load(Ordering::SeqCst), 0); + } +} + +#[tokio::test] +async fn workspace_config_rejects_unstructured_authority_and_invalid_access_before_attach() { + let backend = Arc::new(Backend::new()); + let feature = feature(backend.clone(), true, Some("workspace-1")); + let registry = registry(feature.clone()); + assert!( + registry + .invoke(&invocation("bad-1", Some("admin"))) + .await + .is_err() + ); + let projection = wip::attach_projection(&feature); + let args = BTreeMap::from([("workspace_id".into(), Value::String("another".into()))]); + assert!( + matches!(projection.handler.call("attach", &args, context()).await, + Err(WipOperationError::Protocol(error)) if error.code == ProtocolErrorCode::InvalidArguments) + ); + let mut bad = invocation("bad-2", None); + bad.arguments.push(InvocationArgumentValue { + name: "access".into(), + value: InvocationValue::String("true".into()), + }); + assert!(registry.invoke(&bad).await.is_err()); + assert_eq!(backend.calls.load(Ordering::SeqCst), 0); +} + +#[test] +fn workspace_config_controller_gate_requires_configured_workspace_wip_and_not_a_grant() { + for enabled in [false, true] { + for wip in [false, true] { + for workspace in [None, Some(""), Some("workspace-1"), Some("bad\nworkspace")] { + for available in [false, true] { + let client = Arc::new(Client { + workspace, + available, + }); + let expected = enabled && wip && available && workspace == Some("workspace-1"); + // Registration needs identity/mode, but performs no grant or + // filesystem mutation. Runtime authorization is still Backend's. + assert_eq!( + WorkspaceConfigFeature::configured(client, enabled, wip).is_some(), + expected + ); + } + } + } + } +} diff --git a/crates/worker/src/feature/invocation.rs b/crates/worker/src/feature/invocation.rs new file mode 100644 index 000000000..a4a71a1fe --- /dev/null +++ b/crates/worker/src/feature/invocation.rs @@ -0,0 +1,967 @@ +use std::collections::{BTreeMap, BTreeSet}; +use std::fmt; +use std::sync::Arc; + +use async_trait::async_trait; +use protocol::{ + CompletionEntry, FeatureInvocation, FeatureInvocationDescriptor, FeatureInvocationIdentity, + FeatureInvocationResult, InvocationCompletion, InvocationValue, validate_feature_invocation, +}; + +use super::{FeatureId, FeatureInstallError}; + +pub(crate) const MAX_INVOCATION_MESSAGE_BYTES: usize = 4 * 1024; +pub(crate) const MAX_INVOCATION_CONTEXT_BYTES: usize = 16 * 1024; +const MAX_COMPLETION_ENTRIES: usize = 64; +const MAX_COMPLETION_VALUE_BYTES: usize = 1024; +const MAX_COMPLETION_DESCRIPTION_BYTES: usize = 2 * 1024; +const MAX_COMPLETION_USAGE_BYTES: usize = 4 * 1024; +const MAX_INVOCATION_DESCRIPTOR_BYTES: usize = + protocol::invocation::MAX_FEATURE_INVOCATION_DESCRIPTOR_BYTES; +const MAX_COMPLETION_RESPONSE_BYTES: usize = 64 * 1024; + +pub(crate) fn invocation_result_is_bounded(result: &FeatureInvocationResult) -> bool { + result.message.len() <= MAX_INVOCATION_MESSAGE_BYTES + && result + .context + .as_ref() + .is_none_or(|context| context.len() <= MAX_INVOCATION_CONTEXT_BYTES) +} + +fn truncate_display(text: &mut String, max_bytes: usize) { + let mut end = text.len().min(max_bytes); + while !text.is_char_boundary(end) { + end -= 1; + } + text.truncate(end); +} + +/// Values are never truncated: doing so could silently select a different +/// operation. Display-only text may be shortened; oversized metadata is dropped. +pub(crate) fn bounded_completion_entries( + entries: impl IntoIterator, +) -> Vec { + let mut result = Vec::new(); + let mut bytes = 2; // JSON array delimiters. + for mut entry in entries.into_iter().take(MAX_COMPLETION_ENTRIES * 4) { + if entry.value.len() > MAX_COMPLETION_VALUE_BYTES { + continue; + } + if let Some(description) = &mut entry.description { + truncate_display(description, MAX_COMPLETION_DESCRIPTION_BYTES); + } + if let Some(usage) = &mut entry.usage { + truncate_display(usage, MAX_COMPLETION_USAGE_BYTES); + } + if entry.invocation.as_ref().is_some_and(|descriptor| { + serde_json::to_vec(descriptor).map_or(true, |encoded| { + encoded.len() > MAX_INVOCATION_DESCRIPTOR_BYTES + }) + }) { + continue; + } + let Ok(encoded) = serde_json::to_vec(&entry) else { + continue; + }; + let additional = encoded.len() + usize::from(!result.is_empty()); + if bytes + additional > MAX_COMPLETION_RESPONSE_BYTES { + continue; + } + bytes += additional; + result.push(entry); + if result.len() == MAX_COMPLETION_ENTRIES { + break; + } + } + result +} + +/// Host-owned execution context. `invocation_id` is the idempotency key a +/// handler must use when its operation has external side effects. Workspace, +/// credential, and permission authority remain captured by the handler itself. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct FeatureInvocationContext { + pub invocation_id: String, +} + +#[async_trait] +pub trait FeatureInvocationHandler: Send + Sync { + /// Side-effect-free runtime availability/permission check. Installation is + /// necessary but not sufficient: discovery and execution both consult this. + fn is_available(&self) -> bool { + true + } + + /// Side-effect-free semantic and authority validation. Installed handlers + /// retain Workspace/credential/permission authority; request arguments never + /// supply it. The Worker calls this before persisting an execution start + /// and before reusing successful preparation context on replay or resume. + fn validate( + &self, + _invocation: &FeatureInvocation, + ) -> Result<(), FeatureInvocationHandlerError> { + Ok(()) + } + + async fn invoke( + &self, + context: FeatureInvocationContext, + invocation: &FeatureInvocation, + ) -> Result; +} + +#[derive(Debug, Clone, PartialEq, Eq, thiserror::Error)] +#[error("{message}")] +pub struct FeatureInvocationHandlerError { + pub message: String, + /// True only when the handler cannot determine whether its business-side + /// effect committed. The Worker never automatically retries this outcome. + pub outcome_unknown: bool, +} + +impl FeatureInvocationHandlerError { + pub fn failed(message: impl Into) -> Self { + Self { + message: message.into(), + outcome_unknown: false, + } + } + + pub fn outcome_unknown(message: impl Into) -> Self { + Self { + message: message.into(), + outcome_unknown: true, + } + } +} + +#[async_trait] +pub trait FeatureInvocationCompletionProvider: Send + Sync { + /// The provider selector comes only from installed host metadata, never + /// from completion-request input. Providers retain their own scoped authority. + async fn complete( + &self, + completion: &InvocationCompletion, + argument: &str, + prefix: &str, + ) -> Vec; +} + +#[derive(Clone)] +struct RegisteredInvocation { + descriptor: FeatureInvocationDescriptor, + handler: Arc, + completion_provider: Option>, +} + +/// Installed invocation metadata and executable handlers. Equality/debug expose +/// only public descriptors; implementation objects and credentials never leak. +#[derive(Clone, Default)] +pub struct FeatureInvocationRegistry { + entries: BTreeMap, + names: BTreeMap, +} + +impl fmt::Debug for FeatureInvocationRegistry { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter + .debug_struct("FeatureInvocationRegistry") + .field("descriptors", &self.descriptors()) + .finish() + } +} + +impl PartialEq for FeatureInvocationRegistry { + fn eq(&self, other: &Self) -> bool { + self.descriptors() == other.descriptors() + } +} +impl Eq for FeatureInvocationRegistry {} + +impl FeatureInvocationRegistry { + pub fn descriptors(&self) -> Vec { + self.entries + .values() + .filter(|entry| entry.handler.is_available()) + .map(|entry| entry.descriptor.clone()) + .collect() + } + + pub fn descriptor( + &self, + identity: &FeatureInvocationIdentity, + ) -> Option<&FeatureInvocationDescriptor> { + self.entries + .get(identity) + .filter(|entry| entry.handler.is_available()) + .map(|entry| &entry.descriptor) + } + + pub fn feature_completions(&self, prefix: &str) -> Vec { + bounded_completion_entries( + self.entries + .values() + .filter(|entry| entry.handler.is_available()) + .filter(|entry| { + entry.descriptor.name.starts_with(prefix) + || entry + .descriptor + .aliases + .iter() + .any(|alias| alias.starts_with(prefix)) + }) + .map(|entry| CompletionEntry { + value: entry.descriptor.name.clone(), + is_dir: false, + description: Some(entry.descriptor.description.clone()), + usage: Some(entry.descriptor.usage()), + invocation: Some(entry.descriptor.clone()), + }), + ) + } + + pub fn argument_name_completions( + &self, + identity: &FeatureInvocationIdentity, + prefix: &str, + ) -> Vec { + bounded_completion_entries( + self.entries + .get(identity) + .filter(|entry| entry.handler.is_available()) + .into_iter() + .flat_map(|entry| entry.descriptor.arguments.iter()) + .filter(|argument| argument.name.starts_with(prefix)) + .map(|argument| CompletionEntry { + value: format!("{}=", argument.name), + is_dir: false, + description: argument.description.clone(), + usage: None, + invocation: None, + }), + ) + } + + pub async fn argument_completions( + &self, + identity: &FeatureInvocationIdentity, + argument: &str, + prefix: &str, + ) -> Vec { + let Some(entry) = self + .entries + .get(identity) + .filter(|entry| entry.handler.is_available()) + else { + return Vec::new(); + }; + let Some(descriptor) = entry + .descriptor + .arguments + .iter() + .find(|candidate| candidate.name == argument) + else { + return Vec::new(); + }; + let strings = match &descriptor.completion { + InvocationCompletion::None => match &descriptor.value_type { + protocol::InvocationArgumentType::Boolean => ["true", "false"] + .into_iter() + .filter(|value| value.starts_with(prefix)) + .map(str::to_owned) + .collect(), + protocol::InvocationArgumentType::Enum { values } => values + .iter() + .filter(|value| value.starts_with(prefix)) + .cloned() + .collect(), + _ => Vec::new(), + }, + InvocationCompletion::WorkerFile | InvocationCompletion::ClientFile => Vec::new(), + InvocationCompletion::Static { values } => values + .iter() + .filter(|value| value.starts_with(prefix)) + .cloned() + .collect(), + InvocationCompletion::Provider { .. } => { + return match &entry.completion_provider { + Some(provider) => { + let entries = provider + .complete(&descriptor.completion, argument, prefix) + .await; + if !entry.handler.is_available() { + return Vec::new(); + } + bounded_completion_entries(entries.into_iter().map(|mut candidate| { + // Argument providers contribute values/display text, + // never new executable invocation metadata. + candidate.invocation = None; + candidate + })) + } + None => Vec::new(), + }; + } + }; + bounded_completion_entries(strings.into_iter().map(|value| CompletionEntry { + value, + is_dir: false, + description: descriptor.description.clone(), + usage: None, + invocation: None, + })) + } + + pub(crate) fn validate_metadata( + &self, + invocation: &FeatureInvocation, + ) -> Result<(), FeatureInvocationHandlerError> { + let entry = self + .entries + .get(&invocation.identity) + .filter(|entry| entry.handler.is_available()) + .ok_or_else(|| { + FeatureInvocationHandlerError::failed( + "Feature invocation is not enabled or currently available", + ) + })?; + if entry.descriptor.client_adapter.is_some() { + return Err(FeatureInvocationHandlerError::failed( + "client-local invocation must be staged by the client, not executed by the Worker", + )); + } + validate_feature_invocation(&entry.descriptor, invocation).map_err(|error| { + let mut message = error.to_string(); + truncate_display(&mut message, MAX_INVOCATION_MESSAGE_BYTES); + FeatureInvocationHandlerError::failed(message) + }) + } + + pub fn validate( + &self, + invocation: &FeatureInvocation, + ) -> Result<(), FeatureInvocationHandlerError> { + self.validate_metadata(invocation)?; + self.entries[&invocation.identity] + .handler + .validate(invocation) + .map_err(|mut error| { + // Validation is side-effect-free; bound its diagnostic before + // any start, without claiming an uncertain business outcome. + truncate_display(&mut error.message, MAX_INVOCATION_MESSAGE_BYTES); + error + }) + } + + pub async fn invoke( + &self, + invocation: &FeatureInvocation, + ) -> Result { + self.validate(invocation)?; + let entry = &self.entries[&invocation.identity]; + let result = entry + .handler + .invoke( + FeatureInvocationContext { + invocation_id: invocation.invocation_id.clone(), + }, + invocation, + ) + .await + .map_err(|error| { + if error.message.len() > MAX_INVOCATION_MESSAGE_BYTES { + FeatureInvocationHandlerError::outcome_unknown( + "handler error exceeded the result byte limit; do not retry automatically", + ) + } else { + error + } + })?; + if !invocation_result_is_bounded(&result) { + return Err(FeatureInvocationHandlerError::outcome_unknown( + "handler result exceeded the message/context byte limit; do not retry automatically", + )); + } + if result.invocation_id != invocation.invocation_id + || result.identity != invocation.identity + { + return Err(FeatureInvocationHandlerError::outcome_unknown( + "handler returned a result for a different invocation; do not retry automatically", + )); + } + if result.status != protocol::FeatureInvocationStatus::Succeeded && result.context.is_some() + { + return Err(FeatureInvocationHandlerError::outcome_unknown( + "handler returned preparation context without a successful result", + )); + } + if result.status == protocol::FeatureInvocationStatus::Succeeded + && !entry.handler.is_available() + { + return Err(FeatureInvocationHandlerError::failed( + "handler reported success but the invocation is no longer available; preparation context withheld; do not retry automatically", + )); + } + Ok(result) + } + + pub(crate) fn register( + &mut self, + feature_id: FeatureId, + descriptor: FeatureInvocationDescriptor, + handler: Arc, + completion_provider: Option>, + ) -> Result<(), FeatureInstallError> { + if descriptor.identity.0.len() > 256 + || serde_json::to_vec(&descriptor).map_or(true, |encoded| { + encoded.len() > MAX_INVOCATION_DESCRIPTOR_BYTES + }) + { + return Err(FeatureInstallError::InvalidDescriptor( + "invocation metadata exceeds the host byte limit".into(), + )); + } + descriptor + .validate() + .map_err(|error| FeatureInstallError::InvalidDescriptor(error.to_string()))?; + let expected_prefix = format!("{feature_id}/"); + if !descriptor.identity.0.starts_with(&expected_prefix) { + return Err(FeatureInstallError::InvalidDescriptor(format!( + "invocation identity `{}` must be qualified by feature `{feature_id}`", + descriptor.identity + ))); + } + if self.entries.contains_key(&descriptor.identity) { + return Err(FeatureInstallError::DuplicateInvocationIdentity { + identity: descriptor.identity.to_string(), + }); + } + let public_names = std::iter::once(descriptor.name.as_str()) + .chain(descriptor.aliases.iter().map(String::as_str)) + .collect::>(); + for name in &public_names { + if let Some(existing) = self.names.get(*name) { + return Err(FeatureInstallError::DuplicateInvocationName { + name: (*name).to_string(), + first: existing.to_string(), + duplicate: descriptor.identity.to_string(), + }); + } + } + for name in public_names { + self.names + .insert(name.to_string(), descriptor.identity.clone()); + } + self.entries.insert( + descriptor.identity.clone(), + RegisteredInvocation { + descriptor, + handler, + completion_provider, + }, + ); + Ok(()) + } + + pub fn argument_value<'a>( + invocation: &'a FeatureInvocation, + name: &str, + ) -> Option<&'a InvocationValue> { + invocation + .arguments + .iter() + .find(|argument| argument.name == name) + .map(|argument| &argument.value) + } +} + +pub struct InvocationContributionRegistrar<'a> { + pub(super) feature_id: &'a FeatureId, + pub(super) declarations: &'a [FeatureInvocationDescriptor], + pub(super) registry: &'a mut FeatureInvocationRegistry, + pub(super) installed: &'a mut Vec, +} + +impl InvocationContributionRegistrar<'_> { + pub fn register( + &mut self, + descriptor: FeatureInvocationDescriptor, + handler: H, + ) -> Result<(), FeatureInstallError> + where + H: FeatureInvocationHandler + 'static, + { + self.register_with_provider(descriptor, Arc::new(handler), None) + } + + pub fn register_with_completion( + &mut self, + descriptor: FeatureInvocationDescriptor, + handler: H, + provider: P, + ) -> Result<(), FeatureInstallError> + where + H: FeatureInvocationHandler + 'static, + P: FeatureInvocationCompletionProvider + 'static, + { + self.register_with_provider(descriptor, Arc::new(handler), Some(Arc::new(provider))) + } + + fn register_with_provider( + &mut self, + descriptor: FeatureInvocationDescriptor, + handler: Arc, + provider: Option>, + ) -> Result<(), FeatureInstallError> { + if provider.is_none() + && descriptor.arguments.iter().any(|argument| { + matches!(argument.completion, InvocationCompletion::Provider { .. }) + }) + { + return Err(FeatureInstallError::InvalidDescriptor(format!( + "invocation `{}` declares a completion provider but did not register one", + descriptor.identity + ))); + } + if !self.declarations.contains(&descriptor) { + return Err(FeatureInstallError::UndeclaredContribution { + kind: super::FeatureContributionKind::ChatInvocation, + name: descriptor.identity.to_string(), + feature: self.feature_id.to_string(), + }); + } + self.registry.register( + self.feature_id.clone(), + descriptor.clone(), + handler, + provider, + )?; + self.installed.push(descriptor); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use protocol::{ + FeatureInvocationStatus, FeatureInvocationSyntax, InvocationArgumentDescriptor, + InvocationArgumentType, InvocationArgumentValue, + }; + use std::sync::atomic::{AtomicUsize, Ordering}; + + fn descriptor(feature: &str, name: &str) -> FeatureInvocationDescriptor { + FeatureInvocationDescriptor { + identity: FeatureInvocationIdentity(format!("{feature}/{name}")), + name: name.into(), + aliases: Vec::new(), + display_name: name.into(), + description: format!("invoke {name}"), + syntax: FeatureInvocationSyntax::Parenthesized, + arguments: vec![InvocationArgumentDescriptor { + name: "value".into(), + position: Some(0), + required: true, + value_type: InvocationArgumentType::String, + completion: InvocationCompletion::Static { + values: vec!["alpha".into(), "beta".into()], + }, + description: Some("selected value".into()), + }], + client_adapter: None, + } + } + + struct CountingHandler(Arc); + + #[async_trait] + impl FeatureInvocationHandler for CountingHandler { + async fn invoke( + &self, + _context: FeatureInvocationContext, + invocation: &FeatureInvocation, + ) -> Result { + self.0.fetch_add(1, Ordering::SeqCst); + Ok(FeatureInvocationResult { + invocation_id: invocation.invocation_id.clone(), + identity: invocation.identity.clone(), + status: FeatureInvocationStatus::Succeeded, + message: "done".into(), + context: Some("prepared".into()), + }) + } + } + + fn invocation(descriptor: &FeatureInvocationDescriptor) -> FeatureInvocation { + FeatureInvocation { + invocation_id: "invoke-1".into(), + identity: descriptor.identity.clone(), + name: descriptor.name.clone(), + arguments: vec![InvocationArgumentValue { + name: "value".into(), + value: InvocationValue::String("alpha".into()), + }], + } + } + + #[tokio::test] + async fn installed_registry_validates_before_calling_handler_and_returns_context() { + let calls = Arc::new(AtomicUsize::new(0)); + let descriptor = descriptor("builtin:test", "run"); + let mut registry = FeatureInvocationRegistry::default(); + registry + .register( + FeatureId::builtin("test"), + descriptor.clone(), + Arc::new(CountingHandler(calls.clone())), + None, + ) + .unwrap(); + + let result = registry.invoke(&invocation(&descriptor)).await.unwrap(); + assert_eq!(result.status, FeatureInvocationStatus::Succeeded); + assert_eq!(result.context.as_deref(), Some("prepared")); + assert_eq!(calls.load(Ordering::SeqCst), 1); + + let mut malformed = invocation(&descriptor); + malformed.arguments.clear(); + assert!(registry.invoke(&malformed).await.is_err()); + assert_eq!(calls.load(Ordering::SeqCst), 1); + } + + #[test] + fn registration_rejects_public_name_collisions_across_source_qualified_features() { + let calls = Arc::new(AtomicUsize::new(0)); + let mut registry = FeatureInvocationRegistry::default(); + registry + .register( + FeatureId::builtin("first"), + descriptor("builtin:first", "run"), + Arc::new(CountingHandler(calls.clone())), + None, + ) + .unwrap(); + let error = registry + .register( + FeatureId::builtin("second"), + descriptor("builtin:second", "run"), + Arc::new(CountingHandler(calls)), + None, + ) + .unwrap_err(); + assert!(matches!( + error, + FeatureInstallError::DuplicateInvocationName { .. } + )); + } + + #[tokio::test] + async fn client_local_adapter_metadata_cannot_authorize_worker_execution() { + let calls = Arc::new(AtomicUsize::new(0)); + let descriptor = crate::feature::builtin::chat_invocation::attach_invocation_descriptor(); + let mut registry = FeatureInvocationRegistry::default(); + registry + .register( + FeatureId::builtin("attachments"), + descriptor.clone(), + Arc::new(CountingHandler(calls.clone())), + None, + ) + .unwrap(); + let invocation = FeatureInvocation { + invocation_id: "client-path".into(), + identity: descriptor.identity, + name: descriptor.name, + arguments: vec![InvocationArgumentValue { + name: "file".into(), + value: InvocationValue::String("/client/private/file".into()), + }], + }; + assert!(registry.invoke(&invocation).await.is_err()); + assert_eq!(calls.load(Ordering::SeqCst), 0); + } + + struct CountingCompletion(Arc); + #[async_trait] + impl FeatureInvocationCompletionProvider for CountingCompletion { + async fn complete( + &self, + _completion: &InvocationCompletion, + _argument: &str, + _prefix: &str, + ) -> Vec { + self.0.fetch_add(1, Ordering::SeqCst); + Vec::new() + } + } + + #[tokio::test] + async fn completion_dispatch_uses_installed_argument_metadata_not_provider_presence() { + let calls = Arc::new(AtomicUsize::new(0)); + let provider_calls = Arc::new(AtomicUsize::new(0)); + let descriptor = descriptor("builtin:test", "run"); + let mut registry = FeatureInvocationRegistry::default(); + registry + .register( + FeatureId::builtin("test"), + descriptor.clone(), + Arc::new(CountingHandler(calls.clone())), + Some(Arc::new(CountingCompletion(provider_calls.clone()))), + ) + .unwrap(); + assert_eq!( + registry + .argument_completions(&descriptor.identity, "value", "a") + .await[0] + .value, + "alpha" + ); + assert!( + registry + .argument_completions( + &FeatureInvocationIdentity("foreign:feature/run".into()), + "value", + "" + ) + .await + .is_empty() + ); + assert!( + registry + .argument_completions(&descriptor.identity, "credential", "") + .await + .is_empty() + ); + assert_eq!(provider_calls.load(Ordering::SeqCst), 0); + assert_eq!(calls.load(Ordering::SeqCst), 0); + } + + struct AvailabilityHandler { + available: Arc, + calls: Arc, + } + #[async_trait] + impl FeatureInvocationHandler for AvailabilityHandler { + fn is_available(&self) -> bool { + self.available.load(Ordering::SeqCst) + } + async fn invoke( + &self, + context: FeatureInvocationContext, + invocation: &FeatureInvocation, + ) -> Result { + CountingHandler(self.calls.clone()) + .invoke(context, invocation) + .await + } + } + + struct TestProvider { + entries: Vec, + revoke: Option>, + } + #[async_trait] + impl FeatureInvocationCompletionProvider for TestProvider { + async fn complete( + &self, + _completion: &InvocationCompletion, + _argument: &str, + _prefix: &str, + ) -> Vec { + if let Some(available) = &self.revoke { + available.store(false, Ordering::SeqCst); + } + self.entries.clone() + } + } + + #[tokio::test] + async fn provider_output_is_bounded_and_cannot_inject_invocation_metadata() { + let available = Arc::new(std::sync::atomic::AtomicBool::new(true)); + let mut descriptor = descriptor("builtin:test", "run"); + descriptor.arguments[0].completion = InvocationCompletion::Provider { + provider: "choices".into(), + }; + let entries = (0..128) + .map(|index| CompletionEntry { + value: format!("value-{index}"), + is_dir: false, + description: Some("é".repeat(MAX_COMPLETION_DESCRIPTION_BYTES)), + usage: Some("usage".repeat(MAX_COMPLETION_USAGE_BYTES)), + invocation: Some(descriptor.clone()), + }) + .chain([CompletionEntry { + value: "x".repeat(MAX_COMPLETION_VALUE_BYTES + 1), + is_dir: false, + description: None, + usage: None, + invocation: None, + }]) + .collect(); + let mut registry = FeatureInvocationRegistry::default(); + registry + .register( + FeatureId::builtin("test"), + descriptor.clone(), + Arc::new(AvailabilityHandler { + available, + calls: Arc::new(AtomicUsize::new(0)), + }), + Some(Arc::new(TestProvider { + entries, + revoke: None, + })), + ) + .unwrap(); + let candidates = registry + .argument_completions(&descriptor.identity, "value", "") + .await; + assert!(!candidates.is_empty()); + assert!(candidates.len() <= MAX_COMPLETION_ENTRIES); + assert!(serde_json::to_vec(&candidates).unwrap().len() <= MAX_COMPLETION_RESPONSE_BYTES); + assert!(candidates.iter().all(|candidate| { + candidate.value.len() <= MAX_COMPLETION_VALUE_BYTES + && candidate.invocation.is_none() + && candidate.description.as_ref().unwrap().len() <= MAX_COMPLETION_DESCRIPTION_BYTES + && candidate.usage.as_ref().unwrap().len() <= MAX_COMPLETION_USAGE_BYTES + })); + let simple = (0..128).map(|index| CompletionEntry { + value: format!("value-{index}"), + is_dir: false, + description: None, + usage: None, + invocation: None, + }); + assert_eq!( + bounded_completion_entries(simple).len(), + MAX_COMPLETION_ENTRIES + ); + assert!( + bounded_completion_entries([CompletionEntry { + value: "x".repeat(MAX_COMPLETION_VALUE_BYTES + 1), + is_dir: false, + description: None, + usage: None, + invocation: None, + }]) + .is_empty() + ); + } + + #[tokio::test] + async fn provider_revocation_during_completion_discards_its_output() { + let available = Arc::new(std::sync::atomic::AtomicBool::new(true)); + let calls = Arc::new(AtomicUsize::new(0)); + let mut descriptor = descriptor("builtin:test", "run"); + descriptor.arguments[0].completion = InvocationCompletion::Provider { + provider: "choices".into(), + }; + let mut registry = FeatureInvocationRegistry::default(); + registry + .register( + FeatureId::builtin("test"), + descriptor.clone(), + Arc::new(AvailabilityHandler { + available: available.clone(), + calls: calls.clone(), + }), + Some(Arc::new(TestProvider { + entries: vec![CompletionEntry { + value: "private".into(), + is_dir: false, + description: None, + usage: None, + invocation: None, + }], + revoke: Some(available), + })), + ) + .unwrap(); + assert!( + registry + .argument_completions(&descriptor.identity, "value", "") + .await + .is_empty() + ); + assert!(registry.descriptors().is_empty()); + assert!(registry.invoke(&invocation(&descriptor)).await.is_err()); + assert_eq!(calls.load(Ordering::SeqCst), 0); + } + + #[tokio::test] + async fn enum_and_boolean_completions_derive_from_type_schema_when_no_provider_declared() { + let calls = Arc::new(AtomicUsize::new(0)); + for (value_type, prefix, expected) in [ + (InvocationArgumentType::Boolean, "f", "false"), + ( + InvocationArgumentType::Enum { + values: vec!["read".into(), "write".into()], + }, + "w", + "write", + ), + ] { + let mut descriptor = descriptor("builtin:test", "run"); + descriptor.arguments[0].value_type = value_type; + descriptor.arguments[0].completion = InvocationCompletion::None; + let mut registry = FeatureInvocationRegistry::default(); + registry + .register( + FeatureId::builtin("test"), + descriptor.clone(), + Arc::new(CountingHandler(calls.clone())), + None, + ) + .unwrap(); + let candidates = registry + .argument_completions(&descriptor.identity, "value", prefix) + .await; + assert_eq!(candidates.len(), 1); + assert_eq!(candidates[0].value, expected); + } + assert_eq!(calls.load(Ordering::SeqCst), 0); + } + + #[test] + fn oversized_installed_invocation_metadata_is_rejected() { + let mut descriptor = descriptor("builtin:test", "run"); + descriptor.description = "x".repeat(MAX_INVOCATION_DESCRIPTOR_BYTES); + let mut registry = FeatureInvocationRegistry::default(); + assert!( + registry + .register( + FeatureId::builtin("test"), + descriptor, + Arc::new(CountingHandler(Arc::new(AtomicUsize::new(0)))), + None, + ) + .is_err() + ); + assert!(registry.descriptors().is_empty()); + } + + #[tokio::test] + async fn metadata_and_argument_completion_are_side_effect_free() { + let calls = Arc::new(AtomicUsize::new(0)); + let descriptor = descriptor("builtin:test", "run"); + let mut registry = FeatureInvocationRegistry::default(); + registry + .register( + FeatureId::builtin("test"), + descriptor.clone(), + Arc::new(CountingHandler(calls.clone())), + None, + ) + .unwrap(); + + assert_eq!(registry.feature_completions("ru")[0].value, "run"); + assert_eq!( + registry + .argument_completions(&descriptor.identity, "value", "a") + .await[0] + .value, + "alpha" + ); + assert_eq!(calls.load(Ordering::SeqCst), 0); + } +} diff --git a/crates/worker/src/file_operation.rs b/crates/worker/src/file_operation.rs new file mode 100644 index 000000000..57a5f359f --- /dev/null +++ b/crates/worker/src/file_operation.rs @@ -0,0 +1,43 @@ +//! WIP presentation of the provider-neutral file argument contracts. Providers +//! choose which read surface to publish; routing and preconditions are not args. +use fs_operation::text::TextOperation; +use wip_protocol::{ParameterDeclaration, TypeExpr}; + +pub(crate) fn parameters(name: &str, line_read: bool) -> Option> { + let schema = TextOperation::from_name(name, line_read)?.argument_schema(); + Some(schema_parameters(&schema)) +} + +fn schema_parameters(schema: &schemars::Schema) -> Vec { + let schema = schema.as_value(); + let required = schema["required"].as_array(); + schema["properties"] + .as_object() + .into_iter() + .flat_map(|properties| properties.iter()) + .map(|(name, property)| { + // These core arguments are primitives. Optional Tool read fields + // accept null; WIP keeps its existing optional integer declaration. + let kind = property["type"].as_str().or_else(|| { + property["type"] + .as_array()? + .iter() + .filter_map(|v| v.as_str()) + .find(|t| *t != "null") + }); + let r#type = match kind { + Some("string") => TypeExpr::String, + Some("boolean") => TypeExpr::Boolean, + Some("integer") => TypeExpr::Integer, + _ => panic!("file operation arguments must be primitive: {name}"), + }; + ParameterDeclaration { + name: name.clone(), + required: required + .is_some_and(|items| items.iter().any(|item| item.as_str() == Some(name))), + documentation: None, + r#type, + } + }) + .collect() +} diff --git a/crates/worker/src/hook.rs b/crates/worker/src/hook.rs index 6e0445d87..82f93d469 100644 --- a/crates/worker/src/hook.rs +++ b/crates/worker/src/hook.rs @@ -281,6 +281,20 @@ impl SystemItemAppendHandle { .expect("system-item append queue poisoned") .push(item); } + + /// Queue a revisioned Subject behavior refresh. The caller receives this + /// narrow operation only at a pre-request boundary, and the host commits it + /// before exposing it to the model. + pub fn append_subject_behavior_refresh(&self, body: impl Into, behavior_revision: u64) { + self.pending + .lock() + .expect("system-item append queue poisoned") + .push(SystemItem::SubjectBehaviorRefresh { + body: body.into(), + behavior_revision, + prompt_provenance: None, + }); + } } /// Host-created narrow append authority exposed only at a restore lifecycle diff --git a/crates/worker/src/ipc/interceptor.rs b/crates/worker/src/ipc/interceptor.rs index 2b403cd39..96dd186f8 100644 --- a/crates/worker/src/ipc/interceptor.rs +++ b/crates/worker/src/ipc/interceptor.rs @@ -38,7 +38,9 @@ use crate::hook::{ use crate::ipc::notify_buffer::{NotifyBuffer, build_system_item_with_provenance}; use crate::prompt::catalog::PromptCatalog; use crate::session_history::SessionHistoryMetadata; -use crate::worker::SystemItemCommitter; +use crate::worker::{ + DurableNotificationCommitter, SystemItemCommitter, SystemPromptContributionSource, +}; use agen::HistoryEntry; use agen::token_counter::total_tokens; @@ -109,6 +111,8 @@ pub(crate) struct WorkerInterceptor { /// worker. `None` in tests / `Worker::new` paths where no writer is /// attached. log_writer: Option>, + notification_committer: Option>, + resident_context_source: Option>, pending_committed_history: Arc>>>, /// Next turn index assigned by `on_prompt_submit`. next_turn_index: AtomicUsize, @@ -163,12 +167,22 @@ impl WorkerInterceptor { prompts, prompt_workspace_id: None, log_writer, + notification_committer: None, + resident_context_source: None, pending_committed_history, next_turn_index: AtomicUsize::new(0), tool_calls_this_turn: AtomicUsize::new(0), } } + pub(crate) fn with_notification_committer( + mut self, + committer: Option>, + ) -> Self { + self.notification_committer = committer; + self + } + pub(crate) fn with_usage_tracker(mut self, usage_tracker: Arc) -> Self { self.usage_tracker = Some(usage_tracker); self @@ -179,11 +193,31 @@ impl WorkerInterceptor { self } + pub(crate) fn with_resident_context_source( + mut self, + source: Option>, + ) -> Self { + self.resident_context_source = source; + self + } + pub(crate) fn with_prompt_workspace_id(mut self, workspace_id: Option) -> Self { self.prompt_workspace_id = workspace_id; self } + fn confirm_resident_context_item(&self, item: &SystemItem) { + let Some(source) = self.resident_context_source.as_ref() else { + return; + }; + if let SystemItem::SubjectBehaviorRefresh { + behavior_revision, .. + } = item + { + source.confirm_resident_context_revision(*behavior_revision); + } + } + /// Commit each `SystemItem` as its own `LogEntry::AnnotatedSystemItem` /// entry through the attached writer (no-op when no writer is /// wired). Sync — writes complete before the matching @@ -211,6 +245,7 @@ impl WorkerInterceptor { .lock() .expect("pending committed history poisoned") .push_back(entry); + self.confirm_resident_context_item(item); } Ok(()) } @@ -350,6 +385,11 @@ impl WorkerInterceptor { } if prompt_provenance.is_none() => { *prompt_provenance = Some(provenance("internal.task_reminder")); } + SystemItem::SubjectBehaviorRefresh { + prompt_provenance, .. + } if prompt_provenance.is_none() => { + *prompt_provenance = Some(provenance("internal.subjektiv_resident_context")); + } SystemItem::Interrupt { prompt_provenance, .. } if prompt_provenance.is_none() => { @@ -399,8 +439,23 @@ impl Interceptor for WorkerInterceptor { .lock() .expect("pending_attachments poisoned"), ); + // Preparations can commit before Engine receives the user prompt. The + // annotation queue alone does not materialize their model-visible items: + // return them through the normal prompt lifecycle, without recommitting. + // The annotator consumes their original IDs/provenance as Engine appends. + let mut prepared_items = self + .pending_committed_history + .lock() + .expect("pending committed history poisoned") + .iter() + .map(|entry| entry.item.clone()) + .collect::>(); Ok(if extras.is_empty() { - PromptAction::Continue + if prepared_items.is_empty() { + PromptAction::Continue + } else { + PromptAction::ContinueWith(prepared_items) + } } else { // Commit the typed system items first, then hand the // matching `Item::system_message`s to the worker. Sync @@ -410,7 +465,10 @@ impl Interceptor for WorkerInterceptor { self.attach_prompt_provenance(&mut extras); let items: Vec = extras.iter().map(SystemItem::to_history_item).collect(); match self.commit_system_items(&extras) { - Ok(()) => PromptAction::ContinueWith(items), + Ok(()) => { + prepared_items.extend(items); + PromptAction::ContinueWith(prepared_items) + } Err(error) => PromptAction::Cancel(format!("session persistence failed: {error}")), } }) @@ -434,19 +492,10 @@ impl Interceptor for WorkerInterceptor { projection_digest: projection.catalog_digest.clone(), logical_name: "internal.notify_wrapper".to_string(), }; - let mut system_items: Vec<( - SystemItem, - Vec, - Option, - )> = Vec::with_capacity(drained.len()); - let mut items: Vec = Vec::with_capacity(drained.len()); + let mut system_items = Vec::with_capacity(drained.len()); for entry in &drained { - let system_item = match build_system_item_with_provenance( - entry, - &prompts, - Some(provenance.clone()), - ) { - Ok(system_item) => system_item, + match build_system_item_with_provenance(entry, &prompts, Some(provenance.clone())) { + Ok(item) => system_items.push(item), Err(error) => { self.pending_notifies.requeue_front(drained); return Err(InterceptorError::new( @@ -454,16 +503,50 @@ impl Interceptor for WorkerInterceptor { format!("failed to render notify_wrapper: {error}"), )); } - }; - items.push(system_item.to_history_item()); - system_items.push((system_item, entry.extensions(), entry.history_provenance())); + } } - if let Err(error) = self.commit_system_items_with_extensions(&system_items) { - self.pending_notifies.requeue_front(drained); - return Err(InterceptorError::new( - InterceptorErrorCategory::Dependency, - format!("session persistence failed: {error}"), - )); + let mut items = Vec::with_capacity(drained.len()); + let mut remaining = drained.into_iter(); + for item in system_items { + let entry = remaining.next().expect("rendered notification must exist"); + let committed = if let Some(identity) = entry.notification_commit() { + match self.notification_committer.as_ref() { + Some(committer) => { + committer.commit_notification(identity, item, entry.history_provenance()) + } + None => Err(session_store::StoreError::Io(std::io::Error::other( + "durable notification commit authority is missing", + ))), + } + } else { + self.commit_system_items_with_extensions(&[( + item.clone(), + Vec::new(), + entry.history_provenance(), + )]) + .map(|()| { + items.push(item.to_history_item()); + None + }) + }; + match committed { + Ok(Some(history)) => { + items.push(history.item.clone()); + self.pending_committed_history + .lock() + .expect("pending committed history poisoned") + .push_back(history); + } + Ok(None) => {} + Err(error) => { + self.pending_notifies + .requeue_front(std::iter::once(entry).chain(remaining).collect()); + return Err(InterceptorError::new( + InterceptorErrorCategory::Dependency, + format!("session persistence failed: {error}"), + )); + } + } } Ok(items) } @@ -763,6 +846,7 @@ mod tests { HookTurnEndAction, OnTurnEnd, PostToolCall, PreLlmRequest, PreToolCall, }; use crate::session_history::{WorkerHistoryProvenance, history_entry}; + use crate::worker::SystemPromptContribution; fn test_prompts() -> Arc> { Arc::new(ArcSwap::from(PromptCatalog::builtins_only().unwrap())) @@ -859,6 +943,53 @@ mod tests { } } + struct AppendingSubjectBehaviorHook; + + #[async_trait] + impl Hook for AppendingSubjectBehaviorHook { + async fn call( + &self, + input: &PreRequestContext, + ) -> Result { + input + .system_items() + .expect("test interceptor has durable append authority") + .append_subject_behavior_refresh("current Subject context", 7); + Ok(HookPreRequestAction::Continue) + } + } + + struct ConfirmRecordingResidentSource { + confirmed: Arc>>, + } + + #[async_trait] + impl SystemPromptContributionSource for ConfirmRecordingResidentSource { + async fn load(&self) -> SystemPromptContribution { + SystemPromptContribution::Unavailable + } + + fn confirm_resident_context_revision(&self, revision: u64) { + self.confirmed.lock().unwrap().push(revision); + } + } + + struct FailingSystemItemCommitter; + + impl SystemItemCommitter for FailingSystemItemCommitter { + fn commit_log_entry( + &self, + entry: session_store::LogEntry, + ) -> Result<(), session_store::StoreError> { + if matches!(entry, session_store::LogEntry::AnnotatedSystemItem { .. }) { + return Err(session_store::StoreError::Io(std::io::Error::other( + "synthetic system-item failure", + ))); + } + Ok(()) + } + } + fn task_tool_call_info( name: &str, input: serde_json::Value, @@ -1346,6 +1477,88 @@ mod tests { } } + #[tokio::test] + async fn subject_behavior_revision_is_confirmed_only_after_durable_commit() { + let mut builder = HookRegistryBuilder::new(); + builder.add_pre_llm_request(AppendingSubjectBehaviorHook); + let committed = Arc::new(Mutex::new(Vec::new())); + let confirmed = Arc::new(Mutex::new(Vec::new())); + let source: Arc = + Arc::new(ConfirmRecordingResidentSource { + confirmed: Arc::clone(&confirmed), + }); + let interceptor = WorkerInterceptor::new( + Arc::new(builder.build()), + None, + None, + NotifyBuffer::new(), + Arc::new(Mutex::new(Vec::new())), + test_prompts(), + Some(Arc::new(RecordingSystemItemCommitter { + committed: Arc::clone(&committed), + })), + ) + .with_resident_context_source(Some(source)); + + let mut ctx = Vec::new(); + let action = interceptor + .pre_llm_request(PreLlmRequestContext { + invocation: Default::default(), + items: &mut ctx, + history: &[], + }) + .await + .unwrap(); + + assert!(matches!(action, PreRequestAction::ContinueWith(_))); + assert_eq!(*confirmed.lock().unwrap(), [7]); + assert!(matches!( + committed.lock().unwrap().as_slice(), + [SystemItem::SubjectBehaviorRefresh { + behavior_revision: 7, + prompt_provenance: Some(_), + .. + }] + )); + } + + #[tokio::test] + async fn failed_subject_behavior_persistence_does_not_confirm_revision() { + let mut builder = HookRegistryBuilder::new(); + builder.add_pre_llm_request(AppendingSubjectBehaviorHook); + let confirmed = Arc::new(Mutex::new(Vec::new())); + let source: Arc = + Arc::new(ConfirmRecordingResidentSource { + confirmed: Arc::clone(&confirmed), + }); + let interceptor = WorkerInterceptor::new( + Arc::new(builder.build()), + None, + None, + NotifyBuffer::new(), + Arc::new(Mutex::new(Vec::new())), + test_prompts(), + Some(Arc::new(FailingSystemItemCommitter)), + ) + .with_resident_context_source(Some(source)); + + let mut ctx = Vec::new(); + let action = interceptor + .pre_llm_request(PreLlmRequestContext { + invocation: Default::default(), + items: &mut ctx, + history: &[], + }) + .await + .unwrap(); + + assert!(matches!( + action, + PreRequestAction::Cancel(reason) if reason.contains("session persistence failed") + )); + assert!(confirmed.lock().unwrap().is_empty()); + } + #[tokio::test] async fn pre_llm_request_without_log_writer_does_not_expose_system_item_handle() { let saw_handle = Arc::new(AtomicBool::new(false)); diff --git a/crates/worker/src/ipc/notify_buffer.rs b/crates/worker/src/ipc/notify_buffer.rs index 699a778cf..6af0e75cb 100644 --- a/crates/worker/src/ipc/notify_buffer.rs +++ b/crates/worker/src/ipc/notify_buffer.rs @@ -24,8 +24,9 @@ use std::collections::VecDeque; use std::sync::{Arc, Mutex}; +use crate::worker::NotificationCommitIdentity; use protocol::WorkerEvent; -use session_store::{LoggedSessionHistoryOrigin, SessionExtension, SystemItem}; +use session_store::{LoggedSessionHistoryOrigin, SystemItem}; use crate::prompt::catalog::{CatalogError, PromptCatalog}; @@ -39,7 +40,7 @@ use crate::prompt::catalog::{CatalogError, PromptCatalog}; pub enum PendingNotify { Notify { message: String, - extensions: Vec, + notification_commit: Option, history_provenance: Option, }, WorkerEvent { @@ -48,10 +49,13 @@ pub enum PendingNotify { } impl PendingNotify { - pub(crate) fn extensions(&self) -> Vec { + pub(crate) fn notification_commit(&self) -> Option<&NotificationCommitIdentity> { match self { - PendingNotify::Notify { extensions, .. } => extensions.clone(), - PendingNotify::WorkerEvent { .. } => Vec::new(), + PendingNotify::Notify { + notification_commit, + .. + } => notification_commit.as_ref(), + PendingNotify::WorkerEvent { .. } => None, } } @@ -85,7 +89,7 @@ impl NotifyBuffer { pub fn push_notify(&self, message: String) { self.push_entry(PendingNotify::Notify { message, - extensions: Vec::new(), + notification_commit: None, history_provenance: None, }); } @@ -94,11 +98,11 @@ impl NotifyBuffer { &self, message: String, history_provenance: LoggedSessionHistoryOrigin, - extension: SessionExtension, + identity: NotificationCommitIdentity, ) { self.push_entry(PendingNotify::Notify { message, - extensions: vec![extension], + notification_commit: Some(identity), history_provenance: Some(history_provenance), }); } @@ -233,7 +237,7 @@ mod tests { fn build_system_item_for_notify_carries_wrapper_body() { let entry = PendingNotify::Notify { message: "hello".into(), - extensions: Vec::new(), + notification_commit: None, history_provenance: None, }; let catalog = PromptCatalog::builtins_only().unwrap(); diff --git a/crates/worker/src/ipc/protocol_session.rs b/crates/worker/src/ipc/protocol_session.rs index a9dd18dd7..718b03aea 100644 --- a/crates/worker/src/ipc/protocol_session.rs +++ b/crates/worker/src/ipc/protocol_session.rs @@ -122,9 +122,22 @@ pub async fn dispatch_worker_protocol_method( method: Method, ) -> Option { match method { - Method::ListCompletions { kind, prefix } => { - let entries = handle.completion_entries(kind, &prefix).await; - Some(Event::Completions { kind, entries }) + Method::ListCompletions { + request_id, + kind, + prefix, + context, + } => { + let entries = handle + .completion_entries(kind, &prefix, context.as_ref()) + .await; + Some(Event::Completions { + request_id, + kind, + prefix, + context, + entries, + }) } method => { let _ = handle.send(method).await; diff --git a/crates/worker/src/lib.rs b/crates/worker/src/lib.rs index 49c277eb9..1dbc8a526 100644 --- a/crates/worker/src/lib.rs +++ b/crates/worker/src/lib.rs @@ -1,9 +1,13 @@ +mod backend_job; +pub use backend_job::BackendJobExecutionBinding; pub mod bootstrap; +pub mod checkout; pub mod compact; pub mod controller; pub mod discovery; pub mod entrypoint; pub mod feature; +mod file_operation; pub mod fs_view; pub mod hook; pub(crate) mod in_flight; diff --git a/crates/worker/src/prompt/catalog.rs b/crates/worker/src/prompt/catalog.rs index f8c52cc47..9b3e30459 100644 --- a/crates/worker/src/prompt/catalog.rs +++ b/crates/worker/src/prompt/catalog.rs @@ -105,6 +105,7 @@ pub enum WorkerPrompt { AgentsMdSection, ResidentMemorySummarySection, ResidentMemoryRestoreSection, + SubjektivResidentContext, WorkerOrchestrationGuidanceSection, SubWorkerSpawnToolDescription, } @@ -128,6 +129,7 @@ impl WorkerPrompt { Self::AgentsMdSection => "internal.agents_md_section", Self::ResidentMemorySummarySection => "internal.resident_memory_summary_section", Self::ResidentMemoryRestoreSection => "internal.resident_memory_restore_section", + Self::SubjektivResidentContext => "internal.subjektiv_resident_context", Self::WorkerOrchestrationGuidanceSection => { "internal.worker_orchestration_guidance_section" } @@ -150,6 +152,7 @@ impl WorkerPrompt { WorkerPrompt::AgentsMdSection, WorkerPrompt::ResidentMemorySummarySection, WorkerPrompt::ResidentMemoryRestoreSection, + WorkerPrompt::SubjektivResidentContext, WorkerPrompt::WorkerOrchestrationGuidanceSection, WorkerPrompt::SubWorkerSpawnToolDescription, ]; @@ -518,6 +521,21 @@ impl PromptCatalog { }, ) } + pub fn subjektiv_resident_context( + &self, + behavior_md: &str, + surface_availability: &str, + surface_body: Option<&str>, + ) -> Result { + self.render( + WorkerPrompt::SubjektivResidentContext, + minijinja::context! { + behavior_md => behavior_md, + surface_availability => surface_availability, + surface_body => surface_body.unwrap_or_default(), + }, + ) + } pub fn worker_orchestration_guidance_section(&self) -> Result { self.render( WorkerPrompt::WorkerOrchestrationGuidanceSection, diff --git a/crates/worker/src/shared_state.rs b/crates/worker/src/shared_state.rs index 46d0690d0..a9106e7e7 100644 --- a/crates/worker/src/shared_state.rs +++ b/crates/worker/src/shared_state.rs @@ -48,6 +48,7 @@ pub struct WorkerSharedState { /// `ListCompletions` queries without going through the controller. It is /// unset only in unit tests that construct `WorkerSharedState` directly. fs_view: OnceLock, + feature_invocations: RwLock, flow_transition_enabled: AtomicBool, } @@ -66,6 +67,7 @@ impl WorkerSharedState { state: RwLock::new(WorkerStateSnapshot::initial()), accepted_commands: RwLock::new(VecDeque::new()), fs_view: OnceLock::new(), + feature_invocations: RwLock::new(crate::feature::FeatureInvocationRegistry::default()), flow_transition_enabled: AtomicBool::new(false), } } @@ -82,6 +84,20 @@ impl WorkerSharedState { self.fs_view.get() } + pub fn feature_invocations(&self) -> crate::feature::FeatureInvocationRegistry { + self.feature_invocations + .read() + .expect("feature invocation registry lock poisoned") + .clone() + } + + pub fn set_feature_invocations(&self, registry: crate::feature::FeatureInvocationRegistry) { + *self + .feature_invocations + .write() + .expect("feature invocation registry lock poisoned") = registry; + } + pub fn enable_flow_transition(&self) { self.flow_transition_enabled.store(true, Ordering::Release); } diff --git a/crates/worker/src/wip.rs b/crates/worker/src/wip.rs index a489cf36e..bbce6e9cd 100644 --- a/crates/worker/src/wip.rs +++ b/crates/worker/src/wip.rs @@ -42,6 +42,13 @@ use wip_protocol::{ use crate::permission::permission_action_for; +#[cfg(test)] +#[path = "checkout_http_tests.rs"] +mod checkout_http_tests; +#[cfg(test)] +#[path = "checkout_wip_tests.rs"] +mod checkout_wip_tests; + const WIP_ENDPOINT: &str = "https://worker.wip.invalid/v1/"; const WIP_ROOT: &str = "/"; const WIP_TOOLS_ROOT: &str = "/tools"; @@ -68,6 +75,14 @@ pub enum WipMountError { RouteCollision { route: String, existing: String }, #[error("WIP interface `{interface}` has conflicting descriptors")] InterfaceCollision { interface: String }, + #[error("WIP operation `{operation}` on `{route}` is already contributed by `{existing}`")] + OperationCollision { + route: String, + operation: String, + existing: String, + }, + #[error("WIP operation contribution target `{route}` is not a mounted Object")] + OperationTargetNotFound { route: String }, #[error("invalid WIP projection for `{route}`: {message}")] InvalidProjection { route: String, message: String }, } @@ -84,6 +99,8 @@ pub struct WipCallContext { pub struct WipOperationOutput { pub value: Value, tool_output: Option, + // Exact post-operation provider state; never substitute a later path stat. + validator: Option>, } impl WipOperationOutput { @@ -91,6 +108,15 @@ impl WipOperationOutput { Self { value, tool_output: None, + validator: None, + } + } + + pub fn native_with_validator(value: Value, validator: Vec) -> Self { + Self { + value, + tool_output: None, + validator: Some(validator), } } @@ -98,6 +124,7 @@ impl WipOperationOutput { Self { value, tool_output: Some(output), + validator: None, } } } @@ -113,6 +140,28 @@ pub enum WipOperationError { /// routing and validators are resolved before this trait is called. #[async_trait] pub trait WipOperationHandler: Send + Sync { + /// Whether the owning provider currently exposes this Object to its subject. + fn is_visible(&self) -> bool { + true + } + + /// Override the Object's registered validator with current provider state. + /// None preserves the registered validator and requires no inventory fetch. + fn object_validator(&self) -> Option> { + None + } + + /// Current target capability and subject permission intersection. Providers + /// must still enforce authority at execution; this hook controls publication. + fn operation_available(&self, _operation: &str) -> bool { + true + } + + /// Opt into path-qualified, freshly resolved interface descriptors. + fn contextual_interface(&self) -> bool { + false + } + async fn call( &self, operation: &str, @@ -123,6 +172,16 @@ pub trait WipOperationHandler: Send + Sync { async fn cancel(&self, _context: &ToolExecutionContext) -> Result<(), ToolError> { Ok(()) } + + /// Cancel the selected operation on this exact handler instance. Providers + /// that share a cancellation boundary may keep implementing `cancel`. + async fn cancel_operation( + &self, + _operation: &str, + context: &ToolExecutionContext, + ) -> Result<(), ToolError> { + self.cancel(context).await + } } /// One route contribution. Native projections replace compatibility projections @@ -139,9 +198,21 @@ pub struct WipProjection { pub handler: Arc, } -/// One dynamically resolved direct-child object beneath a Feature-owned -/// collection route. The Host retains route allocation and descriptor authority; -/// the Feature only binds a validated child segment to an object and handler. +/// An additional Feature's operations for an Object whose existence and route +/// are already owned by the Host registry. Contributions may extend the mounted +/// interface, but cannot replace Object resolution or another operation. +#[derive(Clone)] +pub struct WipOperationContribution { + pub route: String, + pub contributor: String, + pub interface: String, + pub descriptor: InterfaceDescriptor, + pub handler: Arc, +} + +/// One dynamically resolved direct-child object beneath a Host-owned collection +/// route. Object resolution stays unique; operation contribution is independent +/// for statically mounted Objects and can be extended without remounting them. pub struct WipDynamicItem { pub object: Object, pub handler: Arc, @@ -149,6 +220,12 @@ pub struct WipDynamicItem { pub trait WipDynamicItemResolver: Send + Sync { fn resolve(&self, item_reference: &str) -> Option; + + /// Family-level opt-in, needed to deny unqualified descriptor fetches before + /// resolving any item. A contextual collection owner also opts in its family. + fn contextual_interface(&self) -> bool { + false + } } pub struct WipDynamicMount { @@ -160,43 +237,138 @@ pub struct WipDynamicMount { pub resolver: Arc, } +/// Explicit arbitrary-depth request-time subtree provider. Unlike item families, +/// this resolves both files and directories and enumerates authorized children. +#[async_trait] +pub trait WipSubtreeProvider: Send + Sync { + /// Provider-specific observation bounds, capped by the Host global bounds. + fn max_depth(&self) -> u32 { + 32 + } + fn max_nodes(&self) -> usize { + 1024 + } + async fn projection(&self, path: &str) -> Result, ProtocolError>; + async fn children(&self, path: &str) -> Result, ProtocolError>; +} + +pub struct WipSubtreeMount { + pub root: String, + pub provider: Arc, +} + +pub trait WipDynamicOperationResolver: Send + Sync { + fn handler(&self, item_reference: &str) -> Arc; +} + +pub struct WipDynamicOperationContribution { + pub collection_route: String, + pub contributor: String, + pub interface: String, + pub descriptor: InterfaceDescriptor, + pub resolver: Arc, +} + +struct MountedDynamicOperationContribution { + contributor: String, + operations: Vec, + resolver: Arc, +} + #[derive(Debug, Clone, PartialEq, Eq)] -pub struct WipFeatureRoute { +pub struct WipNamespaceRoute { root: String, } -impl WipFeatureRoute { +impl WipNamespaceRoute { pub fn root(&self) -> &str { &self.root } - - pub fn child(&self, segment: &str) -> Result { - if segment.is_empty() - || segment == "." - || segment == ".." - || segment.contains('/') - || !segment - .bytes() - .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) - { - return Err(WipMountError::InvalidRoute { - route: format!("{}/{}", self.root, segment), - message: "Feature child must be one canonical path segment".into(), - }); - } - Ok(format!("{}/{}", self.root, segment)) - } } struct MountedProjection { projection: WipProjection, + operation_handlers: BTreeMap)>, +} + +impl MountedProjection { + fn resolved(&self) -> WipProjection { + let mut projection = self.projection.clone(); + projection.handler = Arc::new(OperationDispatchHandler { + owner: Arc::clone(&self.projection.handler), + handlers: self + .operation_handlers + .iter() + .map(|(operation, (_, handler))| (operation.clone(), Arc::clone(handler))) + .collect(), + }); + projection + } +} + +struct OperationDispatchHandler { + owner: Arc, + handlers: BTreeMap>, +} + +#[async_trait] +impl WipOperationHandler for OperationDispatchHandler { + fn is_visible(&self) -> bool { + self.owner.is_visible() + } + + fn object_validator(&self) -> Option> { + self.owner.object_validator() + } + + fn operation_available(&self, operation: &str) -> bool { + self.handlers + .get(operation) + .is_some_and(|handler| handler.operation_available(operation)) + } + + fn contextual_interface(&self) -> bool { + self.owner.contextual_interface() + || self + .handlers + .values() + .any(|handler| handler.contextual_interface()) + } + + async fn call( + &self, + operation: &str, + arguments: &BTreeMap, + context: WipCallContext, + ) -> Result { + let handler = self.handlers.get(operation).ok_or_else(|| { + WipOperationError::Protocol(protocol_error( + ProtocolErrorCode::OperationNotFound, + "operation is not published by the selected interface", + )) + })?; + handler.call(operation, arguments, context).await + } + + async fn cancel_operation( + &self, + operation: &str, + context: &ToolExecutionContext, + ) -> Result<(), ToolError> { + let handler = self.handlers.get(operation).ok_or_else(|| { + ToolError::InvalidArgument("WIP cancellation operation is not published".into()) + })?; + handler.cancel_operation(operation, context).await + } } #[derive(Default)] pub struct WipMountRegistry { mounts: BTreeMap, dynamic_mounts: Vec, - feature_routes: BTreeSet, + subtree_mounts: Vec, + dynamic_operation_contributions: BTreeMap>, + namespaces: BTreeMap, replaced_compatibility_capabilities: BTreeMap, } @@ -205,30 +377,57 @@ impl WipMountRegistry { Self::default() } - /// Allocate the Host namespace owned by one enabled Feature. Features join - /// checked relative segments to this route instead of selecting global paths. - pub fn allocate_feature_route( + /// Allocate one Host-owned root namespace to its unique Object provider. + /// The provider identity is registry metadata, never a public path segment. + pub fn allocate_namespace( &mut self, - feature: &str, - ) -> Result { - if feature.is_empty() - || !feature - .bytes() - .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') - { + owner: &str, + namespace: &str, + ) -> Result { + let valid_segment = |value: &str| { + !value.is_empty() + && value + .bytes() + .all(|byte| byte.is_ascii_lowercase() || byte.is_ascii_digit() || byte == b'-') + }; + if !valid_segment(owner) { return Err(WipMountError::InvalidRoute { - route: format!("/features/{feature}"), - message: "Feature name must use lowercase ASCII letters, digits, or '-'".into(), + route: format!("/{namespace}"), + message: "namespace owner must use lowercase ASCII letters, digits, or '-'".into(), }); } - let root = format!("/features/{feature}"); - if !self.feature_routes.insert(root.clone()) { + if !valid_segment(namespace) { + return Err(WipMountError::InvalidRoute { + route: format!("/{namespace}"), + message: "root namespace must be one lowercase ASCII path segment".into(), + }); + } + if matches!(namespace, "features" | "tools") { + return Err(WipMountError::InvalidRoute { + route: format!("/{namespace}"), + message: "the Host reserves this root namespace".into(), + }); + } + let root = format!("/{namespace}"); + if let Some(existing) = self.namespaces.get(&root) { return Err(WipMountError::RouteCollision { route: root, - existing: format!("feature:{feature}"), + existing: format!("namespace owner `{existing}`"), }); } - Ok(WipFeatureRoute { root }) + if self.mounts.contains_key(&root) + || self + .mounts + .keys() + .any(|route| route.starts_with(&format!("{root}/"))) + { + return Err(WipMountError::RouteCollision { + route: root, + existing: "already mounted route".into(), + }); + } + self.namespaces.insert(root.clone(), owner.to_string()); + Ok(WipNamespaceRoute { root }) } pub fn mount( @@ -236,6 +435,49 @@ impl WipMountRegistry { projection: WipProjection, ) -> Result { validate_projection(&projection)?; + if projection.kind == WipProjectionKind::Compatibility + && self.replaces_compatibility(&projection.capability) + { + return Ok(WipMountDisposition::NativeAlreadySelected); + } + if projection.kind == WipProjectionKind::Compatibility && !is_tool_route(&projection.route) + { + return Err(WipMountError::InvalidProjection { + route: projection.route, + message: "compatibility Objects must use /tools/".into(), + }); + } + if projection.kind == WipProjectionKind::Native && !is_tool_route(&projection.route) { + let namespace_root = projection + .route + .strip_prefix('/') + .and_then(|path| path.split('/').next()) + .map(|segment| format!("/{segment}")) + .unwrap_or_default(); + let Some(owner) = self.namespaces.get(&namespace_root) else { + return Err(WipMountError::InvalidProjection { + route: projection.route, + message: "native Object route requires an allocated Host namespace".into(), + }); + }; + if projection.capability.split(':').next() != Some(owner.as_str()) { + return Err(WipMountError::InvalidProjection { + route: projection.route, + message: format!( + "native Object capability does not belong to namespace owner `{owner}`" + ), + }); + } + } + if self.subtree_mounts.iter().any(|mount| { + projection.route == mount.root + || projection.route.starts_with(&format!("{}/", mount.root)) + }) { + return Err(WipMountError::RouteCollision { + route: projection.route, + existing: "request-time subtree".into(), + }); + } if self.dynamic_mounts.iter().any(|dynamic| { projection .route @@ -256,8 +498,15 @@ impl WipMountRegistry { match (existing.projection.kind, projection.kind) { (WipProjectionKind::Compatibility, WipProjectionKind::Native) => { self.ensure_interface_available(&projection, Some(&existing.projection.route))?; - self.mounts - .insert(projection.route.clone(), MountedProjection { projection }); + let operation_handlers = + operation_handlers(&projection, &projection.capability); + self.mounts.insert( + projection.route.clone(), + MountedProjection { + projection, + operation_handlers, + }, + ); return Ok(WipMountDisposition::ReplacedCompatibility); } (WipProjectionKind::Native, WipProjectionKind::Compatibility) => { @@ -273,10 +522,99 @@ impl WipMountRegistry { } self.ensure_interface_available(&projection, None)?; let route = projection.route.clone(); - self.mounts.insert(route, MountedProjection { projection }); + let operation_handlers = operation_handlers(&projection, &projection.capability); + self.mounts.insert( + route, + MountedProjection { + projection, + operation_handlers, + }, + ); Ok(WipMountDisposition::Mounted) } + /// Add operations from another Feature without changing Object ownership or + /// resolution. Interface shape and operation names are global protocol + /// contracts, so incompatible declarations fail at registration time. + pub fn contribute_operations( + &mut self, + contribution: WipOperationContribution, + ) -> Result<(), WipMountError> { + contribution + .descriptor + .validate() + .map_err(|error| WipMountError::InvalidProjection { + route: contribution.route.clone(), + message: error.to_string(), + })?; + let Some(mounted) = self.mounts.get(&contribution.route) else { + return Err(WipMountError::OperationTargetNotFound { + route: contribution.route, + }); + }; + if mounted.projection.interface != contribution.interface { + return Err(WipMountError::InterfaceCollision { + interface: contribution.interface, + }); + } + if !same_interface_shape(&mounted.projection.descriptor, &contribution.descriptor) { + return Err(WipMountError::InterfaceCollision { + interface: contribution.interface, + }); + } + for operation in &contribution.descriptor.operations { + if let Some((existing, _)) = mounted.operation_handlers.get(&operation.name) { + return Err(WipMountError::OperationCollision { + route: contribution.route, + operation: operation.name.clone(), + existing: existing.clone(), + }); + } + } + + let mut merged = mounted.projection.descriptor.clone(); + merged + .operations + .extend(contribution.descriptor.operations.iter().cloned()); + merged + .operations + .sort_by(|left, right| left.name.cmp(&right.name)); + merged + .validate() + .map_err(|error| WipMountError::InvalidProjection { + route: contribution.route.clone(), + message: error.to_string(), + })?; + if self.interface_conflicts( + &contribution.interface, + &merged, + Some(&descriptor_validator(&merged)), + Some(&contribution.route), + ) { + return Err(WipMountError::InterfaceCollision { + interface: contribution.interface, + }); + } + + let mounted = self + .mounts + .get_mut(&contribution.route) + .expect("operation contribution target was checked"); + for operation in contribution.descriptor.operations { + mounted.operation_handlers.insert( + operation.name, + ( + contribution.contributor.clone(), + Arc::clone(&contribution.handler), + ), + ); + } + mounted.projection.descriptor = merged; + mounted.projection.interface_validator = + Some(descriptor_validator(&mounted.projection.descriptor)); + Ok(()) + } + /// Mount a one-segment item family below an already mounted collection. /// Dynamic resolution never creates authority: the collection route must be /// native and every resolved child is validated before it is exposed. @@ -287,6 +625,16 @@ impl WipMountRegistry { message: error.to_string(), } })?; + if self.subtree_mounts.iter().any(|m| { + m.root == mount.collection_route + || m.root.starts_with(&format!("{}/", mount.collection_route)) + || mount.collection_route.starts_with(&format!("{}/", m.root)) + }) { + return Err(WipMountError::RouteCollision { + route: mount.collection_route, + existing: "request-time subtree".into(), + }); + } let Some(collection) = self.mounts.get(&mount.collection_route) else { return Err(WipMountError::InvalidProjection { route: mount.collection_route, @@ -299,6 +647,13 @@ impl WipMountRegistry { message: "dynamic item mount requires a native collection projection".into(), }); } + if mount.capability.split(':').next() != collection.projection.capability.split(':').next() + { + return Err(WipMountError::InvalidProjection { + route: mount.collection_route, + message: "dynamic Object capability must belong to the collection owner".into(), + }); + } mount .descriptor .validate() @@ -344,6 +699,164 @@ impl WipMountRegistry { Ok(()) } + /// Delegate an entire descendant namespace to one request-time provider. + /// The root Object remains registry-owned; nested/static/item collisions fail. + pub fn mount_subtree(&mut self, mount: WipSubtreeMount) -> Result<(), WipMountError> { + let Some(root) = self.mounts.get(&mount.root) else { + return Err(WipMountError::OperationTargetNotFound { route: mount.root }); + }; + if root.projection.kind != WipProjectionKind::Native { + return Err(WipMountError::InvalidProjection { + route: mount.root, + message: "subtree requires a native root Object".into(), + }); + } + let overlaps = |path: &str| { + path == mount.root + || path.starts_with(&format!("{}/", mount.root)) + || mount.root.starts_with(&format!("{path}/")) + }; + if self.subtree_mounts.iter().any(|m| overlaps(&m.root)) + || self + .dynamic_mounts + .iter() + .any(|m| overlaps(&m.collection_route)) + || self + .mounts + .keys() + .any(|p| p != &mount.root && p.starts_with(&format!("{}/", mount.root))) + { + return Err(WipMountError::RouteCollision { + route: mount.root, + existing: "subtree/static/item provider".into(), + }); + } + self.subtree_mounts.push(mount); + Ok(()) + } + + /// Add operations to every item resolved by one existing dynamic Object + /// family. The family keeps exactly one Object resolver; each Feature only + /// supplies handlers for its own disjoint operation names. + pub fn contribute_dynamic_operations( + &mut self, + contribution: WipDynamicOperationContribution, + ) -> Result<(), WipMountError> { + contribution + .descriptor + .validate() + .map_err(|error| WipMountError::InvalidProjection { + route: contribution.collection_route.clone(), + message: error.to_string(), + })?; + let Some(index) = self + .dynamic_mounts + .iter() + .position(|mount| mount.collection_route == contribution.collection_route) + else { + return Err(WipMountError::OperationTargetNotFound { + route: contribution.collection_route, + }); + }; + let mounted = &self.dynamic_mounts[index]; + if mounted.interface != contribution.interface + || !same_interface_shape(&mounted.descriptor, &contribution.descriptor) + { + return Err(WipMountError::InterfaceCollision { + interface: contribution.interface, + }); + } + for operation in &contribution.descriptor.operations { + if mounted + .descriptor + .operations + .iter() + .any(|existing| existing.name == operation.name) + { + let existing = self + .dynamic_operation_contributions + .get(&contribution.collection_route) + .and_then(|contributions| { + contributions.iter().find(|existing| { + existing + .operations + .iter() + .any(|name| name == &operation.name) + }) + }) + .map(|existing| existing.contributor.clone()) + .unwrap_or_else(|| mounted.capability.clone()); + return Err(WipMountError::OperationCollision { + route: contribution.collection_route, + operation: operation.name.clone(), + existing, + }); + } + } + let mut merged = mounted.descriptor.clone(); + merged + .operations + .extend(contribution.descriptor.operations.iter().cloned()); + merged + .operations + .sort_by(|left, right| left.name.cmp(&right.name)); + merged + .validate() + .map_err(|error| WipMountError::InvalidProjection { + route: contribution.collection_route.clone(), + message: error.to_string(), + })?; + let validator = descriptor_validator(&merged); + let conflicts_static = self.registry_interface_conflicts_static( + &contribution.interface, + &merged, + Some(&validator), + ); + let conflicts_dynamic = self.dynamic_mounts.iter().any(|other| { + other.collection_route != contribution.collection_route + && other.interface == contribution.interface + && (other.descriptor != merged + || other.interface_validator.as_deref() != Some(validator.as_slice())) + }); + if conflicts_static || conflicts_dynamic { + return Err(WipMountError::InterfaceCollision { + interface: contribution.interface, + }); + } + + let operations = contribution + .descriptor + .operations + .into_iter() + .map(|operation| operation.name) + .collect(); + let mounted = &mut self.dynamic_mounts[index]; + mounted.descriptor = merged; + mounted.interface_validator = Some(validator); + self.dynamic_operation_contributions + .entry(contribution.collection_route) + .or_default() + .push(MountedDynamicOperationContribution { + contributor: contribution.contributor, + operations, + resolver: contribution.resolver, + }); + Ok(()) + } + + fn registry_interface_conflicts_static( + &self, + interface: &str, + descriptor: &InterfaceDescriptor, + validator: Option<&[u8]>, + ) -> bool { + self.mounts.values().any(|mounted| { + mounted.projection.interface == interface + && (mounted.projection.descriptor != *descriptor + || mounted.projection.interface_validator.as_deref() != validator) + }) + } + /// Declare ordinary tool capabilities fully represented by native mounts. /// Claimed compatibility projections are removed from WIP mode while the /// normal Tool-mode registration remains unchanged. @@ -380,6 +893,10 @@ impl WipMountRegistry { } } for capability in capabilities { + self.mounts.retain(|_, mounted| { + mounted.projection.kind != WipProjectionKind::Compatibility + || mounted.projection.capability != capability + }); self.replaced_compatibility_capabilities .insert(capability, owner_route.to_string()); } @@ -433,6 +950,71 @@ impl WipMountRegistry { } } +fn operation_handlers( + projection: &WipProjection, + contributor: &str, +) -> BTreeMap)> { + projection + .descriptor + .operations + .iter() + .map(|operation| { + ( + operation.name.clone(), + (contributor.to_string(), Arc::clone(&projection.handler)), + ) + }) + .collect() +} + +fn same_interface_shape(left: &InterfaceDescriptor, right: &InterfaceDescriptor) -> bool { + left.format == right.format + && left.documentation == right.documentation + && left.types == right.types +} + +fn descriptor_validator(descriptor: &InterfaceDescriptor) -> Vec { + let mut digest = Sha256::new(); + digest.update(format!("{descriptor:?}").as_bytes()); + digest.finalize().to_vec() +} + +pub(crate) fn contextual_reference(base: &str, path: &str) -> String { + let encoded: String = path + .as_bytes() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(); + format!("{base}/@/{encoded}") +} + +fn decode_contextual_path(encoded: &str) -> Option { + if encoded.len() % 2 != 0 { + return None; + } + let bytes = encoded + .as_bytes() + .chunks_exact(2) + .map(|pair| { + let digit = |byte| match byte { + b'0'..=b'9' => Some(byte - b'0'), + b'a'..=b'f' => Some(byte - b'a' + 10), + _ => None, + }; + Some(digit(pair[0])? * 16 + digit(pair[1])?) + }) + .collect::>>()?; + let path = String::from_utf8(bytes).ok()?; + wip_protocol::validate_path(&path).ok()?; + Some(path) +} + +fn is_tool_route(route: &str) -> bool { + route + .strip_prefix("/tools/") + .is_some_and(|name| !name.is_empty() && !name.contains('/')) +} + fn validate_projection(projection: &WipProjection) -> Result<(), WipMountError> { wip_protocol::validate_path(&projection.route).map_err(|error| { WipMountError::InvalidRoute { @@ -486,36 +1068,284 @@ impl WipHost { } } - fn descriptor(&self, reference: &str) -> Option<(&InterfaceDescriptor, Option<&[u8]>)> { + fn subtree(&self, path: &str) -> Option<&WipSubtreeMount> { self.registry - .mounts - .values() - .find(|mounted| mounted.projection.interface == reference) - .map(|mounted| { - ( - &mounted.projection.descriptor, - mounted.projection.interface_validator.as_deref(), + .subtree_mounts + .iter() + .find(|m| path == m.root || path.starts_with(&format!("{}/", m.root))) + } + + async fn projection_live(&self, path: &str) -> Result, ProtocolError> { + wip_protocol::validate_path(path).map_err(|_| unpublished_path_error(path))?; + if let Some(mount) = self.subtree(path) { + let Some(projection) = mount.provider.projection(path).await? else { + return Ok(None); + }; + validate_projection(&projection).map_err(|_| { + protocol_error( + ProtocolErrorCode::Internal, + "subtree returned an invalid Object", ) - }) - .or_else(|| { - self.registry - .dynamic_mounts - .iter() - .find(|mounted| mounted.interface == reference) - .map(|mounted| (&mounted.descriptor, mounted.interface_validator.as_deref())) - }) + })?; + let owner = &self.registry.mounts[&mount.root].projection.capability; + if projection.route != path + || projection.kind != WipProjectionKind::Native + || projection.capability.split(':').next() != owner.split(':').next() + { + return Err(protocol_error( + ProtocolErrorCode::Internal, + "subtree changed route or ownership", + )); + } + return Ok(Some(projection)); + } + Ok(self.projection(path)) + } + + async fn descriptor_live( + &self, + reference: &str, + ) -> Result>)>, ProtocolError> { + if let Some((_, encoded)) = reference.rsplit_once("/@/") + && let Some(path) = decode_contextual_path(encoded) + && self.subtree(&path).is_some() + { + return Ok(self + .projection_live(&path) + .await? + .filter(|p| p.interface == reference) + .map(|p| (p.descriptor, p.interface_validator))); + } + // Never expose the static registration placeholder descriptor of a subtree. + if self + .registry + .subtree_mounts + .iter() + .any(|m| self.registry.mounts[&m.root].projection.interface == reference) + { + return Ok(None); + } + Ok(self.descriptor(reference)) + } + + async fn fetch_interface_live( + &self, + reference: &str, + ) -> Result { + let (descriptor, validator) = self.descriptor_live(reference).await?.ok_or_else(|| { + protocol_error( + ProtocolErrorCode::InterfaceNotFound, + "interface is not published", + ) + })?; + Ok(FetchInterfaceResponse { + interface: reference.into(), + descriptor, + validator, + }) + } + + async fn observe_live( + &self, + path: &str, + depth: u32, + ) -> Result { + if depth > 32 { + return Err(protocol_error( + ProtocolErrorCode::ResourceLimitExceeded, + "observation depth exceeds 32", + )); + } + let mut remaining = 1024usize; + tokio::time::timeout( + std::time::Duration::from_secs(30), + self.observe_live_node(path, depth, &mut remaining), + ) + .await + .map_err(|_| { + protocol_error( + ProtocolErrorCode::ResourceLimitExceeded, + "observation deadline exceeded", + ) + })? + } + + fn observe_live_node<'a>( + &'a self, + path: &'a str, + depth: u32, + remaining: &'a mut usize, + ) -> std::pin::Pin< + Box> + Send + 'a>, + > { + Box::pin(async move { + if let Some(mount) = self.subtree(path) { + if depth > mount.provider.max_depth() { + return Err(protocol_error( + ProtocolErrorCode::ResourceLimitExceeded, + "observation exceeds provider depth limit", + )); + } + *remaining = (*remaining).min(mount.provider.max_nodes()); + } + if *remaining == 0 { + return Err(protocol_error( + ProtocolErrorCode::ResourceLimitExceeded, + "observation exceeds 1024 nodes", + )); + } + *remaining -= 1; + let object = if self.subtree(path).is_some() { + self.projection_live(path).await?.map(|p| p.object) + } else { + self.object_at(path) + } + .ok_or_else(|| unpublished_path_error(path))?; + let children = if depth == 0 { + None + } else { + let paths = if let Some(mount) = self.subtree(path) { + mount.provider.children(path).await? + } else { + self.children(path) + }; + let mut values = Vec::new(); + let mut seen = BTreeSet::new(); + for child in paths { + let prefix = if path == "/" { + "/".to_string() + } else { + format!("{path}/") + }; + let suffix = child.strip_prefix(&prefix).ok_or_else(|| { + protocol_error(ProtocolErrorCode::Internal, "subtree child outside parent") + })?; + if suffix.is_empty() || suffix.contains('/') || !seen.insert(child.clone()) { + return Err(protocol_error( + ProtocolErrorCode::Internal, + "invalid/duplicate subtree child", + )); + } + values.push(self.observe_live_node(&child, depth - 1, remaining).await?); + } + Some(values) + }; + Ok(ObjectObservation { object, children }) + }) + } + + fn descriptor(&self, reference: &str) -> Option<(InterfaceDescriptor, Option>)> { + for (path, mounted) in &self.registry.mounts { + if mounted.projection.interface == reference { + let Some(projection) = self.projection(path) else { + continue; + }; + if projection.interface == reference { + return Some((projection.descriptor, projection.interface_validator)); + } + } + } + if let Some(descriptor) = self.registry.dynamic_mounts.iter().find_map(|mounted| { + if mounted.interface != reference || self.dynamic_contextual(mounted) { + return None; + } + // Legacy families retain their shared descriptor. Contextual + // families must always resolve a qualified item reference below. + Some(( + mounted.descriptor.clone(), + mounted.interface_validator.clone(), + )) + }) { + return Some(descriptor); + } + let (_, encoded_path) = reference.rsplit_once("/@/")?; + let path = decode_contextual_path(encoded_path)?; + let projection = self.projection(&path)?; + // Re-resolve the exact current Object, not a cached descriptor or an + // arbitrary base interface with an appended path. Exact legacy refs + // above remain opaque even if they contain our contextual delimiter. + if projection.interface != reference { + return None; + } + Some((projection.descriptor, projection.interface_validator)) + } + + fn dynamic_contextual(&self, mounted: &WipDynamicMount) -> bool { + mounted.resolver.contextual_interface() + || self.registry.mounts[&mounted.collection_route] + .resolved() + .handler + .contextual_interface() + } + + fn project_current( + &self, + mut projection: WipProjection, + contextual_family: bool, + ) -> Option { + if !projection.handler.is_visible() { + return None; + } + if let Some(validator) = projection.handler.object_validator() { + projection.object.validator = Some(validator); + } + let contextual = contextual_family || projection.handler.contextual_interface(); + let original_len = projection.descriptor.operations.len(); + projection + .descriptor + .operations + .retain(|operation| projection.handler.operation_available(&operation.name)); + if contextual || projection.descriptor.operations.len() != original_len { + let mut digest = Sha256::new(); + digest.update(descriptor_validator(&projection.descriptor)); + // Keep an explicit provider version authoritative even when the + // filtered shape is unchanged. + if let Some(validator) = &projection.interface_validator { + digest.update(validator); + } + projection.interface_validator = Some(digest.finalize().to_vec()); + } + if contextual { + projection.interface = contextual_reference(&projection.interface, &projection.route); + projection.object.interfaces = vec![projection.interface.clone()]; + } + Some(projection) } fn projection(&self, path: &str) -> Option { if let Some(mounted) = self.registry.mounts.get(path) { - return Some(mounted.projection.clone()); + return self.project_current(mounted.resolved(), false); } self.registry.dynamic_mounts.iter().find_map(|mounted| { let item_reference = path.strip_prefix(&format!("{}/", mounted.collection_route))?; if item_reference.is_empty() || item_reference.contains('/') { return None; } + // Collection listing authority is independent of item read authority. + // The resolved item's owner gates its own visibility below. let item = mounted.resolver.resolve(item_reference)?; + let contributions = self + .registry + .dynamic_operation_contributions + .get(&mounted.collection_route); + let contributed_names = contributions + .into_iter() + .flatten() + .flat_map(|contribution| contribution.operations.iter()) + .collect::>(); + let mut handlers = mounted + .descriptor + .operations + .iter() + .filter(|operation| !contributed_names.contains(&operation.name)) + .map(|operation| (operation.name.clone(), Arc::clone(&item.handler))) + .collect::>(); + for contribution in contributions.into_iter().flatten() { + let handler = contribution.resolver.handler(item_reference); + for operation in &contribution.operations { + handlers.insert(operation.clone(), Arc::clone(&handler)); + } + } let projection = WipProjection { route: path.to_string(), capability: mounted.capability.clone(), @@ -524,14 +1354,32 @@ impl WipHost { interface: mounted.interface.clone(), descriptor: mounted.descriptor.clone(), interface_validator: mounted.interface_validator.clone(), - handler: item.handler, + handler: Arc::new(OperationDispatchHandler { + owner: item.handler, + handlers, + }), }; validate_projection(&projection).ok()?; - Some(projection) + self.project_current(projection, self.dynamic_contextual(mounted)) }) } fn object_at(&self, path: &str) -> Option { + // An explicit Object can also have static descendants. Do not replace + // its identity, interfaces, or validator with a synthetic namespace. + if let Some(projection) = self.projection(path) { + return Some(projection.object); + } + if self.registry.mounts.contains_key(path) + || self + .registry + .dynamic_mounts + .iter() + .any(|mount| path.starts_with(&format!("{}/", mount.collection_route))) + { + // A denied native Object must not reappear as a synthetic namespace. + return None; + } if path == WIP_ROOT || self.is_namespace(path) { let name = if path == WIP_ROOT { String::new() @@ -546,8 +1394,7 @@ impl WipHost { validator: Some(self.generation.clone()), }); } - self.projection(path) - .map(|projection| projection.object.clone()) + None } fn is_namespace(&self, path: &str) -> bool { @@ -559,7 +1406,7 @@ impl WipHost { self.registry .mounts .keys() - .any(|route| route.starts_with(&prefix)) + .any(|route| route.starts_with(&prefix) && self.projection(route).is_some()) } fn children(&self, path: &str) -> Vec { @@ -584,15 +1431,18 @@ impl WipHost { } else { format!("{path}/{segment}") }; - children.insert(child); + if self.object_at(&child).is_some() { + children.insert(child); + } } children.into_iter().collect() } + #[cfg(test)] fn observe(&self, path: &str, depth: u32) -> Result { - let object = self.object_at(path).ok_or_else(|| { - protocol_error(ProtocolErrorCode::NotFound, "target path is not published") - })?; + let object = self + .object_at(path) + .ok_or_else(|| unpublished_path_error(path))?; let children = if depth == 0 { None } else { @@ -605,6 +1455,7 @@ impl WipHost { Ok(ObjectObservation { object, children }) } + #[cfg(test)] fn fetch_interface(&self, reference: &str) -> Result { let (descriptor, validator) = self.descriptor(reference).ok_or_else(|| { protocol_error( @@ -614,22 +1465,33 @@ impl WipHost { })?; Ok(FetchInterfaceResponse { interface: reference.to_string(), - descriptor: descriptor.clone(), - validator: validator.map(<[u8]>::to_vec), + descriptor, + validator, }) } + #[cfg(test)] async fn call( &self, request: CallOperationRequest, context: WipCallContext, ) -> Result { - let projection = self.projection(&request.target.path).ok_or_else(|| { - WipOperationError::Protocol(protocol_error( - ProtocolErrorCode::NotFound, - "target path is not published", - )) - })?; + let projection = self + .projection_live(&request.target.path) + .await + .map_err(WipOperationError::Protocol)? + .ok_or_else(|| { + WipOperationError::Protocol(unpublished_path_error(&request.target.path)) + })?; + self.call_projection(projection, request, context).await + } + + async fn call_projection( + &self, + projection: WipProjection, + request: CallOperationRequest, + context: WipCallContext, + ) -> Result { let expected_object_validator = projection.object.validator.as_deref(); match ( expected_object_validator, @@ -698,13 +1560,15 @@ impl WipHost { .call(&request.operation, &request.arguments, context) .await } +} - async fn cancel(&self, path: &str, context: &ToolExecutionContext) -> Result<(), ToolError> { - let projection = self.projection(path).ok_or_else(|| { - ToolError::InvalidArgument("WIP cancellation target is no longer published".into()) - })?; - projection.handler.cancel(context).await - } +fn unpublished_path_error(path: &str) -> ProtocolError { + let message = if path == "/features" || path.starts_with("/features/") { + "legacy /features/... routes are no longer published; rediscover from `/` and use the root namespace" + } else { + "target path is not published" + }; + protocol_error(ProtocolErrorCode::NotFound, message) } fn protocol_error(code: ProtocolErrorCode, message: impl Into) -> ProtocolError { @@ -953,6 +1817,12 @@ struct ClientState { session: SessionId, } +#[derive(Clone)] +struct ActiveOperation { + operation: String, + handler: Arc, +} + pub struct WipRuntime { endpoint: Endpoint, wire_limits: Limits, @@ -960,12 +1830,25 @@ pub struct WipRuntime { security_context: SecurityContext, state: Arc>, host: Arc, - active: Arc>>, + active: Arc>>, audit: Arc>>, metrics: Arc, } impl WipRuntime { + /// Build an isolated Client/Host endpoint from native application mounts. + /// Useful to embedders and in-process integration tests without an LLM Engine. + pub fn from_mounts( + registry: WipMountRegistry, + security_context: String, + ) -> Result { + Self::new( + WipHost::new(registry), + SecurityContext::new(security_context), + 0, + ) + } + fn new( host: WipHost, security_context: SecurityContext, @@ -1020,7 +1903,7 @@ impl WipRuntime { Ok(()) } - async fn discover( + pub async fn discover( &self, path: String, depth: u32, @@ -1045,10 +1928,11 @@ impl WipRuntime { .fetch_add(1, Ordering::Relaxed); let response = self.dispatch_retrieval(&prepared.request).await?; let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner()); - state + let completion = state .client .complete(prepared.id, response) .map_err(client_tool_error)?; + check_retrieval_completion(completion)?; } let state = self.state.lock().unwrap_or_else(|error| error.into_inner()); let values = state @@ -1072,7 +1956,7 @@ impl WipRuntime { )) } - async fn inspect(&self, interface: String, refresh: bool) -> Result { + pub async fn inspect(&self, interface: String, refresh: bool) -> Result { let prepared = { let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner()); let session = state.session.clone(); @@ -1092,10 +1976,11 @@ impl WipRuntime { .fetch_add(1, Ordering::Relaxed); let response = self.dispatch_retrieval(&prepared.request).await?; let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner()); - state + let completion = state .client .complete(prepared.id, response) .map_err(client_tool_error)?; + check_retrieval_completion(completion)?; } let state = self.state.lock().unwrap_or_else(|error| error.into_inner()); let observed = state @@ -1114,7 +1999,7 @@ impl WipRuntime { )) } - async fn call( + pub async fn call( &self, path: String, interface: String, @@ -1122,7 +2007,12 @@ impl WipRuntime { arguments: Json, execution: ToolExecutionContext, ) -> Result { - let arguments = match self.host.projection(&path) { + let arguments = match self + .host + .projection_live(&path) + .await + .map_err(|e| ToolError::InvalidArgument(e.message))? + { Some(projection) if projection.kind == WipProjectionKind::Compatibility => { BTreeMap::from([( "input".to_string(), @@ -1139,9 +2029,8 @@ impl WipRuntime { ) .map_err(ToolError::InvalidArgument)?, None => { - return Err(ToolError::InvalidArgument(format!( - "WIP target `{path}` is not published" - ))); + let error = unpublished_path_error(&path); + return Err(ToolError::InvalidArgument(error.message)); } }; let prepared = { @@ -1163,10 +2052,6 @@ impl WipRuntime { .operation_round_trips .fetch_add(1, Ordering::Relaxed); let execution_id = execution.execution_id(); - self.active - .lock() - .unwrap_or_else(|error| error.into_inner()) - .insert(execution_id.clone(), path.clone()); let mut guard = DispatchedCallGuard::new( Arc::clone(&self.state), Arc::clone(&self.active), @@ -1208,14 +2093,19 @@ impl WipRuntime { } async fn cancel(&self, execution: &ToolExecutionContext) -> Result<(), ToolError> { - let path = self + let active = self .active .lock() .unwrap_or_else(|error| error.into_inner()) .get(&execution.execution_id()) .cloned(); - match path { - Some(path) => self.host.cancel(&path, execution).await, + match active { + Some(active) => { + active + .handler + .cancel_operation(&active.operation, execution) + .await + } None => Ok(()), } } @@ -1253,7 +2143,11 @@ impl WipRuntime { Route::Observe => { let request_value = decode_observe_request(request.body(), self.wire_limits) .map_err(|error| ToolError::Internal(error.to_string()))?; - match self.host.observe(&request_value.path, request_value.depth) { + match self + .host + .observe_live(&request_value.path, request_value.depth) + .await + { Ok(value) => encode_observe_response(&request_value, &value, self.wire_limits), Err(error) => encode_protocol_error_response( ProtocolInteraction::Observe, @@ -1267,7 +2161,11 @@ impl WipRuntime { let request_value = decode_fetch_interface_request(request.body(), self.wire_limits) .map_err(|error| ToolError::Internal(error.to_string()))?; - match self.host.fetch_interface(&request_value.interface) { + match self + .host + .fetch_interface_live(&request_value.interface) + .await + { Ok(value) => { encode_fetch_interface_response(&request_value, &value, self.wire_limits) } @@ -1301,7 +2199,9 @@ impl WipRuntime { .map_err(|error| ToolError::InvalidArgument(error.to_string()))?; let descriptor = self .host - .descriptor(&metadata.interface.reference) + .descriptor_live(&metadata.interface.reference) + .await + .map_err(|e| ToolError::InvalidArgument(e.message))? .map(|(descriptor, _)| descriptor.clone()) .ok_or_else(|| ToolError::InvalidArgument("interface is not published".into()))?; let request_value = metadata @@ -1311,21 +2211,71 @@ impl WipRuntime { execution, security_context: self.security_context.as_str().to_string(), }; - match self.host.call(request_value.clone(), context).await { + let result = match self + .host + .projection_live(&request_value.target.path) + .await + .map_err(|e| ToolError::InvalidArgument(e.message))? + { + Some(projection) => { + // Pin the exact resolved instance used for dispatch. A dynamic + // resolver may return a fresh stateful handler on each lookup. + self.active + .lock() + .unwrap_or_else(|error| error.into_inner()) + .insert( + context.execution.execution_id(), + ActiveOperation { + operation: request_value.operation.clone(), + handler: Arc::clone(&projection.handler), + }, + ); + self.host + .call_projection(projection, request_value.clone(), context) + .await + } + None => Err(WipOperationError::Protocol(unpublished_path_error( + &request_value.target.path, + ))), + }; + match result { Ok(output) => { let response = encode_call_operation_response( &request_value, &descriptor, &CallOperationResponse { result: output.value, - validator: self - .host - .projection(&request_value.target.path) - .and_then(|projection| projection.object.validator.clone()), + validator: match output.validator { + Some(validator) => Some(validator), + None if self.host.subtree(&request_value.target.path).is_some() => None, + None => self + .host + .projection(&request_value.target.path) + .and_then(|projection| projection.object.validator.clone()), + }, }, self.wire_limits, - ) - .map_err(|error| ToolError::Internal(error.to_string()))?; + ); + let response = match response { + Ok(response) => response, + Err(_) => { + // Dispatch has completed: losing its result is not a pre-effect refusal. + let message = "completed operation result could not be encoded; inspect effects before retry"; + let response = encode_protocol_error_response( + ProtocolInteraction::CallOperation, + &protocol_error(ProtocolErrorCode::OperationOutcomeUnknown, message), + self.wire_limits, + ) + .map_err(|error| ToolError::Internal(error.to_string()))?; + return Ok(( + response, + Some(Err(ToolError::ExecutionFailed(format!( + "WIP operation outcome unknown; do not retry automatically: {message}" + )))), + WipAuditOutcome::OutcomeUnknown, + )); + } + }; Ok(( response, output.tool_output.map(Ok), @@ -1394,7 +2344,7 @@ impl WipRuntime { struct DispatchedCallGuard { state: Arc>, - active: Arc>>, + active: Arc>>, request_id: RequestId, execution_id: String, armed: bool, @@ -1403,7 +2353,7 @@ struct DispatchedCallGuard { impl DispatchedCallGuard { fn new( state: Arc>, - active: Arc>>, + active: Arc>>, request_id: RequestId, execution_id: String, ) -> Self { @@ -1833,6 +2783,19 @@ fn tool_output_value(output: &ToolOutput) -> Value { Value::Record(record) } +fn check_retrieval_completion(completion: Completion) -> Result<(), ToolError> { + match completion { + Completion::Object(_) | Completion::Interface(_) => Ok(()), + Completion::ProtocolFailure(error) => Err(ToolError::ExecutionFailed(error.to_string())), + Completion::StaleResponseRejected(_) => Err(ToolError::ExecutionFailed( + "WIP retrieval superseded; refresh observations".into(), + )), + Completion::Call(_) => Err(ToolError::Internal( + "WIP retrieval returned a call result".into(), + )), + } +} + fn render_call_completion(completion: Completion) -> Result { let Completion::Call(record) = completion else { return Err(ToolError::Internal( @@ -1956,7 +2919,7 @@ mod tests { use agen::llm_client::{ClientError, Request as LlmRequest, ResponseStream}; use futures::stream; use std::collections::VecDeque; - use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::atomic::{AtomicBool, AtomicUsize, Ordering}; use wip_protocol::{EnumCase, FieldDeclaration, TypeDeclaration, UnionCase}; struct EchoTool { @@ -2184,6 +3147,1827 @@ mod tests { )); } + #[tokio::test] + async fn completed_native_result_encoding_failure_is_unknown_and_not_retried() { + struct OversizedResult(Arc); + #[async_trait] + impl WipOperationHandler for OversizedResult { + async fn call( + &self, + _: &str, + _: &BTreeMap, + _: WipCallContext, + ) -> Result { + self.0.fetch_add(1, Ordering::SeqCst); + Ok(WipOperationOutput::native(Value::String( + "x".repeat(3 * 1024 * 1024), + ))) + } + } + let calls = Arc::new(AtomicUsize::new(0)); + let mut p = contribution_projection(Arc::new(OversizedResult(calls.clone()))); + p.descriptor.operations[0].returns.r#type = TypeExpr::String; + p.interface_validator = Some(descriptor_validator(&p.descriptor)); + let mut registry = WipMountRegistry::new(); + registry.allocate_namespace("asset", "assets").unwrap(); + registry.mount(p).unwrap(); + let runtime = WipRuntime::new( + WipHost::new(registry), + SecurityContext::new("encoding-test"), + 0, + ) + .unwrap(); + runtime + .discover("/assets/A-1".into(), 0, true) + .await + .unwrap(); + runtime + .inspect("test.asset/item/v1".into(), true) + .await + .unwrap(); + let error = runtime + .call( + "/assets/A-1".into(), + "test.asset/item/v1".into(), + "read".into(), + Json::Object(Default::default()), + Default::default(), + ) + .await + .unwrap_err(); + assert!(error.to_string().contains("outcome unknown"), "{error}"); + assert_eq!(calls.load(Ordering::SeqCst), 1); + } + + struct ContributionHandler { + calls: Arc, + allowed: Arc, + } + + #[async_trait] + impl WipOperationHandler for ContributionHandler { + async fn call( + &self, + _operation: &str, + _arguments: &BTreeMap, + _context: WipCallContext, + ) -> Result { + if !self.allowed.load(Ordering::SeqCst) { + return Err(WipOperationError::Protocol(protocol_error( + ProtocolErrorCode::PermissionDenied, + "current subject is not authorized", + ))); + } + self.calls.fetch_add(1, Ordering::SeqCst); + Ok(WipOperationOutput::native(Value::Unit)) + } + } + + fn contributed_operation(name: &str) -> OperationDeclaration { + OperationDeclaration { + name: name.into(), + documentation: documentation(name), + parameters: Vec::new(), + returns: ReturnDeclaration { + documentation: None, + r#type: TypeExpr::Unit, + }, + } + } + + fn contribution_projection(handler: Arc) -> WipProjection { + let descriptor = InterfaceDescriptor { + format: INTERFACE_FORMAT_V1.into(), + documentation: documentation("shared object interface"), + types: Vec::new(), + operations: vec![contributed_operation("read")], + }; + WipProjection { + route: "/assets/A-1".into(), + capability: "asset:object".into(), + kind: WipProjectionKind::Native, + object: Object { + name: "A-1".into(), + description: Some("host-resolved asset".into()), + interfaces: vec!["test.asset/item/v1".into()], + r#ref: Some("asset:A-1".into()), + validator: Some(vec![1]), + }, + interface: "test.asset/item/v1".into(), + interface_validator: Some(descriptor_validator(&descriptor)), + descriptor, + handler, + } + } + + #[tokio::test] + async fn host_namespace_object_resolution_and_operation_contributions_are_independent() { + let read_calls = Arc::new(AtomicUsize::new(0)); + let manage_calls = Arc::new(AtomicUsize::new(0)); + let read_allowed = Arc::new(AtomicBool::new(true)); + let manage_allowed = Arc::new(AtomicBool::new(false)); + let mut registry = WipMountRegistry::new(); + assert!(matches!( + registry.allocate_namespace("host", "features"), + Err(WipMountError::InvalidRoute { .. }) + )); + assert!(matches!( + registry.allocate_namespace("host", "tools"), + Err(WipMountError::InvalidRoute { .. }) + )); + let namespace = registry.allocate_namespace("asset", "assets").unwrap(); + assert_eq!(namespace.root(), "/assets"); + assert!(matches!( + registry.allocate_namespace("other", "assets"), + Err(WipMountError::RouteCollision { .. }) + )); + registry + .mount(contribution_projection(Arc::new(ContributionHandler { + calls: Arc::clone(&read_calls), + allowed: Arc::clone(&read_allowed), + }))) + .unwrap(); + + let host_without_management = WipHost::new(registry); + let resolved = host_without_management.projection("/assets/A-1").unwrap(); + assert_eq!(resolved.object.r#ref.as_deref(), Some("asset:A-1")); + assert_eq!( + resolved + .descriptor + .operations + .iter() + .map(|operation| operation.name.as_str()) + .collect::>(), + ["read"] + ); + + let mut registry = host_without_management.registry; + let contribution_descriptor = InterfaceDescriptor { + format: INTERFACE_FORMAT_V1.into(), + documentation: documentation("shared object interface"), + types: Vec::new(), + operations: vec![contributed_operation("manage")], + }; + registry + .contribute_operations(WipOperationContribution { + route: "/assets/A-1".into(), + contributor: "asset-management".into(), + interface: "test.asset/item/v1".into(), + descriptor: contribution_descriptor.clone(), + handler: Arc::new(ContributionHandler { + calls: Arc::clone(&manage_calls), + allowed: Arc::clone(&manage_allowed), + }), + }) + .unwrap(); + assert!(matches!( + registry.contribute_operations(WipOperationContribution { + route: "/assets/A-1".into(), + contributor: "duplicate-management".into(), + interface: "test.asset/item/v1".into(), + descriptor: contribution_descriptor, + handler: Arc::new(ContributionHandler { + calls: Arc::clone(&manage_calls), + allowed: Arc::clone(&manage_allowed), + }), + }), + Err(WipMountError::OperationCollision { .. }) + )); + let host = WipHost::new(registry); + let projection = host.projection("/assets/A-1").unwrap(); + assert_eq!( + projection + .descriptor + .operations + .iter() + .map(|operation| operation.name.as_str()) + .collect::>(), + ["manage", "read"] + ); + + let request = |operation: &str| CallOperationRequest { + target: wip_protocol::Target { + path: "/assets/A-1".into(), + validator: projection.object.validator.clone(), + }, + interface: wip_protocol::InterfaceTarget { + reference: projection.interface.clone(), + validator: projection.interface_validator.clone(), + }, + operation: operation.into(), + arguments: BTreeMap::new(), + }; + let context = || WipCallContext { + execution: ToolExecutionContext::direct(), + security_context: "worker-a".into(), + }; + assert!(matches!( + host.call(request("manage"), context()).await, + Err(WipOperationError::Protocol(ProtocolError { + code: ProtocolErrorCode::PermissionDenied, + .. + })) + )); + manage_allowed.store(true, Ordering::SeqCst); + host.call(request("manage"), context()) + .await + .unwrap_or_else(|_| panic!("authorized management contribution should run")); + manage_allowed.store(false, Ordering::SeqCst); + assert!(matches!( + host.call(request("manage"), context()).await, + Err(WipOperationError::Protocol(ProtocolError { + code: ProtocolErrorCode::PermissionDenied, + .. + })) + )); + host.call(request("read"), context()) + .await + .unwrap_or_else(|_| panic!("independent read contribution should run")); + assert_eq!(manage_calls.load(Ordering::SeqCst), 1); + assert_eq!(read_calls.load(Ordering::SeqCst), 1); + } + + #[test] + fn operation_contributions_reject_interface_and_target_conflicts() { + let calls = Arc::new(AtomicUsize::new(0)); + let allowed = Arc::new(AtomicBool::new(true)); + let handler: Arc = + Arc::new(ContributionHandler { calls, allowed }); + let mut registry = WipMountRegistry::new(); + let projection = contribution_projection(Arc::clone(&handler)); + assert!(matches!( + registry.mount(projection.clone()), + Err(WipMountError::InvalidProjection { .. }) + )); + registry.allocate_namespace("asset", "assets").unwrap(); + registry.mount(projection).unwrap(); + let mut inconsistent = InterfaceDescriptor { + format: INTERFACE_FORMAT_V1.into(), + documentation: documentation("different interface contract"), + types: Vec::new(), + operations: vec![contributed_operation("manage")], + }; + assert!(matches!( + registry.contribute_operations(WipOperationContribution { + route: "/assets/A-1".into(), + contributor: "management".into(), + interface: "test.asset/item/v1".into(), + descriptor: inconsistent.clone(), + handler: Arc::clone(&handler), + }), + Err(WipMountError::InterfaceCollision { .. }) + )); + inconsistent.documentation = documentation("shared object interface"); + assert!(matches!( + registry.contribute_operations(WipOperationContribution { + route: "/assets/A-2".into(), + contributor: "management".into(), + interface: "test.asset/item/v1".into(), + descriptor: inconsistent, + handler, + }), + Err(WipMountError::OperationTargetNotFound { .. }) + )); + } + + fn asset_registry( + read: Arc, + manage: Option>, + calls: Arc, + ) -> WipMountRegistry { + let mut registry = WipMountRegistry::new(); + registry.allocate_namespace("asset", "assets").unwrap(); + let projection = contribution_projection(Arc::new(ContributionHandler { + calls: Arc::clone(&calls), + allowed: read, + })); + let mut descriptor = projection.descriptor.clone(); + descriptor.operations = vec![contributed_operation("manage")]; + registry.mount(projection).unwrap(); + if let Some(allowed) = manage { + registry + .contribute_operations(WipOperationContribution { + route: "/assets/A-1".into(), + contributor: "asset-management".into(), + interface: "test.asset/item/v1".into(), + descriptor, + handler: Arc::new(ContributionHandler { calls, allowed }), + }) + .unwrap(); + } + registry + } + + fn asset_request(projection: &WipProjection, operation: &str) -> CallOperationRequest { + CallOperationRequest { + target: wip_protocol::Target { + path: projection.route.clone(), + validator: projection.object.validator.clone(), + }, + interface: wip_protocol::InterfaceTarget { + reference: projection.interface.clone(), + validator: projection.interface_validator.clone(), + }, + operation: operation.into(), + arguments: BTreeMap::new(), + } + } + + fn direct_wip_context() -> WipCallContext { + WipCallContext { + execution: ToolExecutionContext::direct(), + security_context: "worker-a".into(), + } + } + + #[tokio::test] + async fn management_enablement_is_independent_of_read_and_operation_permissions() { + for enabled in [false, true] { + for can_read in [false, true] { + for can_manage in [false, true] { + let calls = Arc::new(AtomicUsize::new(0)); + let host = WipHost::new(asset_registry( + Arc::new(AtomicBool::new(can_read)), + enabled.then(|| Arc::new(AtomicBool::new(can_manage))), + Arc::clone(&calls), + )); + // Metadata resolution is not authorization to read domain data. + let projection = host.projection("/assets/A-1").unwrap(); + let read = host + .call(asset_request(&projection, "read"), direct_wip_context()) + .await; + if can_read { + assert!(read.is_ok()); + } else { + assert!(matches!( + read, + Err(WipOperationError::Protocol(ProtocolError { + code: ProtocolErrorCode::PermissionDenied, + .. + })) + )); + } + let manage = host + .call(asset_request(&projection, "manage"), direct_wip_context()) + .await; + if !enabled { + assert!(matches!( + manage, + Err(WipOperationError::Protocol(ProtocolError { + code: ProtocolErrorCode::OperationNotFound, + .. + })) + )); + } else if !can_manage { + assert!(matches!( + manage, + Err(WipOperationError::Protocol(ProtocolError { + code: ProtocolErrorCode::PermissionDenied, + .. + })) + )); + } else { + assert!(manage.is_ok()); + } + assert_eq!( + calls.load(Ordering::SeqCst), + usize::from(can_read) + usize::from(enabled && can_manage) + ); + } + } + } + } + + #[tokio::test] + async fn contributed_interface_invalidates_observations_and_restore_requires_discovery() { + let calls = Arc::new(AtomicUsize::new(0)); + let allowed = Arc::new(AtomicBool::new(true)); + let old_host = WipHost::new(asset_registry( + Arc::clone(&allowed), + None, + Arc::clone(&calls), + )); + let old_projection = old_host.projection("/assets/A-1").unwrap(); + let old_runtime = WipRuntime::new(old_host, SecurityContext::new("worker-a"), 0).unwrap(); + old_runtime + .discover("/assets/A-1".into(), 0, false) + .await + .unwrap(); + old_runtime + .inspect(old_projection.interface.clone(), false) + .await + .unwrap(); + let restored_host = WipHost::new(asset_registry( + Arc::clone(&allowed), + Some(Arc::clone(&allowed)), + Arc::clone(&calls), + )); + let current = restored_host.projection("/assets/A-1").unwrap(); + assert_eq!(old_projection.object, current.object); + assert_ne!( + old_projection.interface_validator, + current.interface_validator + ); + assert!(matches!( + restored_host + .call(asset_request(&old_projection, "read"), direct_wip_context()) + .await, + Err(WipOperationError::Protocol(ProtocolError { + code: ProtocolErrorCode::InterfaceValidatorMismatch, + .. + })) + )); + let restored = WipRuntime::new(restored_host, SecurityContext::new("worker-a"), 0).unwrap(); + // Same Worker security identity, fresh runtime: no inherited observations. + assert!( + restored + .call( + "/assets/A-1".into(), + current.interface.clone(), + "read".into(), + json!({}), + ToolExecutionContext::direct() + ) + .await + .is_err() + ); + assert_eq!(calls.load(Ordering::SeqCst), 0); + restored + .discover("/assets/A-1".into(), 0, false) + .await + .unwrap(); + restored + .inspect(current.interface.clone(), false) + .await + .unwrap(); + restored + .call( + "/assets/A-1".into(), + current.interface.clone(), + "manage".into(), + json!({}), + ToolExecutionContext::direct(), + ) + .await + .unwrap(); + allowed.store(false, Ordering::SeqCst); + assert!( + restored + .call( + "/assets/A-1".into(), + current.interface.clone(), + "manage".into(), + json!({}), + ToolExecutionContext::direct() + ) + .await + .unwrap_err() + .to_string() + .contains("PermissionDenied") + ); + restored.reset_observations().unwrap(); + assert!( + restored + .call( + "/assets/A-1".into(), + current.interface, + "manage".into(), + json!({}), + ToolExecutionContext::direct() + ) + .await + .is_err() + ); + assert_eq!(calls.load(Ordering::SeqCst), 1); + } + + #[test] + fn mounted_collection_keeps_identity_and_interface_with_static_children() { + let mut registry = asset_registry( + Arc::new(AtomicBool::new(true)), + None, + Arc::new(AtomicUsize::new(0)), + ); + let mut collection = contribution_projection(Arc::new(ContributionHandler { + calls: Arc::new(AtomicUsize::new(0)), + allowed: Arc::new(AtomicBool::new(true)), + })); + collection.route = "/assets".into(); + collection.object.name = "assets".into(); + collection.object.r#ref = Some("asset:collection".into()); + let expected = collection.object.clone(); + registry.mount(collection).unwrap(); + let observation = WipHost::new(registry).observe("/assets", 1).unwrap(); + assert_eq!(observation.object, expected); + assert_eq!(observation.children.unwrap()[0].object.name, "A-1"); + } + + #[test] + fn compatibility_mounts_cannot_claim_host_or_legacy_namespaces() { + for route in [ + "/features/ticket/tickets", + "/assets/A-1", + "/tools/Echo/child", + ] { + let mut projection = compatibility_projection( + meta("Echo"), + Arc::new(EchoTool { + calls: Arc::new(AtomicUsize::new(0)), + }), + None, + ) + .unwrap(); + projection.route = route.into(); + projection.object.name = route.rsplit('/').next().unwrap().into(); + let mut registry = WipMountRegistry::new(); + registry.allocate_namespace("asset", "assets").unwrap(); + assert!(matches!( + registry.mount(projection), + Err(WipMountError::InvalidProjection { .. }) + )); + assert!(registry.routes().next().is_none()); + } + } + + #[tokio::test] + async fn legacy_feature_routes_fail_with_rediscovery_guidance() { + let host = WipHost::new(WipMountRegistry::new()); + let error = host.observe("/features/ticket/tickets/T-1", 0).unwrap_err(); + assert_eq!(error.code, ProtocolErrorCode::NotFound); + assert!(error.message.contains("rediscover from `/`")); + assert!(error.message.contains("no longer published")); + + let runtime = WipRuntime::new(host, SecurityContext::new("worker-a"), 0).unwrap(); + let error = runtime + .call( + "/features/ticket/tickets/T-1".into(), + "yoi.ticket/item/v1".into(), + "read".into(), + json!({}), + ToolExecutionContext::direct(), + ) + .await + .unwrap_err(); + assert!(error.to_string().contains("rediscover from `/`")); + } + + #[derive(Clone)] + struct ContextualHandler { + visible: Arc, + permitted: Arc, + capable: Arc, + calls: Arc, + } + + impl ContextualHandler { + fn new(permitted: bool, capable: bool) -> Self { + Self { + visible: Arc::new(AtomicBool::new(true)), + permitted: Arc::new(AtomicBool::new(permitted)), + capable: Arc::new(AtomicBool::new(capable)), + calls: Arc::new(AtomicUsize::new(0)), + } + } + } + + #[async_trait] + impl WipOperationHandler for ContextualHandler { + fn is_visible(&self) -> bool { + self.visible.load(Ordering::SeqCst) + } + + fn operation_available(&self, _operation: &str) -> bool { + self.permitted.load(Ordering::SeqCst) && self.capable.load(Ordering::SeqCst) + } + + fn contextual_interface(&self) -> bool { + true + } + + async fn call( + &self, + operation: &str, + _arguments: &BTreeMap, + _context: WipCallContext, + ) -> Result { + if !self.is_visible() || !self.operation_available(operation) { + return Err(WipOperationError::Protocol(protocol_error( + ProtocolErrorCode::PermissionDenied, + "current native authority denied", + ))); + } + self.calls.fetch_add(1, Ordering::SeqCst); + Ok(WipOperationOutput::native(Value::Unit)) + } + } + + fn operation_names(descriptor: &InterfaceDescriptor) -> Vec<&str> { + descriptor + .operations + .iter() + .map(|operation| operation.name.as_str()) + .collect() + } + + fn contextual_static_registry( + owner: Arc, + contributor: Arc, + ) -> WipMountRegistry { + let mut registry = WipMountRegistry::new(); + registry.allocate_namespace("asset", "assets").unwrap(); + let projection = contribution_projection(owner); + let mut descriptor = projection.descriptor.clone(); + descriptor.operations = vec![contributed_operation("manage")]; + registry.mount(projection).unwrap(); + registry + .contribute_operations(WipOperationContribution { + route: "/assets/A-1".into(), + contributor: "management".into(), + interface: "test.asset/item/v1".into(), + descriptor, + handler: contributor, + }) + .unwrap(); + registry + } + + #[tokio::test] + async fn contextual_static_filters_contributors_and_rechecks_old_references() { + let owner = Arc::new(ContextualHandler::new(true, true)); + let contributor = Arc::new(ContextualHandler::new(false, true)); + let mut registry = contextual_static_registry(Arc::clone(&owner), Arc::clone(&contributor)); + // Same base contract, different Object/capability. It must not share the + // first Object's cached contextual descriptor. + let mut other = registry.mounts["/assets/A-1"].projection.clone(); + other.route = "/assets/A-2".into(); + other.object.name = "A-2".into(); + other.handler = Arc::new(ContextualHandler::new(true, false)); + registry.mount(other).unwrap(); + let host = WipHost::new(registry); + let initial = host.projection("/assets/A-1").unwrap(); + let other = host.projection("/assets/A-2").unwrap(); + assert_eq!( + initial.interface, + contextual_reference("test.asset/item/v1", "/assets/A-1") + ); + assert_ne!(initial.interface, other.interface); + assert_eq!(operation_names(&initial.descriptor), ["read"]); + assert!(other.descriptor.operations.is_empty()); + assert!(host.fetch_interface("test.asset/item/v1").is_err()); + assert_eq!( + host.fetch_interface(&initial.interface).unwrap().descriptor, + initial.descriptor + ); + host.call(asset_request(&initial, "read"), direct_wip_context()) + .await + .unwrap_or_else(|_| panic!("available owner operation must run")); + assert!(matches!( + host.call(asset_request(&initial, "manage"), direct_wip_context()) + .await, + Err(WipOperationError::Protocol(ProtocolError { + code: ProtocolErrorCode::OperationNotFound, + .. + })) + )); + + contributor.permitted.store(true, Ordering::SeqCst); + let enabled = host.projection("/assets/A-1").unwrap(); + assert_eq!(enabled.interface, initial.interface); + assert_eq!( + operation_names(&host.fetch_interface(&initial.interface).unwrap().descriptor), + ["manage", "read"] + ); + assert_ne!(enabled.interface_validator, initial.interface_validator); + assert!(matches!( + host.call(asset_request(&initial, "read"), direct_wip_context()) + .await, + Err(WipOperationError::Protocol(ProtocolError { + code: ProtocolErrorCode::InterfaceValidatorMismatch, + .. + })) + )); + host.call(asset_request(&enabled, "manage"), direct_wip_context()) + .await + .unwrap_or_else(|_| panic!("available contributor operation must run")); + contributor.capable.store(false, Ordering::SeqCst); + assert_eq!( + operation_names(&host.fetch_interface(&initial.interface).unwrap().descriptor), + ["read"] + ); + assert!(matches!( + host.call(asset_request(&enabled, "manage"), direct_wip_context()) + .await, + Err(WipOperationError::Protocol(ProtocolError { + code: ProtocolErrorCode::InterfaceValidatorMismatch, + .. + })) + )); + let current = host.projection("/assets/A-1").unwrap(); + assert!(matches!( + host.call(asset_request(¤t, "manage"), direct_wip_context()) + .await, + Err(WipOperationError::Protocol(ProtocolError { + code: ProtocolErrorCode::OperationNotFound, + .. + })) + )); + let mut unqualified = asset_request(¤t, "read"); + unqualified.interface.reference = "test.asset/item/v1".into(); + assert!(matches!( + host.call(unqualified, direct_wip_context()).await, + Err(WipOperationError::Protocol(ProtocolError { + code: ProtocolErrorCode::InterfaceMismatch, + .. + })) + )); + owner.visible.store(false, Ordering::SeqCst); + contributor.permitted.store(true, Ordering::SeqCst); + contributor.capable.store(true, Ordering::SeqCst); + assert!(host.projection("/assets/A-1").is_none()); + assert!(host.fetch_interface(&initial.interface).is_err()); + assert!(host.observe("/assets/A-1", 0).is_err()); + let observed = host.observe("/assets", 1).unwrap(); + assert_eq!( + observed + .children + .unwrap() + .iter() + .map(|item| item.object.name.as_str()) + .collect::>(), + ["A-2"] + ); + assert!(matches!( + host.call(asset_request(&enabled, "manage"), direct_wip_context()) + .await, + Err(WipOperationError::Protocol(ProtocolError { + code: ProtocolErrorCode::NotFound, + .. + })) + )); + assert_eq!(owner.calls.load(Ordering::SeqCst), 1); + assert_eq!(contributor.calls.load(Ordering::SeqCst), 1); + } + + #[test] + fn contextual_visibility_cannot_fall_back_to_namespace_and_provider_versions_stay_current() { + let owner = Arc::new(ContextualHandler::new(true, true)); + let mut registry = contextual_static_registry( + Arc::clone(&owner), + Arc::new(ContextualHandler::new(true, true)), + ); + let mut child = contribution_projection(Arc::new(DynamicHandler { + calls: Arc::new(AtomicUsize::new(0)), + })); + child.route = "/assets/A-1/child".into(); + child.object.name = "child".into(); + // Legacy interface references stay opaque, including this delimiter. + child.interface = "legacy/@/opaque".into(); + child.object.interfaces = vec![child.interface.clone()]; + registry.mount(child.clone()).unwrap(); + let mut host = WipHost::new(registry); + assert_eq!( + host.fetch_interface(&child.interface).unwrap().descriptor, + child.descriptor + ); + let initial = host.projection("/assets/A-1").unwrap(); + host.registry + .mounts + .get_mut("/assets/A-1") + .unwrap() + .projection + .interface_validator = Some(vec![99]); + let current = host.fetch_interface(&initial.interface).unwrap(); + assert_eq!(current.descriptor, initial.descriptor); + assert_ne!(current.validator, initial.interface_validator); + owner.visible.store(false, Ordering::SeqCst); + assert!(host.object_at("/assets/A-1").is_none()); + assert!(host.observe("/assets/A-1", 1).is_err()); + assert!( + host.observe("/assets", 1) + .unwrap() + .children + .unwrap() + .is_empty() + ); + } + + struct ContextualItemResolver { + owner: Arc, + revision: Arc, + contextual_family: bool, + } + + impl WipDynamicItemResolver for ContextualItemResolver { + fn contextual_interface(&self) -> bool { + self.contextual_family + } + + fn resolve(&self, item_reference: &str) -> Option { + if !matches!(item_reference, "O-1" | "O-2") { + return None; + } + let mut handler = self.owner.as_ref().clone(); + if item_reference == "O-2" { + handler.capable = Arc::new(AtomicBool::new(false)); + } + Some(WipDynamicItem { + object: Object { + name: item_reference.into(), + description: None, + interfaces: vec!["test.dynamic/v1".into()], + r#ref: Some(format!("objective:{item_reference}")), + validator: Some(vec![self.revision.load(Ordering::SeqCst) as u8]), + }, + handler: Arc::new(handler), + }) + } + } + + struct ContextualContributionResolver(Arc); + + impl WipDynamicOperationResolver for ContextualContributionResolver { + fn handler(&self, _item_reference: &str) -> Arc { + self.0.clone() + } + } + + fn contextual_dynamic_registry( + owner: Arc, + contributor: Arc, + revision: Arc, + contextual_collection: bool, + ) -> WipMountRegistry { + let mut registry = dynamic_registry(Arc::clone(&revision), Arc::clone(&owner.calls)); + let mut mounted = registry.dynamic_mounts.pop().unwrap(); + mounted.descriptor.operations = vec![contributed_operation("read")]; + mounted.interface_validator = Some(descriptor_validator(&mounted.descriptor)); + mounted.resolver = Arc::new(ContextualItemResolver { + owner, + revision, + contextual_family: !contextual_collection, + }); + if contextual_collection { + let collection = registry.mounts.get_mut("/objectives").unwrap(); + collection.projection.handler = Arc::new(ContextualHandler::new(true, true)); + collection.operation_handlers = + operation_handlers(&collection.projection, "objective:collection"); + } + let mut descriptor = mounted.descriptor.clone(); + descriptor.operations = vec![contributed_operation("manage")]; + registry.mount_dynamic(mounted).unwrap(); + registry + .contribute_dynamic_operations(WipDynamicOperationContribution { + collection_route: "/objectives".into(), + contributor: "management".into(), + interface: "test.dynamic/v1".into(), + descriptor, + resolver: Arc::new(ContextualContributionResolver(contributor)), + }) + .unwrap(); + registry + } + + #[tokio::test] + async fn contextual_dynamic_filters_by_path_and_rechecks_permission_and_object_revision() { + // Both ways to opt in a family must block unqualified descriptor fetches. + for contextual_collection in [false, true] { + let owner = Arc::new(ContextualHandler::new(true, true)); + let contributor = Arc::new(ContextualHandler::new(true, true)); + let revision = Arc::new(AtomicUsize::new(1)); + let host = WipHost::new(contextual_dynamic_registry( + Arc::clone(&owner), + Arc::clone(&contributor), + Arc::clone(&revision), + contextual_collection, + )); + let initial = host.projection("/objectives/O-1").unwrap(); + let other = host.projection("/objectives/O-2").unwrap(); + assert_ne!(initial.interface, other.interface); + assert_eq!( + operation_names(&host.fetch_interface(&initial.interface).unwrap().descriptor), + ["manage", "read"] + ); + assert_eq!( + operation_names(&host.fetch_interface(&other.interface).unwrap().descriptor), + ["manage"] + ); + assert!(host.fetch_interface("test.dynamic/v1").is_err()); + assert!( + host.fetch_interface(&contextual_reference("wrong.base/v1", "/objectives/O-1")) + .is_err() + ); + assert!( + host.fetch_interface(&contextual_reference( + "test.dynamic/v1", + "/objectives/missing" + )) + .is_err() + ); + assert!(host.fetch_interface("test.dynamic/v1/@/ff").is_err()); + assert!(host.fetch_interface("test.dynamic/v1/@/2f0").is_err()); + let mut wrong_path = asset_request(&initial, "read"); + wrong_path.target.path = other.route.clone(); + assert!(matches!( + host.call(wrong_path, direct_wip_context()).await, + Err(WipOperationError::Protocol(ProtocolError { + code: ProtocolErrorCode::InterfaceMismatch, + .. + })) + )); + contributor.permitted.store(false, Ordering::SeqCst); + assert_eq!( + operation_names(&host.fetch_interface(&initial.interface).unwrap().descriptor), + ["read"] + ); + assert!(matches!( + host.call(asset_request(&initial, "manage"), direct_wip_context()) + .await, + Err(WipOperationError::Protocol(ProtocolError { + code: ProtocolErrorCode::InterfaceValidatorMismatch, + .. + })) + )); + let current = host.projection("/objectives/O-1").unwrap(); + assert!(matches!( + host.call(asset_request(¤t, "manage"), direct_wip_context()) + .await, + Err(WipOperationError::Protocol(ProtocolError { + code: ProtocolErrorCode::OperationNotFound, + .. + })) + )); + revision.store(2, Ordering::SeqCst); + assert!(matches!( + host.call(asset_request(¤t, "read"), direct_wip_context()) + .await, + Err(WipOperationError::Protocol(ProtocolError { + code: ProtocolErrorCode::ValidatorMismatch, + .. + })) + )); + let current = host.projection("/objectives/O-1").unwrap(); + host.call(asset_request(¤t, "read"), direct_wip_context()) + .await + .unwrap_or_else(|_| panic!("fresh available dynamic operation must run")); + owner.visible.store(false, Ordering::SeqCst); + assert!(host.observe("/objectives/O-1", 0).is_err()); + assert!(host.fetch_interface(&initial.interface).is_err()); + assert!(host.fetch_interface(&other.interface).is_err()); + assert!(matches!( + host.call(asset_request(¤t, "read"), direct_wip_context()) + .await, + Err(WipOperationError::Protocol(ProtocolError { + code: ProtocolErrorCode::NotFound, + .. + })) + )); + assert_eq!(owner.calls.load(Ordering::SeqCst), 1); + assert_eq!(contributor.calls.load(Ordering::SeqCst), 0); + } + } + + #[tokio::test] + async fn contextual_runtime_cached_descriptor_does_not_authorize_removed_operations() { + let owner = Arc::new(ContextualHandler::new(true, true)); + let contributor = Arc::new(ContextualHandler::new(true, true)); + let runtime = WipRuntime::new( + WipHost::new(contextual_static_registry( + Arc::clone(&owner), + Arc::clone(&contributor), + )), + SecurityContext::new("worker-a"), + 0, + ) + .unwrap(); + runtime + .discover("/assets/A-1".into(), 0, false) + .await + .unwrap(); + let interface = runtime.host.projection("/assets/A-1").unwrap().interface; + runtime.inspect(interface.clone(), false).await.unwrap(); + contributor.permitted.store(false, Ordering::SeqCst); + // The Client still knows the old descriptor; the Host must independently + // check its current filtered descriptor rather than trust Known Space. + let error = runtime + .call( + "/assets/A-1".into(), + interface.clone(), + "read".into(), + json!({}), + ToolExecutionContext::direct(), + ) + .await + .unwrap_err(); + assert!(error.to_string().contains("InterfaceValidatorMismatch")); + assert!( + runtime + .call( + "/assets/A-1".into(), + interface.clone(), + "manage".into(), + json!({}), + ToolExecutionContext::direct(), + ) + .await + .is_err() + ); + assert_eq!(owner.calls.load(Ordering::SeqCst), 0); + assert_eq!(contributor.calls.load(Ordering::SeqCst), 0); + runtime.inspect(interface.clone(), true).await.unwrap(); + runtime + .call( + "/assets/A-1".into(), + interface.clone(), + "read".into(), + json!({}), + ToolExecutionContext::direct(), + ) + .await + .unwrap(); + owner.visible.store(false, Ordering::SeqCst); + assert!(runtime.host.fetch_interface(&interface).is_err()); + let error = runtime.inspect(interface.clone(), true).await.unwrap_err(); + assert!(error.to_string().contains("InterfaceNotFound")); + { + let state = runtime.state.lock().unwrap(); + let observed = state.client.interface(&state.session, &interface).unwrap(); + assert!(matches!(observed.state, ObservationState::Error(_))); + } + assert!( + runtime + .call( + "/assets/A-1".into(), + interface, + "read".into(), + json!({}), + ToolExecutionContext::direct(), + ) + .await + .is_err() + ); + assert_eq!(owner.calls.load(Ordering::SeqCst), 1); + } + + struct CurrentObjectValidatorHandler { + inner: Arc, + revision: Arc, + } + + #[async_trait] + impl WipOperationHandler for CurrentObjectValidatorHandler { + fn contextual_interface(&self) -> bool { + true + } + + fn object_validator(&self) -> Option> { + Some(self.revision.load(Ordering::SeqCst).to_be_bytes().to_vec()) + } + + async fn call( + &self, + operation: &str, + arguments: &BTreeMap, + context: WipCallContext, + ) -> Result { + self.inner.call(operation, arguments, context).await + } + } + + #[tokio::test] + async fn static_object_validator_hook_rejects_current_owner_state_changes() { + let owner = Arc::new(ContextualHandler::new(true, true)); + let revision = Arc::new(AtomicUsize::new(1)); + let mut registry = WipMountRegistry::new(); + registry.allocate_namespace("asset", "assets").unwrap(); + registry + .mount(contribution_projection(Arc::new( + CurrentObjectValidatorHandler { + inner: Arc::clone(&owner), + revision: Arc::clone(&revision), + }, + ))) + .unwrap(); + // Contributors cannot override Object ownership or its current validator. + let mut descriptor = registry.mounts["/assets/A-1"].projection.descriptor.clone(); + descriptor.operations = vec![contributed_operation("manage")]; + registry + .contribute_operations(WipOperationContribution { + route: "/assets/A-1".into(), + contributor: "management".into(), + interface: "test.asset/item/v1".into(), + descriptor, + handler: Arc::new(CurrentObjectValidatorHandler { + inner: Arc::new(ContextualHandler::new(true, true)), + revision: Arc::new(AtomicUsize::new(99)), + }), + }) + .unwrap(); + let runtime = + WipRuntime::new(WipHost::new(registry), SecurityContext::new("worker-a"), 0).unwrap(); + let interface = observed_contextual_interface(&runtime, "/assets/A-1").await; + let initial = runtime.host.projection("/assets/A-1").unwrap(); + assert_eq!( + initial.object.validator, + Some(1usize.to_be_bytes().to_vec()) + ); + revision.store(2, Ordering::SeqCst); + assert_eq!( + runtime.host.object_at("/assets/A-1").unwrap().validator, + Some(2usize.to_be_bytes().to_vec()) + ); + assert!(matches!( + runtime + .host + .call(asset_request(&initial, "read"), direct_wip_context()) + .await, + Err(WipOperationError::Protocol(ProtocolError { + code: ProtocolErrorCode::ValidatorMismatch, + .. + })) + )); + let error = runtime + .call( + "/assets/A-1".into(), + interface.clone(), + "read".into(), + json!({}), + ToolExecutionContext::direct(), + ) + .await + .unwrap_err(); + assert!(error.to_string().contains("ValidatorMismatch")); + assert_eq!(owner.calls.load(Ordering::SeqCst), 0); + assert_eq!( + observed_contextual_interface(&runtime, "/assets/A-1").await, + interface + ); + transport_call_json(&runtime, "/assets/A-1", &interface, "read", json!({})).await; + assert_eq!(owner.calls.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn dynamic_item_read_is_independent_of_collection_visibility_and_list_permission() { + let owner = Arc::new(ContextualHandler::new(true, true)); + let mut registry = contextual_dynamic_registry( + Arc::clone(&owner), + Arc::new(ContextualHandler::new(false, true)), + Arc::new(AtomicUsize::new(1)), + true, + ); + let denied_collection = Arc::new(ContextualHandler::new(false, true)); + denied_collection.visible.store(false, Ordering::SeqCst); + let collection = registry.mounts.get_mut("/objectives").unwrap(); + collection.projection.handler = denied_collection; + collection.operation_handlers = + operation_handlers(&collection.projection, "objective:collection"); + let runtime = + WipRuntime::new(WipHost::new(registry), SecurityContext::new("worker-a"), 0).unwrap(); + assert!(runtime.host.projection("/objectives").is_none()); + let interface = observed_contextual_interface(&runtime, "/objectives/O-1").await; + transport_call_json(&runtime, "/objectives/O-1", &interface, "read", json!({})).await; + owner.visible.store(false, Ordering::SeqCst); + assert!(runtime.host.fetch_interface(&interface).is_err()); + assert!( + runtime + .call( + "/objectives/O-1".into(), + interface, + "read".into(), + json!({}), + ToolExecutionContext::direct() + ) + .await + .is_err() + ); + assert_eq!(owner.calls.load(Ordering::SeqCst), 1); + } + + fn transport_output_json(output: ToolOutput) -> Json { + serde_json::from_str(output.content.as_deref().expect("transport JSON output")).unwrap() + } + + async fn observed_contextual_interface(runtime: &WipRuntime, path: &str) -> String { + let observed = transport_output_json(runtime.discover(path.into(), 0, true).await.unwrap()); + let object = observed["known_space"] + .as_array() + .unwrap() + .iter() + .find(|item| item["path"] == path) + .unwrap(); + assert_eq!(object["state"], "fresh"); + let interface = object["object"]["interfaces"][0] + .as_str() + .unwrap() + .to_string(); + assert!(interface.contains("/@/")); + let inspected = + transport_output_json(runtime.inspect(interface.clone(), true).await.unwrap()); + assert_eq!(inspected["state"], "fresh"); + interface + } + + async fn transport_call_json( + runtime: &WipRuntime, + path: &str, + interface: &str, + operation: &str, + arguments: Json, + ) -> Json { + transport_output_json( + runtime + .call( + path.into(), + interface.into(), + operation.into(), + arguments, + ToolExecutionContext::direct(), + ) + .await + .unwrap(), + ) + } + + fn catalog_transport_runtime( + client: Arc, + manage: bool, + permissions: Option, + ) -> Arc { + use crate::feature::FeatureRegistryBuilder; + use crate::feature::builtin::manage_workdir::{ + ManageWorkdirFeature, wip::mount_workspace_workdir_wip, + }; + let feature = ManageWorkdirFeature::new(client); + let mut engine = Engine::<_, Mutable, ()>::new_annotated(DummyClient); + let report = FeatureRegistryBuilder::new() + .with_module(feature.clone()) + .install_into_engine(&mut engine, &mut crate::hook::HookRegistryBuilder::new()); + assert!(!report.has_errors()); + assert_eq!(report.installed_tool_names().len(), 5); + let mut registry = WipMountRegistry::new(); + mount_workspace_workdir_wip(&mut registry, &feature, true, manage, permissions.clone()) + .unwrap(); + let runtime = + install_wip_mode_with_mounts(&mut engine, permissions, "worker-a".into(), registry) + .unwrap(); + for name in [ + "WorkdirList", + "WorkdirCreate", + "WorkdirAttach", + "WorkdirDetach", + "WorkdirDelete", + ] { + assert!(runtime.host.projection(&format!("/tools/{name}")).is_none()); + } + assert!(runtime.host.object_at("/features/manage-workdir").is_none()); + runtime + } + + #[tokio::test] + async fn catalog_native_transport_pages_all_workdirs_beyond_legacy_200_cap() { + use crate::feature::builtin::manage_workdir::wip::tests::CatalogClient; + let client = Arc::new(CatalogClient::default()); + client.state.lock().unwrap().workdirs = (0..205).map(|n| json!({ + "working_directory_id":format!("wd-{n:03}"), "source":{"kind":"repository", "repository_key":"registered-key"}, "status":"active" + })).collect(); + let runtime = catalog_transport_runtime(client.clone(), false, None); + let interface = observed_contextual_interface(&runtime, "/workdirs").await; + let mut cursor = Json::Null; + let mut ids = BTreeSet::new(); + let mut pages = 0; + loop { + let page = transport_call_json( + &runtime, + "/workdirs", + &interface, + "list", + if cursor.is_null() { + json!({"limit":100}) + } else { + json!({"limit":100, "cursor":cursor}) + }, + ) + .await; + pages += 1; + let items = page["items"].as_array().unwrap(); + assert!(items.len() <= 100); + for item in items { + assert!(ids.insert(item["working_directory_id"].as_str().unwrap().to_owned())); + } + if !page["has_more"].as_bool().unwrap() { + assert!(page["next_cursor"].is_null()); + break; + } + cursor = page["next_cursor"].clone(); + assert!(pages < 4, "cursor must progress"); + } + assert_eq!(pages, 3); + assert_eq!(ids.len(), 205); + assert!(ids.contains("wd-000")); + let item_interface = observed_contextual_interface(&runtime, "/workdirs/wd-000").await; + let read = transport_call_json( + &runtime, + "/workdirs/wd-000", + &item_interface, + "read", + json!({}), + ) + .await; + assert_eq!(read["working_directory_id"], "wd-000"); + assert!( + client + .state + .lock() + .unwrap() + .requests + .iter() + .all(|r| !r.path.ends_with("/working-directories")), + "WIP must not use a capped legacy inventory" + ); + + let permissions = Some(ToolPermissionConfig { + default_action: ToolPermissionAction::Deny, + rules: vec![ + manifest::ToolPermissionRule { + tool: "WorkdirList".into(), + pattern: "*".into(), + action: ToolPermissionAction::Allow, + }, + manifest::ToolPermissionRule { + tool: "WorkdirRead".into(), + pattern: "*wd-204*".into(), + action: ToolPermissionAction::Allow, + }, + ], + }); + let runtime = catalog_transport_runtime(client, false, permissions); + let interface = observed_contextual_interface(&runtime, "/workdirs").await; + let first = transport_call_json( + &runtime, + "/workdirs", + &interface, + "list", + json!({"limit":100}), + ) + .await; + assert_eq!(first["empty"], true); + assert_eq!( + first["has_more"], true, + "empty filtered page is not terminal" + ); + let second = transport_call_json( + &runtime, + "/workdirs", + &interface, + "list", + json!({"limit":100, "cursor":first["next_cursor"]}), + ) + .await; + assert_eq!(second["has_more"], true); + let last = transport_call_json( + &runtime, + "/workdirs", + &interface, + "list", + json!({"limit":100, "cursor":second["next_cursor"]}), + ) + .await; + assert_eq!(last["items"][0]["working_directory_id"], "wd-204"); + assert_eq!(last["has_more"], false); + } + + #[tokio::test] + async fn catalog_native_transport_invalidates_attachment_change_beyond_first_page() { + use crate::feature::builtin::manage_workdir::wip::tests::CatalogClient; + for capability_change in [false, true] { + let client = Arc::new(CatalogClient::default()); + client.state.lock().unwrap().attachments = (0..65).map(|n| json!({ + "connection_id":format!("lifetime-{n:03}"), "alias":format!("alias-{n:03}"), "working_directory_id":"wd", "capabilities":{"bits":25} + })).collect(); + let first_fifty = client.state.lock().unwrap().attachments[..50].to_vec(); + let runtime = catalog_transport_runtime(client.clone(), true, None); + let interface = observed_contextual_interface(&runtime, "/workdir-attachments").await; + let old_validator = runtime + .host + .projection("/workdir-attachments") + .unwrap() + .object + .validator; + if capability_change { + client.state.lock().unwrap().attachments[64]["capabilities"] = json!({"bits":63}); + } else { + client.state.lock().unwrap().attachments[64]["connection_id"] = + json!("new-lifetime"); + } + assert_eq!(client.state.lock().unwrap().attachments[..50], first_fifty); + assert_ne!( + runtime + .host + .projection("/workdir-attachments") + .unwrap() + .object + .validator, + old_validator + ); + let error = runtime + .call( + "/workdir-attachments".into(), + interface.clone(), + "list".into(), + json!({"cursor":"offset:50", "limit":100}), + ToolExecutionContext::direct(), + ) + .await + .unwrap_err(); + assert!(error.to_string().contains("ValidatorMismatch"), "{error}"); + let interface = observed_contextual_interface(&runtime, "/workdir-attachments").await; + let page = transport_call_json( + &runtime, + "/workdir-attachments", + &interface, + "list", + json!({"cursor":"offset:50"}), + ) + .await; + assert_eq!(page["items"].as_array().unwrap().len(), 15); + if capability_change { + assert_eq!(page["items"][14]["capabilities"]["bits"], 63); + } else { + assert_eq!(page["items"][14]["connection_id"], "new-lifetime"); + } + } + } + + #[tokio::test] + async fn catalog_native_transport_discovers_creates_attaches_reads_and_detaches() { + use crate::feature::builtin::manage_workdir::wip::tests::CatalogClient; + use crate::worker::WorkspaceRequestMethod; + let client = Arc::new(CatalogClient::default()); + let runtime = catalog_transport_runtime(client.clone(), true, None); + let root = transport_output_json(runtime.discover("/".into(), 1, false).await.unwrap()); + let paths = root["known_space"] + .as_array() + .unwrap() + .iter() + .map(|item| item["path"].as_str().unwrap()) + .collect::>(); + for path in ["/repositories", "/workdirs", "/workdir-attachments"] { + assert!(paths.contains(&path)); + } + assert!(!paths.iter().any(|path| path.starts_with("/tools/Workdir") + || path.starts_with("/features/manage-workdir"))); + let repositories_interface = observed_contextual_interface(&runtime, "/repositories").await; + let listed = transport_call_json( + &runtime, + "/repositories", + &repositories_interface, + "list", + json!({}), + ) + .await; + let repository = &listed["items"][0]; + assert_eq!(repository["repository_key"], "registered-key"); + assert!(repository["default_selector"].is_null()); + assert!(!listed.to_string().contains("SECRET")); + assert!(!listed.to_string().contains("/secret/host")); + let repository_path = repository["path"].as_str().unwrap(); + let repository_interface = observed_contextual_interface(&runtime, repository_path).await; + let read = transport_call_json( + &runtime, + repository_path, + &repository_interface, + "read", + json!({}), + ) + .await; + assert_eq!(read["repository_key"], repository["repository_key"]); + let workdirs_interface = observed_contextual_interface(&runtime, "/workdirs").await; + let created = transport_call_json( + &runtime, + "/workdirs", + &workdirs_interface, + "create", + json!({"repository_key":read["repository_key"]}), + ) + .await; + assert_eq!(created["item"]["path"], "/workdirs/wd-created"); + { + let state = client.state.lock().unwrap(); + assert_eq!(state.workdirs.len(), 1); + assert!(state.attachments.is_empty(), "creation must not attach"); + assert!( + !state + .requests + .iter() + .any(|request| request.method == WorkspaceRequestMethod::Post + && request.path.ends_with("workdir-attachments")) + ); + } + let workdir_path = created["item"]["path"].as_str().unwrap(); + let workdir_interface = observed_contextual_interface(&runtime, workdir_path).await; + let attached = transport_call_json( + &runtime, + workdir_path, + &workdir_interface, + "attach", + json!({"alias":"checkout"}), + ) + .await; + assert_eq!(attached["attachments_path"], "/workdir-attachments"); + let attachments_interface = + observed_contextual_interface(&runtime, "/workdir-attachments").await; + let listed = transport_call_json( + &runtime, + "/workdir-attachments", + &attachments_interface, + "list", + json!({}), + ) + .await; + let attachment = &listed["items"][0]; + assert_eq!(attachment["alias"], "checkout"); + assert_eq!(attachment["workdir_path"], workdir_path); + let attachment_path = attachment["path"].as_str().unwrap(); + let attachment_interface = observed_contextual_interface(&runtime, attachment_path).await; + let read = transport_call_json( + &runtime, + attachment_path, + &attachment_interface, + "read", + json!({}), + ) + .await; + assert_eq!(read["connection_id"], attachment["connection_id"]); + transport_call_json( + &runtime, + attachment_path, + &attachment_interface, + "detach", + json!({}), + ) + .await; + let attachments_interface = + observed_contextual_interface(&runtime, "/workdir-attachments").await; + let listed = transport_call_json( + &runtime, + "/workdir-attachments", + &attachments_interface, + "list", + json!({}), + ) + .await; + assert_eq!(listed["empty"], true); + assert!(client.state.lock().unwrap().attachments.is_empty()); + assert_eq!( + client.state.lock().unwrap().workdirs.len(), + 1, + "detach must preserve persistent Workdir" + ); + assert!(runtime.host.projection(attachment_path).is_none()); + assert!(runtime.host.fetch_interface(&attachment_interface).is_err()); + let state = client.state.lock().unwrap(); + let writes = state + .requests + .iter() + .filter(|request| request.method != WorkspaceRequestMethod::Get) + .collect::>(); + assert_eq!(writes.len(), 3); + assert!( + writes[2] + .path + .contains("expected_connection_id=connection-1") + ); + drop(state); + // A dirty provider result is a retained lifecycle outcome, not transport + // uncertainty or implicit destructive success. + client.state.lock().unwrap().workdirs[0]["cleanliness"] = json!("dirty"); + let workdir_interface = observed_contextual_interface(&runtime, workdir_path).await; + let read = transport_call_json( + &runtime, + workdir_path, + &workdir_interface, + "read", + json!({}), + ) + .await; + assert_eq!(read["cleanliness"], "dirty"); + let retained = transport_call_json( + &runtime, + workdir_path, + &workdir_interface, + "delete", + json!({"reason":"dirty checkout retained"}), + ) + .await; + assert_eq!(retained["disposition"], "retained"); + assert_eq!(client.state.lock().unwrap().workdirs.len(), 1); + assert_eq!( + runtime.audit().last().unwrap().outcome, + WipAuditOutcome::Success + ); + } + + #[tokio::test] + async fn catalog_native_transport_read_only_and_read_without_list_permissions() { + use crate::feature::builtin::manage_workdir::wip::tests::CatalogClient; + use crate::worker::WorkspaceRequestMethod; + let client = Arc::new(CatalogClient::default()); + client.state.lock().unwrap().workdirs.push(json!({ + "working_directory_id":"wd-read", "status":"active", "cleanliness":"clean", + "source":{"kind":"repository", "repository_key":"registered-key"}, + })); + let runtime = catalog_transport_runtime(client.clone(), false, None); + runtime.discover("/".into(), 1, false).await.unwrap(); + let collection_interface = observed_contextual_interface(&runtime, "/workdirs").await; + assert_eq!( + operation_names( + &runtime + .host + .fetch_interface(&collection_interface) + .unwrap() + .descriptor + ), + ["list"] + ); + let listed = transport_call_json( + &runtime, + "/workdirs", + &collection_interface, + "list", + json!({}), + ) + .await; + let path = listed["items"][0]["path"].as_str().unwrap(); + let interface = observed_contextual_interface(&runtime, path).await; + assert_eq!( + operation_names(&runtime.host.fetch_interface(&interface).unwrap().descriptor), + ["read"] + ); + transport_call_json(&runtime, path, &interface, "read", json!({})).await; + for (target, reference, operation, input) in [ + ( + "/workdirs", + &collection_interface, + "create", + json!({"repository_key":"registered-key"}), + ), + (path, &interface, "attach", json!({"alias":"checkout"})), + (path, &interface, "delete", json!({"reason":"done"})), + ] { + assert!( + runtime + .call( + target.into(), + reference.clone(), + operation.into(), + input, + ToolExecutionContext::direct() + ) + .await + .is_err() + ); + } + assert!( + client + .state + .lock() + .unwrap() + .requests + .iter() + .all(|request| request.method == WorkspaceRequestMethod::Get) + ); + let read_only = ToolPermissionConfig { + default_action: ToolPermissionAction::Deny, + rules: ["RepositoryRead", "WorkdirRead"] + .iter() + .map(|name| manifest::ToolPermissionRule { + tool: (*name).into(), + pattern: "*".into(), + action: ToolPermissionAction::Allow, + }) + .collect(), + }; + let runtime = catalog_transport_runtime(client.clone(), false, Some(read_only)); + for path in ["/repositories/registered-key", "/workdirs/wd-read"] { + let interface = observed_contextual_interface(&runtime, path).await; + assert_eq!( + operation_names(&runtime.host.fetch_interface(&interface).unwrap().descriptor), + ["read"] + ); + transport_call_json(&runtime, path, &interface, "read", json!({})).await; + } + client.state.lock().unwrap().denied = true; + let interface = contextual_reference("yoi.workdir/item/v1", "/workdirs/wd-read"); + assert!(runtime.host.fetch_interface(&interface).is_err()); + assert!( + runtime + .call( + "/workdirs/wd-read".into(), + interface, + "read".into(), + json!({}), + ToolExecutionContext::direct() + ) + .await + .is_err() + ); + assert!( + client + .state + .lock() + .unwrap() + .requests + .iter() + .all(|request| request.method == WorkspaceRequestMethod::Get) + ); + } + + #[derive(Debug)] + struct UncertainCatalogClient { + catalog: Arc, + } + + impl crate::worker::WorkspaceClient for UncertainCatalogClient { + fn workspace_id(&self) -> Option<&str> { + Some("test-workspace") + } + fn kind(&self) -> &str { + "uncertain-catalog-test" + } + fn is_available(&self) -> bool { + true + } + fn execute( + &self, + request: crate::worker::WorkspaceRequest, + ) -> Result { + use crate::worker::{WorkspaceClientError, WorkspaceRequestMethod}; + if request.method == WorkspaceRequestMethod::Post + && request.path.ends_with("/working-directories") + { + self.catalog.state.lock().unwrap().requests.push(request); + return Err(WorkspaceClientError::Unavailable( + "SECRET /host/path uncertain write".into(), + )); + } + self.catalog.execute(request) + } + } + + #[tokio::test] + async fn catalog_native_transport_unknown_mutation_is_terminal_and_not_retried() { + use crate::feature::builtin::manage_workdir::wip::tests::CatalogClient; + use crate::worker::WorkspaceRequestMethod; + let catalog = Arc::new(CatalogClient::default()); + let runtime = catalog_transport_runtime( + Arc::new(UncertainCatalogClient { + catalog: catalog.clone(), + }), + true, + None, + ); + let interface = observed_contextual_interface(&runtime, "/workdirs").await; + let error = runtime + .call( + "/workdirs".into(), + interface, + "create".into(), + json!({"repository_key":"registered-key"}), + ToolExecutionContext::direct(), + ) + .await + .unwrap_err(); + let error = error.to_string(); + assert!(error.contains("outcome unknown")); + assert!(!error.contains("SECRET")); + assert!(!error.contains("/host/path")); + assert_eq!( + runtime.audit().last().unwrap().outcome, + WipAuditOutcome::OutcomeUnknown + ); + let state = runtime.state.lock().unwrap(); + assert!(matches!( + state + .client + .call_history(&state.session) + .unwrap() + .back() + .unwrap() + .outcome, + CallOutcome::Unknown { .. } + )); + let state = catalog.state.lock().unwrap(); + assert_eq!( + state + .requests + .iter() + .filter(|request| request.method == WorkspaceRequestMethod::Post) + .count(), + 1 + ); + assert!(state.workdirs.is_empty()); + assert!(state.attachments.is_empty()); + } + struct NativeTyped; #[async_trait] @@ -2338,6 +5122,7 @@ mod tests { #[tokio::test] async fn native_projection_preserves_descriptor_and_decodes_typed_arguments_end_to_end() { let mut registry = WipMountRegistry::new(); + registry.allocate_namespace("native", "native").unwrap(); registry.mount(native_typed_projection()).unwrap(); let runtime = WipRuntime::new(WipHost::new(registry), SecurityContext::new("worker-a"), 0).unwrap(); @@ -2600,10 +5385,26 @@ mod tests { } } + struct DynamicContributionResolver { + calls: Arc, + allowed: Arc, + } + + impl WipDynamicOperationResolver for DynamicContributionResolver { + fn handler(&self, _item_reference: &str) -> Arc { + Arc::new(ContributionHandler { + calls: Arc::clone(&self.calls), + allowed: Arc::clone(&self.allowed), + }) + } + } + fn dynamic_registry(revision: Arc, calls: Arc) -> WipMountRegistry { let mut registry = WipMountRegistry::new(); - let feature = registry.allocate_feature_route("objective").unwrap(); - let collection_route = feature.child("objectives").unwrap(); + let namespace = registry + .allocate_namespace("objective", "objectives") + .unwrap(); + let collection_route = namespace.root().to_string(); let mut collection = compatibility_projection( meta("Dynamic"), Arc::new(EchoTool { @@ -2616,8 +5417,8 @@ mod tests { collection.capability = "objective:collection".into(); collection.kind = WipProjectionKind::Native; collection.object.name = "objectives".into(); - collection.object.interfaces = vec!["test.dynamic/v1".into()]; - collection.interface = "test.dynamic/v1".into(); + collection.object.interfaces = vec!["test.dynamic.collection/v1".into()]; + collection.interface = "test.dynamic.collection/v1".into(); collection.handler = Arc::new(DynamicHandler { calls: Arc::clone(&calls), }); @@ -2642,17 +5443,9 @@ mod tests { let revision = Arc::new(AtomicUsize::new(1)); let calls = Arc::new(AtomicUsize::new(0)); let host = WipHost::new(dynamic_registry(Arc::clone(&revision), Arc::clone(&calls))); - let initial = host - .projection("/features/objective/objectives/O-3") - .unwrap(); - assert!( - host.projection("/features/objective/objectives/O-3/other") - .is_none() - ); - assert!( - host.projection("/features/objective/objectives/T-3") - .is_none() - ); + let initial = host.projection("/objectives/O-3").unwrap(); + assert!(host.projection("/objectives/O-3/other").is_none()); + assert!(host.projection("/objectives/T-3").is_none()); assert!(host.projection("/hidden/objectives/O-3").is_none()); revision.store(2, Ordering::SeqCst); @@ -2660,7 +5453,7 @@ mod tests { .call( CallOperationRequest { target: wip_protocol::Target { - path: "/features/objective/objectives/O-3".into(), + path: "/objectives/O-3".into(), validator: initial.object.validator, }, interface: wip_protocol::InterfaceTarget { @@ -2689,6 +5482,127 @@ mod tests { assert_eq!(calls.load(Ordering::SeqCst), 0); } + #[test] + fn dynamic_registration_rejects_foreign_owners_and_operation_collisions_atomically() { + let revision = Arc::new(AtomicUsize::new(1)); + let calls = Arc::new(AtomicUsize::new(0)); + let mut registry = dynamic_registry(Arc::clone(&revision), Arc::clone(&calls)); + let mounted = registry.dynamic_mounts.pop().unwrap(); + let foreign = WipDynamicMount { + collection_route: mounted.collection_route.clone(), + capability: "unrelated:item".into(), + interface: mounted.interface.clone(), + descriptor: mounted.descriptor.clone(), + interface_validator: mounted.interface_validator.clone(), + resolver: Arc::new(DynamicResolver { + revision, + calls: Arc::clone(&calls), + }), + }; + assert!(matches!( + registry.mount_dynamic(foreign), + Err(WipMountError::InvalidProjection { .. }) + )); + assert!(registry.dynamic_mounts.is_empty()); + registry.mount_dynamic(mounted).unwrap(); + let original = registry.dynamic_mounts[0].descriptor.clone(); + let contribution = |descriptor| WipDynamicOperationContribution { + collection_route: "/objectives".into(), + contributor: "management".into(), + interface: "test.dynamic/v1".into(), + descriptor, + resolver: Arc::new(DynamicContributionResolver { + calls: Arc::clone(&calls), + allowed: Arc::new(AtomicBool::new(true)), + }), + }; + assert!(matches!( + registry.contribute_dynamic_operations(contribution(original.clone())), + Err(WipMountError::OperationCollision { .. }) + )); + let mut inconsistent = original.clone(); + inconsistent.operations = vec![contributed_operation("manage")]; + inconsistent.documentation = documentation("different shape"); + assert!(matches!( + registry.contribute_dynamic_operations(contribution(inconsistent)), + Err(WipMountError::InterfaceCollision { .. }) + )); + assert_eq!(registry.dynamic_mounts[0].descriptor, original); + assert!(registry.dynamic_operation_contributions.is_empty()); + let mut valid = original; + valid.operations = vec![contributed_operation("manage")]; + registry + .contribute_dynamic_operations(contribution(valid.clone())) + .unwrap(); + assert!(matches!( + registry.contribute_dynamic_operations(contribution(valid)), + Err(WipMountError::OperationCollision { .. }) + )); + assert_eq!( + registry.dynamic_operation_contributions["/objectives"].len(), + 1 + ); + } + + #[tokio::test] + async fn dynamic_object_resolver_accepts_disjoint_feature_operations() { + let revision = Arc::new(AtomicUsize::new(1)); + let base_calls = Arc::new(AtomicUsize::new(0)); + let contribution_calls = Arc::new(AtomicUsize::new(0)); + let allowed = Arc::new(AtomicBool::new(true)); + let mut registry = dynamic_registry(revision, base_calls); + let mounted = registry + .dynamic_mounts + .first() + .expect("dynamic object family is mounted"); + let mut descriptor = mounted.descriptor.clone(); + descriptor.operations = vec![contributed_operation("manage")]; + registry + .contribute_dynamic_operations(WipDynamicOperationContribution { + collection_route: "/objectives".into(), + contributor: "objective-management".into(), + interface: "test.dynamic/v1".into(), + descriptor, + resolver: Arc::new(DynamicContributionResolver { + calls: Arc::clone(&contribution_calls), + allowed, + }), + }) + .unwrap(); + let host = WipHost::new(registry); + let projection = host.projection("/objectives/O-7").unwrap(); + assert_eq!( + projection + .descriptor + .operations + .iter() + .map(|operation| operation.name.as_str()) + .collect::>(), + ["call", "manage"] + ); + host.call( + CallOperationRequest { + target: wip_protocol::Target { + path: "/objectives/O-7".into(), + validator: projection.object.validator, + }, + interface: wip_protocol::InterfaceTarget { + reference: projection.interface, + validator: projection.interface_validator, + }, + operation: "manage".into(), + arguments: BTreeMap::new(), + }, + WipCallContext { + execution: ToolExecutionContext::direct(), + security_context: "worker-a".into(), + }, + ) + .await + .unwrap_or_else(|_| panic!("dynamic contributed operation should run")); + assert_eq!(contribution_calls.load(Ordering::SeqCst), 1); + } + #[test] fn native_capability_claim_hides_only_claimed_compatibility_route() { let calls = Arc::new(AtomicUsize::new(0)); @@ -2699,18 +5613,13 @@ mod tests { let revision = Arc::new(AtomicUsize::new(1)); let mut registry = dynamic_registry(revision, calls); registry - .replace_compatibility_tools("/features/objective/objectives", ["Echo"]) + .replace_compatibility_tools("/objectives", ["Echo"]) .unwrap(); let runtime = install_wip_mode_with_mounts(&mut engine, None, "worker-a".into(), registry).unwrap(); assert!(runtime.host.projection("/tools/Echo").is_none()); assert!(runtime.host.projection("/tools/Other").is_some()); - assert!( - runtime - .host - .projection("/features/objective/objectives") - .is_some() - ); + assert!(runtime.host.projection("/objectives").is_some()); } #[tokio::test] @@ -2752,7 +5661,9 @@ mod tests { authoring_engine.register_tool(definition); } let mut authoring_registry = WipMountRegistry::new(); - let route = authoring_registry.allocate_feature_route("ticket").unwrap(); + let route = authoring_registry + .allocate_namespace("ticket", "tickets") + .unwrap(); crate::feature::builtin::ticket::mount_workspace_http_ticket_wip( &mut authoring_registry, authoring_client.clone(), @@ -2771,14 +5682,15 @@ mod tests { let discovered = authoring.discover("/".into(), 3, false).await.unwrap(); let discovered = discovered.content.unwrap(); - assert!(discovered.contains("/features/ticket/tickets")); + assert!(discovered.contains("/tickets")); + assert!(!discovered.contains("/features")); assert!(!discovered.contains("/tools/QueryTicket")); assert!(!discovered.contains("/tools/TicketCreate")); authoring .inspect("yoi.ticket/collection/v1".into(), false) .await .unwrap(); - let collection = "/features/ticket/tickets"; + let collection = "/tickets"; authoring .call( collection.into(), @@ -2799,13 +5711,8 @@ mod tests { ) .await .unwrap(); - assert!( - created - .content - .unwrap() - .contains("/features/ticket/tickets/T-9") - ); - let item = "/features/ticket/tickets/T-9"; + assert!(created.content.unwrap().contains("/tickets/T-9")); + let item = "/tickets/T-9"; authoring.discover(item.into(), 0, false).await.unwrap(); authoring .inspect("yoi.ticket/item/v1".into(), false) @@ -2862,7 +5769,9 @@ mod tests { workflow_engine.register_tool(definition); } let mut workflow_registry = WipMountRegistry::new(); - let route = workflow_registry.allocate_feature_route("ticket").unwrap(); + let route = workflow_registry + .allocate_namespace("ticket", "tickets") + .unwrap(); crate::feature::builtin::ticket::mount_workspace_http_ticket_wip( &mut workflow_registry, workflow_client.clone(), @@ -2963,7 +5872,7 @@ mod tests { } let mut coder_registry = WipMountRegistry::new(); let route = coder_registry - .allocate_feature_route("merge-request") + .allocate_namespace("merge-request", "merge-requests") .unwrap(); crate::feature::builtin::merge_request::mount_workspace_http_merge_request_wip( &mut coder_registry, @@ -2982,14 +5891,14 @@ mod tests { .unwrap(); let discovered = coder.discover("/".into(), 3, false).await.unwrap(); let discovered = discovered.content.unwrap(); - assert!(discovered.contains("/features/merge-request/merge-requests")); + assert!(discovered.contains("/merge-requests")); assert!(!discovered.contains("/tools/OpenMergeRequest")); assert!(!discovered.contains("/tools/ShowMergeRequest")); coder .inspect("yoi.merge-request/collection/v1".into(), false) .await .unwrap(); - let collection = "/features/merge-request/merge-requests"; + let collection = "/merge-requests"; let opened = coder .call( collection.into(), @@ -3085,7 +5994,7 @@ mod tests { } let mut reviewer_registry = WipMountRegistry::new(); let route = reviewer_registry - .allocate_feature_route("merge-request") + .allocate_namespace("merge-request", "merge-requests") .unwrap(); crate::feature::builtin::merge_request::mount_workspace_http_merge_request_wip( &mut reviewer_registry, @@ -3174,7 +6083,7 @@ mod tests { } let mut orchestrator_registry = WipMountRegistry::new(); let route = orchestrator_registry - .allocate_feature_route("merge-request") + .allocate_namespace("merge-request", "merge-requests") .unwrap(); crate::feature::builtin::merge_request::mount_workspace_http_merge_request_wip( &mut orchestrator_registry, @@ -3348,12 +6257,14 @@ mod tests { }], }); let mut registry = WipMountRegistry::new(); - let feature_route = registry.allocate_feature_route("objective").unwrap(); + let namespace_route = registry + .allocate_namespace("objective", "objectives") + .unwrap(); crate::feature::builtin::objective::mount_workspace_http_objective_wip( &mut registry, client.clone(), permissions.clone(), - &feature_route, + &namespace_route, ) .unwrap(); let runtime = @@ -3362,13 +6273,13 @@ mod tests { let discovered = runtime.discover("/".into(), 3, false).await.unwrap(); let discovered = discovered.content.unwrap(); - assert!(discovered.contains("/features/objective/objectives")); + assert!(discovered.contains("/objectives")); assert!(!discovered.contains("/tools/QueryObjective")); runtime .inspect("yoi.objective/collection/v1".into(), false) .await .unwrap(); - let collection_path = "/features/objective/objectives"; + let collection_path = "/objectives"; runtime .call( collection_path.into(), @@ -3401,14 +6312,9 @@ mod tests { ) .await .unwrap(); - assert!( - created - .content - .unwrap() - .contains("/features/objective/objectives/O-3") - ); + assert!(created.content.unwrap().contains("/objectives/O-3")); - let item_path = "/features/objective/objectives/O-3"; + let item_path = "/objectives/O-3"; runtime.discover(item_path.into(), 0, false).await.unwrap(); runtime .inspect("yoi.objective/item/v1".into(), false) @@ -3479,6 +6385,212 @@ mod tests { runtime } + struct UnrelatedCancellation { + cancellations: Arc, + } + + #[async_trait] + impl WipOperationHandler for UnrelatedCancellation { + async fn call( + &self, + _: &str, + _: &BTreeMap, + _: WipCallContext, + ) -> Result { + panic!("unrelated provider must not execute"); + } + + async fn cancel(&self, _: &ToolExecutionContext) -> Result<(), ToolError> { + self.cancellations.fetch_add(1, Ordering::SeqCst); + Err(ToolError::InvalidArgument( + "unknown execution in unrelated provider".into(), + )) + } + } + + struct PendingCancellation { + started: Arc, + cancelled: tokio::sync::Notify, + was_called: AtomicBool, + cancellations: Arc, + } + + #[async_trait] + impl WipOperationHandler for PendingCancellation { + async fn call( + &self, + _: &str, + _: &BTreeMap, + _: WipCallContext, + ) -> Result { + self.was_called.store(true, Ordering::SeqCst); + self.started.notify_one(); + self.cancelled.notified().await; + Err(WipOperationError::Cancelled( + "provider confirmed cancellation".to_string().into(), + )) + } + + async fn cancel(&self, _: &ToolExecutionContext) -> Result<(), ToolError> { + // A second resolution of the same path would produce another instance. + assert!( + self.was_called.load(Ordering::SeqCst), + "cancel must use the dispatched instance" + ); + self.cancellations.fetch_add(1, Ordering::SeqCst); + self.cancelled.notify_one(); + Ok(()) + } + } + + struct PendingCancellationResolver { + started: Arc, + cancellations: Arc, + } + + impl WipDynamicOperationResolver for PendingCancellationResolver { + fn handler(&self, _: &str) -> Arc { + Arc::new(PendingCancellation { + started: Arc::clone(&self.started), + cancelled: tokio::sync::Notify::new(), + was_called: AtomicBool::new(false), + cancellations: Arc::clone(&self.cancellations), + }) + } + } + + struct CancellationItemResolver { + unrelated: Arc, + } + + impl WipDynamicItemResolver for CancellationItemResolver { + fn resolve(&self, item: &str) -> Option { + if item != "A-1" { + return None; + } + let projection = contribution_projection(Arc::new(UnrelatedCancellation { + cancellations: Arc::clone(&self.unrelated), + })); + Some(WipDynamicItem { + object: projection.object, + handler: projection.handler, + }) + } + } + + async fn assert_contributed_cancellation_is_selected_once(dynamic: bool) { + let started = Arc::new(tokio::sync::Notify::new()); + let cancellations = Arc::new(AtomicUsize::new(0)); + let unrelated = Arc::new(AtomicUsize::new(0)); + let mut registry = WipMountRegistry::new(); + registry.allocate_namespace("asset", "assets").unwrap(); + let projection = contribution_projection(Arc::new(UnrelatedCancellation { + cancellations: Arc::clone(&unrelated), + })); + let mut descriptor = projection.descriptor.clone(); + // Two names share one provider; read sorts first and fails cancellation. + descriptor.operations = vec![ + contributed_operation("write"), + contributed_operation("write_again"), + ]; + if dynamic { + let mut collection = projection.clone(); + collection.route = "/assets".into(); + collection.object.name = "assets".into(); + collection.interface = "test.asset/collection/v1".into(); + collection.object.interfaces = vec![collection.interface.clone()]; + registry.mount(collection).unwrap(); + registry + .mount_dynamic(WipDynamicMount { + collection_route: "/assets".into(), + capability: "asset:item".into(), + interface: projection.interface.clone(), + descriptor: projection.descriptor, + interface_validator: projection.interface_validator, + resolver: Arc::new(CancellationItemResolver { + unrelated: Arc::clone(&unrelated), + }), + }) + .unwrap(); + registry + .contribute_dynamic_operations(WipDynamicOperationContribution { + collection_route: "/assets".into(), + contributor: "management".into(), + interface: projection.interface, + descriptor, + resolver: Arc::new(PendingCancellationResolver { + started: Arc::clone(&started), + cancellations: Arc::clone(&cancellations), + }), + }) + .unwrap(); + } else { + registry.mount(projection).unwrap(); + registry + .contribute_operations(WipOperationContribution { + route: "/assets/A-1".into(), + contributor: "management".into(), + interface: "test.asset/item/v1".into(), + descriptor, + handler: PendingCancellationResolver { + started: Arc::clone(&started), + cancellations: Arc::clone(&cancellations), + } + .handler("A-1"), + }) + .unwrap(); + } + let runtime = + WipRuntime::new(WipHost::new(registry), SecurityContext::new("worker-a"), 0).unwrap(); + runtime + .discover("/assets/A-1".into(), 0, false) + .await + .unwrap(); + runtime + .inspect("test.asset/item/v1".into(), false) + .await + .unwrap(); + let execution = ToolExecutionContext::direct(); + let call = runtime.call( + "/assets/A-1".into(), + "test.asset/item/v1".into(), + "write".into(), + json!({}), + execution.clone(), + ); + let cancel = async { + started.notified().await; + // Unknown executions must not invoke any provider. + runtime + .cancel(&ToolExecutionContext::new("unknown", "other-batch", 0)) + .await + .unwrap(); + runtime.cancel(&execution).await.unwrap(); + }; + let (result, ()) = tokio::time::timeout(std::time::Duration::from_secs(2), async { + tokio::join!(call, cancel) + }) + .await + .expect("selected provider must receive cancellation"); + assert!(matches!(result, Err(ToolError::Cancelled(_)))); + assert_eq!(unrelated.load(Ordering::SeqCst), 0); + assert_eq!(cancellations.load(Ordering::SeqCst), 1); + assert!(runtime.active.lock().unwrap().is_empty()); + assert_eq!(runtime.audit()[0].outcome, WipAuditOutcome::Cancelled); + runtime.cancel(&execution).await.unwrap(); + assert_eq!(cancellations.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn static_contributed_cancellation_ignores_unrelated_and_shared_handlers() { + assert_contributed_cancellation_is_selected_once(false).await; + } + + #[tokio::test] + async fn dynamic_contributed_cancellation_pins_the_dispatched_instance() { + assert_contributed_cancellation_is_selected_once(true).await; + } + #[tokio::test] async fn cancelled_unknown_and_disconnected_calls_have_distinct_terminal_states() { let cancelled = diff --git a/crates/worker/src/worker.rs b/crates/worker/src/worker.rs index 8689e90a4..e3dbde07d 100644 --- a/crates/worker/src/worker.rs +++ b/crates/worker/src/worker.rs @@ -60,8 +60,8 @@ use crate::feature::session::{ SessionExtensionHandle, }; use crate::feature::{ - FeatureInstructionDeclaration, FeatureInstructionId, FeatureRegistryBuilder, - FeatureRegistryInstallReport, dedupe_instruction_contributions, + FeatureInstructionDeclaration, FeatureInstructionId, FeatureInvocationRegistry, + FeatureRegistryBuilder, FeatureRegistryInstallReport, dedupe_instruction_contributions, }; use crate::hook::{ BeforeSessionRewriteAction, BeforeSessionRewriteContext, Hook, HookHistoryRange, @@ -86,6 +86,101 @@ const MAX_PENDING_SUBMISSION_BYTES: u64 = 1024 * 1024; const MAX_PENDING_ARTIFACT_REFS: usize = 64; const MAX_ACTIVATION_REQUEST_ID_BYTES: usize = 128; const MAX_SUBMISSION_RECEIPTS: usize = 128; +const MAX_INVOCATION_RECEIPTS: usize = 1024; +const MAX_INVOCATION_IDENTITY_BYTES: usize = 256; + +/// A durable start without a matching result is intentionally not retryable. +/// This is a conservative fence, not an exactly-once business-operation ledger. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct InvocationReceipt { + invocation_id: String, + identity: protocol::FeatureInvocationIdentity, + payload_digest: String, + result: Option, +} + +impl InvocationReceipt { + fn new( + invocation: &protocol::FeatureInvocation, + result: Option, + ) -> Self { + Self { + invocation_id: invocation.invocation_id.clone(), + identity: invocation.identity.clone(), + payload_digest: invocation_payload_digest(invocation), + result, + } + } + + fn matches(&self, invocation: &protocol::FeatureInvocation) -> bool { + self.invocation_id == invocation.invocation_id + && self.identity == invocation.identity + && self.payload_digest == invocation_payload_digest(invocation) + } + + fn is_valid(&self) -> bool { + !self.invocation_id.trim().is_empty() + && self.invocation_id.len() <= protocol::invocation::MAX_FEATURE_INVOCATION_ID_BYTES + && !self.identity.0.is_empty() + && self.identity.0.len() <= MAX_INVOCATION_IDENTITY_BYTES + && self.payload_digest.len() == 64 + && self + .payload_digest + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + && self.result.as_ref().is_none_or(|result| { + result.invocation_id == self.invocation_id + && result.identity == self.identity + && crate::feature::invocation::invocation_result_is_bounded(result) + && (result.status == protocol::FeatureInvocationStatus::Succeeded + || result.context.is_none()) + }) + } +} + +fn invocation_payload_digest(invocation: &protocol::FeatureInvocation) -> String { + submission_payload_digest(&[Segment::FeatureInvoke { + invocation: invocation.clone(), + }]) +} + +fn validate_activation_invocation_ids(input: &[Segment]) -> Result<(), String> { + if input + .iter() + .any(|segment| matches!(segment, Segment::Unknown)) + { + return Err( + "unknown input segment rejected; unsupported intent must not be discarded".into(), + ); + } + const MAX_INVOCATIONS_PER_SUBMISSION: usize = 8; + let mut ids = std::collections::BTreeSet::new(); + for invocation in input.iter().filter_map(|segment| match segment { + Segment::FeatureInvoke { invocation } => Some(invocation), + _ => None, + }) { + if invocation.invocation_id.trim().is_empty() + || invocation.invocation_id.len() + > protocol::invocation::MAX_FEATURE_INVOCATION_ID_BYTES + || invocation.identity.0.is_empty() + || invocation.identity.0.len() > MAX_INVOCATION_IDENTITY_BYTES + { + return Err( + "invocation id/identity is empty or exceeds the recovery key byte limit".into(), + ); + } + if !ids.insert(&invocation.invocation_id) { + return Err("duplicate invocation id in one submission".into()); + } + } + if ids.len() > MAX_INVOCATIONS_PER_SUBMISSION { + return Err(format!( + "at most {MAX_INVOCATIONS_PER_SUBMISSION} Feature invocations are allowed per submission" + )); + } + Ok(()) +} #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] pub(crate) struct PendingSubmission { @@ -146,6 +241,12 @@ pub(crate) struct PendingActivationState { pending_notifications: VecDeque, receipts: VecDeque, notification_receipts: VecDeque, + /// Retained across queue checkpoints, compaction, rewind and restore. Do not + /// evict starts/results: eviction would silently permit repeated effects. + #[serde(default)] + invocation_receipts: Vec, + #[serde(default)] + invocation_recovery_blocked: bool, } fn deserialize_activating_notifications<'de, D>( @@ -312,6 +413,7 @@ fn pending_submission_preview(input: &[Segment]) -> String { Segment::UploadedFile { file } => ["[Attached file: ", &file.file_name, "]"], Segment::PasteArtifact { .. } => ["[Large paste]", "", ""], Segment::FileRef { path } => ["@", path, ""], + Segment::FeatureInvoke { invocation } => ["[Feature: ", &invocation.name, "]"], Segment::Flow { selector } => ["[Flow: ", selector, "]"], Segment::Unknown => ["[Unknown input]", "", ""], }; @@ -626,6 +728,7 @@ pub enum WorkspaceServerOperation { WorkerControlRestore { runtime_id: String, worker_id: String, + request: server_api::WorkerRestoreRequest, }, WorkerObservationSessions, WorkerObservationCapture(server_api::WorkerObservationSubjectRef), @@ -700,11 +803,15 @@ fn workspace_server_operation_request( WorkspaceServerOperation::WorkerControlRestore { runtime_id, worker_id, - } => Ok(WorkspaceRequest::json( - WorkspaceRequestMethod::Post, - format!("{base}/worker-control/workers/{runtime_id}/{worker_id}/restore"), - "".to_string(), - )), + request, + } => workspace_server_json_request( + format!( + "{base}/worker-control/workers/{}/{}/restore", + encode_workspace_path_segment(&runtime_id), + encode_workspace_path_segment(&worker_id) + ), + &request, + ), WorkspaceServerOperation::WorkerObservationSessions => Ok(WorkspaceRequest::get(format!( "{base}/worker-observation/sessions" ))), @@ -1508,6 +1615,8 @@ pub(crate) enum PendingSubmissionError { RequestIdLimit, #[error("submission input must contain at least one typed segment")] EmptyInput, + #[error("Feature invocation rejected: {0}")] + InvalidInvocation(String), #[error("submission request id was already used with a different payload")] IdempotencyConflict, #[error("pending submission queue is full (maximum {MAX_PENDING_SUBMISSIONS})")] @@ -1667,6 +1776,8 @@ where if input.is_empty() { return Err(PendingSubmissionError::EmptyInput); } + validate_activation_invocation_ids(&input) + .map_err(PendingSubmissionError::InvalidInvocation)?; let payload_digest = submission_payload_digest(&input); let _append_guard = self .writer @@ -1895,7 +2006,7 @@ where pub(crate) fn prepare_notification_batch( &self, - ) -> Vec<(PendingNotification, SessionExtension)> { + ) -> Vec<(PendingNotification, NotificationCommitIdentity)> { let mut state = self .state .lock() @@ -1912,18 +2023,11 @@ where notifications .into_iter() .map(|notification| { - let index = state - .activating_notifications - .iter() - .position(|active| { - active.notification_request_id == notification.notification_request_id - && active.source_namespace == notification.source_namespace - }) - .expect("newly staged notification must be active"); - let mut committed = state.clone(); - committed.activating_notifications.drain(..=index); - committed.revision = committed.revision.saturating_add(1); - (notification, pending_activation_extension(&committed)) + let identity = NotificationCommitIdentity { + request_id: notification.notification_request_id.clone(), + source_namespace: notification.source_namespace.clone(), + }; + (notification, identity) }) .collect() } @@ -2027,17 +2131,6 @@ where } } - pub(crate) fn finish_notification_batch(&self) { - let mut state = self - .state - .lock() - .expect("pending activation state poisoned"); - if !state.activating_notifications.is_empty() { - state.activating_notifications.clear(); - state.revision = state.revision.saturating_add(1); - } - } - pub(crate) fn direct_client_namespace(&self) -> String { format!("direct:{}", self.writer.state.location().session_id) } @@ -2158,6 +2251,93 @@ impl PendingSubmissionHandle { } } +/// Identity of one staged notification, never a frozen full-state checkpoint. +#[derive(Debug, Clone)] +pub struct NotificationCommitIdentity { + pub(crate) request_id: String, + pub(crate) source_namespace: String, +} + +pub(crate) trait DurableNotificationCommitter: Send + Sync { + fn commit_notification( + &self, + identity: &NotificationCommitIdentity, + item: SystemItem, + provenance: Option, + ) -> Result>, StoreError>; +} + +impl DurableNotificationCommitter for PendingSubmissionHandle { + fn commit_notification( + &self, + identity: &NotificationCommitIdentity, + item: SystemItem, + provenance: Option, + ) -> Result>, StoreError> { + let _append = self + .writer + .state + .append_lock + .lock() + .expect("segment append lock poisoned"); + let mut state = self + .state + .lock() + .expect("pending activation state poisoned"); + let matches = |notification: &PendingNotification| { + notification.notification_request_id == identity.request_id + && notification.source_namespace == identity.source_namespace + }; + let Some(index) = state.activating_notifications.iter().position(matches) else { + // A partial drain may already have committed this notification. Its + // original history entry remains authoritative; never append it twice. + if !state.pending_notifications.iter().any(matches) + && state.notification_receipts.iter().any(|receipt| { + receipt.notification_request_id == identity.request_id + && receipt.source_namespace == identity.source_namespace + }) + { + return Ok(None); + } + return Err(StoreError::Io(std::io::Error::other( + "notification is not staged at the commit boundary", + ))); + }; + if index != 0 { + return Err(StoreError::Io(std::io::Error::other( + "notification commit would violate FIFO order", + ))); + } + let notification = &state.activating_notifications[index]; + if !matches!(&item, SystemItem::Notification { message, .. } if message == ¬ification.message) + || provenance.as_ref() != Some(¬ification.provenance) + { + return Err(StoreError::Io(std::io::Error::other( + "notification does not match accepted input authority", + ))); + } + let mut committed = state.clone(); + committed.activating_notifications.pop_front(); + committed.revision = committed.revision.saturating_add(1); + let metadata = new_history_metadata( + provenance.unwrap_or(WorkerHistoryProvenance::LegacyUnknown), + None, + ); + let history_item = item.to_history_item(); + self.writer + .append_entry_locked(LogEntry::AnnotatedSystemItem { + ts: segment_log::now_millis(), + entry: session_store::LoggedSystemHistoryEntry { + item, + metadata: metadata.clone(), + }, + extensions: vec![pending_activation_extension(&committed)], + })?; + *state = committed; + Ok(Some(HistoryEntry::new(history_item, metadata))) + } +} + /// Type-erased commit handle for the interceptor. Lets the interceptor commit `SystemItem`s without being generic over the /// concrete `Store` type. pub trait SystemItemCommitter: Send + Sync { @@ -2305,6 +2485,19 @@ impl WorkerSession { return; }; if let Ok(mut state) = serde_json::from_value::(payload.clone()) { + let mut ids = std::collections::BTreeSet::new(); + if state.invocation_receipts.len() > MAX_INVOCATION_RECEIPTS + || state + .invocation_receipts + .iter() + .any(|receipt| !receipt.is_valid() || !ids.insert(&receipt.invocation_id)) + { + self.pending_activations + .lock() + .expect("pending activation state poisoned") + .invocation_recovery_blocked = true; + return; + } if let Some(activating) = state.activating.take() { state.pending.push_front(activating); state.revision = state.revision.saturating_add(1); @@ -2319,6 +2512,13 @@ impl WorkerSession { .pending_activations .lock() .expect("pending activation state poisoned") = state; + } else { + // A malformed checkpoint must never turn a durable start into a + // new executable invocation on restore. + self.pending_activations + .lock() + .expect("pending activation state poisoned") + .invocation_recovery_blocked = true; } } @@ -2361,9 +2561,30 @@ pub(crate) enum SystemPromptContribution { Unavailable, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) struct ResidentContextRefresh { + pub body: String, + pub revision: u64, +} + #[async_trait::async_trait] pub(crate) trait SystemPromptContributionSource: Send + Sync { async fn load(&self) -> SystemPromptContribution; + + /// Returns a durable model-visible refresh only when user-managed resident + /// context changed since the last successful load. + async fn load_changed_resident_context( + &self, + ) -> Result, String> { + Ok(None) + } + + /// Acknowledges that the revision's typed refresh was durably committed. + fn confirm_resident_context_revision(&self, _revision: u64) {} + + /// Invalidates the in-memory representation fence before history may be + /// compacted or rewound. + fn invalidate_resident_context_representation(&self) {} } #[derive(Debug, Clone)] @@ -2378,6 +2599,8 @@ struct PendingCompactionCleanup { /// `session-store` functions after each turn. pub struct Worker { manifest: WorkerManifest, + /// Host-bound once before Feature installation; never inherited by child Workers. + backend_job: std::sync::OnceLock>, /// Always `Some` outside of `run()`/`resume()`. engine: Option>, /// Sole live authority for committed model-visible history. @@ -2416,6 +2639,8 @@ pub struct Worker { hook_registry: Option>, /// Executable background tasks registered by successfully installed features. feature_background_tasks: FeatureBackgroundTaskRegistry, + /// Enabled chat invocation metadata and executable handlers. + feature_invocations: FeatureInvocationRegistry, /// Internal Workers install an explicit Feature composition and disable /// manifest-derived lifecycle Features before controller startup. manifest_lifecycle_features_enabled: bool, @@ -2561,6 +2786,7 @@ pub struct Worker { /// paths skip SystemItem disk commits but still see the rendered /// `Item::system_message` in worker history. log_writer: Option>, + notification_committer: Option>, } impl Drop for Worker { @@ -2640,10 +2866,14 @@ impl Worker { /// Idempotent: subsequent calls overwrite the previous handle. pub fn attach_log_writer(&mut self, writer: Arc) { self.log_writer = Some(writer); + self.notification_committer = Some(Arc::new(self.pending_submission_handle())); } pub fn attach_in_flight_events(&mut self, in_flight: InFlightEvents) { self.in_flight = Some(in_flight); + if self.notification_committer.is_some() { + self.notification_committer = Some(Arc::new(self.pending_submission_handle())); + } } pub fn clear_in_flight_events(&self) { @@ -2737,6 +2967,7 @@ impl Worker { let scope = SharedScope::new(scope); let workdir_sessions = workdir_sessions_from_authority(&filesystem_authority, &scope); let mut worker = Self { + backend_job: std::sync::OnceLock::new(), manifest, engine: Some(worker), session: WorkerSession::new(session_id, Vec::new()), @@ -2754,6 +2985,7 @@ impl Worker { hook_builder: HookRegistryBuilder::new(), hook_registry: None, feature_background_tasks: FeatureBackgroundTaskRegistry::default(), + feature_invocations: FeatureInvocationRegistry::default(), manifest_lifecycle_features_enabled: true, interceptor_installed: false, compact_state: None, @@ -2788,6 +3020,7 @@ impl Worker { sink: SegmentLogSink::new(), history_persistence_wired: false, log_writer: None, + notification_committer: None, }; worker.apply_permissions_from_manifest(); worker.apply_prune_from_manifest(); @@ -2898,6 +3131,30 @@ impl Worker { self.segment_state.session_id() } + /// Immutable host-owned Job authority, absent for ordinary and child Workers. + pub fn backend_job(&self) -> Option<&crate::BackendJobExecutionBinding> { + self.backend_job.get().and_then(Option::as_ref) + } + + /// Trusted host seam; immutable after binding and unavailable after Feature installation. + pub fn bind_backend_job( + &mut self, + binding: crate::BackendJobExecutionBinding, + ) -> Result<(), &'static str> { + binding.validate()?; + if self.interceptor_installed || self.backend_job.get().is_some() { + return Err("Backend Job capability is already sealed"); + } + if !self.workspace_client().is_available() + || self.workspace_client().workspace_id().is_none() + { + return Err("Backend Job capability requires Backend Workspace API authority"); + } + self.backend_job + .set(Some(binding)) + .map_err(|_| "Backend Job capability is already bound") + } + /// The Worker's manifest. pub fn manifest(&self) -> &WorkerManifest { &self.manifest @@ -3159,17 +3416,23 @@ impl Worker { self.manifest_lifecycle_features_enabled } + pub fn feature_invocations(&self) -> FeatureInvocationRegistry { + self.feature_invocations.clone() + } + /// Install enabled feature modules into the Worker host surfaces. pub fn install_features( &mut self, registry: FeatureRegistryBuilder, ) -> FeatureRegistryInstallReport { + self.backend_job.get_or_init(|| None); let worker = self.engine.as_mut().expect("worker taken during run"); let report = registry.install_into_engine(worker, &mut self.hook_builder); if report.has_errors() { return report; } self.feature_background_tasks = report.background_tasks.clone(); + self.feature_invocations = report.chat_invocations.clone(); for instruction in report.installed_instruction_contributions() { self.register_feature_instruction(instruction); } @@ -3271,6 +3534,13 @@ impl Worker { .prepare_session_rewrite(SessionRewriteKind::Rewind) .await .map_err(|error| RewindError::Invalid(error.to_string()))?; + // Serialize the replacement with queue admission/checkpoint appends so + // a concurrently accepted operation cannot lose its recovery evidence. + let segment_state = self.segment_state.clone(); + let append_guard = segment_state + .append_lock + .lock() + .expect("segment append lock poisoned"); let loc = self.segment_state.location(); if target.segment_id != loc.segment_id { return Err(RewindError::Invalid( @@ -3304,10 +3574,10 @@ impl Worker { tool_side_effect_warning, }; - self.store - .truncate(loc.session_id, loc.segment_id, truncate_entries)?; - self.segment_state.set_entries_written(truncate_entries); - self.sink.truncate_silent(truncate_entries); + // Recovery evidence survives a rewind independently of visible history. + // Persist the prefix and checkpoint in one replacement: truncating first + // would erase receipts if the checkpoint append failed or we crashed. + let mut replacement = retained.clone(); let pending_state = self .session .pending_activations @@ -3320,6 +3590,8 @@ impl Worker { || !pending_state.activating_notifications.is_empty() || !pending_state.receipts.is_empty() || !pending_state.notification_receipts.is_empty() + || !pending_state.invocation_receipts.is_empty() + || pending_state.invocation_recovery_blocked { let checkpoint = LogEntry::Extension { ts: segment_log::now_millis(), @@ -3330,8 +3602,13 @@ impl Worker { )) })?, }; - self.commit_entry(checkpoint)?; + replacement.push(checkpoint); } + self.store + .create_segment(loc.session_id, loc.segment_id, &replacement)?; + self.segment_state.set_entries_written(replacement.len()); + self.sink.replace_silent(replacement); + drop(append_guard); let history_entries = restore_history_entries(loc.session_id, loc.segment_id, &retained) .map_err(|error| RewindError::Invalid(error.into()))?; @@ -3838,8 +4115,10 @@ impl Worker { self.log_writer.clone(), self.pending_committed_history.clone(), ) + .with_notification_committer(self.notification_committer.clone()) .with_usage_tracker(self.usage_tracker.clone()) .with_metrics_tracker(self.metrics_tracker.clone()) + .with_resident_context_source(self.feature_resident_summary_source.clone()) .with_prompt_workspace_id( self.workspace_context .workspace_id() @@ -4322,6 +4601,7 @@ impl Worker { St: Clone + 'static, F: FnOnce(), { + validate_activation_invocation_ids(&input).map_err(WorkerError::FeatureInvocation)?; let (mut input, pending_flow_state, flow_projection) = self.prepare_flow_input(input)?; let projected_entry_ids = if flow_projection.is_some() { (0..input.len()) @@ -4399,6 +4679,27 @@ impl Worker { self.user_segments.push(input.clone()); on_input_committed(); + // Explicit invocation handlers run only after the typed input is durable + // and before any dependent LLM request. They execute in source order; + // the first failure fences later invocations in the same submission. + let has_invocations = input + .iter() + .any(|segment| matches!(segment, Segment::FeatureInvoke { .. })); + match self.execute_feature_invocations(&input).await { + Ok(false) => {} + Ok(true) => { + self.materialize_committed_history(&projected_input); + return Err(WorkerError::FeatureInvocation( + "activation preparation failed or has an unknown outcome; dependent LLM request was not sent".into(), + )); + } + Err(error) => { + self.materialize_committed_history(&projected_input); + return Err(error); + } + } + let committed_activation_input = has_invocations.then(|| projected_input.clone()); + // Resolve `@` file refs to system messages stashed for the // WorkerInterceptor to attach right after the user message. Resolution // failures are non-fatal alerts. @@ -4446,7 +4747,12 @@ impl Worker { self.engine = Some(locked.unlock()); self.session.note_mutation(); - if self.should_rollback_empty_turn(&result, &rollback_snapshot) { + if let Some(input) = &committed_activation_input { + // A prompt hook may cancel before Engine materializes input/results. + // Business preparation is already durable and cannot be rolled back. + self.materialize_committed_history(input); + } + if !has_invocations && self.should_rollback_empty_turn(&result, &rollback_snapshot) { self.rollback_empty_turn(rollback_snapshot)?; return Ok(WorkerRunResult::RolledBack); } @@ -4454,6 +4760,252 @@ impl Worker { self.handle_worker_result(result, history_before).await } + async fn execute_feature_invocations(&self, segments: &[Segment]) -> Result + where + St: Clone + 'static, + { + use protocol::{FeatureInvocationResult, FeatureInvocationStatus}; + let invocations = segments + .iter() + .filter_map(|segment| match segment { + Segment::FeatureInvoke { invocation } => Some(invocation), + _ => None, + }) + .collect::>(); + let mut prior_failure = false; + for invocation in invocations { + let failed = |status, message: String| FeatureInvocationResult { + invocation_id: invocation.invocation_id.clone(), + identity: invocation.identity.clone(), + status, + message, + context: None, + }; + let handle = self.pending_submission_handle(); + // Use the queue's append/state lock order. Never hold a lock while + // awaiting business execution. A durable start without result fences + // both restart and same-process replay. + let (existing, recovery_blocked) = { + let state = handle + .state + .lock() + .expect("pending activation state poisoned"); + ( + state + .invocation_receipts + .iter() + .find(|receipt| receipt.invocation_id == invocation.invocation_id) + .cloned(), + state.invocation_recovery_blocked, + ) + }; + let result = if recovery_blocked { + failed( + FeatureInvocationStatus::OutcomeUnknown, + "invocation recovery checkpoint is invalid; execution is fenced".into(), + ) + } else if prior_failure { + failed( + FeatureInvocationStatus::Failed, + "not executed because an earlier invocation in this submission failed".into(), + ) + } else if let Some(receipt) = &existing { + if !receipt.matches(invocation) { + failed( + FeatureInvocationStatus::Failed, + "invocation id was already used with a different payload".into(), + ) + } else { + let replay = receipt.result.clone().unwrap_or_else(|| { + failed( + FeatureInvocationStatus::OutcomeUnknown, + "a durable invocation start has no result; do not retry automatically" + .into(), + ) + }); + if replay.status == FeatureInvocationStatus::Succeeded { + // Recorded success is reusable context, not authority to + // resurrect a removed capability or revoked argument + // permission. Revalidate without repeating the effect. + match self.feature_invocations.validate(invocation) { + Ok(()) => replay, + Err(error) => failed(FeatureInvocationStatus::Failed, error.message), + } + } else { + replay + } + } + } else if let Err(error) = self.feature_invocations.validate(invocation) { + failed(FeatureInvocationStatus::Failed, error.message) + } else { + let started = { + let _append = handle + .writer + .state + .append_lock + .lock() + .expect("segment append lock poisoned"); + let mut state = handle + .state + .lock() + .expect("pending activation state poisoned"); + if state.invocation_receipts.len() >= MAX_INVOCATION_RECEIPTS { + false + } else { + let mut started = state.clone(); + started + .invocation_receipts + .push(InvocationReceipt::new(invocation, None)); + handle + .persist_locked(&started) + .map_err(|error| WorkerError::FeatureInvocation(error.to_string()))?; + *state = started; + true + } + }; + if !started { + failed(FeatureInvocationStatus::Failed, + "invocation recovery receipt capacity reached; refusing untracked execution".into()) + } else { + match self.feature_invocations.invoke(invocation).await { + Ok(result) => result, + Err(error) => failed( + if error.outcome_unknown { + FeatureInvocationStatus::OutcomeUnknown + } else { + FeatureInvocationStatus::Failed + }, + error.message, + ), + } + } + }; + prior_failure |= result.status != FeatureInvocationStatus::Succeeded; + let body = match (&result.status, &result.context) { + (FeatureInvocationStatus::Succeeded, Some(context)) => format!( + "[Feature invocation succeeded: {}]\n{}", + result.identity, context + ), + (FeatureInvocationStatus::Succeeded, None) => format!( + "[Feature invocation succeeded: {}] {}", + result.identity, result.message + ), + (FeatureInvocationStatus::Failed, _) => format!( + "[Feature invocation failed: {}] {}", + result.identity, result.message + ), + (FeatureInvocationStatus::OutcomeUnknown, _) => format!( + "[Feature invocation outcome unknown: {}] {} Do not retry automatically.", + result.identity, result.message + ), + }; + // Commit result and recovery checkpoint atomically with the existing + // AnnotatedSystemItem lifecycle. Failed appends leave the start fenced. + let _append = handle + .writer + .state + .append_lock + .lock() + .expect("segment append lock poisoned"); + let mut state = handle + .state + .lock() + .expect("pending activation state poisoned"); + let mut completed = state.clone(); + if let Some(receipt) = completed + .invocation_receipts + .iter_mut() + .find(|receipt| receipt.matches(invocation) && receipt.result.is_none()) + { + receipt.result = Some(result.clone()); + } else if existing.is_none() + && completed.invocation_receipts.len() < MAX_INVOCATION_RECEIPTS + { + completed + .invocation_receipts + .push(InvocationReceipt::new(invocation, Some(result.clone()))); + } + let item = SystemItem::FeatureInvocationResult { result, body }; + let metadata = new_history_metadata( + WorkerHistoryProvenance::BackendInstruction { operation_id: None }, + None, + ); + let history = HistoryEntry::new(item.to_history_item(), metadata.clone()); + handle + .writer + .append_entry_locked(LogEntry::AnnotatedSystemItem { + ts: segment_log::now_millis(), + entry: session_store::LoggedSystemHistoryEntry { item, metadata }, + extensions: vec![pending_activation_extension(&completed)], + })?; + *state = completed; + self.pending_committed_history + .lock() + .expect("pending committed history poisoned") + .push_back(history); + } + Ok(prior_failure) + } + + fn materialize_committed_history(&mut self, input: &[HistoryEntry]) { + let pending = self + .pending_committed_history + .lock() + .expect("pending committed history poisoned") + .drain(..) + .collect::>(); + for entry in input.iter().cloned().chain(pending) { + if !self + .session + .history() + .entries() + .iter() + .any(|existing| existing.annotation.entry_id == entry.annotation.entry_id) + { + self.session.history_mut().push_entry(entry); + } + } + self.session.note_mutation(); + } + + pub(crate) fn can_schedule_notification_run(&self) -> bool { + !self.has_failed_segment_activation() + && !self.has_pending_compaction_cleanup() + && self.ensure_invocation_preparation_complete().is_ok() + } + + fn ensure_invocation_preparation_complete(&self) -> Result<(), WorkerError> { + let state = self + .session + .pending_activations + .lock() + .expect("pending activation state poisoned"); + if state.invocation_recovery_blocked { + return Err(WorkerError::FeatureInvocation( + "invocation recovery checkpoint is invalid".into(), + )); + } + if let Some(input) = self.user_segments.last() { + for invocation in input.iter().filter_map(|segment| match segment { + Segment::FeatureInvoke { invocation } => Some(invocation), + _ => None, + }) { + let succeeded = state.invocation_receipts.iter().any(|receipt| { + receipt.matches(invocation) + && receipt.result.as_ref().is_some_and(|result| { + result.status == protocol::FeatureInvocationStatus::Succeeded + }) + }); + if !succeeded || self.feature_invocations.validate(invocation).is_err() { + return Err(WorkerError::FeatureInvocation( + "activation preparation is failed, incomplete, or outcome unknown; submit fresh input instead of resuming".into(), + )); + } + } + } + Ok(()) + } + /// Resolve every `Segment::FileRef` in `segments` to a `[File: ]` /// or shallow `[Dir: ]` system message via `WorkerFsView`. Resolution /// failures (out-of-scope, not-found, binary, I/O, unsupported symlink @@ -4765,6 +5317,7 @@ impl Worker { where St: Clone + 'static, { + self.ensure_invocation_preparation_complete()?; debug_assert!( matches!( kind, @@ -4821,6 +5374,7 @@ impl Worker { where St: Clone + 'static, { + self.ensure_invocation_preparation_complete()?; self.prepare_for_run().await?; self.resume_prepared(source).await } @@ -5020,6 +5574,10 @@ impl Worker { where St: Clone + 'static, { + // A producer may durably append a prefix before a later append fails. + // Preserve that exact annotated prefix even when Engine never got to + // materialize it; retries must neither lose nor reappend it. + self.materialize_committed_history(&[]); let run_id = uuid::Uuid::now_v7().to_string(); let hook_exit = hook_run_exit(&result); if let Some(hooks) = self.hook_registry.clone() { @@ -6719,6 +7277,7 @@ where workdir_sessions_from_authority(&common.filesystem_authority, &scope); let mut worker = Self { + backend_job: std::sync::OnceLock::new(), manifest, engine: Some(worker), session: WorkerSession::new(session_id, Vec::new()), @@ -6736,6 +7295,7 @@ where hook_builder: HookRegistryBuilder::new(), hook_registry: None, feature_background_tasks: FeatureBackgroundTaskRegistry::default(), + feature_invocations: FeatureInvocationRegistry::default(), manifest_lifecycle_features_enabled: true, interceptor_installed: false, compact_state: None, @@ -6770,6 +7330,7 @@ where sink: SegmentLogSink::new(), history_persistence_wired: false, log_writer: None, + notification_committer: None, }; worker.apply_permissions_from_manifest(); worker.apply_prune_from_manifest(); @@ -6818,6 +7379,18 @@ where }); } + if self + .backend_job() + .is_some_and(|binding| binding.subjektiv_consolidation) + { + if metadata.subjektiv_session_attribution.is_some() { + return Err(WorkerError::SubjektivSessionAttribution { + state: "job_attribution_forbidden", + message: "consolidation Job execution must not carry subject-body Session attribution".into(), + }); + } + return Ok(()); + } if !subjektiv_attached { if metadata.subjektiv_session_attribution.is_some() { return Err(WorkerError::SubjektivSessionAttribution { @@ -6996,6 +7569,7 @@ where let workdir_sessions = workdir_sessions_from_authority(&common.filesystem_authority, &scope); let mut worker = Self { + backend_job: std::sync::OnceLock::new(), manifest, engine: Some(engine), session: WorkerSession::new(session_id, Vec::new()), @@ -7013,6 +7587,7 @@ where hook_builder: HookRegistryBuilder::new(), hook_registry: None, feature_background_tasks: FeatureBackgroundTaskRegistry::default(), + feature_invocations: FeatureInvocationRegistry::default(), manifest_lifecycle_features_enabled: false, interceptor_installed: false, compact_state: None, @@ -7047,6 +7622,7 @@ where sink: SegmentLogSink::new(), history_persistence_wired: false, log_writer: None, + notification_committer: None, }; worker.apply_permissions_from_manifest(); worker.apply_prune_from_manifest(); @@ -7121,6 +7697,7 @@ where workdir_sessions_from_authority(&common.filesystem_authority, &scope); let mut worker = Self { + backend_job: std::sync::OnceLock::new(), manifest, engine: Some(worker), session: WorkerSession::new(session_id, Vec::new()), @@ -7138,6 +7715,7 @@ where hook_builder: HookRegistryBuilder::new(), hook_registry: None, feature_background_tasks: FeatureBackgroundTaskRegistry::default(), + feature_invocations: FeatureInvocationRegistry::default(), manifest_lifecycle_features_enabled: true, interceptor_installed: false, compact_state: None, @@ -7172,6 +7750,7 @@ where sink: SegmentLogSink::new(), history_persistence_wired: false, log_writer: None, + notification_committer: None, }; worker.apply_permissions_from_manifest(); worker.apply_prune_from_manifest(); @@ -7500,6 +8079,7 @@ where workdir_sessions_from_authority(&common.filesystem_authority, &scope); let mut worker = Self { + backend_job: std::sync::OnceLock::new(), manifest, engine: Some(worker), session: WorkerSession::new(session_id, restored_history_entries), @@ -7519,6 +8099,7 @@ where hook_builder: HookRegistryBuilder::new(), hook_registry: None, feature_background_tasks: FeatureBackgroundTaskRegistry::default(), + feature_invocations: FeatureInvocationRegistry::default(), manifest_lifecycle_features_enabled: true, interceptor_installed: false, compact_state: None, @@ -7558,6 +8139,7 @@ where sink: SegmentLogSink::with_initial(mirror_entries), history_persistence_wired: false, log_writer: None, + notification_committer: None, }; worker .session @@ -8252,6 +8834,9 @@ fn preview_segments(segments: &[Segment]) -> String { preview.push('@'); preview.push_str(path); } + Segment::FeatureInvoke { invocation } => { + preview.push_str(&invocation.display_input()); + } Segment::Flow { selector } => { preview.push_str("[Flow: "); preview.push_str(selector); @@ -8368,6 +8953,9 @@ pub enum WorkerError { message: String, }, + #[error("Feature invocation rejected: {0}")] + FeatureInvocation(String), + #[error("Flow input rejected: {0}")] FlowInput(String), @@ -9201,6 +9789,7 @@ mod build_summary_prompt_tests { availability: memory::backend::MemoryResidentSummaryAvailability, content: Option, load_count: Arc, + behavior_revision: Option>, } impl WorkspaceClient for RestoreSubjectResidentClient { @@ -9224,16 +9813,23 @@ mod build_summary_prompt_tests { serde_json::from_str(request.body.as_deref().unwrap_or_default()).unwrap(); assert!(matches!( request.operation, - server_api::SubjektivMemoryBackendOperation::ResidentSummary(_) + server_api::SubjektivMemoryBackendOperation::ResidentContext(_) )); self.load_count.fetch_add(1, Ordering::SeqCst); Ok(WorkspaceResponse { status: 200, body: serde_json::to_string( - &server_api::SubjektivMemoryBackendResponse::ResidentSummary( - memory::backend::MemoryResidentSummaryOutput { - availability: self.availability, - content: self.content.clone(), + &server_api::SubjektivMemoryBackendResponse::ResidentContext( + server_api::SubjektivResidentContextOutput { + behavior_md: "restore behavior".to_string(), + behavior_revision: self + .behavior_revision + .as_ref() + .map_or(1, |revision| revision.load(Ordering::SeqCst) as u64), + memory_surface: memory::backend::MemoryResidentSummaryOutput { + availability: self.availability, + content: self.content.clone(), + }, }, ), ) @@ -9252,9 +9848,12 @@ mod build_summary_prompt_tests { availability, content: content.map(str::to_string), load_count: Arc::clone(&load_count), + behavior_revision: None, }); let source = crate::feature::builtin::memory::ordinary_subjektiv_resident_summary_source( - manifest, client, + manifest, + client, + Arc::new(ArcSwap::from(PromptCatalog::builtins_only().unwrap())), ) .unwrap() .unwrap(); @@ -9668,6 +10267,7 @@ mod build_summary_prompt_tests { struct FailRunTerminalStore { inner: session_store::FsStore, fail_run_terminal: Arc, + invocation_failure: Arc, } impl Store for FailRunTerminalStore { @@ -9684,6 +10284,38 @@ mod build_summary_prompt_tests { "synthetic terminal append failure", ))); } + let invocation_kind = match entry { + LogEntry::Extension { + domain, payload, .. + } if domain == SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN + && payload["invocation_receipts"] + .as_array() + .is_some_and(|receipts| { + receipts.iter().any(|receipt| receipt["result"].is_null()) + }) => + { + 1 + } + LogEntry::AnnotatedSystemItem { entry, .. } + if matches!(entry.item, SystemItem::FeatureInvocationResult { .. }) => + { + 2 + } + LogEntry::AnnotatedSystemItem { entry, .. } if matches!(&entry.item, SystemItem::Notification { message, .. } if message == "second") => { + 5 + } + _ => 0, + }; + if invocation_kind != 0 + && self + .invocation_failure + .compare_exchange(invocation_kind, 0, Ordering::SeqCst, Ordering::SeqCst) + .is_ok() + { + return Err(StoreError::Io(std::io::Error::other( + "synthetic invocation append failure", + ))); + } self.inner.append(session_id, segment_id, entry) } @@ -9716,7 +10348,28 @@ mod build_summary_prompt_tests { segment_id: SegmentId, entries: &[LogEntry], ) -> Result<(), StoreError> { - self.inner.create_segment(session_id, segment_id, entries) + if self + .invocation_failure + .compare_exchange(3, 0, Ordering::SeqCst, Ordering::SeqCst) + .is_ok() + { + return Err(StoreError::Io(std::io::Error::other( + "synthetic replacement failure before atomic commit", + ))); + } + self.inner.create_segment(session_id, segment_id, entries)?; + if self + .invocation_failure + .compare_exchange(4, 0, Ordering::SeqCst, Ordering::SeqCst) + .is_ok() + { + // Model interruption after the atomic commit, before Worker + // updates its mirror/counters. Restart sees the complete new log. + return Err(StoreError::Io(std::io::Error::other( + "synthetic interruption after atomic replacement", + ))); + } + Ok(()) } fn exists(&self, session_id: SessionId, segment_id: SegmentId) -> Result { @@ -9876,6 +10529,56 @@ mod build_summary_prompt_tests { (worker, store, temp) } + #[tokio::test] + async fn ordinary_worker_cannot_gain_job_authority_after_feature_installation() { + let client = Arc::new(AttributionLifecycleClient::new([])); + let (mut worker, _store, _temp) = attribution_lifecycle_worker(client).await; + worker.install_features(FeatureRegistryBuilder::new()); + assert!( + worker + .bind_backend_job(crate::BackendJobExecutionBinding { + job_id: "job-1".into(), + attempt_id: "attempt-1".into(), + input_revision: None, + subjektiv_consolidation: false + }) + .is_err() + ); + assert!(worker.backend_job().is_none()); + } + + #[tokio::test] + async fn consolidation_job_binding_is_immutable_and_has_no_subject_body_attribution() { + let client = Arc::new(AttributionLifecycleClient::new([])); + let (mut worker, store, _temp) = attribution_lifecycle_worker(client.clone()).await; + let binding = crate::BackendJobExecutionBinding { + job_id: "job-1".into(), + attempt_id: "attempt-1".into(), + input_revision: Some("revision-1".into()), + subjektiv_consolidation: true, + }; + worker.bind_backend_job(binding.clone()).unwrap(); + assert_eq!(worker.backend_job(), Some(&binding)); + assert!(worker.bind_backend_job(binding).is_err()); + for lifecycle in [ + SubjektivSessionAttributionLifecycle::NewSession, + SubjektivSessionAttributionLifecycle::RestoredSession, + ] { + worker + .finalize_subjektiv_session_attribution(true, lifecycle) + .unwrap(); + } + assert!(client.requests.lock().unwrap().is_empty()); + assert!( + store + .read_by_name(&worker.manifest().worker.name) + .unwrap() + .unwrap() + .subjektiv_session_attribution + .is_none() + ); + } + #[tokio::test] async fn session_attribution_finalizes_once_before_runs_and_persists_subject() { let client = Arc::new(AttributionLifecycleClient::new([ @@ -9994,6 +10697,13 @@ mod build_summary_prompt_tests { #[tokio::test] async fn pending_restore_reuses_persisted_session_and_recovers_unknown_attribution() { let temp = tempfile::tempdir().unwrap(); + // Startup reads the process runtime-dir environment. Share the existing + // sandbox lock with allocation tests so their temporary directories cannot + // disappear between this test's path lookup and lock-file creation. + let runtime_dir = temp.path().join("runtime"); + std::fs::create_dir_all(&runtime_dir).unwrap(); + let _runtime_sandbox = + crate::runtime::worker_allocation::test_util::RuntimeDirSandbox::new(&runtime_dir); let store = session_store::CombinedStore::new( session_store::FsStore::new(temp.path().join("sessions")).unwrap(), session_store::FsWorkerStore::new(temp.path().join("workers")).unwrap(), @@ -10375,6 +11085,7 @@ mod build_summary_prompt_tests { let store = FailRunTerminalStore { inner: session_store::FsStore::new(dir.path().join("sessions")).unwrap(), fail_run_terminal: Arc::new(AtomicBool::new(false)), + invocation_failure: Arc::new(AtomicUsize::new(0)), }; let mut worker = Worker::new( minimal_manifest(), @@ -12219,9 +12930,10 @@ permission = "write" .unwrap() .to_string(); assert!( - empty_boundary.contains("current ready resident memory surface is intentionally empty") + empty_boundary + .contains("current generated Memory surface is ready and intentionally empty") ); - assert!(!empty_boundary.contains("No current ready resident memory surface")); + assert!(!empty_boundary.contains("No generated Memory surface exists yet")); drop(empty_restore); for availability in [ @@ -12273,7 +12985,22 @@ permission = "write" .as_text() .unwrap() .to_string(); - assert!(tombstone.contains("No current ready resident memory surface")); + let unavailable_message = match availability { + memory::backend::MemoryResidentSummaryAvailability::Ungenerated => { + "No generated Memory surface exists yet" + } + memory::backend::MemoryResidentSummaryAvailability::Stale => { + "generated Memory surface is stale" + } + memory::backend::MemoryResidentSummaryAvailability::Failed => { + "Generation of the Memory surface failed" + } + memory::backend::MemoryResidentSummaryAvailability::Unavailable => { + "Host could not fetch the Subject resident context" + } + memory::backend::MemoryResidentSummaryAvailability::Ready => unreachable!(), + }; + assert!(tombstone.contains(unavailable_message)); assert!(!tombstone.contains("latest subject surface")); assert!(!tombstone.contains("LEGACY_MEMORY_SENTINEL_MUST_NOT_REFRESH")); let entries = store.read_all(session_id, segment_id).unwrap(); @@ -13290,6 +14017,84 @@ permission = "write" assert_eq!(state.pending_notifications[1].provenance, account_b); } + #[test] + fn notification_lost_acceptance_retry_after_restore_commits_once_not_by_body() { + let temp = tempfile::tempdir().unwrap(); + let handle = PendingSubmissionHandle::for_test(temp.path()); + let source = "backend:orchestrator-attention"; + let accept = |handle: &PendingSubmissionHandle, id: &str| { + handle + .accept_notification_from_source( + id.into(), + "Queued Tickets require attention".into(), + source.into(), + WorkerHistoryProvenance::BackendInstruction { operation_id: None }, + ) + .unwrap() + }; + // The sender lost this acceptance response; the receipt is already durable. + assert!(accept(&handle, "attention-1")); + let restored_state: PendingActivationState = handle + .persisted_entries_for_test() + .iter() + .rev() + .find_map(|entry| match entry { + LogEntry::Extension { + domain, payload, .. + } if domain == SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN => { + Some(serde_json::from_value(payload.clone()).unwrap()) + } + _ => None, + }) + .unwrap(); + assert_eq!(restored_state.notification_receipts.len(), 1); + let handle = PendingSubmissionHandle { + state: Arc::new(Mutex::new(restored_state)), + writer: handle.writer.clone(), + }; + assert!(!accept(&handle, "attention-1")); + let commit_batch = |handle: &PendingSubmissionHandle| { + let batch = handle.prepare_notification_batch(); + let count = batch.len(); + for (notification, identity) in batch { + handle + .commit_notification( + &identity, + SystemItem::Notification { + body: notification.message.clone(), + message: notification.message, + prompt_provenance: None, + }, + Some(notification.provenance), + ) + .unwrap(); + } + count + }; + assert_eq!(commit_batch(&handle), 1); + assert!(!accept(&handle, "attention-1")); + assert_eq!(commit_batch(&handle), 0); + let notification_count = |handle: &PendingSubmissionHandle| { + handle + .persisted_entries_for_test() + .iter() + .filter(|entry| { + matches!(entry, + LogEntry::AnnotatedSystemItem { entry, .. } + if matches!(entry.item, SystemItem::Notification { .. }) + ) + }) + .count() + }; + assert_eq!(notification_count(&handle), 1); + assert_eq!(handle.state.lock().unwrap().notification_receipts.len(), 1); + // Same body with a new logical ID is legitimate, not content-deduped. + assert!(accept(&handle, "attention-2")); + assert_eq!(commit_batch(&handle), 1); + assert_eq!(notification_count(&handle), 2); + assert_eq!(handle.state.lock().unwrap().notification_receipts.len(), 2); + } + #[test] fn notification_batch_preserves_fifo_order_and_notification_dedupes() { let temp = tempfile::tempdir().unwrap(); @@ -13321,21 +14126,1341 @@ permission = "write" assert_eq!(notifications.len(), 2); assert_eq!(notifications[0].0.message, "notice"); assert_eq!(notifications[1].0.message, "second notice"); - let committed_state: PendingActivationState = - serde_json::from_value(notifications[1].1.payload.clone()).unwrap(); + for (notification, identity) in notifications { + handle + .commit_notification( + &identity, + SystemItem::Notification { + body: notification.message.clone(), + message: notification.message, + prompt_provenance: None, + }, + Some(notification.provenance), + ) + .unwrap(); + } + let committed_state = handle.state.lock().unwrap().clone(); assert!(committed_state.pending_notifications.is_empty()); assert!(committed_state.activating_notifications.is_empty()); - handle.finish_notification_batch(); let submission = handle.prepare_next_activation(None).unwrap().unwrap(); assert!(matches!(submission, PendingActivation::Submission(_))); } + #[test] + fn notification_commit_preserves_later_admission_and_completed_activation() { + let temp = tempfile::tempdir().unwrap(); + let handle = PendingSubmissionHandle::for_test(temp.path()); + handle + .accept_notification("first".into(), "first".into()) + .unwrap(); + let batch = handle.prepare_notification_batch(); + // These transitions occur after staging but before the model boundary. + handle + .accept("submit-1".into(), vec![Segment::text("input")], false) + .unwrap(); + let Some(PendingActivation::Submission(submission)) = + handle.prepare_next_activation(None).unwrap() + else { + panic!("expected submission") + }; + handle.finish_activation(&submission.submission_id); + handle + .accept_notification("later".into(), "later".into()) + .unwrap(); + handle + .accept("submit-2".into(), vec![Segment::text("queued")], false) + .unwrap(); + let (notification, identity) = &batch[0]; + handle + .commit_notification( + identity, + SystemItem::Notification { + body: notification.message.clone(), + message: notification.message.clone(), + prompt_provenance: None, + }, + Some(notification.provenance.clone()), + ) + .unwrap() + .unwrap(); + let entries = handle.persisted_entries_for_test(); + let LogEntry::AnnotatedSystemItem { extensions, .. } = entries.last().unwrap() else { + panic!("expected notification") + }; + let checkpoint: PendingActivationState = + serde_json::from_value(extensions[0].payload.clone()).unwrap(); + assert!( + checkpoint.activating.is_none(), + "must not revive completed submission" + ); + assert!(checkpoint.activating_notifications.is_empty()); + assert_eq!(checkpoint.pending_notifications.len(), 1); + assert_eq!(checkpoint.notification_receipts.len(), 2); + assert_eq!(checkpoint.pending.len(), 1); + assert_eq!(checkpoint.receipts.len(), 2); + assert_eq!(checkpoint.revision, handle.state.lock().unwrap().revision); + let restored = PendingSubmissionHandle { + state: Arc::new(Mutex::new(checkpoint)), + writer: handle.writer.clone(), + }; + assert!( + !restored + .accept_notification("later".into(), "later".into()) + .unwrap() + ); + assert!( + restored + .accept("submit-2".into(), vec![Segment::text("queued")], false) + .unwrap() + .activation + .is_none() + ); + // Stage a newer batch before a delayed completion/retry of the old one. + let later = restored.prepare_notification_batch(); + assert_eq!(later.len(), 1); + assert!( + restored + .commit_notification( + identity, + SystemItem::Notification { + body: notification.message.clone(), + message: notification.message.clone(), + prompt_provenance: None, + }, + Some(notification.provenance.clone()) + ) + .unwrap() + .is_none(), + "retry must not duplicate durable notification" + ); + assert_eq!( + restored + .state + .lock() + .unwrap() + .activating_notifications + .len(), + 1, + "an old batch completion must not clear a concurrently staged notification" + ); + assert_eq!(restored.persisted_entries_for_test().len(), entries.len()); + } + + #[tokio::test] + async fn notification_partial_drain_retries_only_uncommitted_entries() { + use agen::interceptor::{Interceptor, PendingHistoryAppendsContext}; + let temp = tempfile::tempdir().unwrap(); + let original = PendingSubmissionHandle::for_test(temp.path()); + let store = FailRunTerminalStore { + inner: original.writer.store.clone(), + fail_run_terminal: Arc::new(AtomicBool::new(false)), + invocation_failure: Arc::new(AtomicUsize::new(5)), + }; + let handle = PendingSubmissionHandle { + state: original.state.clone(), + writer: LogWriterHandle { + store: store.clone(), + state: original.writer.state.clone(), + sink: original.writer.sink.clone(), + in_flight: None, + }, + }; + let buffer = NotifyBuffer::new(); + for message in ["first", "second"] { + handle + .accept_notification(message.into(), message.into()) + .unwrap(); + } + for (notification, identity) in handle.prepare_notification_batch() { + buffer.push_durable_notify(notification.message, notification.provenance, identity); + } + let history = Arc::new(Mutex::new(VecDeque::new())); + let interceptor = WorkerInterceptor::new_with_history_queue( + Arc::new(HookRegistryBuilder::new().build()), + None, + None, + buffer.clone(), + Arc::new(Mutex::new(Vec::new())), + Arc::new(ArcSwap::from(PromptCatalog::builtins_only().unwrap())), + Some(Arc::new(handle.writer.clone())), + history.clone(), + ) + .with_notification_committer(Some(Arc::new(handle.clone()))); + assert!( + interceptor + .pending_history_appends(PendingHistoryAppendsContext { + invocation: Default::default(), + history: &[] + }) + .await + .is_err() + ); + assert_eq!( + handle.state.lock().unwrap().activating_notifications.len(), + 1 + ); + assert_eq!(history.lock().unwrap().len(), 1); + // New acceptance between failure and retry must survive both commits. + handle + .accept_notification("later".into(), "later".into()) + .unwrap(); + let appends = interceptor + .pending_history_appends(PendingHistoryAppendsContext { + invocation: Default::default(), + history: &[], + }) + .await + .unwrap(); + assert_eq!(appends.len(), 1); + assert_eq!(history.lock().unwrap().len(), 2); + let entries = original.persisted_entries_for_test(); + assert_eq!(entries.iter().filter(|entry| matches!(entry, LogEntry::AnnotatedSystemItem { entry, .. } if matches!(entry.item, SystemItem::Notification { .. }))).count(), 2); + let LogEntry::AnnotatedSystemItem { extensions, .. } = entries.last().unwrap() else { + panic!("expected notification") + }; + let checkpoint: PendingActivationState = + serde_json::from_value(extensions[0].payload.clone()).unwrap(); + assert_eq!(checkpoint.notification_receipts.len(), 3); + assert_eq!(checkpoint.pending_notifications.len(), 1); + assert!(checkpoint.activating_notifications.is_empty()); + assert!(buffer.is_empty()); + } + + #[tokio::test] + async fn notification_partial_drain_preserves_live_and_restored_history() { + let temp = tempfile::tempdir().unwrap(); + let store = FailRunTerminalStore { + inner: session_store::FsStore::new(temp.path()).unwrap(), + fail_run_terminal: Arc::new(AtomicBool::new(false)), + invocation_failure: Arc::new(AtomicUsize::new(5)), + }; + let client = PauseResumeClient { + calls: Arc::new(AtomicUsize::new(1)), + }; + let mut worker = Worker::new( + minimal_manifest(), + Engine::<_, Mutable, SessionHistoryMetadata>::new_annotated(client.clone()), + store.clone(), + WorkerWorkspaceContext::no_workspace(), + WorkerFilesystemAuthority::None, + Scope::empty(), + ) + .await + .unwrap(); + worker.attach_log_writer(Arc::new(worker.log_writer_handle())); + let pending = worker.pending_submission_handle(); + for message in ["first", "second"] { + pending + .accept_notification(message.into(), message.into()) + .unwrap(); + } + for (notification, identity) in pending.prepare_notification_batch() { + worker.pending_notifies.push_durable_notify( + notification.message, + notification.provenance, + identity, + ); + } + let result = worker + .run_for_notification(protocol::InvokeKind::Notify) + .await; + assert!( + matches!(result, Ok(WorkerRunResult::Interrupted { .. }) | Err(_)), + "{result:?}" + ); + assert_eq!( + client.calls.load(Ordering::SeqCst), + 1, + "failed drain must not call the provider" + ); + let history = worker + .session + .history() + .entries() + .iter() + .find(|entry| { + entry + .item + .as_text() + .is_some_and(|text| text.contains("first")) + }) + .unwrap() + .clone(); + worker + .run_for_notification(protocol::InvokeKind::Notify) + .await + .unwrap(); + assert_eq!(client.calls.load(Ordering::SeqCst), 2); + assert_eq!( + worker + .session + .history() + .entries() + .iter() + .filter(|entry| entry.annotation.entry_id == history.annotation.entry_id) + .count(), + 1 + ); + let entries = store + .read_all(worker.session_id(), worker.segment_id()) + .unwrap(); + let restored = + restore_history_entries(worker.session_id(), worker.segment_id(), &entries).unwrap(); + assert_eq!( + restored + .iter() + .filter(|entry| entry.annotation.entry_id == history.annotation.entry_id) + .count(), + 1 + ); + assert_eq!(entries.iter().filter(|entry| matches!(entry, LogEntry::AnnotatedSystemItem { entry, .. } if matches!(entry.item, SystemItem::Notification { .. }))).count(), 2); + assert!( + pending + .state + .lock() + .unwrap() + .activating_notifications + .is_empty() + ); + } + + #[tokio::test] + async fn invocation_fence_suspends_automatic_notification_until_fresh_input() { + for scenario in ["failed", "unknown", "incomplete"] { + let temp = tempfile::tempdir().unwrap(); + let store = FailRunTerminalStore { + inner: session_store::FsStore::new(temp.path()).unwrap(), + fail_run_terminal: Arc::new(AtomicBool::new(false)), + invocation_failure: Arc::new(AtomicUsize::new(if scenario == "unknown" { + 2 + } else { + 0 + })), + }; + let client = PauseResumeClient { + calls: Arc::new(AtomicUsize::new(1)), + }; + let mut worker = Worker::new( + minimal_manifest(), + Engine::<_, Mutable, SessionHistoryMetadata>::new_annotated(client.clone()), + store, + WorkerWorkspaceContext::no_workspace(), + WorkerFilesystemAuthority::None, + Scope::empty(), + ) + .await + .unwrap(); + worker.attach_log_writer(Arc::new(worker.log_writer_handle())); + let effects = Arc::new(AtomicUsize::new(0)); + install_recovery_invocation(&mut worker, effects.clone()); + let pending = worker.pending_submission_handle(); + pending + .accept_notification("held-notify".into(), "held notification".into()) + .unwrap(); + for (notification, identity) in pending.prepare_notification_batch() { + worker.pending_notifies.push_durable_notify( + notification.message, + notification.provenance, + identity, + ); + } + let mut input = recovery_invocation(scenario); + if scenario == "failed" { + let Segment::FeatureInvoke { invocation } = &mut input else { + unreachable!() + }; + invocation.name = "invalid-name".into(); + } + if scenario == "incomplete" { + assert!( + tokio::time::timeout(Duration::from_millis(20), worker.run(vec![input])) + .await + .is_err() + ); + } else { + assert!(worker.run(vec![input]).await.is_err()); + } + let before = worker.sink().subscribe_with_snapshot().0; + for _ in 0..4 { + assert!(!worker.can_schedule_notification_run(), "{scenario}"); + assert!( + worker + .run_for_notification(protocol::InvokeKind::Notify) + .await + .is_err() + ); + } + assert_eq!( + worker.sink().subscribe_with_snapshot().0.len(), + before.len() + ); + assert_eq!(client.calls.load(Ordering::SeqCst), 1); + assert!(!worker.pending_notifies.is_empty()); + assert_eq!( + effects.load(Ordering::SeqCst), + usize::from(scenario != "failed") + ); + worker + .run(vec![Segment::text("explicit fresh input")]) + .await + .unwrap(); + assert!(worker.can_schedule_notification_run()); + assert!(worker.pending_notifies.is_empty()); + assert_eq!(client.calls.load(Ordering::SeqCst), 2); + assert_eq!( + effects.load(Ordering::SeqCst), + usize::from(scenario != "failed"), + "no unknown effect retry" + ); + } + } + + #[derive(Clone)] + struct RecoveryInvocationFeature { + calls: Arc, + sink: SegmentLogSink, + } + + fn recovery_invocation_descriptor() -> protocol::FeatureInvocationDescriptor { + protocol::FeatureInvocationDescriptor { + identity: protocol::FeatureInvocationIdentity("builtin:recovery-test/prepare".into()), + name: "prepare".into(), + aliases: Vec::new(), + display_name: "Prepare".into(), + description: "Test recovery".into(), + syntax: protocol::FeatureInvocationSyntax::Parenthesized, + arguments: Vec::new(), + client_adapter: None, + } + } + + fn recovery_invocation(id: &str) -> Segment { + Segment::FeatureInvoke { + invocation: protocol::FeatureInvocation { + invocation_id: id.into(), + identity: recovery_invocation_descriptor().identity, + name: "prepare".into(), + arguments: Vec::new(), + }, + } + } + + impl crate::feature::FeatureModule for RecoveryInvocationFeature { + fn descriptor(&self) -> crate::feature::FeatureDescriptor { + crate::feature::FeatureDescriptor::builtin("recovery-test", "Recovery test") + .with_chat_invocation(recovery_invocation_descriptor()) + } + fn install( + &self, + context: &mut crate::feature::FeatureInstallContext<'_>, + ) -> Result<(), crate::feature::FeatureInstallError> { + context + .chat_invocations() + .register(recovery_invocation_descriptor(), self.clone()) + } + } + + #[async_trait] + impl crate::feature::FeatureInvocationHandler for RecoveryInvocationFeature { + async fn invoke( + &self, + context: crate::feature::FeatureInvocationContext, + invocation: &protocol::FeatureInvocation, + ) -> Result + { + let entries = self.sink.subscribe_with_snapshot().0; + let start = entries + .iter() + .rev() + .find_map(|entry| match entry { + LogEntry::Extension { + domain, payload, .. + } if domain == SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN => { + serde_json::from_value::(payload.clone()).ok() + } + _ => None, + }) + .expect("durable start must precede handler"); + assert!( + start + .invocation_receipts + .iter() + .any(|receipt| { receipt.matches(invocation) && receipt.result.is_none() }) + ); + assert_eq!(context.invocation_id, invocation.invocation_id); + self.calls.fetch_add(1, Ordering::SeqCst); + if invocation.invocation_id == "incomplete" { + futures::future::pending::<()>().await; + } + Ok(protocol::FeatureInvocationResult { + invocation_id: invocation.invocation_id.clone(), + identity: invocation.identity.clone(), + status: protocol::FeatureInvocationStatus::Succeeded, + message: "done".into(), + context: Some("prepared recovery context".into()), + }) + } + } + + fn install_recovery_invocation( + worker: &mut Worker, + calls: Arc, + ) { + let feature = RecoveryInvocationFeature { + calls, + sink: worker.sink(), + }; + assert!( + !worker + .install_features(FeatureRegistryBuilder::new().with_module(feature)) + .has_errors() + ); + } + + #[derive(Clone)] + struct ResidentInvocationClient { + sink: Arc>>, + requests: Arc, + } + + #[async_trait] + impl LlmClient for ResidentInvocationClient { + async fn stream( + &self, + request: agen::llm_client::Request, + ) -> Result< + std::pin::Pin< + Box< + dyn futures::Stream< + Item = Result< + agen::llm_client::event::Event, + agen::llm_client::ClientError, + >, + > + Send, + >, + >, + agen::llm_client::ClientError, + > { + let entries = self + .sink + .lock() + .unwrap() + .as_ref() + .unwrap() + .subscribe_with_snapshot() + .0; + let preparation = entries + .iter() + .position(|entry| { + matches!(entry, + LogEntry::AnnotatedSystemItem { entry, .. } + if matches!(entry.item, SystemItem::FeatureInvocationResult { .. }) + ) + }) + .expect("invocation result must be durable before the model request"); + let behavior = entries + .iter() + .position(|entry| { + matches!(entry, + LogEntry::AnnotatedSystemItem { entry, .. } + if matches!(entry.item, SystemItem::SubjectBehaviorRefresh { .. }) + ) + }) + .expect("Subject behavior refresh must be durable before the model request"); + assert!(preparation < behavior); + let request = format!("{request:?}"); + assert!(request.contains("prepared recovery context")); + assert!(request.contains("restore behavior")); + assert!(request.contains("literal /prepare()")); + self.requests.fetch_add(1, Ordering::SeqCst); + Ok(Box::pin(futures::stream::iter([Ok( + agen::llm_client::event::Event::Status(agen::llm_client::event::StatusEvent { + status: agen::llm_client::event::ResponseStatus::Completed, + }), + )]))) + } + + fn clone_boxed(&self) -> Box { + Box::new(self.clone()) + } + } + + #[tokio::test] + async fn feature_invocation_and_subject_refresh_remain_append_only_across_restore_replay() { + let runtime = tempfile::tempdir().unwrap(); + let _sandbox = worker_allocation::test_util::RuntimeDirSandbox::new(runtime.path()); + let dir = tempfile::tempdir().unwrap(); + let cwd = dir.path().join("workspace"); + std::fs::create_dir_all(&cwd).unwrap(); + let mut manifest = minimal_manifest(); + manifest.scope.allow[0].target = cwd.clone(); + manifest.feature.subjektiv.profile.enabled = true; + manifest + .feature + .subjektiv + .bind_workspace_settings(manifest::WorkspaceMemorySettingsSnapshot { + workspace_id: "workspace-test".into(), + settings_revision: 1, + language: "English".into(), + }) + .unwrap(); + let client = ResidentInvocationClient { + sink: Arc::default(), + requests: Arc::default(), + }; + let mut engine = + Engine::<_, Mutable, SessionHistoryMetadata>::new_annotated(client.clone()); + engine.set_system_prompt("fixed initial system prompt"); + let store = session_store::CombinedStore::new( + session_store::FsStore::new(dir.path().join("sessions")).unwrap(), + session_store::FsWorkerStore::new(dir.path().join("workers")).unwrap(), + ); + let authority = WorkerFilesystemAuthority::local(cwd.clone(), cwd.clone()); + let mut original = Worker::new( + manifest.clone(), + engine, + store.clone(), + WorkerWorkspaceContext::no_workspace(), + authority.clone(), + Scope::writable(&cwd).unwrap(), + ) + .await + .unwrap(); + let calls = Arc::new(AtomicUsize::new(0)); + let behavior_revision = Arc::new(AtomicUsize::new(1)); + let resident_source = || { + crate::feature::builtin::memory::ordinary_subjektiv_resident_summary_source( + &manifest, + Arc::new(RestoreSubjectResidentClient { + availability: memory::backend::MemoryResidentSummaryAvailability::Ready, + content: Some("literal /prepare()".into()), + load_count: Arc::default(), + behavior_revision: Some(behavior_revision.clone()), + }), + Arc::new(ArcSwap::from(PromptCatalog::builtins_only().unwrap())), + ) + .unwrap() + .unwrap() + }; + original.attach_log_writer(Arc::new(original.log_writer_handle())); + let source = resident_source(); + let report = original.install_features( + FeatureRegistryBuilder::new() + .with_module(RecoveryInvocationFeature { + calls: calls.clone(), + sink: original.sink(), + }) + .with_module( + crate::feature::builtin::memory::SubjektivResidentRestoreRefreshFeature::new( + source.clone(), + original.prompts(), + "workspace-test", + ), + ), + ); + assert!(!report.has_errors(), "{report:?}"); + original.install_system_prompt_contribution(Some(source), None); + original.set_system_prompt_template( + SystemPromptTemplate::parse( + "default", + crate::prompt::source::PromptCatalogSource::builtins_only(), + ) + .unwrap(), + ); + original.materialize_durable_session_head().await.unwrap(); + behavior_revision.store(2, Ordering::SeqCst); + *client.sink.lock().unwrap() = Some(original.sink()); + let pending = original.pending_submission_handle(); + pending + .accept_notification( + "composed-notify".into(), + "resident context notification".into(), + ) + .unwrap(); + for (notification, extension) in pending.prepare_notification_batch() { + original.pending_notifies.push_durable_notify( + notification.message, + notification.provenance, + extension, + ); + } + let input = vec![recovery_invocation("composed")]; + pending + .accept("composed-submit".into(), input.clone(), false) + .unwrap(); + let Some(PendingActivation::Submission(submission)) = + pending.prepare_next_activation(None).unwrap() + else { + panic!("expected submission") + }; + let (sender, mut receiver) = tokio::sync::oneshot::channel(); + let result = original + .run_with_input_extensions_and_commit_hook( + submission.input, + vec![pending.activation_extension()], + submission.provenance, + crate::controller::input_committed_hook( + pending.clone(), + submission.submission_id, + sender, + ), + ) + .await + .unwrap(); + // No Controller receiver was polled while the invocation/refresh ran. + assert!(receiver.try_recv().is_ok()); + assert!(pending.state.lock().unwrap().activating.is_none()); + assert_eq!( + client.requests.load(Ordering::SeqCst), + 1, + "{result:?}; history={:?}", + original.history() + ); + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "resident text must not invoke a handler" + ); + let history_before = original.history().to_vec(); + let session_id = original.session_id(); + let segment_id = original.segment_id(); + drop(original); + + let mut restored = + Worker::, _>::restore_from_manifest_with_context_and_model_client( + session_id, + segment_id, + manifest.clone(), + store, + crate::prompt::source::PromptCatalogSource::builtins_only(), + WorkerWorkspaceContext::no_workspace(), + authority, + Some(Box::new(client.clone())), + ) + .await + .unwrap(); + restored.attach_log_writer(Arc::new(restored.log_writer_handle())); + assert!( + restored + .pending_submission_handle() + .prepare_next_activation(None) + .unwrap() + .is_none(), + "notification/invocation checkpoints must not resurrect committed input after restore" + ); + assert_eq!( + restored + .session + .pending_activations + .lock() + .unwrap() + .invocation_receipts + .len(), + 1 + ); + assert_eq!( + restored + .session + .pending_activations + .lock() + .unwrap() + .notification_receipts + .len(), + 1 + ); + let source = resident_source(); + let report = restored.install_features( + FeatureRegistryBuilder::new() + .with_module(RecoveryInvocationFeature { + calls: calls.clone(), + sink: restored.sink(), + }) + .with_module( + crate::feature::builtin::memory::SubjektivResidentRestoreRefreshFeature::new( + source.clone(), + restored.prompts(), + "workspace-test", + ), + ), + ); + assert!(!report.has_errors(), "{report:?}"); + restored.install_system_prompt_contribution(Some(source), None); + *client.sink.lock().unwrap() = Some(restored.sink()); + restored.run(input).await.unwrap(); + assert_eq!(client.requests.load(Ordering::SeqCst), 2); + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "restore/replay must not repeat the business effect" + ); + assert_eq!(&restored.history()[..history_before.len()], history_before); + let entries = restored.sink().subscribe_with_snapshot().0; + assert_eq!( + entries + .iter() + .filter(|entry| matches!(entry, + LogEntry::AnnotatedSystemItem { entry, .. } + if matches!(entry.item, SystemItem::FeatureInvocationResult { .. }) + )) + .count(), + 2 // Explicit replay displays the cached result; it does not repeat the effect. + ); + assert!(entries.iter().any(|entry| matches!(entry, + LogEntry::AnnotatedSystemItem { entry, .. } + if matches!(entry.item, SystemItem::ResidentSummaryRefresh { .. }) + ))); + } + + #[tokio::test] + async fn feature_invocation_restart_reuses_completed_results_and_fences_incomplete_starts() { + let runtime = tempfile::tempdir().unwrap(); + let _sandbox = worker_allocation::test_util::RuntimeDirSandbox::new(runtime.path()); + for id in ["completed", "incomplete"] { + let dir = tempfile::tempdir().unwrap(); + let cwd = dir.path().join("workspace"); + std::fs::create_dir_all(&cwd).unwrap(); + let mut engine = + Engine::<_, Mutable, SessionHistoryMetadata>::new_annotated(NoopClient); + engine.set_system_prompt("recovery test system prompt"); + let mut manifest = minimal_manifest(); + manifest.scope.allow[0].target = cwd.clone(); + let mut original = Worker::new( + manifest, + engine, + session_store::FsStore::new(dir.path().join("sessions")).unwrap(), + WorkerWorkspaceContext::local_filesystem(None), + WorkerFilesystemAuthority::local(cwd.clone(), cwd.clone()), + Scope::writable(&cwd).unwrap(), + ) + .await + .unwrap(); + let calls = Arc::new(AtomicUsize::new(0)); + install_recovery_invocation(&mut original, calls.clone()); + let input = vec![recovery_invocation(id)]; + if id == "incomplete" { + assert!( + tokio::time::timeout(Duration::from_millis(20), original.run(input.clone())) + .await + .is_err() + ); + } else { + let _ = original.run(input.clone()).await; + let entries = original.sink().subscribe_with_snapshot().0; + assert!(entries.iter().any(|entry| match entry { + LogEntry::AnnotatedSystemItem { + entry, extensions, .. + } => { + matches!(entry.item, SystemItem::FeatureInvocationResult { .. }) + && extensions.iter().any(|extension| { + extension.domain == SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN + && serde_json::from_value::( + extension.payload.clone(), + ) + .unwrap() + .invocation_receipts + .iter() + .any(|receipt| receipt.result.is_some()) + }) + } + _ => false, + })); + } + assert_eq!(calls.load(Ordering::SeqCst), 1); + let session = original.session_id(); + let segment = original.segment_id(); + let store = session_store::CombinedStore::new( + original.store.clone(), + session_store::FsWorkerStore::new(dir.path().join("workers")).unwrap(), + ); + let manifest = original.manifest.clone(); + drop(original); + let cwd = dir.path().join("workspace"); + let mut restored = Worker::, _>::restore_from_manifest_with_context_and_model_client( + session, segment, manifest, store, + crate::prompt::source::PromptCatalogSource::builtins_only(), + WorkerWorkspaceContext::local_filesystem(None), + WorkerFilesystemAuthority::local(cwd.clone(), cwd), + Some(Box::new(NoopClient)), + ).await.unwrap(); + install_recovery_invocation(&mut restored, calls.clone()); + if id == "incomplete" { + assert!(restored.resume().await.is_err()); + assert!( + restored + .run_for_notification(protocol::InvokeKind::Notify) + .await + .is_err() + ); + assert!(restored.run(input).await.is_err()); + let last = restored + .session + .pending_activations + .lock() + .unwrap() + .invocation_receipts[0] + .result + .clone() + .unwrap(); + assert_eq!( + last.status, + protocol::FeatureInvocationStatus::OutcomeUnknown + ); + } else { + let _ = restored.run(input).await; + assert!(restored.ensure_invocation_preparation_complete().is_ok()); + } + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "restore/replay must not repeat business execution" + ); + } + } + + #[tokio::test] + async fn feature_invocation_rewind_atomic_replacement_preserves_recovery_on_disk_restart() { + let runtime = tempfile::tempdir().unwrap(); + let _sandbox = worker_allocation::test_util::RuntimeDirSandbox::new(runtime.path()); + for queued in [false, true] { + for scenario in ["completed", "incomplete", "result-fault", "corrupt"] { + for replacement_fault in [0, 1, 3, 4] { + let dir = tempfile::tempdir().unwrap(); + let cwd = dir.path().join("workspace"); + std::fs::create_dir_all(&cwd).unwrap(); + let store = FailRunTerminalStore { + inner: session_store::FsStore::new(dir.path().join("sessions")).unwrap(), + fail_run_terminal: Arc::new(AtomicBool::new(false)), + invocation_failure: Arc::new(AtomicUsize::new(0)), + }; + let mut engine = + Engine::<_, Mutable, SessionHistoryMetadata>::new_annotated(NoopClient); + engine.set_system_prompt("recovery rewind test system prompt"); + let mut manifest = minimal_manifest(); + manifest.scope.allow[0].target = cwd.clone(); + let mut original = Worker::new( + manifest, + engine, + store.clone(), + WorkerWorkspaceContext::local_filesystem(None), + WorkerFilesystemAuthority::local(cwd.clone(), cwd.clone()), + Scope::writable(&cwd).unwrap(), + ) + .await + .unwrap(); + let calls = Arc::new(AtomicUsize::new(0)); + install_recovery_invocation(&mut original, calls.clone()); + let input = vec![recovery_invocation(scenario)]; + if scenario == "corrupt" { + let _ = original.run(vec![Segment::text("rewind target")]).await; + let payload = serde_json::json!({"invocation_receipts": "corrupt"}); + original + .commit_entry(LogEntry::Extension { + ts: segment_log::now_millis(), + domain: SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN.into(), + payload: payload.clone(), + }) + .unwrap(); + original.session.restore_pending_activations(&[( + SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN.into(), + payload, + )]); + } else { + if scenario == "result-fault" { + store.invocation_failure.store(2, Ordering::SeqCst); + } + let (run_input, extensions, provenance, activation) = if queued { + // Exercise the durable admission/activation path used + // by Controller, in addition to direct Worker.run. + let handle = original.pending_submission_handle(); + let accepted = handle + .accept("request-1".into(), input.clone(), true) + .unwrap(); + let pending = accepted.activation.unwrap(); + let id = pending.submission_id.clone(); + ( + pending.input, + vec![handle.activation_extension()], + pending.provenance, + Some((handle, id)), + ) + } else { + ( + input.clone(), + Vec::new(), + WorkerHistoryProvenance::LegacyUnknown, + None, + ) + }; + let run = original.run_with_input_extensions_and_commit_hook( + run_input, + extensions, + provenance, + move || { + if let Some((handle, id)) = activation { + handle.finish_activation(&id); + } + }, + ); + if scenario == "incomplete" { + assert!( + tokio::time::timeout(Duration::from_millis(20), run) + .await + .is_err() + ); + } else { + let _ = run.await; + } + assert_eq!(calls.load(Ordering::SeqCst), 1); + } + let pending = original.session.pending_activations.lock().unwrap().clone(); + if !queued { + assert!(pending.pending.is_empty()); + assert!(pending.activating.is_none()); + assert!(pending.receipts.is_empty()); + assert!(pending.notification_receipts.is_empty()); + assert!(pending.pending_notifications.is_empty()); + assert!(pending.activating_notifications.is_empty()); + } + let (head, targets) = original.list_rewind_targets().unwrap(); + let target = targets.last().unwrap(); + let old = store + .read_all(original.session_id(), original.segment_id()) + .unwrap(); + let old_mirror = original.sink.subscribe_with_snapshot().0; + let old_count = original.segment_state.entries_written(); + store + .invocation_failure + .store(replacement_fault, Ordering::SeqCst); + let applied = original.rewind_to(target.id.clone(), head).await; + let disk = store + .read_all(original.session_id(), original.segment_id()) + .unwrap(); + if replacement_fault == 3 { + assert!(applied.is_err()); + assert_eq!( + serde_json::to_value(&disk).unwrap(), + serde_json::to_value(&old).unwrap() + ); + assert_eq!( + serde_json::to_value(original.sink.subscribe_with_snapshot().0) + .unwrap(), + serde_json::to_value(old_mirror).unwrap() + ); + assert_eq!(original.segment_state.entries_written(), old_count); + } else { + if replacement_fault <= 1 { + // An injected checkpoint-append fault is no longer + // relevant: rewind must never append after truncation. + assert_eq!( + store.invocation_failure.load(Ordering::SeqCst), + replacement_fault + ); + let applied = applied.unwrap(); + assert_eq!( + applied.summary.truncated_to_entries, + target.truncate_entries + ); + assert_eq!( + applied.summary.discarded_entries, + head - target.truncate_entries + ); + assert_eq!(applied.entries.len(), target.truncate_entries); + assert_eq!(original.segment_state.entries_written(), disk.len()); + assert_eq!( + serde_json::to_value(original.sink.subscribe_with_snapshot().0) + .unwrap(), + serde_json::to_value(&disk).unwrap() + ); + } else { + assert!(applied.is_err()); + } + assert_eq!(disk.len(), target.truncate_entries + 1); + assert_eq!( + serde_json::to_value(&disk[..target.truncate_entries]).unwrap(), + serde_json::to_value(&old[..target.truncate_entries]).unwrap() + ); + let LogEntry::Extension { + domain, payload, .. + } = disk.last().unwrap() + else { + panic!("complete replacement must include recovery checkpoint") + }; + assert_eq!(domain, SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN); + assert_eq!(*payload, serde_json::to_value(&pending).unwrap()); + } + let session = original.session_id(); + let segment = original.segment_id(); + let manifest = original.manifest.clone(); + drop(original); + // Reopen from the path: no retained in-memory queue/Store state. + let reopened = session_store::CombinedStore::new( + session_store::FsStore::new(dir.path().join("sessions")).unwrap(), + session_store::FsWorkerStore::new(dir.path().join("workers")).unwrap(), + ); + let mut restored = Worker::, _>::restore_from_manifest_with_context_and_model_client( + session, segment, manifest, reopened, + crate::prompt::source::PromptCatalogSource::builtins_only(), + WorkerWorkspaceContext::local_filesystem(None), + WorkerFilesystemAuthority::local(cwd.clone(), cwd), + Some(Box::new(NoopClient)), + ).await.unwrap(); + install_recovery_invocation(&mut restored, calls.clone()); + let replay = restored.run(input).await; + if scenario == "completed" { + assert!(restored.ensure_invocation_preparation_complete().is_ok()); + } else { + assert!(replay.is_err()); + assert!(restored.resume().await.is_err()); + assert!( + restored + .run_for_notification(protocol::InvokeKind::Notify) + .await + .is_err() + ); + } + assert_eq!( + calls.load(Ordering::SeqCst), + usize::from(scenario != "corrupt"), + "queued={queued} scenario={scenario} replacement_fault={replacement_fault}" + ); + } + } + } + } + + #[tokio::test] + async fn feature_invocation_queue_receipts_prevent_activation_and_business_replay() { + let (_dir, mut worker) = rewind_test_worker().await; + let calls = Arc::new(AtomicUsize::new(0)); + install_recovery_invocation(&mut worker, calls.clone()); + let input = vec![recovery_invocation("queued")]; + let handle = worker.pending_submission_handle(); + let accepted = handle + .accept("request-1".into(), input.clone(), false) + .unwrap(); + let repeated = handle + .accept("request-1".into(), input.clone(), false) + .unwrap(); + assert_eq!(accepted.submission_id, repeated.submission_id); + assert_eq!(handle.snapshot().submissions.len(), 1); + let PendingActivation::Submission(pending) = + handle.prepare_next_activation(None).unwrap().unwrap(); + let id = pending.submission_id.clone(); + let extension = handle.activation_extension(); + let finished = handle.clone(); + let _ = worker + .run_with_input_extensions_and_commit_hook( + pending.input, + vec![extension], + pending.provenance, + move || finished.finish_activation(&id), + ) + .await; + assert_eq!(calls.load(Ordering::SeqCst), 1); + let replay = handle + .accept("request-1".into(), input.clone(), false) + .unwrap(); + assert!(replay.activation.is_none()); + assert!(handle.prepare_next_activation(None).unwrap().is_none()); + let _ = worker.run(input).await; + assert_eq!(calls.load(Ordering::SeqCst), 1); + } + + #[tokio::test] + async fn feature_invocation_start_and_result_append_failures_fence_execution_and_replay() { + for failure in [1, 2] { + let dir = tempfile::tempdir().unwrap(); + let store = FailRunTerminalStore { + inner: session_store::FsStore::new(dir.path().join("sessions")).unwrap(), + fail_run_terminal: Arc::new(AtomicBool::new(false)), + invocation_failure: Arc::new(AtomicUsize::new(failure)), + }; + let mut worker = Worker::new( + minimal_manifest(), + Engine::<_, Mutable, SessionHistoryMetadata>::new_annotated(NoopClient), + store, + WorkerWorkspaceContext::no_workspace(), + WorkerFilesystemAuthority::None, + Scope::empty(), + ) + .await + .unwrap(); + let calls = Arc::new(AtomicUsize::new(0)); + install_recovery_invocation(&mut worker, calls.clone()); + let input = vec![recovery_invocation("fault")]; + assert!(worker.run(input.clone()).await.is_err()); + assert_eq!( + calls.load(Ordering::SeqCst), + if failure == 1 { 0 } else { 1 } + ); + assert!(worker.resume().await.is_err()); + if failure == 1 { + // No start became durable and no effect ran; an explicit fresh + // submission may execute after the append fault is corrected. + let _ = worker.run(input).await; + } else { + // Effect ran but result did not become durable. Do not invoke + // the handler a second time even when the Store is healthy. + assert!(worker.run(input).await.is_err()); + let result = worker + .session + .pending_activations + .lock() + .unwrap() + .invocation_receipts[0] + .result + .clone() + .unwrap(); + assert_eq!( + result.status, + protocol::FeatureInvocationStatus::OutcomeUnknown + ); + } + assert_eq!(calls.load(Ordering::SeqCst), 1); + } + } + + #[tokio::test] + async fn feature_invocation_corrupt_checkpoint_fails_closed() { + let (_dir, mut worker) = rewind_test_worker().await; + let calls = Arc::new(AtomicUsize::new(0)); + install_recovery_invocation(&mut worker, calls.clone()); + worker.session.restore_pending_activations(&[( + SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN.into(), + serde_json::json!({"invocation_receipts": "corrupt"}), + )]); + assert!(worker.run(vec![recovery_invocation("new")]).await.is_err()); + assert_eq!(calls.load(Ordering::SeqCst), 0); + } + + #[test] + fn feature_invocation_receipts_retain_digest_not_arguments_and_detect_payload_changes() { + let Segment::FeatureInvoke { mut invocation } = recovery_invocation("large") else { + unreachable!() + }; + invocation.arguments = (0..32) + .map(|index| protocol::InvocationArgumentValue { + name: format!("arg{index}"), + value: protocol::InvocationValue::String("private-argument".repeat(1024)), + }) + .collect(); + let receipt = InvocationReceipt::new(&invocation, None); + assert!(receipt.matches(&invocation)); + assert!(receipt.is_valid()); + let serialized = serde_json::to_vec(&receipt).unwrap(); + assert!(serialized.len() < 512); + assert!( + !String::from_utf8(serialized) + .unwrap() + .contains("private-argument") + ); + let mut altered = invocation.clone(); + altered.arguments[0].value = protocol::InvocationValue::String("different".into()); + assert!(!receipt.matches(&altered)); + altered = invocation.clone(); + altered.name = "alias".into(); + assert!(!receipt.matches(&altered)); + altered = invocation.clone(); + altered.arguments.reverse(); + assert!(!receipt.matches(&altered)); + let state = PendingActivationState { + invocation_receipts: vec![receipt; MAX_INVOCATION_RECEIPTS], + ..PendingActivationState::default() + }; + assert!(serde_json::to_vec(&state).unwrap().len() < 512 * MAX_INVOCATION_RECEIPTS); + } + + #[test] + fn feature_invocation_structurally_valid_corrupt_receipts_fail_closed_on_restore() { + let Segment::FeatureInvoke { invocation } = recovery_invocation("corrupt") else { + unreachable!() + }; + let receipt = InvocationReceipt::new(&invocation, None); + let valid = serde_json::to_value(PendingActivationState { + invocation_receipts: vec![receipt], + ..PendingActivationState::default() + }) + .unwrap(); + let mut variants = Vec::new(); + let mut digest = valid.clone(); + digest["invocation_receipts"][0]["payload_digest"] = serde_json::json!("not-sha256"); + variants.push(digest); + let mut duplicate = valid.clone(); + duplicate["invocation_receipts"] + .as_array_mut() + .unwrap() + .push(valid["invocation_receipts"][0].clone()); + variants.push(duplicate); + let mut identity = valid.clone(); + identity["invocation_receipts"][0]["result"] = + serde_json::to_value(protocol::FeatureInvocationResult { + invocation_id: "foreign".into(), + identity: invocation.identity.clone(), + status: protocol::FeatureInvocationStatus::Succeeded, + message: "done".into(), + context: None, + }) + .unwrap(); + variants.push(identity); + let mut oversized = valid.clone(); + oversized["invocation_receipts"][0]["result"] = + serde_json::to_value(protocol::FeatureInvocationResult { + invocation_id: invocation.invocation_id.clone(), + identity: invocation.identity.clone(), + status: protocol::FeatureInvocationStatus::Succeeded, + message: "done".into(), + context: Some("x".repeat(16385)), + }) + .unwrap(); + variants.push(oversized); + let mut old_shape = valid; + old_shape["invocation_receipts"][0]["invocation"] = + serde_json::to_value(invocation).unwrap(); + variants.push(old_shape); + for payload in variants { + let mut session = WorkerSession::new(session_store::new_session_id(), Vec::new()); + session.restore_pending_activations(&[( + SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN.into(), + payload, + )]); + assert!( + session + .pending_activations + .lock() + .unwrap() + .invocation_recovery_blocked + ); + } + } + + #[tokio::test] + async fn feature_invocation_unknown_segments_rejected_at_queue_admission() { + let (_dir, worker) = rewind_test_worker().await; + let handle = worker.pending_submission_handle(); + for activate_now in [true, false] { + let before = handle.persisted_entries_for_test().len(); + assert!( + handle + .accept( + "unknown".into(), + vec![Segment::text("intent"), Segment::Unknown], + activate_now + ) + .is_err() + ); + assert_eq!(handle.persisted_entries_for_test().len(), before); + assert!(handle.snapshot().submissions.is_empty()); + } + } + #[test] fn restoring_an_in_flight_activation_requeues_it_at_the_fifo_head() { let mut session = WorkerSession::new(session_store::new_session_id(), Vec::new()); let state = PendingActivationState { revision: 4, next_activation_sequence: 3, + invocation_receipts: Vec::new(), + invocation_recovery_blocked: false, activating: Some(PendingSubmission { submission_request_id: "request-1".into(), source_namespace: "direct:test".into(), diff --git a/crates/worker/tests/controller_test.rs b/crates/worker/tests/controller_test.rs index c1b4668e4..f79ab3fdf 100644 --- a/crates/worker/tests/controller_test.rs +++ b/crates/worker/tests/controller_test.rs @@ -838,36 +838,126 @@ async fn feature_flags_default_to_core_tool_surface_only() { } #[tokio::test] -async fn backend_job_profile_exposes_only_structured_result_capability() { +async fn backend_job_capability_is_explicit_and_preserves_profile_features() { + for bound in [false, true] { + let workspace = tempfile::tempdir().unwrap(); + let mut resolved = ProfileResolver::new() + .with_workspace_base(workspace.path()) + .resolve( + &ProfileSelector::source_named(ProfileRegistrySource::Builtin, "backend-job"), + ProfileResolveOptions::with_worker_name("backend-job-worker"), + ) + .unwrap(); + // A custom instruction and enabled feature must survive trusted Job binding. + if bound { + resolved.manifest.engine.instruction = "default".into(); + resolved.manifest.feature.task.enabled = true; + } + let instruction = resolved.manifest.engine.instruction.clone(); + let client = MockClient::new(simple_text_events()); + let client_for_assert = client.clone(); + let (mut worker, _pwd) = make_worker_with_manifest_and_workspace_context( + client, + resolved.manifest, + WorkerWorkspaceContext::with_client(None, Arc::new(AvailableWorkspaceClient)), + ) + .await; + if bound { + worker + .bind_backend_job(worker::BackendJobExecutionBinding { + job_id: "job-1".into(), + attempt_id: "attempt-1".into(), + input_revision: Some("input-1".into()), + subjektiv_consolidation: false, + }) + .unwrap(); + } + assert_eq!(worker.manifest().engine.instruction, instruction); + let handle = spawn_controller(worker).await; + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "Run the bounded Backend Job.", + )) + .await + .unwrap(); + wait_for_status(&handle, WorkerStatus::Idle).await; + let request = wait_for_captured_request(&client_for_assert).await; + let names = request_tool_names(&request); + assert_eq!( + names.iter().any(|name| name == "SubmitBackendJobResult"), + bound + ); + assert_eq!(names.iter().any(|name| name == "TaskCreate"), bound); + } +} + +#[tokio::test] +async fn consolidation_job_preserves_custom_instruction_and_explicit_tool_policy() { let workspace = tempfile::tempdir().unwrap(); - let resolved = ProfileResolver::new() + let mut manifest = ProfileResolver::new() .with_workspace_base(workspace.path()) .resolve( - &ProfileSelector::source_named(ProfileRegistrySource::Builtin, "backend-job"), - ProfileResolveOptions::with_worker_name("backend-job-worker"), + &ProfileSelector::source_named( + ProfileRegistrySource::Builtin, + "subjektiv-memory-consolidation", + ), + ProfileResolveOptions::with_worker_name("custom-consolidation-job"), ) + .unwrap() + .manifest; + manifest.engine.instruction = "default".into(); + manifest.feature.task.enabled = true; + manifest + .feature + .subjektiv + .bind_workspace_settings(manifest::WorkspaceMemorySettingsSnapshot { + workspace_id: "workspace-1".into(), + settings_revision: 1, + language: "English".into(), + }) .unwrap(); let client = MockClient::new(simple_text_events()); - let client_for_assert = client.clone(); - let (worker, _pwd) = make_worker_with_manifest_and_workspace_context( + let captured = client.clone(); + let (mut worker, _pwd) = make_worker_with_manifest_and_workspace_context( client, - resolved.manifest, + manifest, WorkerWorkspaceContext::with_client(None, Arc::new(AvailableWorkspaceClient)), ) .await; + worker + .engine_mut() + .set_system_prompt("chosen custom instruction"); + worker + .bind_backend_job(worker::BackendJobExecutionBinding { + job_id: "job-1".into(), + attempt_id: "attempt-1".into(), + input_revision: None, + subjektiv_consolidation: true, + }) + .unwrap(); let handle = spawn_controller(worker).await; - handle .send(Method::submit_text( protocol::new_submission_request_id(), - "Run the bounded Backend Job.", + "Inspect the bounded batch.", )) .await .unwrap(); wait_for_status(&handle, WorkerStatus::Idle).await; - - let request = wait_for_captured_request(&client_for_assert).await; - assert_eq!(request_tool_names(&request), vec!["SubmitBackendJobResult"]); + let request = wait_for_captured_request(&captured).await; + assert_eq!( + request.system_prompt.as_deref(), + Some("chosen custom instruction") + ); + let tools = request_tool_names(&request); + for tool in [ + "SubmitBackendJobResult", + "MemoryApplyCandidate", + "TaskCreate", + ] { + assert!(tools.iter().any(|name| name == tool)); + } } #[tokio::test] diff --git a/crates/worker/tests/feature_invocation_test.rs b/crates/worker/tests/feature_invocation_test.rs new file mode 100644 index 000000000..a08cffda0 --- /dev/null +++ b/crates/worker/tests/feature_invocation_test.rs @@ -0,0 +1,613 @@ +//! Installed host-feature execution, not lexical slash-command dispatch. +use std::pin::Pin; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, Mutex}; + +use agen::Engine; +use agen::llm_client::event::{Event as LlmEvent, ResponseStatus, StatusEvent}; +use agen::llm_client::{ClientError, LlmClient, Request}; +use async_trait::async_trait; +use futures::Stream; +use protocol::{ + FeatureInvocation, FeatureInvocationDescriptor, FeatureInvocationIdentity, + FeatureInvocationResult, FeatureInvocationStatus, FeatureInvocationSyntax, + InvocationArgumentDescriptor, InvocationArgumentType, InvocationArgumentValue, + InvocationCompletion, InvocationValue, Segment, +}; +use session_store::{CombinedStore, FsStore, FsWorkerStore, LogEntry, SystemItem}; +use worker::feature::{ + FeatureDescriptor, FeatureInstallContext, FeatureInstallError, FeatureInvocationContext, + FeatureInvocationHandler, FeatureInvocationHandlerError, FeatureModule, FeatureRegistryBuilder, +}; +use worker::{Worker, WorkerFilesystemAuthority, WorkerManifest, WorkerWorkspaceContext}; + +#[derive(Clone)] +struct Trace { + calls: Arc>>, + requests: Arc>>, + available: Arc, + argument_allowed: Arc, +} + +impl Default for Trace { + fn default() -> Self { + Self { + calls: Arc::default(), + requests: Arc::default(), + available: Arc::new(AtomicBool::new(true)), + argument_allowed: Arc::new(AtomicBool::new(true)), + } + } +} + +#[async_trait] +impl LlmClient for Trace { + fn clone_boxed(&self) -> Box { + Box::new(self.clone()) + } + async fn stream( + &self, + request: Request, + ) -> Result> + Send>>, ClientError> + { + self.calls.lock().unwrap().push("llm".into()); + self.requests.lock().unwrap().push(request); + Ok(Box::pin(futures::stream::iter(vec![ + Ok(LlmEvent::text_block_start(0)), + Ok(LlmEvent::text_delta(0, "done")), + Ok(LlmEvent::text_block_stop(0, None)), + Ok(LlmEvent::Status(StatusEvent { + status: ResponseStatus::Completed, + })), + ]))) + } +} + +fn descriptor() -> FeatureInvocationDescriptor { + FeatureInvocationDescriptor { + identity: FeatureInvocationIdentity("builtin:fake/prepare".into()), + name: "prepare".into(), + aliases: vec!["prep".into()], + display_name: "Prepare".into(), + description: "Prepare test context".into(), + syntax: FeatureInvocationSyntax::Parenthesized, + arguments: vec![InvocationArgumentDescriptor { + name: "value".into(), + position: Some(0), + required: true, + value_type: InvocationArgumentType::String, + completion: InvocationCompletion::None, + description: None, + }], + client_adapter: None, + } +} + +#[derive(Clone)] +struct FakeFeature(Trace); +impl FeatureModule for FakeFeature { + fn descriptor(&self) -> FeatureDescriptor { + FeatureDescriptor::builtin("fake", "Fake").with_chat_invocation(descriptor()) + } + fn install(&self, context: &mut FeatureInstallContext<'_>) -> Result<(), FeatureInstallError> { + context + .chat_invocations() + .register(descriptor(), self.clone()) + } +} + +#[async_trait] +impl FeatureInvocationHandler for FakeFeature { + fn is_available(&self) -> bool { + self.0.available.load(Ordering::SeqCst) + } + + fn validate( + &self, + invocation: &FeatureInvocation, + ) -> Result<(), FeatureInvocationHandlerError> { + if invocation.arguments[0].value == InvocationValue::String("denied".into()) + || (invocation.arguments[0].value == InvocationValue::String("first".into()) + && !self.0.argument_allowed.load(Ordering::SeqCst)) + { + return Err(FeatureInvocationHandlerError::failed( + "host permission denied", + )); + } + Ok(()) + } + + async fn invoke( + &self, + context: FeatureInvocationContext, + invocation: &FeatureInvocation, + ) -> Result { + assert_eq!(context.invocation_id, invocation.invocation_id); + let InvocationValue::String(value) = &invocation.arguments[0].value else { + panic!("validated string") + }; + self.0.calls.lock().unwrap().push(value.clone()); + match value.as_str() { + "fail" => { + return Err(FeatureInvocationHandlerError::failed( + "test preparation failed", + )); + } + "unknown" => { + return Err(FeatureInvocationHandlerError::outcome_unknown( + "test effect uncertain", + )); + } + "revoke-during-effect" => self.0.available.store(false, Ordering::SeqCst), + "large-error" => return Err(FeatureInvocationHandlerError::failed("x".repeat(4097))), + _ => {} + } + Ok(FeatureInvocationResult { + invocation_id: if value == "wrong" { + "wrong-id".into() + } else { + invocation.invocation_id.clone() + }, + identity: invocation.identity.clone(), + status: FeatureInvocationStatus::Succeeded, + message: match value.as_str() { + "large-message" => "é".repeat(2049), + "limit-message" => "x".repeat(4096), + _ => "prepared".into(), + }, + context: Some(match value.as_str() { + "large-context" => "é".repeat(8193), + "limit-context" => "x".repeat(16384), + _ => format!("trusted-preparation-{value}"), + }), + }) + } +} + +fn invoke(id: &str, value: &str) -> Segment { + Segment::FeatureInvoke { + invocation: FeatureInvocation { + invocation_id: id.into(), + identity: descriptor().identity, + name: "prepare".into(), + arguments: vec![InvocationArgumentValue { + name: "value".into(), + value: InvocationValue::String(value.into()), + }], + }, + } +} + +type TestStore = CombinedStore; +async fn fixture(trace: Trace) -> (tempfile::TempDir, Worker) { + let dir = tempfile::tempdir().unwrap(); + let manifest = WorkerManifest::from_toml( + r#" +[worker] +name = "invocation-test" +[model] +scheme = "anthropic" +model_id = "test-model" +[engine] +max_tokens = 100 +[scope] +allow = [] +"#, + ) + .unwrap(); + let store = CombinedStore::new( + FsStore::new(dir.path().join("sessions")).unwrap(), + FsWorkerStore::new(dir.path().join("workers")).unwrap(), + ); + let mut worker = Worker::new( + manifest, + Engine::<_, agen::state::Mutable, worker::SessionHistoryMetadata>::new_annotated( + trace.clone(), + ), + store, + WorkerWorkspaceContext::no_workspace(), + WorkerFilesystemAuthority::None, + manifest::Scope::empty(), + ) + .await + .unwrap(); + let report = + worker.install_features(FeatureRegistryBuilder::new().with_module(FakeFeature(trace))); + assert!(!report.has_errors(), "{report:?}"); + (dir, worker) +} + +fn results(worker: &Worker) -> Vec { + worker + .sink() + .subscribe_with_snapshot() + .0 + .into_iter() + .filter_map(|entry| match entry { + LogEntry::AnnotatedSystemItem { entry, .. } => match entry.item { + SystemItem::FeatureInvocationResult { result, .. } => Some(result), + _ => None, + }, + _ => None, + }) + .collect() +} + +#[derive(Clone)] +struct TracePromptHook { + trace: Trace, + cancel: bool, +} +#[async_trait] +impl worker::hook::Hook for TracePromptHook { + async fn call( + &self, + _input: &worker::hook::PromptSubmitInfo, + ) -> Result { + self.trace.calls.lock().unwrap().push("prompt-hook".into()); + Ok(if self.cancel { + worker::hook::HookPromptAction::Cancel("test cancellation".into()) + } else { + worker::hook::HookPromptAction::Continue + }) + } +} + +#[tokio::test] +async fn installed_feature_invocations_execute_in_order_before_model_and_context_is_visible() { + let trace = Trace::default(); + let (_dir, mut worker) = fixture(trace.clone()).await; + worker.add_on_prompt_submit_hook(TracePromptHook { + trace: trace.clone(), + cancel: false, + }); + worker + .run(vec![ + invoke("one", "first"), + Segment::text("explain"), + invoke("two", "second"), + ]) + .await + .unwrap(); + assert_eq!( + *trace.calls.lock().unwrap(), + ["first", "second", "prompt-hook", "llm"] + ); + let requests = trace.requests.lock().unwrap(); + let request = format!("{:?}", requests[0]); + assert!(request.contains("trusted-preparation-first")); + assert!(request.contains("trusted-preparation-second")); + assert!(request.find("trusted-preparation-first") < request.find("trusted-preparation-second")); + assert_eq!(results(&worker).len(), 2); + // Preparation results must be model-visible in the same order as replay. + let history = worker.history(); + let first = history + .iter() + .position(|item| { + item.as_text() + .is_some_and(|text| text.contains("trusted-preparation-first")) + }) + .unwrap(); + let second = history + .iter() + .position(|item| { + item.as_text() + .is_some_and(|text| text.contains("trusted-preparation-second")) + }) + .unwrap(); + assert!(first < second); +} + +#[tokio::test] +async fn feature_failure_fences_later_invocations_model_resume_and_notifications() { + for value in ["fail", "unknown", "wrong"] { + let trace = Trace::default(); + let (_dir, mut worker) = fixture(trace.clone()).await; + assert!( + worker + .run(vec![ + invoke("one", "first"), + invoke("two", value), + invoke("three", "never") + ]) + .await + .is_err() + ); + assert_eq!(*trace.calls.lock().unwrap(), ["first", value]); + let results = results(&worker); + assert_eq!(results.len(), 3); + assert_eq!(results[0].status, FeatureInvocationStatus::Succeeded); + assert_eq!( + results[1].status, + if value == "fail" { + FeatureInvocationStatus::Failed + } else { + FeatureInvocationStatus::OutcomeUnknown + } + ); + assert_eq!(results[1].invocation_id, "two"); + assert!(results[1].context.is_none()); + assert_eq!(results[2].status, FeatureInvocationStatus::Failed); + assert!(worker.resume().await.is_err()); + assert!( + worker + .run_for_notification(protocol::InvokeKind::Notify) + .await + .is_err() + ); + assert!(trace.requests.lock().unwrap().is_empty()); + } +} + +#[tokio::test] +async fn feature_replay_reuses_result_and_rejects_id_payload_changes() { + let trace = Trace::default(); + let (_dir, mut worker) = fixture(trace.clone()).await; + worker.run(vec![invoke("same", "first")]).await.unwrap(); + worker.run(vec![invoke("same", "first")]).await.unwrap(); + assert_eq!(*trace.calls.lock().unwrap(), ["first", "llm", "llm"]); + assert!(worker.run(vec![invoke("same", "different")]).await.is_err()); + assert_eq!(*trace.calls.lock().unwrap(), ["first", "llm", "llm"]); + assert_eq!( + results(&worker).last().unwrap().status, + FeatureInvocationStatus::Failed + ); +} + +#[tokio::test] +async fn feature_argument_permission_revocation_fences_success_replay_and_resume() { + let trace = Trace::default(); + let (_dir, mut worker) = fixture(trace.clone()).await; + worker.run(vec![invoke("same", "first")]).await.unwrap(); + trace.argument_allowed.store(false, Ordering::SeqCst); + // The feature remains available: only this argument's authority was revoked. + assert_eq!(worker.feature_invocations().descriptors().len(), 1); + assert!(worker.resume().await.is_err()); + assert!( + worker + .run_for_notification(protocol::InvokeKind::Notify) + .await + .is_err() + ); + assert!(worker.run(vec![invoke("same", "first")]).await.is_err()); + let replay = results(&worker).pop().unwrap(); + assert_eq!(replay.status, FeatureInvocationStatus::Failed); + assert!(replay.context.is_none()); + assert!(replay.message.contains("host permission denied")); + assert_eq!(*trace.calls.lock().unwrap(), ["first", "llm"]); + // Reauthorization allows context reuse, never another business execution. + trace.argument_allowed.store(true, Ordering::SeqCst); + worker.run(vec![invoke("same", "first")]).await.unwrap(); + assert_eq!(*trace.calls.lock().unwrap(), ["first", "llm", "llm"]); +} + +#[tokio::test] +async fn feature_success_replay_does_not_resurrect_disabled_installed_capability() { + let trace = Trace::default(); + let (_dir, mut worker) = fixture(trace.clone()).await; + worker.run(vec![invoke("same", "first")]).await.unwrap(); + assert!( + !worker + .install_features(FeatureRegistryBuilder::new()) + .has_errors() + ); + assert!( + worker + .feature_invocations() + .feature_completions("") + .is_empty() + ); + assert!(worker.run(vec![invoke("same", "first")]).await.is_err()); + assert_eq!(*trace.calls.lock().unwrap(), ["first", "llm"]); + assert_eq!( + results(&worker).last().unwrap().status, + FeatureInvocationStatus::Failed + ); +} + +#[tokio::test] +async fn feature_invocation_text_is_not_executable_and_duplicate_ids_are_rejected_before_effects() { + let trace = Trace::default(); + let (_dir, mut worker) = fixture(trace.clone()).await; + worker + .run(vec![Segment::text("/prepare(first) quoted /prep(fail)")]) + .await + .unwrap(); + assert_eq!(*trace.calls.lock().unwrap(), ["llm"]); + assert!(results(&worker).is_empty()); + assert!( + worker + .run(vec![invoke("same", "first"), invoke("same", "second")]) + .await + .is_err() + ); + assert_eq!(*trace.calls.lock().unwrap(), ["llm"]); +} + +#[tokio::test] +async fn feature_handler_validation_rejects_semantics_and_forged_authority_before_effects() { + let trace = Trace::default(); + let (_dir, mut worker) = fixture(trace.clone()).await; + assert!(worker.run(vec![invoke("denied", "denied")]).await.is_err()); + let mut forged = invoke("forged", "first"); + if let Segment::FeatureInvoke { invocation } = &mut forged { + invocation.arguments.push(InvocationArgumentValue { + name: "workspace_id".into(), + value: InvocationValue::String("foreign".into()), + }); + } + assert!(worker.run(vec![forged]).await.is_err()); + assert!(trace.calls.lock().unwrap().is_empty()); + assert_eq!(results(&worker).len(), 2); + assert!( + results(&worker) + .iter() + .all(|result| result.status == FeatureInvocationStatus::Failed) + ); +} + +#[tokio::test] +async fn feature_prompt_hook_cancellation_does_not_rollback_durable_business_preparation() { + let trace = Trace::default(); + let (_dir, mut worker) = fixture(trace.clone()).await; + worker.add_on_prompt_submit_hook(TracePromptHook { + trace: trace.clone(), + cancel: true, + }); + let _ = worker.run(vec![invoke("one", "first")]).await; + assert_eq!(*trace.calls.lock().unwrap(), ["first", "prompt-hook"]); + assert_eq!(results(&worker).len(), 1); + assert!(worker.history().iter().any(|item| { + item.as_text() + .is_some_and(|text| text.contains("trusted-preparation-first")) + })); + let _ = worker.run(vec![invoke("one", "first")]).await; + assert_eq!( + *trace.calls.lock().unwrap(), + ["first", "prompt-hook", "prompt-hook"] + ); +} + +#[tokio::test] +async fn feature_unknown_input_rejected_before_effects_commit_or_model() { + let trace = Trace::default(); + let (_dir, mut worker) = fixture(trace.clone()).await; + let unknown: Segment = serde_json::from_value(serde_json::json!({ + "kind": "future_side_effect", "args": { "target": "important" } + })) + .unwrap(); + assert!(matches!(unknown, Segment::Unknown)); + let before = worker.sink().subscribe_with_snapshot().0.len(); + assert!( + worker + .run(vec![ + invoke("one", "first"), + unknown, + Segment::text("explain") + ]) + .await + .is_err() + ); + assert_eq!(worker.sink().subscribe_with_snapshot().0.len(), before); + assert!(trace.calls.lock().unwrap().is_empty()); +} + +#[tokio::test] +async fn feature_oversized_results_become_bounded_unknown_and_do_not_repeat_effects() { + for value in ["large-message", "large-context", "large-error"] { + let trace = Trace::default(); + let (_dir, mut worker) = fixture(trace.clone()).await; + assert!( + worker + .run(vec![invoke("large", value), invoke("later", "never")]) + .await + .is_err() + ); + assert_eq!(*trace.calls.lock().unwrap(), [value]); + for result in results(&worker) { + assert!(result.message.len() <= 4096); + assert!(result.context.is_none()); + } + assert_eq!( + results(&worker)[0].status, + FeatureInvocationStatus::OutcomeUnknown + ); + assert!(worker.run(vec![invoke("large", value)]).await.is_err()); + assert!(worker.resume().await.is_err()); + assert_eq!(*trace.calls.lock().unwrap(), [value]); + } + for value in ["limit-message", "limit-context"] { + let trace = Trace::default(); + let (_dir, mut worker) = fixture(trace.clone()).await; + worker.run(vec![invoke("limit", value)]).await.unwrap(); + worker.run(vec![invoke("limit", value)]).await.unwrap(); + assert_eq!(*trace.calls.lock().unwrap(), [value, "llm", "llm"]); + } +} + +#[tokio::test] +async fn feature_dynamic_permission_revocation_hides_discovery_and_fences_execution_and_replay() { + let trace = Trace::default(); + let (_dir, mut worker) = fixture(trace.clone()).await; + let registry = worker.feature_invocations(); + assert_eq!(registry.descriptors().len(), 1); + worker.run(vec![invoke("same", "first")]).await.unwrap(); + trace.available.store(false, Ordering::SeqCst); + assert!(registry.descriptors().is_empty()); + assert!(registry.descriptor(&descriptor().identity).is_none()); + assert!(registry.feature_completions("").is_empty()); + assert!( + registry + .argument_name_completions(&descriptor().identity, "") + .is_empty() + ); + assert!( + registry + .argument_completions(&descriptor().identity, "value", "") + .await + .is_empty() + ); + assert!(worker.run(vec![invoke("same", "first")]).await.is_err()); + assert!(worker.run(vec![invoke("new", "second")]).await.is_err()); + assert!(worker.resume().await.is_err()); + assert_eq!(*trace.calls.lock().unwrap(), ["first", "llm"]); + trace.available.store(true, Ordering::SeqCst); + assert_eq!(registry.descriptors().len(), 1); + worker.run(vec![invoke("same", "first")]).await.unwrap(); + assert_eq!(*trace.calls.lock().unwrap(), ["first", "llm", "llm"]); +} + +#[tokio::test] +async fn feature_revocation_during_effect_withholds_preparation_without_retrying_committed_effect() +{ + let trace = Trace::default(); + let (_dir, mut worker) = fixture(trace.clone()).await; + assert!( + worker + .run(vec![invoke("same", "revoke-during-effect")]) + .await + .is_err() + ); + assert_eq!(*trace.calls.lock().unwrap(), ["revoke-during-effect"]); + let result = &results(&worker)[0]; + assert_eq!(result.status, FeatureInvocationStatus::Failed); + assert!(result.context.is_none()); + assert!(result.message.contains("reported success")); + trace.available.store(true, Ordering::SeqCst); + assert!( + worker + .run(vec![invoke("same", "revoke-during-effect")]) + .await + .is_err() + ); + assert_eq!(*trace.calls.lock().unwrap(), ["revoke-during-effect"]); +} + +#[test] +fn notify_rejects_typed_side_effect_message_payloads() { + let methods = [ + protocol::Method::Notify { + notification_request_id: "n-1".into(), + message: "/prepare(first)".into(), + }, + protocol::Method::NotifyTracked { + notification_request_id: "n-1".into(), + message: "/prepare(first)".into(), + source: protocol::AuthenticatedInputSource::UntrustedWire, + }, + ]; + for method in methods { + // Notify accepts an advisory string, not typed input or invocation + // objects. Ordinary text spelling an invocation remains advisory text. + let mut request = serde_json::to_value(method).unwrap(); + assert!(serde_json::from_value::(request.clone()).is_ok()); + let mut with_input = request.clone(); + with_input["params"]["input"] = serde_json::json!([invoke("one", "first")]); + assert!( + serde_json::from_value::(with_input).is_err(), + "Notify must reject typed input rather than silently discard it" + ); + request["params"]["message"] = serde_json::json!([invoke("one", "first")]); + assert!(serde_json::from_value::(request).is_err()); + } +} diff --git a/crates/workspace-server/src/backend_job.rs b/crates/workspace-server/src/backend_job.rs index 08effd749..f5b54d7a4 100644 --- a/crates/workspace-server/src/backend_job.rs +++ b/crates/workspace-server/src/backend_job.rs @@ -199,6 +199,34 @@ const fn default_max_attempts() -> u8 { DEFAULT_MAX_ATTEMPTS } +/// Explicit Backend-issued domain capabilities, never inferred from purpose or profile. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct BackendJobGrants { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub subjektiv_consolidation: Option, +} + +impl BackendJobGrants { + pub fn is_empty(&self) -> bool { + self.subjektiv_consolidation.is_none() + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct SubjektivConsolidationGrant { + pub subject_id: String, + /// Exact immutable candidate scope. Empty is a surface-only refresh grant: + /// the subject is still locked, but no candidate access is authorized. + pub candidate_ids: Vec, +} + +/// Common runner lock identity for a subject, independent of the candidate snapshot. +pub fn subjektiv_serialization_key(subject_id: &str) -> String { + format!("subjektiv-consolidation:{subject_id}") +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(deny_unknown_fields)] pub struct BackendJobRequest { @@ -216,6 +244,13 @@ pub struct BackendJobRequest { pub instruction: String, /// Existing configured profile selector. Job dispatch never accepts raw profile source. pub profile: String, + /// Immutable domain grants; profile resolution alone never grants access. + #[serde(default, skip_serializing_if = "BackendJobGrants::is_empty")] + pub grants: BackendJobGrants, + /// Optional common-runner single-flight resource, not a separate domain state machine. + /// Consolidation grants always derive their subject lock; an explicit key must match it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub serialization_key: Option, /// Writer provenance and optional advisory destination, not parent ownership. #[serde(default, skip_serializing_if = "Option::is_none")] pub source_worker: Option, @@ -233,10 +268,34 @@ impl BackendJobRequest { validate_bounded_text("input_ref", &self.input_ref, MAX_JOB_REFERENCE_BYTES)?; validate_bounded_text("instruction", &self.instruction, MAX_JOB_INSTRUCTION_BYTES)?; validate_identifier("profile", &self.profile, 256)?; - if self.profile != "builtin:backend-job" { - return Err(Error::InvalidInput( - "Backend Jobs must use the dedicated builtin:backend-job profile".to_string(), - )); + self.profile_selector()?; + if let Some(key) = &self.serialization_key { + validate_identifier("serialization_key", key, MAX_JOB_REFERENCE_BYTES)?; + } + if let Some(grant) = &self.grants.subjektiv_consolidation { + validate_identifier("grant subject_id", &grant.subject_id, 256)?; + if grant.candidate_ids.len() > 256 { + return Err(Error::InvalidInput( + "Backend Job consolidation grant requires 0..=256 candidate ids".into(), + )); + } + let mut seen = std::collections::HashSet::new(); + for id in &grant.candidate_ids { + validate_identifier("grant candidate_id", id, 256)?; + if !seen.insert(id) { + return Err(Error::InvalidInput( + "Backend Job consolidation grant has duplicate candidate ids".into(), + )); + } + } + if let Some(key) = &self.serialization_key + && *key != subjektiv_serialization_key(&grant.subject_id) + { + return Err(Error::InvalidInput( + "Backend Job serialization_key must match the consolidation subject lock" + .into(), + )); + } } let input = serde_json::to_vec(&self.input).map_err(|error| { Error::InvalidInput(format!("serialize Backend Job input: {error}")) @@ -249,6 +308,44 @@ impl BackendJobRequest { self.limits.validate() } + /// Validate selector syntax only. Registry existence and resolution are dispatch authority. + pub fn profile_selector(&self) -> Result { + use manifest::ProfileSelector; + validate_identifier("profile", &self.profile, 256)?; + let selector = ProfileSelector::parse_cli(&self.profile); + let valid = match &selector { + ProfileSelector::Default => true, + ProfileSelector::Path { .. } => false, + ProfileSelector::Named { source, name } => { + !name.is_empty() + && !name.starts_with('.') + && !name.contains(['/', '\\', ':']) + && name + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || matches!(ch, '-' | '_' | '.')) + && ![".toml", ".json", ".dcdl", ".nix"] + .iter() + .any(|suffix| name.ends_with(suffix)) + && (source.is_some() || name != "inherit") + } + }; + if !valid { + return Err(Error::InvalidInput( + "Backend Job profile must be a manifest registry selector, not raw source or a path".into(), + )); + } + Ok(selector) + } + + /// Effective single-flight key. Typed grants cannot bypass their subject fence. + pub fn resource_key(&self) -> Option { + self.grants + .subjektiv_consolidation + .as_ref() + .map(|grant| subjektiv_serialization_key(&grant.subject_id)) + .or_else(|| self.serialization_key.clone()) + } + pub fn fingerprint(&self) -> Result { self.validate()?; let encoded = serde_json::to_vec(self).map_err(|error| { @@ -346,7 +443,12 @@ pub struct BackendJobAttemptRecord { pub struct BackendJobReservation { pub job: BackendJobRecord, pub attempt: BackendJobAttemptRecord, + /// Exact replay of the caller-selected immutable intent. pub replayed: bool, + /// A different intent already fences this resource. The returned request is + /// the original snapshot; the caller must defer its new input to a followup. + #[serde(default)] + pub resource_reused: bool, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] @@ -486,6 +588,8 @@ mod tests { input: serde_json::json!({"title": "Check me"}), instruction: "Check this immutable Ticket snapshot.".to_string(), profile: "builtin:backend-job".to_string(), + grants: BackendJobGrants::default(), + serialization_key: None, source_worker: Some(RuntimeWorkerRef::new("runtime-a", "worker-a")), notification_target: Some(RuntimeWorkerRef::new("runtime-a", "worker-a")), limits: BackendJobLimits::default(), @@ -509,6 +613,133 @@ mod tests { } } + #[test] + fn registry_profile_syntax_accepts_configured_selectors_without_an_allowlist() { + for profile in [ + "builtin:backend-job", + "builtin:subjektiv-memory-consolidation", + "project:custom", + "user:custom", + "custom", + "default", + "builtin:not-yet-configured", + ] { + let mut request = request(); + request.profile = profile.into(); + request.validate().unwrap(); + } + for profile in [ + "", + " builtin:backend-job", + "path:custom", + "./custom.toml", + "/tmp/custom", + "project:../custom", + "project:custom.toml", + "builtin:", + "unknown:custom", + "project:a:b", + "C:\\custom", + "{\"worker\":{}}", + "inherit", + ] { + let mut request = request(); + request.profile = profile.into(); + assert!(request.validate().is_err(), "accepted {profile}"); + } + } + + #[test] + fn explicit_grants_are_immutable_bounded_and_derive_a_subject_fence() { + let mut request = request(); + let original = request.fingerprint().unwrap(); + request.grants.subjektiv_consolidation = Some(SubjektivConsolidationGrant { + subject_id: "subject-a".into(), + candidate_ids: vec!["candidate-1".into()], + }); + assert_ne!(original, request.fingerprint().unwrap()); + assert_eq!( + request.resource_key(), + Some(subjektiv_serialization_key("subject-a")) + ); + let granted = request.fingerprint().unwrap(); + request + .grants + .subjektiv_consolidation + .as_mut() + .unwrap() + .candidate_ids + .push("candidate-2".into()); + assert_ne!(granted, request.fingerprint().unwrap()); + request.serialization_key = Some("another-subject".into()); + assert!(request.validate().is_err()); + request.serialization_key = None; + request + .grants + .subjektiv_consolidation + .as_mut() + .unwrap() + .candidate_ids + .push("candidate-2".into()); + assert!(request.validate().is_err()); + } + + #[test] + fn empty_consolidation_grant_keeps_subject_lock_without_candidate_authority() { + let mut request = request(); + request.grants.subjektiv_consolidation = Some(SubjektivConsolidationGrant { + subject_id: "subject-a".into(), + candidate_ids: vec![], + }); + request.validate().unwrap(); + assert!( + !request.grants.is_empty(), + "empty candidate scope still carries a subject grant" + ); + assert_eq!( + request.resource_key(), + Some(subjektiv_serialization_key("subject-a")) + ); + let encoded = serde_json::to_string(&request).unwrap(); + assert!(encoded.contains("\"candidate_ids\":[]")); + let decoded: BackendJobRequest = serde_json::from_str(&encoded).unwrap(); + assert_eq!(decoded, request); + let surface_only = request.fingerprint().unwrap(); + request + .grants + .subjektiv_consolidation + .as_mut() + .unwrap() + .candidate_ids = (0..256).map(|index| format!("candidate-{index}")).collect(); + request.validate().unwrap(); + assert_ne!(surface_only, request.fingerprint().unwrap()); + request + .grants + .subjektiv_consolidation + .as_mut() + .unwrap() + .candidate_ids + .push("candidate-256".into()); + assert!( + request.validate().is_err(), + "candidate batch remains bounded" + ); + } + + #[test] + fn legacy_request_replay_keeps_its_exact_serialization() { + let request = request(); + let original_json = serde_json::to_string(&request).unwrap(); + assert!(!original_json.contains("grants")); + assert!(!original_json.contains("serialization_key")); + let decoded: BackendJobRequest = serde_json::from_str(&original_json).unwrap(); + assert_eq!(decoded.grants, BackendJobGrants::default()); + assert_eq!( + request.fingerprint().unwrap(), + decoded.fingerprint().unwrap() + ); + } + #[test] fn tracked_runtime_ids_are_fixed_size_for_maximum_job_ids() { let job_id = "j".repeat(256); diff --git a/crates/workspace-server/src/config_source.rs b/crates/workspace-server/src/config_source.rs index 3d3ec8071..2a127bd9c 100644 --- a/crates/workspace-server/src/config_source.rs +++ b/crates/workspace-server/src/config_source.rs @@ -116,6 +116,52 @@ pub struct EvaluatedConfigCandidate { pub evaluation: EvaluationResult, } +/// Validate feature semantics before committing a schema-valid candidate. +/// +/// Keep this shared by the UI and logical config operations. Schema evaluation +/// alone does not resolve Profile sources or reject blocking Skill diagnostics. +/// This function has no publication or repository-secret side effects. +pub(crate) fn validate_workspace_config_candidate_projections( + workspace_id: &str, + candidate: &EvaluatedConfigCandidate, +) -> Result<()> { + let state = WorkspaceConfigState { + snapshot: candidate.snapshot.clone(), + contract: candidate.contract.clone(), + projection_digest: candidate.evaluation.projection_digest.clone(), + }; + let has_provider = |id: &str| { + state + .contract + .schema_bundle + .contributions + .iter() + .any(|provider| provider.provider_id == id) + }; + crate::prompt_settings::validate_evaluated_prompt_catalog(&candidate.evaluation)?; + if has_provider("builtin:profile") { + crate::profile_settings::project_profiles_from_workspace_config(workspace_id, &state)?; + } + if has_provider("builtin:runtime") { + crate::runtime_settings::project_runtime_from_workspace_config(workspace_id, &state)?; + } + if has_provider("builtin:skills") { + let catalog = crate::skills::catalog(&state) + .map_err(|_| Error::InvalidInput("invalid Workspace Skill projection".to_string()))?; + if catalog.entries.iter().any(|entry| { + entry + .diagnostics + .iter() + .any(|diagnostic| diagnostic.severity == server_api::SkillDiagnosticSeverity::Error) + }) { + return Err(Error::InvalidInput( + "Workspace Skills have blocking diagnostics".to_string(), + )); + } + } + Ok(()) +} + #[derive(Debug, Clone, PartialEq, Serialize, Deserialize, ts_rs::TS)] pub struct ConfigCommitRequest { #[ts(type = "number")] @@ -911,6 +957,93 @@ mod tests { } } + fn semantic_candidate( + contribution: ConfigSchemaContribution, + source: &str, + ) -> EvaluatedConfigCandidate { + let schema = WorkspaceConfigSchemaBundle::compose([ + crate::prompt_settings::PromptConfigSchemaProvider + .contribution() + .unwrap(), + contribution, + ]) + .unwrap(); + let current = initial_state_with_schema(schema.clone()).unwrap(); + let change = update_main(¤t, source); + evaluate_candidate(current, &[change], schema).unwrap() + } + + #[test] + fn candidate_semantics_reject_unknown_default_profile() { + let candidate = semantic_candidate( + crate::profile_settings::ProfileConfigSchemaProvider + .contribution() + .unwrap(), + r#"{ profile = { default_profile = "project:missing"; }; }"#, + ); + let error = + validate_workspace_config_candidate_projections("w-config", &candidate).unwrap_err(); + assert!(error.to_string().contains("unknown_default_profile")); + let valid = semantic_candidate( + crate::profile_settings::ProfileConfigSchemaProvider + .contribution() + .unwrap(), + "{}", + ); + validate_workspace_config_candidate_projections("w-config", &valid).unwrap(); + } + + #[test] + fn candidate_semantics_reject_invalid_runtime_identifier() { + let candidate = semantic_candidate( + crate::runtime_settings::RuntimeConfigSchemaProvider + .contribution() + .unwrap(), + r#"{ runtime = { default_runtime_id = "bad\nidentifier"; }; }"#, + ); + assert!(matches!( + validate_workspace_config_candidate_projections("w-config", &candidate), + Err(Error::InvalidRuntimeIdentifier { .. }) + )); + let valid = semantic_candidate( + crate::runtime_settings::RuntimeConfigSchemaProvider + .contribution() + .unwrap(), + "{}", + ); + validate_workspace_config_candidate_projections("w-config", &valid).unwrap(); + } + + #[test] + fn candidate_semantics_reject_skill_without_canonical_source() { + let candidate = semantic_candidate( + crate::skills::SkillConfigSchemaProvider + .contribution() + .unwrap(), + r#"{ + skills = { + debug_rust = { + frontmatter = { name = "debug-rust"; description = "Debug Rust"; }; + content = "inline"; + }; + }; + }"#, + ); + let error = + validate_workspace_config_candidate_projections("w-config", &candidate).unwrap_err(); + assert!(error.to_string().contains("blocking diagnostics")); + // Do not copy Skill source bodies or diagnostic paths into this error. + assert!(!error.to_string().contains("inline")); + assert!(!error.to_string().contains("SKILL.md")); + let valid = semantic_candidate( + crate::skills::SkillConfigSchemaProvider + .contribution() + .unwrap(), + "{}", + ); + validate_workspace_config_candidate_projections("w-config", &valid).unwrap(); + } + #[tokio::test] async fn schema_registry_applies_normal_decodal_composition() { struct WebSchema; diff --git a/crates/workspace-server/src/hosts.rs b/crates/workspace-server/src/hosts.rs index b9f3ed39a..a91b2b160 100644 --- a/crates/workspace-server/src/hosts.rs +++ b/crates/workspace-server/src/hosts.rs @@ -242,6 +242,8 @@ pub struct InternalWorkerImplementationSummary { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct InternalWorkerSummary { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub restore_observation_token: Option, #[serde(flatten)] pub worker: RuntimeWorkerRef, pub host_id: String, @@ -356,6 +358,7 @@ pub(crate) fn workspace_worker_summary( workdir_attachments: Vec, ) -> server_api::WorkerSummary { server_api::WorkerSummary { + restore_observation_token: summary.restore_observation_token, runtime_id: summary.worker.runtime_id, worker_id: summary.worker.worker_id, resource_key, @@ -561,6 +564,8 @@ pub enum WorkerSpawnIntent { job_id: String, attempt_id: String, purpose: String, + input_revision: String, + subjektiv_consolidation: bool, }, TicketRole { ticket_id: String, @@ -735,6 +740,10 @@ pub struct WorkerCompletionsRequest { pub kind: protocol::CompletionKind, #[serde(default)] pub prefix: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub context: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub request_id: Option, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] @@ -743,6 +752,10 @@ pub struct WorkerCompletionsResult { pub worker: RuntimeWorkerRef, pub kind: protocol::CompletionKind, pub prefix: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub context: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub request_id: Option, pub entries: Vec, pub diagnostics: Vec, } @@ -826,11 +839,17 @@ impl RuntimeRegistryError { runtime_id, code, message, - } => Error::RuntimeOperationFailed { - runtime_id, - code, - message, - }, + } => { + if code == "restore_observation_conflict" { + Error::RestoreObservationConflict + } else { + Error::RuntimeOperationFailed { + runtime_id, + code, + message, + } + } + } } } } @@ -930,7 +949,29 @@ pub trait WorkspaceWorkerRuntime: Send + Sync { fn worker(&self, worker_id: &str) -> WorkerLookupResult; - fn restore_worker(&self, worker_id: &str) -> InternalWorkerRestoreResult { + fn coordinate_worker_restore( + &self, + _worker_id: &str, + _request: runtime_api::WorkerRestoreCoordinationRequest, + ) -> Result< + ( + Option, + Option, + ), + RuntimeDiagnostic, + > { + Err(diagnostic( + "worker_restore_coordination_unsupported", + HostDiagnosticSeverity::Error, + "Runtime does not support guarded Restore coordination", + )) + } + + fn restore_worker( + &self, + worker_id: &str, + _request: runtime_api::WorkerRestoreRequest, + ) -> InternalWorkerRestoreResult { InternalWorkerRestoreResult { state: server_api::WorkerRestoreState::Rejected, worker: None, @@ -1327,6 +1368,8 @@ pub trait WorkspaceWorkerRuntime: Send + Sync { worker: RuntimeWorkerRef::new(self.runtime_id().to_string(), worker_id.to_string()), kind: request.kind, prefix: request.prefix, + context: request.context, + request_id: request.request_id, entries: Vec::new(), diagnostics: vec![diagnostic( "worker_completions_unsupported", @@ -1592,16 +1635,39 @@ impl RuntimeRegistry { Ok(worker) } + pub fn coordinate_worker_restore( + &self, + worker: &RuntimeWorkerRef, + request: runtime_api::WorkerRestoreCoordinationRequest, + ) -> Result< + ( + Option, + Option, + ), + RuntimeRegistryError, + > { + validate_backend_identifier("runtime_id", &worker.runtime_id)?; + validate_backend_identifier("worker_id", &worker.worker_id)?; + self.runtime(&worker.runtime_id)? + .coordinate_worker_restore(&worker.worker_id, request) + .map_err(|error| RuntimeRegistryError::RuntimeOperationFailed { + runtime_id: worker.runtime_id.clone(), + code: error.code, + message: error.message, + }) + } + pub fn restore_worker( &self, worker: &RuntimeWorkerRef, + request: runtime_api::WorkerRestoreRequest, ) -> Result { let runtime_id = worker.runtime_id.as_str(); let worker_id = worker.worker_id.as_str(); validate_backend_identifier("runtime_id", runtime_id)?; validate_backend_identifier("worker_id", worker_id)?; let runtime = self.runtime(runtime_id)?; - Ok(runtime.restore_worker(worker_id)) + Ok(runtime.restore_worker(worker_id, request)) } pub fn replace_worker_workspace_api( @@ -2362,6 +2428,7 @@ impl EmbeddedWorkerRuntime { true, ); InternalWorkerSummary { + restore_observation_token: summary.restore_observation_token.clone(), worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id.clone()), host_id: self.host_id.clone(), display_name: display.display_name.clone(), @@ -2406,6 +2473,7 @@ impl EmbeddedWorkerRuntime { true, ); InternalWorkerSummary { + restore_observation_token: detail.restore_observation_token.clone(), worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id.clone()), host_id: self.host_id.clone(), display_name: display.display_name.clone(), @@ -2572,7 +2640,48 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime { } } - fn restore_worker(&self, worker_id: &str) -> InternalWorkerRestoreResult { + fn coordinate_worker_restore( + &self, + worker_id: &str, + request: runtime_api::WorkerRestoreCoordinationRequest, + ) -> Result< + ( + Option, + Option, + ), + RuntimeDiagnostic, + > { + let worker_ref = self.worker_ref(worker_id).ok_or_else(|| { + diagnostic( + "worker_not_found", + HostDiagnosticSeverity::Warning, + "Unknown Worker", + ) + })?; + let (result, preparation) = self + .runtime + .coordinate_worker_restore_operation(&worker_ref, request, None) + .map_err(|error| embedded_runtime_diagnostic(&error))?; + Ok(( + result.map(|result| InternalWorkerRestoreResult { + state: result.state, + worker: result.worker.map(|detail| self.map_worker_detail(detail)), + diagnostics: match (result.reason_code, result.message) { + (Some(code), Some(message)) => { + vec![diagnostic(code, HostDiagnosticSeverity::Warning, message)] + } + _ => Vec::new(), + }, + }), + preparation, + )) + } + + fn restore_worker( + &self, + worker_id: &str, + request: runtime_api::WorkerRestoreRequest, + ) -> InternalWorkerRestoreResult { let Some(worker_ref) = self.worker_ref(worker_id) else { return InternalWorkerRestoreResult { state: server_api::WorkerRestoreState::Rejected, @@ -2584,7 +2693,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime { )], }; }; - match self.runtime.restore_worker_operation(&worker_ref) { + match self.runtime.restore_worker_operation(&worker_ref, request) { Ok(result) => { let diagnostics = match (result.reason_code, result.message) { (Some(code), Some(message)) => { @@ -2598,6 +2707,17 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime { diagnostics, } } + Err(worker_runtime::error::RuntimeError::RestoreObservationConflict { .. }) => { + InternalWorkerRestoreResult { + state: server_api::WorkerRestoreState::Rejected, + worker: None, + diagnostics: vec![diagnostic( + "restore_observation_conflict", + HostDiagnosticSeverity::Warning, + "Restore observation changed; refresh before a new intent".to_string(), + )], + } + } Err(err) => InternalWorkerRestoreResult { state: server_api::WorkerRestoreState::ReconciliationRequired, worker: None, @@ -3310,6 +3430,8 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime { worker: RuntimeWorkerRef::new(self.runtime_id.clone(), worker_id.to_string()), kind: request.kind, prefix: request.prefix, + context: request.context, + request_id: request.request_id, entries: Vec::new(), diagnostics: vec![diagnostic( "embedded_worker_execution_unavailable", @@ -3325,6 +3447,8 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime { worker: RuntimeWorkerRef::new(self.runtime_id.clone(), worker_id.to_string()), kind: request.kind, prefix: request.prefix, + context: request.context, + request_id: request.request_id, entries: Vec::new(), diagnostics: vec![diagnostic( "embedded_worker_id_invalid", @@ -3333,14 +3457,18 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime { )], }; }; - match self - .runtime - .worker_completions(&worker_ref, request.kind, &request.prefix) - { + match self.runtime.worker_completions( + &worker_ref, + request.kind, + &request.prefix, + request.context.as_ref(), + ) { Ok(entries) => WorkerCompletionsResult { worker: RuntimeWorkerRef::new(self.runtime_id.clone(), worker_id.to_string()), kind: request.kind, prefix: request.prefix, + context: request.context, + request_id: request.request_id, entries, diagnostics: Vec::new(), }, @@ -3348,6 +3476,8 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime { worker: RuntimeWorkerRef::new(self.runtime_id.clone(), worker_id.to_string()), kind: request.kind, prefix: request.prefix, + context: request.context, + request_id: request.request_id, entries: Vec::new(), diagnostics: vec![embedded_runtime_diagnostic(&error)], }, @@ -4355,6 +4485,7 @@ impl RemoteWorkerRuntime { false, ); InternalWorkerSummary { + restore_observation_token: summary.restore_observation_token.clone(), worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id.clone()), host_id: self.host_id.clone(), display_name: display.display_name.clone(), @@ -4397,6 +4528,7 @@ impl RemoteWorkerRuntime { false, ); InternalWorkerSummary { + restore_observation_token: detail.restore_observation_token.clone(), worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id.clone()), host_id: self.host_id.clone(), display_name: display.display_name.clone(), @@ -4688,17 +4820,58 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime { } } - fn restore_worker(&self, worker_id: &str) -> InternalWorkerRestoreResult { + fn coordinate_worker_restore( + &self, + worker_id: &str, + request: runtime_api::WorkerRestoreCoordinationRequest, + ) -> Result< + ( + Option, + Option, + ), + RuntimeDiagnostic, + > { + let worker_id = worker_id.to_string(); + let response = self.run_runtime_api( + self.request_timeout, + MAX_REMOTE_RUNTIME_RESPONSE_BYTES, + move |client| async move { client.coordinate_worker_restore(worker_id, request).await }, + )?; + let result = response + .result + .map(|result| -> Result<_, RuntimeDiagnostic> { + let worker = result + .worker + .map(runtime_contract_convert::<_, worker_runtime::catalog::WorkerDetail>) + .transpose()?; + let state = + runtime_contract_convert::<_, server_api::WorkerRestoreState>(result.state)?; + Ok(InternalWorkerRestoreResult { + state, + worker: worker.map(|detail| self.map_worker_detail(detail)), + diagnostics: match (result.reason_code, result.message) { + (Some(code), Some(message)) => { + vec![diagnostic(code, HostDiagnosticSeverity::Warning, message)] + } + _ => Vec::new(), + }, + }) + }) + .transpose()?; + Ok((result, response.preparation)) + } + + fn restore_worker( + &self, + worker_id: &str, + request: runtime_api::WorkerRestoreRequest, + ) -> InternalWorkerRestoreResult { let worker_id_owned = worker_id.to_string(); match self .run_runtime_api( self.request_timeout, MAX_REMOTE_RUNTIME_RESPONSE_BYTES, - move |client| async move { - client - .restore_worker(worker_id_owned, runtime_api::EmptyObjectRequest::default()) - .await - }, + move |client| async move { client.restore_worker(worker_id_owned, request).await }, ) .and_then(|value| { let worker = value @@ -5415,9 +5588,13 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime { let request = runtime_api::CompletionRequest { kind: request.kind, prefix: request.prefix, + context: request.context, + request_id: request.request_id, }; let failure_kind = request.kind; let failure_prefix = request.prefix.clone(); + let failure_context = request.context.clone(); + let failure_request_id = request.request_id.clone(); let worker_id_owned = worker_id.to_string(); match self.run_runtime_api( self.request_timeout, @@ -5432,6 +5609,8 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime { worker: RuntimeWorkerRef::new(self.runtime_id.clone(), worker_id.to_string()), kind: response.kind, prefix: response.prefix, + context: response.context, + request_id: response.request_id, entries: response.entries, diagnostics: Vec::new(), }, @@ -5439,6 +5618,8 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime { worker: RuntimeWorkerRef::new(self.runtime_id.clone(), worker_id.to_string()), kind: failure_kind, prefix: failure_prefix, + context: failure_context, + request_id: failure_request_id, entries: Vec::new(), diagnostics: vec![diagnostic], }, @@ -5529,6 +5710,21 @@ fn runtime_create_worker_request( workspace_api: Some(workspace_api), memory_settings: request.resolved_memory_settings.clone(), subjektiv_attached: request.resolved_subjektiv_attached, + backend_job: match &request.intent { + WorkerSpawnIntent::BackendJob { + job_id, + attempt_id, + input_revision, + subjektiv_consolidation, + .. + } => Some(worker_runtime::catalog::BackendJobExecutionBinding { + job_id: job_id.clone(), + attempt_id: attempt_id.clone(), + input_revision: Some(input_revision.clone()), + subjektiv_consolidation: *subjektiv_consolidation, + }), + _ => None, + }, } } @@ -5858,6 +6054,11 @@ fn sanitize_embedded_execution_detail(summary: &str, message: &str) -> String { fn embedded_runtime_diagnostic(error: &EmbeddedRuntimeError) -> RuntimeDiagnostic { match error { + EmbeddedRuntimeError::RestoreObservationConflict { .. } => diagnostic( + "restore_observation_conflict", + HostDiagnosticSeverity::Warning, + "Restore observation changed; refresh before a new intent".to_string(), + ), EmbeddedRuntimeError::RuntimeStopped => diagnostic( "embedded_runtime_stopped", HostDiagnosticSeverity::Warning, @@ -6264,6 +6465,7 @@ fn worker_spawn_intent_label(intent: &WorkerSpawnIntent) -> &'static str { pub fn placeholder_worker(host_id: impl Into) -> InternalWorkerSummary { let host_id = host_id.into(); InternalWorkerSummary { + restore_observation_token: None, worker: RuntimeWorkerRef::new("placeholder", "worker-placeholder"), host_id, display_name: "Worker runtime actions are not implemented".to_string(), @@ -6341,7 +6543,14 @@ mod tests { ) .unwrap(); - let result = provider.restore_worker("worker-transport-uncertain"); + let result = provider.restore_worker( + "worker-transport-uncertain", + runtime_api::WorkerRestoreRequest { + expected_observation_token: "observation".into(), + request_id: "transport-uncertain".into(), + preparation: None, + }, + ); assert_eq!( result.state, @@ -6711,6 +6920,46 @@ mod tests { } } + #[test] + fn feature_invocation_and_completion_context_survive_runtime_contract() { + let segments = vec![ + protocol::Segment::FeatureInvoke { + invocation: protocol::FeatureInvocation { + invocation_id: "stable-request".into(), + identity: protocol::FeatureInvocationIdentity("builtin:test/prepare".into()), + name: "prepare".into(), + arguments: vec![protocol::InvocationArgumentValue { + name: "path".into(), + value: protocol::InvocationValue::String("資料/a b".into()), + }], + }, + }, + protocol::Segment::text(" subsequent prose"), + ]; + let input = EmbeddedWorkerInput { + kind: EmbeddedWorkerInputKind::User, + content: protocol::Segment::flatten_to_text(&segments), + submission_request_id: Some("submit-request".into()), + segments: Some(segments), + }; + let wire: runtime_api::WorkerInput = runtime_contract_convert(input.clone()).unwrap(); + let restored: EmbeddedWorkerInput = runtime_contract_convert(wire).unwrap(); + assert_eq!(restored, input); + let request = WorkerCompletionsRequest { + kind: protocol::CompletionKind::FeatureArgument, + prefix: "資料/".into(), + context: Some(protocol::CompletionContext { + invocation: protocol::FeatureInvocationIdentity("builtin:test/prepare".into()), + argument: Some("path".into()), + }), + request_id: Some("completion-nonce".into()), + }; + let wire: runtime_api::CompletionRequest = + runtime_contract_convert(request.clone()).unwrap(); + let restored: WorkerCompletionsRequest = runtime_contract_convert(wire).unwrap(); + assert_eq!(restored, request); + } + #[test] fn remote_profile_source_uses_workspace_config_archive_reference() { let request = embedded_spawn_request(); @@ -6801,7 +7050,7 @@ mod tests { fn dispatch_input( &self, - _handle: &worker_runtime::execution::WorkerExecutionHandle, + _worker_ref: &EmbeddedWorkerRef, _input: EmbeddedWorkerInput, ) -> worker_runtime::execution::WorkerExecutionResult { worker_runtime::execution::WorkerExecutionResult::rejected( @@ -6830,7 +7079,7 @@ mod tests { fn dispatch_input( &self, - _handle: &worker_runtime::execution::WorkerExecutionHandle, + _worker_ref: &EmbeddedWorkerRef, _input: EmbeddedWorkerInput, ) -> worker_runtime::execution::WorkerExecutionResult { unreachable!("Repository observation test does not dispatch Worker input") @@ -6910,10 +7159,6 @@ mod tests { .unwrap() .insert(request.worker_ref.clone(), request.context); worker_runtime::execution::WorkerExecutionSpawnResult::Connected { - handle: worker_runtime::execution::WorkerExecutionHandle::new( - request.worker_ref, - self.backend_id(), - ), worker_state: protocol::WorkerStateSnapshot { ..protocol::WorkerStatus::Idle.into() }, @@ -6932,15 +7177,10 @@ mod tests { fn dispatch_input( &self, - handle: &worker_runtime::execution::WorkerExecutionHandle, + worker_ref: &EmbeddedWorkerRef, input: EmbeddedWorkerInput, ) -> worker_runtime::execution::WorkerExecutionResult { - let context = self - .contexts - .lock() - .unwrap() - .get(handle.worker_ref()) - .cloned(); + let context = self.contexts.lock().unwrap().get(worker_ref).cloned(); let Some(context) = context else { return worker_runtime::execution::WorkerExecutionResult::rejected( worker_runtime::execution::WorkerExecutionOperation::Input, @@ -6993,6 +7233,7 @@ mod tests { runtime_id: runtime_id.to_string(), host_id: host_id.to_string(), workers: vec![InternalWorkerSummary { + restore_observation_token: None, worker: RuntimeWorkerRef::new(runtime_id, worker_id), host_id: host_id.to_string(), display_name: label.to_string(), @@ -7639,7 +7880,16 @@ mod tests { let detail = registry.worker(&worker.worker).unwrap(); - let json = serde_json::to_string(&(embedded_summary, worker, input, detail)).unwrap(); + // The public observation generation is intentionally exposed, unlike + // private authentication tokens. Whitelist its key, not arbitrary token + // values or other token-bearing fields. + for summary in [&worker, &detail] { + let token = summary.restore_observation_token.as_ref().unwrap(); + assert_eq!(uuid::Uuid::parse_str(token).unwrap().get_version_num(), 7); + } + let json = serde_json::to_string(&(embedded_summary, worker, input, detail)) + .unwrap() + .replace("\"restore_observation_token\":", "\"restore_observation\":"); for forbidden in [ "/workspace/project", "metadata.json", diff --git a/crates/workspace-server/src/latest_schema.sql b/crates/workspace-server/src/latest_schema.sql index 16d98f1f4..64275a3ab 100644 --- a/crates/workspace-server/src/latest_schema.sql +++ b/crates/workspace-server/src/latest_schema.sql @@ -668,7 +668,7 @@ CREATE TABLE "workdir_registry" ( workspace_id TEXT NOT NULL, workdir_id TEXT NOT NULL, display_name TEXT, - source_kind TEXT NOT NULL CHECK (source_kind IN ('repository', 'external_grant')), + source_kind TEXT NOT NULL CHECK (source_kind IN ('repository', 'external_grant', 'workspace_config')), runtime_id TEXT, repository_id TEXT, external_grant_id TEXT, @@ -680,16 +680,20 @@ CREATE TABLE "workdir_registry" ( updated_at TEXT NOT NULL, current_selector TEXT, current_ref TEXT, creation_tree TEXT, current_tree TEXT, observed_at_epoch_seconds INTEGER, + workspace_config_grant_id TEXT, PRIMARY KEY (workspace_id, workdir_id), CHECK ( - (source_kind = 'repository' AND runtime_id IS NOT NULL AND repository_id IS NOT NULL AND external_grant_id IS NULL) - OR (source_kind = 'external_grant' AND runtime_id IS NULL AND repository_id IS NULL AND external_grant_id IS NOT NULL) + (source_kind = 'repository' AND runtime_id IS NOT NULL AND repository_id IS NOT NULL AND external_grant_id IS NULL AND workspace_config_grant_id IS NULL) + OR (source_kind = 'external_grant' AND runtime_id IS NULL AND repository_id IS NULL AND external_grant_id IS NOT NULL AND workspace_config_grant_id IS NULL) + OR (source_kind = 'workspace_config' AND runtime_id IS NULL AND repository_id IS NULL AND external_grant_id IS NULL AND workspace_config_grant_id IS NOT NULL) ), FOREIGN KEY (workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE, FOREIGN KEY (workspace_id, repository_id) REFERENCES "repositories"(workspace_id, repository_id), FOREIGN KEY (workspace_id, external_grant_id) - REFERENCES external_workdir_grants(workspace_id, grant_id) DEFERRABLE INITIALLY DEFERRED + REFERENCES external_workdir_grants(workspace_id, grant_id) DEFERRABLE INITIALLY DEFERRED, + FOREIGN KEY (workspace_id, workspace_config_grant_id) + REFERENCES workspace_config_grants(workspace_id, grant_id) DEFERRABLE INITIALLY DEFERRED ); CREATE TABLE external_workdir_grants ( grant_id TEXT NOT NULL, @@ -841,6 +845,7 @@ CREATE TABLE backend_jobs ( purpose TEXT NOT NULL, input_revision TEXT NOT NULL, input_ref TEXT NOT NULL, + resource_key TEXT, request_json TEXT NOT NULL, intent_fingerprint TEXT NOT NULL, state TEXT NOT NULL CHECK (state IN ('pending', 'completed', 'failed', 'unknown')), @@ -859,6 +864,8 @@ CREATE TABLE backend_jobs ( ); CREATE INDEX backend_jobs_active ON backend_jobs(workspace_id, state, updated_at); +CREATE INDEX backend_jobs_resource + ON backend_jobs(workspace_id, resource_key, created_at); CREATE TABLE backend_job_attempts ( workspace_id TEXT NOT NULL, job_id TEXT NOT NULL, @@ -1605,3 +1612,20 @@ CREATE TRIGGER ticket_worker_assignments_validate_update BEGIN SELECT RAISE(ABORT, 'Ticket assignment principal is not valid in this Workspace'); END; + +CREATE TABLE workspace_config_grants ( + workspace_id TEXT NOT NULL, + grant_id TEXT NOT NULL, + runtime_id TEXT NOT NULL, + worker_id TEXT NOT NULL, + workdir_id TEXT NOT NULL, + access TEXT NOT NULL CHECK(access IN ('read_only', 'read_write')), + revoked INTEGER NOT NULL CHECK(revoked IN (0, 1)), + created_by TEXT NOT NULL, + created_at TEXT NOT NULL, + PRIMARY KEY(workspace_id, grant_id), + UNIQUE(workspace_id, workdir_id), + FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE +); +CREATE UNIQUE INDEX workspace_config_grants_active_worker + ON workspace_config_grants(workspace_id, runtime_id, worker_id) WHERE revoked = 0; diff --git a/crates/workspace-server/src/lib.rs b/crates/workspace-server/src/lib.rs index c23670ced..652c6ad65 100644 --- a/crates/workspace-server/src/lib.rs +++ b/crates/workspace-server/src/lib.rs @@ -139,6 +139,10 @@ pub enum Error { RuntimeBindingFingerprintConflict { fingerprint: String }, #[error("Runtime binding was not found for {runtime_id}")] RuntimeBindingNotFound { runtime_id: String }, + #[error( + "restore_observation_conflict: Restore observation changed; refresh before a new intent" + )] + RestoreObservationConflict, #[error("Repository conflict: {0}")] RepositoryConflict(String), #[error("Registry inconsistency: {0}")] diff --git a/crates/workspace-server/src/profile_settings.rs b/crates/workspace-server/src/profile_settings.rs index 0e350cc42..5c38a099b 100644 --- a/crates/workspace-server/src/profile_settings.rs +++ b/crates/workspace-server/src/profile_settings.rs @@ -251,6 +251,27 @@ pub fn selector_for_workspace_candidate( .then(|| worker_runtime::catalog::ProfileSelector::Named(profile.to_string())) } +/// Resolve a Host-owned launch selector against the complete existing registry. +/// Browser candidates remain a narrower UI policy; this does not widen that surface. +pub(crate) fn selector_for_registered_profile( + projection: &ProfileConfigProjection, + profile: &str, +) -> Option { + let catalog = builtin_profile_catalog_snapshot(); + let builtin = if profile.starts_with("builtin:") { + profile.to_string() + } else { + format!("builtin:{profile}") + }; + if catalog.entrypoints.contains_key(&builtin) { + return Some(worker_runtime::catalog::ProfileSelector::Builtin(builtin)); + } + projection + .entries + .contains_key(profile) + .then(|| worker_runtime::catalog::ProfileSelector::Named(profile.into())) +} + fn validate_prompt_projection_matches_state( workspace_id: &str, state: &WorkspaceConfigState, @@ -401,8 +422,12 @@ pub fn build_virtual_profile_config_bundle_with_prompt_projection( ) -> Result> { validate_prompt_projection_matches_state(workspace_id, state, prompt_projection)?; let prompt_catalog = prompt_projection.catalog().clone(); - let profile_selector = selector_for_builtin_candidate(selector) - .unwrap_or_else(|| worker_runtime::catalog::ProfileSelector::Named(selector.to_string())); + let profile_selector = + selector_for_registered_profile(projection, selector).ok_or_else(|| { + Error::InvalidInput(format!( + "Profile `{selector}` is not a registered launch selector" + )) + })?; let archive = match projection.entries.get(selector) { Some(entry) => Some(build_virtual_profile_archive( selector, diff --git a/crates/workspace-server/src/resource_broker.rs b/crates/workspace-server/src/resource_broker.rs index 90bbc3efe..9bb68fdb2 100644 --- a/crates/workspace-server/src/resource_broker.rs +++ b/crates/workspace-server/src/resource_broker.rs @@ -61,6 +61,16 @@ impl Drop for StoredResource { } impl BackendResourceBroker { + #[cfg(test)] + pub(crate) fn repository_ssh_access_handle_count(&self) -> usize { + self.resources + .lock() + .unwrap() + .values() + .filter(|resource| resource.handle.kind == BackendResourceKind::RepositorySshAccess) + .count() + } + pub fn issue_profile_source_archive_handle( &self, workspace_id: impl Into, diff --git a/crates/workspace-server/src/runtime_subscription_tests.rs b/crates/workspace-server/src/runtime_subscription_tests.rs index b58cd991b..c07ca6450 100644 --- a/crates/workspace-server/src/runtime_subscription_tests.rs +++ b/crates/workspace-server/src/runtime_subscription_tests.rs @@ -3,10 +3,10 @@ use protocol::subscription::{SubscriptionWorkerIds, SubscriptionWorkerState}; use worker_runtime::Runtime; use worker_runtime::catalog::{CreateWorkerRequest, ProfileSelector, ProfileSourceArchiveSource}; use worker_runtime::execution::{ - WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation, WorkerExecutionResult, + WorkerExecutionBackend, WorkerExecutionOperation, WorkerExecutionResult, WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult, }; -use worker_runtime::identity::WorkerId; +use worker_runtime::identity::{WorkerId, WorkerRef}; #[derive(Debug)] struct TestExecutionBackend; @@ -16,9 +16,8 @@ impl WorkerExecutionBackend for TestExecutionBackend { "runtime-subscription-test" } - fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult { + fn spawn_worker(&self, _request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult { WorkerExecutionSpawnResult::connected( - WorkerExecutionHandle::new(request.worker_ref, self.backend_id()), protocol::WorkerStateSnapshot { ..protocol::WorkerStatus::Idle.into() }, @@ -28,7 +27,7 @@ impl WorkerExecutionBackend for TestExecutionBackend { fn dispatch_input( &self, - _handle: &WorkerExecutionHandle, + _worker_ref: &WorkerRef, input: worker_runtime::interaction::WorkerInput, ) -> WorkerExecutionResult { if let Some(submission_request_id) = input.submission_request_id { @@ -43,7 +42,7 @@ impl WorkerExecutionBackend for TestExecutionBackend { } } - fn stop_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult { + fn stop_worker(&self, _worker_ref: &WorkerRef) -> WorkerExecutionResult { WorkerExecutionResult::accepted(WorkerExecutionOperation::Stop) } } @@ -75,6 +74,7 @@ fn create_request(name: &str) -> CreateWorkerRequest { language: "English".to_string(), }), subjektiv_attached: false, + backend_job: None, } } diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 39e0163ba..84fb046e1 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -1,3 +1,12 @@ +mod backend_job_profile; +mod subjektiv_job; +mod worker_operations; +pub(crate) use worker_operations::{ + WorkerOperationContext, WorkspaceWorker, WorkspaceWorkerMethodSender, +}; + +#[path = "server_workspace_config.rs"] +mod workspace_config; use std::collections::{BTreeMap, HashMap, HashSet}; use std::net::IpAddr; use std::path::{Component, Path, PathBuf}; @@ -173,8 +182,8 @@ use crate::memory_staging::{ list_memory_staging_from_authority, memory_staging_backlog_from_authority, }; use crate::observation::{ - BackendObservationProxy, ObservationProxyError, RuntimeObservationClient, - RuntimeObservationSource, RuntimeObservationSourceConfig, + BackendObservationProxy, ObservationProxyError, RuntimeObservationSource, + RuntimeObservationSourceConfig, }; use crate::records::{ MergeRequestListItem, MergeRequestListResponse, MergeRequestRefDiagnostic, ObjectiveDetail, @@ -1077,6 +1086,9 @@ fn external_workdir_operation_kind(operation: &WorkdirSessionOperation) -> &'sta match operation { WorkdirSessionOperation::AuthorizeScope(_) => "authorize_scope", WorkdirSessionOperation::ScopeRulesOverlap(_) => "scope_rules_overlap", + WorkdirSessionOperation::CheckoutObserve(_) => "checkout_observe", + WorkdirSessionOperation::CheckoutExecute(_) => "checkout_execute", + WorkdirSessionOperation::CheckoutSearch(_) => "checkout_search", WorkdirSessionOperation::Stat(_) => "stat", WorkdirSessionOperation::Read(_) => "read", WorkdirSessionOperation::ReadBytes(_) => "read_bytes", @@ -1127,6 +1139,107 @@ impl workdir::WorkdirSession for ExternalProviderWorkdirSession { _ => Err(Self::mismatch("scope_rules_overlap")), } } + async fn checkout_observe( + &self, + path: workdir::WorkdirPath, + ) -> std::result::Result { + self.ensure_capability(workdir::WorkdirSessionCapability::Read)?; + match self + .operate(WorkdirSessionOperation::CheckoutObserve(path.clone())) + .await? + { + WorkdirSessionOperationResult::CheckoutObserve(mut value) + if value.path == path + && !value.validator.is_empty() + && value.validator.len() <= 256 => + { + value.capabilities = value.capabilities.intersection(self.capabilities()); + Ok(value) + } + _ => Err(Self::mismatch("checkout_observe")), + } + } + async fn checkout_execute( + &self, + request: workdir::CheckoutRequest, + ) -> std::result::Result { + let cap = request.operation.capability(); + self.ensure_capability(cap)?; + let target = request.target.clone(); + let mutation = cap != workdir::WorkdirSessionCapability::Read; + match self + .operate(WorkdirSessionOperation::CheckoutExecute(request)) + .await + { + Ok(WorkdirSessionOperationResult::CheckoutExecute(mut value)) + if value.observation.path == target + && !value.observation.validator.is_empty() + && value.observation.validator.len() <= 256 + && matches!( + (&value.output, cap), + ( + workdir::CheckoutOutput::Read(_), + workdir::WorkdirSessionCapability::Read + ) | ( + workdir::CheckoutOutput::Write(_), + workdir::WorkdirSessionCapability::Write + ) | ( + workdir::CheckoutOutput::Edit(_), + workdir::WorkdirSessionCapability::Edit + ) + ) => + { + value.observation.capabilities = value + .observation + .capabilities + .intersection(self.capabilities()); + Ok(value) + } + Ok(_) if mutation => Err(workdir::WorkdirError::OutcomeUnknown( + "mismatched checkout mutation response".into(), + )), + Ok(_) => Err(Self::mismatch("checkout_execute")), + Err(workdir::WorkdirError::Unavailable(_) | workdir::WorkdirError::Transport(_)) + if mutation => + { + Err(workdir::WorkdirError::OutcomeUnknown( + "checkout provider response lost; inspect effects before retry".into(), + )) + } + Err(error) => Err(error), + } + } + async fn checkout_search( + &self, + request: workdir::CheckoutSearchRequest, + ) -> std::result::Result { + use workdir::{ + CheckoutSearchOperation as Op, CheckoutSearchResult as Result, + WorkdirSessionCapability as Cap, + }; + let cap = match &request.operation { + Op::List(_) => Cap::Read, + Op::Glob(_) => Cap::Glob, + Op::Grep(_) => Cap::Grep, + }; + self.ensure_capability(cap)?; + match self + .operate(WorkdirSessionOperation::CheckoutSearch(request)) + .await? + { + WorkdirSessionOperationResult::CheckoutSearch(value) + if matches!( + (&value, cap), + (Result::List(_), Cap::Read) + | (Result::Glob(_), Cap::Glob) + | (Result::Grep(_), Cap::Grep) + ) => + { + Ok(value) + } + _ => Err(Self::mismatch("checkout_search")), + } + } async fn stat( &self, request: workdir::StatRequest, @@ -1379,6 +1492,56 @@ impl workdir::WorkdirSession for ExternalProviderWorkdirSession { } } +// Per-recipient reservation. No lock is held across Runtime calls: subscription +// callbacks may reenter the attention path while send_input runs. +#[derive(Default)] +struct OrchestratorAttentionState { + sending: bool, + restore_recheck: bool, + accepted: Option, + pending: Option, +} + +#[derive(Clone)] +struct OrchestratorAttentionDelivery { + fingerprint: String, + request_id: String, + content: String, +} + +struct OrchestratorAttentionReservation(Option>>); + +impl OrchestratorAttentionReservation { + fn complete(mut self) -> bool { + let state = self.0.take().expect("active attention reservation"); + let mut state = state.lock().unwrap_or_else(|p| p.into_inner()); + // Release and take deferred work atomically. A Restore arriving after + // release can reserve its own check; one arriving before is ours to drain. + state.sending = false; + std::mem::take(&mut state.restore_recheck) + } +} + +impl Drop for OrchestratorAttentionReservation { + fn drop(&mut self) { + if let Some(state) = &self.0 { + state.lock().unwrap_or_else(|p| p.into_inner()).sending = false; + } + } +} + +fn orchestrator_attention_state( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, +) -> Arc> { + api.orchestrator_attention + .lock() + .unwrap_or_else(|p| p.into_inner()) + .entry(worker.clone()) + .or_default() + .clone() +} + #[derive(Clone)] pub struct WorkspaceApi { pub(crate) config: ServerConfig, @@ -1396,7 +1559,8 @@ pub struct WorkspaceApi { runtime_binding_expectations: Arc>>, companion: Arc, orchestrator_spawn_lock: Arc>, - orchestrator_attention_fingerprint: Arc>>, + orchestrator_attention: + Arc>>>>, backend_job_drain_requests: Arc, observation_proxy: BackendObservationProxy, runtime_subscription_broker: RuntimeSubscriptionBroker, @@ -1973,35 +2137,6 @@ impl WorkerRemovalService { } } -impl crate::worker_source::VerifiedWorkerRemoveExecutor for WorkerRemovalService { - fn execute( - &self, - source: crate::worker_source::VerifiedWorkerMutationSource, - target_runtime_id: &str, - target_worker_id: &str, - reason: &str, - ) -> std::result::Result { - let executor = self.clone(); - let target_runtime_id = target_runtime_id.to_string(); - let target_worker_id = target_worker_id.to_string(); - let reason = reason.to_string(); - std::thread::spawn(move || { - tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .map_err(|error| error.to_string())? - .block_on(executor.execute_async( - source, - &target_runtime_id, - &target_worker_id, - &reason, - )) - }) - .join() - .map_err(|_| "embedded WorkerRemove executor thread panicked".to_string())? - } -} - #[derive(Clone)] pub struct WorkspaceServerApi { template: Arc, @@ -2422,6 +2557,23 @@ fn repository_api_rejection(path: &str, status: StatusCode, message: &str) -> Re status.into_response() } +/// Authored configuration editors are user-facing authority, not a generic +/// Runtime workspace.request capability. Worker configuration access is only +/// through the grant- and connection-bound WIP adapter. Keep runtime projections +/// (Prompt/Skill/config bundles) on their existing, separate consumer contracts. +fn is_workspace_config_editor_path(path: &str) -> bool { + let Some((_, resource)) = path + .strip_prefix("/api/w/") + .and_then(|scoped| scoped.split_once('/')) + else { + return false; + }; + resource == "config/source-tree" + || resource.starts_with("config/source-tree/") + || resource == "settings/profiles" + || resource.starts_with("settings/profiles/") +} + async fn authorize_scoped_workspace_request( api: &WorkspaceServerApi, workspace_id: &str, @@ -2482,6 +2634,13 @@ async fn authorize_scoped_workspace_request( "invalid runtime request proof", ) })?; + if is_workspace_config_editor_path(&request_path) { + return Err(repository_api_rejection( + &request_path, + StatusCode::FORBIDDEN, + "Runtime configuration editor access denied", + )); + } request.extensions_mut().insert(source); if !matches!( *request.method(), @@ -2614,6 +2773,13 @@ async fn authorize_workspace_api_request( "invalid runtime request proof", ); }; + if is_workspace_config_editor_path(&request_path) { + return repository_api_rejection( + &request_path, + StatusCode::FORBIDDEN, + "Runtime configuration editor access denied", + ); + } request.extensions_mut().insert(source); if !matches!( *request.method(), @@ -3247,7 +3413,7 @@ impl WorkspaceApi { runtime_binding_expectations: Arc::new(RwLock::new(HashMap::new())), companion, orchestrator_spawn_lock: Arc::new(std::sync::Mutex::new(())), - orchestrator_attention_fingerprint: Arc::new(Mutex::new(None)), + orchestrator_attention: Arc::new(Mutex::new(HashMap::new())), backend_job_drain_requests: Arc::new(AtomicU64::new(0)), observation_proxy, runtime_subscription_broker, @@ -3265,7 +3431,9 @@ impl WorkspaceApi { }; if let Some(dispatcher) = worker_remove_dispatcher { dispatcher - .install_executor(Arc::new(WorkerRemovalService::new(&api))) + .install_executor(Arc::new( + worker_operations::EmbeddedWorkspaceWorkerRemoveExecutor::new(&api), + )) .map_err(|message| Error::Config(message.to_string()))?; } let expired_external_workdirs = api.store.reconcile_external_workdir_grants_after_restart( @@ -3334,6 +3502,7 @@ impl WorkspaceApi { &self, request: &BackendJobRequest, ) -> Result { + self.resolve_backend_job_profile(request)?; let now = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); let reservation = self.store @@ -3382,6 +3551,19 @@ impl WorkspaceApi { return Ok(reservation); } let request = &reservation.job.request; + let (profile, config_bundle) = match self.resolve_backend_job_profile(request) { + Ok(resolved) => resolved, + Err(error) => { + self.finish_backend_job_attempt( + &request.job_id, + &reservation.attempt.attempt_id, + BackendJobAttemptState::Failed, + "profile_requirements", + &error.to_string(), + )?; + return Err(error); + } + }; let worker_input = request.worker_input(&reservation.attempt.attempt_id)?; let control_operation = WorkerControlOperation { operation_id: allocation_key(&reservation.attempt.attempt_id), @@ -3396,22 +3578,24 @@ impl WorkspaceApi { job_id: request.job_id.clone(), attempt_id: reservation.attempt.attempt_id.clone(), purpose: request.purpose.clone(), + input_revision: request.input_revision.clone(), + subjektiv_consolidation: request.grants.subjektiv_consolidation.is_some(), }, acceptance: WorkerSpawnAcceptanceRequirement::RunAccepted { expected_segments: 0, }, - profile: ProfileSelector::Builtin(request.profile.clone()), + profile, ticket_assignment: None, initial_submit: Vec::new(), workdir_attachment_requests: Vec::new(), resolved_workdir_attachment_requests: Vec::new(), resolved_workdir_attachments: Vec::new(), - resolved_config_bundle: None, + resolved_config_bundle: Some(config_bundle), resolved_worker_observation_enabled: false, resolved_worker_observation_grants: Vec::new(), resolved_workspace_api: None, resolved_memory_settings: None, - resolved_subjektiv_attached: false, + resolved_subjektiv_attached: request.grants.subjektiv_consolidation.is_some(), resolved_control_operation: Some(control_operation), }, ) { @@ -3481,6 +3665,7 @@ impl WorkspaceApi { job, attempt, replayed: true, + resource_reused: false, }); } @@ -3572,6 +3757,7 @@ impl WorkspaceApi { job, attempt, replayed: reservation.replayed, + resource_reused: reservation.resource_reused, }) } @@ -3635,6 +3821,7 @@ impl WorkspaceApi { job, attempt: attempt.clone(), replayed: true, + resource_reused: false, }) { Ok(reservation) => { made_progress |= @@ -4021,6 +4208,7 @@ impl WorkspaceApi { job, attempt, replayed: true, + resource_reused: false, }) { tracing::warn!(%error, "Backend Job reserved-attempt recovery failed"); @@ -4126,6 +4314,7 @@ impl WorkspaceApi { submission: &BackendJobResultSubmission, ) -> Result { let accepted_at = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); + let mut verified_submission = submission.clone(); if let Some(job) = self .store .get_backend_job(&self.config.workspace_id, &submission.job_id)? @@ -4168,10 +4357,41 @@ impl WorkspaceApi { return Err(error); } } + if let Some(job) = self + .store + .get_backend_job(&self.config.workspace_id, &submission.job_id)? + && job.request.grants.subjektiv_consolidation.is_some() + { + let attempt = self + .store + .get_backend_job_attempt( + &self.config.workspace_id, + &submission.job_id, + &submission.attempt_id, + )? + .ok_or_else(|| Error::InvalidInput("unknown consolidation attempt".into()))?; + if attempt.worker.as_ref() != Some(source_worker) + || job.current_attempt != attempt.attempt + || attempt.input_revision != submission.input_revision + || !matches!( + attempt.state, + BackendJobAttemptState::Dispatched | BackendJobAttemptState::Completed + ) + { + return Err(Error::WorkspacePermissionDenied( + "consolidation result requires the exact bound attempt".into(), + )); + } + if attempt.state != BackendJobAttemptState::Completed { + subjektiv_job::validate_result(self, &job.request, &attempt, &submission.result)?; + } + verified_submission.result = + subjektiv_job::with_dispositions(self, &job.request, &submission.result)?; + } let acceptance = self.store.accept_backend_job_result( &self.config.workspace_id, source_worker, - submission, + &verified_submission, &accepted_at, )?; if let Some(target) = acceptance.job.request.notification_target.as_ref() { @@ -4739,6 +4959,7 @@ impl WorkspaceApi { for (alias, workdir_id, reservation_id, capabilities) in &reserved_attachments { let attachment = WorkerWorkdirLinkRecord { + connection_id: String::new(), workspace_id: self.config.workspace_id.clone(), worker: worker_ref.clone(), workdir_id: workdir_id.clone(), @@ -4793,100 +5014,235 @@ impl WorkspaceApi { Ok(result) } + fn current_restore_summary( + &self, + worker: &RuntimeWorkerRef, + has_summary: bool, + ) -> Option { + if !has_summary { + return None; + } + let mut summary = self.runtime.worker(worker).ok()?; + // Historical receipts are not observations. Read current metadata without + // requiring current singleton eligibility and never write it back to catalog. + summary.singleton_key = self.config_store.with_conn(|conn| { + conn.query_row("SELECT singleton_key FROM worker_singleton_owners WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3", + rusqlite::params![self.workspace_id(), worker.runtime_id, worker.worker_id], |row| row.get::<_, String>(0)) + .map(Some).or_else(|error| if matches!(error, rusqlite::Error::QueryReturnedNoRows) { Ok(None) } else { Err(error) }).map_err(Error::from) + }).ok().flatten(); + Some(summary) + } + fn restore_workspace_worker( &self, worker: &RuntimeWorkerRef, + request: server_api::WorkerRestoreRequest, ) -> ApiResult { - // Fence displaced singleton generations before renewing credentials, replacing bindings, - // or asking any Runtime to launch execution. - let singleton_lease = self - .store - .require_current_worker_singleton_owner(&self.config.workspace_id, worker)?; - for link in self - .store - .list_worker_workdir_links(&self.config.workspace_id, worker)? - .into_iter() - .filter(|link| link.unlinked_at.is_none()) + if request.request_id.trim().is_empty() + || request.request_id.len() > 512 + || request.expected_observation_token.is_empty() + || request.expected_observation_token.len() > 256 { - let record = self + return Err(Error::InvalidInput( + "Restore requires a bounded request_id and expected_observation_token".into(), + ) + .into()); + } + // Auth is checked by WorkspaceWorker before this read-only owner dispatch. + // Existing owners reach receipts/reconciliation before NEW launch eligibility. + let (recovered, owned_preparation) = self + .runtime + .coordinate_worker_restore( + worker, + runtime_api::WorkerRestoreCoordinationRequest { + expected_observation_token: request.expected_observation_token.clone(), + request_id: request.request_id.clone(), + preparation: None, + }, + ) + .map_err(RuntimeRegistryError::into_error)?; + if let Some(mut result) = recovered { + result.worker = self.current_restore_summary(worker, result.worker.is_some()); + return Ok(result); + } + let mut preparation = if let Some(preparation) = owned_preparation { + preparation + } else { + self.store + .require_current_worker_singleton_owner(&self.config.workspace_id, worker)?; + let mut repository_access_workdirs = Vec::new(); + for link in self .store - .get_workdir_registry(&self.config.workspace_id, &link.workdir_id)? - .ok_or_else(|| Error::RuntimeOperationFailed { - runtime_id: "workspace-backend".to_string(), - code: "working_directory_not_found".to_string(), - message: format!("Unknown Workdir `{}`", link.workdir_id), - })?; - match record.source { - WorkdirRegistrySource::Repository { runtime_id, .. } => { - if let Some(access) = repository_access_request_for_workdir( - self, - &runtime_id, - &link.workdir_id, - &format!("worker-restore:{}", WorkerId::now_v7()), - )? { - self.runtime - .authorize_working_directory_repository_access(&runtime_id, access) - .map_err(RuntimeRegistryError::into_error)?; + .list_worker_workdir_links(&self.config.workspace_id, worker)? + .into_iter() + .filter(|link| link.unlinked_at.is_none()) + { + let record = self + .store + .get_workdir_registry(&self.config.workspace_id, &link.workdir_id)? + .ok_or_else(|| Error::RuntimeOperationFailed { + runtime_id: "workspace-backend".to_string(), + code: "working_directory_not_found".to_string(), + message: format!("Unknown Workdir `{}`", link.workdir_id), + })?; + match record.source { + WorkdirRegistrySource::Repository { + runtime_id, + repository_id, + } => { + if self + .require_configured_workspace_repository(&repository_id)? + .source + .kind + == server_api::RepositorySourceKind::Ssh + { + repository_access_workdirs.push( + runtime_api::RepositoryAccessWorkdirReference { + runtime_id, + working_directory_id: link.workdir_id.clone(), + }, + ); + } } - } - WorkdirRegistrySource::ExternalGrant { grant_id } => { - let grant = self - .store - .get_external_workdir_grant(&self.config.workspace_id, &grant_id)? - .ok_or_else(|| Error::RuntimeOperationFailed { - runtime_id: "external-workdir-provider".to_string(), - code: "external_workdir_unavailable".to_string(), - message: "External Workdir grant is unavailable".to_string(), - })?; - let grant_expired = external_workdir_grant_is_expired(&grant)?; - let connection = self - .external_workdir_providers - .lock() - .expect("External Workdir provider registry poisoned") - .connection(&grant_id); - if grant.status != "online" - || grant_expired - || !connection.is_some_and(|connection| { - connection.generation == grant.generation - && connection.provider_instance_id == grant.provider_instance_id - && connection.workdir_id == link.workdir_id - && connection - .expires_at - .is_none_or(|expires_at| expires_at > Utc::now()) - }) - { - return Err(Error::RuntimeOperationFailed { - runtime_id: "external-workdir-provider".to_string(), - code: "external_workdir_unavailable".to_string(), - message: "External Workdir provider is offline".to_string(), + WorkdirRegistrySource::WorkspaceConfig { .. } => continue, + WorkdirRegistrySource::ExternalGrant { grant_id } => { + let grant = self + .store + .get_external_workdir_grant(&self.config.workspace_id, &grant_id)? + .ok_or_else(|| Error::RuntimeOperationFailed { + runtime_id: "external-workdir-provider".to_string(), + code: "external_workdir_unavailable".to_string(), + message: "External Workdir grant is unavailable".to_string(), + })?; + let grant_expired = external_workdir_grant_is_expired(&grant)?; + let connection = self + .external_workdir_providers + .lock() + .expect("External Workdir provider registry poisoned") + .connection(&grant_id); + if grant.status != "online" + || grant_expired + || !connection.is_some_and(|connection| { + connection.generation == grant.generation + && connection.provider_instance_id == grant.provider_instance_id + && connection.workdir_id == link.workdir_id + && connection + .expires_at + .is_none_or(|expires_at| expires_at > Utc::now()) + }) + { + return Err(Error::RuntimeOperationFailed { + runtime_id: "external-workdir-provider".to_string(), + code: "external_workdir_unavailable".to_string(), + message: "External Workdir provider is offline".to_string(), + } + .into()); } - .into()); } } } - } - let mut binding = self - .runtime - .replace_worker_workspace_api(worker, self.workspace_api_ref(&worker.runtime_id)) - .map_err(|error| error.into_error())?; - if let Some(summary) = binding.worker.as_mut() { - summary.singleton_key = singleton_lease.as_ref().map(|lease| lease.key.clone()); - } - if binding.state != InternalWorkerOperationState::Accepted { + let preparation = runtime_api::WorkerRestorePreparation { + workspace_api: Some(runtime_contract_restore( + self.workspace_api_ref(&worker.runtime_id), + )?), + workdir_attachments: Some(runtime_contract_restore( + restore_logical_workdir_attachments(self, worker)?, + )?), + repository_access: Vec::new(), + repository_access_workdirs, + }; + let (result, preparation) = self + .runtime + .coordinate_worker_restore( + worker, + runtime_api::WorkerRestoreCoordinationRequest { + expected_observation_token: request.expected_observation_token.clone(), + request_id: request.request_id.clone(), + preparation: Some(preparation), + }, + ) + .map_err(RuntimeRegistryError::into_error)?; + if let Some(mut result) = result { + result.worker = self.current_restore_summary(worker, result.worker.is_some()); + return Ok(result); + } + preparation.ok_or_else(|| { + Error::Config("Runtime did not retain admitted Restore preparation".into()) + })? + }; + // Only the admitted operation may issue SSH handles. Reissue from the pinned + // credential snapshot after interruption; never renew candidate bindings. + let completion = + complete_restore_repository_access(self, &request.request_id, &mut preparation); + if let Err(error) = completion { return Ok(InternalWorkerRestoreResult { - state: server_api::WorkerRestoreState::Rejected, - worker: binding.worker, - diagnostics: binding.diagnostics, + state: server_api::WorkerRestoreState::ReconciliationRequired, + worker: None, + diagnostics: vec![RuntimeDiagnostic::new( + "worker_restore_preparation_pending", + "warning", + sanitize_backend_error(&error.error.to_string()), + )], }); } - sync_runtime_worker_workdir_attachments(self, worker)?; + // Persist the first resource snapshot at the admitted owner BEFORE any + // Workdir Runtime authorization effect. Replays renew only its opaque handles. + let (result, retained) = self + .runtime + .coordinate_worker_restore( + worker, + runtime_api::WorkerRestoreCoordinationRequest { + expected_observation_token: request.expected_observation_token.clone(), + request_id: request.request_id.clone(), + preparation: Some(preparation), + }, + ) + .map_err(RuntimeRegistryError::into_error)?; + if let Some(mut result) = result { + result.worker = self.current_restore_summary(worker, result.worker.is_some()); + return Ok(result); + } + let preparation = retained.ok_or_else(|| { + Error::Config("Runtime did not retain Restore resource snapshot".into()) + })?; + for access in &preparation.repository_access { + let access: worker_runtime::catalog::WorkingDirectoryRepositoryAccessRequest = + runtime_contract_restore(access.clone())?; + let runtime_id = access.materialization.runtime_id.clone(); + if let Err(error) = self + .runtime + .authorize_working_directory_repository_access(&runtime_id, access) + { + return Ok(InternalWorkerRestoreResult { + state: server_api::WorkerRestoreState::ReconciliationRequired, + worker: None, + diagnostics: vec![RuntimeDiagnostic::new( + "worker_restore_repository_access_pending", + "warning", + sanitize_backend_error(&error.message()), + )], + }); + } + } let mut restored = self .runtime - .restore_worker(worker) - .map_err(|error| error.into_error())?; - if let Some(summary) = restored.worker.as_mut() { - summary.singleton_key = singleton_lease.as_ref().map(|lease| lease.key.clone()); + .restore_worker( + worker, + runtime_api::WorkerRestoreRequest { + expected_observation_token: request.expected_observation_token, + request_id: request.request_id, + preparation: Some(preparation), + }, + ) + .map_err(RuntimeRegistryError::into_error)?; + if restored + .diagnostics + .iter() + .any(|d| d.code == "restore_observation_conflict") + { + return Err(Error::RestoreObservationConflict.into()); } + restored.worker = self.current_restore_summary(worker, restored.worker.is_some()); Ok(restored) } @@ -4995,7 +5351,8 @@ impl WorkspaceApi { runtime_id, repository_id, } => (runtime_id, repository_id), - WorkdirRegistrySource::ExternalGrant { .. } => { + WorkdirRegistrySource::ExternalGrant { .. } + | WorkdirRegistrySource::WorkspaceConfig { .. } => { return Err(Error::InvalidInput(format!( "Ticket `{ticket_id}` attachment `{}` is an ExternalGrant and cannot satisfy a persisted Repository target", claim.alias @@ -5138,6 +5495,9 @@ fn workdir_source_capabilities( ) -> Result { match &workdir.source { WorkdirRegistrySource::Repository { .. } => Ok(workdir::WorkdirSessionCapabilities::ALL), + WorkdirRegistrySource::WorkspaceConfig { grant_id } => { + workspace_config::grant_capabilities(api, grant_id) + } WorkdirRegistrySource::ExternalGrant { grant_id } => api .store .get_external_workdir_grant(&api.config.workspace_id, grant_id)? @@ -5243,6 +5603,14 @@ fn ticket_target_capabilities_for_workdir( workdir: &WorkdirRegistryRecord, ) -> Result { let source_capabilities = workdir_source_capabilities(api, workdir)?; + if let WorkdirRegistrySource::WorkspaceConfig { grant_id } = &workdir.source { + if !workspace_config::granted_to(api, worker, grant_id) { + return Err(Error::WorkspacePermissionDenied( + "Workspace config grant unavailable".into(), + )); + } + return Ok(source_capabilities); + } let Some(assignment) = api .store .get_current_ticket_role_assignment_for_worker(&api.config.workspace_id, worker)? @@ -5345,12 +5713,23 @@ fn validate_manual_ticket_coder_workdir_binding( )) .into()); } + if let WorkdirRegistrySource::WorkspaceConfig { grant_id } = &workdir.source { + if !workspace_config::granted_to(api, worker, grant_id) { + return Err(Error::WorkspacePermissionDenied( + "Workspace config grant unavailable".into(), + ) + .into()); + } + effective_links.push(link.clone()); + continue; + } let (workdir_runtime_id, repository_id) = match &workdir.source { WorkdirRegistrySource::Repository { runtime_id, repository_id, } => (runtime_id, repository_id), - WorkdirRegistrySource::ExternalGrant { .. } => { + WorkdirRegistrySource::ExternalGrant { .. } + | WorkdirRegistrySource::WorkspaceConfig { .. } => { return Err(Error::InvalidInput(format!( "Ticket `{ticket_id}` attachment `{}` is an ExternalGrant and cannot satisfy a persisted Repository target", link.alias @@ -5555,6 +5934,49 @@ fn contract_request_headers( Ok(headers) } +/// Self APIs derive identity from authenticated middleware, never caller-supplied hints. +fn current_worker_contract_headers( + context: &server_api::ServerRequestContext, +) -> std::result::Result { + let source = context.runtime_source.as_ref().ok_or_else(|| { + ApiError::from(Error::WorkspacePermissionDenied( + "self Workdir requests require a verified Runtime Worker source".to_string(), + )) + .into_repository_api_error() + })?; + let worker_id = source.worker_id.as_deref().ok_or_else(|| { + ApiError::from(Error::WorkspacePermissionDenied( + "self Workdir requests require a Runtime-bound Worker identity".to_string(), + )) + .into_repository_api_error() + })?; + let mut headers = contract_request_headers(context)?; + for (name, expected) in [ + ("x-yoi-runtime-id", source.runtime_id.as_str()), + ("x-yoi-worker-id", worker_id), + ] { + if headers + .get(name) + .is_some_and(|value| value.as_bytes() != expected.as_bytes()) + { + return Err(ApiError::from(Error::WorkspacePermissionDenied( + "self Workdir request identity does not match its verified source".to_string(), + )) + .into_repository_api_error()); + } + headers.insert( + name, + HeaderValue::from_str(expected).map_err(|_| { + ApiError::from(Error::WorkerSourceIdentity( + "invalid verified Worker identity".to_string(), + )) + .into_repository_api_error() + })?, + ); + } + Ok(headers) +} + async fn attach_server_origin_context(mut request: Request, next: Next) -> Response { let actor = request.extensions().get::().cloned(); let origin = request_origin(request.headers()); @@ -5760,6 +6182,7 @@ fn generated_workspace_contract_router(service: ServerApiContractService) -> Rou .merge(server_api::server_api_axum::subjektiv_subject_get( service.clone(), )) + .merge(server_api::server_api_axum::subjektiv_subject_behavior_update(service.clone())) .merge(server_api::server_api_axum::subjektiv_resident_surface( service.clone(), )) @@ -5841,6 +6264,23 @@ fn generated_workspace_contract_router(service: ServerApiContractService) -> Rou .merge(server_api::server_api_axum::runtime_workdir_cleanup( service.clone(), )) + .merge(server_api::server_api_axum::current_worker_workdir_catalog( + service.clone(), + )) + .merge(server_api::server_api_axum::current_worker_workspace_config_get(service.clone())) + .merge(server_api::server_api_axum::current_worker_workspace_config_attach(service.clone())) + .merge( + server_api::server_api_axum::current_worker_workspace_config_observe(service.clone()), + ) + .merge(server_api::server_api_axum::current_worker_workspace_config_read(service.clone())) + .merge(server_api::server_api_axum::current_worker_workspace_config_commit(service.clone())) + .merge(server_api::server_api_axum::workspace_config_grant_create( + service.clone(), + )) + .merge(server_api::server_api_axum::workspace_config_grant_revoke( + service.clone(), + )) + .merge(server_api::server_api_axum::current_worker_workdir_attachment_list(service.clone())) .merge(server_api::server_api_axum::current_worker_workdir_attach( service.clone(), )) @@ -6304,23 +6744,11 @@ fn worker_input_request_from_api( ))); } }; - let segments = request - .segments - .map(|segments| { - segments - .into_iter() - .map(|segment| { - serde_json::from_value::(segment) - .map_err(|error| invalid_contract_value("Worker input segment", error)) - }) - .collect::>>() - }) - .transpose()?; Ok(WorkerInputRequest { kind, content: request.content, submission_request_id: None, - segments, + segments: request.segments, }) } @@ -6339,9 +6767,10 @@ fn worker_completions_request_from_api( request: server_api::RuntimeWorkerCompletionsRequest, ) -> Result { Ok(WorkerCompletionsRequest { - kind: serde_json::from_value(request.kind) - .map_err(|error| invalid_contract_value("Worker completion kind", error))?, + kind: request.kind, prefix: request.prefix, + context: request.context, + request_id: request.request_id, }) } @@ -6411,7 +6840,18 @@ fn worker_spawn_result_to_api( }) } -fn worker_input_result_to_api(result: WorkerInputResult) -> server_api::RuntimeWorkerInputResult { +fn worker_input_result_to_api( + mut result: WorkerInputResult, +) -> server_api::RuntimeWorkerInputResult { + // The existing wire result has no disposition field. Keep uncertain delivery + // distinguishable without collapsing it into a definite refusal or replaying input. + if result.disposition == WorkerInputDisposition::Unknown { + result.diagnostics.push(RuntimeDiagnostic { + code: "worker_input_outcome_unknown".into(), + severity: HostDiagnosticSeverity::Warning, + message: "Worker input delivery outcome is unknown; inspect the current session before retrying".into(), + }); + } server_api::RuntimeWorkerInputResult { state: result.state.into(), runtime_id: result.worker.runtime_id, @@ -6437,17 +6877,11 @@ fn worker_completions_result_to_api( Ok(server_api::RuntimeWorkerCompletionsResult { runtime_id: result.worker.runtime_id, worker_id: result.worker.worker_id, - kind: serde_json::to_value(result.kind) - .map_err(|error| invalid_contract_value("Worker completion kind", error))?, + kind: result.kind, prefix: result.prefix, - entries: result - .entries - .into_iter() - .map(|entry| { - serde_json::to_value(entry) - .map_err(|error| invalid_contract_value("Worker completion entry", error)) - }) - .collect::>>()?, + context: result.context, + request_id: result.request_id, + entries: result.entries, diagnostics: result.diagnostics.into_iter().map(Into::into).collect(), }) } @@ -7435,6 +7869,26 @@ impl server_api::ServerApi for ServerApiContractService { .map_err(ApiError::into_repository_api_error) } + async fn subjektiv_subject_behavior_update( + &self, + workspace_id: String, + subject_id: String, + request: server_api::SubjektivSubjectBehaviorUpdateRequest, + ) -> std::result::Result + { + scoped_update_subjektiv_subject_behavior( + State(self.workspace_api()?.clone()), + AxumPath(ScopedSubjektivSubjectPath { + workspace_id, + subject_id, + }), + Json(request), + ) + .await + .map(|Json(response)| response) + .map_err(ApiError::into_repository_api_error) + } + async fn subjektiv_resident_surface( &self, workspace_id: String, @@ -7850,7 +8304,7 @@ impl server_api::ServerApi for ServerApiContractService { let Json(response) = list_known_workers( State(self.workspace_api()?.clone()), AxumPath(ScopedWorkspacePath { workspace_id }), - contract_request_headers(&context)?, + current_worker_contract_headers(&context)?, ) .await .map_err(ApiError::into_repository_api_error)?; @@ -7867,7 +8321,7 @@ impl server_api::ServerApi for ServerApiContractService { spawn_known_worker( State(self.workspace_api()?.clone()), AxumPath(ScopedWorkspacePath { workspace_id }), - contract_request_headers(&context)?, + current_worker_contract_headers(&context)?, Json(request), ) .await @@ -7890,7 +8344,7 @@ impl server_api::ServerApi for ServerApiContractService { workspace_id, worker: RuntimeWorkerRef::new(runtime_id, worker_id), }), - contract_request_headers(&context)?, + current_worker_contract_headers(&context)?, Json(request), ) .await @@ -7913,7 +8367,7 @@ impl server_api::ServerApi for ServerApiContractService { workspace_id, worker: RuntimeWorkerRef::new(runtime_id, worker_id), }), - contract_request_headers(&context)?, + current_worker_contract_headers(&context)?, Json(request), ) .await @@ -7936,7 +8390,7 @@ impl server_api::ServerApi for ServerApiContractService { workspace_id, worker: RuntimeWorkerRef::new(runtime_id, worker_id), }), - contract_request_headers(&context)?, + current_worker_contract_headers(&context)?, Json(request), ) .await @@ -7950,6 +8404,7 @@ impl server_api::ServerApi for ServerApiContractService { workspace_id: String, runtime_id: String, worker_id: String, + request: server_api::WorkerRestoreRequest, ) -> std::result::Result { restore_known_worker( @@ -7958,7 +8413,8 @@ impl server_api::ServerApi for ServerApiContractService { workspace_id, worker: RuntimeWorkerRef::new(runtime_id, worker_id), }), - contract_request_headers(&context)?, + current_worker_contract_headers(&context)?, + request, ) .await .map(|Json(response)| response) @@ -7976,7 +8432,7 @@ impl server_api::ServerApi for ServerApiContractService { let Json(response) = scoped_list_worker_observation_sessions( State(self.workspace_api()?.clone()), AxumPath(ScopedWorkspacePath { workspace_id }), - contract_request_headers(&context)?, + current_worker_contract_headers(&context)?, ) .await .map_err(ApiError::into_repository_api_error)?; @@ -7995,7 +8451,7 @@ impl server_api::ServerApi for ServerApiContractService { let Json(response) = scoped_capture_worker_observation_session( State(self.workspace_api()?.clone()), AxumPath(ScopedWorkspacePath { workspace_id }), - contract_request_headers(&context)?, + current_worker_contract_headers(&context)?, Json(request), ) .await @@ -8670,34 +9126,43 @@ impl server_api::ServerApi for ServerApiContractService { async fn runtime_worker_restore_alias( &self, + context: server_api::ServerRequestContext, runtime_id: String, worker_id: String, + request: server_api::WorkerRestoreRequest, ) -> std::result::Result { - restore_runtime_worker( - State(self.workspace_api()?.clone()), - AxumPath((runtime_id, worker_id)), - ) - .await - .map(|Json(response)| response) - .map_err(ApiError::into_repository_api_error) + let api = self.workspace_api()?.clone(); + let operation_context = WorkerOperationContext::from_request(&api, &context)?; + restore_runtime_worker_with_context(api, runtime_id, worker_id, operation_context, request) + .await + .map(|Json(response)| response) + .map_err(ApiError::into_repository_api_error) } async fn runtime_worker_restore( &self, + context: server_api::ServerRequestContext, workspace_id: String, runtime_id: String, worker_id: String, query: server_api::RestoreTicketAssignmentQuery, + request: server_api::WorkerRestoreRequest, ) -> std::result::Result { - scoped_restore_runtime_worker( - State(self.workspace_api()?.clone()), - AxumPath(ScopedRuntimeWorkerPath { + let api = self.workspace_api()?.clone(); + let operation_context = WorkerOperationContext::from_request(&api, &context)?; + validate_workspace_scope(&api, &workspace_id) + .map_err(ApiError::into_repository_api_error)?; + scoped_restore_runtime_worker_with_context( + api, + ScopedRuntimeWorkerPath { workspace_id, worker: RuntimeWorkerRef::new(runtime_id, worker_id), - }), - Query(query), + }, + query, + operation_context, + request, ) .await .map(|Json(response)| response) @@ -8706,40 +9171,42 @@ impl server_api::ServerApi for ServerApiContractService { async fn runtime_worker_pin( &self, + context: server_api::ServerRequestContext, workspace_id: String, runtime_id: String, worker_id: String, ) -> std::result::Result { - let Json(response) = scoped_pin_runtime_worker( - State(self.workspace_api()?.clone()), - AxumPath(ScopedRuntimeWorkerPath { - workspace_id, - worker: RuntimeWorkerRef::new(runtime_id, worker_id), - }), - ) - .await - .map_err(ApiError::into_repository_api_error)?; - Ok(response) + let api = self.workspace_api()?.clone(); + let operation_context = WorkerOperationContext::from_request(&api, &context)?; + validate_workspace_scope(&api, &workspace_id) + .map_err(ApiError::into_repository_api_error)?; + let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) + .map_err(|error| ApiError::from(error).into_repository_api_error())?; + worker + .set_pinned(&operation_context, true) + .await + .map_err(ApiError::into_repository_api_error) } async fn runtime_worker_unpin( &self, + context: server_api::ServerRequestContext, workspace_id: String, runtime_id: String, worker_id: String, ) -> std::result::Result { - let Json(response) = scoped_unpin_runtime_worker( - State(self.workspace_api()?.clone()), - AxumPath(ScopedRuntimeWorkerPath { - workspace_id, - worker: RuntimeWorkerRef::new(runtime_id, worker_id), - }), - ) - .await - .map_err(ApiError::into_repository_api_error)?; - Ok(response) + let api = self.workspace_api()?.clone(); + let operation_context = WorkerOperationContext::from_request(&api, &context)?; + validate_workspace_scope(&api, &workspace_id) + .map_err(ApiError::into_repository_api_error)?; + let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) + .map_err(|error| ApiError::from(error).into_repository_api_error())?; + worker + .set_pinned(&operation_context, false) + .await + .map_err(ApiError::into_repository_api_error) } async fn runtime_cleanup_plan( @@ -8762,6 +9229,7 @@ impl server_api::ServerApi for ServerApiContractService { async fn runtime_cleanup_execute( &self, + context: server_api::ServerRequestContext, workspace_id: String, runtime_id: String, request: server_api::ExecuteRuntimeCleanupRequest, @@ -8769,55 +9237,58 @@ impl server_api::ServerApi for ServerApiContractService { server_api::RuntimeCleanupExecutionResponse, server_api::RepositoryApiError, > { - let Json(response) = scoped_execute_runtime_cleanup( - State(self.workspace_api()?.clone()), - AxumPath(ScopedRuntimePath { - workspace_id, - runtime_id, - }), - Json(request), - ) - .await - .map_err(ApiError::into_repository_api_error)?; - Ok(response) + let api = self.workspace_api()?.clone(); + let operation_context = WorkerOperationContext::from_request(&api, &context)?; + validate_workspace_scope(&api, &workspace_id) + .map_err(ApiError::into_repository_api_error)?; + execute_runtime_cleanup_with_context(&api, &runtime_id, request, &operation_context) + .await + .map_err(ApiError::into_repository_api_error) } async fn runtime_worker_input_alias( &self, + context: server_api::ServerRequestContext, runtime_id: String, worker_id: String, request: server_api::RuntimeWorkerInputRequest, ) -> std::result::Result { - let Json(response) = send_runtime_worker_input( - State(self.workspace_api()?.clone()), - AxumPath((runtime_id, worker_id)), - Json(request), - ) - .await - .map_err(ApiError::into_repository_api_error)?; - Ok(response) + let api = self.workspace_api()?.clone(); + let operation_context = WorkerOperationContext::from_request(&api, &context)?; + let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) + .map_err(|error| ApiError::from(error).into_repository_api_error())?; + let request = worker_input_request_from_api(request) + .map_err(|error| ApiError::from(error).into_repository_api_error())?; + worker + .input(&operation_context, request) + .await + .map(worker_input_result_to_api) + .map_err(ApiError::into_repository_api_error) } async fn runtime_worker_input( &self, + context: server_api::ServerRequestContext, workspace_id: String, runtime_id: String, worker_id: String, request: server_api::RuntimeWorkerInputRequest, ) -> std::result::Result { - let Json(response) = scoped_send_runtime_worker_input( - State(self.workspace_api()?.clone()), - AxumPath(ScopedRuntimeWorkerPath { - workspace_id, - worker: RuntimeWorkerRef::new(runtime_id, worker_id), - }), - Json(request), - ) - .await - .map_err(ApiError::into_repository_api_error)?; - Ok(response) + let api = self.workspace_api()?.clone(); + let operation_context = WorkerOperationContext::from_request(&api, &context)?; + validate_workspace_scope(&api, &workspace_id) + .map_err(ApiError::into_repository_api_error)?; + let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) + .map_err(|error| ApiError::from(error).into_repository_api_error())?; + let request = worker_input_request_from_api(request) + .map_err(|error| ApiError::from(error).into_repository_api_error())?; + worker + .input(&operation_context, request) + .await + .map(worker_input_result_to_api) + .map_err(ApiError::into_repository_api_error) } async fn runtime_worker_attachment_upload_grant( @@ -8962,78 +9433,96 @@ impl server_api::ServerApi for ServerApiContractService { async fn runtime_worker_stop_alias( &self, + context: server_api::ServerRequestContext, runtime_id: String, worker_id: String, request: server_api::RuntimeWorkerLifecycleRequest, ) -> std::result::Result { - let Json(response) = stop_runtime_worker( - State(self.workspace_api()?.clone()), - AxumPath((runtime_id, worker_id)), - Json(request), - ) - .await - .map_err(ApiError::into_repository_api_error)?; - Ok(response) + let api = self.workspace_api()?.clone(); + let operation_context = WorkerOperationContext::from_request(&api, &context)?; + let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) + .map_err(|error| ApiError::from(error).into_repository_api_error())?; + worker + .stop( + &operation_context, + worker_lifecycle_request_from_api(request), + ) + .await + .map(worker_lifecycle_result_to_api) + .map_err(ApiError::into_repository_api_error) } async fn runtime_worker_stop( &self, + context: server_api::ServerRequestContext, workspace_id: String, runtime_id: String, worker_id: String, request: server_api::RuntimeWorkerLifecycleRequest, ) -> std::result::Result { - let Json(response) = scoped_stop_runtime_worker( - State(self.workspace_api()?.clone()), - AxumPath(ScopedRuntimeWorkerPath { - workspace_id, - worker: RuntimeWorkerRef::new(runtime_id, worker_id), - }), - Json(request), - ) - .await - .map_err(ApiError::into_repository_api_error)?; - Ok(response) + let api = self.workspace_api()?.clone(); + let operation_context = WorkerOperationContext::from_request(&api, &context)?; + validate_workspace_scope(&api, &workspace_id) + .map_err(ApiError::into_repository_api_error)?; + let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) + .map_err(|error| ApiError::from(error).into_repository_api_error())?; + worker + .stop( + &operation_context, + worker_lifecycle_request_from_api(request), + ) + .await + .map(worker_lifecycle_result_to_api) + .map_err(ApiError::into_repository_api_error) } async fn runtime_worker_cancel_alias( &self, + context: server_api::ServerRequestContext, runtime_id: String, worker_id: String, request: server_api::RuntimeWorkerLifecycleRequest, ) -> std::result::Result { - let Json(response) = cancel_runtime_worker( - State(self.workspace_api()?.clone()), - AxumPath((runtime_id, worker_id)), - Json(request), - ) - .await - .map_err(ApiError::into_repository_api_error)?; - Ok(response) + let api = self.workspace_api()?.clone(); + let operation_context = WorkerOperationContext::from_request(&api, &context)?; + let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) + .map_err(|error| ApiError::from(error).into_repository_api_error())?; + worker + .cancel( + &operation_context, + worker_lifecycle_request_from_api(request), + ) + .await + .map(worker_lifecycle_result_to_api) + .map_err(ApiError::into_repository_api_error) } async fn runtime_worker_cancel( &self, + context: server_api::ServerRequestContext, workspace_id: String, runtime_id: String, worker_id: String, request: server_api::RuntimeWorkerLifecycleRequest, ) -> std::result::Result { - let Json(response) = scoped_cancel_runtime_worker( - State(self.workspace_api()?.clone()), - AxumPath(ScopedRuntimeWorkerPath { - workspace_id, - worker: RuntimeWorkerRef::new(runtime_id, worker_id), - }), - Json(request), - ) - .await - .map_err(ApiError::into_repository_api_error)?; - Ok(response) + let api = self.workspace_api()?.clone(); + let operation_context = WorkerOperationContext::from_request(&api, &context)?; + validate_workspace_scope(&api, &workspace_id) + .map_err(ApiError::into_repository_api_error)?; + let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) + .map_err(|error| ApiError::from(error).into_repository_api_error())?; + worker + .cancel( + &operation_context, + worker_lifecycle_request_from_api(request), + ) + .await + .map(worker_lifecycle_result_to_api) + .map_err(ApiError::into_repository_api_error) } async fn repository_list( @@ -10372,6 +10861,162 @@ impl server_api::ServerApi for ServerApiContractService { .map_err(ApiError::into_repository_api_error) } + async fn current_worker_workspace_config_get( + &self, + context: server_api::ServerRequestContext, + workspace_id: String, + ) -> std::result::Result< + server_api::WorkspaceConfigCurrentResponse, + server_api::WorkspaceConfigApiError, + > { + let api = self.workspace_api().map_err(|_| { + workspace_config::pre_error(Error::Store("Workspace unavailable".into())) + })?; + let worker = workspace_config::identity(api, &context, &workspace_id)?; + workspace_config::get(api, &worker) + .await + .map(server_api::WorkspaceConfigCurrentResponse) + } + async fn current_worker_workspace_config_attach( + &self, + context: server_api::ServerRequestContext, + workspace_id: String, + request: server_api::WorkspaceConfigAttachRequest, + ) -> std::result::Result< + server_api::WorkspaceConfigAttachment, + server_api::WorkspaceConfigApiError, + > { + let api = self.workspace_api().map_err(|_| { + workspace_config::pre_error(Error::Store("Workspace unavailable".into())) + })?; + let worker = workspace_config::identity(api, &context, &workspace_id)?; + workspace_config::attach(api, &worker, request).await + } + async fn current_worker_workspace_config_observe( + &self, + context: server_api::ServerRequestContext, + workspace_id: String, + request: server_api::WorkspaceConfigObserveRequest, + ) -> std::result::Result< + server_api::WorkspaceConfigObserveResponse, + server_api::WorkspaceConfigApiError, + > { + let api = self.workspace_api().map_err(|_| { + workspace_config::pre_error(Error::Store("Workspace unavailable".into())) + })?; + let worker = workspace_config::identity(api, &context, &workspace_id)?; + workspace_config::observe(api, &worker, request).await + } + async fn current_worker_workspace_config_read( + &self, + context: server_api::ServerRequestContext, + workspace_id: String, + request: server_api::WorkspaceConfigReadRequest, + ) -> std::result::Result< + server_api::WorkspaceConfigReadResponse, + server_api::WorkspaceConfigApiError, + > { + let api = self.workspace_api().map_err(|_| { + workspace_config::pre_error(Error::Store("Workspace unavailable".into())) + })?; + let worker = workspace_config::identity(api, &context, &workspace_id)?; + workspace_config::read(api, &worker, request).await + } + async fn current_worker_workspace_config_commit( + &self, + context: server_api::ServerRequestContext, + workspace_id: String, + request: server_api::WorkspaceConfigCommitRequest, + ) -> std::result::Result< + server_api::WorkspaceConfigCommitResponse, + server_api::WorkspaceConfigApiError, + > { + let api = self.workspace_api().map_err(|_| { + workspace_config::pre_error(Error::Store("Workspace unavailable".into())) + })?; + let worker = workspace_config::identity(api, &context, &workspace_id)?; + workspace_config::commit(api, &worker, request).await + } + async fn workspace_config_grant_create( + &self, + actor: RequestActor, + workspace_id: String, + request: server_api::WorkspaceConfigGrantCreateRequest, + ) -> std::result::Result< + server_api::WorkspaceConfigGrantResponse, + server_api::WorkspaceConfigApiError, + > { + let api = self.workspace_api().map_err(|_| { + workspace_config::pre_error(Error::Store("Workspace unavailable".into())) + })?; + workspace_config::create_grant(api, &actor, &workspace_id, request).await + } + async fn workspace_config_grant_revoke( + &self, + actor: RequestActor, + workspace_id: String, + grant_id: String, + ) -> std::result::Result< + server_api::WorkspaceConfigGrantResponse, + server_api::WorkspaceConfigApiError, + > { + let api = self.workspace_api().map_err(|_| { + workspace_config::pre_error(Error::Store("Workspace unavailable".into())) + })?; + workspace_config::revoke_grant(api, &actor, &workspace_id, &grant_id).await + } + async fn current_worker_workdir_catalog( + &self, + context: server_api::ServerRequestContext, + workspace_id: String, + query: server_api::CurrentWorkerWorkdirCatalogQuery, + ) -> std::result::Result< + server_api::CurrentWorkerWorkdirCatalogResponse, + server_api::RepositoryApiError, + > { + let api = self.workspace_api()?; + validate_workspace_scope(api, &workspace_id) + .map_err(ApiError::into_repository_api_error)?; + let worker = current_worker_identity( + api, + &workspace_id, + ¤t_worker_contract_headers(&context)?, + ) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; + { + let session_lock = current_worker_session_lock(api, &worker); + let _guard = session_lock.lock().await; + list_current_worker_workdir_catalog_for_worker(api, &worker, query) + } + .map_err(ApiError::into_repository_api_error) + } + + async fn current_worker_workdir_attachment_list( + &self, + context: server_api::ServerRequestContext, + workspace_id: String, + query: server_api::CurrentWorkerWorkdirAttachmentListQuery, + ) -> std::result::Result< + server_api::CurrentWorkerWorkdirAttachmentListResponse, + server_api::RepositoryApiError, + > { + let api = self.workspace_api()?; + validate_workspace_scope(api, &workspace_id) + .map_err(ApiError::into_repository_api_error)?; + let worker = current_worker_identity( + api, + &workspace_id, + ¤t_worker_contract_headers(&context)?, + ) + .map_err(ApiError::from) + .map_err(ApiError::into_repository_api_error)?; + let session_lock = current_worker_session_lock(api, &worker); + let _session_guard = session_lock.lock().await; + list_current_worker_workdir_attachments(api, &worker, query) + .map_err(ApiError::into_repository_api_error) + } + async fn current_worker_workdir_attach( &self, context: server_api::ServerRequestContext, @@ -10384,7 +11029,7 @@ impl server_api::ServerApi for ServerApiContractService { let Json(response) = scoped_attach_current_worker_workdir( State(self.workspace_api()?.clone()), AxumPath(ScopedWorkspacePath { workspace_id }), - contract_request_headers(&context)?, + current_worker_contract_headers(&context)?, Json(AttachCurrentWorkerWorkdirRequest { alias: request.alias, working_directory_id: request.working_directory_id, @@ -10400,6 +11045,7 @@ impl server_api::ServerApi for ServerApiContractService { context: server_api::ServerRequestContext, workspace_id: String, alias: String, + query: server_api::CurrentWorkerWorkdirDetachQuery, ) -> std::result::Result< server_api::CurrentWorkerWorkdirAttachmentResponse, server_api::RepositoryApiError, @@ -10407,7 +11053,8 @@ impl server_api::ServerApi for ServerApiContractService { let Json(response) = scoped_detach_current_worker_workdir( State(self.workspace_api()?.clone()), AxumPath((workspace_id, alias)), - contract_request_headers(&context)?, + current_worker_contract_headers(&context)?, + query, ) .await .map_err(ApiError::into_repository_api_error)?; @@ -10430,7 +11077,7 @@ impl server_api::ServerApi for ServerApiContractService { .clone(), ), AxumPath(ScopedWorkspacePath { workspace_id }), - contract_request_headers(&context) + current_worker_contract_headers(&context) .map_err(server_api::WorkdirOperationApiError::api)?, Json(WorkspaceWorkdirSessionOperationRequest { target_workdir: request.target_workdir, @@ -11929,37 +12576,70 @@ async fn scoped_commit_workspace_config_tree( ) -> ApiResult<(StatusCode, Json)> { let request = config_commit_request_from_api(request)?; validate_workspace_scope(&api, &path.workspace_id)?; - let candidate = api - .config_store - .evaluate_workspace_config_candidate_with_schema( - &path.workspace_id, - &request, - api.config_schema_registry.compose()?, - )?; - crate::prompt_settings::validate_evaluated_prompt_catalog(&candidate.evaluation)?; - project_repository_access_candidate( - &*api.store, - &api.repository_secrets, - &path.workspace_id, - &candidate, - )?; - let state = api - .config_store - .commit_evaluated_workspace_config(&path.workspace_id, &candidate)?; - if let Ok(projection) = api - .prompt_projection_cache - .resolve(&path.workspace_id, &state) - { - let _diagnostics = api - .runtime - .observe_workspace_prompt_projection((*projection).clone()); - } + let state = commit_workspace_config_tree(&api, &path.workspace_id, &request)?; Ok(( StatusCode::CREATED, Json(workspace_config_state_to_api(state)), )) } +/// Shared canonical commit and activation boundary for UI and logical config operations. +/// Callers must authorize access before entering this boundary. +/// +/// Repository access projection may materialize a default credential before the +/// config CAS commits. An error here therefore does not promise that every +/// auxiliary side effect was rolled back. Prompt publication is best effort, +/// exactly as it is for UI saves, and must not turn a committed save into an error. +fn commit_workspace_config_tree( + api: &WorkspaceApi, + workspace_id: &str, + request: &ConfigCommitRequest, +) -> Result { + let candidate = prepare_workspace_config_tree(api, workspace_id, request)?; + persist_workspace_config_candidate(api, workspace_id, &candidate) +} + +fn prepare_workspace_config_tree( + api: &WorkspaceApi, + workspace_id: &str, + request: &ConfigCommitRequest, +) -> Result { + let candidate = api + .config_store + .evaluate_workspace_config_candidate_with_schema( + workspace_id, + request, + api.config_schema_registry.compose()?, + )?; + crate::config_source::validate_workspace_config_candidate_projections( + workspace_id, + &candidate, + )?; + Ok(candidate) +} + +fn persist_workspace_config_candidate( + api: &WorkspaceApi, + workspace_id: &str, + candidate: &crate::config_source::EvaluatedConfigCandidate, +) -> Result { + project_repository_access_candidate( + &*api.store, + &api.repository_secrets, + workspace_id, + candidate, + )?; + let state = api + .config_store + .commit_evaluated_workspace_config(workspace_id, &candidate)?; + if let Ok(projection) = api.prompt_projection_cache.resolve(workspace_id, &state) { + let _diagnostics = api + .runtime + .observe_workspace_prompt_projection((*projection).clone()); + } + Ok(state) +} + async fn scoped_get_profile_settings( State(api): State, AxumPath(path): AxumPath, @@ -12110,6 +12790,7 @@ async fn compensate_manual_ticket_coder_binding( api: &WorkspaceApi, worker: &RuntimeWorkerRef, binding: &ManualTicketCoderWorkdirBinding, + operation_id: &str, worker_may_be_live: bool, ) -> Vec { let mut diagnostics = Vec::new(); @@ -12193,7 +12874,11 @@ async fn compensate_manual_ticket_coder_binding( return diagnostics; } } - match api.restore_workspace_worker(worker) { + match restore_internal_worker_intent( + api, + worker, + format!("restore-compensation:{operation_id}"), + ) { Ok(result) if result.state == server_api::WorkerRestoreState::Accepted => {} Ok(result) => diagnostics.push(spawn_compensation_diagnostic( "manual_coder_worker_restore_failed", @@ -12271,7 +12956,11 @@ async fn start_manual_ticket_coder_assignment( .into()); } if let Err(error) = close_current_worker_session_locked(api, &worker).await { - let diagnostics = match api.restore_workspace_worker(&worker) { + let diagnostics = match restore_internal_worker_intent( + &api, + &worker, + format!("manual-coder-recover:{operation_id}"), + ) { Ok(result) if result.state == server_api::WorkerRestoreState::Accepted => Vec::new(), Ok(result) => vec![spawn_compensation_diagnostic( "manual_coder_worker_restore_failed", @@ -12293,7 +12982,8 @@ async fn start_manual_ticket_coder_assignment( Ok(result) if result.state == InternalWorkerOperationState::Accepted => {} Ok(result) => { let diagnostics = - compensate_manual_ticket_coder_binding(api, &worker, &binding, false).await; + compensate_manual_ticket_coder_binding(api, &worker, &binding, operation_id, false) + .await; return Err(ApiError::with_diagnostics( Error::RuntimeOperationFailed { runtime_id: worker.runtime_id.clone(), @@ -12305,7 +12995,8 @@ async fn start_manual_ticket_coder_assignment( } Err(error) => { let diagnostics = - compensate_manual_ticket_coder_binding(api, &worker, &binding, false).await; + compensate_manual_ticket_coder_binding(api, &worker, &binding, operation_id, false) + .await; return Err(ApiError::with_diagnostics(error, diagnostics)); } } @@ -12316,15 +13007,21 @@ async fn start_manual_ticket_coder_assignment( &binding.effective_links, ) { let diagnostics = - compensate_manual_ticket_coder_binding(api, &worker, &binding, false).await; + compensate_manual_ticket_coder_binding(api, &worker, &binding, operation_id, false) + .await; return Err(ApiError::with_diagnostics(error, diagnostics)); } - match api.restore_workspace_worker(&worker) { + match restore_internal_worker_intent( + &api, + &worker, + format!("manual-coder-restore:{operation_id}"), + ) { Ok(result) if result.state == server_api::WorkerRestoreState::Accepted => {} Ok(result) => { let diagnostics = - compensate_manual_ticket_coder_binding(api, &worker, &binding, false).await; + compensate_manual_ticket_coder_binding(api, &worker, &binding, operation_id, false) + .await; return Err(ApiError::with_diagnostics( Error::RuntimeOperationFailed { runtime_id: worker.runtime_id.clone(), @@ -12336,7 +13033,8 @@ async fn start_manual_ticket_coder_assignment( } Err(error) => { let diagnostics = - compensate_manual_ticket_coder_binding(api, &worker, &binding, false).await; + compensate_manual_ticket_coder_binding(api, &worker, &binding, operation_id, false) + .await; return Err(ApiError::with_diagnostics(error.error, diagnostics)); } } @@ -12348,7 +13046,8 @@ async fn start_manual_ticket_coder_assignment( Ok(assignment) => Ok(assignment), Err(error) => { let diagnostics = - compensate_manual_ticket_coder_binding(api, &worker, &binding, true).await; + compensate_manual_ticket_coder_binding(api, &worker, &binding, operation_id, true) + .await; Err(ApiError::with_diagnostics(error, diagnostics)) } } @@ -15928,6 +16627,14 @@ async fn open_current_worker_workdir_session_locked( link.workdir_id ), })?; + if matches!( + workdir.source, + WorkdirRegistrySource::WorkspaceConfig { .. } + ) { + return Err(Error::WorkspacePermissionDenied( + "Logical config has no filesystem session".into(), + )); + } let effective_capabilities = effective_worker_workdir_capabilities(api, worker, &workdir, link.capabilities)?; if let WorkdirRegistrySource::ExternalGrant { grant_id } = &workdir.source { @@ -16007,7 +16714,8 @@ async fn open_current_worker_workdir_session_locked( runtime_id, repository_id, } => (runtime_id.as_str(), repository_id.as_str()), - WorkdirRegistrySource::ExternalGrant { .. } => { + WorkdirRegistrySource::ExternalGrant { .. } + | WorkdirRegistrySource::WorkspaceConfig { .. } => { return Err(Error::RuntimeOperationFailed { runtime_id: "workspace-backend".to_string(), code: "external_workdir_provider_unavailable".to_string(), @@ -16124,6 +16832,50 @@ async fn close_current_worker_session_locked( }) } +fn restore_logical_workdir_attachments( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, +) -> ApiResult> { + api.store + .list_worker_workdir_links(&api.config.workspace_id, worker)? + .into_iter() + .filter(|link| link.unlinked_at.is_none()) + .filter(|link| { + api.store + .get_workdir_registry(&api.config.workspace_id, &link.workdir_id) + .ok() + .flatten() + .is_none_or(|record| { + !matches!(record.source, WorkdirRegistrySource::WorkspaceConfig { .. }) + }) + }) + .map(|link| { + let workdir = api + .store + .get_workdir_registry(&api.config.workspace_id, &link.workdir_id)? + .ok_or_else(|| { + Error::RegistryInconsistency(format!( + "Worker attachment references unknown Workdir {}", + link.workdir_id + )) + })?; + let capabilities = link + .capabilities + .intersection(workdir_source_capabilities(api, &workdir)?) + .intersection(ticket_target_capabilities_for_workdir( + api, worker, &workdir, + )?); + Ok(LogicalWorkdirAttachment { + alias: workdir::WorkdirAttachmentAlias::new(link.alias) + .map_err(|error| Error::InvalidInput(error.to_string()))?, + working_directory_id: link.workdir_id, + capabilities, + }) + }) + .collect::>>() + .map_err(ApiError::from) +} + fn sync_runtime_worker_workdir_attachments( api: &WorkspaceApi, worker: &RuntimeWorkerRef, @@ -16133,6 +16885,15 @@ fn sync_runtime_worker_workdir_attachments( .list_worker_workdir_links(&api.config.workspace_id, worker)? .into_iter() .filter(|link| link.unlinked_at.is_none()) + .filter(|link| { + api.store + .get_workdir_registry(&api.config.workspace_id, &link.workdir_id) + .ok() + .flatten() + .is_none_or(|record| { + !matches!(record.source, WorkdirRegistrySource::WorkspaceConfig { .. }) + }) + }) .map(|mut link| { let workdir = api .store @@ -16190,6 +16951,15 @@ async fn refresh_current_worker_session_locked( .list_worker_workdir_links(&api.config.workspace_id, worker)? .into_iter() .filter(|link| link.unlinked_at.is_none()) + .filter(|link| { + api.store + .get_workdir_registry(&api.config.workspace_id, &link.workdir_id) + .ok() + .flatten() + .is_none_or(|record| { + !matches!(record.source, WorkdirRegistrySource::WorkspaceConfig { .. }) + }) + }) .collect::>(); if remaining.len() == 1 { open_current_worker_workdir_session_locked(api, worker, &remaining[0]).await?; @@ -16273,6 +17043,12 @@ async fn scoped_attach_current_worker_workdir( } match &workdir.source { WorkdirRegistrySource::Repository { .. } => {} + WorkdirRegistrySource::WorkspaceConfig { .. } => { + return Err(Error::WorkspacePermissionDenied( + "Use the grant-bound Workspace config attachment operation".into(), + ) + .into()); + } WorkdirRegistrySource::ExternalGrant { grant_id } => { let grant = api .store @@ -16354,6 +17130,7 @@ async fn scoped_attach_current_worker_workdir( } close_current_worker_attachment_session_locked(&api, &worker).await?; let link = api.store.attach_worker_workdir(&WorkerWorkdirLinkRecord { + connection_id: String::new(), workspace_id: api.config.workspace_id.clone(), worker: worker.clone(), workdir_id: workdir_id.to_string(), @@ -16376,6 +17153,15 @@ async fn scoped_attach_current_worker_workdir( .list_worker_workdir_links(&api.config.workspace_id, &worker)? .into_iter() .filter(|link| link.unlinked_at.is_none()) + .filter(|link| { + api.store + .get_workdir_registry(&api.config.workspace_id, &link.workdir_id) + .ok() + .flatten() + .is_none_or(|record| { + !matches!(record.source, WorkdirRegistrySource::WorkspaceConfig { .. }) + }) + }) .count(); if active_count == 1 && let Err(error) = open_current_worker_workdir_session_locked(&api, &worker, &link).await @@ -16413,56 +17199,196 @@ async fn scoped_attach_current_worker_workdir( })) } +#[cfg(test)] +fn list_current_worker_workdir_catalog( + api: &WorkspaceApi, + query: server_api::CurrentWorkerWorkdirCatalogQuery, +) -> ApiResult { + list_current_worker_workdir_catalog_for_worker( + api, + &RuntimeWorkerRef::new("catalog-test", "ungranted"), + query, + ) +} + +fn list_current_worker_workdir_catalog_for_worker( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + query: server_api::CurrentWorkerWorkdirCatalogQuery, +) -> ApiResult { + // No Runtime inventory refresh: both projection inputs and complete digest are authoritative + // Backend records captured by the store in one read transaction. + let page = api.store.workdir_catalog_page( + &api.config.workspace_id, + query.limit.unwrap_or(50), + query.cursor.as_deref(), + )?; + let items = page + .entries + .into_iter() + .filter_map(|entry| { + if let WorkdirRegistrySource::WorkspaceConfig { grant_id } = &entry.record.source { + if !workspace_config::granted_to(api, worker, grant_id) { + return None; + } + let grant = api + .store + .get_workspace_config_grant(&api.config.workspace_id, grant_id) + .ok() + .flatten()?; + let mut summary = workspace_config::summary(&grant, &entry.record); + summary.occupied_by = entry.occupied_by.map(Into::into); + return Some(summary); + } + if matches!( + entry.record.source, + WorkdirRegistrySource::ExternalGrant { .. } + ) && entry.external_grant_permissions.is_none() + { + return None; + } + let mut summary = workdir_summary_from_record( + &entry.record, + entry.repository_key.as_deref(), + entry.external_grant_permissions, + ); + summary.occupied_by = entry.occupied_by; + Some(server_api::WorkingDirectorySummary::from(summary)) + }) + .collect(); + Ok(server_api::CurrentWorkerWorkdirCatalogResponse { + workspace_id: api.config.workspace_id.clone(), + items, + next_cursor: page.next_cursor, + revision: page.revision, + }) +} + +fn list_current_worker_workdir_attachments( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + query: server_api::CurrentWorkerWorkdirAttachmentListQuery, +) -> ApiResult { + let limit = query.limit.unwrap_or(50); + let offset = query.offset.unwrap_or(0); + if limit == 0 || limit > 100 { + return Err( + Error::InvalidInput("attachment page limit must be 1..=100".to_string()).into(), + ); + } + let (mut links, revision) = api.store.list_worker_workdir_links_page_with_revision( + &api.config.workspace_id, + worker, + limit + 1, + offset, + query.connection_id.as_deref(), + )?; + let next_offset = + if links.len() > limit as usize { + Some(offset.checked_add(limit).ok_or_else(|| { + Error::InvalidInput("attachment page offset overflow".to_string()) + })?) + } else { + None + }; + links.truncate(limit as usize); + Ok(server_api::CurrentWorkerWorkdirAttachmentListResponse { + workspace_id: api.config.workspace_id.clone(), + items: links + .into_iter() + .map(|link| server_api::CurrentWorkerWorkdirAttachmentItem { + alias: link.alias, + working_directory_id: link.workdir_id, + capabilities: link.capabilities, + connection_id: link.connection_id, + }) + .collect(), + next_offset, + revision, + }) +} + async fn scoped_detach_current_worker_workdir( State(api): State, AxumPath((workspace_id, alias)): AxumPath<(String, String)>, headers: HeaderMap, + query: server_api::CurrentWorkerWorkdirDetachQuery, ) -> ApiResult> { validate_workspace_scope(&api, &workspace_id)?; let alias = workdir::WorkdirAttachmentAlias::new(alias) .map_err(|error| Error::InvalidInput(error.to_string()))?; let worker = current_worker_identity(&api, &workspace_id, &headers)?; - let session_lock = current_worker_session_lock(&api, &worker); + detach_current_worker_workdir( + &api, + &worker, + alias.as_str(), + query.expected_connection_id.as_deref(), + ) + .await + .map(Json) +} + +async fn detach_current_worker_workdir( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + alias: &str, + expected_connection_id: Option<&str>, +) -> ApiResult { + let session_lock = current_worker_session_lock(api, worker); let _session_guard = session_lock.lock().await; let link = api .store - .list_worker_workdir_links(&api.config.workspace_id, &worker)? + .list_worker_workdir_links(&api.config.workspace_id, worker)? .into_iter() - .find(|link| link.unlinked_at.is_none() && link.alias == alias.as_str()) + .find(|link| link.unlinked_at.is_none() && link.alias == alias) .ok_or_else(|| { Error::WorkdirAttachmentConflict(format!( "Worker {}:{} has no Workdir attachment `{alias}`", worker.runtime_id, worker.worker_id )) })?; - close_current_worker_attachment_session_locked(&api, &worker).await?; - close_current_worker_command_sessions_for_alias_locked(&api, &worker, alias.as_str()).await?; - api.store.detach_worker_workdir( + if expected_connection_id.is_some_and(|expected| expected != link.connection_id) { + return Err(Error::WorkdirAttachmentConflict(format!( + "Workdir attachment `{alias}` connection changed during detach" + )) + .into()); + } + close_current_worker_attachment_session_locked(api, worker).await?; + close_current_worker_command_sessions_for_alias_locked(api, worker, alias).await?; + // Even unconditional HTTP requests condition the mutation on the lifetime just read + // under the session lock, so a concurrent ledger replacement cannot be removed. + api.store.detach_worker_workdir_connection( &api.config.workspace_id, - &worker, - Some(&link.workdir_id), + worker, + alias, + &link.connection_id, &Utc::now().to_rfc3339_opts(SecondsFormat::Millis, true), )?; - if let Err(error) = sync_runtime_worker_workdir_attachments(&api, &worker) { - let mut restored = link.clone(); - restored.unlinked_at = None; - api.store.attach_worker_workdir(&restored)?; + if let Err(error) = sync_runtime_worker_workdir_attachments(api, worker) { + api.store.restore_worker_workdir_connection(&link)?; return Err(error); } let remaining = api .store - .list_worker_workdir_links(&api.config.workspace_id, &worker)? + .list_worker_workdir_links(&api.config.workspace_id, worker)? .into_iter() .filter(|link| link.unlinked_at.is_none()) + .filter(|link| { + api.store + .get_workdir_registry(&api.config.workspace_id, &link.workdir_id) + .ok() + .flatten() + .is_none_or(|record| { + !matches!(record.source, WorkdirRegistrySource::WorkspaceConfig { .. }) + }) + }) .collect::>(); if remaining.len() == 1 && let Err(error) = - open_current_worker_workdir_session_locked(&api, &worker, &remaining[0]).await + open_current_worker_workdir_session_locked(api, worker, &remaining[0]).await { - let mut restored = link.clone(); - restored.unlinked_at = None; - api.store.attach_worker_workdir(&restored)?; - let _ = sync_runtime_worker_workdir_attachments(&api, &worker); + api.store.restore_worker_workdir_connection(&link)?; + let _ = sync_runtime_worker_workdir_attachments(api, worker); return Err(error.into()); } if let Err(error) = api @@ -16470,20 +17396,18 @@ async fn scoped_detach_current_worker_workdir( .refresh(&worker) .map_err(ApiError::from) { - let mut restored = link.clone(); - restored.unlinked_at = None; - api.store.attach_worker_workdir(&restored)?; - sync_runtime_worker_workdir_attachments(&api, &worker)?; - refresh_current_worker_session_locked(&api, &worker).await?; + api.store.restore_worker_workdir_connection(&link)?; + sync_runtime_worker_workdir_attachments(api, worker)?; + refresh_current_worker_session_locked(api, worker).await?; return Err(error); } - Ok(Json(server_api::CurrentWorkerWorkdirAttachmentResponse { + Ok(server_api::CurrentWorkerWorkdirAttachmentResponse { workspace_id: api.config.workspace_id.clone(), alias: alias.to_string(), working_directory_id: link.workdir_id, capabilities: workdir::WorkdirSessionCapabilities::EMPTY, attached: false, - })) + }) } fn validated_current_worker_attachment( @@ -16644,6 +17568,9 @@ async fn scoped_execute_current_worker_workdir_operation( } operation @ (WorkdirSessionOperation::AuthorizeScope(_) | WorkdirSessionOperation::ScopeRulesOverlap(_) + | WorkdirSessionOperation::CheckoutObserve(_) + | WorkdirSessionOperation::CheckoutExecute(_) + | WorkdirSessionOperation::CheckoutSearch(_) | WorkdirSessionOperation::Stat(_) | WorkdirSessionOperation::Read(_) | WorkdirSessionOperation::ReadBytes(_) @@ -16705,6 +17632,18 @@ async fn execute_workdir_session_operation( .scope_rules_overlap(request) .await .map(|overlaps| WorkdirSessionOperationResult::ScopeRulesOverlap { overlaps }), + WorkdirSessionOperation::CheckoutObserve(path) => session + .checkout_observe(path) + .await + .map(WorkdirSessionOperationResult::CheckoutObserve), + WorkdirSessionOperation::CheckoutExecute(request) => session + .checkout_execute(request) + .await + .map(WorkdirSessionOperationResult::CheckoutExecute), + WorkdirSessionOperation::CheckoutSearch(request) => session + .checkout_search(request) + .await + .map(WorkdirSessionOperationResult::CheckoutSearch), WorkdirSessionOperation::Stat(request) => session .stat(request) .await @@ -16765,48 +17704,54 @@ async fn run_orchestrator_turn_end_hook(api: WorkspaceApi) { }; let mut worker_states = HashMap::new(); while let Some(update) = subscription.recv().await { - match update { - crate::runtime_subscription::BrokerSubscriptionEvent::Snapshot { snapshot, .. } => { - if let protocol::subscription::SubscriptionSnapshot::Workers { workers } = snapshot - { - worker_states.clear(); - for worker in workers { - let worker_id = worker.worker_id.to_string(); - maybe_dispatch_orchestrator_turn_end(&api, &worker_id, None, worker.state); - worker_states.insert(worker_id, worker.state); - } - } - } - crate::runtime_subscription::BrokerSubscriptionEvent::Event { payload, .. } => { - match payload { - protocol::subscription::SubscriptionEventPayload::WorkerUpserted { worker } => { - let worker_id = worker.worker_id.to_string(); - let previous = worker_states.insert(worker_id.clone(), worker.state); - maybe_dispatch_orchestrator_turn_end( - &api, - &worker_id, - previous, - worker.state, - ); - } - protocol::subscription::SubscriptionEventPayload::WorkerRemoved { - worker_id, - .. - } => { - worker_states.remove(worker_id.as_str()); - } - _ => {} - } - } - crate::runtime_subscription::BrokerSubscriptionEvent::Disconnected { .. } => { - worker_states.clear(); - } - crate::runtime_subscription::BrokerSubscriptionEvent::Rejected { .. } - | crate::runtime_subscription::BrokerSubscriptionEvent::Closed { .. } => return, + if !observe_orchestrator_turn_end(&api, &mut worker_states, update) { + return; } } } +// A subscription snapshot establishes an observation baseline, not a lifecycle +// event. Reconnection deliberately discards the old baseline. +fn observe_orchestrator_turn_end( + api: &WorkspaceApi, + worker_states: &mut HashMap, + update: crate::runtime_subscription::BrokerSubscriptionEvent, +) -> bool { + match update { + crate::runtime_subscription::BrokerSubscriptionEvent::Snapshot { snapshot, .. } => { + if let protocol::subscription::SubscriptionSnapshot::Workers { workers } = snapshot { + worker_states.clear(); + for worker in workers { + let worker_id = worker.worker_id.to_string(); + worker_states.insert(worker_id, worker.state); + } + } + } + crate::runtime_subscription::BrokerSubscriptionEvent::Event { payload, .. } => { + match payload { + protocol::subscription::SubscriptionEventPayload::WorkerUpserted { worker } => { + let worker_id = worker.worker_id.to_string(); + let previous = worker_states.insert(worker_id.clone(), worker.state); + maybe_dispatch_orchestrator_turn_end(api, &worker_id, previous, worker.state); + } + protocol::subscription::SubscriptionEventPayload::WorkerRemoved { + worker_id, + .. + } => { + worker_states.remove(worker_id.as_str()); + } + _ => {} + } + } + crate::runtime_subscription::BrokerSubscriptionEvent::Disconnected { .. } => { + worker_states.clear(); + } + crate::runtime_subscription::BrokerSubscriptionEvent::Rejected { .. } + | crate::runtime_subscription::BrokerSubscriptionEvent::Closed { .. } => return false, + } + true +} + fn maybe_dispatch_orchestrator_turn_end( api: &WorkspaceApi, worker_id: &str, @@ -16816,12 +17761,7 @@ fn maybe_dispatch_orchestrator_turn_end( use protocol::subscription::SubscriptionWorkerState; if current != SubscriptionWorkerState::Idle - || !matches!( - previous, - None | Some(SubscriptionWorkerState::Running) - | Some(SubscriptionWorkerState::Stopped) - | Some(SubscriptionWorkerState::Paused) - ) + || previous != Some(SubscriptionWorkerState::Running) { return; } @@ -16831,14 +17771,62 @@ fn maybe_dispatch_orchestrator_turn_end( if orchestrator.worker.runtime_id == EMBEDDED_WORKER_RUNTIME_ID && orchestrator.worker.worker_id == worker_id { - dispatch_orchestrator_queue_attention(api); + dispatch_orchestrator_queue_attention(api, &orchestrator.worker, false); } } -fn dispatch_orchestrator_queue_attention(api: &WorkspaceApi) { - let Some(orchestrator) = find_workspace_orchestrator(api) else { +fn dispatch_orchestrator_queue_attention( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + retry_only: bool, +) { + with_orchestrator_attention_reservation(api, worker, retry_only, |state, pending| { + check_orchestrator_queue_attention(api, worker, state, pending); + }); +} + +fn with_orchestrator_attention_reservation( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + mut retry_only: bool, + mut check: impl FnMut( + &Arc>, + Option, + ), +) { + let state = orchestrator_attention_state(api, worker); + loop { + let pending = { + let mut state = state.lock().unwrap_or_else(|p| p.into_inner()); + if state.sending || (retry_only && state.pending.is_none()) { + return; + } + state.sending = true; + state.pending.clone() + }; + let reservation = OrchestratorAttentionReservation(Some(state.clone())); + check(&state, pending); + if !reservation.complete() { + return; + } + // Accepted Restore is an explicit recheck even when the current check + // was a no-op or an already-online retry. Never infer it from Idle. + retry_only = false; + } +} + +fn check_orchestrator_queue_attention( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + state: &Arc>, + pending: Option, +) { + // Unknown outcomes keep the original immutable payload and ID, even if the + // backlog or prompt projection changed before this retry. + if let Some(delivery) = pending { + send_orchestrator_attention(api, worker, &state, delivery); return; - }; + } let Ok(backend) = browser_ticket_backend(api) else { return; }; @@ -16867,9 +17855,7 @@ fn dispatch_orchestrator_queue_attention(api: &WorkspaceApi) { queued.extend(inprogress); queued.sort_by(|left, right| left.id.cmp(&right.id)); if queued.is_empty() { - *api.orchestrator_attention_fingerprint - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) = None; + state.lock().unwrap_or_else(|p| p.into_inner()).accepted = None; return; } let fingerprint = queued @@ -16877,11 +17863,12 @@ fn dispatch_orchestrator_queue_attention(api: &WorkspaceApi) { .map(|ticket| ticket.id.as_str()) .collect::>() .join("|"); - if api - .orchestrator_attention_fingerprint + if state .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) - .as_deref() + .unwrap_or_else(|p| p.into_inner()) + .accepted + .as_ref() + .map(|delivery| delivery.fingerprint.as_str()) == Some(fingerprint.as_str()) { return; @@ -16932,22 +17919,42 @@ fn dispatch_orchestrator_queue_attention(api: &WorkspaceApi) { return; } }; - let accepted = api - .runtime - .send_input( - &orchestrator.worker, - WorkerInputRequest { - kind: WorkerInputKind::Notify, - content, - submission_request_id: None, - segments: None, - }, - ) - .is_ok_and(|result| result.state == InternalWorkerOperationState::Accepted); - if accepted { - *api.orchestrator_attention_fingerprint - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) = Some(fingerprint); + let delivery = OrchestratorAttentionDelivery { + fingerprint, + request_id: uuid::Uuid::now_v7().to_string(), + content, + }; + state.lock().unwrap_or_else(|p| p.into_inner()).pending = Some(delivery.clone()); + send_orchestrator_attention(api, worker, &state, delivery); +} + +fn send_orchestrator_attention( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + state: &Mutex, + delivery: OrchestratorAttentionDelivery, +) { + // Bounded delivery-side retry, not an Idle-snapshot retry. Rejection and + // transport ambiguity leave pending available for the next attempt. + for _ in 0..2 { + let accepted = api + .runtime + .send_input( + worker, + WorkerInputRequest { + kind: WorkerInputKind::Notify, + content: delivery.content.clone(), + submission_request_id: Some(delivery.request_id.clone()), + segments: None, + }, + ) + .is_ok_and(|result| result.state == InternalWorkerOperationState::Accepted); + if accepted { + let mut state = state.lock().unwrap_or_else(|p| p.into_inner()); + state.accepted = Some(delivery); + state.pending = None; + return; + } } } @@ -17222,6 +18229,13 @@ fn subjektiv_subject_scope( ) })?; let worker = RuntimeWorkerRef::new(&source.runtime_id, worker_id); + if let Some(grant) = subjektiv_job::active_grant(api, workspace_id, &worker)? { + return Ok(( + grant.subject_id, + worker, + SubjektivWorkerAuthority::Consolidation, + )); + } let lease = api .store .require_current_worker_singleton_owner(workspace_id, &worker)? @@ -17328,6 +18342,8 @@ fn subjektiv_subject_response( server_api::SubjektivSubjectResponse { id: subject.id, role: subject.role.as_str().to_string(), + behavior_md: subject.behavior_md, + behavior_revision: subject.behavior_revision, state: match subject.state { crate::subjektiv::SubjectState::Active => server_api::SubjektivSubjectState::Active, crate::subjektiv::SubjectState::Retired => server_api::SubjektivSubjectState::Retired, @@ -17347,9 +18363,11 @@ async fn scoped_create_subjektiv_subject( validate_workspace_scope(&api, &path.workspace_id)?; let role = crate::subjektiv::SubjectRole::new(request.role) .map_err(|error| Error::InvalidInput(error.to_string()))?; + crate::subjektiv::validate_subject_behavior(&request.behavior_md) + .map_err(|error| Error::InvalidInput(error.to_string()))?; let subject = open_subjektiv_store(&api)? - .create_subject(role) - .map_err(|error| Error::Store(error.to_string()))?; + .create_subject_with_behavior(role, request.behavior_md) + .map_err(subjektiv_store_error)?; Ok(( StatusCode::CREATED, Json(subjektiv_subject_response(&api, subject)), @@ -17441,6 +18459,24 @@ async fn scoped_get_subjektiv_subject( Ok(Json(subjektiv_subject_response(&api, subject))) } +async fn scoped_update_subjektiv_subject_behavior( + State(api): State, + AxumPath(path): AxumPath, + Json(request): Json, +) -> ApiResult> { + validate_workspace_scope(&api, &path.workspace_id)?; + crate::subjektiv::validate_subject_behavior(&request.behavior_md) + .map_err(|error| Error::InvalidInput(error.to_string()))?; + let subject = open_subjektiv_store(&api)? + .update_subject_behavior( + &path.subject_id, + request.expected_behavior_revision, + request.behavior_md, + ) + .map_err(subjektiv_store_error)?; + Ok(Json(subjektiv_subject_response(&api, subject))) +} + fn require_subjektiv_subject( store: &crate::subjektiv::SubjektivStore, subject_id: &str, @@ -19031,7 +20067,8 @@ async fn scoped_subjektiv_memory_backend( Json(request): Json, ) -> ApiResult> { validate_workspace_scope(&api, &path.workspace_id)?; - let (subject_id, _, authority) = subjektiv_subject_scope(&api, &path.workspace_id, &context)?; + let (subject_id, source_worker, authority) = + subjektiv_subject_scope(&api, &path.workspace_id, &context)?; let store = open_subjektiv_store(&api)?; let response = match request.operation { server_api::SubjektivMemoryBackendOperation::ResidentSummary(_) => { @@ -19047,6 +20084,23 @@ async fn scoped_subjektiv_memory_backend( resident, )) } + server_api::SubjektivMemoryBackendOperation::ResidentContext(_) => { + require_subjektiv_worker_authority( + authority, + SubjektivWorkerAuthority::Subject, + "resident subject context read", + )?; + let resident = store + .resident_context(&subject_id) + .map_err(subjektiv_store_error)?; + server_api::SubjektivMemoryBackendResponse::ResidentContext( + server_api::SubjektivResidentContextOutput { + behavior_md: resident.subject.behavior_md, + behavior_revision: resident.subject.behavior_revision, + memory_surface: resident_summary_output(resident.surface), + }, + ) + } server_api::SubjektivMemoryBackendOperation::Query(input) => { server_api::SubjektivMemoryBackendResponse::Query(subjektiv_memory_query( &store, @@ -19126,9 +20180,30 @@ async fn scoped_subjektiv_memory_backend( "candidate listing", )?; let limit = bounded_subjektiv_limit(input.limit, SUBJEKTIV_QUERY_DEFAULT_LIMIT)?; - let mut candidates = store - .pending_staging_candidates(&subject_id, limit.saturating_add(1)) - .map_err(subjektiv_store_error)?; + let mut candidates = if let Some(grant) = + subjektiv_job::active_grant(&api, &path.workspace_id, &source_worker)? + { + let mut pending = Vec::new(); + for id in &grant.candidate_ids { + if store + .staging_resolution(&subject_id, id) + .map_err(subjektiv_store_error)? + .is_none() + { + if let Some(candidate) = store + .staging_candidate(&subject_id, id) + .map_err(subjektiv_store_error)? + { + pending.push(candidate); + } + } + } + pending + } else { + store + .pending_staging_candidates(&subject_id, limit.saturating_add(1)) + .map_err(subjektiv_store_error)? + }; let has_more = candidates.len() > limit; candidates.truncate(limit); server_api::SubjektivMemoryBackendResponse::Candidates( @@ -19147,6 +20222,12 @@ async fn scoped_subjektiv_memory_backend( SubjektivWorkerAuthority::Consolidation, "candidate read", )?; + subjektiv_job::require_candidate_grant( + &api, + &path.workspace_id, + &source_worker, + &input.candidate_id, + )?; let candidate = store .staging_candidate(&subject_id, &input.candidate_id) .map_err(subjektiv_store_error)? @@ -19175,6 +20256,12 @@ async fn scoped_subjektiv_memory_backend( SubjektivWorkerAuthority::Consolidation, "candidate decision", )?; + subjektiv_job::require_candidate_grant( + &api, + &path.workspace_id, + &source_worker, + &input.candidate_id, + )?; let receipt = store .decide_candidate(&subject_id, subjektiv_candidate_decision(input)?) .map_err(subjektiv_store_error)?; @@ -19188,12 +20275,42 @@ async fn scoped_subjektiv_memory_backend( SubjektivWorkerAuthority::Consolidation, "surface generation preparation", )?; + if let Some(grant) = + subjektiv_job::active_grant(&api, &path.workspace_id, &source_worker)? + { + for id in &grant.candidate_ids { + if store + .staging_resolution(&subject_id, id) + .map_err(subjektiv_store_error)? + .is_none() + { + return Err(Error::InvalidInput(format!("surface generation requires a durable disposition for batch candidate `{id}`")).into()); + } + } + } + let binding = api + .store + .worker_registry_projection(&path.workspace_id, &source_worker)? + .and_then(|p| p.job); let generation = store - .prepare_surface_generation(&subject_id) + .prepare_job_surface_generation( + &subject_id, + binding + .as_ref() + .map(|b| (b.job_id.as_str(), b.attempt_id.as_str())), + ) .map_err(subjektiv_store_error)?; - server_api::SubjektivMemoryBackendResponse::SurfacePrepared( - subjektiv_surface_prepare_response(generation), - ) + let resident = store + .resident_surface(&subject_id) + .map_err(subjektiv_store_error)?; + let mut response = subjektiv_surface_prepare_response(generation); + if resident.availability == crate::subjektiv::SurfaceAvailability::Ready { + response.current_snapshot_id = resident + .snapshot + .filter(|s| s.built_from_store_revision == response.store_revision) + .map(|s| s.id); + } + server_api::SubjektivMemoryBackendResponse::SurfacePrepared(response) } server_api::SubjektivMemoryBackendOperation::PublishSurface(input) => { require_subjektiv_worker_authority( @@ -19201,6 +20318,20 @@ async fn scoped_subjektiv_memory_backend( SubjektivWorkerAuthority::Consolidation, "surface publication", )?; + if let Some(binding) = api + .store + .worker_registry_projection(&path.workspace_id, &source_worker)? + .and_then(|p| p.job) + { + store + .require_job_surface_generation( + &subject_id, + &input.generation_id, + &binding.job_id, + &binding.attempt_id, + ) + .map_err(subjektiv_store_error)?; + } let points = input .points .into_iter() @@ -19233,13 +20364,52 @@ async fn scoped_subjektiv_memory_backend( SubjektivWorkerAuthority::Consolidation, "surface generation failure", )?; + if let Some(binding) = api + .store + .worker_registry_projection(&path.workspace_id, &source_worker)? + .and_then(|p| p.job) + { + store + .require_job_surface_generation( + &subject_id, + &input.generation_id, + &binding.job_id, + &binding.attempt_id, + ) + .map_err(subjektiv_store_error)?; + } let revision = store .fail_surface_generation(&subject_id, &input.generation_id, &input.reason_code) .map_err(subjektiv_store_error)?; + let status = if let Some(binding) = api + .store + .worker_registry_projection(&path.workspace_id, &source_worker)? + .and_then(|p| p.job) + { + store + .validate_job_surface_outcome( + &subject_id, + &input.generation_id, + revision, + "failed", + None, + Some(&input.reason_code), + &binding.job_id, + &binding.attempt_id, + ) + .map_err(|e| { + Error::RepositoryConflict(format!( + "surface failure was not recorded for this attempt: {e}" + )) + })?; + "failed_confirmed" + } else { + "failed" + }; server_api::SubjektivMemoryBackendResponse::SurfaceFailed( server_api::SubjektivSurfaceFailureResponse { store_revision: revision, - status: "failed".into(), + status: status.into(), }, ) } @@ -19252,6 +20422,7 @@ fn subjektiv_surface_prepare_response( ) -> server_api::SubjektivSurfacePrepareResponse { server_api::SubjektivSurfacePrepareResponse { generation_id: generation.id, + current_snapshot_id: None, store_revision: generation.store_revision, active_memory_count: generation.active_memory_count, materials: generation @@ -20421,6 +21592,7 @@ fn subjektiv_store_error(error: crate::subjektiv::SubjektivError) -> Error { Error::WorkspacePermissionDenied(format!("subject_scope_mismatch: {error}")) } crate::subjektiv::SubjektivError::RevisionConflict { .. } + | crate::subjektiv::SubjektivError::SubjectBehaviorConflict { .. } | crate::subjektiv::SubjektivError::SurfaceGenerationConflict(_) => { Error::RepositoryConflict(format!("revision_conflict: {error}")) } @@ -20455,11 +21627,9 @@ async fn scoped_subjektiv_memory_consolidation( SubjektivWorkerAuthority::Subject, "consolidation request", )?; - Ok(Json(start_subjektiv_staging_consolidation( - api, - &subject_id, - operation, - )?)) + Ok(Json( + start_subjektiv_staging_consolidation(api, &subject_id, operation).await?, + )) } const SUBJECT_CONSOLIDATION_THRESHOLD_FILES: usize = 5; @@ -20518,7 +21688,7 @@ mod subject_surface_consolidation_policy_tests { } } -fn start_subjektiv_staging_consolidation( +async fn start_subjektiv_staging_consolidation( api: WorkspaceApi, subject_id: &str, operation: MemoryConsolidateStagingOperation, @@ -20527,6 +21697,13 @@ fn start_subjektiv_staging_consolidation( let (candidate_count, total_bytes) = store .pending_staging_backlog(subject_id) .map_err(subjektiv_store_error)?; + if !subjektiv_job::retire_legacy_worker(&api, subject_id).await? { + return Ok(MemoryConsolidationOutput { + status: "skipped_legacy_inflight".into(), + summary: "Legacy owned subject consolidator has not completed foreground work; no Job may run concurrently.".into(), + candidate_count, total_bytes, + }); + } let surface_availability = store .resident_surface(subject_id) .map_err(subjektiv_store_error)? @@ -20557,135 +21734,7 @@ fn start_subjektiv_staging_consolidation( }); } - let runtime_id = select_memory_consolidation_runtime(&api)?; - let singleton_key = format!("{SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX}{subject_id}"); - let input_content = format!( - "Process the delegated subject's {candidate_count} pending Memory candidate(s) ({total_bytes} bytes) with MemoryStagingList, MemoryStagingRead, subject Memory reads, and MemoryApplyCandidate. After this committed consolidation turn, the Host will rebuild the {:?} resident surface from confirmed Memory in a separate clean context.", - surface_availability - ); - if let Some(output) = try_reuse_subject_consolidation_worker( - &api, - &singleton_key, - &input_content, - candidate_count, - total_bytes, - )? { - return Ok(output); - } - let result = api.spawn_workspace_worker( - &runtime_id, - WorkerSpawnRequest { - requested_worker_name: Some(SUBJEKTIV_MEMORY_CONSOLIDATION_PROFILE.to_string()), - singleton_key: Some(singleton_key), - intent: WorkerSpawnIntent::WorkspaceOrchestrator, - acceptance: WorkerSpawnAcceptanceRequirement::RunAccepted { - expected_segments: 1, - }, - profile: ProfileSelector::Builtin(SUBJEKTIV_MEMORY_CONSOLIDATION_PROFILE.to_string()), - ticket_assignment: None, - initial_submit: vec![Segment::text(input_content)], - workdir_attachment_requests: Vec::new(), - resolved_workdir_attachment_requests: Vec::new(), - resolved_workdir_attachments: Vec::new(), - resolved_config_bundle: None, - resolved_worker_observation_enabled: false, - resolved_worker_observation_grants: Vec::new(), - resolved_workspace_api: None, - resolved_memory_settings: None, - resolved_subjektiv_attached: true, - resolved_control_operation: None, - }, - )?; - if result.state != InternalWorkerOperationState::Accepted { - return Ok(MemoryConsolidationOutput { - status: "skipped_spawn_rejected".into(), - summary: "Runtime rejected subject Memory consolidater spawn.".into(), - candidate_count, - total_bytes, - }); - } - let Some(worker) = result.worker else { - return Ok(MemoryConsolidationOutput { - status: "skipped_spawn_missing_worker".into(), - summary: "Runtime accepted subject Memory consolidater spawn without returning a Worker summary.".into(), - candidate_count, - total_bytes, - }); - }; - Ok(MemoryConsolidationOutput { - status: "started".into(), - summary: format!( - "Started subject Memory consolidater '{}' for {candidate_count} staging candidate(s).", - worker.worker.worker_id - ), - candidate_count, - total_bytes, - }) -} - -fn try_reuse_subject_consolidation_worker( - api: &WorkspaceApi, - singleton_key: &str, - input_content: &str, - candidate_count: usize, - total_bytes: u64, -) -> ApiResult> { - let Some(owner) = api - .store - .current_worker_singleton_owner(&api.config.workspace_id, singleton_key)? - else { - return Ok(None); - }; - let mut worker = api - .runtime - .worker(&owner.worker) - .map_err(|error| error.into_error())?; - worker.singleton_key = Some(owner.key); - require_dedicated_subjektiv_consolidation_worker( - &worker, - &api.config.workspace_id, - singleton_key, - )?; - if worker.state != "idle" { - return Ok(Some(MemoryConsolidationOutput { - status: "skipped_existing_not_idle".into(), - summary: format!( - "Existing subject Memory consolidater '{}' is '{}', not confirmed idle.", - worker.worker.worker_id, worker.state - ), - candidate_count, - total_bytes, - })); - } - let input = api - .runtime - .send_input( - &worker.worker, - WorkerInputRequest { - kind: WorkerInputKind::User, - content: input_content.to_string(), - submission_request_id: None, - segments: None, - }, - ) - .map_err(|error| error.into_error())?; - if input.state != InternalWorkerOperationState::Accepted { - return Ok(Some(MemoryConsolidationOutput { - status: "skipped_existing_input_rejected".into(), - summary: "Existing subject Memory consolidater rejected new input.".into(), - candidate_count, - total_bytes, - })); - } - Ok(Some(MemoryConsolidationOutput { - status: "reused".into(), - summary: format!( - "Reused subject Memory consolidater '{}' for {candidate_count} staging candidate(s).", - worker.worker.worker_id - ), - candidate_count, - total_bytes, - })) + subjektiv_job::dispatch(&api, subject_id, candidate_count, total_bytes) } async fn scoped_memory_consolidation( @@ -21457,13 +22506,17 @@ async fn scoped_workspace_protocol_ws( State(api): State, Extension(actor): Extension, AxumPath(workspace_id): AxumPath, + headers: HeaderMap, ws: axum::extract::ws::WebSocketUpgrade, ) -> std::result::Result { validate_workspace_scope(&api, &workspace_id).map_err(|error| error.into_response())?; - let input_source = authenticated_browser_input_source(&actor); + let context = WorkerOperationContext::Browser { + headers, + source: authenticated_browser_input_source(&actor), + }; Ok(ws .on_upgrade(move |socket| { - crate::workspace_subscription::serve_workspace_subscription(api, socket, input_source) + crate::workspace_subscription::serve_workspace_subscription(api, socket, context) }) .into_response()) } @@ -21608,13 +22661,22 @@ async fn worker_remove_contract( None, ) })?; - let response = WorkerRemovalService::new(api) - .execute_async( - source, - &request.target_runtime_id, - &request.target_worker_id, - &request.reason, - ) + let worker = + WorkspaceWorker::resolve(api, &request.target_runtime_id, &request.target_worker_id) + .map_err(|error| match error { + Error::UnknownWorker { .. } => server_api::WorkerRemoveApiError::new( + StatusCode::NOT_FOUND.as_u16(), + None, + Some("unknown_worker".into()), + Some("The target Worker is not known in this Workspace".into()), + None, + ), + error => worker_remove_repository_error( + ApiError::from(error).into_repository_api_error(), + ), + })?; + let response = worker + .remove(source, &request.reason) .await .unwrap_or_else(|_| { worker_remove_error_response( @@ -21964,36 +23026,16 @@ async fn send_known_worker_input( ) -> ApiResult> { validate_workspace_scope(&api, &path.workspace_id)?; let source = authenticate_worker_mutation_source(&api, &path.workspace_id, &headers)?; - let controller = RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id); - let permission = if request.kind.as_deref() == Some("notify") { - "notify" - } else { - "send_input" + let context = WorkerOperationContext::WorkerControl { + controller: RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id), }; - let grant = authorize_known_worker_permission( - &api, - &path.workspace_id, - &controller, - &path.worker, - permission, - )?; - let lock = worker_control_lock(&api, &grant.grant_id); - let _guard = lock.lock().await; - authorize_known_worker_permission( - &api, - &path.workspace_id, - &controller, - &path.worker, - permission, - )?; - let mut request = worker_input_request_from_api(request)?; + let worker = WorkspaceWorker::resolve(&api, &path.worker.runtime_id, &path.worker.worker_id)?; + let request = worker_input_request_from_api(request)?; let idle_only = request.kind == WorkerInputKind::User; - if idle_only { - request.kind = WorkerInputKind::UserIfIdle; - } - let Json(result) = - dispatch_runtime_worker_input(api, path.worker.runtime_id, path.worker.worker_id, request)?; - if result.state != server_api::WorkerOperationState::Accepted { + let result = worker.input(&context, request).await?; + let outcome_unknown = result.disposition == WorkerInputDisposition::Unknown; + let result = worker_input_result_to_api(result); + if result.state != server_api::WorkerOperationState::Accepted && !outcome_unknown { let detail = result .diagnostics .iter() @@ -22020,24 +23062,15 @@ async fn cancel_known_worker( ) -> ApiResult> { validate_workspace_scope(&api, &path.workspace_id)?; let source = authenticate_worker_mutation_source(&api, &path.workspace_id, &headers)?; - let controller = RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id); - let grant = authorize_known_worker_permission( - &api, - &path.workspace_id, - &controller, - &path.worker, - "cancel", - )?; - let lock = worker_control_lock(&api, &grant.grant_id); - let _guard = lock.lock().await; - authorize_known_worker_permission( - &api, - &path.workspace_id, - &controller, - &path.worker, - "cancel", - )?; - scoped_cancel_runtime_worker(State(api), AxumPath(path), Json(request)).await + let context = WorkerOperationContext::WorkerControl { + controller: RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id), + }; + let worker = WorkspaceWorker::resolve(&api, &path.worker.runtime_id, &path.worker.worker_id)?; + Ok(Json(worker_lifecycle_result_to_api( + worker + .cancel(&context, worker_lifecycle_request_from_api(request)) + .await?, + ))) } async fn stop_known_worker( @@ -22048,42 +23081,34 @@ async fn stop_known_worker( ) -> ApiResult> { validate_workspace_scope(&api, &path.workspace_id)?; let source = authenticate_worker_mutation_source(&api, &path.workspace_id, &headers)?; - let subject = path.worker.clone(); - let controller = RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id); - let grant = - authorize_known_worker_permission(&api, &path.workspace_id, &controller, &subject, "stop")?; - let lock = worker_control_lock(&api, &grant.grant_id); - let _guard = lock.lock().await; - authorize_known_worker_permission(&api, &path.workspace_id, &controller, &subject, "stop")?; - scoped_stop_runtime_worker(State(api), AxumPath(path), Json(request)).await + let context = WorkerOperationContext::WorkerControl { + controller: RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id), + }; + let worker = WorkspaceWorker::resolve(&api, &path.worker.runtime_id, &path.worker.worker_id)?; + Ok(Json(worker_lifecycle_result_to_api( + worker + .stop(&context, worker_lifecycle_request_from_api(request)) + .await?, + ))) } async fn restore_known_worker( State(api): State, AxumPath(path): AxumPath, headers: HeaderMap, + request: server_api::WorkerRestoreRequest, ) -> ApiResult> { validate_workspace_scope(&api, &path.workspace_id)?; let source = authenticate_worker_mutation_source(&api, &path.workspace_id, &headers)?; - let subject = path.worker.clone(); - let controller = RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id); - let grant = authorize_known_worker_permission( - &api, - &path.workspace_id, - &controller, - &subject, - "restore", - )?; - let lock = worker_control_lock(&api, &grant.grant_id); - let _guard = lock.lock().await; - authorize_known_worker_permission(&api, &path.workspace_id, &controller, &subject, "restore")?; - scoped_restore_runtime_worker( - State(api), - AxumPath(path), - Query(RestoreTicketAssignmentQuery { - ticket_id: None, - assignment_operation_id: None, - }), + let context = WorkerOperationContext::WorkerControl { + controller: RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id), + }; + restore_runtime_worker_with_context( + api, + path.worker.runtime_id, + path.worker.worker_id, + context, + request, ) .await } @@ -22230,14 +23255,38 @@ async fn scoped_start_workspace_orchestrator( let mut disposition = "created"; if let Some(existing) = find_workspace_orchestrator(&api) { if workspace_orchestrator_is_online(&existing) { + drop(_guard); + // Already-online is not a new startup. Only retry outstanding delivery. + dispatch_orchestrator_queue_attention(&api, &existing.worker, true); return Ok(Json(workspace_orchestrator_response(&api, "existing"))); } - let restored = api.restore_workspace_worker(&existing.worker)?; + let attention = orchestrator_attention_state(&api, &existing.worker); + let before_restore = attention + .lock() + .unwrap_or_else(|p| p.into_inner()) + .accepted + .as_ref() + .map(|delivery| delivery.request_id.clone()); + let restored = + restore_internal_observed_worker(&api, &existing, "orchestrator-restore".to_owned())?; if restored.state == server_api::WorkerRestoreState::Accepted { - *api.orchestrator_attention_fingerprint - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) = None; - dispatch_orchestrator_queue_attention(&api); + { + let mut attention = attention.lock().unwrap_or_else(|p| p.into_inner()); + // Retire only pre-Restore success. Concurrent reservations, unknown + // outcomes and newly accepted deliveries must survive Restore. + if attention.pending.is_none() + && attention.accepted.as_ref().map(|d| &d.request_id) == before_restore.as_ref() + { + attention.accepted = None; + } + // A reserved caller may already have decided not to send. Carry + // the explicit Restore check through its completion in that case. + if attention.sending { + attention.restore_recheck = true; + } + } + drop(_guard); + dispatch_orchestrator_queue_attention(&api, &existing.worker, false); return Ok(Json(workspace_orchestrator_response(&api, "restored"))); } if !diagnostics_indicate_unrecoverable_pending_workspace_restore(&restored.diagnostics) { @@ -22316,10 +23365,8 @@ async fn scoped_start_workspace_orchestrator( Some("builtin:orchestrator".to_string()), WorkerRegistryDisplayNamePolicy::UseProvided, )?; - *api.orchestrator_attention_fingerprint - .lock() - .unwrap_or_else(|poisoned| poisoned.into_inner()) = None; - dispatch_orchestrator_queue_attention(&api); + drop(_guard); + dispatch_orchestrator_queue_attention(&api, &worker.worker, false); Ok(Json(workspace_orchestrator_response(&api, disposition))) } @@ -23639,6 +24686,20 @@ async fn scoped_working_directory_detail( code: "working_directory_not_found".to_string(), message: format!("Unknown Workdir `{}`", path.working_directory_id), })?; + if let WorkdirRegistrySource::WorkspaceConfig { grant_id } = &record.source { + let grant = api + .store + .get_workspace_config_grant(&api.config.workspace_id, grant_id)? + .ok_or_else(|| { + Error::WorkspacePermissionDenied("Workspace config grant unavailable".into()) + })?; + return Ok(Json(BrowserWorkingDirectoryDetailResponse { + workspace_id: api.config.workspace_id.clone(), + runtime_id: None, + item: workspace_config::summary(&grant, &record), + diagnostics: vec![], + })); + } if let Some(runtime_id) = record.source.runtime_id().map(str::to_string) { working_directory_detail_for_runtime(api, &runtime_id, &path.working_directory_id) } else { @@ -24581,48 +25642,19 @@ fn recover_workdir_removals(api: &WorkspaceApi) -> Result<()> { Ok(()) } +#[cfg(test)] async fn set_worker_retention( api: WorkspaceApi, runtime_id: String, runtime_worker_id: String, pinned: bool, ) -> ApiResult> { - parse_runtime_worker_id_for_registry(&runtime_worker_id)?; - let worker_ref = RuntimeWorkerRef::new(runtime_id.clone(), runtime_worker_id.clone()); - if api - .store - .get_worker_registry(&api.config.workspace_id, &worker_ref)? - .is_none() - { - if let Ok(worker) = api.runtime.worker(&worker_ref) { - let _ = sync_worker_observation(&api, &worker); - } - } - let retention_state = if pinned { "pinned" } else { "normal" }; - let changed = api.worker_projection.publish_ordered(|| { - api.store - .update_worker_retention( - &api.config.workspace_id, - &worker_ref, - retention_state, - now_registry_timestamp().as_str(), - ) - .map(|commit| (!commit.changes.is_empty(), commit)) - })?; - if !changed { - return Err(cleanup_api_error( - runtime_id.as_str(), - "workspace_worker_retention_unknown_worker", - "Worker is not known to the Backend registry", - )); - } - Ok(Json(WorkerRetentionResponse { - workspace_id: api.config.workspace_id, - runtime_id: worker_ref.runtime_id, - worker_id: worker_ref.worker_id, - pinned, - retention_state: retention_state.to_string(), - })) + let worker = WorkspaceWorker::resolve(&api, &runtime_id, &runtime_worker_id)?; + Ok(Json( + worker + .set_pinned(&WorkerOperationContext::Backend, pinned) + .await?, + )) } fn build_runtime_cleanup_plan( @@ -24849,11 +25881,30 @@ fn cleanup_plan_digest( Ok(format!("sha256:{digest}")) } +#[cfg(test)] async fn execute_runtime_cleanup( api: &WorkspaceApi, runtime_id: &str, request: ExecuteRuntimeCleanupRequest, ) -> ApiResult { + execute_runtime_cleanup_with_context(api, runtime_id, request, &WorkerOperationContext::Backend) + .await +} + +async fn execute_runtime_cleanup_with_context( + api: &WorkspaceApi, + runtime_id: &str, + request: ExecuteRuntimeCleanupRequest, + context: &WorkerOperationContext, +) -> ApiResult { + if matches!(context, WorkerOperationContext::WorkerControl { .. }) { + return Err(Error::WorkspacePermissionDenied( + "Worker control uses target-bound WorkerRemove authority, not aggregate cleanup".into(), + ) + .into()); + } + + context.authorize_workspace(api).await?; let plan = build_runtime_cleanup_plan(api, runtime_id)?; if request.expected_plan_revision != plan.revision || request.expected_plan_digest != plan.digest @@ -24906,8 +25957,8 @@ async fn execute_runtime_cleanup( )); } parse_runtime_worker_id_for_registry(&candidate.runtime_worker_id)?; - WorkerRemovalService::new(api) - .execute_cleanup_removal(candidate) + WorkspaceWorker::resolve(api, &worker.runtime_id, &worker.worker_id)? + .delete_from_plan(context, candidate) .await?; api.workdir_session_locks .lock() @@ -25256,15 +26307,27 @@ async fn scoped_get_runtime_worker( .await } -async fn scoped_restore_runtime_worker( - State(api): State, - AxumPath(path): AxumPath, - Query(query): Query, +async fn scoped_restore_runtime_worker_with_context( + api: WorkspaceApi, + path: ScopedRuntimeWorkerPath, + query: RestoreTicketAssignmentQuery, + context: WorkerOperationContext, + request: server_api::WorkerRestoreRequest, ) -> ApiResult> { validate_workspace_scope(&api, &path.workspace_id)?; + let resolved = WorkspaceWorker::resolve(&api, &path.worker.runtime_id, &path.worker.worker_id)?; + let _authorization = resolved.authorize_operation(&context, "restore").await?; let workspace_id = path.workspace_id.clone(); - let runtime_id = path.worker.runtime_id.clone(); - let worker_id = path.worker.worker_id.clone(); + let runtime_id = resolved.identity().runtime_id.clone(); + let worker_id = resolved.identity().worker_id.clone(); + if matches!(context, WorkerOperationContext::WorkerControl { .. }) + && (query.ticket_id.is_some() || query.assignment_operation_id.is_some()) + { + return Err(Error::WorkspacePermissionDenied( + "Worker control grants do not authorize Ticket assignment".into(), + ) + .into()); + } let assignment_request = match ( query.ticket_id.clone(), query.assignment_operation_id.clone(), @@ -25304,32 +26367,14 @@ async fn scoped_restore_runtime_worker( &fingerprint, &Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true), )?; - if let Some(worker) = existing_lifecycle_assignment_worker(&api, assignment, &runtime_id)? { - if worker.worker.worker_id != worker_id { - return Err(Error::TicketAssignmentConflict(format!( - "assignment operation {} belongs to worker {}, not {}", - assignment.operation_id, worker.worker.worker_id, worker_id - )) - .into()); - } - assign_ticket_worker_from_lifecycle(&api, assignment, &runtime_id, &worker_id)?; - accept_queued_ticket_after_worker_spawn(&api, assignment)?; - let worker = project_workspace_worker(&api, worker)?; - return Ok(Json(server_api::WorkerRestoreResponse { - workspace_id, - runtime_id: runtime_id.clone(), - worker_id: worker_id.clone(), - result: server_api::WorkerRestoreResult { - state: server_api::WorkerRestoreState::Accepted, - worker: Some(worker), - diagnostics: Vec::new(), - }, - })); - } } - let response = restore_runtime_worker( - State(api.clone()), - AxumPath((runtime_id.clone(), worker_id.clone())), + drop(_authorization); + let response = restore_runtime_worker_with_context( + api.clone(), + runtime_id.clone(), + worker_id.clone(), + context, + request, ) .await?; if let Some(assignment) = assignment_request.as_ref() { @@ -25339,22 +26384,6 @@ async fn scoped_restore_runtime_worker( Ok(response) } -async fn scoped_pin_runtime_worker( - State(api): State, - AxumPath(path): AxumPath, -) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - set_worker_retention(api, path.worker.runtime_id, path.worker.worker_id, true).await -} - -async fn scoped_unpin_runtime_worker( - State(api): State, - AxumPath(path): AxumPath, -) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - set_worker_retention(api, path.worker.runtime_id, path.worker.worker_id, false).await -} - async fn scoped_runtime_cleanup_plan( State(api): State, AxumPath(path): AxumPath, @@ -25364,6 +26393,7 @@ async fn scoped_runtime_cleanup_plan( Ok(Json(plan)) } +#[cfg(test)] async fn scoped_execute_runtime_cleanup( State(api): State, AxumPath(path): AxumPath, @@ -25596,6 +26626,7 @@ async fn scoped_delete_runtime_worker_uploaded_file( .await } +#[cfg(test)] async fn scoped_send_runtime_worker_input( State(api): State, AxumPath(path): AxumPath, @@ -25624,36 +26655,9 @@ async fn scoped_runtime_worker_completions( .await } -async fn scoped_stop_runtime_worker( - State(api): State, - AxumPath(path): AxumPath, - Json(request): Json, -) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - stop_runtime_worker( - State(api), - AxumPath((path.worker.runtime_id, path.worker.worker_id)), - Json(request), - ) - .await -} - -async fn scoped_cancel_runtime_worker( - State(api): State, - AxumPath(path): AxumPath, - Json(request): Json, -) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - cancel_runtime_worker( - State(api), - AxumPath((path.worker.runtime_id, path.worker.worker_id)), - Json(request), - ) - .await -} - async fn scoped_worker_protocol_ws( ws: WebSocketUpgrade, + headers: HeaderMap, State(api): State, Extension(actor): Extension, AxumPath(path): AxumPath, @@ -25665,6 +26669,7 @@ async fn scoped_worker_protocol_ws( State(api), Extension(actor), AxumPath((path.worker.runtime_id, path.worker.worker_id)), + headers, ws, ) .await @@ -27680,7 +28685,7 @@ fn restore_reused_subject_worker( } let worker_ref = worker.worker.clone(); - let restored = api.restore_workspace_worker(&worker_ref)?; + let restored = restore_internal_observed_worker(api, worker, "subject-restore".to_owned())?; if restored.state != server_api::WorkerRestoreState::Accepted { let message = runtime_diagnostics_message(&restored.diagnostics); return Err(ApiError::with_diagnostics( @@ -28124,32 +29129,85 @@ async fn post_companion_cancel( Ok(Json(api.companion.cancel(request))) } +fn runtime_contract_restore( + value: T, +) -> ApiResult { + serde_json::from_value( + serde_json::to_value(value).map_err(|e| Error::InvalidInput(e.to_string()))?, + ) + .map_err(|e| Error::InvalidInput(e.to_string()).into()) +} + +fn restore_internal_worker_intent( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + request_id: String, +) -> ApiResult { + let observed = api + .runtime + .worker(worker) + .map_err(RuntimeRegistryError::into_error)?; + restore_internal_observed_worker(api, &observed, request_id) +} + +fn restore_internal_observed_worker( + api: &WorkspaceApi, + observed: &InternalWorkerSummary, + request_id: String, +) -> ApiResult { + let token = observed.restore_observation_token.as_deref(); + let explicit = !matches!( + request_id.as_str(), + "orchestrator-restore" | "subject-restore" + ); + let request = api.config_store.pin_internal_worker_restore_intent( + api.workspace_id(), + &observed.worker, + &request_id, + token, + explicit.then_some(request_id.as_str()), + )?; + let request_id = request.request_id.clone(); + let result = match WorkspaceWorker::resolve( + api, + &observed.worker.runtime_id, + &observed.worker.worker_id, + )? + .restore_internal(request) + { + Ok(result) => result, + Err(error) => { + // A definitive non-admission conflict is not unknown delivery. Retire + // only this pinned intent, without retrying it or exchanging its + // observation. A later deliberate invocation may observe anew. + if matches!(error.error, Error::RestoreObservationConflict) { + api.config_store.settle_internal_worker_restore_intent( + api.workspace_id(), + &observed.worker, + &request_id, + )?; + } + return Err(error); + } + }; + if result.state != server_api::WorkerRestoreState::ReconciliationRequired { + api.config_store.settle_internal_worker_restore_intent( + api.workspace_id(), + &observed.worker, + &request_id, + )?; + } + Ok(result) +} + fn resolve_workspace_worker_reference( api: &WorkspaceApi, runtime_id: &str, reference: &str, ) -> ApiResult { - let direct = RuntimeWorkerRef::new(runtime_id, reference); - if let Some(record) = api - .store - .get_worker_registry(&api.config.workspace_id, &direct)? - { - return Ok(record.worker); - } - let worker = api - .store - .resolve_worker_resource_reference(&api.config.workspace_id, reference)? - .filter(|worker| worker.runtime_id == runtime_id) - .ok_or_else(|| Error::UnknownWorker { - worker: RuntimeWorkerRef::new(runtime_id, reference), - })?; - let record = api - .store - .get_worker_registry(&api.config.workspace_id, &worker)? - .ok_or_else(|| Error::UnknownWorker { - worker: RuntimeWorkerRef::new(runtime_id, reference), - })?; - Ok(record.worker) + Ok(WorkspaceWorker::resolve(api, runtime_id, reference)? + .identity() + .clone()) } async fn get_runtime_worker( @@ -28177,27 +29235,40 @@ async fn get_runtime_worker( })) } -async fn restore_runtime_worker( - State(api): State, - AxumPath((runtime_id, worker_id)): AxumPath<(String, String)>, +async fn restore_runtime_worker_with_context( + api: WorkspaceApi, + runtime_id: String, + worker_id: String, + context: WorkerOperationContext, + request: server_api::WorkerRestoreRequest, ) -> ApiResult> { - let worker = resolve_workspace_worker_reference(&api, &runtime_id, &worker_id)?; - let result = api.restore_workspace_worker(&worker)?; + let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id)?; + let mut result = worker.restore(&context, request).await?; let projected_worker = if let Some(worker) = result.worker.as_ref() { - let record = sync_worker_observation(&api, worker)?; + // A replayed terminal receipt is historical evidence, not a new Runtime observation. + let record = api + .store + .get_worker_registry(api.workspace_id(), &worker.worker)? + .ok_or_else(|| Error::UnknownWorker { + worker: worker.worker.clone(), + })?; let links = api .store .list_worker_workdir_links(&api.config.workspace_id, &record.worker)?; let workdirs = api .store .list_workdir_registry(&api.config.workspace_id, 500)?; - Some(project_worker_registry_projection( - &api, - Some(worker), - &record, - links, - &workdirs, - )?) + match project_worker_registry_projection(&api, Some(worker), &record, links, &workdirs) { + Ok(worker) => Some(worker), + Err(error) => { + result.diagnostics.push(RuntimeDiagnostic::new( + "worker_restore_current_projection_unavailable", + "warning", + sanitize_backend_error(&error.error.to_string()), + )); + None + } + } } else { None }; @@ -28858,37 +29929,27 @@ async fn check_runtime_config_bundle( )?)) } +#[cfg(test)] async fn send_runtime_worker_input( State(api): State, AxumPath((runtime_id, worker_id)): AxumPath<(String, String)>, Json(request): Json, ) -> ApiResult> { let request = worker_input_request_from_api(request)?; - dispatch_runtime_worker_input(api, runtime_id, worker_id, request) + dispatch_runtime_worker_input(api, runtime_id, worker_id, request).await } -fn dispatch_runtime_worker_input( +#[cfg(test)] +async fn dispatch_runtime_worker_input( api: WorkspaceApi, runtime_id: String, worker_id: String, request: WorkerInputRequest, ) -> ApiResult> { - let worker = resolve_workspace_worker_reference(&api, &runtime_id, &worker_id)?; - if api - .store - .worker_registry_projection(&api.config.workspace_id, &worker)? - .and_then(|projection| projection.job) - .is_some() - { - return Err(Error::InvalidInput( - "Backend Job Workers accept only their Backend-owned immutable input".to_string(), - ) - .into()); - } - let result = api - .runtime - .send_input(&worker, request) - .map_err(|err| err.into_error())?; + let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id)?; + let result = worker + .input(&WorkerOperationContext::Backend, request) + .await?; Ok(Json(worker_input_result_to_api(result))) } @@ -28917,44 +29978,6 @@ async fn runtime_worker_completions( Ok(Json(worker_completions_result_to_api(result)?)) } -async fn stop_runtime_worker( - State(api): State, - AxumPath((runtime_id, worker_id)): AxumPath<(String, String)>, - Json(request): Json, -) -> ApiResult> { - let request = worker_lifecycle_request_from_api(request); - let worker = resolve_workspace_worker_reference(&api, &runtime_id, &worker_id)?; - let result = api - .runtime - .stop_worker(&worker, request) - .map_err(|err| err.into_error())?; - parse_runtime_worker_id_for_registry(&worker.worker_id)?; - let session_lock = current_worker_session_lock(&api, &worker); - let _session_guard = session_lock.lock().await; - close_current_worker_session_locked(&api, &worker).await?; - if let Some(record) = api - .store - .get_worker_registry(&api.config.workspace_id, &worker)? - { - sync_linked_workdir_after_worker_stop(&api, &worker.runtime_id, &record)?; - } - Ok(Json(worker_lifecycle_result_to_api(result))) -} - -async fn cancel_runtime_worker( - State(api): State, - AxumPath((runtime_id, worker_id)): AxumPath<(String, String)>, - Json(request): Json, -) -> ApiResult> { - let request = worker_lifecycle_request_from_api(request); - let worker = resolve_workspace_worker_reference(&api, &runtime_id, &worker_id)?; - let result = api - .runtime - .cancel_worker(&worker, request) - .map_err(|err| err.into_error())?; - Ok(Json(worker_lifecycle_result_to_api(result))) -} - fn authenticated_browser_input_source(actor: &RequestActor) -> protocol::AuthenticatedInputSource { protocol::AuthenticatedInputSource::Account { account_id: actor.account_id.clone(), @@ -28984,10 +30007,25 @@ pub(crate) fn authorize_browser_worker_method( }), protocol::Method::SubmitTracked { .. } | protocol::Method::SubmitIfIdle { .. } - | protocol::Method::NotifyTracked { .. } => { + | protocol::Method::NotifyTracked { .. } + | protocol::Method::WorkerEvent(_) => { Err("authenticated Worker input source is server-owned") } - other => Ok(other), + other @ (protocol::Method::ListPendingSubmissions + | protocol::Method::CancelPendingSubmission { .. } + | protocol::Method::ClearPendingSubmissions { .. } + | protocol::Method::ContinuePending { .. } + | protocol::Method::Resume { .. } + | protocol::Method::Cancel { .. } + | protocol::Method::Pause { .. } + | protocol::Method::Compact { .. } + | protocol::Method::ListRewindTargets + | protocol::Method::RewindTo { .. } + | protocol::Method::Shutdown { .. } + | protocol::Method::ListCompletions { .. } + | protocol::Method::ListWorkers + | protocol::Method::RestoreWorker { .. } + | protocol::Method::RegisterPeer { .. }) => Ok(other), } } @@ -28995,39 +30033,59 @@ async fn worker_protocol_ws( State(api): State, Extension(actor): Extension, AxumPath((runtime_id, worker_id)): AxumPath<(String, String)>, + headers: HeaderMap, ws: WebSocketUpgrade, -) -> impl IntoResponse { - let worker = RuntimeWorkerRef::new(&runtime_id, &worker_id); - let job_read_only = match api - .store - .worker_registry_projection(&api.config.workspace_id, &worker) - { - Ok(projection) => projection.and_then(|projection| projection.job).is_some(), +) -> Response { + let worker = match WorkspaceWorker::resolve(&api, &runtime_id, &worker_id) { + Ok(worker) => worker, Err(error) => return ApiError::from(error).into_response(), }; - let source = match api.observation_proxy.source(&worker) { - Ok(source) => source, - Err(ObservationProxyError::WorkerNotFound(_)) => { - match api.runtime.observation_source(&worker) { - Ok(source) => source, - Err(error) => return ApiError::from(error.into_error()).into_response(), + let context = WorkerOperationContext::Browser { + headers, + source: authenticated_browser_input_source(&actor), + }; + ws.on_upgrade(move |mut socket| async move { + match worker.connect_protocol(&context).await { + Ok(connection) => workspace_worker_protocol_session(connection, socket, context).await, + Err(error) => { + let _ = send_protocol_event(&mut socket, &protocol_error_event(error.to_string())) + .await; + let _ = socket.send(WsMessage::Close(None)).await; } } - Err(error) => { - return ( - StatusCode::BAD_REQUEST, - Json(serde_json::json!({ - "error": error.code(), - "message": error.message(), - })), - ) - .into_response(); - } - }; - let input_source = authenticated_browser_input_source(&actor); - ws.on_upgrade(move |socket| { - worker_protocol_ws_session(source, socket, input_source, job_read_only) }) + .into_response() +} + +async fn workspace_worker_protocol_session( + mut connection: worker_operations::WorkspaceWorkerConnection, + mut socket: WebSocket, + context: WorkerOperationContext, +) { + loop { + tokio::select! { + inbound = socket.next() => match inbound { + Some(Ok(WsMessage::Text(text))) => { + let result = match decode_method(&text) { + Ok(method) => connection.sender.send(&context, method).await.map_err(|error| error.to_string()), + Err(error) => Err(error.to_string()), + }; + if let Err(error) = result { + if !send_protocol_event(&mut socket, &protocol_error_event(error)).await { break; } + } + } + Some(Ok(WsMessage::Ping(value))) => { if socket.send(WsMessage::Pong(value)).await.is_err() { break; } } + Some(Ok(WsMessage::Pong(_))) => {} + _ => break, + }, + event = connection.events.recv() => { + let Some(event) = event else { break; }; + let shutdown = matches!(event, protocol::Event::Shutdown); + if !send_protocol_event(&mut socket, &event).await || shutdown { break; } + } + } + } + let _ = socket.send(WsMessage::Close(None)).await; } pub(crate) struct WorkspaceWorkerProtocolConnection { @@ -29059,7 +30117,9 @@ pub(crate) async fn connect_workspace_worker_protocol( connect_remote_worker_protocol(config, input_source).await } RuntimeObservationSource::Embedded(source) => { - connect_embedded_worker_protocol(source).await + let scope = + worker_runtime::RuntimeWorkspaceScope::new(api.workspace_id(), "embedded-backend"); + connect_embedded_worker_protocol(source, scope).await } } } @@ -29141,38 +30201,55 @@ async fn connect_remote_worker_protocol( async fn connect_embedded_worker_protocol( source: crate::observation::EmbeddedRuntimeObservationSource, + scope: worker_runtime::RuntimeWorkspaceScope, ) -> Result { - let mut upstream = - RuntimeObservationClient::connect(&RuntimeObservationSource::Embedded(source.clone())) - .await - .map_err(|error| Error::RuntimeOperationFailed { - runtime_id: source.worker.runtime_id.clone(), - code: error.code().to_string(), - message: error.message().to_string(), - })?; + // Capture one real execution endpoint, including its greeting/snapshot. + // The WorkerRef-wide Workspace observation bus is a separate catalogue + // channel and must not reconnect this operational session after Restore. + let mut transport = source + .runtime + .attach_worker_protocol_scoped(&scope, &source.worker_ref) + .map_err(|error| Error::RuntimeOperationFailed { + runtime_id: source.worker.runtime_id.clone(), + code: "embedded_worker_protocol_connect_failed".to_string(), + message: error.to_string(), + })?; let (methods, mut method_receiver) = tokio::sync::mpsc::channel(256); let (event_sender, events) = tokio::sync::mpsc::channel(512); tokio::spawn(async move { + if event_sender.send(transport.snapshot.clone()).await.is_err() { + return; + } loop { tokio::select! { + _ = event_sender.closed() => break, method = method_receiver.recv() => { let Some(method) = method else { break }; - match source.runtime.send_protocol_method(&source.worker_ref, method) { + let shutdown = matches!(&method, protocol::Method::Shutdown { .. }); + match source.runtime.send_connected_protocol_method_scoped( + &scope, &source.worker_ref, &transport, method, + ) { Ok(direct_events) => { for event in direct_events { - if event_sender.send(event).await.is_err() { return; } + let terminal = matches!(event, protocol::Event::Shutdown); + if event_sender.send(event).await.is_err() || terminal { return; } } } Err(error) => { - if event_sender.send(protocol_error_event(error.to_string())).await.is_err() { return; } + let terminal = embedded_protocol_dispatch_is_terminal(&error, &transport); + if event_sender.send(protocol_error_event(error.to_string())).await.is_err() || terminal { return; } } } + if shutdown { break; } } - event = upstream.next_event() => match event { - Ok(event) => { - if event_sender.send(event.payload).await.is_err() { break; } + event = transport.events.recv() => match event { + Some(event) => { + let terminal = matches!(event, protocol::Event::Shutdown); + if event_sender.send(event).await.is_err() || terminal { break; } } - Err(_) => break, + // The execution relay closes on Stop, disconnect or lag. + // Uncertain streams are terminal, never grounds to reattach. + None => break, } } } @@ -29180,22 +30257,43 @@ async fn connect_embedded_worker_protocol( Ok(WorkspaceWorkerProtocolConnection { methods, events }) } +fn embedded_protocol_dispatch_is_terminal( + error: &worker_runtime::error::RuntimeError, + transport: &worker_runtime::execution::WorkerProtocolTransport, +) -> bool { + // A settled, side-effect-free method rejection can leave the connection + // usable. Uncertain lifecycle/store/transport errors cannot. + !matches!( + error, + worker_runtime::error::RuntimeError::WorkerExecutionRejected { + outcome: worker_runtime::execution::WorkerExecutionOutcome::Busy + | worker_runtime::execution::WorkerExecutionOutcome::Rejected + | worker_runtime::execution::WorkerExecutionOutcome::Unsupported, + .. + } + ) || transport.validate().is_err() +} + +#[cfg(test)] async fn worker_protocol_ws_session( source: RuntimeObservationSource, socket: WebSocket, input_source: protocol::AuthenticatedInputSource, read_only: bool, + scope: worker_runtime::RuntimeWorkspaceScope, ) { match source { RuntimeObservationSource::RemoteWs(config) => { remote_worker_protocol_ws_session(config, socket, input_source, read_only).await; } RuntimeObservationSource::Embedded(source) => { - embedded_worker_protocol_ws_session(source, socket, input_source, read_only).await; + embedded_worker_protocol_ws_session(source, socket, input_source, read_only, scope) + .await; } } } +#[cfg(test)] async fn remote_worker_protocol_ws_session( config: RuntimeObservationSourceConfig, socket: WebSocket, @@ -29346,24 +30444,29 @@ async fn remote_worker_protocol_ws_session( } } +#[cfg(test)] async fn embedded_worker_protocol_ws_session( source: crate::observation::EmbeddedRuntimeObservationSource, mut socket: WebSocket, input_source: protocol::AuthenticatedInputSource, read_only: bool, + scope: worker_runtime::RuntimeWorkspaceScope, ) { - let mut upstream = match RuntimeObservationClient::connect(&RuntimeObservationSource::Embedded( - source.clone(), - )) - .await + let mut transport = match source + .runtime + .attach_worker_protocol_scoped(&scope, &source.worker_ref) { - Ok(client) => client, + Ok(transport) => transport, Err(error) => { - let event = protocol_error_event(error.message()); + let event = protocol_error_event(error.to_string()); let _ = send_protocol_event(&mut socket, &event).await; + let _ = socket.send(WsMessage::Close(None)).await; return; } }; + if !send_protocol_event(&mut socket, &transport.snapshot).await { + return; + } loop { tokio::select! { @@ -29377,24 +30480,44 @@ async fn embedded_worker_protocol_ws_session( } Some(Ok(WsMessage::Text(text))) => match decode_method(&text) { Ok(method) => match authorize_browser_worker_method(method, &input_source) { - Ok(method) => match source.runtime.send_protocol_method(&source.worker_ref, method) { - Ok(events) => { - for event in events { + Ok(method) => { + let shutdown = matches!(&method, protocol::Method::Shutdown { .. }); + match source.runtime.send_connected_protocol_method_scoped( + &scope, &source.worker_ref, &transport, method, + ) { + Ok(events) => { + for event in events { + let terminal = matches!(event, protocol::Event::Shutdown); + if !send_protocol_event(&mut socket, &event).await { + return; + } + if terminal { + let _ = socket.send(WsMessage::Close(None)).await; + return; + } + } + } + Err(error) => { + let terminal = embedded_protocol_dispatch_is_terminal(&error, &transport); + let event = protocol_error_event(error.to_string()); if !send_protocol_event(&mut socket, &event).await { return; } + if terminal { + let _ = socket.send(WsMessage::Close(None)).await; + return; + } } } - Err(error) => { - let event = protocol_error_event(error.to_string()); - if !send_protocol_event(&mut socket, &event).await { - return; - } + if shutdown { + let _ = socket.send(WsMessage::Close(None)).await; + return; } }, Err(message) => { let event = protocol_error_event(message); let _ = send_protocol_event(&mut socket, &event).await; + let _ = socket.send(WsMessage::Close(None)).await; return; } }, @@ -29421,16 +30544,22 @@ async fn embedded_worker_protocol_ws_session( } } } - upstream_event = upstream.next_event() => { + upstream_event = transport.events.recv() => { match upstream_event { - Ok(event) => { - if !send_protocol_event(&mut socket, &event.payload).await { + Some(event) => { + let terminal = matches!(event, protocol::Event::Shutdown); + if !send_protocol_event(&mut socket, &event).await { + return; + } + if terminal { + let _ = socket.send(WsMessage::Close(None)).await; return; } } - Err(error) => { - let event = protocol_error_event(error.message()); - let _ = send_protocol_event(&mut socket, &event).await; + // Stop/disconnect/lag closes the execution-owned relay. + // Never follow the WorkerRef to a restored Controller. + None => { + let _ = socket.send(WsMessage::Close(None)).await; return; } } @@ -29483,52 +30612,6 @@ async fn list_host_workers( })) } -fn project_workspace_worker( - api: &WorkspaceApi, - summary: InternalWorkerSummary, -) -> ApiResult { - let links = api - .store - .list_worker_workdir_links(&api.config.workspace_id, &summary.worker)?; - let workdir_attachments = summary - .workdir_attachments - .iter() - .map(|attachment| { - let record = workdir_record_from_summary( - api, - &summary.worker.runtime_id, - &attachment.working_directory.summary, - ); - let link = links - .iter() - .find(|link| { - link.unlinked_at.is_none() - && link.alias == attachment.alias.as_str() - && link.workdir_id == record.workdir_id - }) - .ok_or_else(|| { - Error::RegistryInconsistency(format!( - "Worker {}:{} attachment `{}` has no active durable Workdir link", - summary.worker.runtime_id, summary.worker.worker_id, attachment.alias - )) - })?; - Ok(server_api::WorkerWorkdirAttachmentSummary { - alias: attachment.alias.to_string(), - effective_permissions: workdir_permission_summary( - effective_worker_workdir_capabilities( - api, - &summary.worker, - &record, - link.capabilities, - )?, - ), - working_directory: projected_workdir_summary_from_record(api, &record)?.into(), - }) - }) - .collect::>>()?; - project_workspace_worker_with_attachments(api, summary, workdir_attachments) -} - fn project_workspace_worker_with_attachments( api: &WorkspaceApi, mut summary: InternalWorkerSummary, @@ -30286,6 +31369,9 @@ fn working_directory_summaries( .list_workdir_registry(&api.config.workspace_id, 200)?; let mut items = Vec::new(); for record in records { + if matches!(record.source, WorkdirRegistrySource::WorkspaceConfig { .. }) { + continue; + } if let Some(grant_id) = legacy_external_workdir_grant_id(api, &record)? { diagnostics.push(RuntimeDiagnostic { code: "external_workdir_legacy_permissions".to_string(), @@ -30314,6 +31400,9 @@ fn available_working_directory_summaries( .list_workdir_registry(&api.config.workspace_id, 200)?; let mut available = Vec::new(); for record in records { + if matches!(record.source, WorkdirRegistrySource::WorkspaceConfig { .. }) { + continue; + } if legacy_external_workdir_grant_id(api, &record)?.is_some() { continue; } @@ -30323,6 +31412,7 @@ fn available_working_directory_summaries( summary.cleanliness.as_deref() == Some("clean") } WorkdirRegistrySource::ExternalGrant { .. } => true, + WorkdirRegistrySource::WorkspaceConfig { .. } => false, }; if summary.status != WorkingDirectoryStatusKind::Active || !source_is_available { continue; @@ -30431,6 +31521,7 @@ fn record_worker_summary( _ => protocol::subscription::SubscriptionWorkerState::Idle, }; let observation = protocol::subscription::SubscriptionWorker { + restore_observation_token: worker.restore_observation_token.clone(), worker_id, runtime_id: Some(worker.worker.runtime_id.clone()), resource_key: None, @@ -30463,6 +31554,7 @@ fn record_worker_summary( fn worker_summary_from_registry(record: &WorkerRegistryRecord) -> InternalWorkerSummary { InternalWorkerSummary { + restore_observation_token: None, worker: record.worker.clone(), host_id: "backend-registry".to_string(), display_name: record.display_name.clone(), @@ -30505,6 +31597,9 @@ fn worker_summary_from_projection( let observed = observation.availability == protocol::subscription::SubscriptionWorkerAvailability::Observed; summary.availability = observation.availability; + summary.restore_observation_token = observed + .then(|| observation.worker.restore_observation_token.clone()) + .flatten(); summary.state = if observed { match observation.worker.state { protocol::subscription::SubscriptionWorkerState::Idle => "idle", @@ -30553,6 +31648,30 @@ fn project_worker_registry_projection( else { continue; }; + if let WorkdirRegistrySource::WorkspaceConfig { grant_id } = &workdir.source { + if !workspace_config::granted_to(api, &record.worker, grant_id) { + continue; + } + let grant = api + .store + .get_workspace_config_grant(&api.config.workspace_id, grant_id)? + .ok_or_else(|| { + Error::WorkspacePermissionDenied("Workspace config grant unavailable".into()) + })?; + workdir_attachments.push(server_api::WorkerWorkdirAttachmentSummary { + alias: link.alias, + effective_permissions: workdir_permission_summary( + effective_worker_workdir_capabilities( + api, + &record.worker, + workdir, + link.capabilities, + )?, + ), + working_directory: workspace_config::summary(&grant, workdir), + }); + continue; + } let effective_permissions = workdir_permission_summary( effective_worker_workdir_capabilities(api, &record.worker, workdir, link.capabilities)?, ); @@ -30954,6 +32073,9 @@ fn workdir_summary_from_record( _ => WorkingDirectoryStatusKind::Unknown, }; let (source, materializer_kind, cleanup_target) = match &record.source { + WorkdirRegistrySource::WorkspaceConfig { .. } => { + unreachable!("logical config never enters filesystem summary") + } WorkdirRegistrySource::Repository { .. } => { let repository_key = repository_key.expect("Repository Workdir projection requires key"); @@ -31033,6 +32155,11 @@ fn projected_workdir_summary_from_record( record: &WorkdirRegistryRecord, ) -> Result { let (repository_key, external_grant_permissions) = match &record.source { + WorkdirRegistrySource::WorkspaceConfig { .. } => { + return Err(Error::WorkspacePermissionDenied( + "Logical config is not a filesystem Workdir".into(), + )); + } WorkdirRegistrySource::Repository { repository_id, .. } => ( Some( api.store @@ -31079,6 +32206,7 @@ fn link_worker_to_workdir( ) -> ApiResult<()> { let timestamp = now_registry_timestamp(); let record = WorkerWorkdirLinkRecord { + connection_id: String::new(), workspace_id: api.config.workspace_id.clone(), worker: worker_record.worker.clone(), workdir_id: workdir_id.to_string(), @@ -31598,6 +32726,82 @@ fn authorize_repository_materialization( Ok(()) } +fn complete_restore_repository_access( + api: &WorkspaceApi, + request_id: &str, + preparation: &mut runtime_api::WorkerRestorePreparation, +) -> ApiResult<()> { + if preparation.repository_access.is_empty() { + for reference in &preparation.repository_access_workdirs { + if let Some(access) = repository_access_request_for_workdir( + api, + &reference.runtime_id, + &reference.working_directory_id, + &format!("worker-restore:{request_id}"), + )? { + preparation + .repository_access + .push(runtime_contract_restore(access)?); + } + } + return Ok(()); + } + for prepared in &mut preparation.repository_access { + let mut access: worker_runtime::catalog::WorkingDirectoryRepositoryAccessRequest = + runtime_contract_restore(prepared.clone())?; + let Some(ssh) = access.materialization.ssh.as_mut() else { + continue; + }; + let mut leases = Vec::new(); + for candidate in &ssh.credential_candidates { + leases.push( + api.repository_secrets + .lease_ssh_materialization_access_revision( + api.workspace_id(), + &candidate.credential_id, + candidate.credential_revision, + &ssh.host_trust_id, + ssh.host_trust_revision, + )?, + ); + } + let known_hosts_entry = leases + .first() + .ok_or_else(|| Error::Config("Admitted SSH snapshot has no candidates".into()))? + .known_hosts_entry + .clone(); + ssh.expires_at_epoch_seconds = repository_access_expiry(); + ssh.secret_resource = api + .resource_broker + .issue_repository_ssh_access_handle( + api.workspace_id(), + &access.materialization.runtime_id, + format!( + "repository-ssh-access:{}", + access.materialization.operation_id + ), + ssh.secret_resource.revision.clone(), + i64::try_from(ssh.expires_at_epoch_seconds).unwrap_or(i64::MAX), + worker_runtime::resource::RepositorySshAccessSecret { + credential_candidates: leases + .into_iter() + .map( + |lease| worker_runtime::resource::RepositorySshAccessSecretCandidate { + credential_id: lease.credential_id, + credential_revision: lease.credential_revision, + private_key: lease.private_key.as_str().to_owned(), + }, + ) + .collect(), + known_hosts_entry, + }, + ) + .map_err(|error| Error::Config(error.to_string()))?; + *prepared = runtime_contract_restore(access)?; + } + Ok(()) +} + fn repository_access_request_for_workdir( api: &WorkspaceApi, runtime_id: &str, @@ -32085,7 +33289,11 @@ impl ApiError { let message = api_error_response_message(&self.error); server_api::RepositoryApiError::new( status.as_u16(), - status.canonical_reason().unwrap_or("error"), + if matches!(self.error, Error::RestoreObservationConflict) { + "restore_observation_conflict" + } else { + status.canonical_reason().unwrap_or("error") + }, message, working_directory_diagnostics(self.diagnostics), ) @@ -32103,7 +33311,8 @@ fn api_error_status(error: &Error) -> StatusCode { | Error::RuntimeBindingConflict(_) | Error::RuntimeBindingRevisionConflict { .. } | Error::RuntimeBindingFingerprintConflict { .. } - | Error::RepositoryConflict(_) => StatusCode::CONFLICT, + | Error::RepositoryConflict(_) + | Error::RestoreObservationConflict => StatusCode::CONFLICT, Error::WorkerSourceIdentity(_) => StatusCode::BAD_REQUEST, Error::InvalidInput(message) if message.starts_with("memory_not_found:") @@ -32256,6 +33465,9 @@ impl IntoResponse for ApiError { #[cfg(test)] mod tests { + mod subjektiv_jobs_tests; + mod worker_operations_tests; + include!("server_workspace_config_tests.rs"); use super::*; use axum::body::{Body, to_bytes}; use axum::http::Request; @@ -32419,6 +33631,7 @@ mod tests { resource_key: Some("W-1".to_string()), observation: Some(WorkerRegistryObservationRecord { worker: protocol::subscription::SubscriptionWorker { + restore_observation_token: None, worker_id: protocol::subscription::SubscriptionWorkerId::new("worker-a") .unwrap(), runtime_id: Some("runtime-a".to_string()), @@ -32596,6 +33809,96 @@ mod tests { assert!(commands.try_recv().is_err()); } + #[tokio::test] + async fn checkout_external_provider_typed_boundary_and_unknown_response() { + let (sender, mut commands) = tokio::sync::mpsc::channel(1); + let connection = Arc::new(ExternalProviderConnection { + grant_id: "checkout-grant".into(), + workdir_id: "checkout-external".into(), + provider_instance_id: "checkout-provider".into(), + generation: 7, + expires_at: None, + capabilities: workdir::WorkdirSessionCapabilities::READ_WRITE, + admission: Arc::new(tokio::sync::Semaphore::new(16)), + shutdown_confirmed: Arc::new(std::sync::atomic::AtomicBool::new(false)), + sender, + }); + let session = Arc::new(ExternalProviderWorkdirSession::new(connection, None)); + let path = workdir::WorkdirPath::new("file.txt").unwrap(); + let observe = { + let session = session.clone(); + let path = path.clone(); + tokio::spawn(async move { session.checkout_observe(path).await }) + }; + let Some(ExternalProviderCommand::Operation { + operation, + response, + .. + }) = commands.recv().await + else { + panic!("expected operation") + }; + assert!(matches!( + operation, + WorkdirSessionOperation::CheckoutObserve(_) + )); + response + .send(Ok(WorkdirSessionOperationResult::CheckoutObserve( + workdir::CheckoutObservation { + path: path.clone(), + kind: workdir::EntryKind::File, + size: 4, + validator: vec![1], + capabilities: workdir::WorkdirSessionCapabilities::READ_WRITE, + }, + ))) + .unwrap(); + let observed = observe.await.unwrap().unwrap(); + for lose_response in [false, true] { + let request = workdir::CheckoutRequest { + target: path.clone(), + validator: observed.validator.clone(), + operation: workdir::CheckoutOperation::Write { + content: b"next".to_vec(), + expected_hash: [0; 32], + }, + }; + let write = { + let session = session.clone(); + tokio::spawn(async move { session.checkout_execute(request).await }) + }; + let Some(ExternalProviderCommand::Operation { + operation, + response, + .. + }) = commands.recv().await + else { + panic!("expected operation") + }; + assert!(matches!( + operation, + WorkdirSessionOperation::CheckoutExecute(_) + )); + if lose_response { + drop(response); + } else { + // A valid ordinary result is still the wrong subject/operation for this checked call. + response + .send(Ok(WorkdirSessionOperationResult::Write( + workdir::WriteResult { + bytes_written: 4, + created: false, + }, + ))) + .unwrap(); + } + assert!(matches!( + write.await.unwrap(), + Err(workdir::WorkdirError::OutcomeUnknown(_)) + )); + } + } + #[tokio::test] async fn external_operation_cancel_retries_without_blocking_a_saturated_provider_queue() { let (sender, mut commands) = tokio::sync::mpsc::channel(1); @@ -33218,6 +34521,7 @@ mod tests { .unwrap(); schedule_external_workdir_expiry(&api, &grant).unwrap(); let link = WorkerWorkdirLinkRecord { + connection_id: String::new(), workspace_id: TEST_WORKSPACE_ID.to_string(), worker: worker.clone(), workdir_id: grant.workdir_id.clone(), @@ -34186,6 +35490,7 @@ mod tests { }) .unwrap(); let repository_link = WorkerWorkdirLinkRecord { + connection_id: String::new(), workspace_id: TEST_WORKSPACE_ID.to_string(), worker: worker.clone(), workdir_id: "analysis-repository".to_string(), @@ -34195,6 +35500,7 @@ mod tests { unlinked_at: None, }; let external_link = WorkerWorkdirLinkRecord { + connection_id: String::new(), workspace_id: TEST_WORKSPACE_ID.to_string(), worker: worker.clone(), workdir_id: grant.working_directory_id.clone(), @@ -34420,6 +35726,9 @@ mod tests { reject_next_spawn: Arc>, workdir_repositories: Arc>>, logical_attachments: Arc>>>, + repository_access_requests: + Arc>>, + reject_repository_access: Arc>, } impl WorkdirlessFixtureRuntime { @@ -34502,6 +35811,10 @@ mod tests { request: &WorkerSpawnRequest, ) -> InternalWorkerSummary { InternalWorkerSummary { + restore_observation_token: Some(format!( + "fixture-generation:{}", + binding.worker_id + )), worker: RuntimeWorkerRef::new(Self::RUNTIME_ID, binding.worker_id.to_string()), host_id: "workdirless-host".to_string(), display_name: request @@ -34730,7 +36043,51 @@ mod tests { } } - fn restore_worker(&self, worker_id: &str) -> InternalWorkerRestoreResult { + fn authorize_working_directory_repository_access( + &self, + request: worker_runtime::catalog::WorkingDirectoryRepositoryAccessRequest, + ) -> std::result::Result<(), Error> { + assert_eq!(request.materialization.runtime_id, Self::RUNTIME_ID); + self.repository_access_requests + .lock() + .unwrap() + .push(request); + if *self.reject_repository_access.lock().unwrap() { + Err(Error::InvalidInput("repository access unavailable".into())) + } else { + Ok(()) + } + } + + fn coordinate_worker_restore( + &self, + worker_id: &str, + request: runtime_api::WorkerRestoreCoordinationRequest, + ) -> std::result::Result< + ( + Option, + Option, + ), + RuntimeDiagnostic, + > { + if self.worker(worker_id).worker.is_none() { + return Ok(( + Some(InternalWorkerRestoreResult { + state: server_api::WorkerRestoreState::Rejected, + worker: None, + diagnostics: Vec::new(), + }), + None, + )); + } + Ok((None, request.preparation)) + } + + fn restore_worker( + &self, + worker_id: &str, + _request: runtime_api::WorkerRestoreRequest, + ) -> InternalWorkerRestoreResult { let worker = self.worker(worker_id).worker; InternalWorkerRestoreResult { state: if worker.is_some() { @@ -35524,7 +36881,7 @@ mod tests { updated_at: "2026-01-01T00:00:00Z".to_owned(), revoked_at: None, }, - false, + true, ) .unwrap(); } @@ -35648,6 +37005,33 @@ mod tests { } } + #[test] + fn feature_completion_api_conversions_preserve_nonce_and_context_for_aba_requests() { + for nonce in ["first", "second", "first"] { + let json = serde_json::json!({"kind":"feature_argument", "prefix":"資料/", "request_id":nonce, + "context":{"invocation":"builtin:test/prepare", "argument":"path"}}); + let wire: server_api::RuntimeWorkerCompletionsRequest = + serde_json::from_value(json.clone()).unwrap(); + let request = worker_completions_request_from_api(wire).unwrap(); + assert_eq!(serde_json::to_value(&request).unwrap(), json); + let result = worker_completions_result_to_api(WorkerCompletionsResult { + worker: RuntimeWorkerRef::new("runtime", "worker"), + kind: request.kind, + prefix: request.prefix, + context: request.context, + request_id: request.request_id, + entries: Vec::new(), + diagnostics: Vec::new(), + }) + .unwrap(); + assert_eq!(result.request_id.as_deref(), Some(nonce)); + assert_eq!( + serde_json::to_value(result.context).unwrap(), + json["context"] + ); + } + } + #[test] fn flow_or_generic_worker_state_change_is_not_ticket_completion_authority() { let change = || { @@ -35939,6 +37323,7 @@ mod tests { }; api.store.upsert_workdir_registry(&workdir).unwrap(); let link = WorkerWorkdirLinkRecord { + connection_id: String::new(), workspace_id: TEST_WORKSPACE_ID.to_string(), worker: worker.worker.clone(), workdir_id: workdir.workdir_id.clone(), @@ -36043,6 +37428,7 @@ mod tests { .create_external_workdir_grant(&grant, &workdir) .unwrap(); let link = WorkerWorkdirLinkRecord { + connection_id: String::new(), workspace_id: TEST_WORKSPACE_ID.to_string(), worker: worker.worker.clone(), workdir_id: workdir.workdir_id.clone(), @@ -36056,6 +37442,7 @@ mod tests { .seed_observation( EMBEDDED_WORKER_RUNTIME_ID, &protocol::subscription::SubscriptionWorker { + restore_observation_token: None, worker_id: protocol::subscription::SubscriptionWorkerId::new("paused-worker") .unwrap(), runtime_id: Some(EMBEDDED_WORKER_RUNTIME_ID.to_string()), @@ -36943,6 +38330,7 @@ mod tests { .unwrap(); api.store .attach_worker_workdir(&WorkerWorkdirLinkRecord { + connection_id: String::new(), workspace_id: TEST_WORKSPACE_ID.to_string(), worker: worker.clone(), workdir_id: workdir_id.to_string(), @@ -37481,6 +38869,11 @@ mod tests { Request::builder() .method("POST") .uri(format!("/api/w/{TEST_WORKSPACE_ID}/worker-control/workers")) + // Unit fixture stands below proof middleware, so inject its verified subject. + .extension(crate::worker_source::VerifiedRuntimeRequestSource { + runtime_id: fixture.controller.runtime_id.clone(), + worker_id: Some(fixture.controller.worker_id.clone()), + }) .header("content-type", "application/json") .header("x-yoi-runtime-id", &fixture.controller.runtime_id) .header("x-yoi-worker-id", &fixture.controller.worker_id) @@ -37796,6 +39189,10 @@ mod tests { Request::builder() .method("POST") .uri(format!("/api/w/{TEST_WORKSPACE_ID}/worker-control/workers")) + .extension(crate::worker_source::VerifiedRuntimeRequestSource { + runtime_id: controller.runtime_id.clone(), + worker_id: Some(controller.worker_id.clone()), + }) .header("content-type", "application/json") .header("x-yoi-runtime-id", &controller.runtime_id) .header("x-yoi-worker-id", &controller.worker_id) @@ -38209,7 +39606,12 @@ mod tests { Error::RepositoryConflict(_) )); - let restored = api.restore_workspace_worker(&worker).unwrap(); + let restored = restore_internal_worker_intent( + &api, + &worker, + format!("restore-internal:{}", WorkerId::now_v7()), + ) + .unwrap(); assert_eq!(restored.state, server_api::WorkerRestoreState::Rejected); assert_eq!( api.store @@ -39059,18 +40461,80 @@ mod tests { app.clone(), "POST", &format!("/api/w/{TEST_WORKSPACE_ID}/subjektiv/subjects"), - Some(serde_json::json!({ "role": " Browser author " })), + Some(serde_json::json!({ + "role": " Browser author ", + "behavior_md": "Prefer explicit evidence.\nAsk when uncertain." + })), &owner_token, StatusCode::CREATED, ) .await; assert_eq!(created["role"], " Browser author "); + assert_eq!( + created["behavior_md"], + "Prefer explicit evidence.\nAsk when uncertain." + ); + assert_eq!(created["behavior_revision"], 0); assert_eq!(created["state"], "active"); assert_eq!(created["store_revision"], 0); assert!(created.get("current_worker").is_none()); let created_id = created["id"].as_str().unwrap(); let persisted = store.subject(created_id).unwrap().unwrap(); assert_eq!(persisted.role.as_str(), " Browser author "); + assert_eq!(persisted.behavior_revision, 0); + let behavior_uri = + format!("/api/w/{TEST_WORKSPACE_ID}/subjektiv/subjects/{created_id}/behavior"); + request_json( + app.clone(), + "PATCH", + &behavior_uri, + Some(serde_json::json!({ + "expected_behavior_revision": 0, + "behavior_md": "Updated without rewriting history." + })), + StatusCode::UNAUTHORIZED, + ) + .await; + let updated = request_json_authenticated( + app.clone(), + "PATCH", + &behavior_uri, + Some(serde_json::json!({ + "expected_behavior_revision": 0, + "behavior_md": "Updated without rewriting history." + })), + &owner_token, + StatusCode::OK, + ) + .await; + assert_eq!(updated["behavior_md"], "Updated without rewriting history."); + assert_eq!(updated["behavior_revision"], 1); + request_json_authenticated( + app.clone(), + "PATCH", + &behavior_uri, + Some(serde_json::json!({ + "expected_behavior_revision": 0, + "behavior_md": "stale write" + })), + &owner_token, + StatusCode::CONFLICT, + ) + .await; + let cleared = request_json_authenticated( + app.clone(), + "PATCH", + &behavior_uri, + Some(serde_json::json!({ + "expected_behavior_revision": 1, + "behavior_md": "" + })), + &owner_token, + StatusCode::OK, + ) + .await; + assert_eq!(cleared["behavior_md"], ""); + assert_eq!(cleared["behavior_revision"], 2); assert!( api.store .current_worker_singleton_owner( @@ -39171,6 +40635,90 @@ mod tests { ); } + #[tokio::test] + async fn subjektiv_browser_behavior_create_update_clear_and_conflict_are_typed() { + let workspace = tempfile::tempdir().unwrap(); + init_clean_git_workspace(workspace.path()); + let api = test_api(workspace.path()).await; + + let (status, Json(created)) = scoped_create_subjektiv_subject( + State(api.clone()), + AxumPath(ScopedWorkspacePath { + workspace_id: TEST_WORKSPACE_ID.to_string(), + }), + Json(server_api::SubjektivSubjectCreateRequest { + role: "companion".to_string(), + behavior_md: "Prefer explicit evidence.".to_string(), + }), + ) + .await + .unwrap(); + assert_eq!(status, StatusCode::CREATED); + assert_eq!(created.behavior_revision, 0); + assert_eq!(created.behavior_md, "Prefer explicit evidence."); + + let Json(updated) = scoped_update_subjektiv_subject_behavior( + State(api.clone()), + AxumPath(ScopedSubjektivSubjectPath { + workspace_id: TEST_WORKSPACE_ID.to_string(), + subject_id: created.id.clone(), + }), + Json(server_api::SubjektivSubjectBehaviorUpdateRequest { + expected_behavior_revision: 0, + behavior_md: "Ask when uncertain.".to_string(), + }), + ) + .await + .unwrap(); + assert_eq!(updated.behavior_revision, 1); + assert_eq!(updated.behavior_md, "Ask when uncertain."); + + let conflict = scoped_update_subjektiv_subject_behavior( + State(api.clone()), + AxumPath(ScopedSubjektivSubjectPath { + workspace_id: TEST_WORKSPACE_ID.to_string(), + subject_id: created.id.clone(), + }), + Json(server_api::SubjektivSubjectBehaviorUpdateRequest { + expected_behavior_revision: 0, + behavior_md: "Stale edit".to_string(), + }), + ) + .await + .unwrap_err(); + assert_eq!(conflict.into_response().status(), StatusCode::CONFLICT); + + let Json(cleared) = scoped_update_subjektiv_subject_behavior( + State(api.clone()), + AxumPath(ScopedSubjektivSubjectPath { + workspace_id: TEST_WORKSPACE_ID.to_string(), + subject_id: created.id, + }), + Json(server_api::SubjektivSubjectBehaviorUpdateRequest { + expected_behavior_revision: 1, + behavior_md: String::new(), + }), + ) + .await + .unwrap(); + assert_eq!(cleared.behavior_revision, 2); + assert!(cleared.behavior_md.is_empty()); + + let oversized = scoped_create_subjektiv_subject( + State(api), + AxumPath(ScopedWorkspacePath { + workspace_id: TEST_WORKSPACE_ID.to_string(), + }), + Json(server_api::SubjektivSubjectCreateRequest { + role: "reviewer".to_string(), + behavior_md: "x".repeat(server_api::SUBJEKTIV_MAX_BEHAVIOR_BYTES + 1), + }), + ) + .await + .unwrap_err(); + assert_eq!(oversized.into_response().status(), StatusCode::BAD_REQUEST); + } + #[tokio::test] async fn subjektiv_browser_reads_are_bounded_scoped_and_revision_exact() { let workspace = tempfile::tempdir().unwrap(); @@ -39593,7 +41141,10 @@ mod tests { assert_eq!(missing.status(), StatusCode::NOT_FOUND); let subject = open_subjektiv_store(&api) .unwrap() - .create_subject(crate::subjektiv::SubjectRole::new("companion").unwrap()) + .create_subject_with_behavior( + crate::subjektiv::SubjectRole::new("companion").unwrap(), + "Prefer explicit evidence.".to_string(), + ) .unwrap(); let request = CreateWorkspaceWorkerRequest { runtime_id: EMBEDDED_WORKER_RUNTIME_ID.to_string(), @@ -39712,6 +41263,33 @@ mod tests { } ) )); + let Json(resident_context) = scoped_subjektiv_memory_backend( + State(api.clone()), + AxumPath(ScopedWorkspacePath { + workspace_id: TEST_WORKSPACE_ID.to_string(), + }), + context.clone(), + Json(server_api::SubjektivMemoryBackendRequest { + operation: server_api::SubjektivMemoryBackendOperation::ResidentContext( + Default::default(), + ), + }), + ) + .await + .unwrap(); + assert!(matches!( + resident_context, + server_api::SubjektivMemoryBackendResponse::ResidentContext( + server_api::SubjektivResidentContextOutput { + behavior_md, + behavior_revision: 0, + memory_surface: memory::backend::MemoryResidentSummaryOutput { + availability: memory::backend::MemoryResidentSummaryAvailability::Ungenerated, + content: None, + }, + } + ) if behavior_md == "Prefer explicit evidence." + )); let missing_legacy = scoped_record_subjektiv_session( State(api.clone()), AxumPath(ScopedWorkspacePath { @@ -40862,7 +42440,12 @@ mod tests { Error::RepositoryConflict(message) if message.contains("worker_singleton_owned") )); assert_eq!(runtime.spawn_requests().len(), spawn_count); - let restored = api.restore_workspace_worker(&worker.worker).unwrap(); + let restored = restore_internal_observed_worker( + &api, + &worker, + format!("restore-test:{}", WorkerId::now_v7()), + ) + .unwrap(); assert_eq!(restored.state, server_api::WorkerRestoreState::Accepted); assert_eq!( restored.worker.unwrap().singleton_key.as_deref(), @@ -40874,7 +42457,12 @@ mod tests { let disconnected_runtime = WorkdirlessFixtureRuntime::default(); api.runtime .register_or_replace(disconnected_runtime.clone()); - let disconnected_restore = api.restore_workspace_worker(&worker.worker).unwrap(); + let disconnected_restore = restore_internal_observed_worker( + &api, + &worker, + format!("restore-test:{}", WorkerId::now_v7()), + ) + .unwrap(); assert_eq!( disconnected_restore.state, server_api::WorkerRestoreState::Rejected @@ -40903,7 +42491,13 @@ mod tests { // any Runtime restore side effect. api.runtime.register_or_replace(runtime.clone()); assert_eq!( - api.restore_workspace_worker(&worker.worker).unwrap().state, + restore_internal_observed_worker( + &api, + &worker, + format!("restore-test:{}", WorkerId::now_v7()) + ) + .unwrap() + .state, server_api::WorkerRestoreState::Accepted ); let stale_worker = runtime @@ -40925,8 +42519,14 @@ mod tests { .unwrap_or_else(std::sync::PoisonError::into_inner) .push(stale_worker); assert!(matches!( - api.restore_workspace_worker(&worker.worker).unwrap_err().error, - Error::RepositoryConflict(message) if message.contains("worker_singleton_fenced") + restore_internal_observed_worker( + &api, + &worker, + format!("restore-test:{}", WorkerId::now_v7()) + ) + .unwrap_err() + .error, + Error::UnknownWorker { .. } )); let replacement = api .spawn_workspace_worker( @@ -40938,8 +42538,14 @@ mod tests { .unwrap(); assert_ne!(replacement.worker.worker_id, worker.worker.worker_id); assert!(matches!( - api.restore_workspace_worker(&worker.worker).unwrap_err().error, - Error::RepositoryConflict(message) if message.contains("worker_singleton_fenced") + restore_internal_observed_worker( + &api, + &worker, + format!("restore-test:{}", WorkerId::now_v7()) + ) + .unwrap_err() + .error, + Error::UnknownWorker { .. } )); assert_eq!( api.store @@ -41307,6 +42913,7 @@ mod tests { .unwrap(); api.store .attach_worker_workdir(&WorkerWorkdirLinkRecord { + connection_id: String::new(), workspace_id: TEST_WORKSPACE_ID.to_string(), worker: RuntimeWorkerRef::new(EMBEDDED_WORKER_RUNTIME_ID, "7"), workdir_id: "managed".to_string(), @@ -41823,11 +43430,125 @@ mod tests { assert!(!workdir_runtime_miss_is_not_found(&unrelated)); } + // The receiver belongs to one mock execution, not to a Runtime WorkerRecord. + // Bound protocol senders become terminal when Stop drops this execution. + struct DeterministicWorkerExecution { + context: worker_runtime::execution::WorkerExecutionContext, + state: std::sync::Mutex, + method_tx: tokio::sync::mpsc::Sender, + method_rx: std::sync::Mutex>>, + events: tokio::sync::broadcast::Sender, + protocol_methods: + Arc>>, + } + + impl DeterministicWorkerExecution { + fn new( + context: worker_runtime::execution::WorkerExecutionContext, + protocol_methods: Arc< + std::sync::Mutex>, + >, + ) -> Self { + let (method_tx, method_rx) = tokio::sync::mpsc::channel(32); + let (events, _) = tokio::sync::broadcast::channel(32); + Self { + context, + state: std::sync::Mutex::new(protocol::WorkerStateSnapshot::initial()), + method_tx, + method_rx: std::sync::Mutex::new(Some(method_rx)), + events, + protocol_methods, + } + } + + fn unavailable() -> worker_runtime::execution::WorkerExecutionResult { + worker_runtime::execution::WorkerExecutionResult::rejected( + worker_runtime::execution::WorkerExecutionOperation::ProtocolMethod, + "deterministic test execution protocol is closed", + ) + } + + fn dispatch( + &self, + method: protocol::Method, + ) -> std::result::Result< + Vec, + worker_runtime::execution::WorkerExecutionResult, + > { + // Drive the actual mock receiver synchronously; no method is routed + // through a WorkerRef lookup after a protocol connection is attached. + let mut receiver = self.method_rx.lock().unwrap(); + let receiver = receiver.as_mut().ok_or_else(Self::unavailable)?; + self.method_tx + .try_send(method) + .map_err(|_| Self::unavailable())?; + let method = receiver.try_recv().map_err(|_| Self::unavailable())?; + self.protocol_methods + .lock() + .unwrap() + .push((self.context.worker_ref().clone(), method.clone())); + Ok(match method { + protocol::Method::ListCompletions { + kind, + prefix, + request_id, + context, + } => vec![protocol::Event::Completions { + kind, + prefix, + request_id, + context, + entries: Vec::new(), + }], + _ => Vec::new(), + }) + } + + fn publish_event( + &self, + event: protocol::Event, + ) -> std::result::Result<(), worker_runtime::error::RuntimeError> { + if let protocol::Event::WorkerState { snapshot } = &event { + *self.state.lock().unwrap() = snapshot.clone(); + } + let _ = self.events.send(event.clone()); + self.context.publish_protocol_event(event).map(|_| ()) + } + + fn snapshot(&self) -> protocol::Event { + protocol::Event::Snapshot { + session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), + entries: Vec::new(), + }, + greeting: protocol::Greeting { + worker_name: self.context.worker_ref().worker_id.to_string(), + cwd: String::new(), + provider: "deterministic-workspace-server-test".to_string(), + model: "deterministic-workspace-server-test".to_string(), + reasoning: None, + scope_summary: "test execution snapshot".to_string(), + tools: Vec::new(), + context_window: 0, + context_tokens: 0, + context_usage: None, + }, + state: self.state.lock().unwrap().clone(), + in_flight: protocol::InFlightSnapshot { + blocks: Vec::new(), + commands: Vec::new(), + compaction: None, + }, + internal_workers: Vec::new(), + } + } + } + struct DeterministicExecutionBackend { contexts: std::sync::Mutex< std::collections::HashMap< worker_runtime::identity::WorkerRef, - worker_runtime::execution::WorkerExecutionContext, + Arc, >, >, materializer: worker_runtime::working_directory::RuntimeGitMaterializer, @@ -41838,9 +43559,16 @@ mod tests { >, input_request_ids: std::sync::Mutex>>, accept_restores: std::sync::atomic::AtomicBool, + restore_failure: + std::sync::Mutex>, + restore_operations: + std::sync::Mutex>, + reconcile_operations: + std::sync::Mutex>, inputs: std::sync::Mutex>, protocol_methods: - std::sync::Mutex>, + Arc>>, + stops: std::sync::Mutex>, } impl Default for DeterministicExecutionBackend { @@ -41863,8 +43591,12 @@ mod tests { input_hook: std::sync::Mutex::new(None), input_request_ids: std::sync::Mutex::new(Vec::new()), accept_restores: std::sync::atomic::AtomicBool::new(false), + restore_failure: std::sync::Mutex::new(None), + restore_operations: std::sync::Mutex::new(Vec::new()), + reconcile_operations: std::sync::Mutex::new(Vec::new()), inputs: std::sync::Mutex::new(Vec::new()), - protocol_methods: std::sync::Mutex::new(Vec::new()), + protocol_methods: Arc::new(std::sync::Mutex::new(Vec::new())), + stops: std::sync::Mutex::new(Vec::new()), } } } @@ -41891,19 +43623,34 @@ mod tests { worker: &RuntimeWorkerRef, snapshot: protocol::WorkerStateSnapshot, ) { - let context = self + let execution = self .contexts .lock() .unwrap() .iter() .find(|(worker_ref, _)| worker_ref.worker_id.to_string() == worker.worker_id) - .map(|(_, context)| context.clone()) - .expect("execution context"); - context - .publish_protocol_event(protocol::Event::WorkerState { snapshot }) + .map(|(_, execution)| execution.clone()) + .expect("mock execution"); + execution + .publish_event(protocol::Event::WorkerState { snapshot }) .unwrap(); } + fn emit_worker_event( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + event: protocol::Event, + ) { + let execution = self + .contexts + .lock() + .unwrap() + .get(worker_ref) + .cloned() + .expect("mock execution"); + execution.publish_event(event).unwrap(); + } + fn take_input_request_ids(&self) -> Vec> { std::mem::take( &mut *self @@ -41991,15 +43738,14 @@ mod tests { working_directory: binding.status(), }) .collect(); - self.contexts - .lock() - .unwrap() - .insert(request.worker_ref.clone(), request.context); + self.contexts.lock().unwrap().insert( + request.worker_ref.clone(), + Arc::new(DeterministicWorkerExecution::new( + request.context, + self.protocol_methods.clone(), + )), + ); worker_runtime::execution::WorkerExecutionSpawnResult::Connected { - handle: worker_runtime::execution::WorkerExecutionHandle::new( - request.worker_ref, - self.backend_id(), - ), worker_state: protocol::WorkerStateSnapshot { ..protocol::WorkerStatus::Idle.into() }, @@ -42007,10 +43753,31 @@ mod tests { } } + fn preflight_restore( + &self, + request: &worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result<(), worker_runtime::execution::WorkerExecutionResult> { + if let Some(execution) = self.contexts.lock().unwrap().get(&request.worker_ref) { + return Err(worker_runtime::execution::WorkerExecutionResult::busy( + worker_runtime::execution::WorkerExecutionOperation::Restore, + "deterministic test backend already has a live execution", + ) + .with_worker_state(execution.state.lock().unwrap().clone())); + } + Ok(()) + } + fn restore_worker( &self, request: worker_runtime::execution::WorkerExecutionRestoreRequest, ) -> worker_runtime::execution::WorkerExecutionSpawnResult { + self.restore_operations + .lock() + .unwrap() + .push(request.operation_id); + if let Some(result) = self.restore_failure.lock().unwrap().take() { + return result; + } if !self .accept_restores .load(std::sync::atomic::Ordering::SeqCst) @@ -42030,69 +43797,114 @@ mod tests { working_directory: binding.status(), }) .collect(); - self.contexts - .lock() - .unwrap() - .insert(request.worker_ref.clone(), request.context); + self.contexts.lock().unwrap().insert( + request.worker_ref.clone(), + Arc::new(DeterministicWorkerExecution::new( + request.context, + self.protocol_methods.clone(), + )), + ); worker_runtime::execution::WorkerExecutionSpawnResult::Connected { - handle: worker_runtime::execution::WorkerExecutionHandle::new( - request.worker_ref, - self.backend_id(), - ), worker_state: protocol::WorkerStateSnapshot::initial(), workdir_attachments, } } - fn dispatch_method( + fn reconcile_restore( &self, - handle: &worker_runtime::execution::WorkerExecutionHandle, - method: protocol::Method, - ) -> worker_runtime::execution::WorkerExecutionResult { - self.protocol_methods + request: worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> worker_runtime::execution::WorkerExecutionSpawnResult { + self.reconcile_operations .lock() .unwrap() - .push((handle.worker_ref().clone(), method)); - worker_runtime::execution::WorkerExecutionResult::accepted( - worker_runtime::execution::WorkerExecutionOperation::ProtocolMethod, - ) + .push(request.operation_id); + self.restore_worker(request) + } + + fn dispatch_method( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + method: protocol::Method, + ) -> worker_runtime::execution::WorkerExecutionResult { + let execution = self.contexts.lock().unwrap().get(worker_ref).cloned(); + let Some(execution) = execution else { + return DeterministicWorkerExecution::unavailable(); + }; + match execution.dispatch(method) { + Ok(_) => worker_runtime::execution::WorkerExecutionResult::accepted( + worker_runtime::execution::WorkerExecutionOperation::ProtocolMethod, + ), + Err(result) => result, + } + } + + fn attach_worker_protocol( + self: Arc, + worker_ref: &worker_runtime::identity::WorkerRef, + ) -> std::result::Result< + worker_runtime::execution::WorkerProtocolTransport, + worker_runtime::execution::WorkerExecutionResult, + > { + let execution = self + .contexts + .lock() + .unwrap() + .get(worker_ref) + .cloned() + .ok_or_else(DeterministicWorkerExecution::unavailable)?; + let mut execution_events = execution.events.subscribe(); + let snapshot = execution.snapshot(); + let (event_tx, events) = tokio::sync::mpsc::channel(32); + tokio::spawn(async move { + while let Ok(event) = execution_events.recv().await { + if event_tx.send(event).await.is_err() { + break; + } + } + }); + let bound_execution = Arc::downgrade(&execution); + let method_tx = execution.method_tx.clone(); + Ok(worker_runtime::execution::WorkerProtocolTransport::new( + worker_ref.clone(), + snapshot, + events, + Arc::new(move |method| { + bound_execution + .upgrade() + .ok_or_else(DeterministicWorkerExecution::unavailable)? + .dispatch(method) + }), + Arc::new(move || { + if method_tx.is_closed() { + Err(DeterministicWorkerExecution::unavailable()) + } else { + Ok(()) + } + }), + )) } fn worker_snapshot( &self, - handle: &worker_runtime::execution::WorkerExecutionHandle, + worker_ref: &worker_runtime::identity::WorkerRef, ) -> Option { - Some(protocol::Event::Snapshot { - session: protocol::SessionSnapshot { - pending_submissions: protocol::PendingSubmissionsSnapshot::default(), - entries: Vec::new(), - }, - greeting: protocol::Greeting { - worker_name: handle.worker_ref().worker_id.to_string(), - cwd: String::new(), - provider: "deterministic-workspace-server-test".to_string(), - model: "deterministic-workspace-server-test".to_string(), - reasoning: None, - scope_summary: "test execution snapshot".to_string(), - tools: Vec::new(), - context_window: 0, - context_tokens: 0, - context_usage: None, - }, - state: protocol::WorkerStateSnapshot::initial(), - in_flight: protocol::InFlightSnapshot { - blocks: Vec::new(), - commands: Vec::new(), - compaction: None, - }, - internal_workers: Vec::new(), - }) + self.contexts + .lock() + .unwrap() + .get(worker_ref) + .map(|execution| execution.snapshot()) } fn stop_worker( &self, - _handle: &worker_runtime::execution::WorkerExecutionHandle, + worker_ref: &worker_runtime::identity::WorkerRef, ) -> worker_runtime::execution::WorkerExecutionResult { + self.stops.lock().unwrap().push(worker_ref.clone()); + if let Some(execution) = self.contexts.lock().unwrap().remove(worker_ref) { + // Drop the actual receiver even if an in-flight operation still + // briefly owns the execution. Old protocol senders stay closed. + execution.method_rx.lock().unwrap().take(); + } worker_runtime::execution::WorkerExecutionResult::accepted( worker_runtime::execution::WorkerExecutionOperation::Stop, ) @@ -42100,7 +43912,7 @@ mod tests { fn cancel_worker( &self, - _handle: &worker_runtime::execution::WorkerExecutionHandle, + _worker_ref: &worker_runtime::identity::WorkerRef, ) -> worker_runtime::execution::WorkerExecutionResult { worker_runtime::execution::WorkerExecutionResult::accepted( worker_runtime::execution::WorkerExecutionOperation::Cancel, @@ -42109,7 +43921,7 @@ mod tests { fn dispatch_input( &self, - handle: &worker_runtime::execution::WorkerExecutionHandle, + worker_ref: &worker_runtime::identity::WorkerRef, input: worker_runtime::interaction::WorkerInput, ) -> worker_runtime::execution::WorkerExecutionResult { self.input_request_ids @@ -42119,9 +43931,18 @@ mod tests { self.inputs .lock() .expect("inputs lock") - .push((handle.worker_ref().clone(), input.content.clone())); + .push((worker_ref.clone(), input.content.clone())); + let execution = self.contexts.lock().unwrap().get(worker_ref).cloned(); + let Some(execution) = execution else { + return worker_runtime::execution::WorkerExecutionResult::rejected( + worker_runtime::execution::WorkerExecutionOperation::Input, + "deterministic test backend has no live Worker", + ); + }; + let bound_execution = Arc::downgrade(&execution); + drop(execution); if let Some(hook) = self.input_hook.lock().unwrap().take() { - hook(handle.worker_ref()); + hook(worker_ref); } if let Some(message) = self.input_failure.lock().unwrap().clone() { return worker_runtime::execution::WorkerExecutionResult::errored( @@ -42129,21 +43950,16 @@ mod tests { message, ); } - let context = self - .contexts - .lock() - .unwrap() - .get(handle.worker_ref()) - .cloned() - .expect("execution context"); let submission_request_id = input.submission_request_id.clone(); let input_kind = input.kind.clone(); let content = input.content.clone(); std::thread::spawn(move || { std::thread::sleep(std::time::Duration::from_millis(25)); - let _ = context.publish_protocol_event(protocol::Event::TextDone { - text: format!("server companion echoed: {content}"), - }); + if let Some(execution) = bound_execution.upgrade() { + let _ = execution.publish_event(protocol::Event::TextDone { + text: format!("server companion echoed: {content}"), + }); + } }); if input_kind == worker_runtime::interaction::WorkerInputKind::Notify { worker_runtime::execution::WorkerExecutionResult::accepted_notification( @@ -43103,6 +44919,8 @@ mod tests { input: serde_json::json!({"value": "immutable"}), instruction: "Return a structured check result.".to_string(), profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, source_worker: None, notification_target: None, limits: crate::backend_job::BackendJobLimits::default(), @@ -43204,6 +45022,8 @@ mod tests { input: serde_json::json!({"value": "immutable"}), instruction: "Return a structured check result.".to_string(), profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, source_worker: Some(notification_target.clone()), notification_target: Some(notification_target.clone()), limits: crate::backend_job::BackendJobLimits::default(), @@ -43310,6 +45130,8 @@ mod tests { input: serde_json::json!({"value": "immutable"}), instruction: "Return a structured check result.".to_string(), profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, source_worker: None, notification_target: None, limits: crate::backend_job::BackendJobLimits::default(), @@ -43346,10 +45168,10 @@ mod tests { Some("backend_job_cleanup_removal_rejected") ); - let second = api.retry_backend_job(&request.job_id).unwrap(); - let second_worker = second.attempt.worker.clone().unwrap(); - seed_worker_session_for_cleanup(temp.path(), &second_worker); - assert_ne!(first_worker, second_worker); + assert!( + api.retry_backend_job(&request.job_id).is_err(), + "retry must not overlap an incompletely cleaned attempt" + ); let _ = set_worker_retention( api.clone(), first_worker.runtime_id.clone(), @@ -43366,6 +45188,11 @@ mod tests { BackendJobWorkerCleanupState::Completed, ) .await; + let second = api.retry_backend_job(&request.job_id).unwrap(); + let second_worker = second.attempt.worker.clone().unwrap(); + seed_worker_session_for_cleanup(temp.path(), &second_worker); + assert_ne!(first_worker, second_worker); + api.recover_backend_jobs().unwrap(); assert!(api.runtime.worker(&second_worker).is_ok()); api.accept_backend_job_result( @@ -43412,6 +45239,8 @@ mod tests { input: serde_json::json!({"value": "immutable"}), instruction: "Return a structured check result.".to_string(), profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, source_worker: None, notification_target: None, limits: crate::backend_job::BackendJobLimits::default(), @@ -43453,6 +45282,7 @@ mod tests { workspace_api: Some(test_worker_workspace_api(EMBEDDED_WORKER_RUNTIME_ID)), memory_settings: Some(test_worker_memory_settings()), subjektiv_attached: false, + backend_job: None, }, ) .unwrap(); @@ -43589,6 +45419,8 @@ mod tests { input: serde_json::json!({"value": "immutable"}), instruction: "Return a structured check result.".to_string(), profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, source_worker: None, notification_target: None, limits: crate::backend_job::BackendJobLimits::default(), @@ -43651,6 +45483,8 @@ mod tests { input: serde_json::json!({"value": "immutable"}), instruction: "Return a structured check result.".to_string(), profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, source_worker: None, notification_target: None, limits: crate::backend_job::BackendJobLimits::default(), @@ -43784,6 +45618,8 @@ mod tests { input: serde_json::json!({"value": "immutable"}), instruction: "Return a structured check result.".to_string(), profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, source_worker: None, notification_target: Some(target.clone()), limits: crate::backend_job::BackendJobLimits { @@ -43866,6 +45702,8 @@ mod tests { input: serde_json::json!({"value": "immutable"}), instruction: "Return a structured check result.".to_string(), profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, source_worker: None, notification_target: None, limits: crate::backend_job::BackendJobLimits::default(), @@ -43904,6 +45742,8 @@ mod tests { input: serde_json::json!({"value": "immutable"}), instruction: "Return a structured check result.".to_string(), profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, source_worker: None, notification_target: None, limits: crate::backend_job::BackendJobLimits::default(), @@ -43969,6 +45809,8 @@ mod tests { input: serde_json::json!({"index": index}), instruction: "Return a structured fixture result.".to_string(), profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, source_worker: None, notification_target: None, limits: crate::backend_job::BackendJobLimits { @@ -44014,6 +45856,8 @@ mod tests { input: serde_json::json!({"value": "immutable"}), instruction: "Return a structured check result.".to_string(), profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, source_worker: None, notification_target: None, limits: crate::backend_job::BackendJobLimits::default(), @@ -44057,6 +45901,8 @@ mod tests { input: serde_json::json!({"value": "immutable"}), instruction: "Return a structured check result.".to_string(), profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, source_worker: None, notification_target: None, limits: crate::backend_job::BackendJobLimits::default(), @@ -44146,6 +45992,8 @@ mod tests { input: serde_json::json!({"value": "immutable"}), instruction: "Wait for a structured result.".to_string(), profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, source_worker: None, notification_target: None, limits: crate::backend_job::BackendJobLimits { @@ -46566,6 +48414,7 @@ mod tests { .unwrap(); api.store .attach_worker_workdir(&WorkerWorkdirLinkRecord { + connection_id: String::new(), workspace_id: TEST_WORKSPACE_ID.to_string(), worker: worker.clone(), workdir_id: workdir_id.clone(), @@ -48063,6 +49912,822 @@ mod tests { assert_eq!(attempts[0].0.worker_id.to_string(), orchestrator.worker_id); } + async fn attention_test_api() -> ( + tempfile::TempDir, + WorkspaceApi, + Arc, + ) { + let dir = tempfile::tempdir().unwrap(); + init_clean_git_workspace(dir.path()); + let (api, execution) = test_api_with_recording_backend(dir.path()).await; + let mut input = ticket::NewTicket::new("Attention backlog"); + input.workflow_state = Some(TicketWorkflowState::Queued); + set_test_ticket_target(&mut input, "test-repository", "HEAD"); + let ticket = browser_ticket_backend(&api).unwrap().create(input).unwrap(); + assign_test_orchestrator(&api, &ticket.id); + (dir, api, execution) + } + + async fn start_attention_test_orchestrator( + api: &WorkspaceApi, + ) -> BrowserWorkspaceOrchestratorResponse { + scoped_start_workspace_orchestrator( + State(api.clone()), + AxumPath(ScopedWorkspacePath { + workspace_id: TEST_WORKSPACE_ID.to_string(), + }), + ) + .await + .unwrap() + .0 + } + + #[tokio::test] + async fn orchestrator_attention_parallel_start_and_idle_observation_send_once() { + let (_dir, api, execution) = attention_test_api().await; + let reentrant_api = api.clone(); + execution.before_input_returns(move |worker| { + use protocol::subscription::SubscriptionWorkerState as S; + // The callback is inside Runtime send_input. Neither initial Idle nor + // a real turn end may deadlock or issue another in-flight delivery. + maybe_dispatch_orchestrator_turn_end( + &reentrant_api, + &worker.worker_id.to_string(), + None, + S::Idle, + ); + maybe_dispatch_orchestrator_turn_end( + &reentrant_api, + &worker.worker_id.to_string(), + Some(S::Running), + S::Idle, + ); + }); + let barrier = Arc::new(std::sync::Barrier::new(8)); + let results = std::thread::scope(|scope| { + let joins = (0..8) + .map(|_| { + let api = api.clone(); + let barrier = barrier.clone(); + scope.spawn(move || { + barrier.wait(); + futures::executor::block_on(start_attention_test_orchestrator(&api)) + }) + }) + .collect::>(); + joins + .into_iter() + .map(|j| j.join().unwrap()) + .collect::>() + }); + assert_eq!( + results + .iter() + .filter(|r| r.disposition == "created") + .count(), + 1 + ); + assert_eq!( + results + .iter() + .filter(|r| r.disposition == "existing") + .count(), + 7 + ); + assert_eq!(execution.take_inputs().len(), 1); + assert_eq!(execution.take_input_request_ids().len(), 1); + assert_eq!( + start_attention_test_orchestrator(&api).await.disposition, + "existing" + ); + assert!(execution.take_inputs().is_empty()); + } + + #[tokio::test] + async fn orchestrator_attention_barrier_callers_share_unsent_reservation() { + let (_dir, api, execution) = attention_test_api().await; + let started = start_attention_test_orchestrator(&api).await; + let worker = RuntimeWorkerRef::new( + EMBEDDED_WORKER_RUNTIME_ID, + started.worker.unwrap().worker_id, + ); + let state = orchestrator_attention_state(&api, &worker); + state.lock().unwrap().accepted = None; + execution.take_inputs(); + execution.take_input_request_ids(); + let barrier = Arc::new(std::sync::Barrier::new(8)); + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + execution.before_input_returns(move |_| { + entered_tx.send(()).unwrap(); + release_rx + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + }); + std::thread::scope(|scope| { + let joins = (0..8) + .map(|_| { + let api = api.clone(); + let worker = worker.clone(); + let barrier = barrier.clone(); + scope.spawn(move || { + barrier.wait(); + dispatch_orchestrator_queue_attention(&api, &worker, false); + }) + }) + .collect::>(); + entered_rx + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + assert!(state.lock().unwrap().sending); + assert!(state.lock().unwrap().accepted.is_none()); + // The success record does not exist yet. A ninth caller must still + // return immediately rather than compare/send twice or wait on Runtime. + dispatch_orchestrator_queue_attention(&api, &worker, false); + release_tx.send(()).unwrap(); + for join in joins { + join.join().unwrap(); + } + }); + assert_eq!(execution.take_inputs().len(), 1); + assert_eq!(execution.take_input_request_ids().len(), 1); + assert!(!state.lock().unwrap().sending); + assert!(state.lock().unwrap().accepted.is_some()); + } + + #[tokio::test] + async fn orchestrator_attention_restore_rechecks_reserved_noop_before_and_after_comparison() { + for pause_before_comparison in [true, false] { + let (_dir, api, execution) = attention_test_api().await; + let started = start_attention_test_orchestrator(&api).await; + let worker = RuntimeWorkerRef::new( + EMBEDDED_WORKER_RUNTIME_ID, + started.worker.unwrap().worker_id, + ); + let state = orchestrator_attention_state(&api, &worker); + let old_id = state + .lock() + .unwrap() + .accepted + .as_ref() + .unwrap() + .request_id + .clone(); + execution.take_inputs(); + execution.take_input_request_ids(); + api.runtime + .stop_worker( + &worker, + WorkerLifecycleRequest { + reason: None, + ticket_assignment: None, + }, + ) + .unwrap(); + execution.accept_restores(); + let (entered_tx, entered_rx) = std::sync::mpsc::channel(); + let (release_tx, release_rx) = std::sync::mpsc::channel(); + std::thread::scope(|scope| { + let api_for_caller = api.clone(); + let worker_for_caller = worker.clone(); + let caller = scope.spawn(move || { + let mut first = true; + with_orchestrator_attention_reservation( + &api_for_caller, + &worker_for_caller, + false, + |state, pending| { + if first && pause_before_comparison { + entered_tx.send(()).unwrap(); + release_rx + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + } + check_orchestrator_queue_attention( + &api_for_caller, + &worker_for_caller, + state, + pending, + ); + if first && !pause_before_comparison { + // Old fingerprint matched: check returned without sending, + // but the exact same production reservation still exists. + assert_eq!( + state.lock().unwrap().accepted.as_ref().unwrap().request_id, + old_id + ); + entered_tx.send(()).unwrap(); + release_rx + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + } + first = false; + }, + ); + }); + entered_rx + .recv_timeout(std::time::Duration::from_secs(5)) + .unwrap(); + assert!(state.lock().unwrap().sending); + let restored = futures::executor::block_on(start_attention_test_orchestrator(&api)); + assert_eq!(restored.disposition, "restored"); + let deferred = state.lock().unwrap().restore_recheck; + let old_retired = state.lock().unwrap().accepted.is_none(); + let inputs_before_release = execution.take_inputs(); + release_tx.send(()).unwrap(); + caller.join().unwrap(); + assert!(deferred); + assert!(old_retired); + assert!(inputs_before_release.is_empty()); + }); + assert_eq!(execution.take_inputs().len(), 1); + assert_eq!(execution.take_input_request_ids().len(), 1); + let state = state.lock().unwrap(); + assert!(state.accepted.is_some()); + assert!(!state.sending); + assert!(!state.restore_recheck); + } + } + + #[tokio::test] + async fn orchestrator_attention_restore_keeps_pending_unknown_delivery_and_id() { + let (_dir, api, execution) = attention_test_api().await; + execution.reject_inputs("unknown acceptance response"); + let started = start_attention_test_orchestrator(&api).await; + let worker = RuntimeWorkerRef::new( + EMBEDDED_WORKER_RUNTIME_ID, + started.worker.unwrap().worker_id, + ); + let state = orchestrator_attention_state(&api, &worker); + let pending = state.lock().unwrap().pending.clone().unwrap(); + execution.take_inputs(); + execution.take_input_request_ids(); + api.runtime + .stop_worker( + &worker, + WorkerLifecycleRequest { + reason: None, + ticket_assignment: None, + }, + ) + .unwrap(); + execution.accept_restores(); + *execution.input_failure.lock().unwrap() = None; + let restored = start_attention_test_orchestrator(&api).await; + assert_eq!(restored.disposition, "restored"); + let attempts = execution.take_inputs(); + assert_eq!(attempts.len(), 1); + assert_eq!(attempts[0].1, pending.content); + assert_eq!( + execution.take_input_request_ids(), + vec![Some(pending.request_id.clone())] + ); + assert!(state.lock().unwrap().pending.is_none()); + assert_eq!( + state.lock().unwrap().accepted.as_ref().unwrap().request_id, + pending.request_id + ); + } + + #[tokio::test] + async fn orchestrator_attention_unknown_response_retry_keeps_id_and_payload() { + let (_dir, api, execution) = attention_test_api().await; + execution.reject_inputs("accepted response lost (unknown transport outcome)"); + let started = start_attention_test_orchestrator(&api).await; + let worker = RuntimeWorkerRef::new( + EMBEDDED_WORKER_RUNTIME_ID, + started.worker.unwrap().worker_id, + ); + let state = orchestrator_attention_state(&api, &worker); + assert!(state.lock().unwrap().accepted.is_none()); + let pending = state.lock().unwrap().pending.clone().unwrap(); + let attempts = execution.take_inputs(); + let ids = execution.take_input_request_ids(); + assert_eq!(attempts.len(), 2); + assert_eq!(ids, vec![Some(pending.request_id.clone()); 2]); + // Change the backlog before retry: uncertain acceptance must never reuse + // the ID for a newly rendered payload. + let mut next = ticket::NewTicket::new("Changed backlog"); + next.workflow_state = Some(TicketWorkflowState::Queued); + set_test_ticket_target(&mut next, "test-repository", "HEAD"); + let next = browser_ticket_backend(&api).unwrap().create(next).unwrap(); + assign_test_orchestrator(&api, &next.id); + *execution.input_failure.lock().unwrap() = None; + assert_eq!( + start_attention_test_orchestrator(&api).await.disposition, + "existing" + ); + let retry = execution.take_inputs(); + assert_eq!(retry.len(), 1); + assert_eq!(retry[0].1, pending.content); + assert_eq!( + execution.take_input_request_ids(), + vec![Some(pending.request_id)] + ); + assert!(state.lock().unwrap().pending.is_none()); + assert!(state.lock().unwrap().accepted.is_some()); + use protocol::subscription::SubscriptionWorkerState as S; + maybe_dispatch_orchestrator_turn_end(&api, &worker.worker_id, Some(S::Running), S::Idle); + let changed = execution.take_inputs(); + assert_eq!(changed.len(), 1); + assert!(changed[0].1.contains("Changed backlog")); + } + + #[tokio::test] + async fn orchestrator_attention_subscription_snapshots_only_establish_baselines() { + use crate::runtime_subscription::BrokerSubscriptionEvent as B; + use protocol::subscription::{ + SubscriptionEventPayload, SubscriptionSnapshot, SubscriptionWorker, + SubscriptionWorkerState as S, + }; + let (_dir, api, execution) = attention_test_api().await; + let started = start_attention_test_orchestrator(&api).await; + let worker = RuntimeWorkerRef::new( + EMBEDDED_WORKER_RUNTIME_ID, + started.worker.unwrap().worker_id, + ); + let attention = orchestrator_attention_state(&api, &worker); + attention.lock().unwrap().accepted = None; + execution.take_inputs(); + let summary = |state: S| -> SubscriptionWorker { + serde_json::from_value(serde_json::json!({ + "worker_id": worker.worker_id, "state": state, + })) + .unwrap() + }; + let snapshot = |state| B::Snapshot { + snapshot: SubscriptionSnapshot::Workers { + workers: vec![summary(state)], + }, + }; + let event = |state| B::Event { + payload: SubscriptionEventPayload::WorkerUpserted { + worker: summary(state), + }, + }; + let mut states = HashMap::new(); + assert!(observe_orchestrator_turn_end( + &api, + &mut states, + snapshot(S::Idle) + )); + assert!(observe_orchestrator_turn_end( + &api, + &mut states, + event(S::Idle) + )); + observe_orchestrator_turn_end(&api, &mut states, event(S::Running)); + // Neither snapshot itself nor an Idle event after reconnect is turn-end evidence. + observe_orchestrator_turn_end( + &api, + &mut states, + B::Disconnected { + message: "fixture reconnect".into(), + }, + ); + observe_orchestrator_turn_end(&api, &mut states, snapshot(S::Idle)); + observe_orchestrator_turn_end(&api, &mut states, event(S::Idle)); + observe_orchestrator_turn_end(&api, &mut states, snapshot(S::Running)); + observe_orchestrator_turn_end(&api, &mut states, snapshot(S::Idle)); + assert!(attention.lock().unwrap().accepted.is_none()); + assert!(execution.take_inputs().is_empty()); + observe_orchestrator_turn_end(&api, &mut states, event(S::Running)); + observe_orchestrator_turn_end(&api, &mut states, event(S::Idle)); + assert_eq!(execution.take_inputs().len(), 1); + } + + // Real Controller and canonical Session storage with isolated local config. + // A closed loopback model endpoint prevents any production-provider access; + // notifications commit at the model boundary before that request can fail. + struct AttentionWorkerFactory { + root: PathBuf, + } + + impl AttentionWorkerFactory { + async fn controller( + &self, + worker_id: String, + restore: bool, + ) -> std::result::Result + { + let name = format!("attention-{worker_id}"); + let root = self.root.join("workers").join(&worker_id); + let store = session_store::CombinedStore::new( + session_store::WorkerSessionStore::new(root.join("session")) + .map_err(|e| e.to_string())?, + session_store::WorkerAggregateStore::new(&root, &name) + .map_err(|e| e.to_string())?, + ); + let manifest = manifest::WorkerManifest::from_toml(&format!( + r#" + [worker] + name = "{name}" + pwd = "./" + [model] + scheme = "anthropic" + model_id = "attention-test" + base_url = "http://127.0.0.1:1" + auth = {{ kind = "none" }} + [engine] + max_tokens = 10 + [scope] + allow = [] + "# + )) + .map_err(|e| e.to_string())?; + let context = worker::WorkerWorkspaceContext::local_filesystem(None); + let workspace_client = context.client_handle(); + let loader = worker::PromptCatalogSource::builtins_only(); + let worker = if restore { + worker::Worker::restore_from_worker_metadata_with_context( + &name, + manifest, + store, + loader, + context, + worker::WorkerFilesystemAuthority::None, + ) + .await + } else { + worker::Worker::from_manifest_with_context( + manifest, + store, + loader, + context, + worker::WorkerFilesystemAuthority::None, + ) + .await + } + .map_err(|e| e.to_string())?; + let started = worker::PreparedWorker::new( + worker.with_notification_coalesce_delay(std::time::Duration::from_millis(10)), + worker::WorkerBootstrapLayout::RuntimeManagedRun { + run_dir: root.join(uuid::Uuid::now_v7().to_string()), + bash_output_dir: root.join("bash-output"), + }, + worker::WorkerControllerTransport::InProcess, + ) + .start() + .await + .map_err(|e| e.to_string())?; + Ok(worker_runtime::worker_backend::RuntimeWorkerController { + handle: started.handle, + shutdown: Arc::new(tokio::sync::Mutex::new(Some(started.shutdown))), + controller_task: started.controller_task, + workspace_client, + }) + } + } + + #[async_trait::async_trait] + impl worker_runtime::worker_backend::RuntimeWorkerFactory for AttentionWorkerFactory { + async fn spawn_controller( + &self, + request: worker_runtime::execution::WorkerExecutionSpawnRequest, + ) -> std::result::Result + { + self.controller(request.worker_ref.worker_id.to_string(), false) + .await + } + async fn restore_controller( + &self, + request: worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result + { + self.controller(request.worker_ref.worker_id.to_string(), true) + .await + } + } + + use worker_runtime::execution as attention_execution; + + // Fault injection occurs AFTER the real Controller durably accepted input. + struct AttentionResponseLossBackend { + inner: Arc, + lose_next: std::sync::atomic::AtomicBool, + attempts: Mutex>, + } + + impl attention_execution::WorkerExecutionBackend for AttentionResponseLossBackend { + fn backend_id(&self) -> &str { + self.inner.backend_id() + } + fn spawn_worker( + &self, + r: attention_execution::WorkerExecutionSpawnRequest, + ) -> attention_execution::WorkerExecutionSpawnResult { + self.inner.spawn_worker(r) + } + fn preflight_restore( + &self, + r: &attention_execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result<(), attention_execution::WorkerExecutionResult> { + self.inner.preflight_restore(r) + } + fn reconcile_restore( + &self, + r: attention_execution::WorkerExecutionRestoreRequest, + ) -> attention_execution::WorkerExecutionSpawnResult { + self.inner.reconcile_restore(r) + } + fn restore_worker( + &self, + r: attention_execution::WorkerExecutionRestoreRequest, + ) -> attention_execution::WorkerExecutionSpawnResult { + self.inner.restore_worker(r) + } + fn activate_restored_worker( + &self, + id: attention_execution::WorkerLifecycleOperationId, + worker_ref: &worker_runtime::identity::WorkerRef, + ) -> std::result::Result<(), String> { + self.inner.activate_restored_worker(id, worker_ref) + } + fn dispatch_input( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + input: worker_runtime::interaction::WorkerInput, + ) -> attention_execution::WorkerExecutionResult { + self.attempts + .lock() + .unwrap() + .push(input.submission_request_id.clone().unwrap()); + let result = self.inner.dispatch_input(worker_ref, input); + assert_eq!( + result.outcome, + attention_execution::WorkerExecutionOutcome::Accepted + ); + if self + .lose_next + .swap(false, std::sync::atomic::Ordering::SeqCst) + { + attention_execution::WorkerExecutionResult::errored( + attention_execution::WorkerExecutionOperation::Input, + "injected loss of accepted response", + ) + } else { + result + } + } + fn dispatch_method( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + method: protocol::Method, + ) -> attention_execution::WorkerExecutionResult { + self.inner.dispatch_method(worker_ref, method) + } + fn stop_worker_operation( + &self, + r: attention_execution::WorkerExecutionStopRequest, + ) -> attention_execution::WorkerExecutionResult { + self.inner.stop_worker_operation(r) + } + fn worker_snapshot( + &self, + worker_ref: &worker_runtime::identity::WorkerRef, + ) -> Option { + self.inner.worker_snapshot(worker_ref) + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn orchestrator_attention_real_worker_start_restore_accept_and_commit_once_each() { + real_attention_worker_start_restore(false).await; + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn orchestrator_attention_real_worker_lost_response_retry_commits_once() { + real_attention_worker_start_restore(true).await; + } + + async fn real_attention_worker_start_restore(lose_response: bool) { + use session_store::{LogEntry, Store, WorkerSessionStore}; + let dir = tempfile::tempdir().unwrap(); + init_clean_git_workspace(dir.path()); + let config = test_server_config(dir.path()); + let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()); + seed_test_registered_workspace(store.as_ref(), &config) + .await + .unwrap(); + // Real Worker Runtime adapter/controller/storage, isolated from profile + // workspace callbacks and real LLM credentials. + let execution = Arc::new( + worker_runtime::worker_backend::WorkerRuntimeExecutionBackend::new( + AttentionWorkerFactory { + root: config.embedded_runtime_store_root.clone(), + }, + ) + .unwrap(), + ); + let execution = Arc::new(AttentionResponseLossBackend { + inner: execution, + lose_next: std::sync::atomic::AtomicBool::new(lose_response), + attempts: Mutex::new(Vec::new()), + }); + let api = WorkspaceApi::new_with_execution_backend(config, store, execution.clone()) + .await + .unwrap(); + let mut input = ticket::NewTicket::new("Real notification backlog"); + input.workflow_state = Some(TicketWorkflowState::Queued); + set_test_ticket_target(&mut input, "test-repository", "HEAD"); + let ticket = browser_ticket_backend(&api).unwrap().create(input).unwrap(); + assign_test_orchestrator(&api, &ticket.id); + let started = start_attention_test_orchestrator(&api).await; + let worker = RuntimeWorkerRef::new( + EMBEDDED_WORKER_RUNTIME_ID, + started.worker.unwrap().worker_id, + ); + let session = WorkerSessionStore::open_read_only( + api.config + .embedded_runtime_store_root + .join("workers") + .join(&worker.worker_id) + .join("session"), + ) + .unwrap(); + let entries = || { + let id = session.session_id().unwrap().unwrap(); + session + .list_segments(id) + .unwrap() + .into_iter() + .flat_map(|segment| session.read_all_read_only(id, segment).unwrap()) + .collect::>() + }; + let receipts = || { + entries() + .iter() + .rev() + .find_map(|entry| match entry { + LogEntry::Extension { + domain, payload, .. + } if domain == "worker.pending_activations.v1" => { + Some(payload["notification_receipts"].as_array().unwrap().clone()) + } + LogEntry::AnnotatedSystemItem { extensions, .. } => extensions + .iter() + .find(|e| e.domain == "worker.pending_activations.v1") + .map(|e| { + e.payload["notification_receipts"] + .as_array() + .unwrap() + .clone() + }), + _ => None, + }) + .unwrap() + }; + let commits = || { + entries().iter().filter(|entry| matches!(entry, + LogEntry::AnnotatedSystemItem { entry, .. } + if matches!(&entry.item, session_store::SystemItem::Notification { message, .. } + if message.starts_with("Queued Tickets require attention:")) + )).count() + }; + async fn wait_for_commits usize>(count: F, expected: usize) { + tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + let actual = count(); + assert!(actual <= expected, "duplicate canonical notifications"); + if actual == expected { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + }) + .await + .expect("controlled Worker notification commit"); + } + assert_eq!(receipts().len(), 1); + wait_for_commits(&commits, 1).await; + assert_eq!( + execution.attempts.lock().unwrap().len(), + if lose_response { 2 } else { 1 } + ); + let first_id = orchestrator_attention_state(&api, &worker) + .lock() + .unwrap() + .accepted + .as_ref() + .unwrap() + .request_id + .clone(); + use protocol::subscription::SubscriptionWorkerState as S; + for previous in [ + None, + Some(S::Idle), + Some(S::Stopped), + Some(S::Paused), + Some(S::Running), + ] { + maybe_dispatch_orchestrator_turn_end(&api, &worker.worker_id, previous, S::Idle); + } + assert_eq!( + start_attention_test_orchestrator(&api).await.disposition, + "existing" + ); + assert_eq!(receipts().len(), 1); + api.runtime + .stop_worker( + &worker, + WorkerLifecycleRequest { + reason: None, + ticket_assignment: None, + }, + ) + .unwrap(); + let restored = start_attention_test_orchestrator(&api).await; + assert_eq!(restored.disposition, "restored"); + let second_id = orchestrator_attention_state(&api, &worker) + .lock() + .unwrap() + .accepted + .as_ref() + .unwrap() + .request_id + .clone(); + assert_ne!(first_id, second_id); + wait_for_commits(&commits, 2).await; + let attempts = execution.attempts.lock().unwrap(); + assert_eq!(attempts.len(), if lose_response { 3 } else { 2 }); + assert_eq!( + attempts.iter().filter(|id| **id == first_id).count(), + if lose_response { 2 } else { 1 } + ); + assert_eq!(attempts.iter().filter(|id| **id == second_id).count(), 1); + drop(attempts); + let receipts = receipts(); + assert_eq!(receipts.len(), 2); + assert_eq!( + receipts + .iter() + .filter(|r| r["notification_request_id"] == first_id) + .count(), + 1 + ); + assert_eq!( + receipts + .iter() + .filter(|r| r["notification_request_id"] == second_id) + .count(), + 1 + ); + // Stop only this isolated test fixture; no live dogfooding Worker is touched. + api.runtime + .stop_worker( + &worker, + WorkerLifecycleRequest { + reason: None, + ticket_assignment: None, + }, + ) + .unwrap(); + } + + #[tokio::test] + async fn orchestrator_attention_restore_and_replacement_have_first_delivery() { + let (_dir, api, execution) = attention_test_api().await; + let started = start_attention_test_orchestrator(&api).await; + let worker = RuntimeWorkerRef::new( + EMBEDDED_WORKER_RUNTIME_ID, + started.worker.unwrap().worker_id, + ); + assert_eq!(execution.take_inputs().len(), 1); + api.runtime + .stop_worker( + &worker, + WorkerLifecycleRequest { + reason: None, + ticket_assignment: None, + }, + ) + .unwrap(); + execution.accept_restores(); + let restored = start_attention_test_orchestrator(&api).await; + assert_eq!(restored.disposition, "restored"); + assert_eq!(restored.worker.unwrap().worker_id, worker.worker_id); + assert_eq!(execution.take_inputs().len(), 1); + use protocol::subscription::SubscriptionWorkerState as S; + maybe_dispatch_orchestrator_turn_end(&api, &worker.worker_id, None, S::Idle); + assert_eq!( + start_attention_test_orchestrator(&api).await.disposition, + "existing" + ); + assert!(execution.take_inputs().is_empty()); + // This is a fixture Worker, not the running dogfooding Orchestrator. + api.runtime.delete_worker(&worker).unwrap(); + api.worker_projection + .publish_ordered(|| { + api.store + .delete_worker_registry(TEST_WORKSPACE_ID, &worker) + .map(|commit| ((), commit)) + }) + .unwrap(); + let replacement = start_attention_test_orchestrator(&api).await; + assert_ne!(replacement.worker.unwrap().worker_id, worker.worker_id); + assert_eq!(execution.take_inputs().len(), 1); + } + #[tokio::test] async fn orchestrator_running_to_idle_recovers_queued_ticket_without_notification_memory() { let dir = tempfile::tempdir().unwrap(); @@ -48074,7 +50739,6 @@ mod tests { set_test_ticket_target(&mut input, "test-repository", "HEAD"); let ticket_ref = backend.create(input).unwrap(); assign_test_orchestrator(&api, &ticket_ref.id); - *api.orchestrator_attention_fingerprint.lock().unwrap() = Some(ticket_ref.id.clone()); let Json(started) = scoped_start_workspace_orchestrator( State(api.clone()), @@ -48087,41 +50751,39 @@ mod tests { assert!(started.online); let startup_inputs = execution.take_inputs(); assert_eq!(startup_inputs.len(), 1); - assert_eq!( - api.orchestrator_attention_fingerprint - .lock() - .unwrap() - .as_deref(), - Some(ticket_ref.id.as_str()) - ); - *api.orchestrator_attention_fingerprint.lock().unwrap() = None; let worker_id = started.worker.as_ref().unwrap().worker_id.clone(); - maybe_dispatch_orchestrator_turn_end( - &api, - &worker_id, - Some(protocol::subscription::SubscriptionWorkerState::Idle), - protocol::subscription::SubscriptionWorkerState::Idle, - ); - assert!( - api.orchestrator_attention_fingerprint - .lock() - .unwrap() - .is_none() - ); - maybe_dispatch_orchestrator_turn_end( - &api, - &worker_id, - Some(protocol::subscription::SubscriptionWorkerState::Running), - protocol::subscription::SubscriptionWorkerState::Idle, - ); - + let worker = RuntimeWorkerRef::new(EMBEDDED_WORKER_RUNTIME_ID, &worker_id); + let attention = orchestrator_attention_state(&api, &worker); assert_eq!( - api.orchestrator_attention_fingerprint + attention .lock() .unwrap() - .as_deref(), - Some(ticket_ref.id.as_str()) + .accepted + .as_ref() + .unwrap() + .fingerprint, + ticket_ref.id ); + attention.lock().unwrap().accepted = None; + use protocol::subscription::SubscriptionWorkerState as S; + // Snapshot/reconnection and non-turn transitions cannot stand in for startup. + for previous in [None, Some(S::Idle), Some(S::Stopped), Some(S::Paused)] { + maybe_dispatch_orchestrator_turn_end(&api, &worker_id, previous, S::Idle); + assert!(attention.lock().unwrap().accepted.is_none()); + assert!(execution.take_inputs().is_empty()); + } + maybe_dispatch_orchestrator_turn_end(&api, &worker_id, Some(S::Running), S::Idle); + assert_eq!( + attention + .lock() + .unwrap() + .accepted + .as_ref() + .unwrap() + .fingerprint, + ticket_ref.id + ); + maybe_dispatch_orchestrator_turn_end(&api, &worker_id, Some(S::Running), S::Idle); let notifications = execution.take_inputs(); assert_eq!(notifications.len(), 1); assert_eq!(notifications[0].0.worker_id.to_string(), worker_id); @@ -49298,7 +51960,27 @@ mod tests { #[tokio::test] async fn embedded_worker_remove_executes_retention_and_returns_bounded_result() { let temp = tempfile::tempdir().unwrap(); - let api = test_api(temp.path()).await; + let config = test_server_config(temp.path()); + let store = Arc::new(test_control_store(&config)); + seed_test_registered_workspace(store.as_ref(), &config) + .await + .unwrap(); + let dispatcher = Arc::new( + crate::worker_source::EmbeddedServerWorkerMutationDispatcher::new( + config.clone(), + store.clone(), + ), + ); + // Use the same constructor-installed executor as the production embedded factory. + let api = WorkspaceApi::new_with_execution_backend_and_broker( + config, + store, + Arc::new(DeterministicExecutionBackend::default()), + BackendResourceBroker::default(), + Some(dispatcher.clone()), + ) + .await + .unwrap(); let Json(orchestrator) = scoped_start_workspace_orchestrator( State(api.clone()), AxumPath(ScopedWorkspacePath { @@ -49339,6 +52021,27 @@ mod tests { ) .unwrap(); let target = spawned.worker.unwrap().worker; + seed_worker_control_grant(&api, &source, &target, "embedded-valid-proof"); + let target_key = api + .store + .worker_resource_key(TEST_WORKSPACE_ID, &target) + .unwrap() + .unwrap(); + let forwarder = worker_runtime::worker_source::RuntimeWorkerMutationForwarder::embedded( + &source.runtime_id, + worker_runtime::RuntimeWorkspaceScope::new(TEST_WORKSPACE_ID, "embedded-backend"), + &source.worker_id, + dispatcher, + ); + let running = forwarder + .execute_worker_remove( + &target.runtime_id, + &target_key, + "must retain running Worker", + ) + .unwrap(); + assert_eq!(running.status, StatusCode::CONFLICT.as_u16()); + assert!(running.body.contains("worker_not_stopped")); let stopped = api .runtime .stop_worker( @@ -49371,25 +52074,30 @@ mod tests { .unwrap(); let summary = api.runtime.worker(&target).unwrap(); sync_worker_observation(&api, &summary).unwrap(); - seed_worker_control_grant(&api, &source, &target, "embedded-valid-proof"); + let mut subscriber = api.worker_projection.subscribe_ordered(10).unwrap(); let _ = subscriber.take_snapshot(); - let response = WorkerRemovalService::new(&api) - .execute_async( - crate::worker_source::VerifiedWorkerMutationSource { - runtime_id: source.runtime_id, - worker_id: source.worker_id, - actor_kind: worker_runtime::auth::WorkerMutationActorKind::Worker, - permission: worker_runtime::auth::WORKER_REMOVE_PERMISSION.to_string(), - jti: "embedded-valid-proof".to_string(), - }, - &target.runtime_id, - &target.worker_id, - "retire completed Worker", - ) + let handle = WorkspaceWorker::resolve(&api, &target.runtime_id, &target_key).unwrap(); + handle + .set_pinned(&WorkerOperationContext::Backend, true) .await .unwrap(); + let pinned = forwarder + .execute_worker_remove(&target.runtime_id, &target_key, "must retain pinned Worker") + .unwrap(); + assert_eq!(pinned.status, StatusCode::CONFLICT.as_u16()); + assert!(pinned.body.contains("worker_removal_blocked")); + handle + .set_pinned(&WorkerOperationContext::Backend, false) + .await + .unwrap(); + + // A resource key must be normalized by common reception after its exact + // target-bound proof is verified; the old raw service executor cannot do this. + let response = forwarder + .execute_worker_remove(&target.runtime_id, &target_key, "retire completed Worker") + .unwrap(); assert_eq!( response.status, StatusCode::OK.as_u16(), @@ -50104,6 +52812,7 @@ mod tests { let runtime_worker_id = runtime_worker_id.parse::().unwrap(); api.store .attach_worker_workdir(&WorkerWorkdirLinkRecord { + connection_id: String::new(), workspace_id: api.config.workspace_id.clone(), worker: RuntimeWorkerRef::new("runtime-test", runtime_worker_id.to_string()), workdir_id: workdir_id.to_string(), @@ -50526,6 +53235,260 @@ mod tests { ); } + #[tokio::test] + async fn current_worker_workdir_catalog_enumerates_complete_stable_inventory() { + let mut fixture = manual_coder_assignment_fixture().await; + let identity = + worker_runtime::auth::RuntimeIdentityMaterial::generate(&fixture.worker.runtime_id) + .unwrap(); + configure_runtime_request_auth(&mut fixture.api, &identity, &fixture.worker.runtime_id); + let api = &fixture.api; + let template = api + .store + .list_workdir_registry(TEST_WORKSPACE_ID, 10) + .unwrap() + .remove(0); + let mut expected = api + .store + .list_workdir_registry(TEST_WORKSPACE_ID, 10) + .unwrap() + .into_iter() + .map(|record| record.workdir_id) + .collect::>(); + let mut oldest = None; + for index in 0..233 { + let legacy = index < 3; + let mut record = WorkdirRegistryRecord { + workdir_id: format!("000-catalog-{index:03}"), + display_name: Some(format!("Catalog {index}")), + // Timestamp order intentionally differs from stable ID order. + updated_at: format!("{index:05}"), + ..template.clone() + }; + if legacy || index % 2 == 1 { + let grant = ExternalWorkdirGrantRecord { + grant_id: format!("catalog-grant-{index}"), + workspace_id: TEST_WORKSPACE_ID.to_string(), + workdir_id: record.workdir_id.clone(), + provider_instance_id: "private-provider-instance".to_string(), + display_name: record.display_name.clone().unwrap(), + permissions: if legacy { "command_only" } else { "read_write" }.to_string(), + created_by: "test-account".to_string(), + created_at: "1".to_string(), + expires_at: None, + generation: 1, + status: "online".to_string(), + updated_at: "1".to_string(), + }; + record.source = WorkdirRegistrySource::ExternalGrant { + grant_id: grant.grant_id.clone(), + }; + api.store + .create_external_workdir_grant(&grant, &record) + .unwrap(); + } else { + api.store.upsert_workdir_registry(&record).unwrap(); + } + if !legacy { + expected.insert(record.workdir_id.clone()); + if oldest.is_none() { + oldest = Some(record.clone()); + } + } + } + let base = format!("/api/w/{TEST_WORKSPACE_ID}/workers/self/workdir-catalog"); + let request = |path: &str| { + runtime_source_request( + &identity, + Some(&fixture.worker.worker_id), + "GET", + path, + Vec::new(), + ) + }; + let response = build_router(api.clone()) + .oneshot(request(&format!("{base}?limit=3"))) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let empty: server_api::CurrentWorkerWorkdirCatalogResponse = + serde_json::from_slice(&to_bytes(response.into_body(), usize::MAX).await.unwrap()) + .unwrap(); + assert!(empty.items.is_empty()); + assert_eq!(empty.next_cursor.as_deref(), Some("000-catalog-002")); + let mut cursor = None; + let mut seen = Vec::new(); + let mut pages = 0; + loop { + let path = match &cursor { + Some(cursor) => format!("{base}?limit=37&cursor={cursor}"), + None => format!("{base}?limit=37"), + }; + let response = build_router(api.clone()) + .oneshot(request(&path)) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + assert!(!String::from_utf8_lossy(&bytes).contains("private-provider-instance")); + let page: server_api::CurrentWorkerWorkdirCatalogResponse = + serde_json::from_slice(&bytes).unwrap(); + assert!(page.items.len() <= 37); + assert_eq!(page.revision, empty.revision); + seen.extend(page.items.into_iter().map(|item| item.working_directory_id)); + pages += 1; + if let Some(next) = page.next_cursor { + assert!(cursor.as_ref().is_none_or(|previous| previous < &next)); + cursor = Some(next); + } else { + break; + } + assert!(pages < 20); + } + assert!(seen.windows(2).all(|pair| pair[0] < pair[1])); + assert_eq!(seen.into_iter().collect::>(), expected); + assert!(pages > 6); + let oldest = oldest.unwrap(); + // Existing direct lookup agrees for the oldest visible External Workdir. + let Json(detail) = scoped_working_directory_detail( + State(api.clone()), + AxumPath(ScopedWorkingDirectoryPath { + workspace_id: TEST_WORKSPACE_ID.to_string(), + working_directory_id: oldest.workdir_id.clone(), + }), + ) + .await + .unwrap(); + let oldest_page = list_current_worker_workdir_catalog( + api, + server_api::CurrentWorkerWorkdirCatalogQuery { + limit: Some(1), + cursor: Some("000-catalog-002".to_string()), + }, + ) + .unwrap(); + assert_eq!(oldest_page.items, vec![detail.item]); + assert!( + !api.store + .list_workdir_registry(TEST_WORKSPACE_ID, 200) + .unwrap() + .iter() + .any(|record| record.workdir_id == oldest.workdir_id) + ); + for query in ["limit=0", "limit=101", "limit=-1", "cursor=", "cursor=%0A"] { + assert_eq!( + build_router(api.clone()) + .oneshot(request(&format!("{base}?{query}"))) + .await + .unwrap() + .status(), + StatusCode::BAD_REQUEST + ); + } + let default = list_current_worker_workdir_catalog(api, Default::default()).unwrap(); + assert_eq!(default.items.len(), 47); + let before = list_current_worker_workdir_catalog( + api, + server_api::CurrentWorkerWorkdirCatalogQuery { + limit: Some(3), + cursor: None, + }, + ) + .unwrap(); + let mut last = api + .store + .get_workdir_registry(TEST_WORKSPACE_ID, "000-catalog-232") + .unwrap() + .unwrap(); + last.display_name = Some("Updated outside the first page".to_string()); + api.store.upsert_workdir_registry(&last).unwrap(); + let changed = list_current_worker_workdir_catalog( + api, + server_api::CurrentWorkerWorkdirCatalogQuery { + limit: Some(3), + cursor: None, + }, + ) + .unwrap(); + assert_eq!(before.items, changed.items); + assert_ne!(before.revision, changed.revision); + let template_link = api + .store + .list_worker_workdir_links(TEST_WORKSPACE_ID, &fixture.worker) + .unwrap() + .remove(0); + let link = api + .store + .attach_worker_workdir(&WorkerWorkdirLinkRecord { + workdir_id: last.workdir_id.clone(), + alias: "catalog-occupancy".to_string(), + ..template_link + }) + .unwrap(); + let occupied = list_current_worker_workdir_catalog( + api, + server_api::CurrentWorkerWorkdirCatalogQuery { + limit: Some(3), + cursor: None, + }, + ) + .unwrap(); + assert_eq!(changed.items, occupied.items); + assert_ne!(changed.revision, occupied.revision); + api.store + .detach_worker_workdir_connection( + TEST_WORKSPACE_ID, + &fixture.worker, + &link.alias, + &link.connection_id, + "detach", + ) + .unwrap(); + assert_eq!( + changed.revision, + list_current_worker_workdir_catalog(api, Default::default()) + .unwrap() + .revision + ); + let restarted = SqliteWorkspaceStore::open(&api.config.database_path).unwrap(); + assert_eq!( + changed.revision, + restarted + .workdir_catalog_page(TEST_WORKSPACE_ID, 100, None) + .unwrap() + .revision + ); + let no_worker = runtime_source_request(&identity, None, "GET", &base, Vec::new()); + assert_eq!( + build_router(api.clone()) + .oneshot(no_worker) + .await + .unwrap() + .status(), + StatusCode::FORBIDDEN + ); + let mut forged = request(&base); + forged + .headers_mut() + .insert("x-yoi-worker-id", "foreign-worker".parse().unwrap()); + assert_eq!( + build_router(api.clone()) + .oneshot(forged) + .await + .unwrap() + .status(), + StatusCode::FORBIDDEN + ); + assert_eq!( + build_router(api.clone()) + .oneshot(Request::builder().uri(&base).body(Body::empty()).unwrap()) + .await + .unwrap() + .status(), + StatusCode::UNAUTHORIZED + ); + } + #[tokio::test] async fn current_worker_workdir_routes_require_a_live_runtime_worker_identity() { let workspace = tempfile::tempdir().unwrap(); @@ -50546,7 +53509,571 @@ mod tests { ) .await .unwrap(); - assert_eq!(response.status(), StatusCode::BAD_REQUEST); + assert_eq!(response.status(), StatusCode::FORBIDDEN); + } + + #[tokio::test] + async fn current_worker_attachment_list_and_stale_detach_use_signed_identity() { + let mut fixture = manual_coder_assignment_fixture().await; + let identity = + worker_runtime::auth::RuntimeIdentityMaterial::generate(&fixture.worker.runtime_id) + .unwrap(); + configure_runtime_request_auth(&mut fixture.api, &identity, &fixture.worker.runtime_id); + let api = &fixture.api; + let base = format!("/api/w/{TEST_WORKSPACE_ID}/workers/self/workdir-attachments"); + let request = |method: &str, path: &str| { + runtime_source_request( + &identity, + Some(&fixture.worker.worker_id), + method, + path, + Vec::new(), + ) + }; + let response = build_router(api.clone()) + .oneshot(request("GET", &format!("{base}?limit=1"))) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let page: server_api::CurrentWorkerWorkdirAttachmentListResponse = + serde_json::from_slice(&to_bytes(response.into_body(), usize::MAX).await.unwrap()) + .unwrap(); + assert_eq!(page.workspace_id, TEST_WORKSPACE_ID); + assert_eq!(page.items.len(), 1); + assert_eq!(page.items[0].alias, "checkout"); + assert_eq!(page.next_offset, Some(1)); + let old = api + .store + .list_worker_workdir_links(TEST_WORKSPACE_ID, &fixture.worker) + .unwrap() + .into_iter() + .find(|link| link.alias == "checkout") + .unwrap(); + assert_eq!(page.items[0].connection_id, old.connection_id); + let response = build_router(api.clone()) + .oneshot(request("GET", &format!("{base}?limit=1&offset=1"))) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let page: serde_json::Value = + serde_json::from_slice(&to_bytes(response.into_body(), usize::MAX).await.unwrap()) + .unwrap(); + assert_eq!(page["items"][0]["alias"], "docs"); + assert!(page["next_offset"].is_null()); + assert_eq!( + page["items"][0].as_object().unwrap().len(), + 4, + "no paths, credentials or materializer data" + ); + for query in ["limit=0", "limit=101", "offset=-1"] { + let response = build_router(api.clone()) + .oneshot(request("GET", &format!("{base}?{query}"))) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + } + let mut forged = request("GET", &base); + forged + .headers_mut() + .insert("x-yoi-worker-id", "foreign-worker".parse().unwrap()); + assert_eq!( + build_router(api.clone()) + .oneshot(forged) + .await + .unwrap() + .status(), + StatusCode::FORBIDDEN + ); + let no_worker = runtime_source_request(&identity, None, "GET", &base, Vec::new()); + assert_eq!( + build_router(api.clone()) + .oneshot(no_worker) + .await + .unwrap() + .status(), + StatusCode::FORBIDDEN + ); + + api.store + .detach_worker_workdir_connection( + TEST_WORKSPACE_ID, + &fixture.worker, + "checkout", + &old.connection_id, + "first-detach", + ) + .unwrap(); + let replacement = api.store.attach_worker_workdir(&old).unwrap(); + assert_ne!(replacement.connection_id, old.connection_id); + // Leave one attachment so successful detach does not need a fixture provider session. + api.store + .detach_worker_workdir( + TEST_WORKSPACE_ID, + &fixture.worker, + Some(&fixture.docs_workdir_id), + "remove-docs", + ) + .unwrap(); + let root = tempfile::tempdir().unwrap(); + let session: WorkdirSessionHandle = Arc::new(workdir::LocalWorkdirSession::new( + manifest::Scope::writable(root.path()).unwrap(), + root.path().to_path_buf(), + )); + let provider_handle = session + .start_command(workdir::CommandRequest { + command: "sleep 30".to_string(), + timeout_secs: 60, + output_limit: 4096, + cwd: workdir::WorkdirPath::root(), + spill_dir: None, + tool_call_id: None, + }) + .await + .unwrap(); + let external = api.workdir_sessions.lock().unwrap().register_command( + fixture.worker.clone(), + "checkout".to_string(), + session.clone(), + provider_handle.clone(), + ); + let stale_path = format!( + "{base}/checkout?expected_connection_id={}", + old.connection_id + ); + let response = build_router(api.clone()) + .oneshot(request("DELETE", &stale_path)) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::CONFLICT); + assert!( + api.workdir_sessions + .lock() + .unwrap() + .command(&fixture.worker, "checkout", &external) + .is_some() + ); + assert_eq!( + session + .command_status(provider_handle.clone()) + .await + .unwrap(), + workdir::CommandStatus::Running + ); + assert_eq!( + api.store + .list_worker_workdir_links(TEST_WORKSPACE_ID, &fixture.worker) + .unwrap(), + vec![replacement.clone()] + ); + + // The same lock excludes command/SubWorker session activity before checking identity. + let lock = current_worker_session_lock(api, &fixture.worker); + let guard = lock.lock().await; + let mut detach = Box::pin(detach_current_worker_workdir( + api, + &fixture.worker, + "checkout", + Some(&old.connection_id), + )); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(20), &mut detach) + .await + .is_err() + ); + assert_eq!( + session + .command_status(provider_handle.clone()) + .await + .unwrap(), + workdir::CommandStatus::Running + ); + drop(guard); + assert!(matches!( + detach.await.unwrap_err().error, + Error::WorkdirAttachmentConflict(_) + )); + + let current_path = format!( + "{base}/checkout?expected_connection_id={}", + replacement.connection_id + ); + let response = build_router(api.clone()) + .oneshot(request("DELETE", ¤t_path)) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let response: server_api::CurrentWorkerWorkdirAttachmentResponse = + serde_json::from_slice(&to_bytes(response.into_body(), usize::MAX).await.unwrap()) + .unwrap(); + assert!(!response.attached); + assert!( + session.command_status(provider_handle).await.is_err(), + "matching detach cancels and closes command authority" + ); + assert!( + api.workdir_sessions + .lock() + .unwrap() + .command(&fixture.worker, "checkout", &external) + .is_none() + ); + assert!( + api.store + .list_worker_workdir_links(TEST_WORKSPACE_ID, &fixture.worker) + .unwrap() + .is_empty() + ); + let new_lifetime = api.store.attach_worker_workdir(&old).unwrap(); + assert_ne!(new_lifetime.connection_id, replacement.connection_id); + let response = build_router(api.clone()) + .oneshot(request("DELETE", &format!("{base}/checkout"))) + .await + .unwrap(); + assert_eq!( + response.status(), + StatusCode::OK, + "normal Tool's query-free detach stays unconditional" + ); + } + + #[tokio::test] + async fn current_worker_attachment_connection_lookup_precedes_paging_and_is_scoped() { + let mut fixture = manual_coder_assignment_fixture().await; + let identity = + worker_runtime::auth::RuntimeIdentityMaterial::generate(&fixture.worker.runtime_id) + .unwrap(); + configure_runtime_request_auth(&mut fixture.api, &identity, &fixture.worker.runtime_id); + let api = &fixture.api; + let template = api + .store + .list_worker_workdir_links(TEST_WORKSPACE_ID, &fixture.worker) + .unwrap() + .remove(0); + let workdir = api + .store + .get_workdir_registry(TEST_WORKSPACE_ID, &template.workdir_id) + .unwrap() + .unwrap(); + let mut target = None; + for index in 0..50 { + let workdir = WorkdirRegistryRecord { + workdir_id: format!("lookup-workdir-{index:03}"), + ..workdir.clone() + }; + api.store.upsert_workdir_registry(&workdir).unwrap(); + target = Some( + api.store + .attach_worker_workdir(&WorkerWorkdirLinkRecord { + workdir_id: workdir.workdir_id, + alias: format!("zz-lookup-{index:03}"), + ..template.clone() + }) + .unwrap(), + ); + } + let target = target.unwrap(); + let base = format!("/api/w/{TEST_WORKSPACE_ID}/workers/self/workdir-attachments"); + let request = |path: &str| { + runtime_source_request( + &identity, + Some(&fixture.worker.worker_id), + "GET", + path, + Vec::new(), + ) + }; + let response = build_router(api.clone()) + .oneshot(request(&base)) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let first: server_api::CurrentWorkerWorkdirAttachmentListResponse = + serde_json::from_slice(&to_bytes(response.into_body(), usize::MAX).await.unwrap()) + .unwrap(); + assert_eq!(first.items.len(), 50); + assert_eq!(first.next_offset, Some(50)); + assert!( + !first + .items + .iter() + .any(|item| item.connection_id == target.connection_id) + ); + let lookup_path = format!("{base}?limit=1&connection_id={}", target.connection_id); + let response = build_router(api.clone()) + .oneshot(request(&lookup_path)) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let page: server_api::CurrentWorkerWorkdirAttachmentListResponse = + serde_json::from_slice(&to_bytes(response.into_body(), usize::MAX).await.unwrap()) + .unwrap(); + assert_eq!(page.items.len(), 1); + assert_eq!(page.items[0].alias, target.alias); + assert_eq!(page.items[0].working_directory_id, target.workdir_id); + assert_eq!(page.items[0].connection_id, target.connection_id); + assert_eq!(page.next_offset, None); + assert_eq!( + page.revision, first.revision, + "lookup does not narrow the collection revision" + ); + + let foreign_worker_id = seed_cleanup_worker(api, 902, "normal"); + seed_cleanup_workdir(api, "foreign-lookup-workdir", "present", "clean"); + seed_cleanup_link(api, &foreign_worker_id, "foreign-lookup-workdir"); + let foreign = api + .store + .list_worker_workdir_links( + TEST_WORKSPACE_ID, + &RuntimeWorkerRef::new("runtime-test", &foreign_worker_id), + ) + .unwrap() + .remove(0); + for query in [ + format!("limit=1&connection_id={}&offset=1", target.connection_id), + format!("limit=1&connection_id={}", foreign.connection_id), + "limit=1&connection_id=nonexistent".to_string(), + format!("limit=1&connection_id={}", "x".repeat(128)), + ] { + let response = build_router(api.clone()) + .oneshot(request(&format!("{base}?{query}"))) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let page: server_api::CurrentWorkerWorkdirAttachmentListResponse = + serde_json::from_slice(&to_bytes(response.into_body(), usize::MAX).await.unwrap()) + .unwrap(); + assert!(page.items.is_empty()); + assert_eq!(page.next_offset, None); + assert_eq!(page.revision, first.revision); + } + let mut forged = request(&lookup_path); + forged + .headers_mut() + .insert("x-yoi-worker-id", foreign_worker_id.parse().unwrap()); + assert_eq!( + build_router(api.clone()) + .oneshot(forged) + .await + .unwrap() + .status(), + StatusCode::FORBIDDEN + ); + for invalid in [ + String::new(), + "%20".to_string(), + "%0A".to_string(), + "x".repeat(129), + ] { + let response = build_router(api.clone()) + .oneshot(request(&format!("{base}?limit=1&connection_id={invalid}"))) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::BAD_REQUEST); + } + + let unchanged = + list_current_worker_workdir_attachments(api, &fixture.worker, Default::default()) + .unwrap(); + assert_eq!( + unchanged, first, + "another caller's connection cannot change this revision" + ); + api.store + .detach_worker_workdir_connection( + TEST_WORKSPACE_ID, + &fixture.worker, + &target.alias, + &target.connection_id, + "lookup-detach", + ) + .unwrap(); + let replacement = api.store.attach_worker_workdir(&target).unwrap(); + assert_ne!(replacement.connection_id, target.connection_id); + let reattached = + list_current_worker_workdir_attachments(api, &fixture.worker, Default::default()) + .unwrap(); + assert_eq!(reattached.items, first.items); + assert_eq!(reattached.next_offset, first.next_offset); + assert_ne!( + reattached.revision, first.revision, + "same alias and Workdir, new lifetime beyond page zero" + ); + let downgraded = api + .store + .attach_worker_workdir(&WorkerWorkdirLinkRecord { + capabilities: workdir::WorkdirSessionCapabilities::READ_ONLY, + ..replacement.clone() + }) + .unwrap(); + assert_eq!(downgraded.connection_id, replacement.connection_id); + let capability_changed = + list_current_worker_workdir_attachments(api, &fixture.worker, Default::default()) + .unwrap(); + assert_eq!(capability_changed.items, reattached.items); + assert_ne!(capability_changed.revision, reattached.revision); + let restarted = SqliteWorkspaceStore::open(&api.config.database_path).unwrap(); + let (restarted_page, restarted_revision) = restarted + .list_worker_workdir_links_page_with_revision( + TEST_WORKSPACE_ID, + &fixture.worker, + 51, + 0, + None, + ) + .unwrap(); + assert_eq!(restarted_page.len(), 51); + assert_eq!(restarted_revision, capability_changed.revision); + // A SQLite backup restores the same authoritative content/revision, without a sync store. + let source = rusqlite::Connection::open(&api.config.database_path).unwrap(); + let mut snapshot = rusqlite::Connection::open_in_memory().unwrap(); + rusqlite::backup::Backup::new(&source, &mut snapshot) + .unwrap() + .run_to_completion(128, std::time::Duration::from_millis(1), None) + .unwrap(); + let restored = SqliteWorkspaceStore::from_connection(snapshot).unwrap(); + assert_eq!( + restored + .list_worker_workdir_links_page_with_revision( + TEST_WORKSPACE_ID, + &fixture.worker, + 51, + 0, + None + ) + .unwrap(), + (restarted_page, restarted_revision) + ); + // Independent SQLite connections race a last-row capability update against exact lookup. + // The page and complete revision must always describe the same snapshot, not two reads. + let mut read_write_links = api + .store + .list_worker_workdir_links(TEST_WORKSPACE_ID, &fixture.worker) + .unwrap(); + read_write_links + .iter_mut() + .find(|link| link.connection_id == replacement.connection_id) + .unwrap() + .capabilities = workdir::WorkdirSessionCapabilities::READ_WRITE; + let read_only_links = api + .store + .list_worker_workdir_links(TEST_WORKSPACE_ID, &fixture.worker) + .unwrap(); + restarted + .replace_worker_workdir_link_capabilities( + TEST_WORKSPACE_ID, + &fixture.worker, + &read_write_links, + ) + .unwrap(); + let (_, read_write_revision) = restarted + .list_worker_workdir_links_page_with_revision( + TEST_WORKSPACE_ID, + &fixture.worker, + 1, + 0, + Some(&replacement.connection_id), + ) + .unwrap(); + assert_ne!(read_write_revision, capability_changed.revision); + let reader = SqliteWorkspaceStore::open(&api.config.database_path).unwrap(); + let barrier = Arc::new(std::sync::Barrier::new(2)); + std::thread::scope(|scope| { + let worker = &fixture.worker; + let writer_barrier = barrier.clone(); + scope.spawn(move || { + writer_barrier.wait(); + for index in 0..64 { + restarted + .replace_worker_workdir_link_capabilities( + TEST_WORKSPACE_ID, + worker, + if index % 2 == 0 { + &read_write_links + } else { + &read_only_links + }, + ) + .unwrap(); + } + }); + barrier.wait(); + for _ in 0..100 { + let (page, revision) = reader + .list_worker_workdir_links_page_with_revision( + TEST_WORKSPACE_ID, + worker, + 1, + 0, + Some(&replacement.connection_id), + ) + .unwrap(); + assert_eq!(page.len(), 1); + if revision == read_write_revision { + assert_eq!( + page[0].capabilities, + workdir::WorkdirSessionCapabilities::READ_WRITE + ); + } else { + assert_eq!(revision, capability_changed.revision); + assert_eq!( + page[0].capabilities, + workdir::WorkdirSessionCapabilities::READ_ONLY + ); + } + } + }); + let response = build_router(api.clone()) + .oneshot(request(&lookup_path)) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let stale: server_api::CurrentWorkerWorkdirAttachmentListResponse = + serde_json::from_slice(&to_bytes(response.into_body(), usize::MAX).await.unwrap()) + .unwrap(); + assert!(stale.items.is_empty()); + assert_eq!(stale.next_offset, None); + let response = build_router(api.clone()) + .oneshot(request(&format!( + "{base}?limit=1&connection_id={}", + replacement.connection_id, + ))) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let current: server_api::CurrentWorkerWorkdirAttachmentListResponse = + serde_json::from_slice(&to_bytes(response.into_body(), usize::MAX).await.unwrap()) + .unwrap(); + assert_eq!(current.items.len(), 1); + assert_eq!(current.items[0].connection_id, replacement.connection_id); + assert_eq!(current.next_offset, None); + } + + #[tokio::test] + async fn current_worker_attachment_failed_detach_preserves_connection_identity() { + let workspace = tempfile::tempdir().unwrap(); + init_clean_git_workspace(workspace.path()); + let api = test_api(workspace.path()).await; + let worker_id = seed_cleanup_worker(&api, 901, "normal"); + seed_cleanup_workdir(&api, "compensation-workdir", "present", "clean"); + seed_cleanup_link(&api, &worker_id, "compensation-workdir"); + let worker = RuntimeWorkerRef::new("runtime-test", worker_id); + let link = api + .store + .list_worker_workdir_links(TEST_WORKSPACE_ID, &worker) + .unwrap() + .remove(0); + // No live Runtime provider: runtime synchronization rejects the detach after mutation. + assert!( + detach_current_worker_workdir(&api, &worker, "attachment", Some(&link.connection_id)) + .await + .is_err() + ); + assert_eq!( + api.store + .list_worker_workdir_links(TEST_WORKSPACE_ID, &worker) + .unwrap(), + vec![link] + ); } #[tokio::test] @@ -52037,6 +55564,7 @@ mod tests { visibility: &str, ) -> server_api::WorkerSummary { server_api::WorkerSummary { + restore_observation_token: None, runtime_id: "runtime-test".to_string(), worker_id: worker_id.to_string(), resource_key: key.to_string(), @@ -52131,6 +55659,7 @@ mod tests { let discovered = workspace_worker_discovery_page( vec![server_api::WorkerSummary { + restore_observation_token: None, runtime_id: target.runtime_id.clone(), worker_id: target.worker_id.clone(), resource_key: "W-2".to_string(), @@ -52492,13 +56021,24 @@ mod tests { workspace_api: None, memory_settings: Some(memory_settings), subjektiv_attached: false, + backend_job: None, } } fn runtime_with_worker() -> (worker_runtime::Runtime, worker_runtime::identity::WorkerRef) { + let (runtime, worker_ref, _) = runtime_with_worker_backend(); + (runtime, worker_ref) + } + + fn runtime_with_worker_backend() -> ( + worker_runtime::Runtime, + worker_runtime::identity::WorkerRef, + Arc, + ) { + let execution = Arc::new(DeterministicExecutionBackend::default()); let runtime = worker_runtime::Runtime::with_execution_backend( worker_runtime::RuntimeOptions::default(), - Arc::new(DeterministicExecutionBackend::default()), + execution.clone(), ) .unwrap(); runtime.store_config_bundle(runtime_test_bundle()).unwrap(); @@ -52508,7 +56048,7 @@ mod tests { runtime_create_request(), ) .unwrap(); - (runtime, worker.worker_ref) + (runtime, worker.worker_ref, execution) } #[test] @@ -54508,10 +58048,11 @@ mod tests { tokio::time::sleep(std::time::Duration::from_millis(50)).await; let restored_store = test_control_store(&config); + let restored_execution = Arc::new(DeterministicExecutionBackend::default()); let restored = WorkspaceApi::new_with_execution_backend( config, Arc::new(restored_store), - Arc::new(DeterministicExecutionBackend::default()), + restored_execution.clone(), ) .await .expect("restored fs-backed api starts"); @@ -54519,10 +58060,34 @@ mod tests { .runtime .worker(&worker_ref) .expect("restored worker"); - // The durable projection keeps an observer attached, so the restored Runtime - // catalog remains controllable while its failed execution is represented in - // `worker_state` rather than by removing the catalog Worker. + // A read-only automatic restore rejection retains the active catalog Worker, + // but cannot invent a live execution state or protocol snapshot. assert_eq!(restored_worker.state, "idle"); + assert!(restored_worker.worker_state.is_none()); + let crate::observation::RuntimeObservationSource::Embedded(observation) = restored + .runtime + .observation_source(&worker_ref) + .expect("catalog Worker retains its observation source") + else { + panic!("expected embedded observation source"); + }; + let diagnostics = observation.runtime.diagnostics().unwrap(); + assert!( + diagnostics.iter().any(|diagnostic| { + diagnostic.worker_ref.as_ref() == Some(&observation.worker_ref) + && diagnostic + .message + .contains("deterministic test backend restore disabled") + }), + "rejected automatic restore must retain its concrete backend diagnostic: {diagnostics:?}" + ); + assert!(matches!( + observation + .runtime + .worker_observation_snapshot(&observation.worker_ref), + Err(worker_runtime::error::RuntimeError::WorkerExecutionUnavailable { .. }) + )); + assert!(restored_execution.contexts.lock().unwrap().is_empty()); assert!( serde_json::to_value(&restored_worker) .unwrap() @@ -54545,13 +58110,108 @@ mod tests { &worker_ref, WorkerInputRequest { kind: WorkerInputKind::User, - content: "should not be routed to corrupted handle".to_string(), + content: "reject input without live Worker".to_string(), submission_request_id: None, segments: None, }, ) .expect("stale worker input is projected as an operation result"); assert_eq!(rejected_input.state, InternalWorkerOperationState::Rejected); + assert_eq!(rejected_input.disposition, WorkerInputDisposition::Rejected); + assert!(rejected_input.runtime_run_id.is_none()); + assert!(rejected_input.diagnostics.iter().any(|diagnostic| { + diagnostic.code == "embedded_worker_execution_rejected" + && diagnostic + .message + .contains("deterministic test backend has no live Worker") + })); + assert_eq!( + restored_execution.take_inputs(), + vec![( + observation.worker_ref.clone(), + "reject input without live Worker".to_string() + )] + ); + assert!( + restored + .runtime + .worker(&worker_ref) + .unwrap() + .worker_state + .is_none() + ); + + // Exercise the ordinary guarded removal service, not direct catalog deletion. + let Json(orchestrator) = scoped_start_workspace_orchestrator( + State(restored.clone()), + AxumPath(ScopedWorkspacePath { + workspace_id: TEST_WORKSPACE_ID.to_string(), + }), + ) + .await + .unwrap(); + let source = orchestrator.worker.expect("removal controller"); + let source = RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id); + sync_worker_observation(&restored, &restored_worker).unwrap(); + seed_worker_control_grant(&restored, &source, &worker_ref, "rejected-restore-remove"); + let verified_source = || crate::worker_source::VerifiedWorkerMutationSource { + runtime_id: source.runtime_id.clone(), + worker_id: source.worker_id.clone(), + actor_kind: worker_runtime::auth::WorkerMutationActorKind::Worker, + permission: worker_runtime::auth::WORKER_REMOVE_PERMISSION.to_string(), + jti: "rejected-restore-remove".to_string(), + }; + let removal = WorkerRemovalService::new(&restored); + let before_stop = removal + .execute_async( + verified_source(), + &worker_ref.runtime_id, + &worker_ref.worker_id, + "rejected restore is not Stop", + ) + .await + .unwrap(); + assert_eq!(before_stop.status, StatusCode::CONFLICT.as_u16()); + assert!(before_stop.body.contains("worker_not_stopped")); + + let stopped = restored + .runtime + .stop_worker( + &worker_ref, + WorkerLifecycleRequest { + reason: Some("retire Worker after rejected automatic restore".to_string()), + ticket_assignment: None, + }, + ) + .unwrap(); + assert_eq!(stopped.state, InternalWorkerOperationState::Accepted); + assert_eq!( + restored_execution.stops.lock().unwrap().as_slice(), + &[observation.worker_ref.clone()] + ); + let stopped_worker = restored.runtime.worker(&worker_ref).unwrap(); + assert_eq!(stopped_worker.state, "stopped"); + sync_worker_observation(&restored, &stopped_worker).unwrap(); + seed_worker_session_for_cleanup(&dir.path().join("workspace"), &worker_ref); + let removed = removal + .execute_async( + verified_source(), + &worker_ref.runtime_id, + &worker_ref.worker_id, + "remove stopped Worker after rejected automatic restore", + ) + .await + .unwrap(); + assert_eq!(removed.status, StatusCode::OK.as_u16(), "{}", removed.body); + assert!(removed.body.contains("\"removed\":true")); + assert!(restored.runtime.worker(&worker_ref).is_err()); + assert!( + restored + .store + .get_worker_registry(TEST_WORKSPACE_ID, &worker_ref) + .unwrap() + .is_none() + ); } #[tokio::test] @@ -54692,7 +58352,7 @@ mod tests { ) .await .unwrap(); - let app = build_inner_router(api.clone()); + let app = authenticated_protocol_fixture_router(api.clone()); let runtimes = get_json(app.clone(), "/api/runtimes").await; let embedded_summary = runtimes["items"] @@ -54773,6 +58433,7 @@ mod tests { .unwrap(); api.store .attach_worker_workdir(&WorkerWorkdirLinkRecord { + connection_id: String::new(), workspace_id: TEST_WORKSPACE_ID.to_string(), worker: RuntimeWorkerRef::new("embedded-worker-runtime", &worker_id), workdir_id: workdir_id.to_string(), @@ -54801,7 +58462,10 @@ mod tests { let restored = post_json( app.clone(), &format!("/api/runtimes/embedded-worker-runtime/workers/{worker_id}/restore"), - json!({}), + json!({ + "expected_observation_token": worker["restore_observation_token"], + "request_id": "embedded-route-fixture-restore" + }), ) .await; assert_eq!(restored["result"]["state"], "accepted"); @@ -54858,10 +58522,17 @@ mod tests { .unwrap(); assert_eq!(wrong_runtime.status(), StatusCode::NOT_FOUND); + // Only the public observation generation key is exempt from this + // credential non-leak check. Its values must be Runtime UUIDv7 tokens. + for summary in [&worker, &restored["result"]["worker"]] { + let token = summary["restore_observation_token"].as_str().unwrap(); + assert_eq!(uuid::Uuid::parse_str(token).unwrap().get_version_num(), 7); + } let projected = format!( "{}{}{}{}{}{}", embedded_summary, spawned, worker, stopped_workers, restored, accepted - ); + ) + .replace("\"restore_observation_token\":", "\"restore_observation\":"); for forbidden in [ dir.path().to_string_lossy().as_ref(), "metadata.json", @@ -54880,7 +58551,7 @@ mod tests { #[tokio::test] async fn proxies_worker_protocol_ws_as_raw_events() { - let (runtime, worker_ref, endpoint) = spawn_runtime_worker().await; + let (_runtime, worker_ref, endpoint, execution) = spawn_runtime_worker().await; let source = RuntimeObservationSourceConfig { worker: RuntimeWorkerRef::new("runtime-a", "worker-a"), endpoint, @@ -54898,6 +58569,8 @@ mod tests { serde_json::to_string(&protocol::Method::ListCompletions { kind: protocol::CompletionKind::File, prefix: String::new(), + request_id: None, + context: None, }) .unwrap() .into(), @@ -54909,23 +58582,656 @@ mod tests { protocol::Event::Completions { .. } )); - runtime - .observe_worker_event( - &worker_ref, - protocol::Event::TextDelta { - text: "live".into(), - }, - ) - .unwrap(); + execution.emit_worker_event( + &worker_ref, + protocol::Event::TextDelta { + text: "live".into(), + }, + ); assert!(matches!( next_client_frame(&mut stream).await, protocol::Event::TextDelta { .. } )); } + fn embedded_protocol_test_source( + runtime: &worker_runtime::Runtime, + worker_ref: &worker_runtime::identity::WorkerRef, + ) -> crate::observation::EmbeddedRuntimeObservationSource { + crate::observation::EmbeddedRuntimeObservationSource { + worker: RuntimeWorkerRef::new("runtime-a", worker_ref.worker_id.to_string()), + runtime: runtime.clone(), + worker_ref: worker_ref.clone(), + } + } + + fn embedded_protocol_test_scope() -> worker_runtime::RuntimeWorkspaceScope { + worker_runtime::RuntimeWorkspaceScope::new("local", "local-token") + } + + fn protocol_test_completion(request_id: &str) -> protocol::Method { + protocol::Method::ListCompletions { + kind: protocol::CompletionKind::File, + prefix: String::new(), + request_id: Some(request_id.to_string()), + context: None, + } + } + + fn delayed_protocol_test_methods() -> Vec { + vec![ + protocol::Method::Submit { + submission_request_id: "delayed-submit-a".to_string(), + input: vec![protocol::Segment::Text { + content: "must not reach restored B".to_string(), + }], + }, + protocol_test_completion("delayed-completion-a"), + protocol::Method::Shutdown { + command: protocol::WorkerCommandEnvelope::new(700), + }, + ] + } + + fn assert_embedded_protocol_snapshot( + event: protocol::Event, + worker_ref: &worker_runtime::identity::WorkerRef, + ) { + let protocol::Event::Snapshot { + greeting, state, .. + } = event + else { + panic!("protocol connection must start with its attached execution snapshot"); + }; + assert_eq!(greeting.worker_name, worker_ref.worker_id.to_string()); + assert_eq!(greeting.provider, "deterministic-workspace-server-test"); + assert_eq!(state, protocol::WorkerStateSnapshot::initial()); + } + + async fn next_embedded_protocol_event( + connection: &mut WorkspaceWorkerProtocolConnection, + ) -> protocol::Event { + tokio::time::timeout(std::time::Duration::from_secs(2), connection.events.recv()) + .await + .expect("embedded protocol event timeout") + .expect("embedded protocol unexpectedly closed") + } + + async fn assert_embedded_protocol_closed(connection: &mut WorkspaceWorkerProtocolConnection) { + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while let Some(event) = connection.events.recv().await { + // A late method may be rejected before the relay closure is + // consumed. No completion, snapshot or B event is permitted. + assert!(matches!( + event, + protocol::Event::Error { .. } | protocol::Event::Shutdown + )); + } + }) + .await + .expect("old embedded protocol connection did not close"); + assert!(connection.methods.is_closed()); + } + + async fn spawn_embedded_protocol_test_ws( + source: crate::observation::EmbeddedRuntimeObservationSource, + scope: worker_runtime::RuntimeWorkspaceScope, + read_only: bool, + ) -> (TestWebSocket, tokio::task::JoinHandle<()>) { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let address = listener.local_addr().unwrap(); + let input_source = authenticated_browser_input_source(&test_browser_request_actor()); + let app = Router::new().route( + "/protocol/ws", + get(move |ws: WebSocketUpgrade| { + let source = source.clone(); + let scope = scope.clone(); + let input_source = input_source.clone(); + async move { + ws.on_upgrade(move |socket| { + worker_protocol_ws_session( + RuntimeObservationSource::Embedded(source), + socket, + input_source, + read_only, + scope, + ) + }) + } + }), + ); + let server = tokio::spawn(async move { + axum::serve(listener, app).await.unwrap(); + }); + let (socket, _) = connect_async(format!("ws://{address}/protocol/ws")) + .await + .unwrap(); + (socket, server) + } + + async fn send_protocol_test_ws_method(socket: &mut TestWebSocket, method: &protocol::Method) { + socket + .send(Message::Text( + protocol::stream::encode_method(method).unwrap().into(), + )) + .await + .unwrap(); + } + + async fn next_embedded_protocol_ws_event(socket: &mut TestWebSocket) -> protocol::Event { + tokio::time::timeout(std::time::Duration::from_secs(2), next_client_frame(socket)) + .await + .expect("embedded protocol WebSocket event timeout") + } + + async fn assert_embedded_protocol_ws_closed(socket: &mut TestWebSocket) { + tokio::time::timeout(std::time::Duration::from_secs(2), async { + loop { + match socket.next().await { + Some(Ok(Message::Text(text))) => { + let event = protocol::stream::decode_event(text.as_ref()).unwrap(); + assert!(matches!( + event, + protocol::Event::Error { .. } | protocol::Event::Shutdown + )); + } + Some(Ok(Message::Close(_))) | None => break, + other => panic!("expected terminal WebSocket frame, got {other:?}"), + } + } + }) + .await + .expect("old embedded protocol WebSocket did not close"); + } + + #[tokio::test] + async fn embedded_internal_protocol_bridge_closes_across_stop_restore_without_retargeting() { + // Each delayed method gets its own A connection: a rejected Submit + // must not hide whether completion or Shutdown can retarget B. + for delayed in delayed_protocol_test_methods() { + let (runtime, worker_ref, execution) = runtime_with_worker_backend(); + let source = embedded_protocol_test_source(&runtime, &worker_ref); + let mut old = + connect_embedded_worker_protocol(source.clone(), embedded_protocol_test_scope()) + .await + .unwrap(); + assert_embedded_protocol_snapshot( + next_embedded_protocol_event(&mut old).await, + &worker_ref, + ); + old.methods + .send(protocol_test_completion("execution-a")) + .await + .unwrap(); + assert!(matches!( + next_embedded_protocol_event(&mut old).await, + protocol::Event::Completions { request_id: Some(id), .. } if id == "execution-a" + )); + execution.emit_worker_event( + &worker_ref, + protocol::Event::TextDelta { + text: "execution-a".into(), + }, + ); + assert!(matches!( + next_embedded_protocol_event(&mut old).await, + protocol::Event::TextDelta { text } if text == "execution-a" + )); + + // No await here: the old loop has not yet observed A's closure when + // B is installed and a delayed method enters its existing queue. + runtime.stop_worker(&worker_ref, None).unwrap(); + execution.accept_restores(); + runtime + .restore_worker( + &worker_ref, + runtime_api::WorkerRestoreRequest { + expected_observation_token: runtime + .worker_detail(&worker_ref) + .unwrap() + .restore_observation_token + .unwrap(), + request_id: WorkerId::now_v7().to_string(), + preparation: Some(runtime_api::WorkerRestorePreparation::default()), + }, + ) + .unwrap(); + old.methods.try_send(delayed).unwrap(); + execution.emit_worker_event( + &worker_ref, + protocol::Event::TextDelta { + text: "execution-b".into(), + }, + ); + assert_embedded_protocol_closed(&mut old).await; + assert_eq!( + execution.protocol_methods().len(), + 1, + "delayed A method reached B" + ); + assert_eq!( + runtime.worker_detail(&worker_ref).unwrap().status, + worker_runtime::catalog::WorkerStatus::Idle + ); + assert_eq!( + execution.stops.lock().unwrap().as_slice(), + &[worker_ref.clone()] + ); + + let mut current = + connect_embedded_worker_protocol(source, embedded_protocol_test_scope()) + .await + .unwrap(); + assert_embedded_protocol_snapshot( + next_embedded_protocol_event(&mut current).await, + &worker_ref, + ); + current + .methods + .send(protocol_test_completion("execution-b")) + .await + .unwrap(); + assert!(matches!( + next_embedded_protocol_event(&mut current).await, + protocol::Event::Completions { request_id: Some(id), .. } if id == "execution-b" + )); + execution.emit_worker_event( + &worker_ref, + protocol::Event::TextDelta { + text: "fresh-b".into(), + }, + ); + assert!(matches!( + next_embedded_protocol_event(&mut current).await, + protocol::Event::TextDelta { text } if text == "fresh-b" + )); + } + } + + #[tokio::test] + async fn embedded_browser_protocol_ws_closes_across_stop_restore_without_retargeting() { + for delayed in delayed_protocol_test_methods() { + let (runtime, worker_ref, execution) = runtime_with_worker_backend(); + let source = embedded_protocol_test_source(&runtime, &worker_ref); + let (mut old, server) = spawn_embedded_protocol_test_ws( + source.clone(), + embedded_protocol_test_scope(), + false, + ) + .await; + assert_embedded_protocol_snapshot( + next_embedded_protocol_ws_event(&mut old).await, + &worker_ref, + ); + let submit = protocol::Method::Submit { + submission_request_id: "authenticated-a".to_string(), + input: vec![protocol::Segment::Text { + content: "from browser".into(), + }], + }; + send_protocol_test_ws_method(&mut old, &submit).await; + send_protocol_test_ws_method(&mut old, &protocol_test_completion("execution-a")).await; + assert!(matches!( + next_embedded_protocol_ws_event(&mut old).await, + protocol::Event::Completions { .. } + )); + assert!(matches!( + &execution.protocol_methods()[0].1, + protocol::Method::SubmitTracked { source, .. } + if source == &authenticated_browser_input_source(&test_browser_request_actor()) + )); + execution.emit_worker_event( + &worker_ref, + protocol::Event::TextDelta { + text: "execution-a".into(), + }, + ); + assert!( + matches!(next_embedded_protocol_ws_event(&mut old).await, protocol::Event::TextDelta { text } if text == "execution-a") + ); + + // Stop and restore happen before this socket loop processes closure. + runtime.stop_worker(&worker_ref, None).unwrap(); + execution.accept_restores(); + runtime + .restore_worker( + &worker_ref, + runtime_api::WorkerRestoreRequest { + expected_observation_token: runtime + .worker_detail(&worker_ref) + .unwrap() + .restore_observation_token + .unwrap(), + request_id: WorkerId::now_v7().to_string(), + preparation: Some(runtime_api::WorkerRestorePreparation::default()), + }, + ) + .unwrap(); + // A peer close may win this race; otherwise the old connected + // dispatcher must reject the frame without touching B. + let _ = old + .send(Message::Text( + protocol::stream::encode_method(&delayed).unwrap().into(), + )) + .await; + execution.emit_worker_event( + &worker_ref, + protocol::Event::TextDelta { + text: "execution-b".into(), + }, + ); + assert_embedded_protocol_ws_closed(&mut old).await; + assert_eq!( + execution.protocol_methods().len(), + 2, + "delayed A method reached B" + ); + assert_eq!( + runtime.worker_detail(&worker_ref).unwrap().status, + worker_runtime::catalog::WorkerStatus::Idle + ); + assert_eq!( + execution.stops.lock().unwrap().as_slice(), + &[worker_ref.clone()] + ); + server.abort(); + + let (mut current, current_server) = + spawn_embedded_protocol_test_ws(source, embedded_protocol_test_scope(), false) + .await; + assert_embedded_protocol_snapshot( + next_embedded_protocol_ws_event(&mut current).await, + &worker_ref, + ); + send_protocol_test_ws_method(&mut current, &protocol_test_completion("execution-b")) + .await; + assert!(matches!( + next_embedded_protocol_ws_event(&mut current).await, + protocol::Event::Completions { request_id: Some(id), .. } if id == "execution-b" + )); + execution.emit_worker_event( + &worker_ref, + protocol::Event::TextDelta { + text: "fresh-b".into(), + }, + ); + assert!(matches!( + next_embedded_protocol_ws_event(&mut current).await, + protocol::Event::TextDelta { text } if text == "fresh-b" + )); + current.close(None).await.unwrap(); + current_server.abort(); + } + } + + #[tokio::test] + async fn embedded_protocol_paths_close_on_shutdown_event_and_lagged_relay() { + for lagged in [false, true] { + let (runtime, worker_ref, execution) = runtime_with_worker_backend(); + let source = embedded_protocol_test_source(&runtime, &worker_ref); + let mut bridge = + connect_embedded_worker_protocol(source.clone(), embedded_protocol_test_scope()) + .await + .unwrap(); + assert_embedded_protocol_snapshot( + next_embedded_protocol_event(&mut bridge).await, + &worker_ref, + ); + let (mut socket, server) = + spawn_embedded_protocol_test_ws(source, embedded_protocol_test_scope(), false) + .await; + assert_embedded_protocol_snapshot( + next_embedded_protocol_ws_event(&mut socket).await, + &worker_ref, + ); + if lagged { + // Overflow the execution broadcast before either relay runs. + // A lost terminal event is uncertain, never a reason to follow + // the Runtime WorkerRef-wide observation bus instead. + for _ in 0..40 { + execution.emit_worker_event( + &worker_ref, + protocol::Event::TextDelta { + text: "overflow".into(), + }, + ); + } + } else { + execution.emit_worker_event(&worker_ref, protocol::Event::Shutdown); + } + assert_embedded_protocol_closed(&mut bridge).await; + assert_embedded_protocol_ws_closed(&mut socket).await; + server.abort(); + } + } + + #[tokio::test] + async fn embedded_protocol_paths_close_after_connected_shutdown_method() { + let shutdown = protocol::Method::Shutdown { + command: protocol::WorkerCommandEnvelope::new(701), + }; + let (runtime, worker_ref, execution) = runtime_with_worker_backend(); + let mut bridge = connect_embedded_worker_protocol( + embedded_protocol_test_source(&runtime, &worker_ref), + embedded_protocol_test_scope(), + ) + .await + .unwrap(); + assert_embedded_protocol_snapshot( + next_embedded_protocol_event(&mut bridge).await, + &worker_ref, + ); + bridge.methods.try_send(shutdown.clone()).unwrap(); + bridge + .methods + .try_send(protocol_test_completion("after-shutdown")) + .unwrap(); + assert_embedded_protocol_closed(&mut bridge).await; + assert!(matches!( + execution.protocol_methods().as_slice(), + [(_, protocol::Method::Shutdown { .. })] + )); + assert_eq!( + runtime.worker_detail(&worker_ref).unwrap().status, + worker_runtime::catalog::WorkerStatus::Stopped + ); + + let (runtime, worker_ref, execution) = runtime_with_worker_backend(); + let (mut socket, server) = spawn_embedded_protocol_test_ws( + embedded_protocol_test_source(&runtime, &worker_ref), + embedded_protocol_test_scope(), + false, + ) + .await; + assert_embedded_protocol_snapshot( + next_embedded_protocol_ws_event(&mut socket).await, + &worker_ref, + ); + send_protocol_test_ws_method(&mut socket, &shutdown).await; + let _ = socket + .send(Message::Text( + protocol::stream::encode_method(&protocol_test_completion("after-shutdown")) + .unwrap() + .into(), + )) + .await; + assert_embedded_protocol_ws_closed(&mut socket).await; + assert!(matches!( + execution.protocol_methods().as_slice(), + [(_, protocol::Method::Shutdown { .. })] + )); + assert_eq!( + runtime.worker_detail(&worker_ref).unwrap().status, + worker_runtime::catalog::WorkerStatus::Stopped + ); + server.abort(); + } + + #[tokio::test] + async fn embedded_protocol_paths_enforce_workspace_scope_and_browser_method_authority() { + let (runtime, worker_ref, execution) = runtime_with_worker_backend(); + let source = embedded_protocol_test_source(&runtime, &worker_ref); + let wrong_scope = + worker_runtime::RuntimeWorkspaceScope::new("other-workspace", "local-token"); + assert!( + connect_embedded_worker_protocol(source.clone(), wrong_scope.clone()) + .await + .is_err() + ); + let (mut socket, server) = + spawn_embedded_protocol_test_ws(source.clone(), wrong_scope, false).await; + assert!(matches!( + next_embedded_protocol_ws_event(&mut socket).await, + protocol::Event::Error { .. } + )); + assert_embedded_protocol_ws_closed(&mut socket).await; + server.abort(); + + let (mut socket, server) = + spawn_embedded_protocol_test_ws(source.clone(), embedded_protocol_test_scope(), true) + .await; + assert_embedded_protocol_snapshot( + next_embedded_protocol_ws_event(&mut socket).await, + &worker_ref, + ); + send_protocol_test_ws_method(&mut socket, &protocol_test_completion("read-only")).await; + assert!(matches!( + next_embedded_protocol_ws_event(&mut socket).await, + protocol::Event::Error { .. } + )); + socket.close(None).await.unwrap(); + server.abort(); + + let (mut socket, server) = + spawn_embedded_protocol_test_ws(source, embedded_protocol_test_scope(), false).await; + assert_embedded_protocol_snapshot( + next_embedded_protocol_ws_event(&mut socket).await, + &worker_ref, + ); + let forged = protocol::Method::SubmitTracked { + submission_request_id: "forged-source".into(), + input: Vec::new(), + source: authenticated_browser_input_source(&test_browser_request_actor()), + }; + send_protocol_test_ws_method(&mut socket, &forged).await; + assert!(matches!( + next_embedded_protocol_ws_event(&mut socket).await, + protocol::Event::Error { .. } + )); + assert_embedded_protocol_ws_closed(&mut socket).await; + assert!(execution.protocol_methods().is_empty()); + server.abort(); + } + + #[tokio::test] + async fn deterministic_protocol_transport_remains_bound_across_stop_and_restore() { + use worker_runtime::execution::WorkerExecutionBackend; + + let (runtime, worker_ref, execution) = runtime_with_worker_backend(); + let completion = |request_id: &str| protocol::Method::ListCompletions { + kind: protocol::CompletionKind::File, + prefix: String::new(), + request_id: Some(request_id.to_string()), + context: None, + }; + let mut old = execution + .clone() + .attach_worker_protocol(&worker_ref) + .unwrap(); + assert!(matches!(old.snapshot, protocol::Event::Snapshot { .. })); + old.validate().unwrap(); + assert!(matches!( + old.dispatch(completion("execution-a")).unwrap().as_slice(), + [protocol::Event::Completions { .. }] + )); + + // Runtime observations are not this execution's protocol event stream. + runtime + .observe_worker_event( + &worker_ref, + protocol::Event::TextDelta { + text: "runtime-only".to_string(), + }, + ) + .unwrap(); + execution.emit_worker_event( + &worker_ref, + protocol::Event::TextDelta { + text: "execution-a".to_string(), + }, + ); + let event = tokio::time::timeout(std::time::Duration::from_secs(2), old.events.recv()) + .await + .unwrap() + .unwrap(); + assert!(matches!(event, protocol::Event::TextDelta { text } if text == "execution-a")); + + runtime.stop_worker(&worker_ref, None).unwrap(); + assert!(old.validate().is_err()); + assert!(old.dispatch(completion("stopped-a")).is_err()); + assert!( + tokio::time::timeout(std::time::Duration::from_secs(2), old.events.recv()) + .await + .unwrap() + .is_none() + ); + + execution.accept_restores(); + runtime + .restore_worker( + &worker_ref, + runtime_api::WorkerRestoreRequest { + expected_observation_token: runtime + .worker_detail(&worker_ref) + .unwrap() + .restore_observation_token + .unwrap(), + request_id: WorkerId::now_v7().to_string(), + preparation: Some(runtime_api::WorkerRestorePreparation::default()), + }, + ) + .unwrap(); + let mut current = execution + .clone() + .attach_worker_protocol(&worker_ref) + .unwrap(); + current.validate().unwrap(); + current.dispatch(completion("execution-b")).unwrap(); + assert!(old.validate().is_err()); + assert!(old.dispatch(completion("must-not-reach-b")).is_err()); + execution.emit_worker_event( + &worker_ref, + protocol::Event::TextDelta { + text: "execution-b".to_string(), + }, + ); + let event = tokio::time::timeout(std::time::Duration::from_secs(2), current.events.recv()) + .await + .unwrap() + .unwrap(); + assert!(matches!(event, protocol::Event::TextDelta { text } if text == "execution-b")); + let delivered: Vec<_> = execution + .protocol_methods() + .into_iter() + .map(|(worker, method)| { + let protocol::Method::ListCompletions { request_id, .. } = method else { + panic!("expected completion method"); + }; + (worker, request_id.unwrap()) + }) + .collect(); + assert_eq!( + delivered, + vec![ + (worker_ref.clone(), "execution-a".to_string()), + (worker_ref, "execution-b".to_string()), + ] + ); + } + #[tokio::test] async fn proxy_maps_runtime_worker_not_found_http_404_to_protocol_error_event() { - let (_runtime, _worker_ref, endpoint) = spawn_runtime_worker().await; + let (_runtime, _worker_ref, endpoint, _execution) = spawn_runtime_worker().await; let endpoint = endpoint.replace("/protocol/ws", "/missing-worker/protocol/ws"); let source = RuntimeObservationSourceConfig { worker: RuntimeWorkerRef::new("runtime-a", "worker-a"), @@ -55020,8 +59326,9 @@ mod tests { worker_runtime::Runtime, worker_runtime::identity::WorkerRef, String, + Arc, ) { - let (runtime, worker_ref) = runtime_with_worker(); + let (runtime, worker_ref, execution) = runtime_with_worker_backend(); let runtime_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let runtime_addr = runtime_listener.local_addr().unwrap(); tokio::spawn({ @@ -55040,7 +59347,7 @@ mod tests { "ws://{runtime_addr}/v1/workers/{}/protocol/ws", worker_ref.worker_id ); - (runtime, worker_ref, endpoint) + (runtime, worker_ref, endpoint, execution) } fn test_browser_request_actor() -> RequestActor { @@ -55053,6 +59360,25 @@ mod tests { } } + // Transport fixtures below the authentication router still need real credentials + // for operation-time reauthorization. Public-router tests use real proofs/tokens. + fn authenticated_protocol_fixture_router(api: WorkspaceApi) -> Router { + let token = seed_test_api_token(api.store.as_ref(), TEST_WORKSPACE_ID); + build_inner_router(api) + .layer(Extension(test_browser_request_actor())) + .layer(axum::middleware::from_fn( + move |mut request: axum::extract::Request, next: Next| { + let value = format!("Bearer {token}"); + async move { + request + .headers_mut() + .insert(axum::http::header::AUTHORIZATION, value.parse().unwrap()); + next.run(request).await + } + }, + )) + } + async fn spawn_workspace_proxy( source: RuntimeObservationSourceConfig, ) -> (String, tempfile::TempDir) { @@ -55071,7 +59397,8 @@ mod tests { .unwrap(); let app_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let app_addr = app_listener.local_addr().unwrap(); - let app = build_inner_router(api).layer(Extension(test_browser_request_actor())); + seed_worker_source_member(&api, &runtime_id, &worker_id); + let app = authenticated_protocol_fixture_router(api); tokio::spawn(async move { axum::serve(app_listener, app).await.unwrap() }); ( format!("ws://{app_addr}/api/runtimes/{runtime_id}/workers/{worker_id}/protocol/ws"), @@ -55084,8 +59411,7 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); - let app = build_inner_router(test_api(dir.path()).await) - .layer(Extension(test_browser_request_actor())); + let app = authenticated_protocol_fixture_router(test_api(dir.path()).await); let server = tokio::spawn(async move { let _ = axum::serve(listener, app).await; }); @@ -55168,6 +59494,7 @@ mod tests { .unwrap(); api.store .attach_worker_workdir(&WorkerWorkdirLinkRecord { + connection_id: String::new(), workspace_id: TEST_WORKSPACE_ID.to_string(), worker: RuntimeWorkerRef::new(EMBEDDED_WORKER_RUNTIME_ID, &worker_id), workdir_id: workdir_id.to_string(), @@ -55181,7 +59508,7 @@ mod tests { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); let projection_api = api.clone(); - let app = build_inner_router(api).layer(Extension(test_browser_request_actor())); + let app = authenticated_protocol_fixture_router(api); let server = tokio::spawn(async move { let _ = axum::serve(listener, app).await; }); @@ -55377,6 +59704,8 @@ mod tests { method: protocol::Method::ListCompletions { kind: protocol::CompletionKind::File, prefix: String::new(), + request_id: None, + context: None, }, }, ), @@ -56041,14 +60370,9 @@ VALUES ('0192f0e8-4d84-7d6e-a000-000000000001', 'ticket', 3); let error = subjektiv_subject_scope(&api, TEST_WORKSPACE_ID, &forged_context).unwrap_err(); assert_eq!(error.into_response().status(), StatusCode::FORBIDDEN); assert!( - try_reuse_subject_consolidation_worker( - &api, - forged_key, - "process pending subject Memory", - 1, - 1, - ) - .is_err() + subjektiv_job::retire_legacy_worker(&api, "forged-subject") + .await + .is_err() ); } diff --git a/crates/workspace-server/src/server/backend_job_profile.rs b/crates/workspace-server/src/server/backend_job_profile.rs new file mode 100644 index 000000000..4c8d155dc --- /dev/null +++ b/crates/workspace-server/src/server/backend_job_profile.rs @@ -0,0 +1,101 @@ +//! Profile recipes are selected by the caller. Execution grants remain Host owned. +use super::*; + +impl WorkspaceApi { + pub(super) fn resolve_backend_job_profile( + &self, + request: &BackendJobRequest, + ) -> Result<(ProfileSelector, worker_runtime::config_bundle::ConfigBundle)> { + request.validate()?; + let state = self + .config_store + .load_workspace_config(self.workspace_id())? + .ok_or_else(|| { + Error::Config("Backend Job requires Workspace Profile configuration".into()) + })?; + let projection = crate::profile_settings::project_profiles_from_workspace_config( + self.workspace_id(), + &state, + )?; + let requested_profile = if request.profile == "default" { + projection + .settings + .default_profile + .as_deref() + .ok_or_else(|| { + Error::InvalidInput("Workspace has no configured default Job Profile".into()) + })? + } else { + request.profile.as_str() + }; + let selector = crate::profile_settings::selector_for_registered_profile( + &projection, + requested_profile, + ) + .ok_or_else(|| { + Error::InvalidInput(format!( + "Backend Job Profile `{}` is not a published registry candidate", + request.profile + )) + })?; + let profile_key = match &selector { + ProfileSelector::Builtin(key) | ProfileSelector::Named(key) => key.as_str(), + }; + let prompts = self + .prompt_projection_cache + .resolve(self.workspace_id(), &state)?; + let bundle = + crate::profile_settings::build_virtual_profile_config_bundle_with_prompt_projection( + &projection, + &state, + self.workspace_id(), + &self.config.workspace_created_at, + profile_key, + prompts.as_ref(), + )? + .ok_or_else(|| Error::Config("Backend Job Profile bundle is missing".into()))?; + let manifest = crate::profile_settings::resolve_profile_manifest_from_config_bundle( + &bundle, + profile_key, + )?; + validate_backend_job_profile_requirements(&manifest, request)?; + Ok((selector, bundle)) + } +} + +fn validate_backend_job_profile_requirements( + manifest: &manifest::WorkerManifest, + request: &BackendJobRequest, +) -> Result<()> { + let f = &manifest.feature; + // These tools require domain/parent/Workdir authority which a bounded Job + // deliberately does not inherit. Reject, rather than disabling features or + // substituting a different recipe under the caller's selector. + if f.memory.enabled() + || f.worker.enabled + || f.sub_worker.enabled + || f.flow.enabled + || f.workspace_worker_discovery.enabled + || f.objective.enabled + || f.manage_workdir.enabled + || f.workspace_config.enabled + || f.ticket.enabled + || f.merge_request.show + || f.merge_request.open + || f.merge_request.review + || f.merge_request.readiness_check + || f.merge_request.complete + || f.orchestration.enabled + || !manifest.mcp.stdio_servers.is_empty() + { + return Err(Error::WorkspacePermissionDenied("Backend Job Profile requires authority not granted to this Job (Workspace mutation, Memory, child/peer, Workdir, Flow or MCP)".into())); + } + let consolidation = request.grants.subjektiv_consolidation.is_some(); + if f.subjektiv.enabled() != consolidation + || f.subjektiv.profile.consolidation_tools != consolidation + || (consolidation && f.subjektiv.profile.extraction.enabled) + { + return Err(Error::WorkspacePermissionDenied("Backend Job Profile subjektiv requirements do not match the explicit consolidation grant".into())); + } + Ok(()) +} diff --git a/crates/workspace-server/src/server/subjektiv_job.rs b/crates/workspace-server/src/server/subjektiv_job.rs new file mode 100644 index 000000000..66317f7ca --- /dev/null +++ b/crates/workspace-server/src/server/subjektiv_job.rs @@ -0,0 +1,369 @@ +//! subjektiv is a Job client, not a second runner or Worker registry. +use super::*; +use crate::backend_job::{BackendJobGrants, SubjektivConsolidationGrant}; + +const PURPOSE: &str = "subjektiv_consolidation"; +const CANDIDATE_BATCH_LIMIT: usize = 100; + +/// Load Host authority from the authenticated Worker registry and current attempt. +/// Model input, Profile tags, display names and a subject-body lease are not grants. +pub(super) fn active_grant( + api: &WorkspaceApi, + workspace_id: &str, + worker: &RuntimeWorkerRef, +) -> ApiResult> { + let Some(binding) = api + .store + .worker_registry_projection(workspace_id, worker)? + .and_then(|p| p.job) + else { + return Ok(None); + }; + let job = api + .store + .get_backend_job(workspace_id, &binding.job_id)? + .ok_or_else(|| Error::WorkspacePermissionDenied("Job binding has no intent".into()))?; + let attempt = api + .store + .get_backend_job_attempt(workspace_id, &binding.job_id, &binding.attempt_id)? + .ok_or_else(|| Error::WorkspacePermissionDenied("Job binding has no attempt".into()))?; + if job.state != BackendJobState::Pending + || attempt.state != BackendJobAttemptState::Dispatched + || attempt.worker.as_ref() != Some(worker) + || job.current_attempt != attempt.attempt + || attempt.input_revision != job.request.input_revision + || chrono::DateTime::parse_from_rfc3339(&attempt.deadline_at) + .map_err(|e| Error::Store(format!("invalid Job deadline: {e}")))? + <= Utc::now() + { + return Err(Error::WorkspacePermissionDenied( + "subjektiv grant requires the current live Job attempt".into(), + ) + .into()); + } + Ok(job.request.grants.subjektiv_consolidation) +} + +pub(super) fn require_candidate_grant( + api: &WorkspaceApi, + workspace_id: &str, + worker: &RuntimeWorkerRef, + candidate_id: &str, +) -> ApiResult<()> { + if let Some(grant) = active_grant(api, workspace_id, worker)? { + if !grant.candidate_ids.iter().any(|id| id == candidate_id) { + return Err(Error::WorkspacePermissionDenied( + "candidate is outside this Job's immutable batch".into(), + ) + .into()); + } + } + Ok(()) // legacy owned consolidators may finish before the cutover fence +} + +/// Retire only the persisted old consolidation owner. Never interrupt a foreground +/// candidate decision. Once Idle is confirmed, stop cancels any legacy surface +/// background work; the new Job regenerates it and does not infer surface success. +pub(super) async fn retire_legacy_worker(api: &WorkspaceApi, subject_id: &str) -> ApiResult { + let key = format!("{SUBJEKTIV_CONSOLIDATION_SINGLETON_PREFIX}{subject_id}"); + let Some(owner) = api + .store + .current_worker_singleton_owner(api.workspace_id(), &key)? + else { + return Ok(true); + }; + let mut summary = api + .runtime + .worker(&owner.worker) + .map_err(|e| e.into_error())?; + summary.singleton_key = Some(owner.key.clone()); + require_dedicated_subjektiv_consolidation_worker(&summary, api.workspace_id(), &key)?; + let stopped = summary.state.eq_ignore_ascii_case("stopped"); + let idle = summary.state.eq_ignore_ascii_case("idle") + && summary + .worker_state + .as_ref() + .is_some_and(|s| s.state == protocol::WorkerState::Idle); + if !stopped && !idle { + return Ok(false); + } + if api + .store + .current_worker_singleton_owner(api.workspace_id(), &key)? + .as_ref() + .map(|o| &o.worker) + != Some(&owner.worker) + { + return Err(Error::RepositoryConflict( + "legacy consolidation owner moved during cutover".into(), + ) + .into()); + } + if !summary.state.eq_ignore_ascii_case("stopped") { + let stopped = api + .runtime + .stop_worker( + &owner.worker, + WorkerLifecycleRequest { + reason: Some( + "Retire legacy subjektiv consolidator after foreground completion".into(), + ), + ticket_assignment: None, + }, + ) + .map_err(|e| e.into_error())?; + if stopped.state != InternalWorkerOperationState::Accepted { + return Ok(false); + } + } + if !api + .runtime + .worker(&owner.worker) + .map_err(|e| e.into_error())? + .state + .eq_ignore_ascii_case("stopped") + { + return Ok(false); + } + let removal = WorkerRemovalService::new(api) + .execute_target_removal( + api.runtime.as_ref(), + &owner.worker, + "subjektiv consolidation Job cutover", + None, + ) + .await + .map_err(Error::Store)?; + if removal.status != StatusCode::OK.as_u16() { + return Err(Error::Store(format!( + "legacy consolidation cleanup rejected: {}", + removal.body + )) + .into()); + } + Ok(api + .store + .current_worker_singleton_owner(api.workspace_id(), &key)? + .is_none()) +} + +fn bounded_candidate_batch( + ids: impl IntoIterator, + max_result_bytes: u32, +) -> Result> { + // Reserve the bounded subject and Host surface envelope. Each candidate ID + // appears twice (input claims and verified outcome); account for JSON escaping, + // not raw UTF-8 bytes. The fixed per-item allowance covers action and framing. + let mut estimated_bytes = 2_048_usize; + let mut selected = Vec::new(); + for id in ids.into_iter().take(CANDIDATE_BATCH_LIMIT) { + let id_bytes = serde_json::to_vec(&id) + .map_err(|e| Error::InvalidInput(e.to_string()))? + .len(); + let additional = 2 * id_bytes + 128; + if estimated_bytes + additional > max_result_bytes as usize { + break; + } + estimated_bytes += additional; + selected.push(id); + } + Ok(selected) +} + +pub(super) fn dispatch( + api: &WorkspaceApi, + subject_id: &str, + candidate_count: usize, + total_bytes: u64, +) -> ApiResult { + let store = open_subjektiv_store(api)?; + let subject = store + .subject(subject_id) + .map_err(subjektiv_store_error)? + .ok_or_else(|| Error::SubjektivSubjectNotFound(subject_id.into()))?; + let candidate_ids = bounded_candidate_batch( + store + .pending_staging_candidates(subject_id, CANDIDATE_BATCH_LIMIT) + .map_err(subjektiv_store_error)? + .into_iter() + .map(|c| c.id), + crate::backend_job::DEFAULT_MAX_RESULT_BYTES, + )?; + let batch_len = candidate_ids.len(); + let request = BackendJobRequest { + job_id: format!("subjektiv-consolidation:{}", Uuid::new_v4()), + purpose: PURPOSE.into(), + input_revision: subject.store_revision.to_string(), + input_ref: format!("subjektiv://{subject_id}/consolidation"), + input: serde_json::json!({"subject_id": subject_id, "candidate_ids": candidate_ids}), + instruction: "Resolve ONLY the immutable candidate_ids batch using MemoryStagingList/Read, confirmed Memory reads and MemoryApplyCandidate. Already resolved candidates are retained receipts, not work to repeat. Leave newly arriving candidates to a subsequent Job. When every batch candidate has a durable disposition, call SubmitBackendJobResult with {subject_id, candidate_ids}; its Host capability awaits bounded clean-context surface generation and adds the actual surface outcome before submission. Final prose is not completion.".into(), + // Selection belongs here, never in the common runner's purpose switch. + profile: format!("builtin:{SUBJEKTIV_MEMORY_CONSOLIDATION_PROFILE}"), + grants: BackendJobGrants { subjektiv_consolidation: Some(SubjektivConsolidationGrant { subject_id: subject_id.into(), candidate_ids }) }, + serialization_key: None, + source_worker: None, + notification_target: None, + limits: crate::backend_job::BackendJobLimits { timeout_seconds: 600, ..Default::default() }, + }; + let reservation = api.dispatch_backend_job(&request)?; + Ok(MemoryConsolidationOutput { + status: if reservation.resource_reused { + "skipped_existing_job" + } else { + "started" + } + .into(), + summary: format!( + "Subject consolidation Job '{}' ({:?}) owns an immutable batch of {} candidate(s); remaining/new candidates stay pending for the next request. No persistent subject consolidator is reused.", + reservation.job.request.job_id, + reservation.attempt.state, + if reservation.resource_reused { + reservation + .job + .request + .grants + .subjektiv_consolidation + .as_ref() + .map_or(0, |g| g.candidate_ids.len()) + } else { + batch_len + } + ), + candidate_count, + total_bytes, + }) +} + +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct ConsolidationResult { + subject_id: String, + candidate_ids: Vec, + surface: SurfaceResult, +} +#[derive(Debug, Deserialize)] +#[serde(deny_unknown_fields)] +struct SurfaceResult { + availability: String, + generation_id: String, + store_revision: u64, + #[serde(default)] + snapshot_id: Option, + #[serde(default)] + reason_code: Option, +} + +pub(super) fn validate_result( + api: &WorkspaceApi, + request: &BackendJobRequest, + attempt: &BackendJobAttemptRecord, + value: &serde_json::Value, +) -> Result<()> { + let Some(grant) = &request.grants.subjektiv_consolidation else { + return Ok(()); + }; + let result: ConsolidationResult = serde_json::from_value(value.clone()) + .map_err(|e| Error::InvalidInput(format!("invalid consolidation result: {e}")))?; + if result.subject_id != grant.subject_id || result.candidate_ids != grant.candidate_ids { + return Err(Error::InvalidInput( + "consolidation result does not match the immutable Subject/batch grant".into(), + )); + } + let store = + crate::subjektiv::SubjektivStore::open(&api.feature_storage, &api.subjektiv_registration) + .map_err(|e| Error::Store(e.to_string()))?; + for candidate_id in &grant.candidate_ids { + if store + .staging_resolution(&grant.subject_id, candidate_id) + .map_err(|e| Error::Store(e.to_string()))? + .is_none() + { + return Err(Error::InvalidInput(format!( + "consolidation candidate `{candidate_id}` is unresolved" + ))); + } + } + store + .validate_job_surface_outcome( + &grant.subject_id, + &result.surface.generation_id, + result.surface.store_revision, + &result.surface.availability, + result.surface.snapshot_id.as_deref(), + result.surface.reason_code.as_deref(), + &request.job_id, + &attempt.attempt_id, + ) + .map_err(|e| Error::InvalidInput(format!("consolidation surface result mismatch: {e}"))) +} + +/// Persist actual candidate dispositions alongside the separate surface outcome. +/// Dispositions are immutable existing records, not model assertions or another +/// apply ledger. This enrichment is deterministic on exact result replay. +pub(super) fn with_dispositions( + api: &WorkspaceApi, + request: &BackendJobRequest, + value: &serde_json::Value, +) -> Result { + let grant = request + .grants + .subjektiv_consolidation + .as_ref() + .ok_or_else(|| Error::InvalidInput("missing consolidation grant".into()))?; + let store = + crate::subjektiv::SubjektivStore::open(&api.feature_storage, &api.subjektiv_registration) + .map_err(|e| Error::Store(e.to_string()))?; + let mut dispositions = Vec::new(); + for id in &grant.candidate_ids { + let resolution = store + .staging_resolution(&grant.subject_id, id) + .map_err(|e| Error::Store(e.to_string()))? + .ok_or_else(|| { + Error::InvalidInput(format!("candidate `{id}` has no durable disposition")) + })?; + // Full reasons, affected revisions and source edges remain in the existing + // immutable receipt store. Duplicating them makes a valid bounded batch's + // completion unbounded; this result is its verified compact projection. + dispositions.push(serde_json::json!({ "candidate_id": id, "action": resolution.action })); + } + let mut enriched = value.clone(); + enriched + .as_object_mut() + .ok_or_else(|| Error::InvalidInput("consolidation result must be an object".into()))? + .insert( + "candidate_dispositions".into(), + serde_json::Value::Array(dispositions), + ); + Ok(enriched) +} + +#[cfg(test)] +mod result_budget_tests { + use super::*; + + #[test] + fn escaped_maximum_ids_leave_a_bounded_complete_result() { + let ids: Vec<_> = (0..100) + .map(|i| format!("{i:03}{}", "\\\"".repeat(126))) + .collect(); + let batch = + bounded_candidate_batch(ids.clone(), crate::backend_job::DEFAULT_MAX_RESULT_BYTES) + .unwrap(); + assert!(!batch.is_empty()); + assert!(batch.len() < ids.len()); + assert_eq!(batch, ids[..batch.len()]); + let dispositions: Vec<_> = batch + .iter() + .map(|id| serde_json::json!({"candidate_id":id,"action":"already_covered"})) + .collect(); + let result = serde_json::json!({ + "subject_id":"\\\"".repeat(128),"candidate_ids":batch, + "candidate_dispositions":dispositions, + "surface":{"availability":"failed","generation_id":"g".repeat(100),"store_revision":u64::MAX,"reason_code":"\\\"".repeat(128)}, + }); + assert!( + serde_json::to_vec(&result).unwrap().len() + <= crate::backend_job::DEFAULT_MAX_RESULT_BYTES as usize + ); + } +} diff --git a/crates/workspace-server/src/server/tests/subjektiv_jobs_tests.rs b/crates/workspace-server/src/server/tests/subjektiv_jobs_tests.rs new file mode 100644 index 000000000..78c191a4d --- /dev/null +++ b/crates/workspace-server/src/server/tests/subjektiv_jobs_tests.rs @@ -0,0 +1,732 @@ +use super::*; +use crate::subjektiv::{ + CandidateDecision, CandidateDecisionRequest, MemoryDraft, MemoryRevisionTarget, SubjectRole, + SubjectStagingRecord, SurfaceAvailability, +}; +use memory::extract::{CandidateKind, ExtractedCandidate, StagingEvidence, StagingRecord}; +use memory::schema::SourceRef; +use memory::schema::{EvidenceKind, EvidenceOrigin, EvidenceOriginKind}; + +fn stage(store: &crate::subjektiv::SubjektivStore, subject: &str, id: &str) { + let origin = EvidenceOrigin { + kind: EvidenceOriginKind::HumanInput, + account_id: Some("test-human".into()), + workspace_id: Some(TEST_WORKSPACE_ID.into()), + runtime_id: None, + worker_id: None, + flow_selector: None, + flow_definition_id: None, + flow_definition_revision: None, + }; + let record = StagingRecord::from_candidate( + id, + "test-extract", + SourceRef { + segment_id: "segment-1".into(), + range: [1, 2], + }, + ExtractedCandidate { + kind: CandidateKind::Constraint, + claim: "Keep receipts and confirmed memory".into(), + why_useful: "No repeated application".into(), + staleness: None, + evidence_ids: vec!["e1".into()], + }, + vec![StagingEvidence { + id: "e1".into(), + kind: EvidenceKind::new(EvidenceKind::MESSAGE), + entry_range: Some([1, 2]), + origin: Some(origin), + excerpt: Some("Keep confirmed memory".into()), + summary: None, + }], + vec![], + ); + store + .stage_candidate(SubjectStagingRecord::attach(subject, record)) + .unwrap(); +} +fn context(worker: &RuntimeWorkerRef) -> server_api::ServerRequestContext { + server_api::ServerRequestContext { + actor: None, + worker_source: None, + runtime_source: Some(server_api::ServerRuntimeSource { + runtime_id: worker.runtime_id.clone(), + worker_id: Some(worker.worker_id.clone()), + }), + origin: None, + transport_headers: vec![], + } +} +async fn op( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + operation: server_api::SubjektivMemoryBackendOperation, +) -> ApiResult> { + scoped_subjektiv_memory_backend( + State(api.clone()), + AxumPath(ScopedWorkspacePath { + workspace_id: TEST_WORKSPACE_ID.into(), + }), + context(worker), + Json(server_api::SubjektivMemoryBackendRequest { operation }), + ) + .await +} +fn resource(api: &WorkspaceApi, subject: &str) -> crate::backend_job::BackendJobRecord { + api.store + .find_active_backend_job_for_resource( + TEST_WORKSPACE_ID, + &crate::backend_job::subjektiv_serialization_key(subject), + ) + .unwrap() + .unwrap() +} +fn current_attempt( + api: &WorkspaceApi, + job: &crate::backend_job::BackendJobRecord, +) -> BackendJobAttemptRecord { + api.store + .get_backend_job_attempt( + TEST_WORKSPACE_ID, + &job.request.job_id, + &crate::backend_job::attempt_id(&job.request.job_id, job.current_attempt), + ) + .unwrap() + .unwrap() +} + +#[tokio::test] +async fn subjektiv_job_bounds_batch_auth_partial_receipts_surface_and_cleanup() { + let dir = tempfile::tempdir().unwrap(); + let (api, execution) = test_api_with_recording_backend(dir.path()).await; + let store = open_subjektiv_store(&api).unwrap(); + let subject = store + .create_subject(SubjectRole::new("test-subject").unwrap()) + .unwrap(); + stage(&store, &subject.id, "candidate-first"); + let started = start_subjektiv_staging_consolidation( + api.clone(), + &subject.id, + MemoryConsolidateStagingOperation { force: true }, + ) + .await + .unwrap(); + assert_eq!(started.status, "started"); + let job = resource(&api, &subject.id); + assert_eq!( + job.request.profile, + "builtin:subjektiv-memory-consolidation" + ); + let attempt = current_attempt(&api, &job); + let worker = attempt.worker.as_ref().unwrap(); + assert_eq!(execution.take_inputs().len(), 1); + assert!( + api.store + .require_current_worker_singleton_owner(TEST_WORKSPACE_ID, worker) + .unwrap() + .is_none() + ); + assert_eq!( + subjektiv_subject_scope(&api, TEST_WORKSPACE_ID, &context(worker)) + .unwrap() + .0, + subject.id + ); + assert!(subjektiv_subject_scope(&api, "foreign", &context(worker)).is_err()); + let ordinary = spawn_ticket_check_source(&api, "not-a-consolidator"); + assert!(subjektiv_subject_scope(&api, TEST_WORKSPACE_ID, &context(&ordinary)).is_err()); + // A premature result cannot generate a surface or complete unresolved work. + assert!( + op( + &api, + worker, + server_api::SubjektivMemoryBackendOperation::PrepareSurface( + server_api::SubjektivSurfacePrepareRequest {} + ) + ) + .await + .is_err() + ); + stage(&store, &subject.id, "candidate-arrived-later"); + let again = start_subjektiv_staging_consolidation( + api.clone(), + &subject.id, + MemoryConsolidateStagingOperation { force: true }, + ) + .await + .unwrap(); + assert_eq!(again.status, "skipped_existing_job"); + assert!(execution.take_inputs().is_empty()); + assert_eq!(resource(&api, &subject.id).request, job.request); + assert!( + subjektiv_job::require_candidate_grant( + &api, + TEST_WORKSPACE_ID, + worker, + "candidate-arrived-later" + ) + .is_err() + ); + let decision = CandidateDecisionRequest { + request_id: "exact-apply".into(), + candidate_id: "candidate-first".into(), + reason: "retain the learned constraint".into(), + decision: CandidateDecision::Apply { + target: MemoryRevisionTarget::Create, + draft: MemoryDraft::active( + CandidateKind::Constraint, + "Keep committed memory", + "Applicable across failures", + "Prevents duplicate application", + "confirmed from candidate", + ), + }, + }; + let receipt = store + .decide_candidate(&subject.id, decision.clone()) + .unwrap(); + let replay = store.decide_candidate(&subject.id, decision).unwrap(); + assert_eq!( + receipt.memory.as_ref().unwrap().id, + replay.memory.as_ref().unwrap().id + ); + assert_eq!(receipt.store_revision, replay.store_revision); + let generation = store + .prepare_job_surface_generation( + &subject.id, + Some((&job.request.job_id, &attempt.attempt_id)), + ) + .unwrap(); + let Json(failed) = op( + &api, + worker, + server_api::SubjektivMemoryBackendOperation::FailSurface( + server_api::SubjektivSurfaceFailureRequest { + generation_id: generation.id.clone(), + reason_code: "editor_failed".into(), + }, + ), + ) + .await + .unwrap(); + assert!( + matches!(failed, server_api::SubjektivMemoryBackendResponse::SurfaceFailed(output) if output.status == "failed_confirmed" && output.store_revision == generation.store_revision) + ); + let result = serde_json::json!({"subject_id": subject.id, "candidate_ids": ["candidate-first"], "surface": {"availability":"failed", "generation_id":generation.id, "store_revision":generation.store_revision,"reason_code":"editor_failed"}}); + let submission = BackendJobResultSubmission { + job_id: job.request.job_id.clone(), + attempt_id: attempt.attempt_id.clone(), + input_revision: job.request.input_revision.clone(), + result, + }; + assert!( + api.accept_backend_job_result(&ordinary, &submission) + .is_err() + ); + let mut forged = submission.clone(); + forged.result["surface"]["reason_code"] = serde_json::json!("not_recorded"); + assert!(api.accept_backend_job_result(worker, &forged).is_err()); + seed_worker_session_for_cleanup(dir.path(), worker); + let accepted = api.accept_backend_job_result(worker, &submission).unwrap(); + assert_eq!(accepted.job.state, BackendJobState::Completed); + assert_eq!( + store.resident_surface(&subject.id).unwrap().availability, + SurfaceAvailability::Failed + ); + assert_eq!(store.pending_staging_backlog(&subject.id).unwrap().0, 1); + assert!( + store + .memory(&subject.id, &receipt.memory.unwrap().id) + .unwrap() + .is_some() + ); + assert!(subjektiv_subject_scope(&api, TEST_WORKSPACE_ID, &context(worker)).is_err()); + wait_for_backend_job_worker_cleanup( + &api, + &job.request.job_id, + &attempt.attempt_id, + BackendJobWorkerCleanupState::Completed, + ) + .await; + assert!( + api.store + .get_backend_job(TEST_WORKSPACE_ID, &job.request.job_id) + .unwrap() + .unwrap() + .result + .is_some() + ); + let next = start_subjektiv_staging_consolidation( + api.clone(), + &subject.id, + MemoryConsolidateStagingOperation { force: true }, + ) + .await + .unwrap(); + assert_eq!(next.status, "started"); + assert_eq!( + resource(&api, &subject.id) + .request + .grants + .subjektiv_consolidation + .unwrap() + .candidate_ids, + ["candidate-arrived-later"] + ); +} + +#[tokio::test] +async fn subjektiv_job_surface_only_requires_attempt_owned_generation_and_real_outcome() { + let dir = tempfile::tempdir().unwrap(); + let (api, _) = test_api_with_recording_backend(dir.path()).await; + let store = open_subjektiv_store(&api).unwrap(); + let subject = store + .create_subject(SubjectRole::new("surface-only").unwrap()) + .unwrap(); + let old = store.prepare_surface_generation(&subject.id).unwrap(); + assert_eq!( + start_subjektiv_staging_consolidation( + api.clone(), + &subject.id, + MemoryConsolidateStagingOperation { force: false } + ) + .await + .unwrap() + .status, + "started" + ); + let job = resource(&api, &subject.id); + let attempt = current_attempt(&api, &job); + let worker = attempt.worker.as_ref().unwrap(); + assert!( + store + .require_job_surface_generation( + &subject.id, + &old.id, + &job.request.job_id, + &attempt.attempt_id + ) + .is_err() + ); + let generation = store + .prepare_job_surface_generation( + &subject.id, + Some((&job.request.job_id, &attempt.attempt_id)), + ) + .unwrap(); + assert!( + store + .require_job_surface_generation( + &subject.id, + &generation.id, + &job.request.job_id, + "wrong-attempt" + ) + .is_err() + ); + let snapshot = store + .publish_surface_generation(&subject.id, &generation.id, vec![]) + .unwrap(); + assert!( + op( + &api, + worker, + server_api::SubjektivMemoryBackendOperation::FailSurface( + server_api::SubjektivSurfaceFailureRequest { + generation_id: generation.id.clone(), + reason_code: "editor_failed".into() + } + ) + ) + .await + .is_err(), + "ready protection is not a confirmed failed outcome" + ); + let Json(prepared) = op( + &api, + worker, + server_api::SubjektivMemoryBackendOperation::PrepareSurface( + server_api::SubjektivSurfacePrepareRequest {}, + ), + ) + .await + .unwrap(); + assert!( + matches!(prepared, server_api::SubjektivMemoryBackendResponse::SurfacePrepared(output) if output.current_snapshot_id.as_deref() == Some(snapshot.id.as_str())) + ); + seed_worker_session_for_cleanup(dir.path(), worker); + let submission = BackendJobResultSubmission { + job_id: job.request.job_id.clone(), + attempt_id: attempt.attempt_id.clone(), + input_revision: job.request.input_revision.clone(), + result: serde_json::json!({"subject_id":subject.id,"candidate_ids":[],"surface":{"availability":"ready","generation_id":generation.id,"store_revision":generation.store_revision,"snapshot_id":snapshot.id}}), + }; + api.accept_backend_job_result(worker, &submission).unwrap(); + wait_for_backend_job_worker_cleanup( + &api, + &job.request.job_id, + &attempt.attempt_id, + BackendJobWorkerCleanupState::Completed, + ) + .await; + assert_eq!( + start_subjektiv_staging_consolidation( + api.clone(), + &subject.id, + MemoryConsolidateStagingOperation { force: true } + ) + .await + .unwrap() + .status, + "skipped_empty" + ); + assert!( + api.accept_backend_job_result(worker, &submission) + .unwrap() + .replayed + ); +} + +#[tokio::test] +async fn backend_job_profile_resolution_rejects_ungranted_requirements_without_fallback() { + let dir = tempfile::tempdir().unwrap(); + let (api, execution) = test_api_with_recording_backend(dir.path()).await; + let mut request = BackendJobRequest { + job_id: "profile-resolution".into(), + purpose: "any-purpose".into(), + input_revision: "1".into(), + input_ref: "test://profiles".into(), + input: serde_json::json!({}), + instruction: "return structured result".into(), + profile: "builtin:backend-job".into(), + grants: Default::default(), + serialization_key: None, + source_worker: None, + notification_target: None, + limits: Default::default(), + }; + let (selector, bundle) = api.resolve_backend_job_profile(&request).unwrap(); + assert_eq!(selector, ProfileSelector::Builtin(request.profile.clone())); + assert_eq!( + crate::profile_settings::resolve_profile_manifest_from_config_bundle( + &bundle, + &request.profile + ) + .unwrap() + .engine + .instruction, + "internal.backend_job_system" + ); + for profile in [ + "path:/etc/profile", + "builtin:missing", + "project:missing", + "builtin:coder", + "builtin:companion", + "builtin:subjektiv-memory-consolidation", + ] { + request.profile = profile.into(); + assert!(api.dispatch_backend_job(&request).is_err(), "{profile}"); + assert!( + api.store + .get_backend_job(TEST_WORKSPACE_ID, &request.job_id) + .unwrap() + .is_none() + ); + } + assert!(execution.take_inputs().is_empty()); +} + +#[tokio::test] +async fn backend_job_project_recipe_is_resolved_and_delivered_without_system_override() { + let dir = tempfile::tempdir().unwrap(); + let (api, _) = test_api_with_recording_backend(dir.path()).await; + let state = api + .config_store + .load_workspace_config(TEST_WORKSPACE_ID) + .unwrap() + .unwrap(); + let entrypoint = state.contract.entrypoints.first().unwrap(); + let root = state.snapshot.entries.get(entrypoint).unwrap(); + let config = config_commit_request_from_api(server_api::ConfigCommitRequest { + base_revision: state.snapshot.revision, base_digest: state.snapshot.digest.clone(), + entrypoints: state.contract.entrypoints.iter().map(|p| p.as_str().into()).collect(), + changes: vec![server_api::ConfigTreeChange::Update { + path: entrypoint.as_str().into(), expected_digest: root.content_digest.clone(), + content: "{ profile = { entries = [{ selector = \"project:job-recipe\"; source = \"profiles/job-recipe.dcdl\"; }]; }; }".into(), + }, server_api::ConfigTreeChange::Create { + path: "profiles/job-recipe.dcdl".into(), content_type: server_api::ConfigContentType::Decodal, + content: "{ slug = \"job-recipe\"; scope = \"workspace_read\"; model = { id = \"fixture-model\"; }; engine = { instruction = \"role.default\"; }; feature = { task = { enabled = true; }; }; }".into(), + }], + }).unwrap(); + commit_workspace_config_tree(&api, TEST_WORKSPACE_ID, &config).unwrap(); + let request = BackendJobRequest { + job_id: "project-recipe-job".into(), + purpose: "no-profile-switch".into(), + input_revision: "r1".into(), + input_ref: "test://recipe".into(), + input: serde_json::json!({}), + instruction: "Submit the result".into(), + profile: "project:job-recipe".into(), + grants: Default::default(), + serialization_key: None, + source_worker: None, + notification_target: None, + limits: Default::default(), + }; + let (selector, bundle) = api.resolve_backend_job_profile(&request).unwrap(); + assert_eq!( + selector, + ProfileSelector::Named("project:job-recipe".into()) + ); + let manifest = crate::profile_settings::resolve_profile_manifest_from_config_bundle( + &bundle, + &request.profile, + ) + .unwrap(); + assert_eq!(manifest.engine.instruction, "role.default"); + assert!(manifest.feature.task.enabled); + let dispatched = api.dispatch_backend_job(&request).unwrap(); + let worker = dispatched.attempt.worker.unwrap(); + assert_eq!( + api.runtime.worker(&worker).unwrap().profile.as_deref(), + Some("project:job-recipe") + ); +} + +#[tokio::test] +async fn subjektiv_job_legacy_cutover_waits_foreground_then_removes_only_owned_consolidator() { + let dir = tempfile::tempdir().unwrap(); + let (api, execution) = test_api_with_recording_backend(dir.path()).await; + let store = open_subjektiv_store(&api).unwrap(); + let subject = store + .create_subject(SubjectRole::new("legacy-cutover").unwrap()) + .unwrap(); + let subject_body = spawn_ticket_check_source(&api, "subject-body-protected"); + let key = format!("subjektiv-consolidation:{}", subject.id); + let legacy = api + .spawn_workspace_worker( + EMBEDDED_WORKER_RUNTIME_ID, + WorkerSpawnRequest { + requested_worker_name: Some("not-authority-display".into()), + singleton_key: Some(key.clone()), + intent: WorkerSpawnIntent::WorkspaceOrchestrator, + acceptance: WorkerSpawnAcceptanceRequirement::RunAccepted { + expected_segments: 0, + }, + profile: ProfileSelector::Builtin(SUBJEKTIV_MEMORY_CONSOLIDATION_PROFILE.into()), + ticket_assignment: None, + initial_submit: vec![], + workdir_attachment_requests: vec![], + resolved_workdir_attachment_requests: vec![], + resolved_workdir_attachments: vec![], + resolved_config_bundle: None, + resolved_workspace_api: None, + resolved_memory_settings: None, + resolved_subjektiv_attached: false, + resolved_worker_observation_enabled: false, + resolved_worker_observation_grants: vec![], + resolved_control_operation: None, + }, + ) + .unwrap() + .worker + .unwrap() + .worker; + seed_worker_session_for_cleanup(dir.path(), &legacy); + execution.publish_worker_state( + &legacy, + protocol::WorkerStateSnapshot::from(protocol::WorkerStatus::Running), + ); + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while api + .runtime + .worker(&legacy) + .unwrap() + .worker_state + .as_ref() + .is_none_or(|s| s.state == protocol::WorkerState::Idle) + { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!( + start_subjektiv_staging_consolidation( + api.clone(), + &subject.id, + MemoryConsolidateStagingOperation { force: true } + ) + .await + .unwrap() + .status, + "skipped_legacy_inflight" + ); + assert!( + api.store + .find_active_backend_job_for_resource(TEST_WORKSPACE_ID, &key) + .unwrap() + .is_none() + ); + assert!(api.runtime.worker(&legacy).is_ok()); + execution.publish_worker_state(&legacy, protocol::WorkerStateSnapshot::initial()); + tokio::time::timeout(std::time::Duration::from_secs(2), async { + while api + .runtime + .worker(&legacy) + .unwrap() + .worker_state + .as_ref() + .is_none_or(|s| s.state != protocol::WorkerState::Idle) + { + tokio::task::yield_now().await; + } + }) + .await + .unwrap(); + assert_eq!( + start_subjektiv_staging_consolidation( + api.clone(), + &subject.id, + MemoryConsolidateStagingOperation { force: true } + ) + .await + .unwrap() + .status, + "started" + ); + assert!(api.runtime.worker(&legacy).is_err()); + assert!( + api.store + .current_worker_singleton_owner(TEST_WORKSPACE_ID, &key) + .unwrap() + .is_none() + ); + assert!(api.runtime.worker(&subject_body).is_ok()); + assert!(store.subject(&subject.id).unwrap().is_some()); +} + +#[tokio::test] +async fn subjektiv_job_long_receipts_and_escaped_ids_fit_result_budget_without_losing_followup() { + let dir = tempfile::tempdir().unwrap(); + let (api, _) = test_api_with_recording_backend(dir.path()).await; + let store = open_subjektiv_store(&api).unwrap(); + let subject = store + .create_subject(SubjectRole::new("result-budget").unwrap()) + .unwrap(); + for i in 0..100 { + stage( + &store, + &subject.id, + &format!("candidate-{i:03}-{}", "\\\"".repeat(60)), + ); + } + start_subjektiv_staging_consolidation( + api.clone(), + &subject.id, + MemoryConsolidateStagingOperation { force: true }, + ) + .await + .unwrap(); + let job = resource(&api, &subject.id); + let ids = job + .request + .grants + .subjektiv_consolidation + .as_ref() + .unwrap() + .candidate_ids + .clone(); + assert!(!ids.is_empty() && ids.len() < 100); + let attempt = current_attempt(&api, &job); + let worker = attempt.worker.as_ref().unwrap(); + for id in &ids { + store + .decide_candidate( + &subject.id, + CandidateDecisionRequest { + request_id: format!("decision-{}", uuid::Uuid::new_v4()), + candidate_id: id.clone(), + reason: "Existing confirmed detail remains in its immutable receipt. " + .repeat(500), + decision: CandidateDecision::Close { + action: crate::subjektiv::StagingResolutionAction::Discarded, + affected_memory: vec![], + }, + }, + ) + .unwrap(); + } + let generation = store + .prepare_job_surface_generation( + &subject.id, + Some((&job.request.job_id, &attempt.attempt_id)), + ) + .unwrap(); + let snapshot = store + .publish_surface_generation(&subject.id, &generation.id, vec![]) + .unwrap(); + let submission = BackendJobResultSubmission { + job_id: job.request.job_id.clone(), + attempt_id: attempt.attempt_id.clone(), + input_revision: job.request.input_revision.clone(), + result: serde_json::json!({"subject_id":subject.id,"candidate_ids":ids,"surface":{"availability":"ready","generation_id":generation.id,"store_revision":generation.store_revision,"snapshot_id":snapshot.id}}), + }; + seed_worker_session_for_cleanup(dir.path(), worker); + let accepted = api.accept_backend_job_result(worker, &submission).unwrap(); + let result = accepted.job.result.as_ref().unwrap(); + assert!( + serde_json::to_vec(result).unwrap().len() <= job.request.limits.max_result_bytes as usize + ); + let outcomes = result["candidate_dispositions"].as_array().unwrap(); + assert_eq!(outcomes.len(), ids.len()); + assert!( + outcomes + .iter() + .all(|o| o["action"] == "discarded" && o.get("reason").is_none()) + ); + assert_eq!( + store.pending_staging_backlog(&subject.id).unwrap().0, + 100 - ids.len() + ); + assert!( + api.accept_backend_job_result(worker, &submission) + .unwrap() + .replayed + ); + wait_for_backend_job_worker_cleanup( + &api, + &job.request.job_id, + &attempt.attempt_id, + BackendJobWorkerCleanupState::Completed, + ) + .await; + start_subjektiv_staging_consolidation( + api.clone(), + &subject.id, + MemoryConsolidateStagingOperation { force: true }, + ) + .await + .unwrap(); + let next = resource(&api, &subject.id); + assert!( + next.request + .grants + .subjektiv_consolidation + .unwrap() + .candidate_ids + .iter() + .all(|id| !ids.contains(id)) + ); + assert!( + store + .staging_resolution(&subject.id, &ids[0]) + .unwrap() + .unwrap() + .reason + .len() + > 16 * 1024 + ); +} diff --git a/crates/workspace-server/src/server/tests/worker_operations_tests.rs b/crates/workspace-server/src/server/tests/worker_operations_tests.rs new file mode 100644 index 000000000..ebcddc3ab --- /dev/null +++ b/crates/workspace-server/src/server/tests/worker_operations_tests.rs @@ -0,0 +1,1758 @@ +use super::*; + +fn operation_input(kind: WorkerInputKind) -> WorkerInputRequest { + WorkerInputRequest { + kind, + content: "operation-boundary-test".into(), + submission_request_id: None, + segments: None, + } +} + +fn operation_lifecycle() -> WorkerLifecycleRequest { + WorkerLifecycleRequest { + reason: Some("operation-boundary-test".into()), + ticket_assignment: None, + } +} + +fn operation_cleanup_candidate(identity: &RuntimeWorkerRef) -> CleanupWorkerCandidate { + CleanupWorkerCandidate { + target_id: format!("worker:{}:{}", identity.runtime_id, identity.worker_id), + action: server_api::CleanupTargetKind::WorkerDelete, + worker_id: identity.worker_id.clone(), + runtime_worker_id: identity.worker_id.clone(), + runtime_id: identity.runtime_id.clone(), + reason: "operation-boundary-test".into(), + blocking_reason: None, + pinned: false, + retention_state: "normal".into(), + linked_workdir_ids: Vec::new(), + running_linked: false, + estimated_reclaim_bytes: None, + } +} + +async fn browser_operation_context(api: &WorkspaceApi) -> (WorkerOperationContext, String) { + seed_test_api_token(api.store.as_ref(), "worker-operation-browser"); + let token = "worker-operation-browser-session"; + api.store + .create_browser_session(&BrowserSessionRecord { + token_hash: crate::auth::token_hash(token), + session_id: "worker-operation-session".into(), + user_id: "user-worker-operation-browser".into(), + created_at: "2026-01-01T00:00:00Z".into(), + expires_at: "2099-01-01T00:00:00Z".into(), + revoked_at: None, + }) + .unwrap(); + let AuthConfig::Passkey { + cookie_name, + origin, + .. + } = &api.config.auth; + let mut headers = HeaderMap::new(); + headers.insert(ORIGIN, origin.parse().unwrap()); + headers.insert( + axum::http::header::COOKIE, + format!("{cookie_name}={token}").parse().unwrap(), + ); + let actor = resolve_actor(&ServerAuthApi::from(api), &headers) + .await + .unwrap() + .expect("valid browser session"); + ( + WorkerOperationContext::Browser { + headers, + source: authenticated_browser_input_source(&actor), + }, + crate::auth::token_hash(token), + ) +} + +fn browser_operation_methods() -> Vec { + vec![ + protocol::Method::Pause { + command: protocol::WorkerCommandEnvelope::new(701), + }, + protocol::Method::Resume { + command: protocol::WorkerCommandEnvelope::new(702), + }, + protocol::Method::Cancel { + command: protocol::WorkerCommandEnvelope::new(703), + }, + protocol::Method::Compact { + command: protocol::WorkerCommandEnvelope::new(704), + }, + protocol::Method::Submit { + submission_request_id: "worker-operation-submit".into(), + input: vec![protocol::Segment::Text { + content: "must not reach execution".into(), + }], + }, + protocol::Method::Notify { + notification_request_id: "worker-operation-notify".into(), + message: "must not reach execution".into(), + }, + protocol_test_completion("worker-operation-completion"), + ] +} + +#[tokio::test] +async fn handle_resource_keys_preserve_identity_for_equal_local_ids_across_runtimes() { + let workspace = tempfile::tempdir().unwrap(); + let api = test_api(workspace.path()).await; + let a = RuntimeWorkerRef::new("offline-runtime-a", "shared-local-id"); + let b = RuntimeWorkerRef::new("offline-runtime-b", "shared-local-id"); + for identity in [&a, &b] { + seed_worker_source_member(&api, &identity.runtime_id, &identity.worker_id); + } + let key = |identity: &RuntimeWorkerRef| { + api.store + .worker_resource_key(TEST_WORKSPACE_ID, identity) + .unwrap() + .unwrap() + }; + assert_ne!(key(&a), key(&b)); + + for identity in [&a, &b] { + let direct = + WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap(); + let canonical = + WorkspaceWorker::resolve(&api, &identity.runtime_id, &key(identity)).unwrap(); + let pinned = canonical + .set_pinned(&WorkerOperationContext::Backend, true) + .await + .unwrap(); + assert_eq!(pinned.runtime_id, identity.runtime_id); + assert_eq!(pinned.worker_id, identity.worker_id); + let unpinned = direct + .set_pinned(&WorkerOperationContext::Backend, false) + .await + .unwrap(); + assert_eq!(unpinned.runtime_id, pinned.runtime_id); + assert_eq!(unpinned.worker_id, pinned.worker_id); + } + assert!(matches!( + WorkspaceWorker::resolve(&api, &b.runtime_id, &key(&a)), + Err(Error::UnknownWorker { .. }) + )); +} + +#[tokio::test] +async fn handle_resolution_rejects_unknown_foreign_and_internal_only_workers() { + let workspace = tempfile::tempdir().unwrap(); + let api = test_api(workspace.path()).await; + let known = spawn_ticket_check_source(&api, "operation-resolution-parent"); + let mut foreign = api + .store + .get_worker_registry(TEST_WORKSPACE_ID, &known) + .unwrap() + .unwrap(); + let mut other_workspace = api + .store + .get_workspace(TEST_WORKSPACE_ID) + .await + .unwrap() + .unwrap(); + other_workspace.workspace_id = "other-workspace".into(); + api.store.upsert_workspace(&other_workspace).await.unwrap(); + foreign.workspace_id = other_workspace.workspace_id; + foreign.worker.worker_id = "foreign-only-worker".into(); + api.store.upsert_worker_registry(&foreign).unwrap(); + assert!( + api.store + .get_worker_registry(TEST_WORKSPACE_ID, &foreign.worker) + .unwrap() + .is_none() + ); + + let RuntimeObservationSource::Embedded(source) = + api.runtime.observation_source(&known).unwrap() + else { + panic!("expected embedded fixture"); + }; + let internal = protocol::InternalWorkerRef { + session_id: "internal-only-session".into(), + parent_session_id: None, + name: "internal-only-child".into(), + kind: protocol::InternalWorkerKind::SubWorker, + }; + source + .runtime + .observe_worker_event( + &source.worker_ref, + protocol::Event::InternalWorker { + worker: internal.clone(), + revision: 1, + event: Box::new(protocol::Event::WorkerState { + snapshot: protocol::WorkerStateSnapshot::initial(), + }), + }, + ) + .unwrap(); + + for reference in [ + "unknown-worker", + &foreign.worker.worker_id, + &internal.session_id, + &internal.name, + ] { + assert!( + matches!( + WorkspaceWorker::resolve(&api, &known.runtime_id, reference), + Err(Error::UnknownWorker { .. }) + ), + "unexpected registry handle for {reference}" + ); + } +} + +#[tokio::test] +async fn offline_handle_can_pin_but_deleted_handle_cannot_mutate_or_connect() { + let workspace = tempfile::tempdir().unwrap(); + let api = test_api(workspace.path()).await; + let identity = RuntimeWorkerRef::new("unregistered-offline-runtime", "42"); + seed_worker_source_member(&api, &identity.runtime_id, &identity.worker_id); + assert!( + api.runtime.worker(&identity).is_err(), + "fixture must have no live Runtime" + ); + let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap(); + let retained = handle + .set_pinned(&WorkerOperationContext::Backend, true) + .await + .unwrap(); + assert!(retained.pinned); + assert_eq!( + api.store + .get_worker_registry(TEST_WORKSPACE_ID, &identity) + .unwrap() + .unwrap() + .retention_state, + "pinned" + ); + + let mut wrong_candidate = operation_cleanup_candidate(&identity); + wrong_candidate.runtime_worker_id = "different-worker".into(); + assert!(matches!( + handle.delete_from_plan(&WorkerOperationContext::Backend, &wrong_candidate) + .await.unwrap_err().error, + Error::InvalidInput(message) if message.contains("resolved Worker") + )); + + api.store + .delete_worker_registry(TEST_WORKSPACE_ID, &identity) + .unwrap(); + assert!( + matches!(handle.set_pinned(&WorkerOperationContext::Backend, false) + .await.unwrap_err().error, Error::UnknownWorker { worker } if worker == identity) + ); + assert!(matches!( + handle + .restore( + &WorkerOperationContext::Backend, + server_api::WorkerRestoreRequest { + expected_observation_token: "stale".into(), + request_id: "removed-handle".into(), + } + ) + .await + .unwrap_err() + .error, + Error::UnknownWorker { .. } + )); + assert!(matches!( + handle + .stop(&WorkerOperationContext::Backend, operation_lifecycle()) + .await + .unwrap_err() + .error, + Error::UnknownWorker { .. } + )); + assert!(matches!( + handle + .cancel(&WorkerOperationContext::Backend, operation_lifecycle()) + .await + .unwrap_err() + .error, + Error::UnknownWorker { .. } + )); + for kind in [WorkerInputKind::User, WorkerInputKind::Notify] { + assert!(matches!( + handle + .input(&WorkerOperationContext::Backend, operation_input(kind)) + .await + .unwrap_err() + .error, + Error::UnknownWorker { .. } + )); + } + assert!(matches!( + handle + .delete_from_plan( + &WorkerOperationContext::Backend, + &operation_cleanup_candidate(&identity) + ) + .await + .unwrap_err() + .error, + Error::UnknownWorker { .. } + )); + assert!(matches!( + handle + .connect_protocol(&WorkerOperationContext::Backend) + .await, + Err(Error::UnknownWorker { .. }) + )); + assert!( + api.store + .get_worker_registry(TEST_WORKSPACE_ID, &identity) + .unwrap() + .is_none(), + "a stale handle must not recreate its registry row" + ); +} + +#[tokio::test] +async fn worker_control_handle_rechecks_revoked_grants_for_each_operation() { + let workspace = tempfile::tempdir().unwrap(); + let api = test_api(workspace.path()).await; + let controller = RuntimeWorkerRef::new("offline-runtime", "controller"); + let subject = RuntimeWorkerRef::new("offline-runtime", "subject"); + for identity in [&controller, &subject] { + seed_worker_source_member(&api, &identity.runtime_id, &identity.worker_id); + } + api.store + .create_worker_control_grant(&WorkerControlGrantRecord { + workspace_id: TEST_WORKSPACE_ID.into(), + grant_id: "operation-grant".into(), + controller: controller.clone(), + subject: subject.clone(), + relation: "spawned".into(), + origin: "test".into(), + permissions: [ + "restore", + "stop", + "cancel", + "send_input", + "notify", + "pin", + "remove", + ] + .into_iter() + .map(str::to_owned) + .collect(), + operation_id: "operation-grant-test".into(), + created_at: now_registry_timestamp(), + revoked_at: None, + }) + .unwrap(); + let handle = WorkspaceWorker::resolve(&api, &subject.runtime_id, &subject.worker_id).unwrap(); + let context = WorkerOperationContext::WorkerControl { controller }; + handle.set_pinned(&context, true).await.unwrap(); + api.store + .revoke_worker_control_grant( + TEST_WORKSPACE_ID, + "operation-grant", + &now_registry_timestamp(), + ) + .unwrap(); + + for permission in [ + "restore", + "stop", + "cancel", + "send_input", + "notify", + "pin", + "remove", + ] { + assert!( + matches!( + handle.authorize_operation(&context, permission).await, + Err(ApiError { + error: Error::UnknownWorker { .. }, + .. + }) + ), + "revoked grant authorized {permission}" + ); + } + assert!(matches!( + handle + .restore( + &context, + server_api::WorkerRestoreRequest { + expected_observation_token: "stale".into(), + request_id: "revoked-grant".into(), + } + ) + .await + .unwrap_err() + .error, + Error::UnknownWorker { .. } + )); + assert!(matches!( + handle + .stop(&context, operation_lifecycle()) + .await + .unwrap_err() + .error, + Error::UnknownWorker { .. } + )); + assert!(matches!( + handle + .cancel(&context, operation_lifecycle()) + .await + .unwrap_err() + .error, + Error::UnknownWorker { .. } + )); + for kind in [WorkerInputKind::User, WorkerInputKind::Notify] { + assert!(matches!( + handle + .input(&context, operation_input(kind)) + .await + .unwrap_err() + .error, + Error::UnknownWorker { .. } + )); + } + assert!(matches!( + handle.set_pinned(&context, false).await.unwrap_err().error, + Error::UnknownWorker { .. } + )); + assert!(matches!( + handle + .delete_from_plan(&context, &operation_cleanup_candidate(&subject)) + .await + .unwrap_err() + .error, + Error::UnknownWorker { .. } + )); + assert_eq!( + api.store + .get_worker_registry(TEST_WORKSPACE_ID, &subject) + .unwrap() + .unwrap() + .retention_state, + "pinned" + ); +} + +#[tokio::test] +async fn generic_and_scoped_input_aliases_match_handle_idle_only_and_notify_semantics() { + let fixture = guarded_spawn_fixture().await; + let response = post_guarded_spawn( + &fixture, + guarded_spawn_payload(&fixture, "operation-alias-parity"), + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + let bytes = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + let created: BrowserCreateWorkerResponse = serde_json::from_slice(&bytes).unwrap(); + let api = fixture.api.clone(); + let runtime = &fixture.runtime; + let subject = RuntimeWorkerRef::new(created.runtime_id, created.worker_id); + let controller = fixture.controller.clone(); + let key = api + .store + .worker_resource_key(TEST_WORKSPACE_ID, &subject) + .unwrap() + .unwrap(); + let handle = WorkspaceWorker::resolve(&api, &subject.runtime_id, &key).unwrap(); + let context = WorkerOperationContext::WorkerControl { controller }; + let idle_only = handle + .input(&context, operation_input(WorkerInputKind::User)) + .await + .unwrap(); + assert_eq!(idle_only.state, InternalWorkerOperationState::Rejected); + let notify = handle + .input(&context, operation_input(WorkerInputKind::Notify)) + .await + .unwrap(); + assert_eq!(notify.state, InternalWorkerOperationState::Accepted); + + let request = || server_api::RuntimeWorkerInputRequest { + kind: Some("user".into()), + content: "human input".into(), + segments: None, + }; + let Json(generic) = send_runtime_worker_input( + State(api.clone()), + AxumPath((subject.runtime_id.clone(), key.clone())), + Json(request()), + ) + .await + .unwrap(); + let Json(scoped) = scoped_send_runtime_worker_input( + State(api.clone()), + AxumPath(ScopedRuntimeWorkerPath { + workspace_id: TEST_WORKSPACE_ID.into(), + worker: RuntimeWorkerRef::new(&subject.runtime_id, &key), + }), + Json(request()), + ) + .await + .unwrap(); + assert_eq!(generic.state, scoped.state); + assert_eq!(generic.state, server_api::WorkerOperationState::Accepted); + assert_eq!( + runtime + .input_requests + .lock() + .unwrap() + .iter() + .map(|request| request.kind.clone()) + .collect::>(), + vec![ + WorkerInputKind::UserIfIdle, + WorkerInputKind::Notify, + WorkerInputKind::User, + WorkerInputKind::User + ] + ); +} + +#[tokio::test] +async fn browser_protocol_sender_rejects_job_input_and_all_live_methods() { + let workspace = tempfile::tempdir().unwrap(); + let (api, execution) = test_api_with_recording_backend(workspace.path()).await; + let request = BackendJobRequest { + job_id: "worker-operation-read-only-job".into(), + purpose: "operation_boundary_test".into(), + input_revision: "revision-1".into(), + input_ref: "test://worker-operation/input".into(), + input: serde_json::json!({"immutable": true}), + instruction: "Return a structured result.".into(), + profile: "builtin:backend-job".into(), + grants: Default::default(), + serialization_key: None, + source_worker: None, + notification_target: None, + limits: crate::backend_job::BackendJobLimits::default(), + }; + let dispatched = api.dispatch_backend_job(&request).unwrap(); + let identity = dispatched.attempt.worker.unwrap(); + assert_eq!(execution.take_inputs().len(), 1); + let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap(); + let (context, _) = browser_operation_context(&api).await; + let mut connection = handle.connect_protocol(&context).await.unwrap(); + assert!(matches!( + tokio::time::timeout(std::time::Duration::from_secs(2), connection.events.recv()) + .await + .unwrap() + .unwrap(), + protocol::Event::Snapshot { .. } + )); + for method in browser_operation_methods() { + assert!(matches!(connection.sender.send(&context, method).await, + Err(Error::InvalidInput(message)) if message.contains("read-only"))); + } + for kind in [WorkerInputKind::User, WorkerInputKind::Notify] { + assert!( + matches!(handle.input(&context, operation_input(kind)).await.unwrap_err().error, + Error::InvalidInput(message) if message.contains("read-only")) + ); + } + assert!(execution.protocol_methods().is_empty()); + assert!(execution.take_inputs().is_empty()); +} + +#[tokio::test] +async fn connected_browser_sender_rechecks_revoked_credentials_before_each_method() { + let workspace = tempfile::tempdir().unwrap(); + let (api, execution) = test_api_with_recording_backend(workspace.path()).await; + let identity = spawn_ticket_check_source(&api, "operation-browser-revocation"); + let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap(); + let (context, token_hash) = browser_operation_context(&api).await; + let mut connection = handle.connect_protocol(&context).await.unwrap(); + assert!(matches!( + tokio::time::timeout(std::time::Duration::from_secs(2), connection.events.recv()) + .await + .unwrap() + .unwrap(), + protocol::Event::Snapshot { .. } + )); + // Prove these credentials were usable before revocation, then keep the same sender. + handle.set_pinned(&context, true).await.unwrap(); + assert!( + api.store + .revoke_browser_session(&token_hash, &now_registry_timestamp()) + .unwrap() + ); + assert!(matches!( + handle.connect_protocol(&context).await, + Err(Error::WorkspacePermissionDenied(_)) + )); + for method in browser_operation_methods() { + assert!(matches!( + connection.sender.send(&context, method).await, + Err(Error::WorkspacePermissionDenied(_)) + )); + } + assert!(matches!( + handle.set_pinned(&context, false).await.unwrap_err().error, + Error::WorkspacePermissionDenied(_) + )); + assert!( + execution.protocol_methods().is_empty(), + "revoked credentials must not reach the execution channel" + ); + assert_eq!( + api.store + .get_worker_registry(TEST_WORKSPACE_ID, &identity) + .unwrap() + .unwrap() + .retention_state, + "pinned" + ); +} + +fn operation_request_context( + source: Option, +) -> server_api::ServerRequestContext { + server_api::ServerRequestContext { + actor: None, + worker_source: None, + runtime_source: source, + origin: None, + transport_headers: Vec::new(), + } +} + +#[tokio::test] +async fn request_context_uses_verified_subject_not_headers_or_account_fallback() { + let workspace = tempfile::tempdir().unwrap(); + let api = test_api(workspace.path()).await; + let mut headers_only = operation_request_context(None); + headers_only.transport_headers = vec![ + ("x-yoi-runtime-id".into(), b"runtime-a".to_vec()), + ("x-yoi-worker-id".into(), b"controller-a".to_vec()), + ]; + assert!(WorkerOperationContext::from_request(&api, &headers_only).is_err()); + let mut runtime_only = operation_request_context(Some(server_api::ServerRuntimeSource { + runtime_id: "runtime-a".into(), + worker_id: None, + })); + runtime_only.actor = Some(test_browser_request_actor()); + assert!(WorkerOperationContext::from_request(&api, &runtime_only).is_err()); + let mut mismatched = operation_request_context(Some(server_api::ServerRuntimeSource { + runtime_id: "runtime-a".into(), + worker_id: Some("controller-a".into()), + })); + mismatched.actor = Some(test_browser_request_actor()); + mismatched.transport_headers = vec![("x-yoi-worker-id".into(), b"controller-b".to_vec())]; + assert!(WorkerOperationContext::from_request(&api, &mismatched).is_err()); + seed_worker_source_member(&api, "runtime-a", "controller-a"); + let context = operation_request_context(Some(server_api::ServerRuntimeSource { + runtime_id: "runtime-a".into(), + worker_id: Some("controller-a".into()), + })); + assert!( + matches!(WorkerOperationContext::from_request(&api, &context).unwrap(), + WorkerOperationContext::WorkerControl { controller } if controller == RuntimeWorkerRef::new("runtime-a", "controller-a")) + ); +} + +#[tokio::test] +async fn generic_management_aliases_cannot_bypass_worker_grants_or_target_bound_remove() { + use server_api::ServerApi; + let workspace = tempfile::tempdir().unwrap(); + let api = test_api(workspace.path()).await; + seed_worker_source_member(&api, "offline-runtime", "controller"); + seed_worker_source_member(&api, "offline-runtime", "subject"); + let service = ServerApiContractService::Workspace(api.clone()); + let context = || { + operation_request_context(Some(server_api::ServerRuntimeSource { + runtime_id: "offline-runtime".into(), + worker_id: Some("controller".into()), + })) + }; + let input = || server_api::RuntimeWorkerInputRequest { + kind: Some("user".into()), + content: "must not execute".into(), + segments: None, + }; + assert!( + service + .runtime_worker_input_alias( + context(), + "offline-runtime".into(), + "subject".into(), + input() + ) + .await + .is_err() + ); + assert!( + service + .runtime_worker_input( + context(), + TEST_WORKSPACE_ID.into(), + "offline-runtime".into(), + "subject".into(), + input() + ) + .await + .is_err() + ); + assert!( + service + .runtime_worker_restore_alias( + context(), + "offline-runtime".into(), + "subject".into(), + server_api::WorkerRestoreRequest { + expected_observation_token: "old".into(), + request_id: "unauthorized".into(), + } + ) + .await + .is_err() + ); + let lifecycle = || server_api::RuntimeWorkerLifecycleRequest { + reason: None, + ticket_assignment: None, + }; + assert!( + service + .runtime_worker_stop_alias( + context(), + "offline-runtime".into(), + "subject".into(), + lifecycle() + ) + .await + .is_err() + ); + assert!( + service + .runtime_worker_cancel_alias( + context(), + "offline-runtime".into(), + "subject".into(), + lifecycle() + ) + .await + .is_err() + ); + assert!( + service + .runtime_worker_pin( + context(), + TEST_WORKSPACE_ID.into(), + "offline-runtime".into(), + "subject".into() + ) + .await + .is_err() + ); + let cleanup = service + .runtime_cleanup_execute( + context(), + TEST_WORKSPACE_ID.into(), + "offline-runtime".into(), + ExecuteRuntimeCleanupRequest { + expected_plan_revision: "irrelevant".into(), + expected_plan_digest: "irrelevant".into(), + worker_target_ids: Vec::new(), + workdir_target_ids: Vec::new(), + confirm_dirty_discard_target_ids: Vec::new(), + }, + ) + .await + .unwrap_err(); + assert!(cleanup.message.contains("target-bound WorkerRemove")); + assert_eq!( + api.store + .get_worker_registry( + TEST_WORKSPACE_ID, + &RuntimeWorkerRef::new("offline-runtime", "subject") + ) + .unwrap() + .unwrap() + .retention_state, + "normal" + ); +} + +#[test] +fn uncertain_unary_input_keeps_a_distinct_diagnostic_without_claiming_acceptance() { + let result = worker_input_result_to_api(WorkerInputResult { + state: InternalWorkerOperationState::Rejected, + disposition: WorkerInputDisposition::Unknown, + worker: RuntimeWorkerRef::new("runtime-a", "worker-a"), + runtime_run_id: None, + diagnostics: Vec::new(), + }); + assert_eq!(result.state, server_api::WorkerOperationState::Rejected); + assert!( + result + .diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "worker_input_outcome_unknown") + ); +} + +#[tokio::test] +async fn grant_replacement_while_admission_waits_cannot_use_another_grants_lock() { + let workspace = tempfile::tempdir().unwrap(); + let api = test_api(workspace.path()).await; + let controller = RuntimeWorkerRef::new("offline-runtime", "controller"); + let subject = RuntimeWorkerRef::new("offline-runtime", "subject"); + for identity in [&controller, &subject] { + seed_worker_source_member(&api, &identity.runtime_id, &identity.worker_id); + } + let grant = |id: &str| WorkerControlGrantRecord { + workspace_id: TEST_WORKSPACE_ID.into(), + grant_id: id.into(), + controller: controller.clone(), + subject: subject.clone(), + relation: "spawned".into(), + origin: "test".into(), + permissions: vec!["pin".into()], + operation_id: id.into(), + created_at: now_registry_timestamp(), + revoked_at: None, + }; + api.store + .create_worker_control_grant(&grant("original-grant")) + .unwrap(); + let guard = worker_control_lock(&api, "original-grant") + .lock_owned() + .await; + let handle = WorkspaceWorker::resolve(&api, &subject.runtime_id, &subject.worker_id).unwrap(); + let context = WorkerOperationContext::WorkerControl { + controller: controller.clone(), + }; + let admission = handle.set_pinned(&context, true); + tokio::pin!(admission); + // Poll deterministically to the held lock before replacing grant authority. + assert!(futures::poll!(admission.as_mut()).is_pending()); + api.store + .revoke_worker_control_grant( + TEST_WORKSPACE_ID, + "original-grant", + &now_registry_timestamp(), + ) + .unwrap(); + api.store + .create_worker_control_grant(&grant("replacement-grant")) + .unwrap(); + drop(guard); + assert!(matches!( + admission.await.unwrap_err().error, + Error::RepositoryConflict(_) + )); + assert_eq!( + api.store + .get_worker_registry(TEST_WORKSPACE_ID, &subject) + .unwrap() + .unwrap() + .retention_state, + "normal" + ); +} + +#[tokio::test] +async fn browser_protocol_connection_and_cleanup_reject_cookie_origin_mismatch() { + let workspace = tempfile::tempdir().unwrap(); + let (api, execution) = test_api_with_recording_backend(workspace.path()).await; + let identity = spawn_ticket_check_source(&api, "operation-origin"); + let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap(); + let (mut context, _) = browser_operation_context(&api).await; + let WorkerOperationContext::Browser { headers, .. } = &mut context else { + unreachable!() + }; + headers.insert(ORIGIN, "https://untrusted.example".parse().unwrap()); + assert!(matches!( + handle.connect_protocol(&context).await, + Err(Error::WorkspacePermissionDenied(_)) + )); + let result = execute_runtime_cleanup_with_context( + &api, + EMBEDDED_WORKER_RUNTIME_ID, + ExecuteRuntimeCleanupRequest { + expected_plan_revision: "irrelevant".into(), + expected_plan_digest: "irrelevant".into(), + worker_target_ids: Vec::new(), + workdir_target_ids: Vec::new(), + confirm_dirty_discard_target_ids: Vec::new(), + }, + &context, + ) + .await + .unwrap_err(); + assert!(matches!(result.error, Error::WorkspacePermissionDenied(_))); + assert!(execution.protocol_methods().is_empty()); +} + +fn restore_request_for( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + request_id: &str, +) -> server_api::WorkerRestoreRequest { + server_api::WorkerRestoreRequest { + expected_observation_token: api + .runtime + .worker(worker) + .unwrap() + .restore_observation_token + .unwrap(), + request_id: request_id.into(), + } +} + +#[tokio::test] +async fn restore_guard_rejects_stale_list_and_stopped_restored_stopped_aba() { + let workspace = tempfile::tempdir().unwrap(); + let (api, execution) = test_api_with_recording_backend(workspace.path()).await; + execution.accept_restores(); + let identity = spawn_ticket_check_source(&api, "restore-guard-aba"); + let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap(); + handle + .stop(&WorkerOperationContext::Backend, operation_lifecycle()) + .await + .unwrap(); + let original = restore_request_for(&api, &identity, "restore-original"); + let stale_list = server_api::WorkerRestoreRequest { + request_id: "restore-stale-list".into(), + ..original.clone() + }; + assert_eq!( + handle + .restore(&WorkerOperationContext::Backend, original.clone()) + .await + .unwrap() + .state, + server_api::WorkerRestoreState::Accepted + ); + let conflict = handle + .restore(&WorkerOperationContext::Backend, stale_list.clone()) + .await + .unwrap_err(); + assert!(matches!(conflict.error, Error::RestoreObservationConflict)); + let typed = conflict.into_repository_api_error(); + assert_eq!( + api_error_status(&Error::RestoreObservationConflict), + StatusCode::CONFLICT + ); + assert_eq!(typed.error, "restore_observation_conflict"); + handle + .stop(&WorkerOperationContext::Backend, operation_lifecycle()) + .await + .unwrap(); + assert_ne!( + original.expected_observation_token, + api.runtime + .worker(&identity) + .unwrap() + .restore_observation_token + .unwrap() + ); + assert!(matches!( + handle + .restore(&WorkerOperationContext::Backend, stale_list) + .await + .unwrap_err() + .error, + Error::RestoreObservationConflict + )); + assert_eq!(api.runtime.worker(&identity).unwrap().state, "stopped"); + assert!( + execution.contexts.lock().unwrap().is_empty(), + "conflict must not install execution" + ); + // The original request is result replay, not a new intent; it stays accepted + // without resurrecting the Worker after a subsequent stop. + assert_eq!( + handle + .restore(&WorkerOperationContext::Backend, original) + .await + .unwrap() + .state, + server_api::WorkerRestoreState::Accepted + ); + assert_eq!(api.runtime.worker(&identity).unwrap().state, "stopped"); + assert!(execution.contexts.lock().unwrap().is_empty()); +} + +fn restore_test_workdir( + api: &WorkspaceApi, + id: &str, + source: WorkdirRegistrySource, +) -> WorkdirRegistryRecord { + WorkdirRegistryRecord { + workspace_id: api.workspace_id().into(), + workdir_id: id.into(), + display_name: None, + source, + creation_selector: None, + creation_ref: None, + creation_tree: None, + current_selector: None, + current_ref: None, + current_tree: None, + observed_at_epoch_seconds: None, + materialization_status: "present".into(), + cleanliness: "clean".into(), + created_at: TEST_CREATED_AT.into(), + updated_at: TEST_CREATED_AT.into(), + } +} + +fn attach_restore_test_workdir( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + workdir: &WorkdirRegistryRecord, +) { + api.store + .attach_worker_workdir(&WorkerWorkdirLinkRecord { + connection_id: String::new(), + workspace_id: api.workspace_id().into(), + worker: worker.clone(), + workdir_id: workdir.workdir_id.clone(), + alias: workdir.workdir_id.clone(), + capabilities: workdir::WorkdirSessionCapabilities::READ_ONLY, + linked_at: TEST_CREATED_AT.into(), + unlinked_at: None, + }) + .unwrap(); +} + +fn displace_restore_singleton( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + replacement: &RuntimeWorkerRef, +) { + api.config_store.with_conn(|conn| { + conn.execute("UPDATE worker_create_reservations SET singleton_key='restore-test-singleton', singleton_generation=1 WHERE workspace_id=?1 AND worker_id=?2", + rusqlite::params![api.workspace_id(), worker.worker_id])?; + conn.execute("INSERT INTO worker_singleton_owners(workspace_id,singleton_key,runtime_id,worker_id,generation,created_at,updated_at) VALUES(?1,'restore-test-singleton',?2,?3,2,?4,?4)", + rusqlite::params![api.workspace_id(), replacement.runtime_id, replacement.worker_id, TEST_CREATED_AT])?; + Ok(()) + }).unwrap(); + assert!( + api.store + .require_current_worker_singleton_owner(api.workspace_id(), worker) + .is_err() + ); +} + +fn add_offline_restore_workdir(api: &WorkspaceApi, worker: &RuntimeWorkerRef) { + let record = restore_test_workdir( + api, + "restore-offline", + WorkdirRegistrySource::ExternalGrant { + grant_id: "restore-offline-grant".into(), + }, + ); + api.store + .create_external_workdir_grant( + &crate::store::ExternalWorkdirGrantRecord { + grant_id: "restore-offline-grant".into(), + workspace_id: api.workspace_id().into(), + workdir_id: record.workdir_id.clone(), + provider_instance_id: "offline-provider".into(), + display_name: "Offline grant".into(), + permissions: "read_only".into(), + created_by: "owner-account".into(), + created_at: TEST_CREATED_AT.into(), + expires_at: None, + generation: 1, + status: "offline".into(), + updated_at: TEST_CREATED_AT.into(), + }, + &record, + ) + .unwrap(); + attach_restore_test_workdir(api, worker, &record); +} + +#[tokio::test] +async fn restore_guard_stale_ssh_intent_does_not_issue_workspace_resource() { + let workspace = tempfile::tempdir().unwrap(); + let api = test_api_with_remote_repository(workspace.path()).await; + let identity = spawn_ticket_check_source(&api, "restore-stale-ssh"); + let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap(); + let stale = restore_request_for(&api, &identity, "stale-ssh-intent"); + handle + .stop(&WorkerOperationContext::Backend, operation_lifecycle()) + .await + .unwrap(); + api.repository_secrets + .generate_credential( + api.workspace_id(), + GenerateRepositorySshCredentialRequest { + operation_id: "restore-default-key".into(), + credential_id: + crate::repository_access::WORKSPACE_DEFAULT_REPOSITORY_SSH_CREDENTIAL_ID.into(), + name: "Restore test default".into(), + }, + "owner-account", + ) + .unwrap(); + let public_key = api + .repository_secrets + .credential_public_key( + api.workspace_id(), + crate::repository_access::WORKSPACE_DEFAULT_REPOSITORY_SSH_CREDENTIAL_ID, + ) + .unwrap() + .unwrap() + .public_key; + api.repository_secrets + .put_host_trust( + api.workspace_id(), + PutRepositorySshHostTrustRequest { + operation_id: "restore-ssh-trust".into(), + host_trust_id: "restore-ssh-host".into(), + hostname: "example.invalid".into(), + port: 22, + host_key: public_key, + expected_revision: None, + }, + "owner-account", + ) + .unwrap(); + let record = restore_test_workdir( + &api, + "restore-ssh-workdir", + WorkdirRegistrySource::Repository { + runtime_id: identity.runtime_id.clone(), + repository_id: test_repository_id(&api), + }, + ); + api.store.upsert_workdir_registry(&record).unwrap(); + attach_restore_test_workdir(&api, &identity, &record); + // Prove this is a fully issuable SSH fixture; the old pre-admission path would mint another handle. + assert!( + repository_access_request_for_workdir( + &api, + &identity.runtime_id, + &record.workdir_id, + "restore-ssh-positive-control" + ) + .unwrap() + .is_some() + ); + let before = api.resource_broker.repository_ssh_access_handle_count(); + assert!(before > 0); + let conflict = handle + .restore(&WorkerOperationContext::Backend, stale) + .await + .unwrap_err(); + assert!(matches!(conflict.error, Error::RestoreObservationConflict)); + assert_eq!( + api.resource_broker.repository_ssh_access_handle_count(), + before + ); + assert_eq!(api.runtime.worker(&identity).unwrap().state, "stopped"); +} + +#[tokio::test] +async fn restore_guard_historical_replay_ignores_new_eligibility_but_rechecks_auth() { + let workspace = tempfile::tempdir().unwrap(); + let (api, execution) = test_api_with_recording_backend(workspace.path()).await; + execution.accept_restores(); + let identity = spawn_ticket_check_source(&api, "restore-historical"); + let replacement = spawn_ticket_check_source(&api, "restore-replacement"); + let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap(); + handle + .stop(&WorkerOperationContext::Backend, operation_lifecycle()) + .await + .unwrap(); + let request = restore_request_for(&api, &identity, "restore-historical-request"); + assert_eq!( + handle + .restore(&WorkerOperationContext::Backend, request.clone()) + .await + .unwrap() + .state, + server_api::WorkerRestoreState::Accepted + ); + handle + .stop(&WorkerOperationContext::Backend, operation_lifecycle()) + .await + .unwrap(); + displace_restore_singleton(&api, &identity, &replacement); + add_offline_restore_workdir(&api, &identity); + let before = execution.restore_operations.lock().unwrap().len(); + let replay = handle + .restore(&WorkerOperationContext::Backend, request.clone()) + .await + .unwrap(); + assert_eq!(replay.state, server_api::WorkerRestoreState::Accepted); + assert_eq!( + replay.worker.unwrap().state, + "stopped", + "never return the old idle receipt summary" + ); + assert_eq!(execution.restore_operations.lock().unwrap().len(), before); + let projected = restore_runtime_worker_with_context( + api.clone(), + identity.runtime_id.clone(), + identity.worker_id.clone(), + WorkerOperationContext::Backend, + request.clone(), + ) + .await + .unwrap() + .0; + assert_eq!( + projected.result.worker.unwrap().state, + "unavailable", + "current fenced projection, never old idle receipt" + ); + let (mut browser, _) = browser_operation_context(&api).await; + let WorkerOperationContext::Browser { headers, .. } = &mut browser else { + unreachable!() + }; + headers.insert(ORIGIN, "https://untrusted.example".parse().unwrap()); + assert!(matches!( + handle.restore(&browser, request).await.unwrap_err().error, + Error::WorkspacePermissionDenied(_) + )); + assert_eq!(execution.restore_operations.lock().unwrap().len(), before); + assert!( + handle + .restore( + &WorkerOperationContext::Backend, + restore_request_for(&api, &identity, "new-ineligible-intent") + ) + .await + .is_err() + ); +} + +#[tokio::test] +async fn restore_guard_pending_recovery_ignores_new_eligibility_and_reuses_operation() { + let workspace = tempfile::tempdir().unwrap(); + let (api, execution) = test_api_with_recording_backend(workspace.path()).await; + execution.accept_restores(); + let identity = spawn_ticket_check_source(&api, "restore-pending"); + let replacement = spawn_ticket_check_source(&api, "restore-pending-replacement"); + let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap(); + handle + .stop(&WorkerOperationContext::Backend, operation_lifecycle()) + .await + .unwrap(); + *execution.restore_failure.lock().unwrap() = Some( + worker_runtime::execution::WorkerExecutionSpawnResult::Errored( + worker_runtime::execution::WorkerExecutionResult::rejected( + worker_runtime::execution::WorkerExecutionOperation::Restore, + "uncertain launch", + ), + ), + ); + let request = restore_request_for(&api, &identity, "restore-pending-request"); + assert_eq!( + handle + .restore(&WorkerOperationContext::Backend, request.clone()) + .await + .unwrap() + .state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + let original = execution.restore_operations.lock().unwrap()[0]; + displace_restore_singleton(&api, &identity, &replacement); + add_offline_restore_workdir(&api, &identity); + assert_eq!( + handle + .restore(&WorkerOperationContext::Backend, request) + .await + .unwrap() + .state, + server_api::WorkerRestoreState::Accepted + ); + assert_eq!( + execution.reconcile_operations.lock().unwrap().as_slice(), + &[original] + ); + assert!( + execution + .restore_operations + .lock() + .unwrap() + .iter() + .all(|id| id == &original) + ); +} + +#[tokio::test] +async fn restore_guard_internal_stale_conflict_retires_intent_for_later_explicit_restore() { + let workspace = tempfile::tempdir().unwrap(); + let (mut api, execution) = test_api_with_recording_backend(workspace.path()).await; + execution.accept_restores(); + for domain in ["orchestrator-restore", "subject-restore"] { + let identity = spawn_ticket_check_source(&api, domain); + let handle = + WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap(); + handle + .stop(&WorkerOperationContext::Backend, operation_lifecycle()) + .await + .unwrap(); + let observed = api.runtime.worker(&identity).unwrap(); + let pinned = api + .config_store + .pin_internal_worker_restore_intent( + api.workspace_id(), + &identity, + domain, + observed.restore_observation_token.as_deref(), + None, + ) + .unwrap(); + // A distinct operation wins between the internal observation/pin and admission. + let competing = restore_request_for(&api, &identity, &format!("{domain}-competing")); + assert_eq!( + handle + .restore(&WorkerOperationContext::Backend, competing) + .await + .unwrap() + .state, + server_api::WorkerRestoreState::Accepted + ); + handle + .stop(&WorkerOperationContext::Backend, operation_lifecycle()) + .await + .unwrap(); + let count = execution.restore_operations.lock().unwrap().len(); + let error = restore_internal_worker_intent(&api, &identity, domain.into()).unwrap_err(); + assert!(matches!(error.error, Error::RestoreObservationConflict)); + assert_eq!( + execution.restore_operations.lock().unwrap().len(), + count, + "a conflict must not automatically exchange observation and retry" + ); + assert_eq!(api.runtime.worker(&identity).unwrap().state, "stopped"); + // Retirement, unlike in-memory refresh, must survive caller journal restart. + api.config_store = + Arc::new(SqliteWorkspaceStore::open(api.config.database_path.clone()).unwrap()); + let fresh = api.runtime.worker(&identity).unwrap(); + let next = api + .config_store + .pin_internal_worker_restore_intent( + api.workspace_id(), + &identity, + domain, + fresh.restore_observation_token.as_deref(), + None, + ) + .unwrap(); + assert_ne!(pinned.request_id, next.request_id); + assert_ne!( + pinned.expected_observation_token, + next.expected_observation_token + ); + assert_eq!( + next.expected_observation_token, + fresh.restore_observation_token.unwrap() + ); + // Only this separate deliberate invocation dispatches the new intent. + assert_eq!( + restore_internal_worker_intent(&api, &identity, domain.into()) + .unwrap() + .state, + server_api::WorkerRestoreState::Accepted + ); + assert_eq!( + execution.restore_operations.lock().unwrap().len(), + count + 1 + ); + // Explicit historical identity is immutable, never rewritten to the new token. + let original = api + .config_store + .pin_internal_worker_restore_intent( + api.workspace_id(), + &identity, + domain, + None, + Some(&pinned.request_id), + ) + .unwrap(); + assert_eq!(original, pinned); + } +} + +#[tokio::test] +async fn restore_guard_internal_conflict_same_generation_has_distinct_fresh_intents() { + let workspace = tempfile::tempdir().unwrap(); + let (api, execution) = test_api_with_recording_backend(workspace.path()).await; + execution.accept_restores(); + let identity = spawn_ticket_check_source(&api, "internal-other-owner-pending"); + let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap(); + handle + .stop(&WorkerOperationContext::Backend, operation_lifecycle()) + .await + .unwrap(); + let original = restore_request_for(&api, &identity, "original-pending-owner"); + *execution.restore_failure.lock().unwrap() = Some( + worker_runtime::execution::WorkerExecutionSpawnResult::Errored( + worker_runtime::execution::WorkerExecutionResult::errored( + worker_runtime::execution::WorkerExecutionOperation::Restore, + "unknown delivery", + ), + ), + ); + assert_eq!( + handle + .restore(&WorkerOperationContext::Backend, original.clone()) + .await + .unwrap() + .state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + let operation = execution.restore_operations.lock().unwrap()[0]; + let observed = api.runtime.worker(&identity).unwrap(); + let count = execution.restore_operations.lock().unwrap().len(); + for domain in ["orchestrator-restore", "subject-restore"] { + let mut prior = None; + for _ in 0..2 { + let pinned = api + .config_store + .pin_internal_worker_restore_intent( + api.workspace_id(), + &identity, + domain, + observed.restore_observation_token.as_deref(), + None, + ) + .unwrap(); + if let Some(prior) = prior.replace(pinned.clone()) { + assert_ne!(pinned.request_id, prior.request_id); + assert_eq!( + pinned.expected_observation_token, + prior.expected_observation_token + ); + } + let error = restore_internal_worker_intent(&api, &identity, domain.into()).unwrap_err(); + assert!(matches!(error.error, Error::RestoreObservationConflict)); + assert_eq!(execution.restore_operations.lock().unwrap().len(), count); + } + } + // Retiring competing rejected intents does not retire or replace an admitted owner. + assert_eq!( + handle + .restore(&WorkerOperationContext::Backend, original) + .await + .unwrap() + .state, + server_api::WorkerRestoreState::Accepted + ); + assert_eq!( + execution.reconcile_operations.lock().unwrap().as_slice(), + &[operation] + ); + assert!( + execution + .restore_operations + .lock() + .unwrap() + .iter() + .all(|id| *id == operation) + ); +} + +#[tokio::test] +async fn restore_guard_internal_intent_survives_client_store_restart_without_replacing_token() { + let workspace = tempfile::tempdir().unwrap(); + let (mut api, execution) = test_api_with_recording_backend(workspace.path()).await; + execution.accept_restores(); + let identity = spawn_ticket_check_source(&api, "restore-internal-restart"); + WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id) + .unwrap() + .stop(&WorkerOperationContext::Backend, operation_lifecycle()) + .await + .unwrap(); + let observed = api.runtime.worker(&identity).unwrap(); + *execution.restore_failure.lock().unwrap() = Some( + worker_runtime::execution::WorkerExecutionSpawnResult::Errored( + worker_runtime::execution::WorkerExecutionResult::rejected( + worker_runtime::execution::WorkerExecutionOperation::Restore, + "uncertain launch", + ), + ), + ); + assert_eq!( + restore_internal_observed_worker(&api, &observed, "subject-restore".into()) + .unwrap() + .state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + let original = execution.restore_operations.lock().unwrap()[0]; + let current = api.runtime.worker(&identity).unwrap(); + assert_ne!( + observed.restore_observation_token, + current.restore_observation_token + ); + // Reopen the durable caller journal, discarding all client-side memory of request identity. + api.config_store = + Arc::new(SqliteWorkspaceStore::open(api.config.database_path.clone()).unwrap()); + let pinned = api + .config_store + .pin_internal_worker_restore_intent( + api.workspace_id(), + &identity, + "subject-restore", + current.restore_observation_token.as_deref(), + None, + ) + .unwrap(); + assert_eq!( + pinned.expected_observation_token, + observed.restore_observation_token.unwrap() + ); + assert_eq!( + restore_internal_observed_worker(&api, ¤t, "subject-restore".into()) + .unwrap() + .state, + server_api::WorkerRestoreState::Accepted + ); + assert_eq!( + execution.reconcile_operations.lock().unwrap().as_slice(), + &[original] + ); + assert!( + execution + .restore_operations + .lock() + .unwrap() + .iter() + .all(|id| id == &original) + ); +} + +#[tokio::test] +async fn restore_guard_workspace_interrupted_preparation_recovers_admitted_snapshot() { + let workspace = tempfile::tempdir().unwrap(); + let (api, execution) = test_api_with_recording_backend(workspace.path()).await; + execution.accept_restores(); + let identity = spawn_ticket_check_source(&api, "restore-prepare-interrupted"); + let replacement = spawn_ticket_check_source(&api, "restore-prepare-replacement"); + let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap(); + handle + .stop(&WorkerOperationContext::Backend, operation_lifecycle()) + .await + .unwrap(); + let request = restore_request_for(&api, &identity, "restore-prepare-interrupted-request"); + let preparation = runtime_api::WorkerRestorePreparation { + workspace_api: Some( + runtime_contract_restore(api.workspace_api_ref(&identity.runtime_id)).unwrap(), + ), + workdir_attachments: Some(Vec::new()), + ..Default::default() + }; + let (result, retained) = api + .runtime + .coordinate_worker_restore( + &identity, + runtime_api::WorkerRestoreCoordinationRequest { + expected_observation_token: request.expected_observation_token.clone(), + request_id: request.request_id.clone(), + preparation: Some(preparation.clone()), + }, + ) + .unwrap(); + assert!(result.is_none()); + assert_eq!(retained, Some(preparation)); + assert!(execution.restore_operations.lock().unwrap().is_empty()); + // Model interruption between admission and completion, then changed current eligibility. + displace_restore_singleton(&api, &identity, &replacement); + add_offline_restore_workdir(&api, &identity); + let restored = handle + .restore(&WorkerOperationContext::Backend, request.clone()) + .await + .unwrap(); + assert_eq!(restored.state, server_api::WorkerRestoreState::Accepted); + assert_eq!(execution.restore_operations.lock().unwrap().len(), 1); + let original = execution.restore_operations.lock().unwrap()[0]; + assert!( + execution.reconcile_operations.lock().unwrap().is_empty(), + "no execution existed to reconcile yet" + ); + assert_eq!( + handle + .restore(&WorkerOperationContext::Backend, request) + .await + .unwrap() + .state, + server_api::WorkerRestoreState::Accepted + ); + assert_eq!( + execution.restore_operations.lock().unwrap().as_slice(), + &[original] + ); +} + +#[tokio::test] +async fn restore_guard_cross_runtime_ssh_completion_retries_pinned_snapshot_at_workdir_owner() { + let workspace = tempfile::tempdir().unwrap(); + let mut config = test_server_config(workspace.path()); + let source = server_api::RepositorySource { + kind: server_api::RepositorySourceKind::Ssh, + uri: "git@example.invalid:owner/repository.git".into(), + }; + config.repositories[0].source_fingerprint = + crate::repository_source::repository_source_fingerprint(&source); + config.repositories[0].source = source; + config.repositories[0].path = None; + let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap(); + let execution = Arc::new(DeterministicExecutionBackend::default()); + execution.accept_restores(); + let api = WorkspaceApi::new_with_execution_backend(config, Arc::new(store), execution.clone()) + .await + .unwrap(); + let peer = WorkdirlessFixtureRuntime::default(); + *peer.reject_repository_access.lock().unwrap() = true; + api.runtime.register_or_replace(peer.clone()); + let identity = spawn_ticket_check_source(&api, "restore-cross-runtime-ssh"); + let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap(); + handle + .stop(&WorkerOperationContext::Backend, operation_lifecycle()) + .await + .unwrap(); + let original_key = api + .repository_secrets + .generate_credential( + api.workspace_id(), + GenerateRepositorySshCredentialRequest { + operation_id: "cross-runtime-default-key".into(), + credential_id: + crate::repository_access::WORKSPACE_DEFAULT_REPOSITORY_SSH_CREDENTIAL_ID.into(), + name: "Restore default".into(), + }, + "owner-account", + ) + .unwrap(); + let public_key = api + .repository_secrets + .credential_public_key(api.workspace_id(), &original_key.credential_id) + .unwrap() + .unwrap() + .public_key; + api.repository_secrets + .put_host_trust( + api.workspace_id(), + PutRepositorySshHostTrustRequest { + operation_id: "cross-runtime-trust".into(), + host_trust_id: "cross-runtime-host".into(), + hostname: "example.invalid".into(), + port: 22, + host_key: public_key, + expected_revision: None, + }, + "owner-account", + ) + .unwrap(); + let record = restore_test_workdir( + &api, + "cross-runtime-restore-workdir", + WorkdirRegistrySource::Repository { + runtime_id: WorkdirlessFixtureRuntime::RUNTIME_ID.into(), + repository_id: test_repository_id(&api), + }, + ); + peer.set_workdir_repository(&record.workdir_id, &test_repository_id(&api)); + api.store.upsert_workdir_registry(&record).unwrap(); + attach_restore_test_workdir(&api, &identity, &record); + let request = restore_request_for(&api, &identity, "cross-runtime-restore-request"); + assert_eq!( + handle + .restore(&WorkerOperationContext::Backend, request.clone()) + .await + .unwrap() + .state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + assert!( + execution.restore_operations.lock().unwrap().is_empty(), + "credential completion must precede execution" + ); + let first = peer.repository_access_requests.lock().unwrap()[0].clone(); + assert_eq!( + first.materialization.runtime_id, + WorkdirlessFixtureRuntime::RUNTIME_ID + ); + assert_ne!(first.materialization.runtime_id, identity.runtime_id); + // Rotate current host trust while the accepted operation is uncertain. Its + // original credential/trust revisions, source and binding must remain pinned. + let alternate = api + .repository_secrets + .generate_credential( + api.workspace_id(), + GenerateRepositorySshCredentialRequest { + operation_id: "cross-runtime-alternate-key".into(), + credential_id: "cross-runtime-alternate".into(), + name: "Alternate".into(), + }, + "owner-account", + ) + .unwrap(); + let alternate_public = api + .repository_secrets + .credential_public_key(api.workspace_id(), &alternate.credential_id) + .unwrap() + .unwrap() + .public_key; + let original_trust_revision = first + .materialization + .ssh + .as_ref() + .unwrap() + .host_trust_revision; + api.repository_secrets + .put_host_trust( + api.workspace_id(), + PutRepositorySshHostTrustRequest { + operation_id: "cross-runtime-trust-rotate".into(), + host_trust_id: "cross-runtime-host".into(), + hostname: "example.invalid".into(), + port: 22, + host_key: alternate_public, + expected_revision: Some(original_trust_revision), + }, + "owner-account", + ) + .unwrap(); + *peer.reject_repository_access.lock().unwrap() = false; + assert_eq!( + handle + .restore(&WorkerOperationContext::Backend, request.clone()) + .await + .unwrap() + .state, + server_api::WorkerRestoreState::Accepted + ); + let attempts = peer.repository_access_requests.lock().unwrap(); + assert_eq!(attempts.len(), 2); + let second = &attempts[1]; + assert_eq!( + second.materialization.operation_id, + first.materialization.operation_id + ); + let mut normalized = second.clone(); + let old_ssh = first.materialization.ssh.as_ref().unwrap(); + let new_ssh = normalized.materialization.ssh.as_mut().unwrap(); + assert_ne!(new_ssh.secret_resource.nonce, old_ssh.secret_resource.nonce); + new_ssh.secret_resource = old_ssh.secret_resource.clone(); + new_ssh.expires_at_epoch_seconds = old_ssh.expires_at_epoch_seconds; + assert_eq!(normalized, first, "only opaque handle and expiry may renew"); + drop(attempts); + assert_eq!(execution.restore_operations.lock().unwrap().len(), 1); + assert_eq!( + handle + .restore(&WorkerOperationContext::Backend, request) + .await + .unwrap() + .state, + server_api::WorkerRestoreState::Accepted + ); + assert_eq!( + peer.repository_access_requests.lock().unwrap().len(), + 2, + "terminal receipt must not renew access" + ); +} diff --git a/crates/workspace-server/src/server/worker_operations.rs b/crates/workspace-server/src/server/worker_operations.rs new file mode 100644 index 000000000..754787c3c --- /dev/null +++ b/crates/workspace-server/src/server/worker_operations.rs @@ -0,0 +1,601 @@ +//! Workspace identity-bound operation reception. Runtime/controller remain execution authority. +use super::*; + +/// Caller authority supplied by authenticated Server adapters, never public JSON/identity hints. +/// The unchecked Backend variant exists only for lower-level regression fixtures. +#[derive(Clone)] +pub(crate) enum WorkerOperationContext { + #[cfg(test)] + Backend, + WorkerControl { + controller: RuntimeWorkerRef, + }, + Browser { + headers: HeaderMap, + source: protocol::AuthenticatedInputSource, + }, +} + +impl WorkerOperationContext { + pub(super) async fn authorize_workspace(&self, api: &WorkspaceApi) -> ApiResult<()> { + if let Self::Browser { headers, source } = self { + let actor = resolve_actor(&ServerAuthApi::from(api), headers) + .await? + .ok_or_else(|| { + Error::WorkspacePermissionDenied( + "Worker connection credentials were revoked".into(), + ) + })?; + if matches!(actor.auth_method, ActorAuthMethod::BrowserSession) { + let AuthConfig::Passkey { origin, .. } = &api.config.auth; + if headers.get(ORIGIN).and_then(|value| value.to_str().ok()) + != Some(origin.as_str()) + { + return Err(Error::WorkspacePermissionDenied( + "cross-origin Worker mutation denied".into(), + ) + .into()); + } + } + if authenticated_browser_input_source(&actor) != *source { + return Err(Error::WorkspacePermissionDenied( + "Worker connection Account changed".into(), + ) + .into()); + } + if !api + .store + .get_workspace(api.workspace_id()) + .await? + .is_some_and(|workspace| workspace.state == "active") + { + return Err(Error::WorkspacePermissionDenied( + "Workspace is not accepting Worker mutations".into(), + ) + .into()); + } + } + Ok(()) + } + pub(super) fn from_request( + api: &WorkspaceApi, + context: &server_api::ServerRequestContext, + ) -> std::result::Result { + if let Some(source) = context.runtime_source.as_ref() { + // Check hints against middleware's verified subject without replaying its proof + // or requiring the target Runtime to be online just to represent identity. + current_worker_contract_headers(context)?; + let controller = RuntimeWorkerRef::new( + &source.runtime_id, + source + .worker_id + .as_deref() + .expect("source binding required a Worker identity"), + ); + if api + .store + .get_worker_registry(api.workspace_id(), &controller) + .map_err(|error| ApiError::from(error).into_repository_api_error())? + .is_none() + { + return Err(ApiError::from(Error::UnknownWorker { worker: controller }) + .into_repository_api_error()); + } + return Ok(Self::WorkerControl { controller }); + } + let actor = context.actor.as_ref().ok_or_else(|| { + ApiError::from(Error::WorkspacePermissionDenied( + "authenticated Worker operation source required".into(), + )) + .into_repository_api_error() + })?; + let headers = contract_request_headers(context)?; + Ok(Self::Browser { + headers, + source: authenticated_browser_input_source(actor), + }) + } +} + +/// A registry identity, not an authorization lease or a snapshot of live execution. +#[derive(Clone)] +pub(crate) struct WorkspaceWorker { + api: Services, + removal: WorkerRemovalService, + identity: RuntimeWorkerRef, +} + +// The removal-only specialization keeps the installed embedded dispatcher from +// retaining WorkspaceApi -> Runtime -> dispatcher -> WorkspaceApi as a strong cycle. +// Both specializations resolve the same registry identity and call the same remove method. +impl WorkspaceWorker { + fn resolve_with_services( + api: Services, + removal: WorkerRemovalService, + runtime_id: &str, + reference: &str, + ) -> Result { + let direct = RuntimeWorkerRef::new(runtime_id, reference); + let identity = if let Some(record) = removal + .store + .get_worker_registry(&removal.workspace_id, &direct)? + { + record.worker + } else { + let worker = removal + .store + .resolve_worker_resource_reference(&removal.workspace_id, reference)? + .filter(|worker| worker.runtime_id == runtime_id) + .ok_or_else(|| Error::UnknownWorker { + worker: direct.clone(), + })?; + removal + .store + .get_worker_registry(&removal.workspace_id, &worker)? + .ok_or_else(|| Error::UnknownWorker { worker: direct })? + .worker + }; + Ok(Self { + api, + removal, + identity, + }) + } + pub(super) fn identity(&self) -> &RuntimeWorkerRef { + &self.identity + } + + fn current_record(&self) -> ApiResult { + self.removal + .store + .get_worker_registry(&self.removal.workspace_id, &self.identity)? + .ok_or_else(|| { + Error::UnknownWorker { + worker: self.identity.clone(), + } + .into() + }) + } + + pub(super) async fn remove( + &self, + source: crate::worker_source::VerifiedWorkerMutationSource, + reason: &str, + ) -> std::result::Result { + self.current_record() + .map_err(|error| error.error.to_string())?; + self.removal + .execute_async( + source, + &self.identity.runtime_id, + &self.identity.worker_id, + reason, + ) + .await + } +} + +/// Server-owned adapter for Runtime's independent embedded Tool dispatcher. +/// The proof was target-bound and verified before this synchronous transport seam. +pub(super) struct EmbeddedWorkspaceWorkerRemoveExecutor { + removal: WorkerRemovalService, +} +impl EmbeddedWorkspaceWorkerRemoveExecutor { + pub(super) fn new(api: &WorkspaceApi) -> Self { + Self { + removal: WorkerRemovalService::new(api), + } + } +} +impl crate::worker_source::VerifiedWorkerRemoveExecutor for EmbeddedWorkspaceWorkerRemoveExecutor { + fn execute( + &self, + source: crate::worker_source::VerifiedWorkerMutationSource, + target_runtime_id: &str, + target_worker_id: &str, + reason: &str, + ) -> std::result::Result { + let worker = match WorkspaceWorker::resolve_with_services( + (), + self.removal.clone(), + target_runtime_id, + target_worker_id, + ) { + Ok(worker) => worker, + Err(Error::UnknownWorker { .. }) => { + return Ok(worker_remove_error_response( + StatusCode::NOT_FOUND, + "unknown_worker", + "The target Worker is not known in this Workspace", + )); + } + Err(error) => return Err(error.to_string()), + }; + let reason = reason.to_owned(); + std::thread::spawn(move || { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| error.to_string())? + .block_on(worker.remove(source, &reason)) + }) + .join() + .map_err(|_| "embedded WorkerRemove executor thread panicked".to_owned())? + } +} + +impl WorkspaceWorker { + pub(crate) fn resolve(api: &WorkspaceApi, runtime_id: &str, reference: &str) -> Result { + Self::resolve_with_services( + api.clone(), + WorkerRemovalService::new(api), + runtime_id, + reference, + ) + } + + pub(super) async fn authorize_operation( + &self, + context: &WorkerOperationContext, + permission: &str, + ) -> ApiResult>> { + self.current_record()?; + match context { + #[cfg(test)] + WorkerOperationContext::Backend => Ok(None), + WorkerOperationContext::Browser { .. } => { + context.authorize_workspace(&self.api).await?; + Ok(None) + } + WorkerOperationContext::WorkerControl { controller } => { + if self + .api + .store + .get_worker_registry(self.api.workspace_id(), controller)? + .is_none() + { + return Err(Error::UnknownWorker { + worker: controller.clone(), + } + .into()); + } + let grant = authorize_known_worker_permission( + &self.api, + self.api.workspace_id(), + controller, + &self.identity, + permission, + )?; + let guard = worker_control_lock(&self.api, &grant.grant_id) + .lock_owned() + .await; + self.current_record()?; + let current_grant = authorize_known_worker_permission( + &self.api, + self.api.workspace_id(), + controller, + &self.identity, + permission, + )?; + if current_grant.grant_id != grant.grant_id { + return Err(Error::RepositoryConflict( + "Worker control grant changed while waiting for admission".into(), + ) + .into()); + } + Ok(Some(guard)) + } + } + } + + pub(super) async fn restore( + &self, + context: &WorkerOperationContext, + request: server_api::WorkerRestoreRequest, + ) -> ApiResult { + let _guard = self.authorize_operation(context, "restore").await?; + self.restore_internal(request) + } + + /// Trusted Backend callers still supply an explicit observed intent. Runtime + /// admission is shared with browser/Tool callers; this is not an unguarded launch. + pub(super) fn restore_internal( + &self, + request: server_api::WorkerRestoreRequest, + ) -> ApiResult { + self.current_record()?; + self.api.restore_workspace_worker(&self.identity, request) + } + + pub(super) async fn stop( + &self, + context: &WorkerOperationContext, + request: WorkerLifecycleRequest, + ) -> ApiResult { + let _guard = self.authorize_operation(context, "stop").await?; + let result = self + .api + .runtime + .stop_worker(&self.identity, request) + .map_err(|e| e.into_error())?; + if result.state != InternalWorkerOperationState::Accepted { + return Ok(result); + } + parse_runtime_worker_id_for_registry(&self.identity.worker_id)?; + let session_lock = current_worker_session_lock(&self.api, &self.identity); + let _session_guard = session_lock.lock().await; + close_current_worker_session_locked(&self.api, &self.identity).await?; + if let Some(record) = self + .api + .store + .get_worker_registry(self.api.workspace_id(), &self.identity)? + { + sync_linked_workdir_after_worker_stop(&self.api, &self.identity.runtime_id, &record)?; + } + Ok(result) + } + + pub(super) async fn cancel( + &self, + context: &WorkerOperationContext, + request: WorkerLifecycleRequest, + ) -> ApiResult { + let _guard = self.authorize_operation(context, "cancel").await?; + Ok(self + .api + .runtime + .cancel_worker(&self.identity, request) + .map_err(|e| e.into_error())?) + } + + pub(super) async fn input( + &self, + context: &WorkerOperationContext, + mut request: WorkerInputRequest, + ) -> ApiResult { + let permission = if request.kind == WorkerInputKind::Notify { + "notify" + } else { + "send_input" + }; + let _guard = self.authorize_operation(context, permission).await?; + self.ensure_interactive()?; + if matches!(context, WorkerOperationContext::WorkerControl { .. }) + && request.kind == WorkerInputKind::User + { + request.kind = WorkerInputKind::UserIfIdle; + } + Ok(self + .api + .runtime + .send_input(&self.identity, request) + .map_err(|e| e.into_error())?) + } + + fn ensure_interactive(&self) -> ApiResult<()> { + if self + .api + .store + .worker_registry_projection(self.api.workspace_id(), &self.identity)? + .and_then(|projection| projection.job) + .is_some() + { + return Err(Error::InvalidInput("Backend Job Workers accept only their Backend-owned immutable input; Console is read-only".into()).into()); + } + Ok(()) + } + + pub(super) async fn set_pinned( + &self, + context: &WorkerOperationContext, + pinned: bool, + ) -> ApiResult { + let _guard = self.authorize_operation(context, "pin").await?; + let retention_state = if pinned { "pinned" } else { "normal" }; + let changed = self.api.worker_projection.publish_ordered(|| { + self.api + .store + .update_worker_retention( + self.api.workspace_id(), + &self.identity, + retention_state, + now_registry_timestamp().as_str(), + ) + .map(|commit| (!commit.changes.is_empty(), commit)) + })?; + if !changed { + return Err(cleanup_api_error( + &self.identity.runtime_id, + "workspace_worker_retention_unknown_worker", + "Worker is not known to the Backend registry", + )); + } + Ok(WorkerRetentionResponse { + workspace_id: self.api.workspace_id().to_owned(), + runtime_id: self.identity.runtime_id.clone(), + worker_id: self.identity.worker_id.clone(), + pinned, + retention_state: retention_state.into(), + }) + } + + /// The caller owns the full cleanup-plan revision/digest check. The removal service + /// rechecks live assignment/retention/use at its existing commit boundary. + pub(super) async fn delete_from_plan( + &self, + context: &WorkerOperationContext, + candidate: &CleanupWorkerCandidate, + ) -> ApiResult<()> { + let _guard = self.authorize_operation(context, "remove").await?; + if candidate.runtime_id != self.identity.runtime_id + || candidate.runtime_worker_id != self.identity.worker_id + { + return Err(Error::InvalidInput( + "cleanup candidate does not address the resolved Worker".into(), + ) + .into()); + } + WorkerRemovalService::new(&self.api) + .execute_cleanup_removal(candidate) + .await + } + + pub(crate) async fn connect_protocol( + &self, + context: &WorkerOperationContext, + ) -> Result { + let _guard = self + .authorize_operation(context, "observe") + .await + .map_err(|error| error.error)?; + let source = match context { + WorkerOperationContext::Browser { source, .. } => Some(source), + _ => None, + }; + let connection = + connect_workspace_worker_protocol(&self.api, &self.identity, source).await?; + Ok(WorkspaceWorkerConnection { + sender: WorkspaceWorkerMethodSender { + worker: self.clone(), + methods: connection.methods, + }, + events: connection.events, + }) + } +} + +/// An execution connection is bound to exactly the identity which created it. No caller +/// can pair a different target handle with a subscription's raw sending capability. +#[derive(Clone)] +pub(crate) struct WorkspaceWorkerMethodSender { + worker: WorkspaceWorker, + methods: tokio::sync::mpsc::Sender, +} +pub(crate) struct WorkspaceWorkerConnection { + pub(crate) sender: WorkspaceWorkerMethodSender, + pub(crate) events: tokio::sync::mpsc::Receiver, +} +impl WorkspaceWorkerMethodSender { + pub(crate) async fn send( + &self, + context: &WorkerOperationContext, + method: protocol::Method, + ) -> Result<()> { + match method { + protocol::Method::Pause { command } => self.pause(context, command).await, + protocol::Method::Resume { command } => self.resume(context, command).await, + protocol::Method::Cancel { command } => self.cancel(context, command).await, + protocol::Method::Compact { command } => self.compact(context, command).await, + protocol::Method::Submit { + submission_request_id, + input, + } => self.submit(context, submission_request_id, input).await, + protocol::Method::Notify { + notification_request_id, + message, + } => self.notify(context, notification_request_id, message).await, + other => self.dispatch(context, other).await, + } + } + async fn dispatch( + &self, + context: &WorkerOperationContext, + method: protocol::Method, + ) -> Result<()> { + self.send_checked(context, method) + .await + .map_err(|error| error.error) + } + pub(crate) async fn pause( + &self, + context: &WorkerOperationContext, + command: protocol::WorkerCommandEnvelope, + ) -> Result<()> { + self.dispatch(context, protocol::Method::Pause { command }) + .await + } + pub(crate) async fn resume( + &self, + context: &WorkerOperationContext, + command: protocol::WorkerCommandEnvelope, + ) -> Result<()> { + self.dispatch(context, protocol::Method::Resume { command }) + .await + } + pub(crate) async fn cancel( + &self, + context: &WorkerOperationContext, + command: protocol::WorkerCommandEnvelope, + ) -> Result<()> { + self.dispatch(context, protocol::Method::Cancel { command }) + .await + } + pub(crate) async fn compact( + &self, + context: &WorkerOperationContext, + command: protocol::WorkerCommandEnvelope, + ) -> Result<()> { + self.dispatch(context, protocol::Method::Compact { command }) + .await + } + pub(crate) async fn submit( + &self, + context: &WorkerOperationContext, + submission_request_id: String, + input: Vec, + ) -> Result<()> { + self.dispatch( + context, + protocol::Method::Submit { + submission_request_id, + input, + }, + ) + .await + } + pub(crate) async fn notify( + &self, + context: &WorkerOperationContext, + notification_request_id: String, + message: String, + ) -> Result<()> { + self.dispatch( + context, + protocol::Method::Notify { + notification_request_id, + message, + }, + ) + .await + } + + async fn send_checked( + &self, + context: &WorkerOperationContext, + method: protocol::Method, + ) -> ApiResult<()> { + let _guard = self + .worker + .authorize_operation(context, "send_input") + .await?; + self.worker.ensure_interactive()?; + let WorkerOperationContext::Browser { source, .. } = context else { + return Err(Error::WorkspacePermissionDenied( + "Browser Worker protocol context required".into(), + ) + .into()); + }; + let method = authorize_browser_worker_method(method, source) + .map_err(|message| Error::InvalidInput(message.into()))?; + self.methods.send(method).await.map_err(|_| { + Error::RuntimeOperationFailed { + runtime_id: self.worker.identity.runtime_id.clone(), + code: "worker_protocol_closed".into(), + message: + "Worker execution connection closed; no implicit Restore or retry was performed" + .into(), + } + .into() + }) + } +} diff --git a/crates/workspace-server/src/server_workspace_config.rs b/crates/workspace-server/src/server_workspace_config.rs new file mode 100644 index 000000000..eb5935d75 --- /dev/null +++ b/crates/workspace-server/src/server_workspace_config.rs @@ -0,0 +1,652 @@ +//! Logical configuration operations share the durable attachment ledger and canonical CAS. +//! This module never opens a Runtime filesystem session or exports a host path. +//! Files support read/edit/write and non-entrypoint deletion; directories support +//! create, with atomic change sets (including non-entrypoint renames) on the root. +//! The canonical main.dcdl entrypoint cannot be deleted or renamed. +use super::*; +use server_api::{ + WorkspaceConfigAccess as Access, WorkspaceConfigApiError as Failure, + WorkspaceConfigFailureClassification as Classification, WorkspaceConfigNodeKind as NodeKind, +}; +use server_api::{ + WorkspaceConfigAttachRequest, WorkspaceConfigAttachment, WorkspaceConfigCommitRequest, + WorkspaceConfigCommitResponse, WorkspaceConfigGrantCreateRequest, WorkspaceConfigGrantResponse, + WorkspaceConfigNode, WorkspaceConfigObserveRequest, WorkspaceConfigObserveResponse, + WorkspaceConfigReadRequest, WorkspaceConfigReadResponse, +}; + +const ALIAS: &str = "workspace-config"; +const MAX_NODES: usize = 256; +const MAX_TEXT: usize = 256 * 1024; +const DEADLINE: std::time::Duration = std::time::Duration::from_secs(30); +type ConfigResult = std::result::Result; + +fn fail(status: u16, code: &str) -> Failure { + Failure { + status, + code: code.into(), + message: format!("Workspace configuration: {code}"), + classification: Classification::NotCommitted, + } +} +pub(super) fn pre_error(error: impl Into) -> Failure { + let error = error.into(); + let status = api_error_status(&error.error).as_u16(); + fail( + status, + match status { + 401 | 403 => "access_denied", + 404 => "not_found", + 409 => "conflict", + 413 => "limit_exceeded", + 500..=599 => "unavailable", + _ => "invalid_request", + }, + ) +} +fn mutation_error(error: Error) -> Failure { + // Storage/service failures at a mutation boundary may include a commit + // acknowledgement failure. Never describe those as a guaranteed rollback. + if matches!(&error, Error::Sqlite(_) | Error::Store(_)) { + unknown() + } else { + pre_error(error) + } +} +fn unknown() -> Failure { + Failure { + classification: Classification::Unknown, + ..fail(500, "outcome_unknown") + } +} +pub(super) fn identity( + api: &WorkspaceApi, + context: &server_api::ServerRequestContext, + workspace_id: &str, +) -> ConfigResult { + validate_workspace_scope(api, workspace_id).map_err(pre_error)?; + let headers = + current_worker_contract_headers(context).map_err(|_| fail(403, "access_denied"))?; + current_worker_identity(api, workspace_id, &headers).map_err(pre_error) +} +async fn lock( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, +) -> ConfigResult> { + tokio::time::timeout( + DEADLINE, + current_worker_session_lock(api, worker).lock_owned(), + ) + .await + .map_err(|_| fail(503, "busy")) +} +fn grant_for_worker( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, +) -> ConfigResult { + api.store + .current_workspace_config_grant(&api.config.workspace_id, worker) + .map_err(pre_error)? + .filter(|grant| !grant.revoked) + .ok_or_else(|| fail(403, "access_denied")) +} +fn access_capabilities(access: Access) -> workdir::WorkdirSessionCapabilities { + match access { + Access::ReadOnly => workdir::WorkdirSessionCapabilities::READ_ONLY, + Access::ReadWrite => workdir::WorkdirSessionCapabilities::READ_WRITE, + } +} +pub(super) fn grant_capabilities( + api: &WorkspaceApi, + grant_id: &str, +) -> Result { + let grant = api + .store + .get_workspace_config_grant(&api.config.workspace_id, grant_id)? + .filter(|grant| !grant.revoked) + .ok_or_else(|| { + Error::WorkspacePermissionDenied("Workspace config grant unavailable".into()) + })?; + Ok(access_capabilities(grant.access)) +} +pub(super) fn granted_to(api: &WorkspaceApi, worker: &RuntimeWorkerRef, grant_id: &str) -> bool { + grant_for_worker(api, worker).is_ok_and(|grant| grant.grant_id == grant_id) +} +fn active_link( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + grant: &WorkspaceConfigGrantResponse, + connection_id: Option<&str>, +) -> ConfigResult> { + if connection_id.is_some_and(|id| id.is_empty() || id.len() > 128) { + return Err(fail(400, "invalid_connection")); + } + let link = api + .store + // The T697 exclusive-active-Workdir ledger bounds this lookup to one + // current lifetime, regardless of accumulated detached history. + .list_workdir_worker_links(&api.config.workspace_id, &grant.working_directory_id) + .map_err(pre_error)? + .into_iter() + .find(|link| link.unlinked_at.is_none() && link.worker == *worker && link.alias == ALIAS); + if let Some(link) = &link { + let record = api + .store + .get_workdir_registry(&api.config.workspace_id, &link.workdir_id) + .map_err(pre_error)? + .ok_or_else(|| fail(403, "access_denied"))?; + if link.workdir_id != grant.working_directory_id + || !matches!(record.source, WorkdirRegistrySource::WorkspaceConfig { ref grant_id } if grant_id == &grant.grant_id) + { + return Err(fail(403, "access_denied")); + } + } + if let Some(id) = connection_id { + if !link.as_ref().is_some_and(|link| link.connection_id == id) { + return Err(fail(409, "connection_changed")); + } + } + Ok(link) +} +fn effective_access( + grant: &WorkspaceConfigGrantResponse, + link: &WorkerWorkdirLinkRecord, +) -> Access { + if grant.access == Access::ReadWrite + && link + .capabilities + .supports(workdir::WorkdirSessionCapability::Write) + && link + .capabilities + .supports(workdir::WorkdirSessionCapability::Edit) + { + Access::ReadWrite + } else { + Access::ReadOnly + } +} +fn attachment( + grant: &WorkspaceConfigGrantResponse, + link: &WorkerWorkdirLinkRecord, + already_attached: bool, +) -> WorkspaceConfigAttachment { + WorkspaceConfigAttachment { + workspace_id: grant.workspace_id.clone(), + connection_id: link.connection_id.clone(), + alias: ALIAS.into(), + working_directory_id: link.workdir_id.clone(), + access: effective_access(grant, link), + name: "Workspace configuration".into(), + purpose: "Explore and edit this Workspace's canonical configuration".into(), + content_path: "/workspace-config".into(), + already_attached, + } +} +pub(super) async fn get( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, +) -> ConfigResult> { + let _guard = lock(api, worker).await?; + let grant = grant_for_worker(api, worker)?; + Ok(active_link(api, worker, &grant, None)?.map(|link| attachment(&grant, &link, true))) +} +pub(super) async fn attach( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + request: WorkspaceConfigAttachRequest, +) -> ConfigResult { + if request.alias.as_deref().is_some_and(|alias| alias != ALIAS) { + return Err(fail(400, "invalid_alias")); + } + let _guard = lock(api, worker).await?; + let grant = grant_for_worker(api, worker)?; + let access = request.access.unwrap_or(grant.access); + if access == Access::ReadWrite && grant.access != Access::ReadWrite { + return Err(fail(403, "access_denied")); + } + if let Some(mut link) = active_link(api, worker, &grant, None)? { + // Same active alias/identity is idempotent; explicit write never silently downgrades. + if request.access == Some(Access::ReadWrite) + && effective_access(&grant, &link) != Access::ReadWrite + { + return Err(fail(403, "access_denied")); + } + if access == Access::ReadOnly && effective_access(&grant, &link) != Access::ReadOnly { + link.capabilities = access_capabilities(access); + link = api + .store + .attach_worker_workdir(&link) + .map_err(mutation_error)?; + } + return Ok(attachment(&grant, &link, true)); + } + let link = api + .store + .attach_worker_workdir(&WorkerWorkdirLinkRecord { + connection_id: String::new(), + workspace_id: grant.workspace_id.clone(), + worker: worker.clone(), + workdir_id: grant.working_directory_id.clone(), + alias: ALIAS.into(), + capabilities: access_capabilities(access), + linked_at: now_registry_timestamp(), + unlinked_at: None, + }) + .map_err(mutation_error)?; + // No Runtime filesystem/session synchronization for a logical config target. + api.worker_projection + .refresh(worker) + .map_err(|_| unknown())?; + Ok(attachment(&grant, &link, false)) +} +fn resolve( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + connection_id: &str, +) -> ConfigResult<(WorkspaceConfigGrantResponse, WorkerWorkdirLinkRecord)> { + let grant = grant_for_worker(api, worker)?; + let link = active_link(api, worker, &grant, Some(connection_id))? + .ok_or_else(|| fail(409, "connection_changed"))?; + Ok((grant, link)) +} +fn path_valid(path: &str, root: bool) -> ConfigResult<()> { + if root && path.is_empty() { + return Ok(()); + } + config_source::VirtualPath::parse(path).map_err(|_| fail(400, "invalid_path"))?; + if path.chars().any(char::is_control) { + return Err(fail(400, "invalid_path")); + } + Ok(()) +} +fn tree(api: &WorkspaceApi) -> ConfigResult { + api.config_store + .load_workspace_config(&api.config.workspace_id) + .map_err(pre_error)? + .ok_or_else(|| fail(404, "not_found")) +} +fn validator( + grant: &WorkspaceConfigGrantResponse, + link: &WorkerWorkdirLinkRecord, + snapshot: &ConfigTreeSnapshot, + path: &str, +) -> String { + let bound = serde_json::to_vec(&( + "workspace-config:v1", + &grant.workspace_id, + &grant.grant_id, + &grant.runtime_id, + &grant.worker_id, + &link.connection_id, + effective_access(grant, link), + path, + snapshot.revision, + &snapshot.digest, + )) + .expect("bounded configuration validator tuple"); + format!( + "wc1:{}", + Sha256::digest(bound) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::() + ) +} +fn check_validator(actual: &str, expected: &str) -> ConfigResult<()> { + if actual != expected { + Err(fail(409, "stale_validator")) + } else { + Ok(()) + } +} +pub(super) async fn observe( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + request: WorkspaceConfigObserveRequest, +) -> ConfigResult { + if request.paths.is_empty() || request.paths.len() > MAX_NODES || request.depth > 8 { + return Err(fail(400, "limit_exceeded")); + } + for path in &request.paths { + path_valid(path, true)?; + } + let _guard = lock(api, worker).await?; + let (grant, link) = resolve(api, worker, &request.connection_id)?; + let state = tree(api)?; + let snapshot = &state.snapshot; + let writable = effective_access(&grant, &link) == Access::ReadWrite; + // Build only bounded metadata (the canonical tree itself is bounded by config-source). + let mut kinds = BTreeMap::from([(String::new(), NodeKind::Directory)]); + for path in snapshot.entries.keys() { + let text = path.as_str(); + kinds.insert(text.to_string(), NodeKind::File); + let mut cursor = text; + while let Some((parent, _)) = cursor.rsplit_once('/') { + kinds.entry(parent.into()).or_insert(NodeKind::Directory); + cursor = parent; + } + } + let mut selected = BTreeMap::new(); + for path in request.paths { + selected.insert( + path.clone(), + *kinds.get(&path).unwrap_or(&NodeKind::Missing), + ); + let prefix = if path.is_empty() { + String::new() + } else { + format!("{path}/") + }; + for (child, kind) in kinds.range(prefix.clone()..) { + if !child.starts_with(&prefix) { + break; + } + if child == &path { + continue; + } + let relative = &child[prefix.len()..]; + if relative.split('/').count() <= request.depth as usize { + selected.insert(child.clone(), *kind); + } + if selected.len() > MAX_NODES { + return Err(fail(413, "limit_exceeded")); + } + } + } + if selected.len() > MAX_NODES { + return Err(fail(413, "limit_exceeded")); + } + let nodes = selected + .into_iter() + .map(|(path, kind)| { + let entry = snapshot + .entries + .iter() + .find(|(key, _)| key.as_str() == path) + .map(|(_, entry)| entry); + let mut operations = match kind { + NodeKind::File => vec!["read".to_string()], + NodeKind::Directory | NodeKind::Missing => vec![], + }; + if writable { + match kind { + NodeKind::File => { + operations.extend(["edit", "write"].map(str::to_string)); + if path != crate::config_source::MAIN_CONFIG_ENTRYPOINT { + operations.push("delete".into()); + } + } + NodeKind::Missing => operations.push("create".into()), + NodeKind::Directory => { + operations.push("create".into()); + if path.is_empty() { + operations.push("apply_changes".into()); + } + } + } + } + WorkspaceConfigNode { + validator: validator(&grant, &link, snapshot, &path), + path, + kind, + digest: entry.map(|entry| entry.content_digest.clone()), + content_type: entry.map(|entry| match entry.content_type { + config_source::ConfigContentType::Decodal => { + server_api::ConfigContentType::Decodal + } + config_source::ConfigContentType::Text => server_api::ConfigContentType::Text, + }), + operations, + } + }) + .collect(); + Ok(WorkspaceConfigObserveResponse { + connection_id: link.connection_id.clone(), + validator: validator(&grant, &link, snapshot, ""), + revision: snapshot.revision, + digest: snapshot.digest.clone(), + entrypoints: state + .contract + .entrypoints + .iter() + .map(ToString::to_string) + .collect(), + nodes, + }) +} +pub(super) async fn read( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + request: WorkspaceConfigReadRequest, +) -> ConfigResult { + path_valid(&request.path, false)?; + let _guard = lock(api, worker).await?; + let (grant, link) = resolve(api, worker, &request.connection_id)?; + let state = tree(api)?; + let expected = validator(&grant, &link, &state.snapshot, &request.path); + check_validator(&request.validator, &expected)?; + let path = + config_source::VirtualPath::parse(&request.path).map_err(|_| fail(400, "invalid_path"))?; + let entry = state + .snapshot + .entries + .get(&path) + .ok_or_else(|| fail(404, "not_found"))?; + if entry.content.len() > MAX_TEXT { + return Err(fail(413, "limit_exceeded")); + } + Ok(WorkspaceConfigReadResponse { + path: request.path, + content: entry.content.clone(), + content_type: match entry.content_type { + config_source::ConfigContentType::Decodal => server_api::ConfigContentType::Decodal, + config_source::ConfigContentType::Text => server_api::ConfigContentType::Text, + }, + digest: entry.content_digest.clone(), + validator: expected, + }) +} +pub(super) async fn commit( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + request: WorkspaceConfigCommitRequest, +) -> ConfigResult { + if request.request.changes.len() > MAX_NODES { + return Err(fail(413, "limit_exceeded")); + } + for change in &request.request.changes { + match change { + server_api::ConfigTreeChange::Create { path, content, .. } + | server_api::ConfigTreeChange::Update { path, content, .. } => { + path_valid(path, false)?; + if content.len() > MAX_TEXT { + return Err(fail(413, "limit_exceeded")); + } + } + server_api::ConfigTreeChange::Rename { from, to, .. } => { + path_valid(from, false)?; + path_valid(to, false)?; + } + server_api::ConfigTreeChange::Delete { path, .. } => path_valid(path, false)?, + } + } + let _guard = lock(api, worker).await?; + let (grant, link) = resolve(api, worker, &request.connection_id)?; + if effective_access(&grant, &link) != Access::ReadWrite { + return Err(fail(403, "access_denied")); + } + let state = tree(api)?; + check_validator( + &request.validator, + &validator(&grant, &link, &state.snapshot, ""), + )?; + let canonical = config_commit_request_from_api(request.request).map_err(pre_error)?; + // Evaluation/semantic validation are pure preparation. Bound this stage + // without ever dispatching persistence in a timed-out background task. + let preparation_api = api.clone(); + let preparation = tokio::task::spawn_blocking(move || { + prepare_workspace_config_tree( + &preparation_api, + &preparation_api.config.workspace_id, + &canonical, + ) + }); + let candidate = tokio::time::timeout(DEADLINE, preparation) + .await + .map_err(|_| fail(503, "evaluation_timeout"))? + .map_err(|_| fail(500, "evaluation_unavailable"))? + .map_err(pre_error)?; + // Recheck durable authority before the first auxiliary/canonical effects. + let (current_grant, current_link) = resolve(api, worker, &request.connection_id)?; + if current_grant != grant || current_link.capabilities != link.capabilities { + return Err(fail(409, "authority_changed")); + } + // From this boundary repository credential projection may have side effects. + // Never falsely claim rollback or automatically replay a failed/uncertain save. + let state = persist_workspace_config_candidate(api, &api.config.workspace_id, &candidate) + .map_err(|_| unknown())?; + Ok(WorkspaceConfigCommitResponse { + validator: validator(&grant, &link, &state.snapshot, ""), + revision: state.snapshot.revision, + digest: state.snapshot.digest, + }) +} +pub(super) async fn create_grant( + api: &WorkspaceApi, + actor: &RequestActor, + workspace_id: &str, + request: WorkspaceConfigGrantCreateRequest, +) -> ConfigResult { + require_workspace_owner(api, workspace_id, actor, "Workspace config grants") + .await + .map_err(pre_error)?; + let worker = RuntimeWorkerRef::new(&request.runtime_id, &request.worker_id); + let _guard = lock(api, &worker).await?; + let summary = api + .runtime + .worker(&worker) + .map_err(|_| fail(404, "worker_not_found"))?; + if summary.workspace.workspace_id.as_deref() != Some(workspace_id) { + return Err(fail(403, "access_denied")); + } + if let Some(grant) = api + .store + .current_workspace_config_grant(workspace_id, &worker) + .map_err(pre_error)? + { + if grant.access != request.access { + return Err(fail(409, "grant_exists_revoke_first")); + } + return Ok(grant); + } + let grant = WorkspaceConfigGrantResponse { + grant_id: Uuid::now_v7().to_string(), + workspace_id: workspace_id.into(), + runtime_id: worker.runtime_id, + worker_id: worker.worker_id, + working_directory_id: Uuid::now_v7().to_string(), + access: request.access, + revoked: false, + }; + api.store + .create_workspace_config_grant(&grant, &actor.account_id) + .map_err(mutation_error)?; + Ok(grant) +} +pub(super) async fn revoke_grant( + api: &WorkspaceApi, + actor: &RequestActor, + workspace_id: &str, + grant_id: &str, +) -> ConfigResult { + require_workspace_owner(api, workspace_id, actor, "Workspace config grants") + .await + .map_err(pre_error)?; + let mut grant = api + .store + .get_workspace_config_grant(workspace_id, grant_id) + .map_err(pre_error)? + .ok_or_else(|| fail(404, "not_found"))?; + let worker = RuntimeWorkerRef::new(&grant.runtime_id, &grant.worker_id); + let _guard = lock(api, &worker).await?; + api.store + .revoke_workspace_config_grant(workspace_id, grant_id) + .map_err(mutation_error)?; + grant.revoked = true; + // Revocation is durable before unlink; a failed cleanup cannot restore access. + if let Some(link) = api + .store + .list_workdir_worker_links(workspace_id, &grant.working_directory_id) + .map_err(|_| unknown())? + .into_iter() + .find(|link| link.unlinked_at.is_none() && link.worker == worker) + { + api.store + .detach_worker_workdir_connection( + workspace_id, + &worker, + &link.alias, + &link.connection_id, + &now_registry_timestamp(), + ) + .map_err(|_| unknown())?; + } + api.worker_projection + .refresh(&worker) + .map_err(|_| unknown())?; + Ok(grant) +} + +pub(super) fn summary( + grant: &WorkspaceConfigGrantResponse, + record: &WorkdirRegistryRecord, +) -> server_api::WorkingDirectorySummary { + server_api::WorkingDirectorySummary { + working_directory_id: record.workdir_id.clone(), + display_name: Some("Workspace configuration".into()), + source: server_api::WorkingDirectorySource::WorkspaceConfig { + access: grant.access, + content_path: "/workspace-config".into(), + purpose: "Workspace configuration".into(), + }, + creation_selector: None, + creation_ref: None, + creation_tree: None, + current_selector: None, + current_ref: None, + current_tree: None, + observed_at_epoch_seconds: None, + materializer_kind: server_api::WorkingDirectoryMaterializerKind::LogicalWorkspaceConfig, + cleanup_target: None, + status: if grant.revoked { + server_api::WorkingDirectoryStatusKind::NotFound + } else { + server_api::WorkingDirectoryStatusKind::Active + }, + cleanliness: None, + occupied_by: None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn workspace_config_mutation_storage_failures_are_conservatively_unknown() { + assert_eq!( + mutation_error(Error::Sqlite(rusqlite::Error::InvalidQuery)).classification, + Classification::Unknown + ); + assert_eq!( + mutation_error(Error::Store("private /host/storage acknowledgement".into())) + .classification, + Classification::Unknown + ); + let denied = mutation_error(Error::WorkspacePermissionDenied( + "secret permission context".into(), + )); + assert_eq!(denied.classification, Classification::NotCommitted); + assert_eq!(denied.status, 403); + assert!(!denied.message.contains("secret")); + } +} diff --git a/crates/workspace-server/src/server_workspace_config_tests.rs b/crates/workspace-server/src/server_workspace_config_tests.rs new file mode 100644 index 000000000..e65ae4e62 --- /dev/null +++ b/crates/workspace-server/src/server_workspace_config_tests.rs @@ -0,0 +1,1390 @@ +// Included inside server::tests so the existing real Workspace/Runtime fixtures are reused. +mod workspace_config_integration { + include!("server_workspace_config_wip_tests.rs"); + use super::*; + use server_api::{ + WorkspaceConfigAccess as Access, WorkspaceConfigAttachRequest as Attach, + WorkspaceConfigAttachment as Attachment, WorkspaceConfigCommitRequest as Commit, + WorkspaceConfigFailureClassification as Classification, + WorkspaceConfigGrantCreateRequest as Grant, WorkspaceConfigObserveRequest as Observe, + WorkspaceConfigObserveResponse as Observed, WorkspaceConfigReadRequest as Read, + }; + + async fn grant( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + access: Access, + ) -> server_api::WorkspaceConfigGrantResponse { + workspace_config::create_grant( + api, + &test_owner_actor(), + TEST_WORKSPACE_ID, + Grant { + runtime_id: worker.runtime_id.clone(), + worker_id: worker.worker_id.clone(), + access, + }, + ) + .await + .unwrap() + } + async fn attach(api: &WorkspaceApi, worker: &RuntimeWorkerRef) -> Attachment { + workspace_config::attach( + api, + worker, + Attach { + alias: None, + access: None, + }, + ) + .await + .unwrap() + } + async fn observe( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + connection_id: &str, + paths: &[&str], + depth: u32, + ) -> Observed { + workspace_config::observe( + api, + worker, + Observe { + connection_id: connection_id.into(), + paths: paths.iter().map(|path| (*path).into()).collect(), + depth, + }, + ) + .await + .unwrap() + } + fn commit_request( + connection: &str, + snapshot: &Observed, + changes: Vec, + ) -> Commit { + Commit { + connection_id: connection.into(), + validator: snapshot.validator.clone(), + request: server_api::ConfigCommitRequest { + base_revision: snapshot.revision, + base_digest: snapshot.digest.clone(), + changes, + entrypoints: snapshot.entrypoints.clone(), + }, + } + } + fn create(path: &str, content: &str) -> server_api::ConfigTreeChange { + server_api::ConfigTreeChange::Create { + path: path.into(), + content_type: server_api::ConfigContentType::Text, + content: content.into(), + } + } + async fn second_worker(fixture: &ManualCoderAssignmentFixture) -> RuntimeWorkerRef { + let mut summary = fixture.api.runtime.worker(&fixture.worker).unwrap(); + summary.worker.worker_id = Uuid::now_v7().to_string(); + summary.display_name = "Second config editor".into(); + summary.label = summary.display_name.clone(); + summary.workdir_attachments.clear(); + fixture + .runtime + .workers + .lock() + .unwrap() + .push(summary.clone()); + sync_worker_observation(&fixture.api, &summary).unwrap(); + summary.worker + } + async fn signed_call( + api: &WorkspaceApi, + identity: &RuntimeIdentityMaterial, + worker: Option<&str>, + method: &str, + path: &str, + body: Value, + ) -> (StatusCode, Value) { + let bytes = if method == "GET" { + vec![] + } else { + serde_json::to_vec(&body).unwrap() + }; + let response = build_router(api.clone()) + .oneshot(runtime_source_request( + identity, worker, method, path, bytes, + )) + .await + .unwrap(); + let status = response.status(); + let bytes = to_bytes(response.into_body(), 4 * 1024 * 1024) + .await + .unwrap(); + ( + status, + serde_json::from_slice(&bytes) + .unwrap_or_else(|_| Value::String(String::from_utf8_lossy(&bytes).into_owned())), + ) + } + + #[tokio::test] + async fn workspace_config_signed_editor_routes_cannot_bypass_wip_authority() { + let mut fixture = manual_coder_assignment_fixture().await; + let identity = RuntimeIdentityMaterial::generate(&fixture.worker.runtime_id).unwrap(); + configure_runtime_request_auth(&mut fixture.api, &identity, &fixture.worker.runtime_id); + let api = &fixture.api; + let initial = api.config_store.load_workspace_config(TEST_WORKSPACE_ID).unwrap().unwrap(); + let tree = format!("/api/w/{TEST_WORKSPACE_ID}/config/source-tree"); + let change = server_api::ConfigCommitRequest { + base_revision: initial.snapshot.revision, + base_digest: initial.snapshot.digest.clone(), + changes: vec![create("notes/editor-only.txt", "must not persist from Worker")], + entrypoints: vec!["main.dcdl".into()], + }; + let routes = [ + ("GET", tree.clone()), + ("GET", format!("{tree}/entries/main.dcdl")), + ("GET", format!("{tree}/revisions/{}", initial.snapshot.revision)), + ("POST", format!("{tree}/commit")), + ("GET", format!("/api/w/{TEST_WORKSPACE_ID}/settings/profiles")), + ]; + for state in ["ungranted", "read_only", "revoked", "detached", "read_write"] { + match state { + "read_only" => { + grant(api, &fixture.worker, Access::ReadOnly).await; + attach(api, &fixture.worker).await; + } + "revoked" => { + let current = api.store.current_workspace_config_grant(TEST_WORKSPACE_ID, &fixture.worker).unwrap().unwrap(); + workspace_config::revoke_grant(api, &test_owner_actor(), TEST_WORKSPACE_ID, + ¤t.grant_id).await.unwrap(); + } + "detached" => { + grant(api, &fixture.worker, Access::ReadWrite).await; + let attached = attach(api, &fixture.worker).await; + detach_current_worker_workdir(api, &fixture.worker, "workspace-config", + Some(&attached.connection_id)).await.unwrap(); + } + "read_write" => { attach(api, &fixture.worker).await; } + _ => {} + } + for (method, path) in &routes { + let body = if *method == "POST" { serde_json::to_vec(&change).unwrap() } else { vec![] }; + for scoped in [false, true] { + let app = if scoped { + workspace_server_router(WorkspaceServerApi::new(api.config.clone(), api.store.clone())) + } else { + build_router(api.clone()) + }; + let response = app.oneshot(runtime_source_request(&identity, + Some(&fixture.worker.worker_id), method, path, body.clone())).await.unwrap(); + assert_eq!(response.status(), StatusCode::FORBIDDEN, + "{state}: {method} {path}, scoped={scoped}"); + let bytes = to_bytes(response.into_body(), 4096).await.unwrap(); + assert!(!String::from_utf8_lossy(&bytes).contains("must not persist")); + } + } + let after = api.config_store.load_workspace_config(TEST_WORKSPACE_ID).unwrap().unwrap(); + assert_eq!(after.snapshot, initial.snapshot, "{state}: editor rejection must have no effect"); + } + // A Runtime-only proof is not a user editor identity either. + assert_eq!(signed_call(api, &identity, None, "GET", &tree, Value::Null).await.0, + StatusCode::FORBIDDEN); + let attached = attach(api, &fixture.worker).await; + let observed = observe(api, &fixture.worker, &attached.connection_id, &["main.dcdl"], 0).await; + let node = observed.nodes.iter().find(|n| n.path == "main.dcdl").unwrap(); + assert!(workspace_config::read(api, &fixture.worker, Read { + connection_id: attached.connection_id, + path: "main.dcdl".into(), validator: node.validator.clone(), + }).await.is_ok(), "the granted WIP path remains available"); + + // Real API-token authentication still permits the normal UI editor. + let token = seed_test_api_token(api.store.as_ref(), TEST_WORKSPACE_ID); + for (method, path) in &routes { + if *method == "GET" { + request_json_authenticated(build_router(api.clone()), method, path, None, + &token, StatusCode::OK).await; + } + } + request_json_authenticated(build_router(api.clone()), "POST", &format!("{tree}/commit"), + Some(serde_json::to_value(change).unwrap()), &token, StatusCode::CREATED).await; + let saved = api.config_store.load_workspace_config(TEST_WORKSPACE_ID).unwrap().unwrap(); + assert_eq!(saved.snapshot.revision, initial.snapshot.revision + 1); + assert!(saved.snapshot.entries.keys().any(|p| p.as_str() == "notes/editor-only.txt")); + // Prompt and Skill runtime consumption are separate read-only projections, + // not authored-tree editor APIs; bootstrap must remain usable. + assert_eq!(signed_call(api, &identity, Some(&fixture.worker.worker_id), "GET", + &format!("/api/w/{TEST_WORKSPACE_ID}/config/projections/prompts"), Value::Null).await.0, + StatusCode::OK); + assert_eq!(signed_call(api, &identity, Some(&fixture.worker.worker_id), "GET", + &format!("/api/w/{TEST_WORKSPACE_ID}/skills"), Value::Null).await.0, StatusCode::OK); + } + + #[tokio::test] + async fn workspace_config_entrypoint_support_is_exact_and_canonical_mutations_stay_guarded() { + let fixture = manual_coder_assignment_fixture().await; + let api = &fixture.api; + grant(api, &fixture.worker, Access::ReadWrite).await; + let attached = attach(api, &fixture.worker).await; + let observed = observe( + api, + &fixture.worker, + &attached.connection_id, + &["", "main.dcdl"], + 1, + ) + .await; + let main = observed + .nodes + .iter() + .find(|node| node.path == "main.dcdl") + .unwrap(); + assert_eq!(main.operations, ["read", "edit", "write"]); + let before = api + .config_store + .load_workspace_config(TEST_WORKSPACE_ID) + .unwrap() + .unwrap(); + for change in [ + server_api::ConfigTreeChange::Delete { + path: "main.dcdl".into(), + expected_digest: main.digest.clone().unwrap(), + }, + server_api::ConfigTreeChange::Rename { + from: "main.dcdl".into(), + to: "renamed.dcdl".into(), + expected_digest: main.digest.clone().unwrap(), + }, + ] { + let failure = workspace_config::commit( + api, + &fixture.worker, + commit_request(&attached.connection_id, &observed, vec![change]), + ) + .await + .unwrap_err(); + assert_eq!(failure.classification, Classification::NotCommitted); + assert_eq!( + api.config_store + .load_workspace_config(TEST_WORKSPACE_ID) + .unwrap() + .unwrap(), + before + ); + } + workspace_config::commit( + api, + &fixture.worker, + commit_request( + &attached.connection_id, + &observed, + vec![create("notes/deletable.md", "note")], + ), + ) + .await + .unwrap(); + let note = observe( + api, + &fixture.worker, + &attached.connection_id, + &["notes/deletable.md"], + 0, + ) + .await; + assert_eq!( + note.nodes[0].operations, + ["read", "edit", "write", "delete"] + ); + } + + #[tokio::test] + async fn workspace_config_grants_owner_scope_and_signed_routes() { + let mut fixture = manual_coder_assignment_fixture().await; + let identity = RuntimeIdentityMaterial::generate(&fixture.worker.runtime_id).unwrap(); + configure_runtime_request_auth(&mut fixture.api, &identity, &fixture.worker.runtime_id); + let api = &fixture.api; + let base = format!("/api/w/{TEST_WORKSPACE_ID}/workers/self/workspace-config"); + assert_eq!( + signed_call( + api, + &identity, + Some(&fixture.worker.worker_id), + "GET", + &base, + Value::Null + ) + .await + .0, + StatusCode::FORBIDDEN, + "unattached GET must still require a grant" + ); + assert_eq!( + signed_call( + api, + &identity, + Some(&fixture.worker.worker_id), + "POST", + &base, + json!({}) + ) + .await + .0, + StatusCode::FORBIDDEN + ); + assert!( + api.store + .current_workspace_config_grant(TEST_WORKSPACE_ID, &fixture.worker) + .unwrap() + .is_none(), + "attach cannot create a grant" + ); + let mut other_owner = test_owner_actor(); + other_owner.account_id = "not-owner".into(); + let request = Grant { + runtime_id: fixture.worker.runtime_id.clone(), + worker_id: fixture.worker.worker_id.clone(), + access: Access::ReadWrite, + }; + assert_eq!( + workspace_config::create_grant(api, &other_owner, TEST_WORKSPACE_ID, request.clone()) + .await + .unwrap_err() + .status, + 403 + ); + assert!( + workspace_config::create_grant( + api, + &test_owner_actor(), + "foreign-workspace", + request.clone() + ) + .await + .is_err() + ); + // Exercise generated browser grant router with the authenticated owner extension. + let path = format!("/api/w/{TEST_WORKSPACE_ID}/workspace-config-grants"); + let response = build_inner_router(api.clone()) + .layer(Extension(test_owner_actor())) + .oneshot( + Request::builder() + .method("POST") + .uri(&path) + .header(CONTENT_TYPE, "application/json") + .body(Body::from(serde_json::to_vec(&request).unwrap())) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let granted: server_api::WorkspaceConfigGrantResponse = serde_json::from_slice( + &to_bytes(response.into_body(), 4 * 1024 * 1024) + .await + .unwrap(), + ) + .unwrap(); + assert_eq!( + grant(api, &fixture.worker, Access::ReadWrite) + .await + .grant_id, + granted.grant_id + ); + let (status, current) = signed_call( + api, + &identity, + Some(&fixture.worker.worker_id), + "GET", + &base, + Value::Null, + ) + .await; + assert_eq!(status, StatusCode::OK); + assert!(current.is_null()); + let (status, current) = signed_call( + api, + &identity, + Some(&fixture.worker.worker_id), + "POST", + &base, + json!({}), + ) + .await; + assert_eq!(status, StatusCode::OK); + let attached: Attachment = serde_json::from_value(current).unwrap(); + let again: Attachment = serde_json::from_value( + signed_call( + api, + &identity, + Some(&fixture.worker.worker_id), + "POST", + &base, + json!({}), + ) + .await + .1, + ) + .unwrap(); + assert!(again.already_attached); + assert_eq!(again.connection_id, attached.connection_id); + assert_eq!( + signed_call(api, &identity, None, "GET", &base, Value::Null) + .await + .0, + StatusCode::FORBIDDEN + ); + let mut forged = runtime_source_request( + &identity, + Some(&fixture.worker.worker_id), + "GET", + &base, + vec![], + ); + forged + .headers_mut() + .insert("x-yoi-worker-id", "forged-worker".parse().unwrap()); + assert_eq!( + build_router(api.clone()) + .oneshot(forged) + .await + .unwrap() + .status(), + StatusCode::FORBIDDEN + ); + let foreign = base.replace(TEST_WORKSPACE_ID, "foreign-workspace"); + assert_ne!( + signed_call( + api, + &identity, + Some(&fixture.worker.worker_id), + "GET", + &foreign, + Value::Null + ) + .await + .0, + StatusCode::OK + ); + // A Worker proof is not an owner grant capability. + assert_ne!( + signed_call( + api, + &identity, + Some(&fixture.worker.worker_id), + "POST", + &path, + serde_json::to_value(request).unwrap() + ) + .await + .0, + StatusCode::OK + ); + assert_eq!( + workspace_config::attach( + api, + &fixture.worker, + Attach { + alias: Some("other".into()), + access: None + } + ) + .await + .unwrap_err() + .status, + 400 + ); + let (status, metadata) = signed_call( + api, + &identity, + Some(&fixture.worker.worker_id), + "POST", + &format!("{base}/observe"), + json!({"connection_id":attached.connection_id,"paths":[""],"depth":1}), + ) + .await; + assert_eq!(status, StatusCode::OK); + assert!( + metadata["nodes"] + .as_array() + .unwrap() + .iter() + .all(|node| node.get("content").is_none()) + ); + assert_eq!(metadata["entrypoints"], json!(["main.dcdl"])); + assert!( + api.runtime + .worker(&fixture.worker) + .unwrap() + .workdir_attachments + .is_empty(), + "logical source never creates a Runtime filesystem attachment" + ); + let runtime_logical = fixture + .runtime + .logical_attachments(&fixture.worker.worker_id); + assert!( + runtime_logical + .iter() + .all(|item| item.working_directory_id != attached.working_directory_id) + ); + let catalog = list_current_worker_workdir_catalog_for_worker( + api, + &fixture.worker, + Default::default(), + ) + .unwrap(); + let logical = catalog + .items + .iter() + .find(|item| item.working_directory_id == attached.working_directory_id) + .unwrap(); + assert!(logical.cleanup_target.is_none()); + assert_eq!( + logical.materializer_kind, + server_api::WorkingDirectoryMaterializerKind::LogicalWorkspaceConfig + ); + assert!(matches!( + logical.source, + server_api::WorkingDirectorySource::WorkspaceConfig { .. } + )); + assert!( + api.store + .delete_workdir_registry(TEST_WORKSPACE_ID, &attached.working_directory_id) + .is_err() + ); + let generic = format!("/api/w/{TEST_WORKSPACE_ID}/workers/self/workdir-attachments"); + assert_eq!(signed_call(api,&identity,Some(&fixture.worker.worker_id),"POST",&generic,json!({"alias":"workspace-config","working_directory_id":attached.working_directory_id})).await.0,StatusCode::FORBIDDEN); + let link = api + .store + .list_worker_workdir_links(TEST_WORKSPACE_ID, &fixture.worker) + .unwrap() + .into_iter() + .find(|link| link.alias == "workspace-config") + .unwrap(); + assert!( + !link + .capabilities + .supports(workdir::WorkdirSessionCapability::Command) + ); + let lock = current_worker_session_lock(api, &fixture.worker); + let _guard = lock.lock().await; + assert!( + open_current_worker_workdir_session_locked(api, &fixture.worker, &link) + .await + .is_err() + ); + drop(_guard); + assert!( + cleanup_working_directory( + api.clone(), + ScopedWorkingDirectoryPath { + workspace_id: TEST_WORKSPACE_ID.into(), + working_directory_id: attached.working_directory_id.clone() + }, + "owner".into(), + WorkingDirectoryRemovalRequest { + reason: "unsupported logical cleanup".into() + } + ) + .await + .is_err() + ); + let revoke_path = format!( + "/api/w/{TEST_WORKSPACE_ID}/workspace-config-grants/{}", + granted.grant_id + ); + let response = build_inner_router(api.clone()) + .layer(Extension(test_owner_actor())) + .oneshot( + Request::builder() + .method("DELETE") + .uri(&revoke_path) + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + assert_eq!( + workspace_config::get(api, &fixture.worker) + .await + .unwrap_err() + .status, + 403 + ); + } + + #[tokio::test] + async fn workspace_config_readonly_revoke_and_conditional_lifetime_fences() { + let fixture = manual_coder_assignment_fixture().await; + let api = &fixture.api; + let read_grant = grant(api, &fixture.worker, Access::ReadOnly).await; + assert!( + workspace_config::get(api, &fixture.worker) + .await + .unwrap() + .is_none() + ); + assert_eq!( + workspace_config::attach( + api, + &fixture.worker, + Attach { + alias: None, + access: Some(Access::ReadWrite) + } + ) + .await + .unwrap_err() + .status, + 403 + ); + let attached = attach(api, &fixture.worker).await; + let observed = observe( + api, + &fixture.worker, + &attached.connection_id, + &["", "main.dcdl"], + 1, + ) + .await; + assert!( + observed + .nodes + .iter() + .all(|node| node.operations.iter().all(|operation| operation == "read")) + ); + let read_node = observed + .nodes + .iter() + .find(|node| node.path == "main.dcdl") + .unwrap(); + let read = Read { + connection_id: attached.connection_id.clone(), + path: "main.dcdl".into(), + validator: read_node.validator.clone(), + }; + assert!( + !workspace_config::read(api, &fixture.worker, read.clone()) + .await + .unwrap() + .content + .is_empty() + ); + let denied = workspace_config::commit( + api, + &fixture.worker, + commit_request( + &attached.connection_id, + &observed, + vec![create("notes/no.md", "no")], + ), + ) + .await + .unwrap_err(); + assert_eq!(denied.status, 403); + assert_eq!(denied.classification, Classification::NotCommitted); + let second = second_worker(&fixture).await; + assert_eq!( + workspace_config::read(api, &second, read.clone()) + .await + .unwrap_err() + .status, + 403 + ); + let second_grant = grant(api, &second, Access::ReadWrite).await; + assert_ne!( + second_grant.working_directory_id, + read_grant.working_directory_id + ); + assert_eq!( + workspace_config::read(api, &second, read.clone()) + .await + .unwrap_err() + .code, + "connection_changed" + ); + let detached = detach_current_worker_workdir( + api, + &fixture.worker, + "workspace-config", + Some(&attached.connection_id), + ) + .await + .unwrap(); + assert!(!detached.attached); + assert_eq!( + workspace_config::read(api, &fixture.worker, read.clone()) + .await + .unwrap_err() + .code, + "connection_changed" + ); + let reattached = attach(api, &fixture.worker).await; + assert_ne!(reattached.connection_id, attached.connection_id); + assert!( + detach_current_worker_workdir( + api, + &fixture.worker, + "workspace-config", + Some(&attached.connection_id) + ) + .await + .is_err() + ); + assert_eq!( + workspace_config::get(api, &fixture.worker) + .await + .unwrap() + .unwrap() + .connection_id, + reattached.connection_id + ); + workspace_config::revoke_grant( + api, + &test_owner_actor(), + TEST_WORKSPACE_ID, + &read_grant.grant_id, + ) + .await + .unwrap(); + assert_eq!( + workspace_config::get(api, &fixture.worker) + .await + .unwrap_err() + .status, + 403 + ); + assert_eq!( + workspace_config::read(api, &fixture.worker, read) + .await + .unwrap_err() + .status, + 403 + ); + assert_eq!( + workspace_config::attach( + api, + &fixture.worker, + Attach { + alias: None, + access: None + } + ) + .await + .unwrap_err() + .status, + 403 + ); + assert!( + workspace_config::revoke_grant( + api, + &test_owner_actor(), + "foreign-workspace", + &second_grant.grant_id + ) + .await + .is_err() + ); + assert!( + api.store + .current_workspace_config_grant(TEST_WORKSPACE_ID, &second) + .unwrap() + .is_some() + ); + let writable = attach(api, &second).await; + let restricted = workspace_config::attach( + api, + &second, + Attach { + alias: None, + access: Some(Access::ReadOnly), + }, + ) + .await + .unwrap(); + assert_eq!(restricted.connection_id, writable.connection_id); + assert_eq!(restricted.access, Access::ReadOnly); + assert_eq!( + attach(api, &second).await.access, + Access::ReadOnly, + "default repeat must honor current attenuation" + ); + assert_eq!( + workspace_config::attach( + api, + &second, + Attach { + alias: None, + access: Some(Access::ReadWrite) + } + ) + .await + .unwrap_err() + .status, + 403 + ); + } + + #[tokio::test] + async fn workspace_config_two_workers_ui_cas_and_identical_virtual_replacement() { + let fixture = manual_coder_assignment_fixture().await; + let api = &fixture.api; + let second = second_worker(&fixture).await; + grant(api, &fixture.worker, Access::ReadWrite).await; + grant(api, &second, Access::ReadWrite).await; + let a = attach(api, &fixture.worker).await; + let b = attach(api, &second).await; + let first = observe(api, &fixture.worker, &a.connection_id, &[""], 0).await; + let other = observe(api, &second, &b.connection_id, &[""], 0).await; + assert_eq!(first.revision, other.revision); + assert_eq!(first.digest, other.digest); + assert_ne!(first.validator, other.validator); + workspace_config::commit( + api, + &fixture.worker, + commit_request( + &a.connection_id, + &first, + vec![create("notes/concurrent.md", "first")], + ), + ) + .await + .unwrap(); + let rejected = workspace_config::commit( + api, + &second, + commit_request( + &b.connection_id, + &other, + vec![create("notes/second.md", "second")], + ), + ) + .await + .unwrap_err(); + assert_eq!(rejected.code, "stale_validator"); + assert_eq!(rejected.classification, Classification::NotCommitted); + let fresh = observe(api, &second, &b.connection_id, &["notes/concurrent.md"], 0).await; + let file = fresh.nodes.first().unwrap(); + let old_read = Read { + connection_id: b.connection_id.clone(), + path: file.path.clone(), + validator: file.validator.clone(), + }; + let canonical = config_commit_request_from_api(server_api::ConfigCommitRequest { + base_revision: fresh.revision, + base_digest: fresh.digest.clone(), + entrypoints: fresh.entrypoints.clone(), + changes: vec![server_api::ConfigTreeChange::Update { + path: file.path.clone(), + expected_digest: file.digest.clone().unwrap(), + content: "first".into(), + }], + }) + .unwrap(); + // Same bytes, new virtual identity/revision via the ORIGINAL UI commit boundary. + commit_workspace_config_tree(api, TEST_WORKSPACE_ID, &canonical).unwrap(); + let after = observe(api, &second, &b.connection_id, &["notes/concurrent.md"], 0).await; + assert_eq!(after.digest, fresh.digest); + assert!(after.revision > fresh.revision); + assert_ne!(after.validator, fresh.validator); + assert_eq!( + workspace_config::read(api, &second, old_read) + .await + .unwrap_err() + .code, + "stale_validator" + ); + assert_eq!( + workspace_config::commit( + api, + &second, + commit_request( + &b.connection_id, + &fresh, + vec![create("notes/stale.md", "stale")] + ) + ) + .await + .unwrap_err() + .code, + "stale_validator" + ); + let delete = vec![server_api::ConfigTreeChange::Delete { + path: file.path.clone(), + expected_digest: file.digest.clone().unwrap(), + }]; + workspace_config::commit( + api, + &second, + commit_request(&b.connection_id, &after, delete), + ) + .await + .unwrap(); + let deleted = observe(api, &second, &b.connection_id, &[""], 0).await; + workspace_config::commit( + api, + &second, + commit_request( + &b.connection_id, + &deleted, + vec![create(&file.path, "replacement")], + ), + ) + .await + .unwrap(); + let state = api + .config_store + .load_workspace_config(TEST_WORKSPACE_ID) + .unwrap() + .unwrap(); + assert_eq!( + state + .snapshot + .get(&config_source::VirtualPath::parse("notes/concurrent.md").unwrap()) + .unwrap() + .content, + "replacement" + ); + assert!( + state + .snapshot + .get(&config_source::VirtualPath::parse("notes/second.md").unwrap()) + .is_none() + ); + // Canonical immediate transaction still rejects a race after evaluation. + let current = observe(api, &second, &b.connection_id, &[""], 0).await; + let request = config_commit_request_from_api( + commit_request( + &b.connection_id, + ¤t, + vec![create("notes/raced.md", "race")], + ) + .request, + ) + .unwrap(); + let candidate = prepare_workspace_config_tree(api, TEST_WORKSPACE_ID, &request).unwrap(); + let ui = config_commit_request_from_api( + commit_request( + &b.connection_id, + ¤t, + vec![create("notes/ui.md", "ui")], + ) + .request, + ) + .unwrap(); + commit_workspace_config_tree(api, TEST_WORKSPACE_ID, &ui).unwrap(); + assert!( + api.config_store + .commit_evaluated_workspace_config(TEST_WORKSPACE_ID, &candidate) + .is_err() + ); + } + + #[tokio::test] + async fn workspace_config_invalid_bounds_and_failed_save_do_not_corrupt_tree() { + let fixture = manual_coder_assignment_fixture().await; + let api = &fixture.api; + grant(api, &fixture.worker, Access::ReadWrite).await; + let attached = attach(api, &fixture.worker).await; + let before = api + .config_store + .load_workspace_config(TEST_WORKSPACE_ID) + .unwrap() + .unwrap(); + let observed = observe( + api, + &fixture.worker, + &attached.connection_id, + &["", "main.dcdl"], + 0, + ) + .await; + let main = observed + .nodes + .iter() + .find(|node| node.path == "main.dcdl") + .unwrap(); + for changes in [ + vec![server_api::ConfigTreeChange::Update { + path: "main.dcdl".into(), + expected_digest: main.digest.clone().unwrap(), + content: "not valid Decodal".into(), + }], + vec![server_api::ConfigTreeChange::Delete { + path: "main.dcdl".into(), + expected_digest: main.digest.clone().unwrap(), + }], + vec![create("../escape", "no")], + vec![create("bad/\nsecret", "no")], + vec![create(&"x".repeat(513), "no")], + vec![create("notes/large.md", &"x".repeat(256 * 1024 + 1))], + (0..257) + .map(|i| create(&format!("notes/{i}.md"), "no")) + .collect(), + ] { + let error = workspace_config::commit( + api, + &fixture.worker, + commit_request(&attached.connection_id, &observed, changes), + ) + .await + .unwrap_err(); + assert_eq!(error.classification, Classification::NotCommitted); + assert_eq!( + api.config_store + .load_workspace_config(TEST_WORKSPACE_ID) + .unwrap() + .unwrap(), + before + ); + assert!(!error.message.contains("Decodal\"")); + } + for (paths, depth) in [ + (vec!["".to_string()], 9), + (vec!["/host/secret".to_string()], 0), + (vec!["".to_string(); 257], 0), + ] { + assert!( + workspace_config::observe( + api, + &fixture.worker, + Observe { + connection_id: attached.connection_id.clone(), + paths, + depth + } + ) + .await + .is_err() + ); + } + let missing = observe( + api, + &fixture.worker, + &attached.connection_id, + &["notes/missing.md"], + 0, + ) + .await; + assert_eq!( + missing.nodes[0].kind, + server_api::WorkspaceConfigNodeKind::Missing + ); + assert_eq!(missing.nodes[0].operations, ["create"]); + api.config_store.with_conn(|conn| { conn.execute_batch("CREATE TRIGGER config_test_save_failure BEFORE INSERT ON workspace_config_entries WHEN NEW.path='notes/fail.md' BEGIN SELECT RAISE(ABORT,'private /host/path'); END;")?;Ok(()) }).unwrap(); + let error = workspace_config::commit( + api, + &fixture.worker, + commit_request( + &attached.connection_id, + &observed, + vec![create("notes/fail.md", "safe")], + ), + ) + .await + .unwrap_err(); + // The API conservatively marks failures past the auxiliary-effects boundary unknown. + assert_eq!(error.classification, Classification::Unknown); + assert!(!error.message.contains("private")); + assert_eq!( + api.config_store + .load_workspace_config(TEST_WORKSPACE_ID) + .unwrap() + .unwrap(), + before + ); + api.config_store + .with_conn(|conn| { + conn.execute_batch("DROP TRIGGER config_test_save_failure;")?; + Ok(()) + }) + .unwrap(); + workspace_config::commit( + api, + &fixture.worker, + commit_request( + &attached.connection_id, + &observed, + vec![create("notes/ok.md", "ok")], + ), + ) + .await + .unwrap(); + } + + #[tokio::test] + async fn workspace_config_operation_and_revoke_share_the_detach_session_lock() { + let fixture = manual_coder_assignment_fixture().await; + let api = &fixture.api; + let granted = grant(api, &fixture.worker, Access::ReadWrite).await; + let attached = attach(api, &fixture.worker).await; + let observed = observe(api, &fixture.worker, &attached.connection_id, &[""], 0).await; + let lock = current_worker_session_lock(api, &fixture.worker); + let held = lock.lock().await; + let mut operation = Box::pin(workspace_config::commit( + api, + &fixture.worker, + commit_request( + &attached.connection_id, + &observed, + vec![create("notes/locked.md", "no")], + ), + )); + let owner = test_owner_actor(); + let mut revoke = Box::pin(workspace_config::revoke_grant( + api, + &owner, + TEST_WORKSPACE_ID, + &granted.grant_id, + )); + // Poll once, without sleeps/time-based ordering: both must be fenced by held lock. + assert!(matches!( + futures::poll!(operation.as_mut()), + std::task::Poll::Pending + )); + assert!(matches!( + futures::poll!(revoke.as_mut()), + std::task::Poll::Pending + )); + drop(held); + // Commit queued first, then revocation; subsequent calls cannot use its cached validator. + operation.await.unwrap(); + revoke.await.unwrap(); + assert_eq!( + workspace_config::commit( + api, + &fixture.worker, + commit_request( + &attached.connection_id, + &observed, + vec![create("notes/after.md", "no")] + ) + ) + .await + .unwrap_err() + .status, + 403 + ); + } + #[tokio::test] + async fn workspace_config_grant_restart_source_immutability_and_owner_acceptance_guard() { + let fixture = manual_coder_assignment_fixture().await; + let api = &fixture.api; + let granted = grant(api, &fixture.worker, Access::ReadWrite).await; + let attached = attach(api, &fixture.worker).await; + let restarted = SqliteWorkspaceStore::open(&api.config.database_path).unwrap(); + assert_eq!( + restarted + .get_workspace_config_grant(TEST_WORKSPACE_ID, &granted.grant_id) + .unwrap(), + Some(granted.clone()) + ); + assert!( + restarted + .get_workspace_config_grant("foreign-workspace", &granted.grant_id) + .unwrap() + .is_none() + ); + let record = api + .store + .get_workdir_registry(TEST_WORKSPACE_ID, &attached.working_directory_id) + .unwrap() + .unwrap(); + let mut forged = record.clone(); + forged.source = WorkdirRegistrySource::Repository { + runtime_id: fixture.worker.runtime_id.clone(), + repository_id: test_repository_id(api), + }; + assert!( + api.store.upsert_workdir_registry(&forged).is_err(), + "Runtime observation must never replace a logical config source identity" + ); + assert_eq!( + api.store + .get_workdir_registry(TEST_WORKSPACE_ID, &attached.working_directory_id) + .unwrap(), + Some(record) + ); + let second = second_worker(&fixture).await; + let requested = server_api::WorkspaceConfigGrantResponse { + grant_id: Uuid::now_v7().to_string(), + workspace_id: TEST_WORKSPACE_ID.into(), + runtime_id: second.runtime_id.clone(), + worker_id: second.worker_id.clone(), + working_directory_id: Uuid::now_v7().to_string(), + access: Access::ReadOnly, + revoked: false, + }; + assert!( + api.store + .create_workspace_config_grant(&requested, "not-owner") + .is_err(), + "storage transaction rechecks owner acceptance" + ); + api.config_store + .with_conn(|conn| { + conn.execute( + "UPDATE workspaces SET state='deleting' WHERE workspace_id=?1", + [TEST_WORKSPACE_ID], + )?; + Ok(()) + }) + .unwrap(); + assert!( + api.store + .create_workspace_config_grant(&requested, &test_owner_actor().account_id) + .is_err(), + "existing Workspace mutation lifecycle gate must also fence grants" + ); + api.config_store + .with_conn(|conn| { + conn.execute( + "UPDATE workspaces SET state='active' WHERE workspace_id=?1", + [TEST_WORKSPACE_ID], + )?; + Ok(()) + }) + .unwrap(); + assert!( + api.store + .current_workspace_config_grant(TEST_WORKSPACE_ID, &second) + .unwrap() + .is_none() + ); + } + + #[tokio::test] + async fn workspace_config_auxiliary_secret_effects_are_unknown_not_false_rollback() { + let fixture = manual_coder_assignment_fixture().await; + let api = &fixture.api; + grant(api, &fixture.worker, Access::ReadWrite).await; + let attached = attach(api, &fixture.worker).await; + let observed = observe( + api, + &fixture.worker, + &attached.connection_id, + &["", "main.dcdl"], + 0, + ) + .await; + let main = observed + .nodes + .iter() + .find(|node| node.path == "main.dcdl") + .unwrap(); + let before = api + .config_store + .load_workspace_config(TEST_WORKSPACE_ID) + .unwrap() + .unwrap(); + let count = || { + api.config_store + .with_conn(|conn| { + conn.query_row( + "SELECT COUNT(*) FROM repository_ssh_credentials WHERE workspace_id=?1", + [TEST_WORKSPACE_ID], + |row| row.get::<_, i64>(0), + ) + .map_err(Error::from) + }) + .unwrap() + }; + let before_secrets = count(); + let source = r#"{ repository_access = { missing_repository = { ssh = { credential = "workspace-default"; host_trust = "missing-host-trust"; access = "read_write"; }; }; }; } as WorkspaceConfigSchema"#; + let change = server_api::ConfigTreeChange::Update { + path: "main.dcdl".into(), + expected_digest: main.digest.clone().unwrap(), + content: source.into(), + }; + let error = workspace_config::commit( + api, + &fixture.worker, + commit_request(&attached.connection_id, &observed, vec![change]), + ) + .await + .unwrap_err(); + assert_eq!( + error.classification, + Classification::Unknown, + "repository projection can create a default credential before rejecting source/reference validation" + ); + assert_eq!( + api.config_store + .load_workspace_config(TEST_WORKSPACE_ID) + .unwrap() + .unwrap(), + before, + "canonical config stays unchanged" + ); + assert!( + count() > before_secrets, + "auxiliary credential side effect must be demonstrably durable; cannot label this NotCommitted" + ); + assert!(!error.message.contains("host_trust") && !error.message.contains("private")); + } + #[tokio::test] + async fn workspace_config_observe_node_limit_is_complete_or_error_not_silent_truncation() { + let fixture = manual_coder_assignment_fixture().await; + let api = &fixture.api; + grant(api, &fixture.worker, Access::ReadWrite).await; + let attached = attach(api, &fixture.worker).await; + let observed = observe(api, &fixture.worker, &attached.connection_id, &[""], 0).await; + let state = api + .config_store + .load_workspace_config(TEST_WORKSPACE_ID) + .unwrap() + .unwrap(); + let count = config_source::MAX_ENTRY_COUNT - state.snapshot.entries.len(); + let changes = (0..count) + .map(|i| create(&format!("bounded/{i}.md"), "text")) + .collect(); + workspace_config::commit( + api, + &fixture.worker, + commit_request(&attached.connection_id, &observed, changes), + ) + .await + .unwrap(); + let overflow = workspace_config::observe( + api, + &fixture.worker, + Observe { + connection_id: attached.connection_id.clone(), + paths: vec!["".into()], + depth: 8, + }, + ) + .await + .unwrap_err(); + assert_eq!(overflow.status, 413); + let root = observe(api, &fixture.worker, &attached.connection_id, &[""], 0).await; + assert_eq!(root.nodes.len(), 1); + let directory = observe( + api, + &fixture.worker, + &attached.connection_id, + &["bounded"], + 1, + ) + .await; + assert_eq!(directory.nodes.len(), count + 1); + assert!(directory.nodes.len() <= 256); + let direct = observe( + api, + &fixture.worker, + &attached.connection_id, + &["bounded/0.md"], + 0, + ) + .await; + assert_eq!( + direct.nodes[0], + *directory + .nodes + .iter() + .find(|node| node.path == "bounded/0.md") + .unwrap(), + "listing and direct lookup must expose identical metadata/auth/validator" + ); + } +} diff --git a/crates/workspace-server/src/server_workspace_config_wip_tests.rs b/crates/workspace-server/src/server_workspace_config_wip_tests.rs new file mode 100644 index 000000000..28a7c496e --- /dev/null +++ b/crates/workspace-server/src/server_workspace_config_wip_tests.rs @@ -0,0 +1,863 @@ +#[derive(Clone)] +struct ConfigNoopLlmClient { + api: WorkspaceApi, + worker: RuntimeWorkerRef, + calls: Arc, +} +#[async_trait] +impl agen::llm_client::LlmClient for ConfigNoopLlmClient { + fn clone_boxed(&self) -> Box { + Box::new(self.clone()) + } + async fn stream( + &self, + request: agen::llm_client::Request, + ) -> std::result::Result< + std::pin::Pin< + Box< + dyn futures::Stream< + Item = std::result::Result< + agen::llm_client::event::Event, + agen::llm_client::ClientError, + >, + > + Send, + >, + >, + agen::llm_client::ClientError, + > { + let grant = self + .api + .store + .current_workspace_config_grant(TEST_WORKSPACE_ID, &self.worker) + .unwrap() + .unwrap(); + assert!( + self.api + .store + .list_worker_workdir_links(TEST_WORKSPACE_ID, &self.worker) + .unwrap() + .iter() + .any(|link| link.unlinked_at.is_none() + && link.alias == "workspace-config" + && link.workdir_id == grant.working_directory_id), + "real Backend attachment must precede the dependent LLM request" + ); + assert!( + format!("{:?}", request.items).contains("Workspace configuration is available"), + "selected handler's preparation context must reach the actual LLM request" + ); + self.calls.fetch_add(1, Ordering::SeqCst); + Ok(Box::pin(futures::stream::empty())) + } +} +// Real production Feature/WIP adapters over a signed in-process Backend HTTP router. +// No fake config provider, copied tree, or fabricated attachment ledger is used. +#[derive(Clone)] +struct ConfigRouterClient { + api: WorkspaceApi, + worker: RuntimeWorkerRef, + identity: Arc, + runtime: tokio::runtime::Handle, + paths: Arc>>, + drop_next_commit_response: Arc, +} +impl std::fmt::Debug for ConfigRouterClient { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("Authenticated ConfigRouterClient") + } +} +impl worker::WorkspaceClient for ConfigRouterClient { + fn workspace_id(&self) -> Option<&str> { + Some(TEST_WORKSPACE_ID) + } + fn kind(&self) -> &str { + "authenticated-backend-router" + } + fn is_available(&self) -> bool { + true + } + fn execute( + &self, + request: worker::WorkspaceRequest, + ) -> std::result::Result { + use worker::{ + WorkspaceClientError as Failure, WorkspaceRequestMethod as Method, WorkspaceResponse, + }; + let method = match request.method { + Method::Get => "GET", + Method::Post => "POST", + Method::Put => "PUT", + Method::Patch => "PATCH", + Method::Delete => "DELETE", + }; + self.paths + .lock() + .unwrap() + .push(format!("{method} {}", request.path)); + let is_commit = request.path.ends_with("/commit"); + let api = self.api.clone(); + let identity = self.identity.clone(); + let worker = self.worker.clone(); + let runtime = self.runtime.clone(); + // Feature validation is synchronous, including on a Tokio thread. A separate + // thread enters the existing runtime instead of nesting a Tokio runtime. + let response = std::thread::spawn(move || { + runtime.block_on(async move { + let body = request.body.unwrap_or_default().into_bytes(); + let signed = runtime_source_request( + &identity, + Some(&worker.worker_id), + method, + &request.path, + body, + ); + let response = build_router(api) + .oneshot(signed) + .await + .map_err(|_| Failure::Request("router dispatch failed".into()))?; + let status = response.status().as_u16(); + let bytes = to_bytes(response.into_body(), 4 * 1024 * 1024) + .await + .map_err(|_| Failure::Request("response unavailable after dispatch".into()))?; + let body = String::from_utf8(bytes.to_vec()) + .map_err(|_| Failure::Request("invalid response".into()))?; + Ok(WorkspaceResponse { status, body }) + }) + }) + .join() + .map_err(|_| Failure::Request("dispatch completion unavailable".into()))??; + if is_commit + && response.is_success() + && self + .drop_next_commit_response + .swap(false, std::sync::atomic::Ordering::SeqCst) + { + return Err(Failure::Request( + "simulated lost response AFTER committed dispatch".into(), + )); + } + Ok(response) + } +} +fn config_interface(path: &str) -> String { + // Use the published contextual interface identity, not a model-managed validator. + let hex = path + .bytes() + .map(|byte| format!("{byte:02x}")) + .collect::(); + format!("yoi.workspace-config/node/v1/@/{hex}") +} +async fn config_discover(runtime: &worker::wip::WipRuntime, path: &str) { + let observed = runtime + .discover(path.into(), 0, true) + .await + .unwrap() + .content + .unwrap(); + assert!( + observed.contains(&config_interface(path)), + "interface must come from live discovery: {observed}" + ); + runtime.inspect(config_interface(path), true).await.unwrap(); +} +async fn config_call( + runtime: &worker::wip::WipRuntime, + path: &str, + operation: &str, + args: Value, +) -> std::result::Result { + runtime + .call( + path.into(), + config_interface(path), + operation.into(), + args, + agen::tool::ToolExecutionContext::direct(), + ) + .await +} + +async fn catalog_call( + runtime: &worker::wip::WipRuntime, + path: &str, + base: &str, + operation: &str, +) -> Value { + let hex = path + .bytes() + .map(|byte| format!("{byte:02x}")) + .collect::(); + let interface = format!("{base}/@/{hex}"); + let discovered = runtime + .discover(path.into(), 0, true) + .await + .unwrap() + .content + .unwrap(); + assert!(discovered.contains(&interface), "{discovered}"); + runtime.inspect(interface.clone(), true).await.unwrap(); + let output = runtime + .call( + path.into(), + interface, + operation.into(), + json!({}), + agen::tool::ToolExecutionContext::direct(), + ) + .await + .unwrap(); + serde_json::from_str(output.content.as_deref().unwrap()).unwrap() +} + +async fn assert_config_catalog(runtime: &worker::wip::WipRuntime, attached: &Attachment) { + let list = catalog_call( + runtime, + "/workdir-attachments", + "yoi.workdir-attachment/collection/v1", + "list", + ) + .await; + let listed = list["items"] + .as_array() + .unwrap() + .iter() + .find(|item| item["connection_id"] == attached.connection_id) + .unwrap(); + let read = catalog_call( + runtime, + &format!("/workdir-attachments/{}", attached.connection_id), + "yoi.workdir-attachment/item/v1", + "read", + ) + .await; + for item in [listed, &read] { + assert_eq!(item["connection_id"], attached.connection_id); + assert_eq!(item["working_directory_id"], attached.working_directory_id); + assert_eq!(item["alias"], attached.alias); + assert_eq!(item["content_path"], "/workspace-config"); + assert_eq!( + item["access"], + serde_json::to_value(attached.access).unwrap() + ); + assert_eq!(item["name"], attached.name); + assert_eq!(item["purpose"], attached.purpose); + assert!(item.get("checkout_path").is_none() && item.get("checkout_slug").is_none()); + } + let workdir = catalog_call( + runtime, + &format!("/workdirs/{}", attached.working_directory_id), + "yoi.workdir/item/v1", + "read", + ) + .await; + assert_eq!(workdir["source"]["kind"], "workspace_config"); + assert_eq!(workdir["source"]["content_path"], "/workspace-config"); + assert!(workdir["source"].get("grant_id").is_none()); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn workspace_config_real_selected_invoke_backend_wip_edit_create_delete_import_and_unknown() { + use worker::feature::builtin::workspace_config::wip::mount_workspace_config_wip; + use worker::feature::builtin::workspace_config::{CONTENT_ROOT, WorkspaceConfigFeature}; + let mut fixture = manual_coder_assignment_fixture().await; + let identity = RuntimeIdentityMaterial::generate(&fixture.worker.runtime_id).unwrap(); + configure_runtime_request_auth(&mut fixture.api, &identity, &fixture.worker.runtime_id); + let granted = grant(&fixture.api, &fixture.worker, Access::ReadWrite).await; + let client = Arc::new(ConfigRouterClient { + api: fixture.api.clone(), + worker: fixture.worker.clone(), + identity: Arc::new(identity), + runtime: tokio::runtime::Handle::current(), + paths: Default::default(), + drop_next_commit_response: Default::default(), + }); + let feature = WorkspaceConfigFeature::for_workspace(client.clone(), true); + let mut mounts = worker::wip::WipMountRegistry::new(); + mount_workspace_config_wip(&mut mounts, &feature).unwrap(); + worker::feature::builtin::manage_workdir::wip::mount_workspace_workdir_wip( + &mut mounts, + &worker::feature::builtin::manage_workdir::ManageWorkdirFeature::new(client.clone()), + true, + false, + None, + ) + .unwrap(); + let wip = worker::wip::WipRuntime::from_mounts( + mounts, + format!( + "{}:{}@{TEST_WORKSPACE_ID}", + fixture.worker.runtime_id, fixture.worker.worker_id + ), + ) + .unwrap(); + let manifest=worker::WorkerManifest::from_toml("[worker]\nname=\"config-e2e\"\n[model]\nscheme=\"anthropic\"\nmodel_id=\"unused\"\n[engine]\n[scope]\nallow=[]\n").unwrap(); + let calls = Arc::new(AtomicU64::new(0)); + let engine = + agen::Engine::<_, agen::state::Mutable, worker::SessionHistoryMetadata>::new_annotated( + ConfigNoopLlmClient { + api: fixture.api.clone(), + worker: fixture.worker.clone(), + calls: calls.clone(), + }, + ); + let store = + session_store::FsStore::new(fixture._workspace.path().join("config-e2e-sessions")).unwrap(); + let mut host = worker::Worker::new( + manifest, + engine, + store, + worker::WorkerWorkspaceContext::with_client( + Some(worker::WorkspaceId::new(TEST_WORKSPACE_ID).unwrap()), + client.clone(), + ), + worker::WorkerFilesystemAuthority::None, + manifest::Scope::empty(), + ) + .await + .unwrap(); + let report = + host.install_features(worker::feature::FeatureRegistryBuilder::new().with_module(feature)); + assert!( + report.installed_tool_names().is_empty() && !report.has_errors(), + "WIP-only installation cannot expose a compatibility config Tool" + ); + let selected = report + .chat_invocations + .feature_completions("workspace-c") + .remove(0) + .invocation + .unwrap(); + let invocation = protocol::FeatureInvocation { + invocation_id: "selected-config-invoke".into(), + identity: selected.identity, + name: selected.name, + arguments: vec![], + }; + assert!( + workspace_config::get(&fixture.api, &fixture.worker) + .await + .unwrap() + .is_none() + ); + host.run(vec![ + protocol::Segment::FeatureInvoke { + invocation: invocation.clone(), + }, + protocol::Segment::text("Edit this Workspace configuration"), + ]) + .await + .unwrap(); + assert_eq!( + calls.load(Ordering::SeqCst), + 1, + "production Worker run must issue its dependent LLM request only after preparation" + ); + let attached = workspace_config::get(&fixture.api, &fixture.worker) + .await + .unwrap() + .unwrap(); + assert_eq!(attached.working_directory_id, granted.working_directory_id); + assert_config_catalog(&wip, &attached).await; + assert!( + report + .chat_invocations + .invoke(&protocol::FeatureInvocation { + invocation_id: "repeat-config-invoke".into(), + ..invocation.clone() + }) + .await + .unwrap() + .message + .starts_with("Already attached") + ); + assert_eq!( + workspace_config::get(&fixture.api, &fixture.worker) + .await + .unwrap() + .unwrap() + .connection_id, + attached.connection_id + ); + config_discover(&wip, CONTENT_ROOT).await; + config_call( + &wip, + CONTENT_ROOT, + "create", + json!({"path":"notes/wip.md","content_type":"text","content":"original note"}), + ) + .await + .unwrap(); + let path = "/workspace-config/notes/wip.md"; + config_discover(&wip, path).await; + assert!( + config_call(&wip, path, "read", json!({})) + .await + .unwrap() + .content + .unwrap() + .contains("original note") + ); + config_call( + &wip, + path, + "edit", + json!({"old_string":"original","new_string":"edited"}), + ) + .await + .unwrap(); + let state = fixture + .api + .config_store + .load_workspace_config(TEST_WORKSPACE_ID) + .unwrap() + .unwrap(); + assert_eq!( + state + .snapshot + .get(&config_source::VirtualPath::parse("notes/wip.md").unwrap()) + .unwrap() + .content, + "edited note" + ); + config_call(&wip, path, "write", json!({"content":"replacement note"})) + .await + .unwrap(); + config_discover(&wip, CONTENT_ROOT).await; + config_call(&wip,CONTENT_ROOT,"apply_changes",json!({"changes":[ + {"kind":"create","path":"fragments/prompts.dcdl","content_type":"decodal","content":"{ common = { language = \"CONFIG_WIP_E2E\"; }; }"}, + {"kind":"update","path":"main.dcdl","content":"{ prompts = import \"./fragments/prompts.dcdl\"; } as WorkspaceConfigSchema"} + ]})).await.unwrap(); + // Confirm through ORIGINAL config reader/projection, not a copied WIP store. + let state = fixture + .api + .config_store + .load_workspace_config(TEST_WORKSPACE_ID) + .unwrap() + .unwrap(); + let prompts = crate::prompt_settings::project_prompts_from_workspace_config(&state).unwrap(); + assert_eq!(prompts.templates["common.language"], "CONFIG_WIP_E2E"); + config_discover(&wip, path).await; + config_call(&wip, path, "delete", json!({})).await.unwrap(); + assert!( + fixture + .api + .config_store + .load_workspace_config(TEST_WORKSPACE_ID) + .unwrap() + .unwrap() + .snapshot + .get(&config_source::VirtualPath::parse("notes/wip.md").unwrap()) + .is_none() + ); + config_discover(&wip, CONTENT_ROOT).await; + let before = fixture + .api + .config_store + .load_workspace_config(TEST_WORKSPACE_ID) + .unwrap() + .unwrap(); + let dispatched = client + .paths + .lock() + .unwrap() + .iter() + .filter(|path| path.ends_with("/commit")) + .count(); + client + .drop_next_commit_response + .store(true, std::sync::atomic::Ordering::SeqCst); + let unknown=config_call(&wip,CONTENT_ROOT,"create",json!({"path":"notes/lost-response.md","content_type":"text","content":"committed exactly once"})).await.unwrap_err(); + assert!(unknown.to_string().contains("unknown"), "{unknown}"); + let after = fixture + .api + .config_store + .load_workspace_config(TEST_WORKSPACE_ID) + .unwrap() + .unwrap(); + assert_eq!(after.snapshot.revision, before.snapshot.revision + 1); + assert!( + after + .snapshot + .get(&config_source::VirtualPath::parse("notes/lost-response.md").unwrap()) + .is_some() + ); + assert_eq!( + client + .paths + .lock() + .unwrap() + .iter() + .filter(|path| path.ends_with("/commit")) + .count(), + dispatched + 1, + "a lost response cannot trigger automatic retry" + ); + config_discover(&wip, CONTENT_ROOT).await; + let readonly = workspace_config::attach( + &fixture.api, + &fixture.worker, + Attach { + alias: None, + access: Some(Access::ReadOnly), + }, + ) + .await + .unwrap(); + assert_eq!(readonly.connection_id, attached.connection_id); + assert_config_catalog(&wip, &readonly).await; + workspace_config::revoke_grant( + &fixture.api, + &test_owner_actor(), + TEST_WORKSPACE_ID, + &granted.grant_id, + ) + .await + .unwrap(); + assert!( + config_call( + &wip, + CONTENT_ROOT, + "create", + json!({"path":"notes/revoked.md","content":"denied"}) + ) + .await + .is_err(), + "cached interface reference never grants access after revoke" + ); + let after_revoke = catalog_call( + &wip, + "/workdir-attachments", + "yoi.workdir-attachment/collection/v1", + "list", + ) + .await; + assert!( + after_revoke["items"] + .as_array() + .unwrap() + .iter() + .all(|item| item["connection_id"] != attached.connection_id) + ); + assert!( + client + .paths + .lock() + .unwrap() + .iter() + .all(|path| !path.contains("workdir-session") && !path.contains("bash")) + ); +} + +// Capture the client from an actual production embedded controller factory, +// rather than fabricating an "embedded" WorkspaceClient in the test. +struct ConfigCaptureFactory { + inner: worker_runtime::worker_backend::ProfileRuntimeWorkerFactory, + client: Arc>>>, + controller_abort: Arc>>, +} +#[async_trait] +impl worker_runtime::worker_backend::RuntimeWorkerFactory for ConfigCaptureFactory { + async fn spawn_controller( + &self, + mut request: worker_runtime::execution::WorkerExecutionSpawnRequest, + ) -> std::result::Result { + // No LLM request is issued; explicit unauthenticated test model avoids + // depending on developer credentials during production bootstrap. + let archive = worker_runtime::profile_archive::ProfileSourceArchive::build(worker_runtime::profile_archive::ProfileSourceArchiveInput { + id: "profile-source-archive:config-adapter-e2e".into(), + entrypoints: BTreeMap::from([("builtin:companion".into(), "profiles/default.dcdl".into())]), + imports: BTreeMap::new(), + sources: BTreeMap::from([("profiles/default.dcdl".into(), r#"{ + slug = "default"; description = "Config adapter test"; scope = "workspace_read"; + model = { scheme = "anthropic"; model_id = "test-model"; auth = { kind = "none"; }; }; + engine = { max_tokens = 100; }; + }"#.into())]), + }).map_err(|error| error.to_string())?; + request.request.profile_source = + worker_runtime::catalog::ProfileSourceArchiveSource::Embedded { archive }; + let controller = self.inner.spawn_controller(request).await?; + *self.client.lock().unwrap() = Some(controller.workspace_client.clone()); + *self.controller_abort.lock().unwrap() = Some(controller.controller_task.abort_handle()); + Ok(controller) + } + async fn restore_controller( + &self, + request: worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> std::result::Result { + self.inner.restore_controller(request).await + } +} + +async fn assert_config_production_http_adapter( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + client: Arc, +) { + use worker::feature::builtin::workspace_config::backend::WorkspaceConfigBackend; + use worker::feature::builtin::workspace_config::{ + WorkspaceConfigAttachmentBackend, WorkspaceConfigFeature, + }; + assert_eq!(client.kind(), "runtime-owned-workspace-client"); + let backend = WorkspaceConfigBackend::new(client.clone()); + assert!( + backend.current().is_err(), + "valid source proof cannot manufacture a config grant" + ); + grant(api, worker, Access::ReadWrite).await; + backend + .attach("production-adapter-test", None) + .await + .unwrap(); + let attached = workspace_config::get(api, worker).await.unwrap().unwrap(); + let feature = WorkspaceConfigFeature::for_workspace(client.clone(), true); + let mut mounts = worker::wip::WipMountRegistry::new(); + worker::feature::builtin::workspace_config::wip::mount_workspace_config_wip( + &mut mounts, + &feature, + ) + .unwrap(); + worker::feature::builtin::manage_workdir::wip::mount_workspace_workdir_wip( + &mut mounts, + &worker::feature::builtin::manage_workdir::ManageWorkdirFeature::new(client), + true, + false, + None, + ) + .unwrap(); + let wip = worker::wip::WipRuntime::from_mounts( + mounts, + format!( + "{}:{}@{TEST_WORKSPACE_ID}", + worker.runtime_id, worker.worker_id + ), + ) + .unwrap(); + assert_config_catalog(&wip, &attached).await; + config_discover(&wip, "/workspace-config/main.dcdl").await; + let main_interface = wip + .inspect(config_interface("/workspace-config/main.dcdl"), true) + .await + .unwrap() + .content + .unwrap(); + assert!( + !main_interface.contains("\"delete\""), + "required entrypoint cannot advertise delete: {main_interface}" + ); + assert!( + config_call(&wip, "/workspace-config/main.dcdl", "read", json!({})) + .await + .unwrap() + .content + .unwrap() + .contains("WorkspaceConfigSchema") + ); + config_discover(&wip, "/workspace-config").await; + config_call( + &wip, + "/workspace-config", + "create", + json!({"path":"notes/production-adapter.md","content_type":"text","content":"production source proof"}), + ) + .await + .unwrap(); + assert!( + api.config_store + .load_workspace_config(TEST_WORKSPACE_ID) + .unwrap() + .unwrap() + .snapshot + .get(&config_source::VirtualPath::parse("notes/production-adapter.md").unwrap()) + .is_some() + ); + workspace_config::revoke_grant( + api, + &test_owner_actor(), + TEST_WORKSPACE_ID, + &attached_grant_id(api, worker), + ) + .await + .unwrap(); + assert!(backend.current().is_err()); + assert!( + config_call( + &wip, + "/workspace-config", + "create", + json!({"path":"notes/revoked-adapter.md","content":"denied"}) + ) + .await + .is_err() + ); +} +fn attached_grant_id(api: &WorkspaceApi, worker: &RuntimeWorkerRef) -> String { + api.store + .current_workspace_config_grant(TEST_WORKSPACE_ID, worker) + .unwrap() + .unwrap() + .grant_id +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn workspace_config_actual_remote_and_embedded_clients_use_authenticated_workspace_request_scope() + { + use worker_runtime::auth::{ + RUNTIME_REQUEST_SOURCE_PROOF_HEADER, WORKSPACE_REQUEST_PERMISSION, + decode_runtime_request_source_claims, + }; + // Remote Runtime-owned HTTP adapter, including its real request signer. + let mut fixture = manual_coder_assignment_fixture().await; + let identity = RuntimeIdentityMaterial::generate(&fixture.worker.runtime_id).unwrap(); + configure_runtime_request_auth(&mut fixture.api, &identity, &fixture.worker.runtime_id); + let claims = Arc::new(Mutex::new(Vec::new())); + let capture = claims.clone(); + let app = build_router(fixture.api.clone()).layer(axum::middleware::from_fn( + move |request: Request, next: axum::middleware::Next| { + let capture = capture.clone(); + async move { + let proof = request + .headers() + .get(RUNTIME_REQUEST_SOURCE_PROOF_HEADER) + .unwrap() + .to_str() + .unwrap(); + capture + .lock() + .unwrap() + .push(decode_runtime_request_source_claims(proof).unwrap()); + next.run(request).await + } + }, + )); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + let client = worker_runtime::worker_source::RuntimeOwnedWorkspaceClient::new( + TEST_WORKSPACE_ID, + format!("http://{addr}"), + &fixture.worker.runtime_id, + &fixture.worker.worker_id, + ) + .with_runtime_request_source(&identity, "server-test"); + assert_config_production_http_adapter(&fixture.api, &fixture.worker, Arc::new(client)).await; + let observed = claims.lock().unwrap(); + assert!(observed.iter().any(|claim| claim.path.ends_with("/commit"))); + assert!( + observed + .iter() + .all(|claim| claim.permission == WORKSPACE_REQUEST_PERMISSION + && claim.iss == fixture.worker.runtime_id + && claim.worker_id.as_deref() == Some(fixture.worker.worker_id.as_str()) + && claim.workspace_id == TEST_WORKSPACE_ID) + ); + drop(observed); + let path = format!("/api/w/{TEST_WORKSPACE_ID}/workers/self/workspace-config"); + let wrong_scope = worker_runtime::auth::RuntimeRequestSourceSigner::from_identity(&identity) + .issue( + "server-test", + TEST_WORKSPACE_ID, + Some(&fixture.worker.worker_id), + worker_runtime::auth::BACKEND_RESOURCE_FETCH_PERMISSION, + "GET", + &path, + &[], + i64::try_from(worker_runtime::auth::unix_now_seconds()).unwrap(), + 30, + ) + .unwrap(); + let request = Request::builder() + .method("GET") + .uri(&path) + .header(RUNTIME_REQUEST_SOURCE_PROOF_HEADER, wrong_scope) + .body(Body::empty()) + .unwrap(); + assert_eq!( + build_router(fixture.api.clone()) + .oneshot(request) + .await + .unwrap() + .status(), + StatusCode::UNAUTHORIZED, + "valid signature under a different permission is not WorkspaceRequest authority" + ); + server.abort(); + + // Actual embedded Runtime factory -> real Worker controller -> captured + // production WorkspaceClient -> signed HTTP -> production Backend middleware. + let root = tempfile::tempdir().unwrap(); + init_clean_git_workspace(root.path()); + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let base = format!("http://{addr}"); + let mut config = test_server_config(root.path()); + config.backend_base_url = Some("server-test".into()); + let store = Arc::new(SqliteWorkspaceStore::open(&config.database_path).unwrap()); + let captured = Arc::new(Mutex::new(None)); + let abort = Arc::new(Mutex::new(None)); + let factory = ConfigCaptureFactory { + inner: worker_runtime::worker_backend::ProfileRuntimeWorkerFactory::new(root.path()) + .with_embedded_worker_mutation_dispatcher( + EMBEDDED_RUNTIME_ID, + Arc::new( + crate::worker_source::EmbeddedServerWorkerMutationDispatcher::new( + config.clone(), + store.clone(), + ), + ), + ) + .with_workspace_request_client( + worker_runtime::workspace_request::RuntimeWorkspaceRequestClient::new( + TEST_WORKSPACE_ID, + &base, + EMBEDDED_RUNTIME_ID, + ) + .with_runtime_request_source(&EMBEDDED_RUNTIME_REQUEST_IDENTITY, "server-test"), + ) + .with_runtime_store_dir(root.path().join("production-controller-store")), + client: captured.clone(), + controller_abort: abort.clone(), + }; + let execution = Arc::new( + worker_runtime::worker_backend::WorkerRuntimeExecutionBackend::new(factory).unwrap(), + ); + let mut api = WorkspaceApi::new_with_execution_backend(config, store, execution) + .await + .unwrap(); + configure_runtime_request_auth( + &mut api, + &EMBEDDED_RUNTIME_REQUEST_IDENTITY, + EMBEDDED_RUNTIME_ID, + ); + let server_api = api.clone(); + let server = tokio::spawn(async move { + axum::serve(listener, build_router(server_api)) + .await + .unwrap() + }); + let Json(created) = create_workspace_worker( + State(api.clone()), + HeaderMap::new(), + Json(CreateWorkspaceWorkerRequest { + runtime_id: EMBEDDED_RUNTIME_ID.into(), + display_name: "Production config adapter".into(), + singleton_key: None, + profile: Some("builtin:companion".into()), + ticket_assignment: None, + initial_submit: Vec::new(), + workdir_attachments: Vec::new(), + feature_connections: Default::default(), + control_operation_id: None, + }), + ) + .await + .unwrap(); + let worker = RuntimeWorkerRef::new(created.runtime_id, created.worker_id); + let client = captured + .lock() + .unwrap() + .take() + .expect("production embedded factory must inject a WorkspaceClient"); + assert_config_production_http_adapter(&api, &worker, client).await; + abort.lock().unwrap().take().unwrap().abort(); + server.abort(); +} diff --git a/crates/workspace-server/src/store.rs b/crates/workspace-server/src/store.rs index ecfebdec8..13d19cb7f 100644 --- a/crates/workspace-server/src/store.rs +++ b/crates/workspace-server/src/store.rs @@ -28,7 +28,8 @@ use crate::workspace_deletion::WorkspaceDeletionStore; use crate::{Error, Result}; const OLDEST_SCHEMA_VERSION: i64 = 50; -const LATEST_SCHEMA_VERSION: i64 = 77; +const LATEST_SCHEMA_VERSION: i64 = 81; +const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs"; const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline"; const WORKSPACE_RUNTIME_BINDINGS_MIGRATION_NAME: &str = "workspace runtime bindings"; const RUNTIME_BINDING_AUDIT_MIGRATION_NAME: &str = "workspace Runtime binding revision and audit"; @@ -77,6 +78,9 @@ const ARCHIVE_OBSERVE_GRANTS_MIGRATION_NAME: &str = "preserve explicit observe grants for committed Worker Session archives"; const BACKEND_JOB_WORKER_CLEANUP_MIGRATION_NAME: &str = "durable terminal Backend Job Worker cleanup"; +const WORKDIR_CONNECTION_ID_MIGRATION_NAME: &str = "durable Workdir attachment connection identity"; +const WORKER_RESTORE_INTENTS_MIGRATION_NAME: &str = + "durable internal Worker Restore request identity"; const TICKET_SCHEMA_VERSION_WITH_TARGETS: i64 = 7; const TICKET_SCHEMA_BASELINE_NAME: &str = "ticket schema baseline"; const WORKER_REGISTRY_PROJECTION_SCHEMA: &str = r#" @@ -259,6 +263,26 @@ const MIGRATIONS: &[Migration] = &[ name: BACKEND_JOB_WORKER_CLEANUP_MIGRATION_NAME, apply: migrate_backend_job_worker_cleanup_v76_to_v77, }, + Migration { + version: 78, + name: WORKDIR_CONNECTION_ID_MIGRATION_NAME, + apply: migrate_workdir_connection_id_v77_to_v78, + }, + Migration { + version: 79, + name: WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME, + apply: migrate_workspace_config_v78_to_v79, + }, + Migration { + version: 80, + name: WORKER_RESTORE_INTENTS_MIGRATION_NAME, + apply: migrate_worker_restore_intents_v79_to_v80, + }, + Migration { + version: 81, + name: "Backend Job immutable resource serialization", + apply: migrate_backend_job_resources_v80_to_v81, + }, ]; #[derive(Clone, Copy)] @@ -894,6 +918,9 @@ pub struct WorkdirCreateOperationRecord { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(tag = "kind", rename_all = "snake_case")] pub enum WorkdirRegistrySource { + WorkspaceConfig { + grant_id: String, + }, Repository { runtime_id: String, repository_id: String, @@ -907,20 +934,20 @@ impl WorkdirRegistrySource { pub fn runtime_id(&self) -> Option<&str> { match self { Self::Repository { runtime_id, .. } => Some(runtime_id), - Self::ExternalGrant { .. } => None, + Self::ExternalGrant { .. } | Self::WorkspaceConfig { .. } => None, } } pub fn repository_id(&self) -> Option<&str> { match self { Self::Repository { repository_id, .. } => Some(repository_id), - Self::ExternalGrant { .. } => None, + Self::ExternalGrant { .. } | Self::WorkspaceConfig { .. } => None, } } pub fn external_grant_id(&self) -> Option<&str> { match self { - Self::Repository { .. } => None, + Self::Repository { .. } | Self::WorkspaceConfig { .. } => None, Self::ExternalGrant { grant_id } => Some(grant_id), } } @@ -945,6 +972,23 @@ pub struct WorkdirRegistryRecord { pub updated_at: String, } +/// Projection inputs captured together with the catalog revision in one SQLite snapshot. +#[derive(Debug, Clone)] +pub struct WorkdirCatalogEntry { + pub record: WorkdirRegistryRecord, + pub repository_key: Option, + /// None for Repository records and unsupported legacy External grants. + pub external_grant_permissions: Option, + pub occupied_by: Option, +} + +#[derive(Debug, Clone)] +pub struct WorkdirCatalogPage { + pub entries: Vec, + pub next_cursor: Option, + pub revision: String, +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct ExternalWorkdirGrantRecord { pub grant_id: String, @@ -963,6 +1007,8 @@ pub struct ExternalWorkdirGrantRecord { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct WorkerWorkdirLinkRecord { + /// Assigned by the ledger on attach; preserved only by explicit compensation. + pub connection_id: String, pub workspace_id: String, pub worker: RuntimeWorkerRef, pub workdir_id: String, @@ -1542,6 +1588,20 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore { ) -> Result; fn get_backend_job(&self, workspace_id: &str, job_id: &str) -> Result>; + /// Pending/unknown Jobs and terminal Jobs with unfinished Worker cleanup + /// retain their resource. This is a query, not an independent lock ledger. + fn find_active_backend_job_for_resource( + &self, + workspace_id: &str, + resource_key: &str, + ) -> Result>; + /// Current dispatched Job bound to this Worker and Runtime run. Returns the + /// immutable grants for Backend authorization without trusting caller input. + fn get_active_backend_job_for_worker( + &self, + workspace_id: &str, + worker: &RuntimeWorkerRef, + ) -> Result>; fn get_backend_job_attempt( &self, workspace_id: &str, @@ -1888,8 +1948,31 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore { workspace_id: &str, limit: usize, ) -> Result>; + fn workdir_catalog_page( + &self, + workspace_id: &str, + limit: u32, + cursor: Option<&str>, + ) -> Result; fn delete_workdir_registry(&self, workspace_id: &str, workdir_id: &str) -> Result; + fn create_workspace_config_grant( + &self, + grant: &server_api::WorkspaceConfigGrantResponse, + actor: &str, + ) -> Result<()>; + fn get_workspace_config_grant( + &self, + workspace_id: &str, + grant_id: &str, + ) -> Result>; + fn current_workspace_config_grant( + &self, + workspace_id: &str, + worker: &RuntimeWorkerRef, + ) -> Result>; + fn revoke_workspace_config_grant(&self, workspace_id: &str, grant_id: &str) -> Result<()>; + fn create_external_workdir_grant( &self, grant: &ExternalWorkdirGrantRecord, @@ -1983,6 +2066,31 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore { expected_workdir_id: Option<&str>, unlinked_at: &str, ) -> Result>; + fn detach_worker_workdir_connection( + &self, + workspace_id: &str, + worker: &RuntimeWorkerRef, + alias: &str, + expected_connection_id: &str, + unlinked_at: &str, + ) -> Result; + fn restore_worker_workdir_connection(&self, record: &WorkerWorkdirLinkRecord) -> Result<()>; + fn list_worker_workdir_links_page( + &self, + workspace_id: &str, + worker: &RuntimeWorkerRef, + limit: u32, + offset: u32, + connection_id: Option<&str>, + ) -> Result>; + fn list_worker_workdir_links_page_with_revision( + &self, + workspace_id: &str, + worker: &RuntimeWorkerRef, + limit: u32, + offset: u32, + connection_id: Option<&str>, + ) -> Result<(Vec, String)>; fn worker_workdir_link_history_exists( &self, workspace_id: &str, @@ -2085,6 +2193,52 @@ impl SqliteWorkspaceStore { }) } + /// Client intent journal, not a lifecycle state machine: pin the public tuple + /// before dispatch and keep it unchanged until a terminal receipt is obtained. + pub(crate) fn pin_internal_worker_restore_intent( + &self, + workspace_id: &str, + worker: &RuntimeWorkerRef, + domain_key: &str, + observed_token: Option<&str>, + explicit_request_id: Option<&str>, + ) -> Result { + self.with_transaction(|tx| { + let existing: Option<(String, String)> = if let Some(id) = explicit_request_id { + tx.query_row("SELECT request_id, expected_token FROM worker_restore_intents WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND request_id=?4", + params![workspace_id, worker.runtime_id, worker.worker_id, id], |row| Ok((row.get(0)?, row.get(1)?))).optional()? + } else { + tx.query_row("SELECT request_id, expected_token FROM worker_restore_intents WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND domain_key=?4 AND settled=0", + params![workspace_id, worker.runtime_id, worker.worker_id, domain_key], |row| Ok((row.get(0)?, row.get(1)?))).optional()? + }; + if let Some((request_id, expected_observation_token)) = existing { + return Ok(server_api::WorkerRestoreRequest { request_id, expected_observation_token }); + } + let observed_token = observed_token.ok_or(Error::RestoreObservationConflict)?; + // A separate deliberate invocation after a settled conflict may + // still observe the same generation (e.g. another owner is pending). + // Give it a new identity rather than colliding with a retired row; + // retries of unsettled delivery always return the pinned tuple above. + let request_id = explicit_request_id.map(str::to_owned).unwrap_or_else(|| format!("{domain_key}:{}", uuid::Uuid::now_v7())); + tx.execute("INSERT INTO worker_restore_intents(workspace_id,runtime_id,worker_id,domain_key,request_id,expected_token,settled) VALUES(?1,?2,?3,?4,?5,?6,0)", + params![workspace_id, worker.runtime_id, worker.worker_id, domain_key, request_id, observed_token])?; + Ok(server_api::WorkerRestoreRequest { request_id, expected_observation_token: observed_token.to_owned() }) + }) + } + + pub(crate) fn settle_internal_worker_restore_intent( + &self, + workspace_id: &str, + worker: &RuntimeWorkerRef, + request_id: &str, + ) -> Result<()> { + self.with_conn(|conn| { + conn.execute("UPDATE worker_restore_intents SET settled=1 WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND request_id=?4", + params![workspace_id, worker.runtime_id, worker.worker_id, request_id])?; + Ok(()) + }) + } + pub(crate) fn get_workspace_memory_settings( &self, workspace_id: &str, @@ -6337,7 +6491,18 @@ impl ControlPlaneStore for SqliteWorkspaceStore { let attempt = read_backend_job_attempt(&tx, workspace_id, &request.job_id, &attempt_id)? .ok_or_else(|| Error::Store("Backend Job current attempt is missing".to_string()))?; tx.commit()?; - return Ok(BackendJobReservation { job, attempt, replayed: true }); + return Ok(BackendJobReservation { job, attempt, replayed: true, resource_reused: false }); + } + if let Some(key) = request.resource_key() + && let Some(job) = read_active_backend_job_for_resource(&tx, workspace_id, &key)? + { + let id = attempt_id(&job.request.job_id, job.current_attempt); + let attempt = read_backend_job_attempt(&tx, workspace_id, &job.request.job_id, &id)? + .ok_or_else(|| Error::Store("active Backend Job current attempt is missing".into()))?; + tx.commit()?; + return Ok(BackendJobReservation { + job, attempt, replayed: false, resource_reused: true, + }); } for (role, worker) in [ ("source", request.source_worker.as_ref()), @@ -6354,8 +6519,8 @@ impl ControlPlaneStore for SqliteWorkspaceStore { let current_attempt = 1_u8; let attempt_id = attempt_id(&request.job_id, current_attempt); tx.execute( - "INSERT INTO backend_jobs (workspace_id, job_id, purpose, input_revision, input_ref, request_json, intent_fingerprint, state, current_attempt, created_at, updated_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, 'pending', ?8, ?9, ?9)", - params![workspace_id, request.job_id, request.purpose, request.input_revision, request.input_ref, request_json, fingerprint, current_attempt as i64, now], + "INSERT INTO backend_jobs (workspace_id, job_id, purpose, input_revision, input_ref, request_json, intent_fingerprint, state, current_attempt, created_at, updated_at, resource_key) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, 'pending', ?8, ?9, ?9, ?10)", + params![workspace_id, request.job_id, request.purpose, request.input_revision, request.input_ref, request_json, fingerprint, current_attempt as i64, now, request.resource_key()], )?; tx.execute( "INSERT INTO backend_job_attempts (workspace_id, job_id, attempt_id, attempt, input_revision, state, deadline_at, created_at, updated_at) VALUES (?1, ?2, ?3, ?4, ?5, 'reserved', ?6, ?7, ?7)", @@ -6366,7 +6531,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { let attempt = read_backend_job_attempt(&tx, workspace_id, &request.job_id, &attempt_id)? .ok_or_else(|| Error::Store("reserved Backend Job attempt is missing".to_string()))?; tx.commit()?; - Ok(BackendJobReservation { job, attempt, replayed: false }) + Ok(BackendJobReservation { job, attempt, replayed: false, resource_reused: false }) }) } @@ -6383,12 +6548,25 @@ impl ControlPlaneStore for SqliteWorkspaceStore { let previous_id = attempt_id(job_id, job.current_attempt); let previous = read_backend_job_attempt(&tx, workspace_id, job_id, &previous_id)? .ok_or_else(|| Error::Store("Backend Job current attempt is missing".to_string()))?; - if previous.state != BackendJobAttemptState::Failed { + if job.state != BackendJobState::Failed || previous.state != BackendJobAttemptState::Failed { return Err(Error::InvalidInput( "Backend Job re-evaluation requires a definitively failed current attempt; unknown execution outcomes cannot be retried" .to_string(), )); } + if backend_job_has_unfinished_cleanup(&tx, workspace_id, job_id)? { + return Err(Error::InvalidInput( + "Backend Job retry requires completed Worker cleanup for every prior attempt".into(), + )); + } + if let Some(key) = job.request.resource_key() + && let Some(active) = read_active_backend_job_for_resource(&tx, workspace_id, &key)? + && active.request.job_id != job_id + { + return Err(Error::InvalidInput( + "Backend Job retry resource is reserved by another Job".into(), + )); + } if job.current_attempt >= job.request.limits.max_attempts { return Err(Error::InvalidInput(format!( "Backend Job `{job_id}` reached its attempt limit" @@ -6413,7 +6591,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, &next_id)? .ok_or_else(|| Error::Store("retried Backend Job attempt is missing".to_string()))?; tx.commit()?; - Ok(BackendJobReservation { job, attempt, replayed: false }) + Ok(BackendJobReservation { job, attempt, replayed: false, resource_reused: false }) }) } @@ -6425,6 +6603,36 @@ impl ControlPlaneStore for SqliteWorkspaceStore { self.with_conn(|conn| read_backend_job(conn, workspace_id, job_id)) } + fn find_active_backend_job_for_resource( + &self, + workspace_id: &str, + resource_key: &str, + ) -> Result> { + self.with_conn(|conn| { + read_active_backend_job_for_resource(conn, workspace_id, resource_key) + }) + } + + fn get_active_backend_job_for_worker( + &self, + workspace_id: &str, + worker: &RuntimeWorkerRef, + ) -> Result> { + self.with_conn(|conn| { + let id = conn.query_row( + "SELECT job.job_id FROM backend_jobs job JOIN backend_job_attempts attempt + ON attempt.workspace_id = job.workspace_id AND attempt.job_id = job.job_id + AND attempt.attempt = job.current_attempt + WHERE job.workspace_id = ?1 AND job.state = 'pending' + AND attempt.state = 'dispatched' AND attempt.runtime_id = ?2 AND attempt.worker_id = ?3 + AND attempt.runtime_run_id IS NOT NULL", + params![workspace_id, worker.runtime_id.as_str(), worker.worker_id.as_str()], + |row| row.get::<_, String>(0), + ).optional()?; + id.map(|id| read_backend_job(conn, workspace_id, &id)).transpose().map(Option::flatten) + }) + } + fn get_backend_job_attempt( &self, workspace_id: &str, @@ -6479,7 +6687,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { .ok_or_else(|| Error::InvalidInput(format!("unknown Backend Job attempt `{attempt_id}`")))?; if attempt.state != BackendJobAttemptState::Reserved { tx.commit()?; - return Ok((BackendJobReservation { job, attempt, replayed: true }, false)); + return Ok((BackendJobReservation { job, attempt, replayed: true, resource_reused: false }, false)); } let active: i64 = tx.query_row( "SELECT COUNT(*) FROM backend_job_attempts WHERE workspace_id = ?1 AND state IN ('dispatching', 'dispatched')", @@ -6488,7 +6696,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { )?; if active >= i64::from(job.request.limits.max_concurrent_jobs) { tx.commit()?; - return Ok((BackendJobReservation { job, attempt, replayed: true }, false)); + return Ok((BackendJobReservation { job, attempt, replayed: true, resource_reused: false }, false)); } let deadline = backend_job_deadline(now, job.request.limits.timeout_seconds)?; let changed = tx.execute( @@ -6501,7 +6709,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { let attempt = read_backend_job_attempt(&tx, workspace_id, job_id, attempt_id)? .ok_or_else(|| Error::Store("claimed Backend Job attempt disappeared".to_string()))?; tx.commit()?; - Ok((BackendJobReservation { job, attempt, replayed: false }, true)) + Ok((BackendJobReservation { job, attempt, replayed: false, resource_reused: false }, true)) }) } @@ -8793,13 +9001,13 @@ impl ControlPlaneStore for SqliteWorkspaceStore { fn upsert_workdir_registry(&self, record: &WorkdirRegistryRecord) -> Result<()> { self.with_conn(|conn| { - conn.execute( + let changed = conn.execute( r#"INSERT INTO workdir_registry ( workspace_id, workdir_id, display_name, source_kind, runtime_id, repository_id, external_grant_id, creation_selector, creation_ref, creation_tree, current_selector, current_ref, current_tree, observed_at_epoch_seconds, - materialization_status, cleanliness, created_at, updated_at - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18) + materialization_status, cleanliness, created_at, updated_at, workspace_config_grant_id + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19) ON CONFLICT(workspace_id, workdir_id) DO UPDATE SET display_name = excluded.display_name, source_kind = excluded.source_kind, @@ -8815,12 +9023,13 @@ impl ControlPlaneStore for SqliteWorkspaceStore { observed_at_epoch_seconds = excluded.observed_at_epoch_seconds, materialization_status = excluded.materialization_status, cleanliness = excluded.cleanliness, - updated_at = excluded.updated_at"#, + updated_at = excluded.updated_at, workspace_config_grant_id = excluded.workspace_config_grant_id + WHERE workdir_registry.source_kind != 'workspace_config' OR (excluded.source_kind = 'workspace_config' AND workdir_registry.workspace_config_grant_id = excluded.workspace_config_grant_id)"#, params![ record.workspace_id, record.workdir_id, record.display_name, - match &record.source { WorkdirRegistrySource::Repository { .. } => "repository", WorkdirRegistrySource::ExternalGrant { .. } => "external_grant" }, + match &record.source { WorkdirRegistrySource::Repository { .. } => "repository", WorkdirRegistrySource::ExternalGrant { .. } => "external_grant", WorkdirRegistrySource::WorkspaceConfig { .. } => "workspace_config" }, record.source.runtime_id(), record.source.repository_id(), record.source.external_grant_id(), @@ -8835,8 +9044,10 @@ impl ControlPlaneStore for SqliteWorkspaceStore { record.cleanliness, record.created_at, record.updated_at, + match &record.source { WorkdirRegistrySource::WorkspaceConfig { grant_id } => Some(grant_id.as_str()), _ => None }, ], )?; + if changed == 0 { return Err(Error::WorkspacePermissionDenied("Logical config source identity cannot be replaced by a Runtime observation".into())); } Ok(()) }) } @@ -8876,7 +9087,83 @@ impl ControlPlaneStore for SqliteWorkspaceStore { }) } + fn workdir_catalog_page( + &self, + workspace_id: &str, + limit: u32, + cursor: Option<&str>, + ) -> Result { + if limit == 0 || limit > 100 { + return Err(Error::InvalidInput( + "catalog page limit must be 1..=100".to_string(), + )); + } + if cursor.is_some_and(|id| { + id.trim().is_empty() || id.len() > 128 || id.chars().any(char::is_control) + }) { + return Err(Error::InvalidInput( + "catalog cursor must contain 1..=128 bytes without control characters".to_string(), + )); + } + self.with_conn(|conn| { + // Also fence writers using other SQLite connections, not only this mutex. + let tx = conn.unchecked_transaction()?; + let sql = workdir_catalog_select_sql(); + let mut hasher = Sha256::new(); + update_public_collection_digest(&mut hasher, &("workdir-catalog:v1", workspace_id))?; + { + let mut stmt = tx.prepare(&format!("{sql} WHERE wr.workspace_id = ?1 ORDER BY wr.workdir_id"))?; + let mut rows = stmt.query(params![workspace_id])?; + while let Some(row) = rows.next()? { + let (entry, connection_id) = read_workdir_catalog_entry(row)?; + let record = &entry.record; + // Safe projection fields and timestamps only: no paths, provider identity, + // credentials, Repository UUIDs, or raw internal records. + update_public_collection_digest(&mut hasher, &( + &record.workdir_id, &record.display_name, &entry.repository_key, + record.source.external_grant_id(), &entry.external_grant_permissions, + (&record.creation_selector, &record.creation_ref, &record.creation_tree), + (&record.current_selector, &record.current_ref, &record.current_tree), + record.observed_at_epoch_seconds, + (&record.materialization_status, &record.cleanliness), + (&record.created_at, &record.updated_at), + &entry.occupied_by, connection_id, + ))?; + } + } + let mut stmt = tx.prepare(&format!( + "{sql} WHERE wr.workspace_id = ?1 AND (?2 IS NULL OR wr.workdir_id > ?2) ORDER BY wr.workdir_id ASC LIMIT ?3" + ))?; + let mut rows = stmt.query(params![workspace_id, cursor, limit + 1])?; + let mut entries = Vec::new(); + while let Some(row) = rows.next()? { + entries.push(read_workdir_catalog_entry(row)?.0); + } + let next_cursor = if entries.len() > limit as usize { + entries.truncate(limit as usize); + entries.last().map(|entry| entry.record.workdir_id.clone()) + } else { + None + }; + Ok(WorkdirCatalogPage { + entries, + next_cursor, + revision: hasher.finalize().iter().map(|byte| format!("{byte:02x}")).collect::(), + }) + }) + } + fn delete_workdir_registry(&self, workspace_id: &str, workdir_id: &str) -> Result { + if self + .get_workdir_registry(workspace_id, workdir_id)? + .is_some_and(|record| { + matches!(record.source, WorkdirRegistrySource::WorkspaceConfig { .. }) + }) + { + return Err(Error::WorkspacePermissionDenied( + "Logical config Workdir cleanup is unsupported; revoke its grant".into(), + )); + } self.with_conn(|conn| { let tx = rusqlite::Transaction::new_unchecked( conn, @@ -8907,6 +9194,41 @@ impl ControlPlaneStore for SqliteWorkspaceStore { }) } + fn create_workspace_config_grant( + &self, + grant: &server_api::WorkspaceConfigGrantResponse, + actor: &str, + ) -> Result<()> { + self.with_conn_mut(|conn| { + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let active: bool = tx.query_row("SELECT EXISTS(SELECT 1 FROM workspaces WHERE workspace_id=?1 AND owner_account_id=?2 AND state='active')", params![grant.workspace_id, actor], |row|row.get(0))?; + if !active { return Err(Error::WorkspacePermissionDenied("Workspace no longer accepts new grants".into())); } + let now = chrono::Utc::now().to_rfc3339(); + let access = match grant.access { server_api::WorkspaceConfigAccess::ReadOnly => "read_only", server_api::WorkspaceConfigAccess::ReadWrite => "read_write" }; + tx.execute("INSERT INTO workspace_config_grants (workspace_id,grant_id,runtime_id,worker_id,workdir_id,access,revoked,created_by,created_at) VALUES (?1,?2,?3,?4,?5,?6,0,?7,?8)", params![grant.workspace_id,grant.grant_id,grant.runtime_id,grant.worker_id,grant.working_directory_id,access,actor,now])?; + tx.execute("INSERT INTO workdir_registry (workspace_id,workdir_id,display_name,source_kind,workspace_config_grant_id,materialization_status,cleanliness,created_at,updated_at) VALUES (?1,?2,'Workspace configuration','workspace_config',?3,'present','clean',?4,?4)", params![grant.workspace_id,grant.working_directory_id,grant.grant_id,now])?; + tx.commit()?; + Ok(()) + }) + } + fn get_workspace_config_grant( + &self, + workspace_id: &str, + grant_id: &str, + ) -> Result> { + self.with_conn(|conn| conn.query_row("SELECT workspace_id,grant_id,runtime_id,worker_id,workdir_id,access,revoked FROM workspace_config_grants WHERE workspace_id=?1 AND grant_id=?2", params![workspace_id,grant_id], read_workspace_config_grant).optional().map_err(Error::from)) + } + fn current_workspace_config_grant( + &self, + workspace_id: &str, + worker: &RuntimeWorkerRef, + ) -> Result> { + self.with_conn(|conn| conn.query_row("SELECT workspace_id,grant_id,runtime_id,worker_id,workdir_id,access,revoked FROM workspace_config_grants WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND revoked=0", params![workspace_id,worker.runtime_id,worker.worker_id], read_workspace_config_grant).optional().map_err(Error::from)) + } + fn revoke_workspace_config_grant(&self, workspace_id: &str, grant_id: &str) -> Result<()> { + self.with_conn_mut(|conn| { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; tx.execute("UPDATE workspace_config_grants SET revoked=1 WHERE workspace_id=?1 AND grant_id=?2", params![workspace_id,grant_id])?; tx.execute("UPDATE workdir_registry SET materialization_status='not_found', updated_at=?3 WHERE workspace_id=?1 AND workspace_config_grant_id=?2", params![workspace_id,grant_id,chrono::Utc::now().to_rfc3339()])?; tx.commit()?; Ok(()) }) + } + fn create_external_workdir_grant( &self, grant: &ExternalWorkdirGrantRecord, @@ -9349,6 +9671,8 @@ impl ControlPlaneStore for SqliteWorkspaceStore { "reserved attachment finalization requires an active attachment".to_string(), )); } + let mut record = record.clone(); + record.connection_id = uuid::Uuid::now_v7().to_string(); self.with_conn(|conn| { let tx = rusqlite::Transaction::new_unchecked( conn, @@ -9370,11 +9694,12 @@ impl ControlPlaneStore for SqliteWorkspaceStore { } tx.execute( r#"INSERT INTO worker_workdir_links ( - workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, linked_at, unlinked_at - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, NULL) + workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, linked_at, unlinked_at, connection_id + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, NULL, ?8) ON CONFLICT(workspace_id, runtime_id, worker_id, workdir_id, alias) DO UPDATE SET capabilities = excluded.capabilities, linked_at = excluded.linked_at, + connection_id = excluded.connection_id, unlinked_at = NULL"#, params![ record.workspace_id, @@ -9384,6 +9709,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { record.alias, encode_workdir_link_capabilities(record.capabilities)?, record.linked_at, + record.connection_id, ], ) .map_err(|error| { @@ -9421,6 +9747,8 @@ impl ControlPlaneStore for SqliteWorkspaceStore { "a new attachment cannot already be unlinked".to_string(), )); } + let mut record = record.clone(); + record.connection_id = uuid::Uuid::now_v7().to_string(); self.with_conn(|conn| { let tx = rusqlite::Transaction::new_unchecked( conn, @@ -9456,7 +9784,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { } let active_for_alias = tx .query_row( - r#"SELECT workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, linked_at, unlinked_at + r#"SELECT workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, linked_at, unlinked_at, connection_id FROM worker_workdir_links WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3 AND alias = ?4 AND unlinked_at IS NULL"#, @@ -9499,7 +9827,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { } let active_for_workdir = tx .query_row( - r#"SELECT workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, linked_at, unlinked_at + r#"SELECT workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, linked_at, unlinked_at, connection_id FROM worker_workdir_links WHERE workspace_id = ?1 AND workdir_id = ?2 AND unlinked_at IS NULL"#, params![record.workspace_id, record.workdir_id], @@ -9514,11 +9842,12 @@ impl ControlPlaneStore for SqliteWorkspaceStore { } let write = tx.execute( r#"INSERT INTO worker_workdir_links ( - workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, linked_at, unlinked_at - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, NULL) + workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, linked_at, unlinked_at, connection_id + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, NULL, ?8) ON CONFLICT(workspace_id, runtime_id, worker_id, workdir_id, alias) DO UPDATE SET capabilities = excluded.capabilities, linked_at = excluded.linked_at, + connection_id = excluded.connection_id, unlinked_at = NULL"#, params![ record.workspace_id, @@ -9528,6 +9857,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { record.alias, encode_workdir_link_capabilities(record.capabilities)?, record.linked_at, + record.connection_id, ], ); if let Err(error) = write { @@ -9556,7 +9886,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { rusqlite::TransactionBehavior::Immediate, )?; let mut stmt = tx.prepare( - r#"SELECT workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, linked_at, unlinked_at + r#"SELECT workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, linked_at, unlinked_at, connection_id FROM worker_workdir_links WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3 AND unlinked_at IS NULL"#, @@ -9571,7 +9901,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { let mut persisted_identity = persisted .iter() - .map(|link| (link.alias.as_str(), link.workdir_id.as_str())) + .map(|link| (link.alias.as_str(), link.workdir_id.as_str(), link.connection_id.as_str())) .collect::>(); persisted_identity.sort_unstable(); let mut requested_identity = active_links @@ -9592,7 +9922,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { link.alias )) })?; - Ok((link.alias.as_str(), link.workdir_id.as_str())) + Ok((link.alias.as_str(), link.workdir_id.as_str(), link.connection_id.as_str())) }) .collect::>>()?; requested_identity.sort_unstable(); @@ -9644,7 +9974,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { )?; let active = tx .query_row( - r#"SELECT workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, linked_at, unlinked_at + r#"SELECT workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, linked_at, unlinked_at, connection_id FROM worker_workdir_links WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3 AND (?4 IS NULL OR workdir_id = ?4) AND unlinked_at IS NULL"#, @@ -9683,6 +10013,202 @@ impl ControlPlaneStore for SqliteWorkspaceStore { }) } + fn detach_worker_workdir_connection( + &self, + workspace_id: &str, + worker: &RuntimeWorkerRef, + alias: &str, + expected_connection_id: &str, + unlinked_at: &str, + ) -> Result { + self.with_conn(|conn| { + let tx = rusqlite::Transaction::new_unchecked( + conn, + rusqlite::TransactionBehavior::Immediate, + )?; + let active = tx + .query_row( + r#"SELECT workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, + linked_at, unlinked_at, connection_id + FROM worker_workdir_links + WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3 + AND alias = ?4 AND connection_id = ?5 AND unlinked_at IS NULL"#, + params![ + workspace_id, + worker.runtime_id, + worker.worker_id, + alias, + expected_connection_id + ], + read_worker_workdir_link_record, + ) + .optional()? + .ok_or_else(|| { + Error::WorkdirAttachmentConflict(format!( + "Workdir attachment `{alias}` connection changed during detach" + )) + })?; + let changed = tx.execute( + r#"UPDATE worker_workdir_links SET unlinked_at = ?6 + WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3 + AND alias = ?4 AND connection_id = ?5 AND unlinked_at IS NULL"#, + params![ + workspace_id, + worker.runtime_id, + worker.worker_id, + alias, + expected_connection_id, + unlinked_at + ], + )?; + if changed != 1 { + return Err(Error::WorkdirAttachmentConflict(format!( + "Workdir attachment `{alias}` connection changed during detach" + ))); + } + tx.commit()?; + Ok(WorkerWorkdirLinkRecord { + unlinked_at: Some(unlinked_at.to_string()), + ..active + }) + }) + } + + fn restore_worker_workdir_connection(&self, record: &WorkerWorkdirLinkRecord) -> Result<()> { + self.with_conn(|conn| { + // Compensation reactivates only the same lifetime, never an intervening attach. + let changed = conn.execute( + r#"UPDATE worker_workdir_links SET unlinked_at = NULL + WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3 + AND workdir_id = ?4 AND alias = ?5 AND connection_id = ?6 + AND unlinked_at IS NOT NULL + AND NOT EXISTS ( + SELECT 1 FROM worker_workdir_attachment_reservations + WHERE workspace_id = ?1 AND workdir_id = ?4 + )"#, + params![record.workspace_id, record.worker.runtime_id, record.worker.worker_id, + record.workdir_id, record.alias, record.connection_id], + ).map_err(|error| { + if matches!(error, rusqlite::Error::SqliteFailure(ref code, _) if code.extended_code == rusqlite::ffi::SQLITE_CONSTRAINT_UNIQUE) { + Error::WorkdirAttachmentConflict("Workdir acquired another active attachment during compensation".to_string()) + } else { + error.into() + } + })?; + if changed != 1 { + return Err(Error::WorkdirAttachmentConflict( + "Workdir attachment connection changed during compensation".to_string(), + )); + } + Ok(()) + }) + } + + fn list_worker_workdir_links_page( + &self, + workspace_id: &str, + worker: &RuntimeWorkerRef, + limit: u32, + offset: u32, + connection_id: Option<&str>, + ) -> Result> { + self.list_worker_workdir_links_page_with_revision( + workspace_id, + worker, + limit, + offset, + connection_id, + ) + .map(|(links, _)| links) + } + + fn list_worker_workdir_links_page_with_revision( + &self, + workspace_id: &str, + worker: &RuntimeWorkerRef, + limit: u32, + offset: u32, + connection_id: Option<&str>, + ) -> Result<(Vec, String)> { + if limit == 0 || limit > 101 { + return Err(Error::InvalidInput( + "attachment page limit must be 1..=101".to_string(), + )); + } + if connection_id.is_some_and(|id| { + id.trim().is_empty() || id.len() > 128 || id.chars().any(char::is_control) + }) { + return Err(Error::InvalidInput( + "attachment connection_id must contain 1..=128 bytes without control characters" + .to_string(), + )); + } + self.with_conn(|conn| { + let tx = conn.unchecked_transaction()?; + let mut hasher = Sha256::new(); + update_public_collection_digest( + &mut hasher, + &( + "workdir-connections:v1", + workspace_id, + &worker.runtime_id, + &worker.worker_id, + ), + )?; + { + let mut stmt = tx.prepare( + "SELECT alias, connection_id, workdir_id, capabilities FROM worker_workdir_links + WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3 + AND unlinked_at IS NULL ORDER BY alias" + )?; + let mut rows = + stmt.query(params![workspace_id, worker.runtime_id, worker.worker_id])?; + while let Some(row) = rows.next()? { + let alias: String = row.get(0)?; + let id: String = row.get(1)?; + let workdir: String = row.get(2)?; + let capabilities = + decode_workdir_link_capabilities(&row.get::<_, String>(3)?, 3).map_err( + |_| Error::Store("invalid public connection capabilities".to_string()), + )?; + update_public_collection_digest( + &mut hasher, + &(alias, id, workdir, capabilities), + )?; + } + } + let mut stmt = tx.prepare( + r#"SELECT workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, + linked_at, unlinked_at, connection_id + FROM worker_workdir_links + WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3 + AND unlinked_at IS NULL + AND (?6 IS NULL OR connection_id = ?6) + ORDER BY alias LIMIT ?4 OFFSET ?5"#, + )?; + let rows = stmt.query_map( + params![ + workspace_id, + worker.runtime_id, + worker.worker_id, + limit, + offset, + connection_id + ], + read_worker_workdir_link_record, + )?; + let links = rows.collect::, _>>()?; + Ok(( + links, + hasher + .finalize() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::(), + )) + }) + } + fn worker_workdir_link_history_exists( &self, workspace_id: &str, @@ -9708,7 +10234,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { ) -> Result> { self.with_conn(|conn| { let mut stmt = conn.prepare( - r#"SELECT workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, linked_at, unlinked_at + r#"SELECT workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, linked_at, unlinked_at, connection_id FROM worker_workdir_links WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3 AND unlinked_at IS NULL ORDER BY linked_at DESC"#, @@ -9729,7 +10255,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { ) -> Result> { self.with_conn(|conn| { let mut stmt = conn.prepare( - r#"SELECT workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, linked_at, unlinked_at + r#"SELECT workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, linked_at, unlinked_at, connection_id FROM worker_workdir_links WHERE workspace_id = ?1 AND workdir_id = ?2 AND unlinked_at IS NULL ORDER BY linked_at DESC"#, @@ -9750,7 +10276,7 @@ impl ControlPlaneStore for SqliteWorkspaceStore { ) -> Result> { self.with_conn(|conn| { conn.query_row( - r#"SELECT workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, linked_at, unlinked_at + r#"SELECT workspace_id, runtime_id, worker_id, workdir_id, alias, capabilities, linked_at, unlinked_at, connection_id FROM worker_workdir_links WHERE workspace_id = ?1 AND workdir_id = ?2 ORDER BY linked_at DESC, rowid DESC @@ -10497,6 +11023,7 @@ fn read_worker_workdir_link_record( capabilities, linked_at: row.get(6)?, unlinked_at: row.get(7)?, + connection_id: row.get(8)?, }) } @@ -10579,6 +11106,41 @@ fn backend_job_deadline(now: &str, timeout_seconds: u32) -> Result { Ok((now + chrono::Duration::seconds(i64::from(timeout_seconds))).to_rfc3339()) } +fn backend_job_has_unfinished_cleanup( + conn: &Connection, + workspace_id: &str, + job_id: &str, +) -> Result { + Ok(conn.query_row( + "SELECT EXISTS(SELECT 1 FROM backend_job_attempts WHERE workspace_id = ?1 AND job_id = ?2 + AND worker_cleanup_state IS NOT 'completed')", + params![workspace_id, job_id], + |row| row.get(0), + )?) +} + +fn read_active_backend_job_for_resource( + conn: &Connection, + workspace_id: &str, + resource_key: &str, +) -> Result> { + // Resource ownership is computed from common Job/attempt authority. Unknown + // effects block even after Worker removal; cleanup is not outcome resolution. + let id = conn.query_row( + "SELECT job.job_id FROM backend_jobs job WHERE job.workspace_id = ?1 AND job.resource_key = ?2 + AND (job.state IN ('pending', 'unknown') OR EXISTS ( + SELECT 1 FROM backend_job_attempts attempt WHERE attempt.workspace_id = job.workspace_id + AND attempt.job_id = job.job_id + AND attempt.worker_cleanup_state IS NOT 'completed')) + ORDER BY job.created_at, job.job_id LIMIT 1", + params![workspace_id, resource_key], + |row| row.get::<_, String>(0), + ).optional()?; + id.map(|id| read_backend_job(conn, workspace_id, &id)) + .transpose() + .map(Option::flatten) +} + fn read_backend_job( conn: &Connection, workspace_id: &str, @@ -10847,6 +11409,7 @@ fn read_worker_registry_projection( })? } else { SubscriptionWorker { + restore_observation_token: None, worker_id: SubscriptionWorkerId::new(worker.worker_id.to_string()) .map_err(|error| Error::InvalidInput(error.to_string()))?, runtime_id: Some(worker.runtime_id.to_string()), @@ -11442,9 +12005,100 @@ fn require_expected_ticket_assignment( ))) } +// Length-framed canonical JSON prevents ambiguous concatenations and never hashes raw ledgers. +fn update_public_collection_digest(hasher: &mut Sha256, value: &impl Serialize) -> Result<()> { + let bytes = serde_json::to_vec(value) + .map_err(|_| Error::Store("unable to encode public collection metadata".to_string()))?; + hasher.update((bytes.len() as u64).to_be_bytes()); + hasher.update(bytes); + Ok(()) +} + +fn workdir_catalog_select_sql() -> String { + format!( + "SELECT wr.*, r.repository_key, g.permissions, + l.runtime_id, l.worker_id, w.display_name, l.linked_at, l.connection_id + FROM ({}) wr + LEFT JOIN repositories r ON r.workspace_id = wr.workspace_id AND r.repository_id = wr.repository_id + LEFT JOIN external_workdir_grants g ON g.workspace_id = wr.workspace_id AND g.grant_id = wr.external_grant_id + LEFT JOIN worker_workdir_links l ON l.rowid = ( + SELECT rowid FROM worker_workdir_links + WHERE workspace_id = wr.workspace_id AND workdir_id = wr.workdir_id AND unlinked_at IS NULL + ORDER BY linked_at DESC, rowid DESC LIMIT 1 + ) + LEFT JOIN worker_registry w ON w.workspace_id = l.workspace_id AND w.runtime_id = l.runtime_id AND w.worker_id = l.worker_id", + workdir_registry_select_sql("") + ) +} + +fn read_workdir_catalog_entry( + row: &rusqlite::Row<'_>, +) -> Result<(WorkdirCatalogEntry, Option)> { + let record = read_workdir_registry_record(row)?; + let repository_key: Option = row.get(18)?; + let external_grant_permissions = match &record.source { + WorkdirRegistrySource::Repository { .. } => { + if repository_key.is_none() { + return Err(Error::Store( + "Workdir catalog Repository is unavailable".to_string(), + )); + } + None + } + WorkdirRegistrySource::WorkspaceConfig { .. } => None, + WorkdirRegistrySource::ExternalGrant { .. } => { + let permissions: Option = row.get(19)?; + match permissions.as_deref() { + Some("read_only") => Some(workdir::workspace::WorkdirPermissionSummary { + read: true, + write: false, + command: false, + }), + Some("read_write") => Some(workdir::workspace::WorkdirPermissionSummary { + read: true, + write: true, + command: false, + }), + Some("read_write_command") => Some(workdir::workspace::WorkdirPermissionSummary { + read: true, + write: true, + command: true, + }), + Some("command_only" | "read_command") => None, + _ => { + return Err(Error::Store( + "Workdir catalog External grant permissions are unavailable".to_string(), + )); + } + } + } + }; + let runtime_id: Option = row.get(20)?; + let occupied_by = match runtime_id { + Some(runtime_id) => Some(workdir::workspace::WorkingDirectoryOccupancy { + runtime_id, + worker_id: row.get(21)?, + display_name: row.get::<_, Option>(22)?.ok_or_else(|| { + Error::Store("Workdir catalog occupant is unavailable".to_string()) + })?, + linked_at: row.get(23)?, + }), + None => None, + }; + Ok(( + WorkdirCatalogEntry { + record, + repository_key, + external_grant_permissions, + occupied_by, + }, + row.get(24)?, + )) +} + fn workdir_registry_select_sql(where_clause: &str) -> String { format!( - "SELECT workspace_id, workdir_id, display_name, source_kind, runtime_id, repository_id, external_grant_id, \ + "SELECT workspace_id, workdir_id, display_name, source_kind, runtime_id, repository_id, COALESCE(external_grant_id, workspace_config_grant_id) AS external_grant_id, \ creation_selector, creation_ref, creation_tree, \ current_selector, current_ref, current_tree, observed_at_epoch_seconds, \ materialization_status, cleanliness, created_at, updated_at \ @@ -11476,6 +12130,15 @@ fn read_workdir_registry_record( ) })?, }, + "workspace_config" => WorkdirRegistrySource::WorkspaceConfig { + grant_id: external_grant_id.clone().ok_or_else(|| { + rusqlite::Error::InvalidColumnType( + 6, + "workspace_config_grant_id".into(), + rusqlite::types::Type::Null, + ) + })?, + }, "external_grant" => WorkdirRegistrySource::ExternalGrant { grant_id: external_grant_id.ok_or_else(|| { rusqlite::Error::InvalidColumnType( @@ -11512,6 +12175,24 @@ fn read_workdir_registry_record( }) } +fn migrate_worker_restore_intents_v79_to_v80(conn: &Connection) -> Result<()> { + let tx = rusqlite::Transaction::new_unchecked(conn, TransactionBehavior::Exclusive)?; + tx.execute_batch("CREATE TABLE worker_restore_intents ( + workspace_id TEXT NOT NULL, runtime_id TEXT NOT NULL, worker_id TEXT NOT NULL, + domain_key TEXT NOT NULL, request_id TEXT NOT NULL, expected_token TEXT NOT NULL, + settled INTEGER NOT NULL CHECK(settled IN (0,1)), + PRIMARY KEY(workspace_id,runtime_id,worker_id,request_id), + FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE + ); + CREATE UNIQUE INDEX worker_restore_intent_pending_domain ON worker_restore_intents(workspace_id,runtime_id,worker_id,domain_key) WHERE settled=0;")?; + tx.execute( + "INSERT INTO __yoi_schema_migrations(version,name) VALUES(?1,?2)", + params![80_i64, WORKER_RESTORE_INTENTS_MIGRATION_NAME], + )?; + tx.commit()?; + Ok(()) +} + fn prepare_connection(conn: &Connection) -> Result<()> { configure_sqlite(conn)?; let workspace_schema_version = current_schema_version(conn)?; @@ -14447,6 +15128,27 @@ fn migrate_archive_observe_grants_v75_to_v76(conn: &Connection) -> Result<()> { Ok(()) } +fn migrate_backend_job_resources_v80_to_v81(conn: &Connection) -> Result<()> { + if current_schema_version(conn)? != 80 { + return Err(Error::Store( + "Backend Job resource migration requires schema 80".into(), + )); + } + let tx = rusqlite::Transaction::new_unchecked(conn, TransactionBehavior::Exclusive)?; + tx.execute_batch( + "ALTER TABLE backend_jobs ADD COLUMN resource_key TEXT; + CREATE INDEX backend_jobs_resource ON backend_jobs(workspace_id, resource_key, created_at);", + )?; + // Pre-grant requests had no resource lock. Preserve their exact stored JSON + // and fingerprint; serde defaults read them as empty grants / no key. + tx.execute( + "INSERT INTO __yoi_schema_migrations(version, name) VALUES (81, 'Backend Job immutable resource serialization')", + [], + )?; + tx.commit()?; + Ok(()) +} + fn migrate_backend_job_worker_cleanup_v76_to_v77(conn: &Connection) -> Result<()> { let current = current_schema_version(conn)?; if current != 76 { @@ -14486,7 +15188,122 @@ fn migrate_backend_job_worker_cleanup_v76_to_v77(conn: &Connection) -> Result<() Ok(()) } +fn read_workspace_config_grant( + row: &rusqlite::Row<'_>, +) -> rusqlite::Result { + let access: String = row.get(5)?; + Ok(server_api::WorkspaceConfigGrantResponse { + workspace_id: row.get(0)?, + grant_id: row.get(1)?, + runtime_id: row.get(2)?, + worker_id: row.get(3)?, + working_directory_id: row.get(4)?, + access: match access.as_str() { + "read_only" => server_api::WorkspaceConfigAccess::ReadOnly, + "read_write" => server_api::WorkspaceConfigAccess::ReadWrite, + _ => { + return Err(rusqlite::Error::InvalidColumnType( + 5, + "access".into(), + rusqlite::types::Type::Text, + )); + } + }, + revoked: row.get(6)?, + }) +} + +fn migrate_workspace_config_v78_to_v79(conn: &Connection) -> Result<()> { + if current_schema_version(conn)? != 78 { + return Err(Error::Store( + "Workspace config migration requires schema 78".into(), + )); + } + let foreign_keys = + conn.pragma_query_value(None, "foreign_keys", |row| row.get::<_, bool>(0))?; + conn.execute_batch("PRAGMA foreign_keys = OFF;")?; + let result = (|| { + let tx = rusqlite::Transaction::new_unchecked(conn, TransactionBehavior::Exclusive)?; + let dependents = { + let mut stmt = tx.prepare("SELECT sql FROM sqlite_schema WHERE tbl_name='workdir_registry' AND type IN ('index','trigger') AND sql IS NOT NULL")?; + let rows = stmt.query_map([], |row| row.get::<_, String>(0))?; + rows.collect::>>()? + }; + let schema = include_str!("latest_schema.sql"); + let start = schema + .find("CREATE TABLE \"workdir_registry\"") + .ok_or_else(|| Error::Store("Missing Workdir schema".into()))?; + let end = schema[start..] + .find("CREATE TABLE external_workdir_grants") + .ok_or_else(|| Error::Store("Missing Workdir schema end".into()))? + + start; + if column_exists(&tx, "workdir_registry", "workspace_config_grant_id")? + && table_exists(&tx, "workspace_config_grants")? + { + tx.execute("INSERT INTO __yoi_schema_migrations(version,name) VALUES (79,'Workspace config grants and logical Workdirs')", [])?; + tx.commit()?; + return Ok(()); + } + tx.execute_batch(include_str!("workspace_config_grants.sql"))?; + tx.execute_batch(&schema[start..end].replace( + "CREATE TABLE \"workdir_registry\"", + "CREATE TABLE workdir_registry_v79", + ))?; + tx.execute_batch("INSERT INTO workdir_registry_v79 (workspace_id,workdir_id,display_name,source_kind,runtime_id,repository_id,external_grant_id,creation_selector,creation_ref,creation_tree,current_selector,current_ref,current_tree,observed_at_epoch_seconds,materialization_status,cleanliness,created_at,updated_at) SELECT workspace_id,workdir_id,display_name,source_kind,runtime_id,repository_id,external_grant_id,creation_selector,creation_ref,creation_tree,current_selector,current_ref,current_tree,observed_at_epoch_seconds,materialization_status,cleanliness,created_at,updated_at FROM workdir_registry; DROP TABLE workdir_registry; ALTER TABLE workdir_registry_v79 RENAME TO workdir_registry;")?; + for sql in dependents { + tx.execute_batch(&sql)?; + } + let broken: bool = { + let mut stmt = tx.prepare("PRAGMA foreign_key_check")?; + let mut rows = stmt.query([])?; + rows.next()?.is_some() + }; + if broken { + return Err(Error::Store( + "Workspace config migration foreign-key verification failed".into(), + )); + } + tx.execute("INSERT INTO __yoi_schema_migrations(version,name) VALUES (79,'Workspace config grants and logical Workdirs')", [])?; + tx.commit()?; + Ok(()) + })(); + if foreign_keys { + conn.execute_batch("PRAGMA foreign_keys = ON;")?; + } + result +} + +fn migrate_workdir_connection_id_v77_to_v78(conn: &Connection) -> Result<()> { + use rusqlite::TransactionBehavior; + let current = current_schema_version(conn)?; + if current != 77 { + return Err(Error::Store(format!( + "expected schema version 77 before {WORKDIR_CONNECTION_ID_MIGRATION_NAME} migration, found {current}" + ))); + } + let tx = rusqlite::Transaction::new_unchecked(conn, TransactionBehavior::Exclusive)?; + tx.execute_batch( + "ALTER TABLE worker_workdir_links ADD COLUMN connection_id TEXT NOT NULL DEFAULT ''; + UPDATE worker_workdir_links SET connection_id = lower(hex(randomblob(16)));", + )?; + tx.execute( + "INSERT INTO __yoi_schema_migrations (version, name) VALUES (?1, ?2)", + params![78_i64, WORKDIR_CONNECTION_ID_MIGRATION_NAME], + )?; + tx.commit()?; + Ok(()) +} + fn create_latest_workspace_schema(conn: &Connection) -> Result<()> { + conn.execute_batch("CREATE TABLE worker_restore_intents ( + workspace_id TEXT NOT NULL, runtime_id TEXT NOT NULL, worker_id TEXT NOT NULL, + domain_key TEXT NOT NULL, request_id TEXT NOT NULL, expected_token TEXT NOT NULL, + settled INTEGER NOT NULL CHECK(settled IN (0,1)), + PRIMARY KEY(workspace_id,runtime_id,worker_id,request_id), + FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE + ); + CREATE UNIQUE INDEX worker_restore_intent_pending_domain ON worker_restore_intents(workspace_id,runtime_id,worker_id,domain_key) WHERE settled=0;")?; + conn.execute_batch("DROP TABLE IF EXISTS typed_ticket_targets;")?; conn.execute_batch(include_str!("latest_schema.sql"))?; conn.execute_batch( @@ -14546,6 +15363,7 @@ fn create_latest_workspace_schema(conn: &Connection) -> Result<()> { ); CREATE INDEX typed_ticket_targets_workspace_repository ON typed_ticket_targets(workspace_id, repository_key, ticket_id); + ALTER TABLE worker_workdir_links ADD COLUMN connection_id TEXT NOT NULL DEFAULT ''; ALTER TABLE worker_workdir_links ADD COLUMN capabilities TEXT NOT NULL CHECK (capabilities IN ( @@ -15348,6 +16166,7 @@ fn migrate_workdir_credential_candidate_snapshots_v58_to_v59(conn: &Connection) #[cfg(test)] mod tests { + include!("store_workspace_config_tests.rs"); use super::*; #[test] @@ -15593,6 +16412,7 @@ mod tests { } fn downgrade_schema_66_ticket_and_workdir_authority(conn: &Connection) -> Result<()> { + conn.execute_batch("DROP TABLE IF EXISTS worker_restore_intents;")?; conn.execute_batch( "DROP TABLE backend_job_deliveries; DROP TABLE backend_job_attempts; @@ -15610,6 +16430,46 @@ mod tests { Ok(()) } + #[test] + fn workdir_connection_migration_backfills_distinct_durable_lifetimes() { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch( + "CREATE TABLE __yoi_schema_migrations (version INTEGER PRIMARY KEY, name TEXT NOT NULL); + INSERT INTO __yoi_schema_migrations VALUES (77, 'workspace schema baseline'); + CREATE TABLE worker_workdir_links (alias TEXT, linked_at TEXT, unlinked_at TEXT); + INSERT INTO worker_workdir_links VALUES ('checkout', 'same-time', NULL), ('docs', 'same-time', 'removed');", + ).unwrap(); + migrate_workdir_connection_id_v77_to_v78(&conn).unwrap(); + assert_eq!(current_schema_version(&conn).unwrap(), 78); + let rows = || { + let mut stmt = conn.prepare("SELECT alias, linked_at, unlinked_at, connection_id FROM worker_workdir_links ORDER BY alias").unwrap(); + stmt.query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, Option>(2)?, + row.get::<_, String>(3)?, + )) + }) + .unwrap() + .collect::, _>>() + .unwrap() + }; + let migrated = rows(); + assert_eq!(migrated[0].0, "checkout"); + assert_eq!(migrated[0].1, "same-time"); + assert_eq!(migrated[0].2, None); + assert_eq!(migrated[1].2.as_deref(), Some("removed")); + assert!(!migrated[0].3.is_empty()); + assert_ne!(migrated[0].3, migrated[1].3); + assert!(migrate_workdir_connection_id_v77_to_v78(&conn).is_err()); + assert_eq!( + rows(), + migrated, + "a retry cannot replace an already durable connection ID" + ); + } + #[test] fn worker_workdir_link_capability_codec_round_trips_only_canonical_sets() { for (capabilities, encoded) in WORKDIR_LINK_CAPABILITY_ENCODINGS { @@ -17052,6 +17912,22 @@ mod tests { version: 77, name: BACKEND_JOB_WORKER_CLEANUP_MIGRATION_NAME.to_string(), }, + WorkspaceSchemaMigrationStep { + version: 78, + name: WORKDIR_CONNECTION_ID_MIGRATION_NAME.to_string(), + }, + WorkspaceSchemaMigrationStep { + version: 79, + name: WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME.to_string() + }, + WorkspaceSchemaMigrationStep { + version: 80, + name: WORKER_RESTORE_INTENTS_MIGRATION_NAME.to_string() + }, + WorkspaceSchemaMigrationStep { + version: 81, + name: "Backend Job immutable resource serialization".to_string() + }, ] ); @@ -17139,6 +18015,10 @@ mod tests { 77, BACKEND_JOB_WORKER_CLEANUP_MIGRATION_NAME.to_string(), ), + (78, WORKDIR_CONNECTION_ID_MIGRATION_NAME.to_string()), + (79, WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME.to_string()), + (80, WORKER_RESTORE_INTENTS_MIGRATION_NAME.to_string()), + (81, "Backend Job immutable resource serialization".to_string()), ] ); assert!(!table_exists(conn, "trusted_runtime_records")?); @@ -17490,7 +18370,7 @@ mod tests { .collect::>(), vec![ 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, - 73, 74, 75, 76, 77 + 73, 74, 75, 76, 77, 78, 79, 80, 81 ] ); SqliteWorkspaceStore::migrate_database(&path).unwrap(); @@ -17499,7 +18379,7 @@ mod tests { current_schema_version(&conn).unwrap(), LATEST_SCHEMA_VERSION ); - assert_eq!(workspace_schema_migration_history(&conn).unwrap().len(), 28); + assert_eq!(workspace_schema_migration_history(&conn).unwrap().len(), 32); assert!(column_exists(&conn, "worker_workdir_links", "alias").unwrap()); assert!(column_exists(&conn, "worker_workdir_links", "capabilities").unwrap()); assert!(!column_exists(&conn, "worker_workdir_links", "role").unwrap()); @@ -20855,6 +21735,7 @@ INSERT INTO worker_registry ( store.upsert_workdir_registry(&unmanaged_workdir).unwrap(); let link = WorkerWorkdirLinkRecord { + connection_id: String::new(), workspace_id: "local-dev".to_string(), worker: worker.worker.clone(), workdir_id: workdir.workdir_id.clone(), @@ -20883,12 +21764,10 @@ INSERT INTO worker_registry ( store.attach_worker_workdir(&link), Err(Error::WorkdirAttachmentConflict(_)) )); - assert_eq!( - store - .finalize_reserved_worker_workdir_attachment(&link, "spawn-1") - .unwrap(), - link - ); + let link = store + .finalize_reserved_worker_workdir_attachment(&link, "spawn-1") + .unwrap(); + assert!(!link.connection_id.is_empty()); assert_eq!(store.attach_worker_workdir(&link).unwrap(), link); let downgraded_link = WorkerWorkdirLinkRecord { capabilities: workdir::WorkdirSessionCapabilities::READ_ONLY, @@ -20958,10 +21837,7 @@ INSERT INTO worker_registry ( store.attach_worker_workdir(&invalid_capabilities), Err(Error::InvalidInput(_)) )); - assert_eq!( - store.attach_worker_workdir(&second_attachment).unwrap(), - second_attachment - ); + let second_attachment = store.attach_worker_workdir(&second_attachment).unwrap(); assert_eq!( store .list_worker_workdir_links("local-dev", &worker.worker) @@ -20991,6 +21867,157 @@ INSERT INTO worker_registry ( store.detach_worker_workdir("local-dev", &worker.worker, Some("wrong-workdir"), "6",), Err(Error::WorkdirAttachmentConflict(_)) )); + let old_connection = downgraded_link.connection_id.clone(); + let removed = store + .detach_worker_workdir_connection( + "local-dev", + &worker.worker, + "checkout", + &old_connection, + "5", + ) + .unwrap(); + store.restore_worker_workdir_connection(&removed).unwrap(); + assert_eq!( + store + .list_worker_workdir_links_page("local-dev", &worker.worker, 1, 0, None) + .unwrap()[0], + downgraded_link + ); + store + .detach_worker_workdir_connection( + "local-dev", + &worker.worker, + "checkout", + &old_connection, + "5", + ) + .unwrap(); + // Reusing the exact record, including alias, Workdir ID and timestamp, starts a fresh lifetime. + let replacement = store.attach_worker_workdir(&downgraded_link).unwrap(); + assert_ne!(replacement.connection_id, old_connection); + assert!(matches!( + store.detach_worker_workdir_connection( + "local-dev", + &worker.worker, + "checkout", + &old_connection, + "6", + ), + Err(Error::WorkdirAttachmentConflict(_)) + )); + assert!(matches!( + store.restore_worker_workdir_connection(&removed), + Err(Error::WorkdirAttachmentConflict(_)) + )); + assert!(matches!( + store.replace_worker_workdir_link_capabilities( + "local-dev", + &worker.worker, + &[downgraded_link.clone(), second_attachment.clone()], + ), + Err(Error::WorkdirAttachmentConflict(_)) + )); + assert_eq!(store.attach_worker_workdir(&link).unwrap(), replacement); + assert_eq!( + store + .list_worker_workdir_links_page("local-dev", &worker.worker, 1, 0, None) + .unwrap(), + vec![replacement.clone()] + ); + assert_eq!( + store + .list_worker_workdir_links_page("local-dev", &worker.worker, 1, 1, None) + .unwrap(), + vec![second_attachment.clone()] + ); + assert!( + store + .list_worker_workdir_links_page("local-dev", &worker.worker, 1, 2, None) + .unwrap() + .is_empty() + ); + assert!( + store + .list_worker_workdir_links_page("foreign", &worker.worker, 1, 0, None) + .unwrap() + .is_empty() + ); + assert!( + store + .list_worker_workdir_links_page( + "local-dev", + &RuntimeWorkerRef::new("foreign", worker.worker.worker_id.clone()), + 1, + 0, + None + ) + .unwrap() + .is_empty() + ); + assert_eq!( + store + .list_worker_workdir_links_page( + "local-dev", + &worker.worker, + 1, + 0, + Some(&second_attachment.connection_id) + ) + .unwrap(), + vec![second_attachment.clone()], + "connection filtering must precede paging" + ); + for (workspace_id, identity, connection_id) in [ + ( + "foreign", + worker.worker.clone(), + second_attachment.connection_id.as_str(), + ), + ( + "local-dev", + second_worker.worker.clone(), + second_attachment.connection_id.as_str(), + ), + ( + "local-dev", + RuntimeWorkerRef::new("foreign", worker.worker.worker_id.clone()), + second_attachment.connection_id.as_str(), + ), + ("local-dev", worker.worker.clone(), old_connection.as_str()), + ("local-dev", worker.worker.clone(), "nonexistent"), + ] { + assert!( + store + .list_worker_workdir_links_page( + workspace_id, + &identity, + 1, + 0, + Some(connection_id) + ) + .unwrap() + .is_empty() + ); + } + drop(store); + let store = SqliteWorkspaceStore::open(&db).unwrap(); + assert_eq!( + store + .list_worker_workdir_links_page("local-dev", &worker.worker, 1, 0, None) + .unwrap(), + vec![replacement] + ); + assert!(matches!( + store.detach_worker_workdir_connection( + "local-dev", + &worker.worker, + "checkout", + &old_connection, + "6", + ), + Err(Error::WorkdirAttachmentConflict(_)) + )); let detached = store .detach_worker_workdir("local-dev", &worker.worker, Some(&workdir.workdir_id), "6") .unwrap() @@ -21001,10 +22028,7 @@ INSERT INTO worker_registry ( .worker_workdir_link_history_exists("local-dev", &worker.worker) .unwrap() ); - assert_eq!( - store.attach_worker_workdir(&workdir_conflict).unwrap(), - workdir_conflict - ); + let workdir_conflict = store.attach_worker_workdir(&workdir_conflict).unwrap(); drop(store); let reopened = SqliteWorkspaceStore::open(&db).unwrap(); @@ -21057,6 +22081,7 @@ INSERT INTO worker_registry ( for record in [&runtime_a, &runtime_b] { let observed = SubscriptionWorker { + restore_observation_token: None, worker_id: SubscriptionWorkerId::new("shared-local-id").unwrap(), runtime_id: Some(record.worker.runtime_id.clone()), resource_key: None, @@ -21193,6 +22218,7 @@ INSERT INTO worker_registry ( Some("W-1") ); let observed = SubscriptionWorker { + restore_observation_token: None, worker_id: SubscriptionWorkerId::new("known").unwrap(), runtime_id: Some("embedded".to_string()), resource_key: None, @@ -21213,6 +22239,7 @@ INSERT INTO worker_registry ( }; store.upsert_worker_registry(&other_catalog).unwrap(); let other_observed = SubscriptionWorker { + restore_observation_token: None, worker_id: SubscriptionWorkerId::new("other").unwrap(), runtime_id: Some("runtime-other".to_string()), display_name: Some("Other Runtime Worker".to_string()), @@ -21222,6 +22249,7 @@ INSERT INTO worker_registry ( .apply_worker_registry_observation("local-dev", "runtime-other", &other_observed, "2") .unwrap(); let orphan = SubscriptionWorker { + restore_observation_token: None, worker_id: SubscriptionWorkerId::new("orphan").unwrap(), ..observed.clone() }; @@ -21387,6 +22415,7 @@ INSERT INTO worker_registry ( .is_none() ); let event = SubscriptionWorker { + restore_observation_token: None, worker_id: SubscriptionWorkerId::new("removed").unwrap(), runtime_id: Some("embedded".to_string()), resource_key: None, @@ -21783,6 +22812,816 @@ INSERT INTO worker_registry ( ); } + const RESOURCE_JOB_NOW: &str = "2026-09-01T00:00:00Z"; + + async fn seed_resource_job_workspace(store: &SqliteWorkspaceStore) { + store + .upsert_account(&AccountRecord { + account_id: "resource-owner".into(), + kind: "user".into(), + handle: "resource-owner".into(), + display_name: "Resource Owner".into(), + created_at: RESOURCE_JOB_NOW.into(), + updated_at: RESOURCE_JOB_NOW.into(), + }) + .unwrap(); + store + .upsert_workspace(&WorkspaceRecord { + workspace_id: "resource-workspace".into(), + owner_account_id: "resource-owner".into(), + display_name: "Resource Workspace".into(), + state: "active".into(), + created_at: RESOURCE_JOB_NOW.into(), + updated_at: RESOURCE_JOB_NOW.into(), + }) + .await + .unwrap(); + } + + fn resource_job_request(id: &str, subject: &str, candidate: &str) -> BackendJobRequest { + BackendJobRequest { + job_id: id.into(), + purpose: "subjektiv_consolidation".into(), + input_revision: candidate.into(), + input_ref: format!("subject://{subject}/{candidate}"), + input: serde_json::json!({"candidate": candidate}), + instruction: "Consolidate this immutable candidate snapshot.".into(), + profile: "project:custom-consolidation".into(), + grants: crate::backend_job::BackendJobGrants { + subjektiv_consolidation: Some(crate::backend_job::SubjektivConsolidationGrant { + subject_id: subject.into(), + candidate_ids: vec![candidate.into()], + }), + }, + serialization_key: None, + source_worker: None, + notification_target: None, + limits: Default::default(), + } + } + + fn bind_resource_job_worker( + store: &SqliteWorkspaceStore, + reservation: &BackendJobReservation, + id: &str, + ) -> RuntimeWorkerRef { + let worker = RuntimeWorkerRef::new("embedded-worker-runtime", id); + store + .upsert_worker_registry(&WorkerRegistryRecord { + workspace_id: "resource-workspace".into(), + worker: worker.clone(), + display_name: id.into(), + profile: Some(reservation.job.request.profile.clone()), + retention_state: "normal".into(), + transcript_ref: None, + session_ref: None, + summary_ref: None, + diagnostics_ref: None, + created_at: RESOURCE_JOB_NOW.into(), + updated_at: RESOURCE_JOB_NOW.into(), + }) + .unwrap(); + let (_, claimed) = store + .claim_backend_job_attempt_dispatch( + "resource-workspace", + &reservation.job.request.job_id, + &reservation.attempt.attempt_id, + RESOURCE_JOB_NOW, + ) + .unwrap(); + assert!(claimed); + store + .bind_backend_job_attempt_worker( + "resource-workspace", + &reservation.job.request.job_id, + &reservation.attempt.attempt_id, + &worker, + Some("resource-run"), + RESOURCE_JOB_NOW, + ) + .unwrap(); + worker + } + + fn finish_resource_job_cleanup( + store: &SqliteWorkspaceStore, + reservation: &BackendJobReservation, + ) { + let (_, claimed) = store + .claim_backend_job_worker_cleanup( + "resource-workspace", + &reservation.job.request.job_id, + &reservation.attempt.attempt_id, + RESOURCE_JOB_NOW, + ) + .unwrap(); + assert!(claimed); + store + .finish_backend_job_worker_cleanup( + "resource-workspace", + &reservation.job.request.job_id, + &reservation.attempt.attempt_id, + BackendJobWorkerCleanupState::Completed, + None, + None, + RESOURCE_JOB_NOW, + ) + .unwrap(); + } + + #[tokio::test] + async fn backend_job_resource_reserve_converges_across_connections_without_changing_input() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("resource-jobs.sqlite"); + let store = SqliteWorkspaceStore::open(&path).unwrap(); + seed_resource_job_workspace(&store).await; + let other = SqliteWorkspaceStore::open(&path).unwrap(); + let requests = [ + resource_job_request("first", "subject-a", "candidate-1"), + resource_job_request("followup", "subject-a", "candidate-2"), + ]; + let barrier = std::sync::Barrier::new(2); + let reservations = std::thread::scope(|scope| { + let a = scope.spawn(|| { + barrier.wait(); + store + .reserve_backend_job("resource-workspace", &requests[0], RESOURCE_JOB_NOW) + .unwrap() + }); + let b = scope.spawn(|| { + barrier.wait(); + other + .reserve_backend_job("resource-workspace", &requests[1], RESOURCE_JOB_NOW) + .unwrap() + }); + [a.join().unwrap(), b.join().unwrap()] + }); + assert_eq!(reservations.iter().filter(|r| r.resource_reused).count(), 1); + assert!(reservations.iter().all(|r| !r.replayed)); + assert_eq!(reservations[0].job.request, reservations[1].job.request); + let original = &reservations[0]; + let deferred = requests + .iter() + .find(|r| r.job_id != original.job.request.job_id) + .unwrap(); + assert!( + store + .get_backend_job("resource-workspace", &deferred.job_id) + .unwrap() + .is_none() + ); + let replay = other + .reserve_backend_job( + "resource-workspace", + &original.job.request, + RESOURCE_JOB_NOW, + ) + .unwrap(); + assert!(replay.replayed && !replay.resource_reused); + let mut changed = original.job.request.clone(); + changed.input = serde_json::json!({"changed": true}); + assert!( + other + .reserve_backend_job("resource-workspace", &changed, RESOURCE_JOB_NOW) + .is_err() + ); + assert_eq!( + store + .find_active_backend_job_for_resource( + "resource-workspace", + &original.job.request.resource_key().unwrap() + ) + .unwrap() + .unwrap() + .request, + original.job.request + ); + // A separate subject does not collide. + assert!( + !store + .reserve_backend_job( + "resource-workspace", + &resource_job_request("other", "subject-b", "candidate-3"), + RESOURCE_JOB_NOW + ) + .unwrap() + .resource_reused + ); + store + .finish_backend_job_attempt( + "resource-workspace", + &original.job.request.job_id, + &original.attempt.attempt_id, + BackendJobAttemptState::Failed, + "not_started", + "no Worker allocated", + RESOURCE_JOB_NOW, + ) + .unwrap(); + let followup = other + .reserve_backend_job("resource-workspace", deferred, RESOURCE_JOB_NOW) + .unwrap(); + assert!(!followup.replayed && !followup.resource_reused); + assert_eq!(&followup.job.request, deferred); + assert_ne!( + followup.job.intent_fingerprint, + original.job.intent_fingerprint + ); + assert_eq!( + store + .get_backend_job("resource-workspace", &original.job.request.job_id) + .unwrap() + .unwrap() + .request, + original.job.request + ); + assert!( + store + .reserve_backend_job_retry( + "resource-workspace", + &original.job.request.job_id, + RESOURCE_JOB_NOW + ) + .is_err(), + "retry cannot displace a followup that reserved the same resource" + ); + } + + #[tokio::test] + async fn backend_job_resource_retry_and_unknown_are_fenced_by_durable_cleanup() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("cleanup-jobs.sqlite"); + let store = SqliteWorkspaceStore::open(&path).unwrap(); + seed_resource_job_workspace(&store).await; + let request = resource_job_request("failed", "subject-a", "candidate-1"); + let first = store + .reserve_backend_job("resource-workspace", &request, RESOURCE_JOB_NOW) + .unwrap(); + let worker = bind_resource_job_worker(&store, &first, "failed-worker"); + assert_eq!( + store + .get_active_backend_job_for_worker("resource-workspace", &worker) + .unwrap() + .unwrap() + .request + .grants, + request.grants + ); + assert!( + store + .get_active_backend_job_for_worker("other-workspace", &worker) + .unwrap() + .is_none() + ); + store + .finish_backend_job_attempt( + "resource-workspace", + &request.job_id, + &first.attempt.attempt_id, + BackendJobAttemptState::Failed, + "definite_failure", + "known failed", + RESOURCE_JOB_NOW, + ) + .unwrap(); + assert!( + store + .get_active_backend_job_for_worker("resource-workspace", &worker) + .unwrap() + .is_none() + ); + let deferred = resource_job_request("later", "subject-a", "candidate-2"); + for cleanup_state in [ + BackendJobWorkerCleanupState::Pending, + BackendJobWorkerCleanupState::Executing, + BackendJobWorkerCleanupState::Failed, + ] { + assert!( + store + .reserve_backend_job_retry( + "resource-workspace", + &request.job_id, + RESOURCE_JOB_NOW + ) + .is_err() + ); + let reused = store + .reserve_backend_job("resource-workspace", &deferred, RESOURCE_JOB_NOW) + .unwrap(); + assert!(reused.resource_reused && !reused.replayed); + assert_eq!(reused.job.request, request); + match cleanup_state { + BackendJobWorkerCleanupState::Pending => { + store + .claim_backend_job_worker_cleanup( + "resource-workspace", + &request.job_id, + &first.attempt.attempt_id, + RESOURCE_JOB_NOW, + ) + .unwrap(); + } + BackendJobWorkerCleanupState::Executing => { + store + .finish_backend_job_worker_cleanup( + "resource-workspace", + &request.job_id, + &first.attempt.attempt_id, + BackendJobWorkerCleanupState::Failed, + Some("cleanup_failed"), + Some("Worker still exists"), + RESOURCE_JOB_NOW, + ) + .unwrap(); + } + _ => {} + } + } + finish_resource_job_cleanup(&store, &first); + let second = store + .reserve_backend_job_retry("resource-workspace", &request.job_id, RESOURCE_JOB_NOW) + .unwrap(); + assert_eq!(second.job.request, request); + assert_eq!(second.attempt.attempt, 2); + bind_resource_job_worker(&store, &second, "unknown-worker"); + store + .finish_backend_job_attempt( + "resource-workspace", + &request.job_id, + &second.attempt.attempt_id, + BackendJobAttemptState::Unknown, + "unknown_effects", + "domain outcome unknown", + RESOURCE_JOB_NOW, + ) + .unwrap(); + finish_resource_job_cleanup(&store, &second); + drop(store); + let restarted = SqliteWorkspaceStore::open(&path).unwrap(); + let active = restarted + .find_active_backend_job_for_resource( + "resource-workspace", + &request.resource_key().unwrap(), + ) + .unwrap() + .unwrap(); + assert_eq!(active.state, BackendJobState::Unknown); + assert_eq!(active.request, request); + assert!( + restarted + .reserve_backend_job("resource-workspace", &deferred, RESOURCE_JOB_NOW) + .unwrap() + .resource_reused, + "Worker removal must not resolve unknown domain effects" + ); + assert!( + restarted + .reserve_backend_job_retry("resource-workspace", &request.job_id, RESOURCE_JOB_NOW) + .is_err() + ); + } + + #[test] + fn backend_job_resource_migration_preserves_legacy_intent() { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch( + "CREATE TABLE __yoi_schema_migrations(version INTEGER PRIMARY KEY, name TEXT NOT NULL); + INSERT INTO __yoi_schema_migrations VALUES (80, 'workspace schema baseline'); + CREATE TABLE backend_jobs(workspace_id TEXT, job_id TEXT, request_json TEXT, intent_fingerprint TEXT, created_at TEXT); + INSERT INTO backend_jobs VALUES ('workspace', 'legacy', '{\"profile\":\"builtin:backend-job\"}', 'sha256:original', '1');", + ).unwrap(); + migrate_backend_job_resources_v80_to_v81(&conn).unwrap(); + assert_eq!(current_schema_version(&conn).unwrap(), 81); + assert_eq!( + conn.query_row( + "SELECT request_json, intent_fingerprint, resource_key FROM backend_jobs", + [], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, Option>(2)?, + )) + } + ) + .unwrap(), + ( + "{\"profile\":\"builtin:backend-job\"}".into(), + "sha256:original".into(), + None + ) + ); + } + + #[tokio::test] + async fn backend_job_resource_retry_races_new_intent_atomically_and_remains_bounded() { + let dir = tempfile::tempdir().unwrap(); + let path = dir.path().join("retry-race.sqlite"); + let store = SqliteWorkspaceStore::open(&path).unwrap(); + seed_resource_job_workspace(&store).await; + // Generic resources use the same common reservation API without domain grants. + let mut request = resource_job_request("generic", "subject-a", "candidate-1"); + request.grants = Default::default(); + request.serialization_key = Some("custom-resource".into()); + let first = store + .reserve_backend_job("resource-workspace", &request, RESOURCE_JOB_NOW) + .unwrap(); + store + .finish_backend_job_attempt( + "resource-workspace", + &request.job_id, + &first.attempt.attempt_id, + BackendJobAttemptState::Failed, + "not_started", + "no Worker allocated", + RESOURCE_JOB_NOW, + ) + .unwrap(); + let mut followup = request.clone(); + followup.job_id = "generic-followup".into(); + followup.input = serde_json::json!({"candidate": "new"}); + let other = SqliteWorkspaceStore::open(&path).unwrap(); + let barrier = std::sync::Barrier::new(2); + let (retry, reserve) = std::thread::scope(|scope| { + let a = scope.spawn(|| { + barrier.wait(); + store.reserve_backend_job_retry( + "resource-workspace", + &request.job_id, + RESOURCE_JOB_NOW, + ) + }); + let b = scope.spawn(|| { + barrier.wait(); + other + .reserve_backend_job("resource-workspace", &followup, RESOURCE_JOB_NOW) + .unwrap() + }); + (a.join().unwrap(), b.join().unwrap()) + }); + match retry { + Ok(retried) => { + assert!(reserve.resource_reused); + assert_eq!(reserve.job.request, request); + assert_eq!(retried.attempt.attempt, 2); + store + .finish_backend_job_attempt( + "resource-workspace", + &request.job_id, + &retried.attempt.attempt_id, + BackendJobAttemptState::Failed, + "not_started", + "no Worker allocated", + RESOURCE_JOB_NOW, + ) + .unwrap(); + } + Err(_) => { + assert!(!reserve.resource_reused); + assert_eq!(reserve.job.request, followup); + store + .finish_backend_job_attempt( + "resource-workspace", + &followup.job_id, + &reserve.attempt.attempt_id, + BackendJobAttemptState::Failed, + "not_started", + "no Worker allocated", + RESOURCE_JOB_NOW, + ) + .unwrap(); + let retried = store + .reserve_backend_job_retry( + "resource-workspace", + &request.job_id, + RESOURCE_JOB_NOW, + ) + .unwrap(); + store + .finish_backend_job_attempt( + "resource-workspace", + &request.job_id, + &retried.attempt.attempt_id, + BackendJobAttemptState::Failed, + "not_started", + "no Worker allocated", + RESOURCE_JOB_NOW, + ) + .unwrap(); + } + } + assert!( + store + .reserve_backend_job_retry("resource-workspace", &request.job_id, RESOURCE_JOB_NOW) + .is_err(), + "retry limit cannot be reset by new candidates" + ); + assert!( + store + .find_active_backend_job_for_resource("resource-workspace", "custom-resource") + .unwrap() + .is_none() + ); + } + + #[tokio::test] + async fn backend_job_all_older_attempt_cleanups_fence_retry_and_resource_release() { + let store = SqliteWorkspaceStore::in_memory().unwrap(); + seed_resource_job_workspace(&store).await; + let mut request = resource_job_request("legacy-overlap", "subject-a", "candidate-1"); + request.limits.max_attempts = 3; + let first = store + .reserve_backend_job("resource-workspace", &request, RESOURCE_JOB_NOW) + .unwrap(); + bind_resource_job_worker(&store, &first, "old-worker"); + store + .finish_backend_job_attempt( + "resource-workspace", + &request.job_id, + &first.attempt.attempt_id, + BackendJobAttemptState::Failed, + "partial_apply", + "some candidate effects durably applied", + RESOURCE_JOB_NOW, + ) + .unwrap(); + // Model a pre-fence durable journal: the old runner permitted attempt 2 + // before attempt 1's Worker cleanup. Current reservation must fail closed + // on this inherited state, even after the current attempt is cleaned. + let second_id = attempt_id(&request.job_id, 2); + store.with_conn(|conn| { + conn.execute("UPDATE backend_jobs SET state = 'pending', current_attempt = 2, completed_at = NULL WHERE workspace_id = 'resource-workspace' AND job_id = ?1", params![request.job_id])?; + conn.execute("INSERT INTO backend_job_attempts (workspace_id, job_id, attempt_id, attempt, input_revision, state, deadline_at, created_at, updated_at) VALUES ('resource-workspace', ?1, ?2, 2, ?3, 'reserved', ?4, ?4, ?4)", + params![request.job_id, second_id, request.input_revision, RESOURCE_JOB_NOW])?; + Ok(()) + }).unwrap(); + let second = store + .reserve_backend_job("resource-workspace", &request, RESOURCE_JOB_NOW) + .unwrap(); + assert_eq!(second.attempt.attempt, 2); + bind_resource_job_worker(&store, &second, "current-worker"); + store + .finish_backend_job_attempt( + "resource-workspace", + &request.job_id, + &second.attempt.attempt_id, + BackendJobAttemptState::Failed, + "partial_apply", + "remaining candidate effects not yet applied", + RESOURCE_JOB_NOW, + ) + .unwrap(); + finish_resource_job_cleanup(&store, &second); + let followup = resource_job_request("new-batch", "subject-a", "candidate-2"); + assert!( + store + .reserve_backend_job_retry("resource-workspace", &request.job_id, RESOURCE_JOB_NOW) + .is_err(), + "completed current cleanup must not hide incomplete attempt 1" + ); + assert!( + store + .reserve_backend_job("resource-workspace", &followup, RESOURCE_JOB_NOW) + .unwrap() + .resource_reused + ); + assert_eq!( + store + .find_active_backend_job_for_resource( + "resource-workspace", + &request.resource_key().unwrap() + ) + .unwrap() + .unwrap() + .current_attempt, + 2 + ); + finish_resource_job_cleanup(&store, &first); + let third = store + .reserve_backend_job_retry("resource-workspace", &request.job_id, RESOURCE_JOB_NOW) + .unwrap(); + assert_eq!( + third.job.request, request, + "partial apply retry keeps its original snapshot and grants" + ); + assert_eq!(third.attempt.attempt, 3); + let worker = bind_resource_job_worker(&store, &third, "converged-worker"); + let submission = BackendJobResultSubmission { + job_id: request.job_id.clone(), + attempt_id: third.attempt.attempt_id.clone(), + input_revision: request.input_revision.clone(), + result: serde_json::json!({"converged": true}), + }; + store + .accept_backend_job_result("resource-workspace", &worker, &submission, RESOURCE_JOB_NOW) + .unwrap(); + finish_resource_job_cleanup(&store, &third); + // A legacy incomplete older cleanup also fences a completed current Job. + store.with_conn(|conn| { + conn.execute("UPDATE backend_job_attempts SET worker_cleanup_state = 'failed', worker_cleanup_completed_at = NULL WHERE workspace_id = 'resource-workspace' AND attempt_id = ?1", params![first.attempt.attempt_id])?; + Ok(()) + }).unwrap(); + assert!( + store + .reserve_backend_job("resource-workspace", &followup, RESOURCE_JOB_NOW) + .unwrap() + .resource_reused, + "completed Job and current cleanup must not release an older unremoved Worker" + ); + finish_resource_job_cleanup(&store, &first); + assert!( + !store + .reserve_backend_job("resource-workspace", &followup, RESOURCE_JOB_NOW) + .unwrap() + .resource_reused + ); + let replay = store + .accept_backend_job_result("resource-workspace", &worker, &submission, RESOURCE_JOB_NOW) + .unwrap(); + assert!(replay.replayed); + assert_eq!(replay.job.result, Some(submission.result)); + assert!( + store + .reserve_backend_job_retry("resource-workspace", &request.job_id, RESOURCE_JOB_NOW) + .is_err(), + "converged intent cannot be reopened or exceed its bounded attempts" + ); + } + + #[tokio::test] + async fn backend_job_empty_batch_surface_refresh_locks_subject_and_defers_candidates() { + let store = SqliteWorkspaceStore::in_memory().unwrap(); + seed_resource_job_workspace(&store).await; + for surface_state in ["ungenerated", "stale", "failed"] { + let subject = format!("subject-{surface_state}"); + let mut refresh = resource_job_request( + &format!("surface-{surface_state}"), + &subject, + "surface-revision", + ); + refresh + .grants + .subjektiv_consolidation + .as_mut() + .unwrap() + .candidate_ids + .clear(); + refresh.input = serde_json::json!({"subject_id": subject, "candidate_ids": [], "surface_state": surface_state}); + refresh.instruction = + "Regenerate only the Subject surface; this batch grants no candidate access." + .into(); + let first = store + .reserve_backend_job("resource-workspace", &refresh, RESOURCE_JOB_NOW) + .unwrap(); + assert_eq!(first.job.request, refresh); + assert!( + store + .reserve_backend_job("resource-workspace", &refresh, RESOURCE_JOB_NOW) + .unwrap() + .replayed + ); + let worker = bind_resource_job_worker( + &store, + &first, + &format!("surface-worker-{surface_state}"), + ); + let active = store + .get_active_backend_job_for_worker("resource-workspace", &worker) + .unwrap() + .unwrap(); + let grant = active + .request + .grants + .subjektiv_consolidation + .as_ref() + .unwrap(); + assert_eq!(grant.subject_id, subject); + assert!( + grant.candidate_ids.is_empty(), + "surface-only Job grants no candidate capability" + ); + assert!( + active + .request + .worker_input(&first.attempt.attempt_id) + .unwrap() + .contains("Regenerate only the Subject surface") + ); + assert_eq!( + store + .find_active_backend_job_for_resource( + "resource-workspace", + &refresh.resource_key().unwrap() + ) + .unwrap() + .unwrap() + .request, + refresh + ); + let followup = resource_job_request( + &format!("candidates-{surface_state}"), + &subject, + "new-candidate", + ); + let reused = store + .reserve_backend_job("resource-workspace", &followup, RESOURCE_JOB_NOW) + .unwrap(); + assert!(reused.resource_reused && !reused.replayed); + assert_eq!( + reused.job.request, refresh, + "later candidates must not expand surface-only authority" + ); + store + .accept_backend_job_result( + "resource-workspace", + &worker, + &BackendJobResultSubmission { + job_id: refresh.job_id.clone(), + attempt_id: first.attempt.attempt_id.clone(), + input_revision: refresh.input_revision.clone(), + result: serde_json::json!({"surface": {"availability": "available"}}), + }, + RESOURCE_JOB_NOW, + ) + .unwrap(); + assert!( + store + .reserve_backend_job("resource-workspace", &followup, RESOURCE_JOB_NOW) + .unwrap() + .resource_reused, + "completed surface refresh still fences the subject until Worker cleanup" + ); + finish_resource_job_cleanup(&store, &first); + let next = store + .reserve_backend_job("resource-workspace", &followup, RESOURCE_JOB_NOW) + .unwrap(); + assert!(!next.resource_reused); + assert_eq!(next.job.request, followup); + assert_eq!( + store + .get_backend_job("resource-workspace", &refresh.job_id) + .unwrap() + .unwrap() + .request, + refresh + ); + } + } + + #[tokio::test] + async fn backend_job_completed_resource_releases_only_after_worker_cleanup() { + let store = SqliteWorkspaceStore::in_memory().unwrap(); + seed_resource_job_workspace(&store).await; + let request = resource_job_request("completed", "subject-a", "candidate-1"); + let first = store + .reserve_backend_job("resource-workspace", &request, RESOURCE_JOB_NOW) + .unwrap(); + let worker = bind_resource_job_worker(&store, &first, "completed-worker"); + store + .accept_backend_job_result( + "resource-workspace", + &worker, + &BackendJobResultSubmission { + job_id: request.job_id.clone(), + attempt_id: first.attempt.attempt_id.clone(), + input_revision: request.input_revision.clone(), + result: serde_json::json!({"done": true}), + }, + RESOURCE_JOB_NOW, + ) + .unwrap(); + let followup = resource_job_request("new-candidates", "subject-a", "candidate-2"); + assert!( + store + .reserve_backend_job("resource-workspace", &followup, RESOURCE_JOB_NOW) + .unwrap() + .resource_reused + ); + finish_resource_job_cleanup(&store, &first); + assert!( + store + .find_active_backend_job_for_resource( + "resource-workspace", + &request.resource_key().unwrap() + ) + .unwrap() + .is_none() + ); + let next = store + .reserve_backend_job("resource-workspace", &followup, RESOURCE_JOB_NOW) + .unwrap(); + assert!(!next.resource_reused); + assert_eq!(next.job.request, followup); + assert_eq!( + store + .get_backend_job("resource-workspace", &request.job_id) + .unwrap() + .unwrap() + .request, + request + ); + } + #[tokio::test] async fn backend_jobs_fence_results_project_ownership_and_bound_retries() { let store = SqliteWorkspaceStore::in_memory().unwrap(); @@ -21870,6 +23709,8 @@ INSERT INTO worker_registry ( input: serde_json::json!({"title": "check"}), instruction: "Check the immutable input.".to_string(), profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, source_worker: Some(source.clone()), notification_target: Some(source.clone()), limits: crate::backend_job::BackendJobLimits { @@ -22290,6 +24131,35 @@ INSERT INTO worker_registry ( ) .is_err() ); + assert!( + store + .reserve_backend_job_retry( + workspace_id, + &retry_request.job_id, + "2026-09-01T00:01:04Z" + ) + .is_err(), + "failed attempt cannot retry before canonical Worker cleanup" + ); + store + .claim_backend_job_worker_cleanup( + workspace_id, + &retry_request.job_id, + &failed.attempt.attempt_id, + "2026-09-01T00:01:04Z", + ) + .unwrap(); + store + .finish_backend_job_worker_cleanup( + workspace_id, + &retry_request.job_id, + &failed.attempt.attempt_id, + BackendJobWorkerCleanupState::Completed, + None, + None, + "2026-09-01T00:01:04Z", + ) + .unwrap(); let second = store .reserve_backend_job_retry(workspace_id, &retry_request.job_id, "2026-09-01T00:01:04Z") .unwrap(); diff --git a/crates/workspace-server/src/store_workspace_config_tests.rs b/crates/workspace-server/src/store_workspace_config_tests.rs new file mode 100644 index 000000000..7f79fba5a --- /dev/null +++ b/crates/workspace-server/src/store_workspace_config_tests.rs @@ -0,0 +1,145 @@ +// Included in store::tests. Intentionally preserves the v64/v78 physical column +// order, which differs from the latest baseline and must never be copied via '*'. +fn workspace_config_v78_fixture() -> Connection { + let conn = Connection::open_in_memory().unwrap(); + conn.execute_batch(r#" + PRAGMA foreign_keys=ON; + CREATE TABLE __yoi_schema_migrations(version INTEGER PRIMARY KEY,name TEXT NOT NULL); + INSERT INTO __yoi_schema_migrations VALUES(78,'workspace schema baseline'); + CREATE TABLE workspaces(workspace_id TEXT PRIMARY KEY); + INSERT INTO workspaces VALUES('ws'); + CREATE TABLE repositories(workspace_id TEXT,repository_id TEXT,PRIMARY KEY(workspace_id,repository_id)); + INSERT INTO repositories VALUES('ws','repo'); + CREATE TABLE external_workdir_grants(workspace_id TEXT,grant_id TEXT,PRIMARY KEY(workspace_id,grant_id)); + INSERT INTO external_workdir_grants VALUES('ws','external-grant'); + CREATE TABLE workdir_registry( + workspace_id TEXT NOT NULL,workdir_id TEXT NOT NULL,display_name TEXT,source_kind TEXT NOT NULL, + runtime_id TEXT,repository_id TEXT,external_grant_id TEXT,creation_selector TEXT,creation_ref TEXT, + creation_tree TEXT,current_selector TEXT,current_ref TEXT,current_tree TEXT,observed_at_epoch_seconds INTEGER, + materialization_status TEXT NOT NULL,cleanliness TEXT NOT NULL,created_at TEXT NOT NULL,updated_at TEXT NOT NULL, + PRIMARY KEY(workspace_id,workdir_id)); + INSERT INTO workdir_registry VALUES('ws','repo-wd','Repository','repository','runtime','repo',NULL,'develop','a','tree-a','work','b','tree-b',123,'present','clean','created','updated'); + INSERT INTO workdir_registry VALUES('ws','external-wd','External','external_grant',NULL,NULL,'external-grant',NULL,NULL,NULL,NULL,NULL,NULL,NULL,'present','unknown','created-e','updated-e'); + CREATE INDEX registry_display_test ON workdir_registry(display_name); + CREATE TABLE registry_updates(value TEXT); + CREATE TRIGGER registry_update_test AFTER UPDATE ON workdir_registry BEGIN INSERT INTO registry_updates VALUES(NEW.display_name); END; + CREATE TABLE dependent_links(workspace_id TEXT,workdir_id TEXT,connection_id TEXT, + FOREIGN KEY(workspace_id,workdir_id) REFERENCES workdir_registry(workspace_id,workdir_id)); + INSERT INTO dependent_links VALUES('ws','repo-wd','durable-connection'); + "#).unwrap(); + conn +} + +#[test] +fn workspace_config_migration_78_to_79_preserves_sources_refs_dependencies_and_constraints() { + let conn = workspace_config_v78_fixture(); + migrate_workspace_config_v78_to_v79(&conn).unwrap(); + assert_eq!(current_schema_version(&conn).unwrap(), 79); + assert!( + conn.pragma_query_value(None, "foreign_keys", |row| row.get::<_, bool>(0)) + .unwrap() + ); + let row = conn + .query_row( + &workdir_registry_select_sql("WHERE workdir_id='repo-wd'"), + [], + read_workdir_registry_record, + ) + .unwrap(); + assert_eq!(row.creation_tree.as_deref(), Some("tree-a")); + assert_eq!(row.current_tree.as_deref(), Some("tree-b")); + assert_eq!(row.current_selector.as_deref(), Some("work")); + assert_eq!(row.cleanliness, "clean"); + assert_eq!(row.created_at, "created"); + let external = conn + .query_row( + &workdir_registry_select_sql("WHERE workdir_id='external-wd'"), + [], + read_workdir_registry_record, + ) + .unwrap(); + assert!( + matches!(external.source,WorkdirRegistrySource::ExternalGrant{ref grant_id} if grant_id=="external-grant") + ); + assert_eq!( + conn.query_row("SELECT connection_id FROM dependent_links", [], |row| { + row.get::<_, String>(0) + }) + .unwrap(), + "durable-connection" + ); + assert!( + conn.query_row( + "SELECT EXISTS(SELECT 1 FROM sqlite_schema WHERE name='registry_display_test')", + [], + |row| row.get::<_, bool>(0) + ) + .unwrap() + ); + conn.execute( + "UPDATE workdir_registry SET display_name='Updated' WHERE workdir_id='repo-wd'", + [], + ) + .unwrap(); + assert_eq!( + conn.query_row("SELECT value FROM registry_updates", [], |row| row + .get::<_, String>(0)) + .unwrap(), + "Updated" + ); + conn.execute("INSERT INTO workspace_config_grants VALUES('ws','config-grant','runtime','worker','config-wd','read_only',0,'owner','now')",[]).unwrap(); + conn.execute("INSERT INTO workdir_registry(workspace_id,workdir_id,source_kind,workspace_config_grant_id,materialization_status,cleanliness,created_at,updated_at) VALUES('ws','config-wd','workspace_config','config-grant','present','clean','now','now')",[]).unwrap(); + let logical = conn + .query_row( + &workdir_registry_select_sql("WHERE workdir_id='config-wd'"), + [], + read_workdir_registry_record, + ) + .unwrap(); + assert!( + matches!(logical.source,WorkdirRegistrySource::WorkspaceConfig{ref grant_id} if grant_id=="config-grant") + ); + assert!(conn.execute("INSERT INTO workspace_config_grants VALUES('ws','invalid-grant','r','w','invalid-wd','read_write_command',0,'owner','now')",[]).is_err()); + assert!(conn.execute("INSERT INTO workspace_config_grants VALUES('ws','duplicate','runtime','worker','other-wd','read_write',0,'owner','now')",[]).is_err()); + assert!(conn.execute("INSERT INTO workdir_registry(workspace_id,workdir_id,source_kind,workspace_config_grant_id,runtime_id,materialization_status,cleanliness,created_at,updated_at) VALUES('ws','invalid-wd','workspace_config','config-grant','os-runtime','present','clean','now','now')",[]).is_err()); + let mut check = conn.prepare("PRAGMA foreign_key_check").unwrap(); + assert!(check.query([]).unwrap().next().unwrap().is_none()); + assert!( + migrate_workspace_config_v78_to_v79(&conn).is_err(), + "wrong-version rerun must not silently mutate data" + ); +} + +#[test] +fn workspace_config_migration_rolls_back_and_restores_foreign_keys_on_invalid_legacy_data() { + let conn = workspace_config_v78_fixture(); + conn.execute( + "UPDATE workdir_registry SET repository_id='missing' WHERE workdir_id='repo-wd'", + [], + ) + .unwrap(); + assert!(migrate_workspace_config_v78_to_v79(&conn).is_err()); + assert_eq!(current_schema_version(&conn).unwrap(), 78); + assert!(!column_exists(&conn, "workdir_registry", "workspace_config_grant_id").unwrap()); + assert!(!table_exists(&conn, "workspace_config_grants").unwrap()); + assert!( + conn.pragma_query_value(None, "foreign_keys", |row| row.get::<_, bool>(0)) + .unwrap() + ); + assert_eq!( + conn.query_row( + "SELECT repository_id FROM workdir_registry WHERE workdir_id='repo-wd'", + [], + |row| row.get::<_, String>(0) + ) + .unwrap(), + "missing" + ); + assert_eq!( + conn.query_row("SELECT connection_id FROM dependent_links", [], |row| { + row.get::<_, String>(0) + }) + .unwrap(), + "durable-connection" + ); +} diff --git a/crates/workspace-server/src/subjektiv.rs b/crates/workspace-server/src/subjektiv.rs index 1e9564954..83d4cfe63 100644 --- a/crates/workspace-server/src/subjektiv.rs +++ b/crates/workspace-server/src/subjektiv.rs @@ -11,7 +11,7 @@ use std::collections::HashSet; use chrono::{SecondsFormat, Utc}; use memory::extract::{CandidateKind, STAGING_SCHEMA_VERSION, StagingEvidence, StagingRecord}; use memory::schema::{SourceEvidenceRef, SourceRef}; -use rusqlite::{OptionalExtension, Transaction, params}; +use rusqlite::{Connection, OptionalExtension, Transaction, params}; use serde::{Deserialize, Deserializer, Serialize}; use uuid::Uuid; @@ -24,6 +24,8 @@ pub const SUBJEKTIV_SCHEMA_VERSION: u32 = 1; pub const SUBJEKTIV_FEATURE_ID: &str = "subjektiv"; pub const SUBJECT_WORKER_SINGLETON_PREFIX: &str = "subjektiv:"; pub const MAX_STAGING_ANCHORS: usize = 10; +/// Maximum UTF-8 size of the user-managed Subject behavior document. +pub const MAX_SUBJECT_BEHAVIOR_BYTES: usize = server_api::SUBJEKTIV_MAX_BEHAVIOR_BYTES; /// Deterministic surface-generation policy. Input estimates use the repository's /// provider-independent UTF-8 byte estimate (`ceil(bytes / 4)`). The material /// payload is capped below the whole-input budget to reserve room for the fixed @@ -74,6 +76,14 @@ pub enum SubjektivError { expected: u64, actual: u64, }, + #[error( + "subject `{subject_id}` behavior revision conflict: expected {expected}, current {actual}" + )] + SubjectBehaviorConflict { + subject_id: String, + expected: u64, + actual: u64, + }, #[error("surface generation conflict: {0}")] SurfaceGenerationConflict(String), #[error("memory `{memory_id}` cannot transition from {from:?} to {to:?}")] @@ -136,6 +146,14 @@ pub struct SubjectRecord { pub schema_version: u32, pub id: String, pub role: SubjectRole, + /// User-managed, host-injected behavior document. This is not Memory and + /// Memory lifecycle operations must preserve it byte-for-byte. + #[serde(default)] + pub behavior_md: String, + /// Monotonic CAS generation for behavior updates, independent of Memory's + /// `store_revision`. + #[serde(default)] + pub behavior_revision: u64, pub state: SubjectState, /// Monotonic generation incremented exactly once for every committed Memory /// creation or revision for this subject. @@ -500,6 +518,12 @@ pub struct ResidentSurface { pub snapshot: Option, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct SubjectResidentContext { + pub subject: SubjectRecord, + pub surface: ResidentSurface, +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(tag = "kind", rename_all = "snake_case")] pub enum MemoryRevisionTarget { @@ -986,6 +1010,22 @@ END; Ok(()) } +fn add_subject_behavior(transaction: &Transaction<'_>) -> crate::feature_storage::Result<()> { + transaction.execute_batch( + r#" +ALTER TABLE subjects ADD COLUMN behavior_md TEXT NOT NULL DEFAULT ''; +ALTER TABLE subjects ADD COLUMN behavior_revision INTEGER NOT NULL DEFAULT 0 + CHECK (behavior_revision >= 0); +"#, + )?; + Ok(()) +} + +fn add_surface_job_provenance(transaction: &Transaction<'_>) -> crate::feature_storage::Result<()> { + transaction.execute_batch("ALTER TABLE surface_generation_runs ADD COLUMN job_id TEXT; ALTER TABLE surface_generation_runs ADD COLUMN attempt_id TEXT;")?; + Ok(()) +} + static MIGRATIONS: &[FeatureMigration] = &[ FeatureMigration::new(1, "create subjektiv subject memory store", create_schema), FeatureMigration::new( @@ -1003,6 +1043,12 @@ static MIGRATIONS: &[FeatureMigration] = &[ "add bounded surface generation state and runs", add_surface_generation_state, ), + FeatureMigration::new(5, "add user-managed subject behavior", add_subject_behavior), + FeatureMigration::new( + 6, + "bind surface generation provenance to Job attempts", + add_surface_job_provenance, + ), ]; pub const REGISTRATION: FeatureRegistration = @@ -1070,11 +1116,23 @@ impl SubjektivStore { /// Issues a random host-owned subject id. It is never derived from Worker, /// Profile, Session, or Memory content. pub fn create_subject(&self, role: SubjectRole) -> Result { + self.create_subject_with_behavior(role, String::new()) + } + + /// Creates a Subject with its optional user-managed behavior document. + pub fn create_subject_with_behavior( + &self, + role: SubjectRole, + behavior_md: String, + ) -> Result { + validate_subject_behavior(&behavior_md)?; let timestamp = now(); let record = SubjectRecord { schema_version: SUBJEKTIV_SCHEMA_VERSION, id: issued_id("subject"), role, + behavior_md, + behavior_revision: 0, state: SubjectState::Active, store_revision: 0, created_at: timestamp.clone(), @@ -1084,11 +1142,13 @@ impl SubjektivStore { self.database.try_transaction(|transaction| { transaction.execute( "INSERT INTO subjects ( - subject_id, role, state, store_revision, record_json, created_at, updated_at - ) VALUES (?1, ?2, 'active', 0, ?3, ?4, ?5)", + subject_id, role, behavior_md, behavior_revision, state, + store_revision, record_json, created_at, updated_at + ) VALUES (?1, ?2, ?3, 0, 'active', 0, ?4, ?5, ?6)", params![ record.id, record.role.as_str(), + record.behavior_md, raw, record.created_at, record.updated_at @@ -1111,6 +1171,51 @@ impl SubjektivStore { }) } + /// Replaces or clears the user-managed behavior document using its own CAS + /// generation. Memory revisions do not participate in this generation. + pub fn update_subject_behavior( + &self, + subject_id: &str, + expected_behavior_revision: u64, + behavior_md: String, + ) -> Result { + validate_subject_behavior(&behavior_md)?; + self.database.try_transaction(|transaction| { + let mut subject = require_subject(transaction, subject_id)?; + if subject.behavior_revision != expected_behavior_revision { + return Err(SubjektivError::SubjectBehaviorConflict { + subject_id: subject_id.to_string(), + expected: expected_behavior_revision, + actual: subject.behavior_revision, + }); + } + if subject.behavior_md == behavior_md { + return Ok(subject); + } + subject.behavior_md = behavior_md; + subject.behavior_revision = + subject.behavior_revision.checked_add(1).ok_or_else(|| { + SubjektivError::InvalidRecord("subject behavior revision overflow".to_string()) + })?; + subject.updated_at = now(); + let raw = serde_json::to_string(&subject)?; + transaction.execute( + "UPDATE subjects + SET behavior_md = ?2, behavior_revision = ?3, + record_json = ?4, updated_at = ?5 + WHERE subject_id = ?1", + params![ + subject.id, + subject.behavior_md, + to_i64(subject.behavior_revision)?, + raw, + subject.updated_at + ], + )?; + Ok(subject) + }) + } + /// Validates an explicit product launch connection against this Workspace's /// subject store and the already-resolved Profile policy. Subject knowledge /// alone is insufficient: only an active local record produces host launch material. @@ -1863,6 +1968,14 @@ impl SubjektivStore { /// active Memory revisions. Kind order is fixed and selection is round-robin /// across kinds; within each kind, updated_at desc then memory id asc. pub fn prepare_surface_generation(&self, subject_id: &str) -> Result { + self.prepare_job_surface_generation(subject_id, None) + } + + pub(crate) fn prepare_job_surface_generation( + &self, + subject_id: &str, + job_attempt: Option<(&str, &str)>, + ) -> Result { self.database.try_transaction(|transaction| { let subject = require_active_subject(transaction, subject_id)?; let (active_memory_count, materials) = @@ -1878,21 +1991,71 @@ impl SubjektivStore { transaction.execute( "INSERT INTO surface_generation_runs ( subject_id, generation_id, store_revision, active_memory_count, - materials_json, created_at - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6)", + materials_json, created_at, job_id, attempt_id + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", params![ subject_id, generation.id, to_i64(generation.store_revision)?, to_i64(generation.active_memory_count as u64)?, serde_json::to_string(&generation.materials)?, - generation.created_at + generation.created_at, + job_attempt.map(|(job_id, _)| job_id), + job_attempt.map(|(_, attempt_id)| attempt_id), ], )?; Ok(generation) }) } + /// Validate a Job's structured surface claim against the existing generation + /// and publication/failure records. This does not create another receipt or + /// roll back Memory. The generation must have begun inside this attempt. + #[allow(clippy::too_many_arguments)] + pub(crate) fn validate_job_surface_outcome( + &self, + subject_id: &str, + generation_id: &str, + store_revision: u64, + availability: &str, + snapshot_id: Option<&str>, + reason_code: Option<&str>, + job_id: &str, + attempt_id: &str, + ) -> Result<()> { + self.database.try_with_connection(|connection| { + let subject = require_subject_in_connection(connection, subject_id)?; + let generation_revision = require_job_generation(connection, subject_id, generation_id, job_id, attempt_id)?; + if generation_revision != store_revision || subject.store_revision != store_revision { + return Err(SubjektivError::InvalidRecord("surface generation is stale".into())); + } + let state = connection.query_row( + "SELECT store_revision, status, snapshot_id, reason_code FROM surface_generation_state WHERE subject_id = ?1", + [subject_id], + |row| Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?, row.get::<_, Option>(2)?, row.get::<_, Option>(3)?)), + ).optional()?.ok_or_else(|| SubjektivError::InvalidRecord("surface generation has no terminal outcome".into()))?; + if state.0 != to_i64(store_revision)? || state.1 != availability + || !matches!(availability, "ready" | "failed") + || state.2.as_deref() != snapshot_id || state.3.as_deref() != reason_code { + return Err(SubjektivError::InvalidRecord("surface claim differs from the persisted outcome".into())); + } + Ok(()) + }) + } + + pub(crate) fn require_job_surface_generation( + &self, + subject_id: &str, + generation_id: &str, + job_id: &str, + attempt_id: &str, + ) -> Result<()> { + self.database.try_with_connection(|connection| { + require_job_generation(connection, subject_id, generation_id, job_id, attempt_id)?; + Ok(()) + }) + } + /// Validates structured editor output against the exact material snapshot and /// atomically publishes it only while the subject store generation is current. pub fn publish_surface_generation( @@ -2021,60 +2184,17 @@ impl SubjektivStore { pub fn resident_surface(&self, subject_id: &str) -> Result { self.database.try_with_connection(|connection| { let subject = require_subject_in_connection(connection, subject_id)?; - let state = connection - .query_row( - "SELECT store_revision, status, snapshot_id - FROM surface_generation_state WHERE subject_id = ?1", - [subject_id], - |row| { - Ok(( - row.get::<_, i64>(0)?, - row.get::<_, String>(1)?, - row.get::<_, Option>(2)?, - )) - }, - ) - .optional()?; - let Some((revision, status, snapshot_id)) = state else { - return Ok(ResidentSurface { - availability: SurfaceAvailability::Ungenerated, - snapshot: None, - }); - }; - let revision = u64::try_from(revision).map_err(|_| { - SubjektivError::InvalidRecord("surface state revision is negative".into()) - })?; - if revision != subject.store_revision || status == "dirty" { - return Ok(ResidentSurface { - availability: SurfaceAvailability::Stale, - snapshot: None, - }); - } - if status == "failed" { - return Ok(ResidentSurface { - availability: SurfaceAvailability::Failed, - snapshot: None, - }); - } - let snapshot_id = snapshot_id.ok_or_else(|| { - SubjektivError::InvalidRecord("ready surface state has no snapshot id".into()) - })?; - let raw = connection.query_row( - "SELECT snapshot_json FROM surface_snapshots - WHERE subject_id = ?1 AND snapshot_id = ?2", - params![subject_id, snapshot_id], - |row| row.get::<_, String>(0), - )?; - let snapshot = parse_surface_snapshot(&raw)?; - if snapshot.built_from_store_revision != subject.store_revision { - return Err(SubjektivError::InvalidRecord( - "ready surface snapshot generation does not match subject".into(), - )); - } - Ok(ResidentSurface { - availability: SurfaceAvailability::Ready, - snapshot: Some(snapshot), - }) + resident_surface_in_connection(connection, &subject) + }) + } + + /// Atomically reads user-managed behavior and the independently-fresh + /// generated Memory surface from one SQLite connection snapshot. + pub fn resident_context(&self, subject_id: &str) -> Result { + self.database.try_transaction(|transaction| { + let subject = require_subject(transaction, subject_id)?; + let surface = resident_surface_in_connection(transaction, &subject)?; + Ok(SubjectResidentContext { subject, surface }) }) } @@ -3414,6 +3534,87 @@ fn validate_range(kind: &str, range: [u64; 2]) -> Result<()> { } } +fn resident_surface_in_connection( + connection: &Connection, + subject: &SubjectRecord, +) -> Result { + let state = connection + .query_row( + "SELECT store_revision, status, snapshot_id + FROM surface_generation_state WHERE subject_id = ?1", + [&subject.id], + |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, String>(1)?, + row.get::<_, Option>(2)?, + )) + }, + ) + .optional()?; + let Some((revision, status, snapshot_id)) = state else { + return Ok(ResidentSurface { + availability: SurfaceAvailability::Ungenerated, + snapshot: None, + }); + }; + let revision = u64::try_from(revision) + .map_err(|_| SubjektivError::InvalidRecord("surface state revision is negative".into()))?; + if revision != subject.store_revision || status == "dirty" { + return Ok(ResidentSurface { + availability: SurfaceAvailability::Stale, + snapshot: None, + }); + } + if status == "failed" { + return Ok(ResidentSurface { + availability: SurfaceAvailability::Failed, + snapshot: None, + }); + } + let snapshot_id = snapshot_id.ok_or_else(|| { + SubjektivError::InvalidRecord("ready surface state has no snapshot id".into()) + })?; + let raw = connection.query_row( + "SELECT snapshot_json FROM surface_snapshots + WHERE subject_id = ?1 AND snapshot_id = ?2", + params![subject.id, snapshot_id], + |row| row.get::<_, String>(0), + )?; + let snapshot = parse_surface_snapshot(&raw)?; + if snapshot.built_from_store_revision != subject.store_revision { + return Err(SubjektivError::InvalidRecord( + "ready surface snapshot generation does not match subject".into(), + )); + } + Ok(ResidentSurface { + availability: SurfaceAvailability::Ready, + snapshot: Some(snapshot), + }) +} + +pub fn validate_subject_behavior(value: &str) -> Result<()> { + if value.len() > MAX_SUBJECT_BEHAVIOR_BYTES { + return Err(SubjektivError::InvalidRecord(format!( + "subject behavior must be at most {MAX_SUBJECT_BEHAVIOR_BYTES} bytes" + ))); + } + if !value.is_empty() && value.trim().is_empty() { + return Err(SubjektivError::InvalidRecord( + "subject behavior must be empty or contain non-whitespace text".to_string(), + )); + } + if value + .chars() + .any(|character| character.is_control() && !matches!(character, '\n' | '\r' | '\t')) + { + return Err(SubjektivError::InvalidRecord( + "subject behavior contains an unsupported control character".to_string(), + )); + } + Ok(()) +} + fn validate_label(kind: &str, value: &str) -> Result<()> { validate_nonempty(kind, value)?; if value.len() > 256 || value.chars().any(char::is_control) { @@ -3472,6 +3673,26 @@ fn reject_duplicate_memory_ids(kind: &str, values: &[MemoryRevisionRef]) -> Resu Ok(()) } +fn require_job_generation( + connection: &Connection, + subject_id: &str, + generation_id: &str, + job_id: &str, + attempt_id: &str, +) -> Result { + let revision: Option = connection.query_row( + "SELECT store_revision FROM surface_generation_runs WHERE subject_id = ?1 AND generation_id = ?2 AND job_id = ?3 AND attempt_id = ?4", + params![subject_id, generation_id, job_id, attempt_id], |row| row.get(0), + ).optional()?; + revision + .and_then(|value| u64::try_from(value).ok()) + .ok_or_else(|| { + SubjektivError::InvalidRecord( + "surface generation is not owned by this Job attempt".into(), + ) + }) +} + fn to_i64(value: u64) -> Result { i64::try_from(value).map_err(|_| { SubjektivError::InvalidRecord(format!("revision value {value} exceeds SQLite range")) @@ -3668,6 +3889,67 @@ mod tests { assert!(!complete.has_more); } + #[test] + fn subject_behavior_is_bounded_revisioned_and_independent_from_memory_revision() { + let temp = tempfile::tempdir().unwrap(); + let (_manager, _workspace, store) = open_store(temp.path(), "workspace-a"); + let subject = store + .create_subject_with_behavior(role(), "Prefer explicit evidence.".to_string()) + .unwrap(); + assert_eq!(subject.behavior_revision, 0); + assert_eq!(subject.store_revision, 0); + + let updated = store + .update_subject_behavior( + &subject.id, + 0, + "Prefer explicit evidence.\nAsk when uncertain.".to_string(), + ) + .unwrap(); + assert_eq!(updated.behavior_revision, 1); + assert_eq!(updated.store_revision, 0); + + store + .create_memory( + &subject.id, + draft( + "A confirmed observation", + "exercise Memory lifecycle isolation", + ), + ) + .unwrap(); + let after_memory = store.subject(&subject.id).unwrap().unwrap(); + assert_eq!(after_memory.behavior_md, updated.behavior_md); + assert_eq!(after_memory.behavior_revision, 1); + assert_eq!(after_memory.store_revision, 1); + + let (_reopened_manager, _reopened_workspace, reopened) = + open_store(temp.path(), "workspace-a"); + let persisted = reopened.subject(&subject.id).unwrap().unwrap(); + assert_eq!(persisted.behavior_md, updated.behavior_md); + assert_eq!(persisted.behavior_revision, 1); + assert!(matches!( + store.update_subject_behavior(&subject.id, 0, String::new()), + Err(SubjektivError::SubjectBehaviorConflict { + expected: 0, + actual: 1, + .. + }) + )); + + let unchanged = store + .update_subject_behavior(&subject.id, 1, updated.behavior_md.clone()) + .unwrap(); + assert_eq!(unchanged.behavior_revision, 1); + let cleared = store + .update_subject_behavior(&subject.id, 1, String::new()) + .unwrap(); + assert_eq!(cleared.behavior_revision, 2); + assert!(cleared.behavior_md.is_empty()); + assert!(validate_subject_behavior(&"x".repeat(MAX_SUBJECT_BEHAVIOR_BYTES + 1)).is_err()); + assert!(validate_subject_behavior(" ").is_err()); + } + #[test] fn attributed_candidates_preserve_revision_and_surface_reference_continuity() { let temp = tempfile::tempdir().unwrap(); @@ -3892,10 +4174,38 @@ mod tests { )) .unwrap(); let store = SubjektivStore::open(&workspace, ®istration).unwrap(); - store.create_subject(role()).unwrap().id + let timestamp = now(); + let subject_id = issued_id("subject"); + let mut raw = serde_json::json!({ + "schema_version": SUBJEKTIV_SCHEMA_VERSION, + "id": subject_id, + "role": role(), + "state": SubjectState::Active, + "store_revision": 0, + "created_at": timestamp.clone(), + "updated_at": timestamp.clone(), + }); + let raw = serde_json::to_string(&raw.take()).unwrap(); + store + .database + .try_transaction(|transaction| -> Result<()> { + transaction.execute( + "INSERT INTO subjects ( + subject_id, role, state, store_revision, + record_json, created_at, updated_at + ) VALUES (?1, ?2, 'active', 0, ?3, ?4, ?5)", + params![subject_id, role().as_str(), raw, timestamp, timestamp], + )?; + Ok(()) + }) + .unwrap(); + subject_id }; let (_manager, _workspace, store) = open_store(&root, "workspace-a"); + let migrated_subject = store.subject(&subject_id).unwrap().unwrap(); + assert!(migrated_subject.behavior_md.is_empty()); + assert_eq!(migrated_subject.behavior_revision, 0); let lifecycle_attribution = attribution( &subject_id, "runtime-1", @@ -3935,26 +4245,50 @@ mod tests { )) .unwrap(); let store = SubjektivStore::open(&workspace, ®istration).unwrap(); - let subject = store.create_subject(role()).unwrap(); + let subject_id = issued_id("subject"); + let timestamp = now(); + let subject_json = serde_json::to_string(&serde_json::json!({ + "schema_version": SUBJEKTIV_SCHEMA_VERSION, + "id": subject_id, + "role": role(), + "state": SubjectState::Active, + "store_revision": 0, + "created_at": timestamp, + "updated_at": timestamp, + })) + .unwrap(); let snapshot = SurfaceSnapshot { schema_version: SUBJEKTIV_SCHEMA_VERSION, id: "legacy-surface".into(), - subject_id: subject.id.clone(), + subject_id: subject_id.clone(), body_md: "Legacy summary without generation-policy evidence.".into(), memory_refs: Vec::new(), - built_from_store_revision: subject.store_revision, + built_from_store_revision: 0, created_at: now(), }; store .database .try_transaction(|transaction| -> Result<()> { + transaction.execute( + "INSERT INTO subjects ( + subject_id, role, state, store_revision, + record_json, created_at, updated_at + ) VALUES (?1, ?2, 'active', 0, ?3, ?4, ?5)", + params![ + subject_id, + role().as_str(), + subject_json, + timestamp, + timestamp + ], + )?; transaction.execute( "INSERT INTO surface_snapshots ( subject_id, snapshot_id, built_from_store_revision, snapshot_json, created_at ) VALUES (?1, ?2, ?3, ?4, ?5)", params![ - subject.id, + subject_id, snapshot.id, to_i64(snapshot.built_from_store_revision)?, serde_json::to_string(&snapshot)?, @@ -3964,12 +4298,12 @@ mod tests { transaction.execute( "INSERT INTO surface_snapshot_seals (subject_id, snapshot_id) VALUES (?1, ?2)", - params![subject.id, snapshot.id], + params![subject_id, snapshot.id], )?; Ok(()) }) .unwrap(); - (subject.id, snapshot.id) + (subject_id, snapshot.id) }; let (_manager, _workspace, store) = open_store(&root, "workspace-a"); diff --git a/crates/workspace-server/src/ticket_item_checker.rs b/crates/workspace-server/src/ticket_item_checker.rs index 5e7178590..af592f9cc 100644 --- a/crates/workspace-server/src/ticket_item_checker.rs +++ b/crates/workspace-server/src/ticket_item_checker.rs @@ -129,6 +129,8 @@ pub fn request( })?, instruction, profile: "builtin:backend-job".to_string(), + grants: Default::default(), + serialization_key: None, source_worker: Some(source_worker.clone()), notification_target: Some(source_worker), limits: BackendJobLimits { diff --git a/crates/workspace-server/src/workspace_config_grants.sql b/crates/workspace-server/src/workspace_config_grants.sql new file mode 100644 index 000000000..dfdb104dc --- /dev/null +++ b/crates/workspace-server/src/workspace_config_grants.sql @@ -0,0 +1,16 @@ +CREATE TABLE workspace_config_grants ( + workspace_id TEXT NOT NULL, + grant_id TEXT NOT NULL, + runtime_id TEXT NOT NULL, + worker_id TEXT NOT NULL, + workdir_id TEXT NOT NULL, + access TEXT NOT NULL CHECK(access IN ('read_only', 'read_write')), + revoked INTEGER NOT NULL CHECK(revoked IN (0, 1)), + created_by TEXT NOT NULL, + created_at TEXT NOT NULL, + PRIMARY KEY(workspace_id, grant_id), + UNIQUE(workspace_id, workdir_id), + FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE +); +CREATE UNIQUE INDEX workspace_config_grants_active_worker + ON workspace_config_grants(workspace_id, runtime_id, worker_id) WHERE revoked = 0; diff --git a/crates/workspace-server/src/workspace_deletion.rs b/crates/workspace-server/src/workspace_deletion.rs index a19f095a4..b5d997dcf 100644 --- a/crates/workspace-server/src/workspace_deletion.rs +++ b/crates/workspace-server/src/workspace_deletion.rs @@ -76,6 +76,7 @@ const WORKSPACE_DELETION_PURGE_TABLES: &[&str] = &[ "worker_registry_projection_diagnostics", "worker_registry_projection_removals", "worker_removal_operations", + "worker_restore_intents", "worker_retention_audit_events", "worker_session_archive_observe_grants", "worker_session_archives", @@ -84,6 +85,7 @@ const WORKSPACE_DELETION_PURGE_TABLES: &[&str] = &[ "worker_workdir_attachment_reservations", "worker_workdir_links", "workspace_config_entries", + "workspace_config_grants", "workspace_config_tree_revisions", "workspace_config_trees", "workspace_create_operations", @@ -936,6 +938,17 @@ mod tests { ) VALUES (?1, 'runtime-a', 'jti-a', 1, '1')", params![workspace_id], )?; + for (workspace, grant, workdir, revoked) in [ + (workspace_id.as_str(), "config-active", "config-active-wd", 0), + (workspace_id.as_str(), "config-revoked", "config-revoked-wd", 1), + ("workspace-b", "other-config", "other-config-wd", 0), + ] { + conn.execute( + "INSERT INTO workspace_config_grants (workspace_id,grant_id,runtime_id,worker_id,workdir_id,access,revoked,created_by,created_at) + VALUES (?1,?2,'runtime-a','worker-a',?3,'read_only',?4,'owner','1')", + params![workspace, grant, workdir, revoked], + )?; + } Ok(()) }) .expect("non-FK scoped audit fixture"); @@ -1038,6 +1051,23 @@ mod tests { }) .expect("scoped audit read"); assert_eq!(proof_count, 0); + let grants = store + .with_conn(|conn| { + let mut statement = conn.prepare( + "SELECT workspace_id,grant_id FROM workspace_config_grants ORDER BY grant_id", + )?; + statement + .query_map([], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })? + .collect::, _>>() + .map_err(Error::from) + }) + .expect("config grants purge is Workspace-scoped, including revoked history"); + assert_eq!( + grants, + vec![("workspace-b".to_string(), "other-config".to_string())] + ); } #[test] diff --git a/crates/workspace-server/src/workspace_subscription.rs b/crates/workspace-server/src/workspace_subscription.rs index 54f3182ae..9e7698a30 100644 --- a/crates/workspace-server/src/workspace_subscription.rs +++ b/crates/workspace-server/src/workspace_subscription.rs @@ -12,7 +12,7 @@ use worker_runtime::identity::RuntimeWorkerRef; use crate::runtime_subscription::RuntimeSubscriptionBroker; use crate::server::{ - WorkspaceApi, authorize_browser_worker_method, connect_workspace_worker_protocol, + WorkerOperationContext, WorkspaceApi, WorkspaceWorker, WorkspaceWorkerMethodSender, }; const OUTBOUND_CAPACITY: usize = 256; @@ -20,13 +20,13 @@ const WORKER_PROTOCOL_SNAPSHOT_TIMEOUT: std::time::Duration = std::time::Duratio struct ActiveSubscription { task: tokio::task::JoinHandle<()>, - methods: Option>, + methods: Option, } pub(crate) async fn serve_workspace_subscription( api: WorkspaceApi, socket: WebSocket, - input_source: protocol::AuthenticatedInputSource, + context: WorkerOperationContext, ) { let broker = api.runtime_subscription_broker().clone(); let (mut socket_sender, mut socket_receiver) = socket.split(); @@ -90,16 +90,20 @@ pub(crate) async fn serve_workspace_subscription( worker_id, runtime_id: Some(runtime_id), } => { - let worker = RuntimeWorkerRef::new(&runtime_id, worker_id.as_str()); - match connect_workspace_worker_protocol( + let connection = match WorkspaceWorker::resolve( &api, - &worker, - Some(&input_source), - ) - .await - { + &runtime_id, + worker_id.as_str(), + ) { + Ok(worker) => worker + .connect_protocol(&context) + .await + .map_err(|error| error.to_string()), + Err(error) => Err(error.to_string()), + }; + match connection { Ok(connection) => { - let methods = connection.methods.clone(); + let methods = connection.sender.clone(); let task = tokio::spawn(run_worker_protocol( request_id, subscription_id.clone(), @@ -165,12 +169,7 @@ pub(crate) async fn serve_workspace_subscription( else { break; }; - let Ok(method) = - authorize_browser_worker_method(message.method, &input_source) - else { - break; - }; - if methods.send(method).await.is_err() { + if methods.send(&context, message.method).await.is_err() { break; } } @@ -426,6 +425,7 @@ fn project_registry_worker( worker } else { SubscriptionWorker { + restore_observation_token: None, worker_id: protocol::subscription::SubscriptionWorkerId::new( record.registry.worker.worker_id.clone(), ) @@ -601,6 +601,7 @@ mod tests { #[test] fn external_workdir_attachment_does_not_remove_workspace_worker() { let mut worker = SubscriptionWorker { + restore_observation_token: None, worker_id: worker_id(), runtime_id: Some("runtime-1".to_string()), resource_key: Some("W-1".to_string()), diff --git a/docs/README.md b/docs/README.md index 91da1a8d8..7a836749d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -35,6 +35,10 @@ It is not a dumping ground for external research, old plans, API inventories, or 27. [`development/subjektiv-product-cutover.md`](development/subjektiv-product-cutover.md) — legacy Workspace Memoryからsubjektivへのoperator cutover、explicit reset、backup／rollback runbook。 28. [`design/worker-wip-mode.md`](design/worker-wip-mode.md) — explicit-opt-in Worker WIP mode, Worldspace projection, authority, cache, and retry contracts. 29. [`design/objective-wip-projection.md`](design/objective-wip-projection.md) — Host-assigned Objective collection/item routes, operation mapping, Backend authority, bounds, and compatibility suppression. +30. [`design/ticket-wip-projection.md`](design/ticket-wip-projection.md) — Host-assigned Ticket collection/item routes, role-specific operations, route identity, and typed Backend preservation. +31. [`design/merge-request-wip-projection.md`](design/merge-request-wip-projection.md) — Host-assigned Merge Request routes and preserved review/integration authority gates. +32. [`design/feature-chat-invocations.md`](design/feature-chat-invocations.md) — declarative `/` invocation identity, grammar, completion, typed draft, attachment staging, execution, and authority boundaries. +33. [`design/workspace-config-wip-projection.md`](design/workspace-config-wip-projection.md) — Backend-authored configuration as a logical WIP attachment, explicit grants, shared UI conflict domain, and activation boundaries. ## What belongs here diff --git a/docs/architecture/worker-restore-observations.md b/docs/architecture/worker-restore-observations.md new file mode 100644 index 000000000..4efcb74cd --- /dev/null +++ b/docs/architecture/worker-restore-observations.md @@ -0,0 +1,70 @@ +# Guarded Worker Restore and Web observations + +T-701 extends the [registry-bound operation reception](workspace-worker-operations.md), not the Worker creation, Resume or Restore state machines. + +## Different tokens protect different things + +| Operation | Observed identity / expected input | Atomic validation and execution owner | Conflict / uncertain outcome | Shared client behavior / deliberate difference | +| --- | --- | --- | --- | --- | +| Restore | Workspace + Runtime + Worker identity; Runtime-owned `restore_observation_token`; immutable `expected_observation_token` + `request_id` | Existing Runtime per-Worker lifecycle operation lock, durable admission and lifecycle commit. Preparation and execution follow admission, not a frontend GET comparison | Stale new intent is HTTP 409 (`restore_observation_conflict`); operation results remain accepted / rejected / rolled_back / reconciliation_required. A lost reply is unknown, not definite rollback | Both entrances use the same scoped POST/parser/notice classifiers and observation refresh. Never replace a rejected intent's token and automatically retry | +| Stop | Identity and reason, **not** a catalog revision CAS | Runtime per-Worker operation lock and existing backend Stop settlement; Workspace session finalization only after accepted | Existing flat accepted / unsupported / rejected result with diagnostics | Same encoded target paths and bounded HTTP-error messages; refresh observations instead of writing a guessed local stopped state. Stop does not gain Restore's generation guarantee | +| Pause / Resume / Cancel / Compact | Connected execution identity and `WorkerCommandEnvelope.command_id` | Existing bound execution/controller command admission | Command acknowledgement: Accepted / StaleCommandId / Conflict / InvalidState, with authoritative state | Command IDs order commands; they are not catalog observation generations. Resume is not Restore; connections do not silently move to a replacement execution | +| Delete / cleanup | Runtime + selected target IDs; `expected_plan_revision` **and** `expected_plan_digest` | Existing cleanup validation, per-target live/retention revalidation, removal service and catalog commits | Existing cleanup-plan conflicts, per-target status and diagnostics | Preserve plan protection for candidate sets and dependencies. Never substitute a Worker generation. Page plans invalidate at the beginning of every catalog refresh and are unavailable until fresh catalog and plan observations settle | +| Pin / Unpin | Registry Worker identity and explicit boolean | Registry retention update and ordered projection publication | Existing retention result/errors | Preserve retention semantics and full catalog metadata; refresh shared observations and cleanup plan | +| Submit / Notify | Authenticated source + durable submission / notification request ID | Existing controller/session durable receipt admission | Existing accepted/rejected receipts and unknown delivery diagnostics | These are not Restore requests; Restore never manufactures input, Resume or a new Worker | + +There is no universal Worker command token or shared frontend action lock. Workspace reception resolves a stable target and rechecks caller authorization; Runtime/controller/store owners retain operation-specific atomic boundaries. + +## Browser request and result + +Normal explicit Restore is enabled only for `availability = observed`, catalog `stopped`, and an available backend observation token. Sidebar display state and foreground `worker_state` are not eligibility evidence: sidebar uses `lifecycleState`; the page uses the full catalog's `state`. Neither UI guesses snapshot, singleton, Workdir provider or authorization eligibility. + +Both entrances send the generated two-field `WorkerRestoreRequest` to: + +```text +POST /api/w/{workspace_id}/runtimes/{runtime_id}/workers/{worker_id}/restore +``` + +Each ID is independently encoded. No Ticket-assignment query, Submit, Resume or Worker-create call is used. The bounded parser validates wrapper identity, the four states, diagnostics and optional **Workspace WorkerSummary**. A Stop lifecycle result or launch summary is not a Restore response. Unknown states/fields/invalid identities cannot produce a success notice. + +Accepted means the existing Restore operation committed, not that the Worker is necessarily Running or a later run succeeded. A returned summary validates result identity; display is updated from fresh shared observations, not an old POST snapshot. Rejected and rolled_back display diagnostics without success. Reconciliation_required and transport/malformed-response uncertainty retain the original immutable request for an explicit **Retry Restore** action. That action sends the same request; it does not create an operation, exchange its observation token, or automatically loop. A conflict clears this local recovery intent and asks the user to review fresh observations before a new explicit operation. A GET alone does not settle Runtime's pending lifecycle operation. + +## Backend admission, preparation and recovery + +The Workspace `WorkspaceWorker` reception still resolves target identity and rechecks caller authority. Its Restore coordinator first dispatches the public tuple to Runtime without candidate preparation. A terminal owner returns its historical receipt; an admitted pending owner supplies its stored preparation before renewed singleton/Workdir/External eligibility is evaluated. Historical result summaries are read from the current Runtime and are not written back as catalog observations. + +For a new intent, Workspace builds read-only preparation references after the normal eligibility checks. Runtime atomically checks the observation token again under the existing per-Worker lock and persists the owner together with `pending_restore` and its existing lifecycle operation ID. Only this admitted operation can proceed to SSH resource issuance, binding/access preparation and execution. A race between the first observation check and admission therefore cannot issue credentials or launch a new Restore from the old intent. + +Preparation completion is attached to that same pending operation, not a replacement operation. Retained repository-access snapshots and exact request identity allow interrupted preparation to resume without exchanging candidate bindings. Runtime retries use the original owner's payload; a freshly generated preparation cannot rewrite the admitted target or expected observation. Stop and startup reconciliation continue to use the existing pending lifecycle ownership. When startup/legacy persistence has a pending lifecycle operation but no public receipt owner, an explicit request with the current observation may bind recovery to that exact operation ID, without applying candidate preparation. Once bound, a different request cannot take over; terminal replay does not resurrect a later stopped Worker. + +| Caller | Intent identity retained / validation boundary | +| --- | --- | +| Web sidebar and page | Immutable two-field request retained locally for explicit uncertain-result recovery; no shared in-flight lock | +| Workspace alias, scoped and Worker-control HTTP | Mandatory public request; common target/auth reception and the same Runtime admission | +| Embedded / remote Runtime providers and management HTTP | Typed Runtime request and coordination transport, including original observation and lifecycle operation owner | +| `WorkerRestore` Tool | Observed token in Tool arguments and durable Tool-call-derived request identity | +| Existing TUI Restore/resume path | Selected row's token and exact request retained through the existing flow. Unknown/pending results refresh Session observation and prompt for an explicit same-tuple retry; cancellation does not retry and reports the original tuple. Conflict exits for fresh selection, never exchanges the failed intent. No new TUI Restore entrance | +| Internal Coder, Orchestrator, subject reuse and compensation | Domain intent journal pins the request/token before dispatch and retains it through unknown result and restart. A definitive observation conflict retires only that rejected intent without automatic retry; a later deliberate invocation may pin fresh observation with a fresh request ID. It does not own lifecycle status or bypass Runtime validation | +| Runtime startup / pending reconciliation | Private recovery of the existing persisted Restore owner and operation; not an external unguarded new-intent endpoint | + +The Workspace internal intent journal stores client retry identity only. Runtime remains the lifecycle authority; it is not a second Restore state machine, Worker ledger or cross-screen lock. + +## Observation and cleanup-plan ordering + +`workspaceWorkersStore` retains the existing sidebar stream/projection and also fetches the full Workspace Worker catalog for the page. Full catalog metadata preserves retention, attachments, implementation and targets that are not sidebar entries. Stream updates overlay lifecycle/availability/foreground snapshot/token without interpreting absence as a new page exclusion. + +Every stream update or explicit refresh increments a local **observation scheduling version** and publishes `catalogRefreshing` before fetching. This local version fences asynchronous display work; it is **not** Restore admission authority and is never sent to Runtime. A newer event/GET supersedes older GET completion. Old POST summaries and page-loader data cannot overwrite the active catalog. On disposal, requests and subscription callbacks are fenced. Catalog fetch errors have their own stable alert episode rather than clearing transport/Workdir alerts. + +The page invalidates cleanup plans immediately when observations refresh. It does not fetch or install plans while `catalogRefreshing` is true. Successful refresh completion triggers plan reads even when the resulting catalog contents did not change. Workspace/lifetime and cleanup epochs reject late previous plan responses. A failed refresh leaves old candidates unavailable instead of treating old page data as fresh. + +Both components retain local busy state only. Requests pin their starting Workspace/target, and late replies after Workspace switch/disposal cannot update another Workspace's notices, list or busy state. Sidebar and page may issue competing requests: atomic Runtime admission—not a cross-component client lock—decides their outcomes. Console links and existing sidebar filtering remain unchanged. + +## Validation surfaces + +- Runtime admission tests prove stale/ABA and serialization, durable same-request replay and pending reconciliation. +- Runtime management, embedded/remote providers, Workspace aliases/scoped/control routes, Tool and existing TUI callers exercise the guarded API rather than a legacy unguarded Restore. +- `tests/runtime-workers.test.ts` and `test/sidebar/worker-actions.test.ts` prove response/request and notice contracts; the Runtime parser tests are included in the normal Web test task. +- Rendered `worker-stop.browser.spec.ts` and `workers-page.browser.spec.ts` cover menu/action eligibility, busy behavior, four outcomes and errors, explicit retry, Workspace/disposal fences, ordinary Console/Pin/Delete, sidebar-origin updates and cleanup epochs. +- `worker-restore-store.browser.spec.ts` tests the real store's full metadata, GET/event ordering, dependent-plan invalidation, no POST recovery loop, catalog-only target preservation and disposal. + +Exact executed validation counts and immutable source/review evidence belong to the repository Merge Request and final approved Ticket handoff. diff --git a/docs/architecture/workspace-worker-operations.md b/docs/architecture/workspace-worker-operations.md new file mode 100644 index 000000000..ea247b845 --- /dev/null +++ b/docs/architecture/workspace-worker-operations.md @@ -0,0 +1,140 @@ +# Workspace Worker operations + +T-702 documents the implemented Workspace operation-reception boundary. It does not move execution authority into the Workspace registry or make all Worker-related routes use one API. The common implementation is [`worker_operations.rs`](../../crates/workspace-server/src/server/worker_operations.rs); production adapters and service orchestration are in [`server.rs`](../../crates/workspace-server/src/server.rs). + +## Identity reception is not execution authority + +`WorkspaceWorker::resolve` resolves a Workspace registry reference to a `RuntimeWorkerRef` (Runtime ID and Worker ID). Resolution first tries that exact pair, then a Workspace resource reference constrained to the requested Runtime. It returns an identity-bound handle, not a live execution snapshot, authorization lease, or promise that the Worker is running. Each operation rereads the target registry record and authorizes its own context. + +The owners have different responsibilities: + +- **Workspace registry and domain services:** durable Workspace identity, retention/pinning, Workdir links and sessions, Ticket assignments, credentials/bindings, control grants, restore coordination, and removal plans/catalog publication. Registry projections and observation streams describe Workers; they do not admit controller commands. +- **`RuntimeRegistry` and Runtime adapters:** select the Runtime owning the resolved pair and delegate execution. Remote and embedded adapters preserve their existing transport/error behavior. Runtime owns lifecycle operation settlement and checks Workspace scope at the scoped embedded execution boundary. +- **Execution backend and Worker controller/session:** own the actual execution endpoint, command admission, authoritative Idle/Busy state, in-flight run transitions, Submit FIFO and durable receipts, Notify receipts, and compaction. A catalog status is not sufficient evidence for command admission. + +The term *controller* in a `WorkerControl` context means the calling Runtime Worker holding a grant over a target. It is not the target's in-process execution controller. + +## Production ingress → common method → execution owner + +For the HTTP rows below, `R` abbreviates `/api/runtimes/{runtime_id}/workers/{worker_id}` and `W` abbreviates `/api/w/{workspace_id}/runtimes/{runtime_id}/workers/{worker_id}`. Aliases operate in the server's configured Workspace; scoped routes additionally validate the Workspace path. The declarations live in [`server-api`](../../crates/server-api/src/lib.rs), and the generated contract service implementations in `server.rs` are the production unary adapters. + +| Ingress | Common reception / coordination | Execution or persistence owner | +| --- | --- | --- | +| `POST R/restore` and `POST W/restore` | `WorkerOperationContext::from_request` → `restore_runtime_worker_with_context` → `WorkspaceWorker::restore`; scoped route first uses `scoped_restore_runtime_worker_with_context` | `WorkspaceApi::restore_workspace_worker` retains the existing singleton, Workdir, credential/binding and pending-operation reconciliation flow; Runtime/backend owns restored execution | +| `POST R/input` and `POST W/input` | Resolve → convert request → `WorkspaceWorker::input` | `RuntimeRegistry::send_input` → Runtime adapter → execution backend/controller/session | +| `POST R/stop` and `POST W/stop` | `WorkspaceWorker::stop` | Runtime owns Stop; on an **Accepted** result Workspace closes the current Workdir session and synchronizes linked Workdir state. Rejected/Unsupported results do not perform that finalization | +| `POST R/cancel` and `POST W/cancel` | `WorkspaceWorker::cancel` | `RuntimeRegistry::cancel_worker` → target execution controller; not registry removal | +| `PUT W/pin` / `DELETE W/pin` | `WorkspaceWorker::set_pinned(true/false)` | Workspace store retention update, ordered through `worker_projection`; no execution command | +| `GET /api/w/{workspace_id}/runtimes/{runtime_id}/cleanup-plan` | `build_runtime_cleanup_plan` (preview, not a Worker mutation) | Workspace cleanup planner/projections | +| `POST /api/w/{workspace_id}/runtimes/{runtime_id}/cleanup-executions` | `execute_runtime_cleanup_with_context` validates plan revision/digest and selected candidates → each selected Worker's `delete_from_plan` | `WorkerRemovalService::execute_cleanup_removal` → existing retention planning, Runtime retention execution, and Workspace catalog commit. Workdir-only actions remain in the cleanup service | +| `POST /api/w/{workspace_id}/workers/remove` (`WorkerRemove`, not a browser removal alias) | `worker_remove_contract` verifies its source proof against the requested target → `WorkspaceWorker::remove` | `WorkerRemovalService::execute_async` checks the `remove` grant and coordinates retention/removal. It rejects self-removal and requires a stopped target for a new removal; existing durable removal operations have their own recovery path | +| Embedded `WorkerRemove` Tool | Runtime-owned in-process target-bound proof → `EmbeddedServerWorkerMutationDispatcher` verifies/consumes proof → installed `EmbeddedWorkspaceWorkerRemoveExecutor` → `WorkspaceWorker::resolve_with_services` → the same `WorkspaceWorker::remove` | Same `WorkerRemovalService::execute_async`, grant-lock/recheck, self/live/retention guards, durable plan/recovery and catalog commit as HTTP; no Runtime-to-Server dependency | +| Dedicated WS `R/protocol/ws` and `W/protocol/ws` | `worker_protocol_ws` (scoped wrapper for `W`) → `WorkspaceWorker::connect_protocol` → `workspace_worker_protocol_session` → bound sender | One remote WS or embedded execution transport; controller/session owns protocol operation results | +| Multiplexed WS `/api/w/{workspace_id}/protocol/ws`, runtime-scoped `WorkerProtocol` selector | `scoped_workspace_protocol_ws` → `serve_workspace_subscription` → resolve + `connect_protocol`; subscription stores the bound sender | Same connected execution transport as dedicated WS; the subscription layer wraps its snapshot/events, not a different execution implementation | +| Worker-control `POST /api/w/{workspace_id}/worker-control/workers/{runtime_id}/{worker_id}/{input,cancel,stop,restore}` | `send_known_worker_input`, `cancel_known_worker`, `stop_known_worker`, `restore_known_worker` authenticate the Worker source, construct `WorkerControl`, and call the same identity methods/restore adapter | Same Runtime execution owners, under the applicable active target grant; known-worker input adds its existing non-Accepted-result error translation | + +Both WS adapters call `WorkspaceWorkerMethodSender::send`, which selects the following **named methods** rather than implementing separate command semantics in each transport: + +| Public protocol method | Named common sender method | Distinct owner/receipt | +| --- | --- | --- | +| `Pause { command }` | `pause` | Controller command admission/acknowledgement; pause is not Stop | +| `Resume { command }` | `resume` | Controller resumes the paused run; not Restore | +| `Cancel { command }` | `cancel` | Controller command admission for cancelling the current run; distinct from HTTP lifecycle request/result | +| `Compact { command }` | `compact` | Controller maintenance/compaction, with command acknowledgement and subsequent lifecycle events | +| `Submit { submission_request_id, input }` | `submit` | Session durable Submit receipt/FIFO and later run execution | +| `Notify { notification_request_id, message }` | `notify` | Session durable Notify receipt and advisory delivery, not the Submit FIFO | + +All six methods funnel through `dispatch` / `send_checked`. Other browser-allowed protocol methods also use that checked dispatch; the six named methods are not an exhaustive allowlist. For example pending-submission operations, rewind, Shutdown, completions, ListWorkers, RestoreWorker and RegisterPeer are allowed by `authorize_browser_worker_method`. This forwarding does not replace their lower-layer semantics with Workspace registry operations. + +`build_inner_router` installs the three production WS routes using typed `websocket_route` registrations and merges the generated Workspace contract router for unary routes. [`workspace_subscription.rs`](../../crates/workspace-server/src/workspace_subscription.rs) handles multiplexing. Its `WorkspaceWorkers` subscription is a separate catalog stream with no method sender; only a runtime-scoped `WorkerProtocol` subscription carries an operational sender. + +## Context and authorization restrictions + +`WorkerOperationContext` is crate-private. Clients cannot submit this enum to select their authority. + +- **Request-derived Worker control:** `from_request` uses the middleware-verified Runtime Worker subject and current Workspace registry membership. The shared source-binding helper rejects Runtime-only proofs and conflicting identity hints; no hint-only request or Account fallback can impersonate a Worker. Known-worker adapters bind their source through the same verified request extension before constructing the operation context. +- **Account/browser context:** despite the `Browser` name, this also represents account-authenticated unary requests. It retains request headers and the server-derived Account input source. On each common authorization call, the server re-resolves credentials, rejects revoked credentials or an Account change, requires the configured Origin for `BrowserSession` actors, and requires an active Workspace. It does not interpret the operation's permission string as a Worker-control grant check for an Account actor; request middleware remains part of the route's authorization boundary. +- **Worker-control context:** the target and calling Worker must be registered. The server requires an active grant for this controller/target pair and the operation's permission, locks by grant ID, then rereads the target and rechecks permission and the exact grant ID after acquiring the lock. A replacement grant requires new admission under its own lock. A missing active grant is reported as UnknownWorker; a present grant lacking the permission is InvalidInput. Merely knowing an ID does not grant control. +- **Test Backend context:** `WorkerOperationContext::Backend` exists only under `#[cfg(test)]`. It bypasses context authorization after the target-record check. There is no production Backend enum variant for a client, Worker Tool, or generic internal caller to select. Production Backend services continue to own their explicit domain flows. + +Worker-control grants are Workspace-owned records, not inferred parentage or universal access to every Runtime Worker. The production spawn flow persists a controller/subject grant with `send_input`, `notify`, `cancel`, `stop`, `restore`, `remove`, and `observe`; that default does not include `pin`. Known-worker discovery follows active grants, while Internal children use their separate parent-owned registry. Other grant origins/permission sets must be read from their own authority rather than assumed from this spawn default. + +Permissions are `restore`, `stop`, `cancel`, `pin`, and `remove` for those identity operations. HTTP input uses `notify` for Notify and `send_input` for all other accepted kinds. `connect_protocol` checks `observe`; every connected method dispatch checks `send_input`, requires an interactive target, then explicitly requires a **Browser** context. Thus an observe grant is not a mutation grant, and a Worker-control grant does not enable browser WS method forwarding. WS Notify also uses this browser-only `send_input` check, not HTTP input's `notify` grant mapping. + +`input` and all connected protocol method sends reject Workers with a Backend Job projection: their immutable input is Backend-owned and their Console is read-only. Connecting for observation is not itself rejected by that interactive check. + +For Worker-control user input, `input` converts `User` to `UserIfIdle`; admission belongs to the target controller and never queues behind an active run. Browser Submit may durably queue while Running/Paused. Notify is advisory input, not a queued Submit; it can be incorporated at a between-turn boundary or drive the notification flow while Idle. The known-worker input adapter turns definite non-Accepted input results into errors, but keeps uncertain outcomes as the existing typed result with a `worker_input_outcome_unknown` diagnostic, and supplies the Idle-only guidance for user input. + +Scoped Restore may coordinate Ticket assignment only when both `ticket_id` and `assignment_operation_id` are supplied. A Worker-control context cannot use either assignment parameter. Aggregate cleanup rejects all Worker-control requests: Worker callers must retain the stronger target-bound WorkerRemove proof/retention path. A control grant is not authority over Ticket assignments or Workdir cleanup. The cleanup coordinator revalidates Account credentials, Origin, and active Workspace even for a Workdir-only request. + +The HTTP and embedded `WorkerRemove` entry points share the registry-bound removal method. `WorkerRemove` intentionally has a distinct authorization contract from the other operations. It accepts a verified mutation source rather than `WorkerOperationContext`, verifies its target-bound proof before calling the identity method, and keeps source-specific and grant revalidation in the removal service. It is not permission-free just because it does not call `authorize_operation`. + +The embedded executor retains only the removal services with a **weak** RuntimeRegistry reference. Its removal-only `WorkspaceWorker<()>` specialization shares resolution, current-record checks and the removal method with the full `WorkspaceWorker` handle. This service projection avoids a strong `WorkspaceApi → Runtime → dispatcher → WorkspaceApi` ownership cycle; it is not another Worker ledger or an authorization fallback. It cannot call the full handle's restore/input/protocol methods. The existing independent Runtime-owned proof forwarder and verification/replay seam stay unchanged. + +## Tokens and outcomes are operation-specific + +There is no universal Worker command token or universal success result. + +- `RuntimeWorkerRef` identifies the registry target; a Workspace resource reference is an ingress lookup form. Neither identifies a particular run or proves authority. +- `WorkerCommandEnvelope.command_id` orders controller lifecycle commands. `WorkerCommandAcknowledgement` contains command kind, disposition (`Accepted`, `StaleCommandId`, `Conflict`, `InvalidState`) and authoritative state. These IDs belong to controller command admission, not durable Submit identity or Workspace removal planning. +- `submission_request_id` is the caller's Submit retry key in an authenticated-source namespace. A durable `SubmissionAccepted` receipt also contains the Worker-allocated `submission_id` and `Started`/`Queued` disposition. `SubmissionRejected` is correlated by request ID. Acceptance is not a successful completed run; `last_finished_submission_request_id` is a teardown fence, not a result-success claim. +- `notification_request_id` correlates `NotificationAccepted` / `NotificationRejected`. Notify has its own durable receipt and exact-payload retry behavior; it does not allocate a Submit FIFO entry. Browser Submit/Notify are rewritten to `SubmitTracked`/`NotifyTracked` with the server's Account source. Browser-supplied tracked forms, `SubmitIfIdle`, and `WorkerEvent` are rejected. +- Runtime `WorkerLifecycleOperationId` is a durable lifecycle/reconciliation identity, separate from a controller command ID. Ticket `assignment_operation_id`, cleanup plan revision/digest, and retention plan/operation IDs and fingerprints likewise belong to their own domain checks. A multiplexed `subscription_id` only routes frames to an already-bound sender. + +The common sender returns `Result<()>` after authorization and enqueueing on its transport channel. This is **not** a controller acknowledgement, durable receipt, or proof of upstream delivery. Actual outcomes arrive on the event stream. The dedicated WS reports dispatch errors as protocol error events; the multiplexed adapter currently terminates the connection when sending a method fails. It also requires an initial session Snapshot before accepting the WorkerProtocol subscription. + +Unary input/lifecycle responses preserve their existing API projections: `WorkerOperationState` is `Accepted`, `Unsupported`, or `Rejected`, with Runtime/Worker IDs and diagnostics. Restore instead has `Accepted`, `Rejected`, `RolledBack`, and `ReconciliationRequired`. Pin returns retention state; cleanup returns per-target actions/statuses; WorkerRemove has its own response/error body. These are not interchangeable acknowledgements. + +A transport or store failure can leave an **unknown outcome** after effects have started. Lower-layer input results have `WorkerInputDisposition::{Accepted,Rejected,Unknown}` and an optional `runtime_run_id`, but `worker_input_result_to_api` projects only state, target IDs and diagnostics: the public unary input result does not expose those two fields. Unknown delivery is explicitly preserved as a `worker_input_outcome_unknown` diagnostic; neither HTTP adapter changes it into definite rejection or retries it. Runtime execution results also distinguish Busy/Rejected/Errored/Unsupported; error handling and durable restore/removal recovery remain lower-layer responsibilities. Do not infer that a rejected-looking projected response or a closed WS proves that nothing was accepted. The new reception layer does not add automatic replay or an exactly-once guarantee across every ingress. In particular, `worker_input_request_from_api` sets `submission_request_id` to `None`; the unary Workspace input contract does not provide the browser WS Submit retry-key contract. + +## Connected execution lifetime + +`WorkspaceWorkerMethodSender` stores both the resolved Worker handle and the sending channel privately. A WS caller cannot pair a different Worker handle with a subscription's raw sending capability. Registry identity can outlive an execution, but an operational connection does not follow it to a replacement execution. + +`connect_workspace_worker_protocol` obtains an observation source from the Workspace proxy (falling back to the Runtime only for WorkerNotFound). It then connects to one remote execution WS or attaches one embedded `WorkerProtocolTransport`. The embedded path captures the real endpoint and its initial snapshot, not the WorkerRef-wide catalog observation bus. + +Embedded dispatch uses `send_connected_protocol_method_scoped`. Runtime verifies that the transport target equals the authorized WorkerRef, checks scope and pending-operation settlement, and validates the captured endpoint before any durable mutation. Even a delayed terminal-close observation cannot make the old endpoint Stop or send to a new execution under the same identity. + +Shutdown and a closed execution relay terminate the operational stream. Embedded lifecycle/store/transport uncertainty is terminal; a settled Busy/Rejected/Unsupported rejection may leave it usable only if the endpoint still validates. Remote bridge failure closes its channels. `send_checked` reports `worker_protocol_closed` when its channel is closed and explicitly states that no implicit Restore or retry occurred. No bridge in this path retargets, restores, or reconnects to the next execution. The caller must explicitly Restore when authorized and establish a new connection; this is different from catalog observation continuing to describe the same registry identity. + +## Internal Workers are parent-owned + +Internal SubWorkers and service Internal Workers are not Workspace registry targets or independently addressable Runtime protocol subjects. They remain separate, parent-owned sessions. Their visible identity is `InternalWorkerRef.session_id` (`name` is display-only), and snapshots/events are presented through the parent's protocol stream; service-private Internal Workers are excluded from the public child inventory. + +Direct-child control/observation stays in the parent-owned Worker machinery. A Workspace Runtime Worker grant is not a grant to an arbitrary Internal Worker, and this registry handle does not create an Internal Worker Runtime record, independent transport, or implicit Restore path. The Workspace observation capture adapter explicitly rejects a non-RuntimeWorker subject; it does not resolve a SubWorker as a registry Worker. + +## Regression helpers are not production routes + +Lower-level test functions in `server.rs` must not be mistaken for installed unauthenticated endpoints: + +- `set_worker_retention`, `send_runtime_worker_input` / `dispatch_runtime_worker_input`, and `execute_runtime_cleanup` / `scoped_execute_runtime_cleanup` are `#[cfg(test)]` wrappers using the test-only Backend context. +- `worker_protocol_ws_session`, `remote_worker_protocol_ws_session`, and `embedded_worker_protocol_ws_session` are `#[cfg(test)]` transport regression helpers. Production dedicated WS uses `workspace_worker_protocol_session` with the common bound sender; production multiplexed WS uses `serve_workspace_subscription` with that same sender type. +- Production `connect_workspace_worker_protocol`, `connect_remote_worker_protocol`, and `connect_embedded_worker_protocol` are real shared connection helpers, not those test-only sessions. The grant-checked observation capture service also calls the connection helper to read a committed Snapshot; that read-only capture is outside the six operational WS methods. + +Source presence alone is not route wiring. Follow `build_inner_router`, generated contract implementations, and the typed route declarations when auditing production ingress. + +## Minimum scope, limitations, and future guard + +The boundary above covers the T-702 minimum operation set. It is not a claim that every session/file/completion API has been migrated: + +- Session snapshot/history and retained session attachment reads remain their existing observation/retention paths. They are not controller command admission. +- Attachment upload grants and upload cancellation keep their Account/target-bound grant checks; file upload and uploaded-file deletion still resolve the target and delegate through `api.runtime`, rather than calling a new `WorkspaceWorker` file method. +- Unary completions still resolve the registry reference and call `api.runtime.worker_completions`. Protocol `ListCompletions` is also browser-allowed through checked WS dispatch. Those are different ingress paths. +- Worker creation, Backend Job execution, rollback compensation, Workspace deletion, and other domain-owned orchestration are not universally rewritten to this handle. Legitimate lower-level Runtime calls still exist. + +Known limitations worth follow-up, rather than undocumented guarantees: + +1. Browser credentials/Account are rechecked for each common operation or WS method send, but not continuously for every outgoing observation event. This is not a promise of immediate revocation-driven stream teardown. +2. Cleanup is sequential rather than an all-target atomic transaction. The existing cleanup plan revision/digest and each removal service's live revalidation remain distinct protections. +3. Unary input has its existing narrower wire contract, rather than the WS authenticated receipt envelope. Identity unification does not create a new retry contract or manufacture caller request IDs across transports. + +**T-701's stale-observation Restore guard is not implemented here.** T-701 will connect the caller's observed revision/generation to Runtime's existing operation lock, distinguish a new Restore intent from pending reconciliation, and handle UI conflicts. This boundary leaves those Restore inputs/results and the pending-operation state machine intact rather than creating a duplicate guard. + +### Implementation references + +- [`server/worker_operations.rs`](../../crates/workspace-server/src/server/worker_operations.rs): identity handle, contexts, authorization, named methods and private bound sender. +- [`server.rs`](../../crates/workspace-server/src/server.rs): contract adapters, `build_inner_router`, scoped Restore/cleanup, WorkerRemovalService, dedicated WS and shared transport bridges. +- [`workspace_subscription.rs`](../../crates/workspace-server/src/workspace_subscription.rs): multiplexed WorkerProtocol binding and catalog subscriptions. +- [`hosts.rs`](../../crates/workspace-server/src/hosts.rs): Runtime registry/adapters and distinct input/lifecycle result projections. +- [`protocol`](../../crates/protocol/src/lib.rs): command envelopes/acknowledgements, Submit/Notify receipts and Internal Worker presentation. +- [`worker-runtime/runtime.rs`](../../crates/worker-runtime/src/runtime.rs) and [`execution.rs`](../../crates/worker-runtime/src/execution.rs): scoped connected endpoint validation, lifecycle identities and execution results. diff --git a/docs/design/checkout-wip-projection.md b/docs/design/checkout-wip-projection.md new file mode 100644 index 000000000..acb67b05d --- /dev/null +++ b/docs/design/checkout-wip-projection.md @@ -0,0 +1,232 @@ +# Attached checkout WIP projection + +T-699 publishes the contents of **current Worker attachments**, independently of +Repository/Workdir inventory and management. This is a Yoi implementation of +request-time projection, **not a dependency on `wip-fs-server`**. The upstream +read-only source/specification informed the design; its byte `read` and +truncate/write `replace` contracts are not substituted for Yoi's text/edit/save +contracts. No file-tree store, synchronization, or per-file resident registration +is created. + +```text +/ +├── repositories inventory/reference +├── workdirs identity/lifecycle +├── workdir-attachments Backend connection lifetimes +└── checkouts current accessible content entrances; list + └── directory; glob, grep, create_file + ├── README.md file; read, edit, write + └── src directory; glob, grep, create_file +``` + +Operations are Interface declarations, not children or `/features/...` routes. +File tools are native when the normal Worker WIP surface is installed. Bash stays +in compatibility mode. Ordinary Tools mode retains Read/Edit/Write/Glob/Grep. +Feature activation never grants READ, WRITE, EDIT, search, COMMAND, or delegation. + +## Placement and ownership + +`worker::checkout` owns Worldspace binding, contextual Interface declarations, +permission input reconstruction and safe output. It never opens a host pathname. +It resolves the live `WorkdirSessionRouter` on each request. The existing +`WorkdirSession` provider executes `checkout_observe` and `checkout_execute`; +Runtime HTTP and External transports carry typed operation/result envelopes at +their existing authorization, connection-generation and cancellation boundaries. +The Workspace-attached session forwards through the existing Backend broker. +Endpoint paths, credentials and materialized roots do not enter WIP descriptors, +Objects, arguments or results. + +The common Host registry has an **explicit asynchronous subtree provider** +contract, separate from the existing one-segment dynamic item families. It +validates every resolved projection's path/owner/descriptor and every enumerated +child's immediate-parent relationship. Static and dynamic/subtree overlaps are +rejected. Observe resolves only the requested depth; successful observations are +complete through that depth, not partial trees masked as success. Directory +listing and native search use the shared List/Glob/Grep engines through typed +`CheckoutSearchRequest` forwarding. Scoped sessions append intersecting rule +layers, translate their cwd, and require provider-side visibility checks before +entry descent/content reads. Results retain session-logical coordinates, so a +SubWorker cwd does not duplicate a path prefix in its Object links. Unsupported +providers fail closed rather than falling back to a broader source search. +Direct resolution also uses current provider authority, not possession of a +cached Interface. + +## Names and connection isolation + +Aliases use T-697's canonical encoding: nonempty ASCII letters/digits/`-`/`_` are +unchanged; every other UTF-8 alias is `~` plus lowercase UTF-8 hex, with `~` itself +encoded. This is injective, not display-name slugification, Unicode normalization, +URL decoding or case folding. Only canonical spellings are accepted. + +`checkouts.list` returns alias, slug, Workdir ID, Worker-local attachment generation +and `path`. Attachment inventory retains its Backend connection lifetime ID and +can return `checkout_path`/`checkout_slug` for the matching live router attachment. +Attach results point to the attachment catalog and checkout entrance. Workdir +existence alone does not expose its content. Following a reference still requires +current scope/capability and an accessible filesystem projection. + +Objects have no shared inode `ref`: hard-linked names and different attachments +must not be mistaken for one path-independent representation. Interface references +are opaque, path-qualified and include a fresh projection incarnation and live +attachment generation. Validators combine that namespace with the provider's +opaque identity/state validator. Restoring a Worker creates a fresh incarnation; +detach/alias reuse changes the router generation. An old Interface, validator or +read-history entry cannot retarget a new connection. No separate connection ledger +is added. The existing router's active-operation exclusion and begin/finish/cancel +detach paths remain authoritative; detach does not remove the Workdir/files. +Collection enumeration rejects connection-set changes between sampling and +publication instead of pairing old items with a new validator. Search-link +publication checks the captured attachment generation, not merely a reused +alias; response-side provider awaits are covered by the operation deadline. + +## AI Operation mapping + +All operations bind their attachment and target from the WIP route. There is no +`target_workdir`, Workdir ID, host path or replacement `file_path` argument. +Arguments not declared by the descriptor are rejected before dispatch. Paths +below are relative to the **bound directory**, not the Worldspace or endpoint. + +| Object | Operation | Arguments | Result | +| --- | --- | --- | --- | +| `/checkouts` | `list` | none | bounded `items` containing alias/slug/Workdir/generation/path | +| directory | `glob` | `pattern`, optional `path` | normal bounded Glob text/summary plus same-checkout Object links | +| directory | `grep` | `pattern`; optional `path`, `glob`, `type`, `case_insensitive`, `-A`, `-B`, `-C`, `multiline`, `output_mode`, `head_limit`, `offset` | normal Grep grouped/context/count text and typed provider search targets mapped to Object links | +| file | `read` | optional line `offset`, `limit` | line-numbered text and normal Read summary | +| file | `edit` | `old_string`, `new_string`, optional `replace_all` | replacement count/summary and preview | +| file | `write` | `content` | existing-file save summary; never creation | +| directory | `create_file` | `path`, `content` | create-new summary and created-file link; permitted missing parents are created | + +Native operation results use a JSON record with `summary`, nullable `content` and +`items` (`path` is an absolute WIP Object navigation coordinate, not an OS path). +Search links are constructed from **typed provider paths**, not by parsing Grep's +rendered filenames/line delimiters. Provider scope remains authoritative for +search output as well as links. Glob/Grep share normal provider ignore/pattern, +context/output mode, offset and bounded result behavior. Descriptor-safe ignore +loading preserves root/nested rules and prunes ignored directories, checking +scope before opening each rule source. It cannot read global/ambient ignore +configuration outside provider authority or chase linked-worktree administration +pointers; source-class precedence, per-provider Git-marker/error behavior and +text/aggregate limits are detailed in `crates/workdir/CHECKOUT_PROVIDER.md`. +Search does not claim an +atomic snapshot of every descendant: it fences the bound directory before/after +search and each provider read/search retains its own consistency guarantees. + +Read offsets are zero-based **lines**, limit defaults to 2000, displayed numbers +are one-based. The shared Read retains at most 256 KiB of source text and bounds +its numbered presentation to 256 KiB, with an explicit truncation marker when +numbering/Unicode expansion exceeds that budget. No byte-range +operation is published by this native surface. Partial Read records the provider's +full bounded-source hash and total line count, not the hash of only its displayed +slice. Native and ordinary Tool calls use the same original alias/generation +tracker, mutation coordinator, rendering and parameter validation. + +Edit requires a nonempty, different `old_string`/`new_string`; `old_string` must +occur exactly once unless `replace_all` is true. Existing Write/Edit require a +prior successful Read through the same attachment. The read-history hash check +and the provider identity validator are **both** required: a same-content inode +replacement cannot be blessed merely by matching a hash. Failed/conflicting Read +must not record history; successful mutations update hash/change statistics only +after provider success. Create is strictly beneath the bound parent, creates only +an absent file, and must not overwrite an existing destination or symlink. + +Manifest permission identities remain Read/Edit/Write/Glob/Grep; create_file uses +Write. The Host reconstructs the same route-bound input for policy evaluation; +provider read/write scope, capabilities and delegated leases are checked again. +`ask` fails closed. Native replacement claims remove `/tools/Read`, `/tools/Edit`, +`/tools/Write`, `/tools/Glob`, `/tools/Grep`, not unrelated Tools. + +## Consistency, safety and resource policy + +The provider resolves/pins the entry/parent, compares its opaque validator and +executes shared `fs-operation` processing within the same checked boundary. The +validator represents identity and protected metadata, independently of the Read +content hash. Read checks state across its stream and returns a validator for +that same state. The WIP response uses the **execution result's validator**, not a +later independent path stat that could bless a replacement after execution. + +The native checked filesystem projection deliberately rejects symlink traversal +and unsupported/special entries. It does not silently broaden existing resolved +or logical scope. Ordinary Tools retain their existing symlink-policy behavior. +External sessions retain the root descriptor pinned **before approval/grant**; +renaming/replacing its ambient pathname must not change the approved root. +Absolute paths, traversal, alternate alias arguments and special-file mutation +are not native capabilities. UTF-8 names are byte-for-byte; non-UTF-8 names must +not be lossy-normalized into another Object name. Root/directory permissions are +not inferred from catalog visibility or from a descendant pathname supplied by +the model. + +Saves retain Yoi's temporary-file publication rather than in-place truncation. +Thus saving one hard-link pathname replaces that name rather than changing all +aliases of the old inode. Checked preconditions detect known changes before +commit. Advisory locks cannot exclude arbitrary noncooperating external editors; +read-time/commit-time identity checks do not create a filesystem-wide transaction. +The supported race guarantees and remaining external namespace race window are +specified by the provider implementation and tests, not by a claim that a pinned +fd makes any rename-based update universally atomic. + +A guaranteed pre-effect rejection is distinct from **OutcomeUnknown** after +mutation/parent creation may have started, commit succeeds but state cannot be +returned, response validation/encoding fails, or a transport/deadline loses the +result. Never automatically retry uncertain mutations. Inspect current state and +rediscover/reread. Do not report partially created parents as unchanged. + +Host observations are bounded to depth 32 and 1024 nodes with a 30-second total +cooperative deadline. The checkout adapter permits at most 16 concurrent requests +and bounds admission waiting and the complete operation (including result-link +publication) to 30 seconds each; provider path/depth, source/response, +mutation/search and transport bounds apply too. No initial context expands an +entire tree. Resource-limit errors do not turn incomplete observation into a +successful truncated tree; bounded search results may explicitly report +truncation under the existing search contract. + +## Typical sequence + +1. Discover `/checkouts` with depth 1; inspect its Interface and call `list`. +2. Discover an entrance or known deep directory, inspect it, call `glob`/`grep`. +3. Follow a returned file `path`, discover/inspect, call line `read`. +4. Call `edit` or `write`; Client-managed validators protect the exact observed + object while the shared tracker enforces the prior Read. +5. Rediscover after external changes; a stale rejection is not permission to skip + Read. ReRead after successful save to verify text. +6. Inspect a directory and call `create_file` with a descendant relative path, + then discover/Read the returned new file. +7. Use `/workdir-attachments/` for detach. Old checkout + observations and history cannot operate a newly attached alias. + +Workspace configuration (T-695) remains a separate logical provider; it is not +replaced by raw Linux filesystem writes. Bash native conversion, broad POSIX +operations, lifecycle duplication, upstream changes and deployment/restarts are +outside this feature. + +## Reproducible validation matrix + +Run from the main repository, on Linux with `openat2` and `/proc/self/fd`: + +```sh +cargo check --workspace --all-targets +cargo test -p fs-operation -p workdir -p tools -p worker +cargo test -p worker-runtime -p yoi-workspace-server +cargo fmt --all -- --check +git diff --check +``` + +| Boundary | Focused regression locations | +| --- | --- | +| Deep mounts, multiple aliases, route-only arguments, exact permission identities, no duplicate compatibility file Tools | `worker/src/checkout_wip_tests.rs` and existing `wip.rs` registry/controller tests | +| Native search → numbered Read → unique/all Edit → Write → Create; returned validator cached chains | `checkout_wip_tests.rs`, `tools/src/checkout_tests.rs` | +| SubWorker write leases, sparse/nonrecursive/nested scopes, denied basename/content pruning, result-coordinate rebasing, revocation | `workdir/src/checkout_search_tests.rs`, `checkout.rs`, native scoped-child test | +| Approved External root replacement; same-content inode/name/parent substitution; hard-link save isolation; symlinks/special files; Unicode/non-UTF8 paths | `workdir/src/checkout.rs`, `fs-operation/src/checked.rs` | +| Mid-read updates; independent hash and validator fences; parent partial creation and post-save state failure | `checked.rs`, `checkout.rs`, Tools tests | +| Detach/alias reuse/restore references and read history; collection/search response generation fences | `checkout_wip_tests.rs`, `checkout_race_tests.rs`, existing router/scope/catalog lifecycle tests | +| Traversal depth/entry/cancellation bounds; stalled observation/link publication; numbered output expansion; post-dispatch encoding loss | `fs-operation/src/traversal_tests.rs`, `checkout_race_tests.rs`, `tools/src/read.rs`, `wip.rs` | +| Real prefixed HTTP provider → Remote session → router → native WIP; auth, read-only, lost/malformed post-commit response without retry | `worker/src/checkout_http_tests.rs` | +| Workspace proxy/External typed pairing and unknown-outcome propagation; Backend/Runtime Workspace, grant and generation isolation | checkout additions in `manage_workdir.rs` and Backend `server.rs`, plus full existing host/provider suites | +| Root/nested ignore parity, source precedence/negation/anchoring, directory pruning, denied/symlink rule confinement and bounded loading | `fs-operation/src/ignore_tests.rs`, `workdir/src/checkout_search_tests.rs`, native WIP ignore/link regression | +| Ordinary Tools text/hash/edit/write/search and provider symlink policy | full Tools, fs-operation and Workdir suites | + +The HTTP fixture tests actual network/codec boundaries but is not a production +Backend grant-network end-to-end deployment. Production routing/authorization is +covered separately by the existing host suites. Fault tests are deterministic +boundary simulations, not power-loss tests. Metadata/pinned-handle tests do not +close the external-editor final-check/rename window documented above. No live +Worker or deployed environment needs updating/restarting to execute this matrix. diff --git a/docs/design/durable-operations.md b/docs/design/durable-operations.md index 08616d3de..053d4b676 100644 --- a/docs/design/durable-operations.md +++ b/docs/design/durable-operations.md @@ -330,9 +330,30 @@ Exact dispatch retry reuses the deterministic attempt and Worker allocation key. Only explicit re-evaluation of a definitively failed attempt increments the bounded attempt number; unknown outcomes are not replayable. The runner uses the existing embedded Runtime spawn, Worker registry, Session input, -observation, usage, and lifecycle paths with no Workdir attachment. It uses the -dedicated `builtin:backend-job` profile, whose tools cannot mutate Workspace -state, recursively create Workers or Jobs, or inherit caller Workdir authority. +observation, usage, and lifecycle paths with no Workdir attachment. Each client +explicitly selects an existing registry Profile. The runner resolves the complete +Builtin/Workspace project catalog through the normal virtual config bundle and +prompt projection path; browser launch candidates remain a narrower UI policy. +Raw Profile source and filesystem/path selectors are not accepted. A selected +recipe's instruction and features are not replaced with `backend-job` defaults. +Profiles requiring ungranted Workspace mutation, ordinary Memory, child/peer, +Flow, Workdir or MCP authority fail explicitly; no caller scope is inherited. +The Ticket checker continues to select `builtin:backend-job` itself. + +Job identity/result capability comes from an immutable Host-authored create and +restore binding, not the Profile name, tag, instruction, Worker display name or +Runtime placement. Backend checks the exact live Job/attempt/input revision and +authenticated Worker before accepting results or domain operations. Profile +policy alone never confers a subject grant. + +A common optional resource key serializes Jobs using their existing durable +intent/attempt records (schema 81), not a second domain workflow table. Exact +request replay is distinct from a resource collision: the latter returns the +original active immutable request with `resource_reused`, and never claims that +new input was processed. Pending/unknown Jobs and **all** unfinished attempt +cleanups retain the fence. Retry requires definitive failure, remaining attempt +budget, completed prior cleanups and no competing resource owner. Unknown intent +remains attention-required even after its Worker is removed. Success is accepted only by the structured result boundary. Before Runtime input can execute, the attempt is fenced by Workspace, Job ID, attempt ID, input diff --git a/docs/design/feature-chat-invocations.md b/docs/design/feature-chat-invocations.md new file mode 100644 index 000000000..8bfe93554 --- /dev/null +++ b/docs/design/feature-chat-invocations.md @@ -0,0 +1,92 @@ +# Declarative Feature chat invocations + +Feature chat invocations are explicit structured user input. They let an enabled Feature contribute discoverable chat actions without teaching each client Feature-specific syntax or asking the LLM to infer an operation from text. + +## Namespaces and identity + +Composer sigils have separate meanings: + +- `:` is a client control command for the whole draft. +- `@` is a Worker-readable file reference. +- `/` selects a Feature invocation and may occur among ordinary prose. +- `#` has no invocation meaning. + +A public name such as `attach` is presentation, not authority. The payload carries the installed source-qualified `FeatureInvocationIdentity` (for example `builtin:attachments/attach`). Installation rejects duplicate public names and aliases, so clients never choose an arbitrary winner. Disabled, failed-to-install, or unavailable Features are absent from completion and cannot be resolved at execution. + +## Declarative grammar + +The common spelling is: + +```text +/name(positional, named=value) following natural language +``` + +The closing `)` is the explicit argument/prose boundary. String values use JSON double quoting and escaping (including Unicode escapes and escaped control characters). Bare values end at whitespace, comma, or `)`; comma separates arguments and a trailing comma is accepted. Integers are decimal signed 32-bit values (`-?[0-9]+`); booleans are `true` or `false`. Positional indices are contiguous; every argument can also be represented by its stable name in restored chip text. Mixed named/positional input advances the positional cursor only for positional values, and assigning the same argument twice is an error. + +A descriptor is declarative data, for example: + +```json +{ + "identity": "builtin:prepare/prepare", + "name": "prepare", + "aliases": [], + "display_name": "Prepare context", + "description": "Prepare scoped context before processing the request", + "syntax": "parenthesized", + "arguments": [ + {"name": "path", "position": 0, "required": true, "value_type": {"kind": "worker_file"}, "completion": {"kind": "worker_file"}}, + {"name": "mode", "required": false, "value_type": {"kind": "enum", "values": ["safe", "fast"]}, "completion": {"kind": "static", "values": ["safe", "fast"]}} + ] +} +``` + +`/prepare("資料/a b", mode=safe) explain the result` produces a typed invocation and a separate following Text segment only when explicitly confirmed. Missing required arguments, unknown or duplicate arguments, invalid enum/primitive values, and incomplete quotations remain validation errors, not partial business execution. + +Descriptors declare requiredness, value type, positional index, completion strategy, and optional description. Completion strategies are static values, Worker files, client files, or a host-owned provider. Metadata and completion are side-effect free. Arbitrary parser or provider code never crosses into a client. + +Clients may parse this spelling only for input assistance. A call becomes executable only after the user selects a descriptor and the client emits `Segment::FeatureInvoke`. Free text, URLs, paths, pasted content, attachments, assistant output, and tool output are not scanned or promoted by the Host. + +## Composer and completion lifecycle + +TUI and Web composers retain selected invocations as atomic typed chips containing identity and structured arguments. Backspace/Delete and range deletion remove an unsent chip; removal is not an undo for an already executed operation. Draft history, transport retry, queueing, Session history, snapshot/live projections, and rewind restoration preserve the typed segment rather than flattening and reparsing it. + +Completion requests carry a kind, fresh `request_id`, and source-qualified argument context. Host and Runtime echo the request ID unchanged. Clients correlate replies with that nonce plus their current draft revision, cursor, target Worker, authority snapshot, and connection generation; even identical-prefix/context ABA replies are discarded. Old uncorrelated replies cannot establish a selected invocation. Client-file arguments use a declared `InvocationClientAdapter`; they do not ask the Worker to resolve a client-local path. + +## Attachment adapter + +`/attach` is the built-in real use of the generic contract. Its descriptor declares a required client-file argument and the attachment client adapter. + +- TUI may select a path visible to the TUI process, stage an upload, and replace the invocation intent with an `UploadedFile` chip. +- Web opens the existing browser picker; it does not enumerate the browser host filesystem. +- The local path is never sent to or resolved by the Worker. +- Uploading is cancellable draft staging, not invocation execution. Removing an in-progress upload chip by Backspace/Delete, range replacement, or draft abandonment cancels its reservation and adapter upload; the removed upload must no longer block a text-only Submit. Late callbacks and Undo cannot restore that cancelled reservation as sendable input. A completed unsent uploaded-file chip may retain its staged resource while editor Undo can restore it, releasing abandoned resources at the draft boundary; a late completion must never turn a deleted pending reservation into such an Undo-owned resource. +- Authoritative `SubmissionAccepted` (not socket-send success) transfers uploaded references to the durable input lifecycle. Draft removal cannot delete an in-flight reference whose admission may already have succeeded, or an accepted attachment. Rejection restores the exact typed intent for retry; retry retains request/invocation IDs. Upload completion alone is not submission acceptance. +- TUI standalone currently has no client-local upload staging transport. It reports the unsupported adapter and preserves the draft; it does not reinterpret a local path as a Worker host path. + +There is no `/clear-attachments` command. Normal composer editing is the removal mechanism. + +## Installation and execution + +A Feature declares invocation descriptors using `FeatureDescriptor::with_chat_invocation` and registers matching handlers with `context.chat_invocations().register` (or `register_with_completion`) during `FeatureModule::install`. Registration must match the declared descriptor and the installing Feature's source-qualified identity. The invocation registry is checkpointed and rolled back with other Feature contributions. Only successfully installed, currently available descriptors are exposed through completion. + +Authority-sensitive handlers override the side-effect-free `is_available()` discovery hook and `validate()` semantic hook. They must still revalidate authorization and use scoped idempotency at their business commit boundary inside `invoke()`; defaults grant no extra domain authority. Availability is checked again after asynchronous provider/handler work. Reusing a successful preparation result or resuming its dependent model run also rechecks the side-effect-free semantic/argument-specific authority validation; general Feature availability alone is insufficient. Completion strategies come from installed metadata, never from a client-selected provider identifier. Enum and boolean candidates can derive directly from value types when no explicit completion strategy is supplied. + +Completion output is bounded to 64 entries / 64 KiB encoded data, with per-field byte limits. Oversized candidate values are discarded instead of altered. Handler results allow 4 KiB messages and 16 KiB context; oversized results/errors after execution become `OutcomeUnknown`, withholding dependent model context rather than assuming the operation was unexecuted. + +The Worker validates every structured payload again against the installed descriptor. It executes invocations in segment order only when the durable submission is activated, after committing the user input and before the dependent LLM request. A completion choice, draft edit, queued-but-cancelled submission, reconnect, history view, or restored snapshot never executes a handler. + +A submission may contain at most eight invocations with distinct invocation IDs. The first failed or outcome-unknown result fences later invocations **and prevents the dependent LLM run**; no cross-invocation rollback is promised. Every result is committed as typed Session history and rendered as bounded model-visible context. `OutcomeUnknown` is distinct from failure and is never automatically retried. + +Before calling a handler, the Worker appends a start receipt through the existing `worker.pending_activations.v1` extension checkpoint. The result and updated receipt are appended together with the annotated system item. An unresolved start (including handler completion followed by a failed result append) remains a fence across same-process retry, restart, Pause/Resume, compaction, and rewind. Reusing an ID with changed arguments is rejected; exact replay can reuse its recorded result but does not invoke again. Receipt capacity is bounded and refuses untracked execution rather than evicting recovery evidence. Restoring or displaying history never reexecutes a handler. + +Rewind preserves invocation-only receipts and the recovery-blocked fence independently of submission/notification queues. The retained history prefix and recovery checkpoint replace the segment together through the existing atomic Store replacement contract. There is no truncate-then-checkpoint repair window in which a crash can erase a completed or ambiguous business operation's recovery evidence. + +The recovery receipt retains the invocation ID, qualified identity, SHA256 typed-payload digest, and bounded result—not another copy of every argument. Invocation IDs are nonempty and at most 128 UTF-8 bytes; clients generate fresh UUIDs on explicit selection and preserve them through transport retry. The session retains at most 1,024 recovery receipts without eviction. Corrupt, duplicate, inconsistent, or oversized recovery evidence fails closed. + +`submission_request_id` protects submission admission and retry, while `invocation_id` is the stable idempotency key passed to a side-effecting handler. A handler must scope that identity at its domain boundary. This design does not claim a new generic exactly-once substrate. + +## Authority boundaries + +Possession of syntax, metadata, identity, or a typed payload grants no permission. Runtime target, enabled Feature, argument schema, and captured domain authority are checked at execution. Completion providers cannot perform business mutations or expose credentials. Handlers may share an underlying domain operation with Worker tools, but chat discovery does not expose every tool automatically. + +Text-only clients submit text. The Backend and Host never reinterpret text as an invocation. Unknown typed input is visibly unresolved in restoration and **rejected on submission**, not silently flattened or processed by the LLM. Notify remains text-only: extra typed wire fields or typed Runtime input cannot be discarded into an advisory message or executed as a side-effecting call. diff --git a/docs/design/merge-request-wip-projection.md b/docs/design/merge-request-wip-projection.md index 3879b8054..4d314a7a2 100644 --- a/docs/design/merge-request-wip-projection.md +++ b/docs/design/merge-request-wip-projection.md @@ -4,13 +4,15 @@ When a Worker explicitly runs in `wip` mode and its Merge Request Feature has at least one enabled operation, the Worker Host projects that exact operation set as native Worldspace objects. The projection is an adapter over the existing scoped Workspace Merge Request tools and Backend routes. It does not create a second Merge Request store, provider integration path, review grant, approval rule, or completion authority. -The Host allocates `/features/merge-request` and the Feature mounts: +The Host assigns the Merge Request provider the root `merge-requests` namespace through `WipMountRegistry` and mounts: ```text -/features/merge-request/merge-requests -/features/merge-request/merge-requests/ +/merge-requests +/merge-requests/ ``` +There is no Feature Object or `/features/merge-request` compatibility alias. Legacy paths remain historical display data only and fail as current input with rediscovery guidance. + An item identity is a single canonical path segment. Nested paths and percent-encoded target substitution do not resolve. Item operations remove `merge_request_id` from their descriptor and reconstruct it only from the selected object route. Discovery does not mint authority: every call still uses the Worker's scoped `WorkspaceClient`, the enabled Feature flags, ordinary Tool permission rules, and the Backend's Workspace, repository, assignment, provider-ref, review, and integration checks. ## Current Feature inventory diff --git a/docs/design/objective-wip-projection.md b/docs/design/objective-wip-projection.md index a9316e5cc..20c0fe6e3 100644 --- a/docs/design/objective-wip-projection.md +++ b/docs/design/objective-wip-projection.md @@ -4,13 +4,15 @@ When a Worker explicitly runs in `wip` mode and the Objective Feature is enabled, the Worker Host projects Objectives as native Worldspace objects. The projection is an adapter over the existing scoped Workspace API. It does not introduce another Objective store, state machine, reference resolver, or audit path. -The Host allocates `/features/objective` to the Feature through `WipMountRegistry`. The Objective Feature then mounts: +The Host assigns the Objective provider the root `objectives` namespace through `WipMountRegistry` and mounts: ```text -/features/objective/objectives -/features/objective/objectives/ +/objectives +/objectives/ ``` +There is no Feature Object or `/features/objective` compatibility alias. Legacy paths are retained only as historical text and fail as current inputs with rediscovery guidance. + `` is either a canonical `O-` key or an existing canonical internal alphanumeric ID. Only one direct child segment is accepted. Nested suffixes, Ticket keys, and unrelated or hidden prefixes do not resolve as Objective objects. The Backend remains authoritative for existence, `O-*`/internal-ID resolution, Workspace scope, validation, audit events, revision generation, pagination, body/event bounds, and linked-Ticket validation. A discovered route is not independent authority; every operation still uses the Worker's current `WorkspaceClient`. diff --git a/docs/design/subjektiv-consolidation-jobs.md b/docs/design/subjektiv-consolidation-jobs.md new file mode 100644 index 000000000..6121fd9fe --- /dev/null +++ b/docs/design/subjektiv-consolidation-jobs.md @@ -0,0 +1,103 @@ +# Backend subjektiv consolidation Jobs (T-704) + +subjektiv requests the common Backend Job runner with an explicit +`builtin:subjektiv-memory-consolidation` Profile. The runner has no purpose-based +Profile switch. Subject-body Workers and the independent Workspace Memory feature +are not Jobs and are not migrated or reset here. The shared/standalone adapters +planned in T-708/T-709 are outside this Backend change. + +## Input, grants and concurrency + +The existing backlog/byte thresholds and missing/stale/failed surface policy +remain the launch decision. A Job captures an immutable bounded batch (up to 100 +candidate IDs) plus the Subject store revision. Selection also reserves the Job's +16 KiB result budget, accounting for JSON-escaped IDs appearing in both the batch +claim and compact verified outcomes. Excess candidates remain staged for a later +batch. An empty batch is valid for a +surface-only rebuild. The Backend explicitly binds the Subject and candidate +batch grant in durable Job intent before Worker execution. No subject-body +singleton ownership or Session attribution is transferred to the Job. + +Authenticated `(Workspace, Runtime, Worker)` registry binding and the exact +current dispatched attempt authorize each operation. Names, Profile tags and +model-provided `subject_id` are not authority. Candidate listing exposes only +unresolved IDs in the captured batch; reads/decisions outside it are denied. +Surface prepare requires all batch candidates to have durable dispositions. +Surface generation records preserve immutable Job/attempt provenance (subjektiv +schema migration 6); publication, failure and completion reject another attempt's +generation, including pre-Job/legacy generations. + +The common resource key `subjektiv-consolidation:` prevents parallel +Jobs for a Subject. Concurrent requests return the original active intent and +report `skipped_existing_job`; they do not expand its grants. New candidates and +items beyond the bounded batch stay in staging. A subsequent consolidation +request snapshots remaining work; it is not counted as processed by the first +Job. There is no domain-specific status ledger, pool or persistent idle Worker. +Definitive failures use the common bounded retry API with the original intent; +unknown execution stays fenced for attention. A later explicitly requested Job +may snapshot remaining candidates once all previous Workers are safely cleaned. + +## Memory and surface completion + +`MemoryApplyCandidate` still uses the existing expected-revision checks, atomic +apply/close and immutable decision receipts. Partial application survives model +failure, timeout, lost response, retry, Worker deletion and Backend recovery. +Job replay is not a second memory apply: resolved candidates retain their +existing dispositions, and exact candidate decisions converge on their receipts. + +`SubmitBackendJobResult` for a bound consolidation Job receives only the Subject +and captured candidate IDs. Its Host capability awaits the existing bounded +clean-context surface editor and publish-or-recorded-failure, then adds the actual +structured surface outcome. It does **not** schedule a Worker-owned post-Run task. +The result call serializes and caches the exact generated outcome for ambiguous +submission replay. The Backend verifies every batch disposition and the current +store revision, generation ownership and persisted ready/failed surface fields; +it enriches the durable result with actual immutable candidate disposition +summaries (`candidate_id` and `action`). Full reasons, affected revisions and +source edges remain in the existing immutable receipt store; they are not copied +into a bounded Job result. Exact replay uses the same verified compact projection. +Thus confirmed-memory consolidation and surface readiness are separate outcomes. +A recorded surface failure does not undo Memory. A stale/unfinished/fabricated +surface cannot be reported completed. Final prose or Idle is never success. + +Only after durable structured result acceptance does the existing terminal Job +cleanup run. Its tracked Run teardown fence lets the result tool finish before +stop/remove. Timeout/unknown paths cancel unfinished Worker execution through +that same cleanup; later operations/results are rejected. The next request can +regenerate failed/stale surfaces without touching confirmed memory. + +## Legacy cutover and recovery + +The subject-specific direct spawn and Idle resubmission paths are removed. +On a consolidation request the Backend checks the **persisted singleton owner** +for the old `subjektiv-consolidation:` key. It revalidates Workspace, +key and old dedicated recipe provenance, not the display name. A running/paused +legacy foreground operation is left alone and returns `skipped_legacy_inflight`; +no Job is launched alongside it. Legacy owned operations can finish during this +bounded cutover. + +After foreground Idle (or Stopped) is confirmed, Backend stop is confirmed before +canonical WorkerRemovalService removal. Any legacy surface background task is +cancelled at stop; the subsequent Job rebuilds from confirmed Memory and does not +infer successful surface completion from Idle. Busy/unobservable/mismatched +owners fail closed. Empty/ready Subjects also retire their idle legacy owner. +Removal failures remain diagnosable through existing durable removal recovery; +a subsequent request rechecks ownership instead of deleting storage directly. +Subject-body/ordinary/Workspace-Memory Workers are never selected by this key. + +The Job runner uses its existing durable dispatch allocation, tracked input, +T-701 registry-bound restore/recovery, deadlines, unknown outcomes, result replay, +delivery and T-690 cleanup. Job results, candidate receipts, Memory revisions, +provenance and surfaces remain in their existing Backend stores after Worker +removal. No live dogfood Worker or environment is stopped/updated by development +validation. + +## Validation boundary + +Server recording-backend fixtures prove reservation/binding/auth/result/cleanup +contracts, and existing store tests prove candidate CAS/atomic receipts and +surface freshness. Worker scripted-client tests prove the clean-context editor +completion-before-result boundary and result replay. They are not real-provider +LLM or live dogfood process evidence. See the T-704 validation report for exact +commands/results and inherited intermittent failures; later passes do not erase +those failures. diff --git a/docs/design/subjektiv-store.md b/docs/design/subjektiv-store.md index f9b1c2a00..fdce5859d 100644 --- a/docs/design/subjektiv-store.md +++ b/docs/design/subjektiv-store.md @@ -289,11 +289,15 @@ active/resolved→retracted, and resolved→active. Retraction remains terminal. ## Candidate consolidation and corrections (T-670) -The Backend starts a restricted `builtin:subjektiv-memory-consolidation` Worker -for exactly one authenticated subject. The Host binds the subject; model-visible -inputs cannot select a Workspace, subject, Runtime, Worker, or Session. Its tools -list/read pending immutable candidates, query/read confirmed revisions, and make -one candidate decision. Candidate text is evidence rather than authority. +subjektiv requests a bounded Backend-owned Job with the explicit +`builtin:subjektiv-memory-consolidation` Profile and an immutable Subject/candidate +batch grant. The common runner resolves the recipe; Profile names do not grant +permissions. The Host derives authority from the authenticated current +Job/attempt/Worker binding; model-visible inputs cannot select a Workspace, +subject, Runtime, Worker, or Session. Its tools list/read only pending candidates +in that batch, query/read confirmed revisions, and make one candidate decision. +Candidate text is evidence rather than authority. No subject-specific idle Worker +is resubmitted; see [Job execution and cutover](subjektiv-consolidation-jobs.md). Every decision carries a stable request ID. An exact retry returns the committed receipt, while reuse with different input is a conflict. Applying a candidate diff --git a/docs/design/ticket-wip-projection.md b/docs/design/ticket-wip-projection.md index 34f021108..6a76194c2 100644 --- a/docs/design/ticket-wip-projection.md +++ b/docs/design/ticket-wip-projection.md @@ -4,13 +4,15 @@ When a Worker explicitly runs in `wip` mode and its Ticket Feature is enabled, the Worker Host projects the enabled Ticket surface as native Worldspace objects. The projection is an adapter over the existing scoped Workspace Ticket API and typed Ticket tools. It does not add a Ticket store, resolver, audit path, state machine, readiness rule, queue path, or relation implementation. -The Host allocates `/features/ticket` and the Feature mounts: +The Host assigns the Ticket provider the root `tickets` namespace through `WipMountRegistry` and mounts: ```text -/features/ticket/tickets -/features/ticket/tickets/ +/tickets +/tickets/ ``` +There is no Feature Object or `/features/ticket` compatibility alias. A legacy path supplied as current input fails with rediscovery guidance; saved history containing that text is left unchanged. + `` is either a canonical `T-` key or an existing canonical internal alphanumeric ID. Only a direct child is accepted. Nested suffixes, Objective keys, unrelated prefixes, and hidden routes do not resolve. Every call retains the Worker's scoped `WorkspaceClient`, current Ticket Feature access, and ordinary Tool permission policy. Discovery is not mutation authority. A direct call to an unpublished operation fails even when the caller knows its name. `deny` and interactive `ask` permission outcomes fail closed before Backend dispatch. diff --git a/docs/design/workdir-wip-projection.md b/docs/design/workdir-wip-projection.md new file mode 100644 index 000000000..bace39405 --- /dev/null +++ b/docs/design/workdir-wip-projection.md @@ -0,0 +1,186 @@ +# Repository, Workdir and attachment WIP projection + +## Objects and operations + +The Host owns these root namespaces, using the same `WipMountRegistry` +namespace/resolver/contribution contracts as Tickets, Objectives and Merge Requests: + +```text +/ +├── repositories list +│ └── read +├── workdirs list, create +│ └── read, attach, delete (when supported) +└── workdir-attachments list + └── read, detach +``` + +Operations are Interface declarations, **not child files**. There is no +`/features/manage-workdir`, no writable Repository catalog and no Repository +content tree here. `/checkouts` content projection is a separate provider (T-699), +as is Workspace configuration (T-695). Management Objects do not grant access to +unattached contents. A future content provider should return its entrance from +an authorized, live attachment; it must not infer content/command capability +from catalog or lifecycle access. + +`path` in inventory results is an absolute WIP Object path. It is a reference, +not an OS path, a capability grant, or an instruction to reinterpret another +string argument as a route. Source Repository references use `repository_path`; +attachment references use `workdir_path`. Attach results include +`attachments_path` to discover the current connection and its lifetime ID. + +### Identity and route encoding + +Repository identity is the formal Backend `repository_key`, never its display +name or a guessed `main`. Workdir identity is `working_directory_id`, never a +Worker-local alias. A segment containing only ASCII letters, digits, `-`, `_` +is unchanged. All other UTF-8 identities are encoded as `~` followed by lowercase +hexadecimal UTF-8 bytes. The `~` marker is itself always encoded. Only the +canonical encoding is accepted; no alternate spellings, recursive URL decoding, +case folding, slash traversal or namespace fallback are accepted. For example, +`repo/name` is `/repositories/~7265706f2f6e616d65`. This is injective and accepts +existing valid keys, including Unicode and slash-containing identities. + +Attachment identity is a fresh opaque Backend connection ID for each lifetime, +including detach/reattach to the **same** alias and Workdir. Read exposes alias, +Workdir ID and effective attachment capabilities as attributes. Workdir existence +and Worker connection are separate Objects. Detaching a connection does not +remove the Workdir or delete a logical settings space. + +## Feature and authority separation + +`feature.workdir_catalog.enabled` independently supplies read-only Repository, +Workdir and attachment reference operations. It defaults enabled so a WIP Worker +with Workspace authority and appropriate permissions can find registered keys +without Ticket or conversation hints. It does not add ordinary Tools or require +a filesystem/Bash location. Standalone Workers without Workspace authority have +no Workspace catalog mounted. + +`feature.manage_workdir.enabled` supplies management operation implementations +on the **same Objects** using `contribute_operations` and +`contribute_dynamic_operations`. It remains default disabled. It neither owns +another resolver nor confers permission. The object provider is unique; duplicate +namespace, route, operation name or inconsistent Interface contributions fail at +registration through the common registry. + +Native permission identities are: + +| Operation | Manifest permission identity | Bound identity in permission input | +| --- | --- | --- | +| repositories.list | RepositoryList | list arguments | +| repository.read | RepositoryRead | repository_key | +| workdirs.list | WorkdirList | list arguments | +| workdir.read | WorkdirRead | working_directory_id | +| workdirs.create | WorkdirCreate | original create arguments | +| workdir.attach | WorkdirAttach | working_directory_id + alias | +| workdir.delete | WorkdirDelete | working_directory_id + reason | +| attachments.list | WorkdirAttachmentList | list arguments | +| attachment.read | WorkdirAttachmentRead | connection_id | +| attachment.detach | WorkdirDetach | current bound alias | + +Read permissions gate direct item resolution as well as list item visibility. +The manifest's existing default policy is retained; `ask` is denied fail-closed. +An argument-dependent policy may advertise an operation that could be allowed, +but every call checks its exact reconstructed input before execution. Mutation +arguments cannot override path-bound Workdir or connection identity. Having +RepositoryRead is not WorkdirCreate; inventory/management permissions are not +file access, delegation or command execution grants. Backend caller/Workspace +identity, source proof, attachment capability ceilings and lifecycle policy are +checked by the existing paths on every request. + +## Contextual Interfaces and stale observations + +Providers opt into common Host contextual Interfaces. Operation publication is +the intersection of registered implementation, current target support and current +subject policy. The collection reference feature can remain available while the +management feature is disabled. External Workdirs without cleanup authority do +not expose delete. Only currently supported lifecycle operations are advertised; +no delete/command is added to logical attachments solely because they appear as +workspaces. + +A contextual Interface reference is opaque and path-qualified as +`/@/`. Clients must use the returned reference, not +construct it. Fetch and call freshly resolve the same Object and filtered +Interface; base references cannot fetch an unfiltered descriptor. Descriptor and +Object validators invalidate old observations after target/connection/permission +changes. Validators are managed by the WIP Client, not manually by the LLM. +Cached Interface possession and known direct paths never bypass current checks. +Historic session text remains append-only and is not rewritten or replayed. + +## Backend lifecycle and safe output + +Catalogs are request-time projections of existing Backend authority, not another +store or synchronization job. WIP list pages default to 50 and accept 1–100 +items. Repository pages sort the existing complete Repository catalog by stable +key. Workdir pages use the signed caller-bound Backend +`workers/self/workdir-catalog` operation, with SQL keyset paging over the existing +Workdir registry in stable ID order. This is not the legacy bounded +`working-directories` browser/Tool snapshot: Workdirs older than its 200-record +cap remain reachable. `next_cursor` advances over scanned rows even when the +current read policy or unsupported legacy source makes a page sparse or empty; +`empty` describes that page, and `has_more` remains authoritative for progress. +Attachment pages use the Backend's SQL-bounded, alias-ordered limit/offset query +and opaque `offset:` cursors. Both Workdir and attachment responses supply an +opaque complete-set `revision` derived from existing authoritative records in +the same database snapshot as the page, not by hashing the first page. The Host +uses this complete-set revision for the collection Object validator. A lifetime +or capability change beyond row 50 therefore invalidates old collection +observations without adding a ledger or revision store. A cursor is not +a snapshot or authority grant: restart listing after concurrent collection +changes. Filtering by exact `connection_id` is applied in the caller-scoped +Backend ledger before paging, so direct item lookup also works beyond the first +page. Unknown/expired IDs yield no item, never a current alias substitute. +Repository projection uses an explicit field +allowlist: formal key, kind/provider, any existing display metadata and explicit +nullable default selector. Source URI/path, credentials, remotes, Runtime +connection information and internal fingerprints are not model-visible. +Workdir projection includes stable ID, optional display name, public source kind +and Repository key, existing selector/ref/tree, status, cleanliness and occupancy +when available. Cleanup handles and External grant internals are not forwarded. +No missing attributes or implicit selectors are fabricated. Backend errors are +sanitized rather than forwarding raw diagnostic bodies with possible host paths. + +Create and attach reuse the ordinary Workdir backend, request identity, +Worker-session router, mutation lock and Internal SubWorker lifecycle hooks. +Creation does **not** attach. Attach installs the Backend-attenuated session +capabilities, never a fabricated `ALL` capability for External Workdirs. Detach +uses the same begin/finish/cancel-detach path and command/operation exclusion as +Tools; the Backend checks expected connection identity under its existing +current-Worker lifecycle lock before closing sessions, then conditionally +updates that exact row in the existing connection ledger. Alias reuse cannot +retarget an old connection operation. Compensation retains the original +connection identity. Existing removal dispositions `removed`, `retained`, and +`attention_required`, plus retryability, remain intact. + +Stale identity/validator rejection is distinct from unknown operation outcome. +Transport failure, invalid post-dispatch response or uncertain lifecycle failure +is **not** proof that no effect occurred; it must not be automatically retried. +Inspect current state or request operator guidance. No new retry or attachment +ledger is introduced by WIP. + +When native management is mounted, `WorkdirList/Create/Attach/Detach/Delete` are +claimed as represented native capabilities, so `/tools/Workdir*` is not also +published. Ordinary Tools mode keeps its names and inputs. Existing native +Ticket/Objective/Merge Request routes retain their T-698 contracts. + +## First-Workspace example + +1. `WipDiscover(path="/", depth=1)` finds `repositories`, `workdirs` and + `workdir-attachments` when permitted. +2. Inspect the returned Repository collection Interface and call `list` with a + bounded limit. An empty registered/visible catalog explicitly returns an empty + list; do not invent a key. Follow `next_cursor` when supplied. +3. Discover the returned item `path`, inspect its Interface and call `read`. + `default_selector: null` means no configured default, not `main`. +4. Inspect the Workdir collection and `create` using the discovered formal key, + with optional selector, display name and Runtime under the existing contract. +5. Follow created `item.path`, discover/inspect and `attach` with a chosen + Worker-local alias. This is separate from creation. +6. Follow `attachments_path`, `list`, then discover/read the returned connection + path to confirm alias, Workdir and effective capabilities. +7. Call that connection's `detach` with no identity override. The observed + lifetime ID guards alias reuse; the Workdir remains in its inventory. + +Each step uses current WIP observations and no Ticket-derived key, host path, +Runtime endpoint or manual validator. Explicitly refresh/reset discovery after +external changes and before reconstructing a rejected stale call. diff --git a/docs/design/worker-wip-mode.md b/docs/design/worker-wip-mode.md index 012a5d768..f77be0daf 100644 --- a/docs/design/worker-wip-mode.md +++ b/docs/design/worker-wip-mode.md @@ -52,21 +52,47 @@ The interface descriptor uses WIP `Json` for compatibility input and includes th ## Native projection extension and collisions -`worker::wip::WipMountRegistry` is the Host-owned mount allocator. A Feature can independently construct a `WipProjection` with an async `WipOperationHandler`, then pass its aggregate registry through `install_wip_mode_with_mounts` before compatibility finalization. A Feature first requests a Host-owned `/features/` namespace with `allocate_feature_route`; checked direct-child joins prevent a Feature from claiming an ambient global route. +`worker::wip::WipMountRegistry` is the Host-owned namespace, Object, and operation registry. A provider requests a root namespace with `allocate_namespace(owner, namespace)`; the owner is retained as Host metadata and is not inserted into the public path. It then mounts the collection Object directly at `/` and may install a bounded direct-child resolver. Features do not appear as intermediate Objects. -- Routes must be canonical absolute WIP paths. +Object resolution and operations are separate contracts. One mounted `WipProjection` owns a static Object route and its initial operations. Another Feature may add operations with `contribute_operations` without replacing that Object. Dynamic families likewise keep one `WipDynamicItemResolver`, while `contribute_dynamic_operations` supplies disjoint handlers for every resolved item without a second resolver. The Host merges only contributions with the same interface format, documentation, and type declarations; duplicate operation names, inconsistent interfaces/validators, missing targets, duplicate namespace owners, and unrelated route claims fail deterministically at registration. The merged interface validator changes with the effective operation set, so stale Known Space cannot authorize a newly contributed operation. + +- Routes must be canonical absolute WIP paths, and allocated namespaces are one lowercase root segment. - Object names must match the final route segment. -- A projected object exposes exactly its declared interface. -- Unrelated duplicate routes and conflicting interface descriptors or validators fail startup. +- Object existence and resolution have one owner; operation contributors cannot remount or substitute the Object. +- Multiple Features may contribute disjoint operations to the same Object interface. +- Unrelated duplicate routes, duplicate operation names, and conflicting interface shapes fail startup. - A bounded dynamic mount may resolve exactly one item segment below an already mounted native collection; deeper paths never fall through to the resolver. - A native projection replaces a compatibility projection either by the original same-route semantic capability or by an explicit registry capability claim owned by a mounted native route. - Conflicting native claims fail startup. When native is selected, the claimed compatibility tool is hidden rather than exposed through a second entry. Normal Tool mode is unaffected because it does not install the WIP registry. -`WipInspect` renders the complete descriptor, including descriptor-local named declarations and recursive record, list, enum, and union shapes with required flags and declaration documentation. The unit samples `mount_collision_and_native_replacement_are_explicit` and `native_projection_preserves_descriptor_and_decodes_typed_arguments_end_to_end` demonstrate independent native registration, deterministic replacement, collision rejection, complete descriptor inspection, and a descriptor-typed native invocation. +`WipInspect` renders the complete effective descriptor, including contributed operations, descriptor-local named declarations, and recursive record, list, enum, and union shapes with required flags and declaration documentation. The tests `host_namespace_object_resolution_and_operation_contributions_are_independent`, `operation_contributions_reject_interface_and_target_conflicts`, and `native_projection_preserves_descriptor_and_decodes_typed_arguments_end_to_end` demonstrate root allocation, unique resolution, contribution merging, collision rejection, current call-time authorization, complete descriptor inspection, and descriptor-typed invocation. + +The built-in native route migration is: + +| Historical route | Canonical route | +| --- | --- | +| `/features/ticket/tickets[/]` | `/tickets[/]` | +| `/features/objective/objectives[/]` | `/objectives[/]` | +| `/features/merge-request/merge-requests[/]` | `/merge-requests[/]` | + +The built-in Repository/Workdir reference provider allocates `/repositories`, `/workdirs`, and `/workdir-attachments` using this contract; `manage-workdir` contributes operations rather than owning another Object. See [Repository, Workdir and attachment WIP projection](workdir-wip-projection.md) for identity encoding, reference/management permissions, attachment lifetime safety, and first-Workspace discovery. `/checkouts` content projection remains a separate provider. + +### Registering a subsequent provider + +1. Allocate `allocate_namespace("repository", "repositories")`. The returned `WipNamespaceRoute::root()` is `/repositories`; the owner is independent of any Feature registration name. +2. Mount a native collection `WipProjection` at that root with capability `repository:collection`, matching Object name and one declared interface. Static Objects underneath the root use the same capability owner prefix. A mounted collection retains its own identity, interfaces, and validator even when it has static children. +3. For dynamic items, register one `WipDynamicMount` on the collection with capability `repository:item`. A foreign capability owner is rejected. Its `WipDynamicItemResolver` owns reference acceptance, Object identity, and current Object validator. It returns only a bounded direct child; it must not reinterpret a reference as another namespace or grant permission from existence. +4. The reference provider supplies its read operations and handlers independently of management Feature enablement. An Object-only provider may use an empty operation descriptor. Enabled management Features register `WipOperationContribution` for static Objects or `WipDynamicOperationContribution` for a dynamic family; they do not allocate the namespace again or replace its resolver. +5. Contributions use the existing interface reference and identical format, documentation, and type declarations, supplying only their new operations. Interface references are global: extending a descriptor that is also mounted elsewhere must not create different descriptors/validators under the same reference (registration fails). Use separate references for collection and item contracts. Failed contributions leave the registry unchanged. +6. Handlers are responsible for checking current target capability and current subject authorization before dispatch. Registration/discovery is metadata, not a grant. A resolver/handler must retain the original permission identity and domain audit/failure contracts. Revocation after discovery must fail before provider dispatch. Compatibility mounts are restricted to `/tools/` and cannot bypass namespace ownership or republish legacy routes. + +Tests `management_enablement_is_independent_of_read_and_operation_permissions` and `contributed_interface_invalidates_observations_and_restore_requires_discovery` cover all eight enablement/read/manage combinations, direct-call rejection, effective descriptor changes, fresh runtime isolation under the same Worker identity, reset, and post-observation revocation. ## Authority and execution -Discovery is not authority. The Worldspace contains only the tools already registered from enabled Features and the Worker's current execution context. Calling a compatibility operation: +Discovery is not authority. Object publication, operation implementation, target capability, and subject permission are independent checks. An Object may remain discoverable through an enabled read provider when a management Feature contributes no operations; conversely, publishing an operation never bypasses the target's current capability or the subject's current permission. Every handler re-evaluates permission at call time, so revocation after discovery is rejected before provider dispatch. Feature enablement selects implementations and does not itself grant read or mutation authority. + +The Worldspace contains only the operations registered from enabled Features and the Worker's current execution context. Calling a compatibility operation: 1. resolves the exact mounted route; 2. verifies object and interface membership and validators; @@ -85,6 +111,8 @@ The stateful `wip-client::Client` is owned by one Worker WIP runtime. Sessions a Known Space and interface observations are therefore not shared across Workers, endpoints, or security contexts. A restored Worker creates a new runtime/client and must explore again. `WipDiscover(reset = true)` drops all observations before reconnect/authority-change exploration; `refresh = true` explicitly supersedes one cached observation. A normal compaction keeps the live runtime but the three gateway schemas are supplied again on every LLM request, including the instruction to begin discovery at `/`; the compacted transcript is not treated as cache authority. +Saved Session/history entries remain append-only evidence and are never rewritten or replayed as calls. In particular, historical `/features//...` text stays displayable but is not a current route alias. Passing such a legacy path to discovery or call returns `NotFound` with guidance to rediscover from `/`; the Host does not translate it, resolve it under another namespace, or expose old and new routes in parallel. + ## Errors, cancellation, and retries The adapter keeps these outcomes distinct: @@ -96,6 +124,8 @@ The adapter keeps these outcomes distinct: - **disconnected/timeout after dispatch**: `wip-client` records an unknown outcome; - **operation outcome unknown**: an executing provider fails without a rollback/non-execution guarantee, a response cannot be completed after dispatch, or the call future is force-closed. +Cooperative cancellation is scoped to the active execution's selected operation and exact resolved handler instance. Static/dynamic dispatchers forward `cancel_operation` only to that operation's provider, once per cancellation request, never to unrelated contributors or every alias backed by a shared handler. The runtime pins the dispatcher used for execution rather than resolving the Object again on cancellation, since dynamic resolvers may return stateful instances. Providers may implement `cancel_operation` for operation-specific control or the shared `cancel` default. Terminal completion or a dropped execution future releases the active binding; unknown/finished execution IDs do not invoke any provider. A cancellation request is not itself terminal confirmation. + An unknown mutating outcome must never be retried automatically. The model must inspect external state or ask for operator guidance. Retrieval failures may be explicitly refreshed. ## Measurement diff --git a/docs/design/workspace-config-wip-projection.md b/docs/design/workspace-config-wip-projection.md new file mode 100644 index 000000000..a8dfc977c --- /dev/null +++ b/docs/design/workspace-config-wip-projection.md @@ -0,0 +1,246 @@ +# Workspace configuration as a WIP logical filesystem + +## Authority and source mapping + +Workspace configuration is not the Backend host's configuration directory. The +canonical authored source is already a SQLite-backed, revisioned virtual tree: +`workspace_config_trees`, `workspace_config_entries`, and append-only +`workspace_config_tree_revisions`. `WorkspaceConfigState` combines its +`ConfigTreeSnapshot`, toolchain/schema contract and projection digest. The +`main.dcdl` entrypoint and its imports are authored files; evaluated profiles, +prompts and other projections are not replacement source files. + +The builtin Workspace schema composes profile, prompts, runtime, +repository-access and Skills settings. Profile sources include model settings. +Do not assume that all imported Profile sources live under `profiles/`: +`VirtualPath` identifies authored entries, including shared imports. Skill +Markdown and text resources are entries in the same virtual tree. + +The WIP provider must resolve this authority at request time, not copy it to an +OS directory, maintain a synchronized config cache/store, or register every +entry as a permanent Object. A logical attachment selects a Workspace-bound +configuration subject; it is not a Bash cwd or a host filesystem grant. + +## Existing save and activation contract + +The browser source-tree editor uses `ConfigCommitRequest`: a base revision and +tree digest, changes with expected entry digests, and the `main.dcdl` +entrypoint. Candidate evaluation applies changes, formats/normalizes Decodal, +and evaluates the composed schema. Commit rechecks revision **and** digest in +an SQLite Immediate transaction before replacing active entries and appending +the revision manifest. Worker operations must share that conflict domain with +UI and other Worker edits; a per-Worker mutex alone would not prevent lost +updates. + +The Backend commit orchestration validates the prompt projection and +repository-access references, then commits and best-effort publishes the prompt +projection to the embedded Runtime. Saving establishes new authoritative +Workspace configuration, **not universal live application to existing Workers**. +New Worker configuration bundles are revision-bound. In-flight consumers can +retain immutable projections. Skill availability and activation are separate +contracts. A transport failure after dispatch is not proof of rollback and is +not authorization to retry a mutation automatically. + +## Separate authorities and excluded objects + +The following are not files of the authored Workspace source tree: + +- Backend host `server.toml`, OS configuration paths and materializer state. +- Workspace identity, signing material, runtime trust/registration, + credential stores, authentication/session/passkey/token data and private keys. +- Repository registrations and Flow records, which use separate typed APIs. +- Workspace display name and Memory language, which have independent CAS + domains; they must not be presented as one atomic source-tree transaction. +- Worker retention/lifecycle policy and destructive administrative operations. + +Repository-access **references** in authored config are not credential private +material. Granting config access does not grant secret-management operations, +filesystem access, command execution, or arbitrary Workspace access. Authored +text can itself contain sensitive information; configuration access must be an +explicit operator grant, not a consequence of possessing Read/Write Tools. +Existing secret stores must never be serialized into observation, operation +results, diagnostic bodies or history through this provider. + +## Enablement and operator grants + +Choose Worker WIP mode and the Feature implementation independently of authority: + +```toml +[worker] +mode = "wip" + +[feature.workspace_config] +enabled = true +``` + +The Feature defaults disabled; enabling it does not add a settings grant. An +operator authenticated as the Workspace owner grants access to a specific +registered Runtime/Worker through `workspace_config_grant_create`: + +```text +POST /api/w//workspace-config-grants +{"runtime_id":"","worker_id":"","access":"read_write"} +``` + +Use `read_only` for observation/read-only access. Grant creation is a separate +owner administration action, not an approval prompt for each edit. To revoke, +use the owner operation `workspace_config_grant_revoke` on the returned grant +ID. Worker requests carry authenticated source identity, never these IDs as a +substitute for authorization. A grant from another Workspace or another Worker +cannot be adopted by knowing a route or possessing a previous Interface. + +Explicit grants do not replace the UI editor's user authentication. Authored +source-tree read/entry/history/commit routes and the UI Profile settings catalog +reject Runtime/Worker source proofs in both single-Workspace and server-scoped +middleware, including workers with a read-write grant. `workspace.request` is +not a configuration-editor capability: granted Workers use the WIP adapter, +never the UI editor as an alternate access path. User/session/API-token editing +continues to share the canonical validation and CAS boundary with WIP saves. + +The audit found no separate Profile/Prompt/Runtime/Skill source-save endpoint; +authored writes converge on source-tree commit. Existing runtime consumption of +Prompt projections, Skill catalog/activation and resolved config bundles remains +a separate read-only/activation contract needed for Worker startup and normal +operation. Those evaluated projections are not authored source-tree bodies, +revision history or a second editor. Do not fence all `/config` routes as one +capability: that would break the existing Prompt reader's runtime contract. + +The existing connection ledger remains authoritative. Each granted Worker has +an independently identified logical registry target over the **same** authored +configuration tree. This preserves the ordinary Workdir exclusive-owner +invariant without making separately granted Workers maintain separate config +stores. Every attachment has an opaque connection lifetime ID; alias reuse is +not identity reuse. No Runtime filesystem session is created for this target. +Detaching ends that connection's use but deletes neither configuration nor its +registry target. Configuration is not a Workdir-cleanup or command target. + +## User invocation and Worker access + +Select `/workspace-config` using the common Feature completion menu. Complete +its declared parenthesized argument form (close `)` to use the default access), +then send the typed chip with the request, for example: + +```text +/workspace-config() このWorkspaceのモデル設定を変更して +``` + +The composer preserves the selection as a typed invocation chip. Ordinary text +that happens to contain this spelling is not a command. Feature preparation +calls Backend attach before dependent LLM input and displays the selected +logical subject and effective access. Failure/unknown preparation must not be +reported as a successful attachment. Optional declarative `access` values are +`effective` (default), `read_only`, and `read_write`. They restrict existing +rights; explicit `read_write` fails rather than silently downgrading. A Worker +can invoke the native attach operation at `/workspace-config` through the same +Feature/Backend handler. Neither path creates a grant or a second ledger. + +The logical content root is `/workspace-config`, not `/checkouts` or a host +path. Observe exposes requested structure and available Interfaces; contents +are retrieved through Operations. Implicit directories reflect current +`VirtualPath` entries and are not independently stored directories. + +## Backend transport and consistency + +Worker operations use identity-bound routes under +`/api/w//workers/self/workspace-config`: + +| Interaction | Route | Contract | +| --- | --- | --- | +| Resolve connection | GET root | Current grant checked even if authorized but unattached | +| Attach | POST root | Idempotent reuse of the active canonical `workspace-config` alias | +| Observe | POST `observe` | Requested paths/depth; metadata only, no source bodies | +| Read | POST `read` | Connection + path validator; canonical body/type/digest | +| Save | POST `commit` | Connection + root validator + canonical config changes/CAS | + +Observe returns root revision/digest and path validators from the same snapshot. +Opaque validators bind Workspace, Worker, grant, connection lifetime, logical +path and whole-tree revision/digest. Even replacing a file with identical bytes +in a later revision invalidates older observations. Whole-tree CAS is +intentionally conservative: an unrelated UI edit can require fresh observation. +The Client holds validators; the LLM must not manage or fabricate them. + +Authorization, active connection lookup and operation use share the per-Worker +session lock with detach/revoke. The canonical Immediate-transaction CAS remains +the final cross-Worker/UI commit boundary. Failed validation, stale state and +permission rejection do not overwrite configuration. Potential auxiliary +repository-secret effects and uncertain persistence are not described as a +full rollback. Sanitized failures distinguish `not_committed` and `unknown`; +a timeout/disconnect after dispatch never causes automatic mutation retry. + +Structural observation is bounded to depth 0–8 and 256 returned nodes, paths to +512 bytes and each text entry to 256 KiB. Oversized observations fail rather +than presenting a truncated result as complete. Source-tree creation/deletion +is expressed through canonical changes, including atomic multi-file changes +needed to update a source together with its import/registry references. The +`main.dcdl` entrypoint cannot be deleted/renamed. These are configuration +operations, not a generic POSIX filesystem or an OS mount. + +## Native operation mapping + +| WIP target | Operations and model arguments | Canonical action | +| --- | --- | --- | +| Root | `attach(access?)` | Same Backend connection operation as the selected slash invocation | +| Root | `apply_changes(changes)` | One atomic create/update/delete/rename batch; source and import/registry edits can be combined | +| Directory | `create(path, content, content_type?)` | Create an entry below this directory; ancestor directories are implicit | +| File | `read()` | Return canonical text and content type, not a byte-stream or OS descriptor | +| File | `write(content)` | Update this exact entry, preserving its content type | +| File | `edit(old_string, new_string, replace_all?)` | Unique string replacement unless `replace_all` is selected | +| File | `delete()` | Canonical deletion subject to entrypoint/import validation | +| Explicit missing path | `create(content, content_type?)` | Create that exact absence-observed entry, not a substituted path | + +Content type is `decodal` (default for creation) or `text`. File operation +targets come from the resolved WIP route. The root change batch uses canonical +relative source paths; it does **not** accept `expected_digest`, revision, +validator, Workspace, grant or connection overrides as model arguments. The +adapter captures metadata and fills those preconditions automatically, requiring +all paths in a batch to match one observed tree state. Unsupported/invalid +configuration mutations fail instead of performing a partial multi-file save. + +Example `apply_changes` argument: + +```json +{"changes":[{"kind":"create","path":"notes/readme.md","content_type":"text","content":"Configuration notes"}]} +``` + +These are native Interfaces only. There is no configuration-specific +compatibility Tool or fallback under `/tools`, no Bash location, and no +`wip-fs-server`/Linux real-filesystem provider substituted for these operations. + +## Validation scope + +Start with the changed contracts: + +```sh +cargo test -p manifest --lib workspace_config +cargo test -p worker --lib workspace_config +cargo test -p worker --lib manage_workdir::wip +cargo test -p yoi-workspace-server --lib workspace_config_integration +cargo test -p yoi-workspace-server --lib config_source::tests +cargo test -p server-api --lib workspace_config +cargo test -p tui --lib backend_worker_picker +``` + +The Backend integration tests use production Feature completion selection, +structured `Worker::run` input, request preparation and the real Backend router +and SQLite configuration store. At the recording LLM-client boundary they +assert that attach and its durable context happened first, then use the actual +WIP Client/Host to read/edit/write/create/delete and atomically change a source +with its import. The existing Prompt reader confirms the saved projection. +Signed-route, owner-grant, readonly/revoke, detach/lifetime, multi-Worker/UI CAS, +same-content replacement, invalid configuration, storage-failure and dropped +post-save-response cases complement that sequence. + +Web tests exercise the production generic composer component's declaration +completion, selection and typed submission, plus removal/literal-text behavior. +Logical-source parsers and launch/sidebar/TUI tests ensure that a settings +attachment is not treated as a repository revision or process cwd. The two +layers are complementary; the Rust fixture uses a recording LLM client and +in-process transport, not a live browser-to-provider deployment. No live +Backend/Runtime/LLM browser or physical power-loss result is claimed. Bounds +are finite input/tree and lock/request deadlines, not a promise that synchronous +configuration evaluation is forcibly preemptible at a hard wall-clock limit. + +Completion validation also runs the current-source root `cargo check`, complete +changed-crate suites, the server-api TypeScript configuration and generated +contract checks, formatting and diff checks. Updating this code does not update +or restart the running dogfooding environment. diff --git a/docs/development/file-operations.md b/docs/development/file-operations.md new file mode 100644 index 000000000..f86d1eb24 --- /dev/null +++ b/docs/development/file-operations.md @@ -0,0 +1,49 @@ +# Shared text file operations + +`fs-operation::text` owns the provider-neutral read/edit/write argument contracts, +JSON decoding, argument validation, pure text outcomes and byte-bound calculations. +The argument schemas also supply WIP parameter declarations. Ordinary Tools flatten +these arguments alongside their own Workdir routing fields; checkout WIP uses the +same Tool execution functions, not a third editing implementation. + +Edit requires a nonempty `old_string`, a different `new_string`, and a unique +non-overlapping match unless `replace_all` is true. Limits count UTF-8 bytes, are +supplied by each provider, and are checked (including arithmetic/capacity overflow) +before allocating a replacement result. Outcomes carry replacement count and byte +size independently of persistence metadata. + +## Provider-owned differences + +- Workdir read exposes optional line offset/limit and numbered, bounded output. + Workspace settings read remains a whole-text read with no arguments, preserving + content type and digest. Line range defaults and numbered rendering live in the + core but are only selected by the Workdir adapter. +- Workdir routing, attachment generation, Scope and permission identity remain in + Worker/Tools/Workdir. Read history and content hashes stay in the existing + attachment-scoped tracker. Descriptor-protected reads and checked edits/writes + retain their existing validator, identity and save boundary in the provider. +- Workspace settings grants, access mode and connection lifetime remain in Backend. + The settings adapter binds the read to captured entry metadata and generates a + canonical commit using the same tree revision/digest. Decodal validation, + formatting and atomic multi-file persistence remain in the canonical Backend + commit. Content type, create/delete/rename/apply_changes and attach are settings + operations, not ordinary OS file operations. + +The core does not perform I/O, authorize anything, or own revision/existence +mirrors. Providers invoke pure processing inside their existing consistency +boundary; it must not be used to turn a checked edit into an unchecked +Read–Write sequence. + +## External contracts + +Operation names, argument shapes and successful responses remain unchanged. +Workspace settings Edit now rejects identical old/new strings before reading or +committing (the intended behavior change). Pure text failures are explicitly +mapped to Tool argument errors or WIP `InvalidArguments`/`ResourceLimitExceeded` +by the adapters. A mutation's post-dispatch transport or response failure is still +`OutcomeUnknown`; the core does not retry it or reinterpret it as a validation +rejection. + +Pure policy matrices belong in `fs-operation::text` tests. Provider tests cover +entrypoint wiring and the existing authorization, observation, checked-save, +configuration CAS and unknown-outcome boundaries rather than copying that matrix. diff --git a/docs/development/ui-ux/subject-memory-information.md b/docs/development/ui-ux/subject-memory-information.md new file mode 100644 index 000000000..2a2ff687c --- /dev/null +++ b/docs/development/ui-ux/subject-memory-information.md @@ -0,0 +1,133 @@ +# Subject / Memory information audit (T-692) + +## Decision and scope + +The previous Subjects index exposed `Store revision` alongside identity, state and +update time; Subject detail led with technical metadata and raw surface states. +These numbers require implementation knowledge before they help a user choose a +Subject or understand what is available. The revised hierarchy starts with +identity, connection, committed records and generated-context availability. + +This is a presentation change, not a change to storage, freshness comparisons, +permissions, generation, or Worker history. T-688's ordinary Worker launch and +T-689's independent Subject creation remain the actual operation paths. T-693 +owns editable user-managed behavior: that content must remain distinct from this +page's context generated from committed Memories. A successful read or a current +surface does not attest what is already in any running Worker's history. + +## Field classification and before/after priorities + +| Information | Classification and reason | Presentation decision | +| --- | --- | --- | +| Subject role and ID | Normal identification. Similar or identical roles need a stable identifier for selection and comparison. | Keep role as heading and the complete wrapping ID visible in index and detail. | +| Subject active/retired state | Normal operation decision: whether a new Worker connection can use the Subject. | Replace raw `active` with `Available`; keep `Retired`. Detail explains connection eligibility, not an assertion that a Worker is running. | +| Current Worker | Normal connection decision, derived from the existing Backend singleton projection. | Promote `Connected` / `Not connected` and display name. Do not infer ownership from store revision or claim all Workers received current Memory. Worker start remains the ordinary launch form. | +| Committed-Memory presence | Normal reading decision, independent from surface generation or candidates. | Show `Available` / `None` only on the successful first bounded page. On later pages say `N on this page`; errors say `Unavailable`. Never infer counts from revision or scan all pages. | +| List counts | Normal navigation context, not a store-wide statistic. | Label `N shown` and `more available`, preserving existing bounded cursor pagination. | +| Resident availability | Normal context decision. A missing surface is not missing committed Memory. | `Ready to use`, `Ready, no context`, `Not generated`, `Needs refresh`, `Generation failed`, and request `Unavailable` are distinct. These describe the current generated store artifact, not content truth or Worker-history propagation. | +| Resident body and generated time | Normal content/read freshness context. | Keep the shared Markdown renderer, generated date and exact snapshot reference count. Ready-empty explains that a valid snapshot has no body; stale/failed do not display a partial or old snapshot. | +| Subject store revision | Diagnostic only: internal committed-Memory change number, not record count, schema version or quality score. | Remove it from the index's normal columns. Retain the exact value under detail's `Technical details`, with its meaning explained. Backend comparisons are untouched. | +| Subject update time | Secondary metadata; not a context-injection or generation completion timestamp. | Label `Last changed`; on detail put it with Subject diagnostics rather than competing with the availability summary. | +| Snapshot ID and built-from revision | Diagnostic correlation, distinct from per-Memory revisions. | Retain under `Surface sources and diagnostics`, labelled `Generated from Subject revision` and explained. | +| Surface Memory references | Detailed audit/reachability. | Preserve reference IDs, immutable revision values and Memory links in the disclosure. | +| Memory claim, kind, lifecycle state, excerpt and body | Normal content decision. Active, resolved and retracted are not interchangeable. | Retain list scanning and body reading. Keep the Markdown renderer and current/historical distinction. | +| Individual Memory revision and history | Detailed content comparison, also useful for identifying a selected version. | Explicit `Memory revision` labels, separate from Subject counters. Preserve immutable history, selected revision links and revision pagination. | +| Candidate provenance, evidence, origin, source refs and derivations | Detailed audit, required to understand why a record exists. | Retain IDs, ranges, summaries, excerpts, origins and derived-revision links. Counts say `shown` except where the API provides exact per-candidate totals. Preserve body and evidence continuation links. | +| Raw field names / duplicated general revision terminology | Unnecessary in the normal hierarchy; multiple unrelated counters are misleading. | Replace bare revision labels and raw surface enums with purpose-specific labels. Do not delete API fields, stored information, diagnostics or their detailed access. | + +No diagnostic datum was removed from Subject detail, surface detail or Memory +provenance/history. The index no longer shows a store revision, but the exact +value remains one click away in Subject detail. No invented regeneration, +editing or recovery action was added; `Needs refresh` does not mean an update job +is running. Creation still creates only a Subject, and selection still uses the +existing ordinary launch contract. + +## Candidate-only verification + +Candidates are not committed records and are not an input to this page. The +Subject loader performs just three bounded reads: Subject, resident surface and +`/memories?limit=100[&cursor=...]`. It does not read or count staging records. + +The existing real-store regression +`subjektiv::tests::candidate_decisions_reject_conflicts_and_roll_back_partial_writes` +stages a candidate and attempts an invalid apply. It asserts the candidate has no +resolution and the committed Memory list remains empty. The component regression +`candidate-only Subjects show no committed Memories, not an inferred candidate count` +uses that read projection (store revision 0, ungenerated surface, empty successful +committed list) and checks `None`, `No committed Memories yet.`, `Not connected` +and `Not generated`, with no resident body. Thus a staged candidate does not turn +into a committed-Memory count or a claim that context has been generated. + +The store test proves the persistence boundary; the component test proves the +rendering of its public projection. These are separate boundary tests, **not** a +live browser-to-Backend candidate-creation E2E. The existing `ungenerated` capture +represents the same visible projection, but does not itself stage a candidate. + +## Visual review and validation evidence + +The restored implementation is in `fbb09e65` and `4cf6c130`; this report/test +supplement does not change rendered markup, styling, copy or data contracts. + +- Scenarios: `tools/web-ux/scenarios/memory-subjects-{light,dark}.json`. +- Before: `target/web-ux/t-692-before-{light,dark}` at `cc979874`. +- After: `target/web-ux/t-692-final2-{light,dark}` at clean `4cf6c130`. +- Comparisons: `target/web-ux/t-692-final2-{light,dark}-comparison`. +- Browser interactions: `target/web-ux/t-692-browser-interactions`. +- Persona: synthetic Workspace fixture; permitted creation and intentional + permission-rejection responses. No real account or private store data used. +- Viewports: 1440, 768, 390 and 320px; light and dark themes. +- States: connected/unconnected, active/retired, successful empty lists, ready + long Markdown, ready-empty, ungenerated, stale, failed, unavailable HTTP503, + create form and create permission error; expanded technical disclosures. +- The takeover implementer personally inspected individual before/after images + for index at 768px, representative detail at desktop/mobile and Memory detail, + then ready-empty mobile, ungenerated narrow, stale desktop, failed mobile, + endpoint error at 320px, mobile surface sources, compact Subject diagnostics + and mobile permission error. Oversized contact sheets were rejected by the + image provider; individual viewport PNGs were inspected instead. +- Findings/result: primary state groups replace counters without losing IDs or + document access; long IDs wrap, status is textual rather than color-only, + mobile groups stack, diagnostics remain readable, and error state regions stay + separate. No new visual defect identified. **Pass for the audited scope.** +- Final bundles contain 64 captures each. `completed-with-errors` is expected: + intentional creation HTTP403, endpoint HTTP503/associated aborted requests, + and the deliberately failed surface's alert are retained as evidence. Healthy + capture states have empty error arrays; this is not a blanket claim that all + captures were error-free. Review-context diagnostics are empty. +- The focused production-shell browser test proves main-owned vertical scrolling + with wheel/keyboard/touch, 320px/mobile shell behavior, no page-wide overflow, + local table/code scroll, Markdown safety, creation/launch separation, bounded + pagination and Memory/source/revision reachability. + +Independent review at source `4cf6c130` ran root `cargo check`, +`cargo fmt --all -- --check`, diff hygiene, Web check (0 errors/warnings), 12 +Memory API tests, 452 full Web unit tests, 134 component tests (8 Memory tests), +production build, the focused production-shell browser test (1), and Web-UX unit +tests (16), all passing. + +The takeover implementer then validated this report/test supplement with: + +- `cargo test -p yoi-workspace-server --lib candidate_decisions_reject_conflicts_and_roll_back_partial_writes`: 1 passed. +- Root `cargo check`, `cargo fmt --all -- --check`, and `git diff --check HEAD`: passed. +- Targeted Memory component tests: 9 passed, including the added candidate-only projection test. +- Web `deno task check`: 0 errors/warnings; `deno task test`: 452 passed; + `deno task test:component`: 135 passed. +- Scoped `deno fmt --check` on the changed TypeScript test: passed. +- JSON audit of both final2 bundles: 64 captures each, exact clean source + `4cf6c130`, and no errors outside the deliberate permission/error/failed routes. + +The supplement changes only a document and a test, so it does not require new +rendered-UI captures. Rereview records and the final approved Ticket handoff link +this report to the new exact source. + +Known non-blocking validation diagnostics: component runs emitted incidental +localhost:3000 connection-refused messages while passing; the production build +warned about a large chunk. A broader Web-UX typecheck fails in unchanged +`tools/web-ux/src/processes.ts:44` on a `TextDecoderStream` generic incompatibility; +this report does not claim that broad check passed. + +No live-Backend browser interactions, real Worker model-request propagation, +regeneration, or server/runtime process E2E were performed. No Rust production +crate changed in T-692; the supplemental store test exercises an existing +contract without changing that contract. Backend counters, comparison logic, +storage schema and feature behavior remain intact. diff --git a/docs/report/2026-10-05-review-severity-tool-contract.md b/docs/report/2026-10-05-review-severity-tool-contract.md new file mode 100644 index 000000000..69d1a9c64 --- /dev/null +++ b/docs/report/2026-10-05-review-severity-tool-contract.md @@ -0,0 +1,21 @@ +# Review finding severity contract mismatch during T-697 + +The independent Reviewer completed source/requirement analysis and validation, +then its one `ReviewMergeRequest` submission was rejected with HTTP 422 because +it used severity `medium`. The tool exposed a free-form string, while Backend +validation accepts only `blocker`, `major`, `minor`, or `note`. No verdict was +recorded by that failed call. The child preserved the source and reported the +blocked submission rather than loop-retrying a mutation. + +The parent inspected the committed review session and fresh MR authority, +confirmed the same request/source/Ticket snapshot with no verdict, and submitted +a fresh child turn to correct the input contract. The Reviewer independently +selected supported severities and recorded `request_changes` successfully on +the same MR. No alternate token/context, CLI/storage write, source movement or +replacement MR was used to compensate for the rejection. + +Suggested improvement: expose finding severity as the typed Backend enum in the +LLM tool schema, and document allowed values in the Reviewer prompt. Rejecting +an unadvertised enum after a costly review should not prevent durable findings +from reaching the MR. This report is tool feedback, not a T-697 product +constraint or a request to alter review judgments. diff --git a/docs/report/2026-10-05-review-verdict-schema-rejection.md b/docs/report/2026-10-05-review-verdict-schema-rejection.md new file mode 100644 index 000000000..7d5e9ce2e --- /dev/null +++ b/docs/report/2026-10-05-review-verdict-schema-rejection.md @@ -0,0 +1,30 @@ +# Reviewer verdict のvalidation rejectionとone-call制約 + +## 確認した事実 + +T-703のMR `01a10b3b-87ec-7093-b0c4-a00a78095695`、source +`3174142bd1314fe34c5de7b6a66654ff1c2a8af2` の独立ReviewerはRestoreとno-op +attention checkの競合を見つけ、`ReviewMergeRequest`でrequest_changesを提出しようとした。 +severityに`medium`を指定したためHTTP 422となった。Backendの許容値は +`blocker` / `major` / `minor` / `note` だった。 + +親の`ShowMergeRequest`再読でthreadはReviewRequested seq2までであり、verdictが +永続化されていないことを確認した。Reviewerは「ReviewMergeRequestを正確に一度だけ +呼ぶ」という指示に従い、validation rejection後も再提出を行わなかった。親のfollow-upでも +同じ境界を報告したため、再提出を強制せずReviewerを停止した。 + +## 障壁 + +正しいBackend拒否を迂回すべきではない一方、mutationが行われていない入力validation失敗が +一回限りの提出枠を消費する扱いだと、レビュー済みの指摘がMR authorityに残らずpendingになる。 +セッションでの報告をverdictや承認の代わりにはできない。 + +## 改善案（このTicketの実装対象ではない） + +- Reviewer-facing tool schemaにseverityのBackend enumを明示し、不正値を入力時に防ぐ。 +- one-call指示について「一度のdurable verdict」と「validationで拒否された呼出」を区別できるか + 設計上判断する。無制限再提出や記録済みverdictの変更を認める提案ではない。 +- mutationなし／retry可否をtyped errorとして返せると、親とReviewerの双方が推測なしに扱える。 + +T-703では元のMR/source selectorを保持し、指摘を修正したcommitを通常pushした後、fresh +exact-source reviewで修正内容とその証拠をMRに記録する。未記録のverdictを承認扱いしない。 diff --git a/docs/report/2026-10-05-review-verdict-validation.md b/docs/report/2026-10-05-review-verdict-validation.md new file mode 100644 index 000000000..d3cad9724 --- /dev/null +++ b/docs/report/2026-10-05-review-verdict-validation.md @@ -0,0 +1,13 @@ +# Review verdict validation and single-call policy + +During T-696's independent review, `ReviewRequested` captured MR `01a10bc8-59f4-7630-8166-d8cd71943e15`, source `9812e68b4d4021ea778a40d9ead4fe545d09e8bf`, and Ticket revision `00001M4517HE0:5`. The Reviewer reproduced a pending-upload deletion defect and attempted a structured `request_changes` verdict. The request used finding severity `medium`; the Backend returned HTTP 422, listing the accepted values as `blocker`, `major`, `minor`, and `note`. No review ID or verdict event was returned. + +The Reviewer obeyed the role prompt's instruction to call `ReviewMergeRequest` exactly once and did not retry. Its prose recommendation and artifacts are evidence of investigation, not recorded review authority. The parent preserves the finding, fixes the defect, and requests a new independent review bound to the updated source; it does not invent a verdict or bypass the rejected operation. + +Potential improvements: + +- Publish the finding severity vocabulary as a closed tool-schema enum, matching Backend validation, rather than relying on a free-form string. +- Clarify the single-call policy for an unambiguously rejected validation request that committed no event. Keep transport ambiguity, consumed capabilities, and source movement distinct from input validation; any retry permission must come from the authority contract, not the caller's inference. +- Return explicit committed-event/capability-consumption evidence on rejection, so a caller can distinguish an absent verdict from an ambiguous write without inspecting storage. + +These are observed workflow barriers and proposals, not changes to the review authority contract or implementation requirements of T-696. The original rejected request and reproduction are preserved in the parent-observed Reviewer session and `target/review/T-696/review-submission-blocker.md`. diff --git a/docs/report/2026-10-05-reviewer-subworker-stopped-before-run.md b/docs/report/2026-10-05-reviewer-subworker-stopped-before-run.md new file mode 100644 index 000000000..38f6fd395 --- /dev/null +++ b/docs/report/2026-10-05-reviewer-subworker-stopped-before-run.md @@ -0,0 +1,18 @@ +# Reviewer SubWorkerがreview開始前に停止する + +## 状況 + +T-698の公開済みMerge Requestに対して、`builtin:reviewer`を指定し、対象Ticket/Merge Request、writable delegated scope、command grantを明示してReviewer SubWorkerを3回起動した。 + +いずれも`review_requested` eventは正しいsource refとTicket revisionを記録したが、SubWorkerは直後に`Stopped`となった。session observationには最初のuser taskだけが存在し、assistant output、tool call、typed review verdictは記録されなかった。Merge Request threadにもapprove/request_changesは追加されなかった。 + +## 障壁 + +CoderはReviewer approvalを代行できず、親Workerには`ReviewMergeRequest` authorityがないため、実装・公開・検証が完了していてもfresh approvalを取得できない。同じ入力を再送するだけでは診断情報が増えず、review requestだけが複数残る。 + +## 改善案 + +- Reviewer起動がmodel run前に停止した場合、停止理由を親へ構造化して通知する。 +- `review_requested`を記録した後にReviewer sessionの開始に失敗した場合、request eventと失敗eventを関連付け、未解決reviewとして明示する。 +- profile/provider初期化、review worktree準備、capability bindingのどの段階で失敗したかを、credentialやhost pathを露出しないbounded diagnosticとして提供する。 +- 同一sourceへの再試行時は、先行Reviewerが一度も実行されていないことを識別できるようにする。 diff --git a/docs/report/2026-10-05-subworker-cwd-attachment-root.md b/docs/report/2026-10-05-subworker-cwd-attachment-root.md new file mode 100644 index 000000000..61ad4f90b --- /dev/null +++ b/docs/report/2026-10-05-subworker-cwd-attachment-root.md @@ -0,0 +1,33 @@ +# SubWorker cwd attachment-root ambiguity during T-697 + +## Observed barrier + +A delegated backend investigation was spawned with `cwd=crates/workspace-server` +and explicit writable rules for both `crates/workspace-server` and +`crates/server-api`, plus repository-root read access. The child reported that +its available attachment exposed only the workspace-server crate; its observed +file reads used `src/server.rs`/`src/lib.rs` and it could not access server-api. +It also had no `ShowTicket` or parent-notification tool, despite the parent +having typed Ticket authority. No implementation changes were made by that child. + +The parent inspected the committed child session, released the idle child, and +respawned at the repository root with the same delegated write boundaries and +an explicit summary of the authoritative Ticket requirements. That child could +then implement the cross-crate API boundary. No filesystem capability rejection +was worked around through another write path. + +## Suggested improvement + +The advertised SubWorker `cwd` contract says that it changes only the tool +working directory, not attachment roots. Show the effective attachment aliases, +logical roots and readable/writable scopes explicitly to the child and parent, +and test cross-crate reads/writes for a non-root cwd. If cwd intentionally +narrows or rebases an attachment, document that instead of presenting it as +only a command default. A denied/inaccessible sibling diagnostic should identify +the effective logical attachment root without disclosing unauthorized paths. + +Also expose which typed domain and parent-messaging tools will be available +before spawning. Delegated implementation requires authoritative requirements; +where Ticket reads are unavailable, the parent must deliberately provide the +bounded contract rather than assume the child can recover it by ID. This report +records tool ergonomics only; it is not a new T-697 product requirement. diff --git a/docs/report/2026-10-05-subworker-ticket-context.md b/docs/report/2026-10-05-subworker-ticket-context.md new file mode 100644 index 000000000..e05394172 --- /dev/null +++ b/docs/report/2026-10-05-subworker-ticket-context.md @@ -0,0 +1,10 @@ +# Delegated coding context and artifact access + +During T-696 recovery, the parent Coder could read the authoritative Ticket through `ShowTicket`, but default coding SubWorkers did not receive that tool. A delegated task that simply asks a child to reread the Ticket therefore cannot be fulfilled. One child attempted a CLI fallback; this is not an authorized substitute for typed Backend tools. The parent supplied the authoritative revision and relevant requirements instead and retained lifecycle/publish authority. + +Improvement: make the child tool/capability projection explicit in the spawn result, and support passing a bounded authoritative Ticket snapshot with revision into a child without implying mutation authority. Delegation prompts should include the necessary contract rather than assuming the child has the parent's Backend tools. + +The Bash provider also advertised a full-output spill under the parent's declared readable `bash-output` directory, but `Read` rejected the returned absolute path as outside the attachment scope. The parent did not bypass that rejection; it used smaller bounded queries. Improvement: return a typed read capability/artifact reference for spill output, or ensure the advertised spill path is actually readable through the selected attachment. + +A child spawned with `cwd: tools/web-ux` observed its filesystem attachment rooted there, and sibling `docs`, `web/workspace`, and `target/web-ux` reads were rejected despite explicit sibling scope rules. The tool description says `cwd` changes only the default command directory. The parent stopped the unchanged child and created a root-oriented child with an explicit nonrecursive readable root plus the original bounded sibling rules. Improvement: preserve attachment-relative path identity independently of command cwd, or document the actual child path binding in the spawn result. Capability rejections should remain authoritative, not trigger filesystem/CLI fallbacks. + diff --git a/docs/report/2026-10-06-t701-validation.md b/docs/report/2026-10-06-t701-validation.md new file mode 100644 index 000000000..fa9025f75 --- /dev/null +++ b/docs/report/2026-10-06-t701-validation.md @@ -0,0 +1,78 @@ +# T-701 validation ledger + +This records concrete intermediate failures rather than treating later passes as erasing them. Final source and review evidence are on the repository Merge Request. + +## Validation approach + +Start with Restore/parser/UI/serialization boundary filters, then the eight changed crates and the normal root `cargo check` compile closure. Cargo jobs are bounded to three; full crate test concurrency is bounded to four. No all-workspace test, `--all-targets`, Nix/Docker build, deployment, process restart or live Worker repair is part of this validation. + +The Web production bundle, `svelte-check`, normal Deno tests and actual rendered component suites have passed. Those results alone do not prove Backend admission, replay or preparation recovery. Full Rust suites exercise callers, schema migration and dependent inventories that narrow guard filters cannot prove. + +## Intermediate failures and disposition + +1. Early root checks failed because the Runtime request/preparation DTO integration was not yet present. This was an incomplete implementation, not a baseline defect. A subsequent root check passed after the integration. +2. Early Server API full tests reported three stale generated artifacts after the OpenAPI contract changed. Regenerating the affected closure artifacts resolved them; the full default suite passed 70 tests. TypeScript-feature tests were also executed after regeneration. +3. Adding full catalog GETs exposed existing browser fixtures that returned Workdir-only shapes to every endpoint. The fixtures now return the exact `/workers` contract, and the Workdir-error fixture fails only its intended endpoint. Catalog and transport/Workdir failures have separate alert identities. +4. Parent full `cargo test -p yoi-workspace-server -- --test-threads=4` produced **614 passed / 4 failed** (618 library tests): + - `hosts::tests::embedded_runtime_registers_routes_input_without_internal_leaks`: a broad forbidden-word assertion rejected the newly public observation field's word `token`. The private credential/token non-leak contract must be preserved while allowing the explicitly public observation token. + - `server::tests::embedded_runtime_api_routes_by_runtime_and_worker_ids_without_leaking_internals`: an old `{}` Restore fixture received 422 because guarded Restore now requires the observed token and request ID. Update the real route fixture, do not restore an unguarded compatibility path. + - `workspace_deletion::tests::explicit_purge_inventory_covers_every_workspace_scoped_table`: the new internal Restore intent journal was absent from the explicit purge inventory. Integrate its Workspace deletion lifecycle; do not weaken the exhaustive inventory check. + - `server::tests::ticket_checker_suppresses_stale_and_clean_results_and_diagnoses_invalid_output`: the inherited stale-checker `execution.take_inputs().is_empty()` assertion failed. Earlier T-702 evidence reproduced this named instability on an isolated original baseline; see [T-702 validation report](2026-10-06-t702-stale-checker-validation.md). T-701 has not fixed that checker, and this failure is not conflated with the three new integration regressions. + +## Follow-up disposition and final pre-publication results + +The projection fixtures now permit only the explicit `restore_observation_token` key and validate its UUIDv7 value; private token/credential fields remain forbidden. The HTTP fixture sends the real observed token and request ID. The internal intent table is in the exhaustive Workspace deletion inventory. + +The final restart recovery test exposed one obsolete Runtime expectation: an unowned startup/legacy pending operation must be explicitly recoverable, not permanently conflict. Public recovery now binds a receipt to the exact existing lifecycle operation ID, without candidate preparation or a replacement execution. Tests assert a competing request conflicts, recovery preserves the ID, and terminal replay after Stop never launches again. A Tool recovery fixture also used path-shaped IDs rejected by the existing authority-ID validator; it now uses valid authority IDs and proves the two explicit calls send an identical request body. + +After those fixes, a full Server run reported **618 passed / 1 failed**, solely the inherited stale-checker assertion above. Its isolated test subsequently passed. An additional whole-crate run with the same four-thread configuration and **no skips** passed **619 library + 11 binary tests**. This does not erase the earlier failure or claim to fix the checker race. + +| Executed command | Final pre-publication evidence | +| --- | --- | +| root `cargo check` | Passed normal default-member compile closure, including TUI and Server | +| `cargo test -p worker-runtime -- --test-threads=4` | 332 library + 34 binary tests passed | +| `cargo test -p yoi-workspace-server -- --test-threads=4` | 619 library + 11 binary tests passed in the additional no-skip run; previous flaky failure retained above | +| `cargo test -p worker -- --test-threads=4` | 804 library + 109 integration tests passed | +| `cargo test -p runtime-api` | 9 tests passed | +| `cargo test -p server-api` / `--features typescript` | 70 / 77 tests passed, including generated artifact freshness | +| `cargo test -p protocol` / `--features typescript` | 78 / 83 tests passed | +| `cargo test -p client` | 52 library + 2 integration tests passed | +| `cargo test -p tui` | 331 tests passed | +| Web `deno task check` | 0 errors, 0 warnings | +| Web `deno task test` | 504 tests passed | +| Web `deno task test:component` | 249 tests in 25 files passed, including the real paired sidebar/page/store suite | +| Web `deno task build` | Production build passed; existing chunk-size warning remains | +| `cargo fmt --all -- --check`, `git diff --check HEAD` | Passed | + +Additional exploratory Runtime feature checks did **not** pass: `--no-default-features` test/check cannot resolve existing unconditional `workspace_request` / `reqwest` imports; the test target additionally references disabled `fs_store`. HTTP-only check also lacks the existing retention and futures dependencies. Inspection of original HEAD confirms those imports/gates already exist; no isolated baseline build was performed for these configurations. These are recorded as failed exploratory checks, not semantic passes or newly supported configurations. The default Runtime configuration, which contains fs-store and ws-server, is exercised in full above. This Ticket does not redesign unrelated Runtime feature gating. + +Raw session validation logs are retained under ignored `target/t701-validation/`, not shipped as product source. Immutable published source, independent verdict and any subsequent fix/review evidence belong to the explicit repository Merge Request. + +## Independent review and caller recovery fixes + +The independent Reviewer recorded `request_changes` on MR `01a11064-502b-7222-9494-b1952fbcb1d5`, source `fdce12a47bdd408e6e1a0275dfb218951f1a2f1f`, for two concrete omitted recovery scenarios. These are implementation defects, not baseline failures: + +- **`restore_tui_loses_recovery_identity`:** The existing TUI Resume/Restore flow exited on unknown/pending result and discarded the tuple. It now retains the request throughout that flow, refreshes Session observation without treating GET as settlement, and offers explicit same-tuple recovery confirmation. Each additional POST requires user confirmation. Conflict and terminal rejection do not offer same-intent recovery; cancellation makes no additional POST and reports the original identity. No new Restore entrance, new execution operation, input or automatic retry loop is introduced. New TUI tests use the **real Runtime**, with only execution and lost transport acknowledgements faked, to assert identical target/request and unchanged lifecycle operation ID across transport and reconciliation-required recovery. Runtime is a test-only TUI dependency. +- **`restore_internal_conflict_never_retires_intent`:** A definitive stale conflict now settles only its pinned client intent before returning the original conflict; unknown/pending delivery remains pinned. A later deliberate generic orchestrator/subject invocation can use fresh observation. Fresh generic requests use a new UUID identity, including when two rejected invocations see the same generation, without changing unsettled retry identity. Tests cover both generic domains, ABA between pin and admission, journal restart, no automatic retry, immutable historical identity, repeated same-generation conflict, and preservation/reconciliation of a different admitted pending operation. + +The first compile of the new TUI fixture omitted `last_finished_submission_request_id` in two WorkerState snapshots and failed; the fixtures were corrected before the passing focused/full tests below. + +Post-fix validation: + +| Command | Evidence | +| --- | --- | +| root `cargo check` | Passed | +| TUI focused `backend_worker_picker::tests::resume_` | 5 passed, including 3 new real-Runtime recovery tests | +| Server focused `restore_guard_internal_` | 3 passed, including both new conflict retirement tests and existing unknown/restart recovery | +| `cargo test -p tui -- --test-threads=4` | 334 passed | +| `cargo test -p worker-runtime -- --test-threads=4` | 332 library + 34 binary tests passed | +| First post-fix full Server run | 620 passed / 1 failed: the same inherited stale-checker assertion | +| Post-fix isolated checker | Failed the same assertion; not hidden or treated as a pass | +| Additional post-fix full Server run, same four threads and no skips | 621 library + 11 binary tests passed | +| Formatting / diff whitespace checks | Passed | + +The Web/API/Worker/client source is unchanged by this fix; its initial published-source validation above remains applicable. The additional full Server pass does not resolve or erase either post-fix checker failure. Raw fix logs are `target/t701-validation/{tui-recovery-fix-target,server-intent-fix-target,fix-root-check,fix-full-tui,fix-full-worker-runtime,fix-full-yoi-workspace-server,fix-checker-isolated,fix-server-additional}.log`. + +### Review tooling boundary encountered + +A first builtin Reviewer spawn with read-only delegation was rejected with `Merge Request Reviewer SubWorkers must include writable delegated scope`, despite the read-only source-review role. The accepted configuration kept product source read-only and delegated write/command authority only to isolated ignored `target/t701-review-validation` scratch. The Reviewer did not implement changes or delegate implementation. A clearer API distinction between read-only review source and writable validation scratch would avoid this misleading setup failure. diff --git a/docs/report/2026-10-06-t702-command-output-readback.md b/docs/report/2026-10-06-t702-command-output-readback.md new file mode 100644 index 000000000..a9ef6bd48 --- /dev/null +++ b/docs/report/2026-10-06-t702-command-output-readback.md @@ -0,0 +1,7 @@ +# Command-output readback during T-702 + +During a cold `cargo test -p yoi-workspace-server` run, Bash bounded its inline output and reported a spill file under the Worker's declared readable `bash-output` boundary. A subsequent `Grep` of that exact returned absolute path was rejected as outside allowed scope. I did not attempt another path to bypass the rejection; instead I reran the relevant tests with bounded output filtering. + +This made a timed-out crate test's failure details unavailable through the advertised readback path. Suggested improvement: issue a readback capability/reference for Bash spills usable by Read/Grep, or ensure the Workdir attachment routes the declared readable spill directory consistently. Include the permitted tool/attachment in the spill message if it differs from the repository attachment. + +The first whole-crate test also reached the provider's 600-second timeout while an outdated WebSocket fixture awaited a subscription that was rejected by the new operation-time authentication. After providing genuine fixture credentials, the targeted subscription test completed; subsequent whole-crate runs use bounded test concurrency. A timeout was treated as incomplete validation, not a passing suite. diff --git a/docs/report/2026-10-06-t702-stale-checker-validation.md b/docs/report/2026-10-06-t702-stale-checker-validation.md new file mode 100644 index 000000000..c70e53c12 --- /dev/null +++ b/docs/report/2026-10-06-t702-stale-checker-validation.md @@ -0,0 +1,28 @@ +# T-702 validation: intermittent stale Ticket checker assertion + +The full `yoi-workspace-server` library suite intermittently fails +`server::tests::ticket_checker_suppresses_stale_and_clean_results_and_diagnoses_invalid_output` +at its post-stale-result `assert!(execution.take_inputs().is_empty())`. The same unchanged +assertion passed in some isolated/full runs and failed in others during T-702 validation. + +To separate this from Worker operation reception, I archived original base +`b96bd110c623fafdd7e8df1b153e8c640c793abe` under the ignored `target/t702-baseline` +directory without moving/replacing refs or discarding the Ticket Workdir. A first +shared-target build failed with stale dependency-interface artifacts and was **not** +baseline evidence. I then used an isolated `CARGO_TARGET_DIR=target/t702-baseline-build` +with `CARGO_BUILD_JOBS=3`. The baseline compiled successfully; its isolated checker test +passed. Its full library run with `--test-threads=4` reproduced the same assertion: +**599 passed, 1 failed, 600 total** (79.19s), at baseline `server.rs:44331`. +The post-review T-702 fix's full library run reported **611 passed, 1 failed, 612 total** +(79.01s), at the corresponding `server.rs:44266` assertion. This demonstrates an inherited +intermittent validation failure, not a green suite or proof of its root cause. +A subsequent post-review-fix whole-crate run, with the same implementation and no skips, +passed **612 library + 11 binary tests and doc tests** (library 81.24s). That successful +run does not erase the failed runs or resolve the underlying checker race. + +The Worker reception fix's 12 operation-boundary tests, 5 WorkerRemove tests (including +constructor-installed embedded dispatcher, target-bound proof, resource-key normalization, +running/pinned refusals, retention and catalog removal), root `cargo check`, formatting and +diff checks passed. The checker implementation/test was not changed or skipped to conceal +the failure. Follow-up should instrument competing checker-job dispatch and deterministic +execution-input consumption so this assertion has an explicit synchronization boundary. diff --git a/docs/report/2026-10-06-t704-validation.md b/docs/report/2026-10-06-t704-validation.md new file mode 100644 index 000000000..7ba86a81a --- /dev/null +++ b/docs/report/2026-10-06-t704-validation.md @@ -0,0 +1,126 @@ +# T-704 implementation and validation ledger + +Base: `6f9621ca` (clean detached assigned checkout). Parent created +`work/t-704-backend-subjektiv-jobs` without replacing a branch or discarding changes. +Parent owns Backend/Profile/Subject authorization, cutover, documentation and +integration. Two direct implementation SubWorkers supplied the common durable +Job/store and Worker/Runtime contracts; parent inspected their committed sessions, +diffs and test evidence, integrated them and corrected the boundaries below. +Independent MR review is separate from implementation delegation. + +## Implemented contracts + +- Caller-selected existing Builtin/project registry Profiles, normal virtual + config/prompt bundle resolution and explicit execution-requirement rejection. + Chosen instructions/features/model are not replaced with backend-job defaults. +- Immutable Host-authored Job/attempt create/restore binding and result capability, + independent of Profile names/instructions/tags. No inherited Workdir, parent or + subject-body execution authority. +- Common transactional resource serialization (Workspace schema 81), immutable + Subject/candidate batch grants, exact replay vs resource collision, unknown + fences and all-old-attempt cleanup fences before retry/followup. +- subjektiv requests its Profile; immutable bounded batches, empty surface-only + Jobs, candidate authorization, existing CAS/atomic receipts and dispositions. +- Synchronous bounded clean-context surface generation before result submission. + Preparation reports an already-current snapshot to avoid needless conflicting + regeneration. Job failure acknowledgements are `failed_confirmed` only after + checking the actual failed marker/reason/revision and generation provenance. + Subject feature migration 6 adds immutable generation Job/attempt provenance, + without changing Subject/Memory/candidate content. +- Durable results contain actual candidate dispositions and a separate verified + ready/failed surface outcome. Existing terminal cleanup retains results/memory. +- Removed subject direct spawn and Idle resubmission. Persisted legacy ownership + plus actual foreground WorkerState and confirmed stop/removal fence the cutover. + Busy legacy workers are not interrupted or duplicated; subject bodies and the + independent old Workspace Memory feature are not selected for removal. + +## Final validation (bounded `CARGO_BUILD_JOBS=2`) + +| Command/boundary | Result | +| --- | --- | +| `cargo test -p yoi-workspace-server --lib backend_job` | 33 passed (before final surface acknowledgement addition; final whole-crate run below covers it) | +| `cargo test -p yoi-workspace-server --lib subjektiv_jobs_tests` | 5 passed, including project recipe, batch/auth/receipts, actual failed acknowledgement, current surface and legacy cutover | +| `cargo test -p worker --lib memory_surface_lifecycle` | 16 passed, including publication/failure before result, cancellation, definitive correction, ambiguous exact retry and already-current surface | +| Root `cargo check` | Passed; normal TUI/Server/standalone compile closure | +| `cargo test -p worker -p worker-runtime -p manifest -p runtime-api -p server-api -- --test-threads=4` | Passed: manifest 185; runtime-api 9; server-api 70; worker 815 lib + 110 integration; runtime 335 lib + 34 binary tests | +| `cargo test -p yoi-workspace-server -- --test-threads=4` (final) | 637 lib + 11 binary passed, no skips | +| `cargo test -p server-api --features typescript` | 77 passed, including generated/OpenAPI/TypeScript contract freshness; existing ts-rs serde-attribute warnings | +| `cargo fmt --all -- --check` / `git diff --check HEAD` | Passed | + +These are recording-backend/store and scripted-client fixtures, plus real compiler +and test binaries. They are **not** real-provider LLM execution, a live Backend / +Runtime deployment, live dogfood consolidation or an environment update. No live +Worker/environment was stopped or updated. + +## Failed checks and fixes (not erased by later passes) + +- During integration, new request fields, resource replay metadata and surface + validation arguments initially failed compilation; all were corrected before + final validation. +- Initial Backend tests exposed the narrower browser builtin candidate allowlist; + Host launch now resolves the complete existing registry without widening browser + choices or adding a purpose-specific Profile allowlist. +- Initial surface-only test exposed rejection of empty candidate grants; empty + batches now retain the subject lock and grant no candidate access (owned store + tests 9 + contract tests 9 passed). +- Legacy fixture initially compared catalog compatibility state rather than + actual WorkerState. Cutover now requires the actual Idle snapshot, and the test + synchronizes on that authoritative observation before asserting behavior. +- First full Server run: **635 passed / 2 failed**, both stale migration-history + fixtures expecting schema 80. Their explicit retained chain/history is updated + to 81; migration filter passed 14 tests and subsequent whole-crate runs passed. +- Exploratory `cargo check -p worker-runtime --no-default-features` and + `--no-default-features --features fs-store` both **failed**, with 16 errors from + existing unconditional `workspace_request` / `reqwest` imports while HTTP is + disabled (plus unused-import warnings). The source/gates are present in the + original baseline and are recorded in the + [T-701 ledger](2026-10-06-t701-validation.md#validation-results). No isolated + baseline build was performed here. These remain failed exploratory unsupported + configurations, not feature passes or a T-704 feature-gating fix. The default + Runtime fs-store/ws-server configuration is fully tested. +- The inherited intermittent Ticket checker failure described in the + [T-702 ledger](2026-10-06-t702-stale-checker-validation.md) and T-701 report was + not observed in the final T-704 full Server runs, but is **not** claimed fixed. + +Raw parent logs are under `target/t704-validation/` (ignored local artifacts), +including `server-full-initial-migration-fixtures.log`, `server-full-final.log`, +`changed-crates.log`, `root-check.log`, `worker-surface-final.log`, +`server-subjektiv-final.log`, `server-api-typescript.log`, `runtime-no-default.log` +and `runtime-fs-only.log`. The durable report records their concrete results; +local log paths are not a substitute for the published immutable MR source. + +## Bounded-result follow-up after interrupted review + +The first implementation was published as `d12cdc91`. Human interruption ended +its first pending Reviewer before a verdict. On retry, the parent reread Ticket, +MR, clean Workdir and provider source. Before the replacement Reviewer verdict, +parent inspection identified a concrete result-budget defect: copying full +resolution records for a 100-ID batch could exceed the common 16 KiB result bound, +even when candidate decisions and surface generation had actually completed. +That pending old-source review was stopped rather than treated as approval. + +The follow-up keeps full immutable receipts/reasons/revision/source edges in their +existing store and persists only verified `{candidate_id, action}` summaries in +the Job result. Batch selection conservatively budgets the complete result, +including JSON-escaped IDs in both claim and outcome, plus a reserved bounded +Subject/surface envelope. Candidates beyond that byte-bound stay staged. The +common result bound is not enlarged or bypassed. + +A new narrow fixture closes a batch of long/escaped IDs with individual receipt +reasons larger than 16 KiB, successfully accepts a result within the default +bound, verifies exact replay and remaining candidates, and starts a disjoint +followup only after cleanup. A maximum escaped-ID budget unit test proves the +complete projected result fits, not merely its raw input IDs. + +Follow-up evidence (same bounded jobs setting): +- `cargo test -p yoi-workspace-server --lib result_budget`: **2 passed**. +- Root `cargo check`: **passed** (`fix-root-check.log`). +- Full `cargo test -p yoi-workspace-server -- --test-threads=4`: + **639 library + 11 binary passed**, no skips (`fix-server-full.log`). +- Format and `git diff --check HEAD`: **passed**. + +Only Server logic, Server fixtures and documentation changed in this follow-up; +all other changed-crate/generated/Worker/Runtime results above remain applicable. +The new commit is published on the same source selector of the existing MR and +requires a fresh exact-source review. No target ref, tags or unrelated refs are +pushed, and no merge/integration is performed by the Coder. diff --git a/docs/report/T-695-subworker-path-coordinates.md b/docs/report/T-695-subworker-path-coordinates.md new file mode 100644 index 000000000..1562f06a3 --- /dev/null +++ b/docs/report/T-695-subworker-path-coordinates.md @@ -0,0 +1,30 @@ +# SubWorker cwd and logical paths during T-695 + +Two writable child sessions initially reported missing Workdir/Ticket access. +The concrete observations did not justify that conclusion: + +- A default-profile child had no `ShowTicket` tool, while the `inherit` child + could read the Ticket. Product requirements therefore had to be forwarded or + the effective child profile chosen explicitly. +- A child spawned with `cwd: crates/workspace-server` received + `file not found: crates/workspace-server/crates/workspace-server/src/config_source.rs` + for a root-relative `Read` path. `Read("src/config_source.rs")` succeeded. +- `Read("../server-api/src/lib.rs")` was rejected as an invalid logical path, + despite both crate directories being in the declared child scope. +- The read-only investigation child successfully inspected repository sources; + successful spawning was not evidence that every child had the expected tools, + but one invalid absolute output-directory lookup was not evidence of an + inaccessible repository either. + +The parent inspected committed child sessions and preserved their changes. +The server child was released at Idle, then continued under an explicitly +`inherit`, root-cwd child with the same per-crate write grants and root read +scope. No host-path or parent-traversal workaround was used to bypass the +rejected path. The original prerequisite diff and test evidence were retained. + +Improvement: tool descriptions and generated child startup context should agree +on whether relative file paths are attachment-root-relative or cwd-relative. +Expose effective attachment aliases, root/cwd coordinates and granted tool +surface directly to the child, so a profile/tool mismatch is not misdiagnosed +as a repository authority failure. Scope validation should distinguish invalid +path syntax, missing file, missing attachment and actual permission denial. diff --git a/docs/report/T-700-subworker-followup-observation.md b/docs/report/T-700-subworker-followup-observation.md new file mode 100644 index 000000000..3dc6d95d3 --- /dev/null +++ b/docs/report/T-700-subworker-followup-observation.md @@ -0,0 +1,15 @@ +# SubWorker follow-up後のcommitted sessionを確認できなかった + +T-700の実装中、既存のIdleなdirect child `backend`へ`WorkerSendInput`で追加修正を依頼した。toolは成功を返し、その後Idle通知と`WorkerList`のIdle表示を確認した。 + +一方、`ViewSessionOverview`と`SearchSessionEntries`が返した最後のentryは、follow-up前の保存Manifest fixture修正報告（entry range 744）だった。今回送った追加依頼の文字列の検索は0件で、offset 744からもその旧報告1件しか見えなかった。したがって、通知だけから追加作業の完了は認定できなかった。これは最新turnの実行失敗、captureの遅延、観測面のfreshnessのいずれかを切り分ける証拠ではない。 + +親は現在のdiffを直接確認し、子をStopしてdelegated scopeを解放した後、残りのShutdown admission修正を新しいdirect childへ限定して委譲した。新しい子では最初の依頼と修正報告をsession observationから確認できた。 + +改善案: + +- Idle通知に、対象turnとcommitted captureの末尾entryの対応が分かる情報を含める。 +- follow-upの開始またはcaptureに失敗した場合、成功した旧turnだけを返す代わりにbounded diagnosticを表示する。 +- observationで最新のcommitted captureが未取得の場合、古いsnapshotであることを明示する。 + +credential、raw prompt、Backend保存領域の直接読取で観測制限を迂回することは行っていない。 diff --git a/openapi/server-api.json b/openapi/server-api.json index 9fa559d03..c217d3c0f 100644 --- a/openapi/server-api.json +++ b/openapi/server-api.json @@ -1075,6 +1075,70 @@ ], "type": "object" }, + "CompletionContext": { + "properties": { + "argument": { + "type": [ + "string", + "null" + ] + }, + "invocation": { + "$ref": "#/components/schemas/FeatureInvocationIdentity" + } + }, + "required": [ + "invocation" + ], + "type": "object" + }, + "CompletionEntry": { + "description": "One completion candidate for a prefix query.", + "properties": { + "description": { + "type": [ + "string", + "null" + ] + }, + "invocation": { + "anyOf": [ + { + "$ref": "#/components/schemas/FeatureInvocationDescriptor" + }, + { + "type": "null" + } + ] + }, + "is_dir": { + "default": false, + "type": "boolean" + }, + "usage": { + "type": [ + "string", + "null" + ] + }, + "value": { + "type": "string" + } + }, + "required": [ + "value" + ], + "type": "object" + }, + "CompletionKind": { + "description": "Kind of completion requested by `Method::ListCompletions`.\n\n`File` is the existing `@` path lane. `Feature` lists installed chat\ninvocations. `FeatureArgument` resolves a declared argument using the\nsource-qualified [`CompletionContext`].", + "enum": [ + "file", + "feature", + "feature_argument" + ], + "type": "string" + }, "ConfigCommitRequest": { "additionalProperties": false, "description": "Compare-and-swap request for applying a complete set of source-tree changes.", @@ -2232,6 +2296,98 @@ }, "type": "object" }, + "FeatureInvocation": { + "properties": { + "arguments": { + "items": { + "$ref": "#/components/schemas/InvocationArgumentValue" + }, + "type": "array" + }, + "identity": { + "$ref": "#/components/schemas/FeatureInvocationIdentity" + }, + "invocation_id": { + "description": "Stable client-generated id retained across queueing and transport retry.", + "type": "string" + }, + "name": { + "description": "Selected public name, retained for exact draft/history restoration. Host\nresolution is always by `identity`.", + "type": "string" + } + }, + "required": [ + "invocation_id", + "identity", + "name", + "arguments" + ], + "type": "object" + }, + "FeatureInvocationDescriptor": { + "properties": { + "aliases": { + "default": [], + "items": { + "type": "string" + }, + "type": "array" + }, + "arguments": { + "default": [], + "items": { + "$ref": "#/components/schemas/InvocationArgumentDescriptor" + }, + "type": "array" + }, + "client_adapter": { + "anyOf": [ + { + "$ref": "#/components/schemas/InvocationClientAdapter" + }, + { + "type": "null" + } + ] + }, + "description": { + "type": "string" + }, + "display_name": { + "type": "string" + }, + "identity": { + "$ref": "#/components/schemas/FeatureInvocationIdentity" + }, + "name": { + "type": "string" + }, + "syntax": { + "$ref": "#/components/schemas/FeatureInvocationSyntax" + } + }, + "required": [ + "identity", + "name", + "display_name", + "description", + "syntax" + ], + "type": "object" + }, + "FeatureInvocationIdentity": { + "description": "Source-qualified identity of one invocation contribution, for example\n`builtin:attachments/attach`. Display names and aliases are never authority.", + "type": "string" + }, + "FeatureInvocationSyntax": { + "oneOf": [ + { + "const": "parenthesized", + "description": "`/name(value, key=value)` with JSON-style quoted strings and `)` as the\nexplicit boundary before following natural language.", + "type": "string" + } + ] + }, "FindingSeverity": { "enum": [ "blocker", @@ -2609,6 +2765,285 @@ ], "type": "object" }, + "InvocationArgumentDescriptor": { + "properties": { + "completion": { + "$ref": "#/components/schemas/InvocationCompletion", + "default": { + "kind": "none" + } + }, + "description": { + "type": [ + "string", + "null" + ] + }, + "name": { + "type": "string" + }, + "position": { + "description": "Zero-based positional index. `None` means named-only. A positional\nargument may also be supplied by name; this makes chip editing stable.", + "format": "uint16", + "maximum": 65535, + "minimum": 0, + "type": [ + "integer", + "null" + ] + }, + "required": { + "default": false, + "type": "boolean" + }, + "value_type": { + "$ref": "#/components/schemas/InvocationArgumentType" + } + }, + "required": [ + "name", + "value_type" + ], + "type": "object" + }, + "InvocationArgumentType": { + "oneOf": [ + { + "properties": { + "kind": { + "const": "string", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "properties": { + "kind": { + "const": "integer", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "properties": { + "kind": { + "const": "boolean", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "properties": { + "kind": { + "const": "enum", + "type": "string" + }, + "values": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "kind", + "values" + ], + "type": "object" + }, + { + "description": "A Worker-readable path. This does not grant filesystem authority.", + "properties": { + "kind": { + "const": "worker_file", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "description": "A resource selected and staged by the client. It must never be resolved\nas a path on the Worker host.", + "properties": { + "kind": { + "const": "client_file", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + } + ] + }, + "InvocationArgumentValue": { + "properties": { + "name": { + "type": "string" + }, + "value": { + "$ref": "#/components/schemas/InvocationValue" + } + }, + "required": [ + "name", + "value" + ], + "type": "object" + }, + "InvocationClientAdapter": { + "oneOf": [ + { + "const": "attachment", + "description": "The client stages a local file and replaces the draft invocation with an\n`UploadedFile` chip. Local paths are not sent to or persisted by Worker.", + "type": "string" + } + ] + }, + "InvocationCompletion": { + "oneOf": [ + { + "properties": { + "kind": { + "const": "none", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "properties": { + "kind": { + "const": "static", + "type": "string" + }, + "values": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "required": [ + "kind", + "values" + ], + "type": "object" + }, + { + "properties": { + "kind": { + "const": "worker_file", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "properties": { + "kind": { + "const": "client_file", + "type": "string" + } + }, + "required": [ + "kind" + ], + "type": "object" + }, + { + "description": "Host-owned provider selected by source-qualified id. Metadata lookup is\nside-effect free; implementations never cross this wire boundary.", + "properties": { + "kind": { + "const": "provider", + "type": "string" + }, + "provider": { + "type": "string" + } + }, + "required": [ + "kind", + "provider" + ], + "type": "object" + } + ] + }, + "InvocationValue": { + "oneOf": [ + { + "properties": { + "kind": { + "const": "string", + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "kind", + "value" + ], + "type": "object" + }, + { + "properties": { + "kind": { + "const": "integer", + "type": "string" + }, + "value": { + "format": "int32", + "type": "integer" + } + }, + "required": [ + "kind", + "value" + ], + "type": "object" + }, + { + "properties": { + "kind": { + "const": "boolean", + "type": "string" + }, + "value": { + "type": "boolean" + } + }, + "required": [ + "kind", + "value" + ], + "type": "object" + } + ] + }, "LegacyMemoryResetRequest": { "additionalProperties": false, "description": "Explicit destructive reset request. The confirmation must exactly equal\n`RESET LEGACY MEMORY ` for the Workspace named in the route.", @@ -7356,10 +7791,28 @@ "RuntimeWorkerCompletionsRequest": { "additionalProperties": false, "properties": { - "kind": true, + "context": { + "anyOf": [ + { + "$ref": "#/components/schemas/CompletionContext" + }, + { + "type": "null" + } + ] + }, + "kind": { + "$ref": "#/components/schemas/CompletionKind" + }, "prefix": { "default": "", "type": "string" + }, + "request_id": { + "type": [ + "string", + "null" + ] } }, "required": [ @@ -7370,6 +7823,16 @@ "RuntimeWorkerCompletionsResult": { "additionalProperties": false, "properties": { + "context": { + "anyOf": [ + { + "$ref": "#/components/schemas/CompletionContext" + }, + { + "type": "null" + } + ] + }, "diagnostics": { "default": [], "items": { @@ -7378,13 +7841,23 @@ "type": "array" }, "entries": { - "items": true, + "items": { + "$ref": "#/components/schemas/CompletionEntry" + }, "type": "array" }, - "kind": true, + "kind": { + "$ref": "#/components/schemas/CompletionKind" + }, "prefix": { "type": "string" }, + "request_id": { + "type": [ + "string", + "null" + ] + }, "runtime_id": { "type": "string" }, @@ -7415,7 +7888,9 @@ ] }, "segments": { - "items": true, + "items": { + "$ref": "#/components/schemas/Segment" + }, "type": [ "array", "null" @@ -7587,6 +8062,13 @@ "resource_key": { "type": "string" }, + "restore_observation_token": { + "description": "Opaque Runtime-owned observation fence used by explicit Restore intent.", + "type": [ + "string", + "null" + ] + }, "retention_state": { "default": "", "type": "string" @@ -8262,6 +8744,23 @@ ], "type": "object" }, + { + "description": "Explicitly selected, source-qualified Feature invocation. The host never\npromotes free text to this variant and validates it against the installed\ndeclaration before execution.", + "properties": { + "invocation": { + "$ref": "#/components/schemas/FeatureInvocation" + }, + "kind": { + "const": "feature_invoke", + "type": "string" + } + }, + "required": [ + "kind", + "invocation" + ], + "type": "object" + }, { "description": "Source-qualified request for the host to start a Flow before committing\nthis user input. Runtime resolves it through Workspace authority and\nreplaces it with the entered state's instructions before Worker-side\ninput resolution.", "properties": { @@ -9315,9 +9814,32 @@ ], "type": "object" }, + "SubjektivSubjectBehaviorUpdateRequest": { + "additionalProperties": false, + "properties": { + "behavior_md": { + "type": "string" + }, + "expected_behavior_revision": { + "format": "uint64", + "maximum": 9007199254740991, + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "expected_behavior_revision", + "behavior_md" + ], + "type": "object" + }, "SubjektivSubjectCreateRequest": { "additionalProperties": false, "properties": { + "behavior_md": { + "default": "", + "type": "string" + }, "role": { "type": "string" } @@ -9383,6 +9905,15 @@ "SubjektivSubjectResponse": { "additionalProperties": false, "properties": { + "behavior_md": { + "type": "string" + }, + "behavior_revision": { + "format": "uint64", + "maximum": 9007199254740991, + "minimum": 0, + "type": "integer" + }, "created_at": { "type": "string" }, @@ -9419,6 +9950,8 @@ "required": [ "id", "role", + "behavior_md", + "behavior_revision", "state", "store_revision", "created_at", @@ -12427,6 +12960,23 @@ ], "type": "string" }, + "WorkerRestoreRequest": { + "additionalProperties": false, + "description": "One explicit Restore intent. Retries retain both fields; conflicts require a\nfresh observation and a deliberate new intent, not automatic token replacement.", + "properties": { + "expected_observation_token": { + "type": "string" + }, + "request_id": { + "type": "string" + } + }, + "required": [ + "expected_observation_token", + "request_id" + ], + "type": "object" + }, "WorkerRestoreResponse": { "additionalProperties": false, "properties": { @@ -12631,6 +13181,13 @@ "resource_key": { "type": "string" }, + "restore_observation_token": { + "description": "Opaque Runtime-owned observation fence used by explicit Restore intent.", + "type": [ + "string", + "null" + ] + }, "retention_state": { "default": "", "type": "string" @@ -12878,6 +13435,7 @@ "WorkingDirectoryMaterializerKind": { "description": "Public Workdir materializer classification.\n\nThe value identifies stable materialization provenance without exposing a\nprovider path, Runtime handle, or session identity.", "enum": [ + "logical_workspace_config", "runtime_git_clone", "client_hosted_external" ], @@ -12975,6 +13533,31 @@ "WorkingDirectorySource": { "description": "Public Workdir source identity. Provider routing details and host paths are intentionally absent.", "oneOf": [ + { + "additionalProperties": false, + "properties": { + "access": { + "$ref": "#/components/schemas/WorkspaceConfigAccess" + }, + "content_path": { + "type": "string" + }, + "kind": { + "const": "workspace_config", + "type": "string" + }, + "purpose": { + "type": "string" + } + }, + "required": [ + "kind", + "access", + "content_path", + "purpose" + ], + "type": "object" + }, { "additionalProperties": false, "properties": { @@ -13173,6 +13756,103 @@ }, "type": "array" }, + "WorkspaceConfigAccess": { + "enum": [ + "read_only", + "read_write" + ], + "type": "string" + }, + "WorkspaceConfigApiError": { + "additionalProperties": false, + "properties": { + "classification": { + "$ref": "#/components/schemas/WorkspaceConfigFailureClassification" + }, + "code": { + "type": "string" + }, + "message": { + "type": "string" + }, + "status": { + "format": "uint16", + "maximum": 65535, + "minimum": 0, + "type": "integer" + } + }, + "required": [ + "status", + "code", + "message", + "classification" + ], + "type": "object" + }, + "WorkspaceConfigFailureClassification": { + "enum": [ + "not_committed", + "unknown" + ], + "type": "string" + }, + "WorkspaceConfigGrantCreateRequest": { + "additionalProperties": false, + "properties": { + "access": { + "$ref": "#/components/schemas/WorkspaceConfigAccess" + }, + "runtime_id": { + "type": "string" + }, + "worker_id": { + "type": "string" + } + }, + "required": [ + "runtime_id", + "worker_id", + "access" + ], + "type": "object" + }, + "WorkspaceConfigGrantResponse": { + "additionalProperties": false, + "properties": { + "access": { + "$ref": "#/components/schemas/WorkspaceConfigAccess" + }, + "grant_id": { + "type": "string" + }, + "revoked": { + "type": "boolean" + }, + "runtime_id": { + "type": "string" + }, + "worker_id": { + "type": "string" + }, + "working_directory_id": { + "type": "string" + }, + "workspace_id": { + "type": "string" + } + }, + "required": [ + "grant_id", + "workspace_id", + "runtime_id", + "worker_id", + "working_directory_id", + "access", + "revoked" + ], + "type": "object" + }, "WorkspaceConfigSchemaBundle": { "additionalProperties": false, "properties": { @@ -14794,7 +15474,7 @@ "info": { "title": "Yoi Server API", "version": "0.1.0", - "x-yoi-source-digest": "sha256:f9f0f8918dbcce386e94d5368fa938f02565ca98f9f67f07537bd88ed8c49870" + "x-yoi-source-digest": "sha256:fd06f382c6d33a5c3cc98ef9e264545923c53fa95a9be555a85680f98c6065da" }, "openapi": "3.1.0", "paths": { @@ -17367,6 +18047,16 @@ "style": "simple" } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkerRestoreRequest" + } + } + }, + "required": true + }, "responses": { "200": { "content": { @@ -25784,6 +26474,16 @@ "style": "form" } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkerRestoreRequest" + } + } + }, + "required": true + }, "responses": { "200": { "content": { @@ -29131,6 +29831,123 @@ ] } }, + "/api/w/{workspace_id}/subjektiv/subjects/{subject_id}/behavior": { + "patch": { + "operationId": "subjektiv_subject_behavior_update", + "parameters": [ + { + "explode": false, + "in": "path", + "name": "workspace_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/string" + }, + "style": "simple" + }, + { + "explode": false, + "in": "path", + "name": "subject_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/string" + }, + "style": "simple" + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SubjektivSubjectBehaviorUpdateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SubjektivSubjectResponse" + } + } + }, + "description": "Successful response" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RepositoryApiError" + } + } + }, + "description": "Error response" + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RepositoryApiError" + } + } + }, + "description": "Error response" + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RepositoryApiError" + } + } + }, + "description": "Error response" + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RepositoryApiError" + } + } + }, + "description": "Error response" + }, + "409": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RepositoryApiError" + } + } + }, + "description": "Error response" + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/RepositoryApiError" + } + } + }, + "description": "Error response" + } + }, + "security": [ + { + "bearerAuth": [] + }, + { + "browserSession": [] + } + ] + } + }, "/api/w/{workspace_id}/subjektiv/subjects/{subject_id}/memories": { "get": { "operationId": "subjektiv_memory_list", @@ -34518,6 +35335,220 @@ ] } }, + "/api/w/{workspace_id}/workspace-config-grants": { + "post": { + "operationId": "workspace_config_grant_create", + "parameters": [ + { + "explode": false, + "in": "path", + "name": "workspace_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/string" + }, + "style": "simple" + } + ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkspaceConfigGrantCreateRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkspaceConfigGrantResponse" + } + } + }, + "description": "Successful response" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkspaceConfigApiError" + } + } + }, + "description": "Error response" + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkspaceConfigApiError" + } + } + }, + "description": "Error response" + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkspaceConfigApiError" + } + } + }, + "description": "Error response" + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkspaceConfigApiError" + } + } + }, + "description": "Error response" + }, + "409": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkspaceConfigApiError" + } + } + }, + "description": "Error response" + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkspaceConfigApiError" + } + } + }, + "description": "Error response" + } + }, + "security": [ + { + "bearerAuth": [] + }, + { + "browserSession": [] + } + ] + } + }, + "/api/w/{workspace_id}/workspace-config-grants/{grant_id}": { + "delete": { + "operationId": "workspace_config_grant_revoke", + "parameters": [ + { + "explode": false, + "in": "path", + "name": "workspace_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/string" + }, + "style": "simple" + }, + { + "explode": false, + "in": "path", + "name": "grant_id", + "required": true, + "schema": { + "$ref": "#/components/schemas/string" + }, + "style": "simple" + } + ], + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkspaceConfigGrantResponse" + } + } + }, + "description": "Successful response" + }, + "400": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkspaceConfigApiError" + } + } + }, + "description": "Error response" + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkspaceConfigApiError" + } + } + }, + "description": "Error response" + }, + "403": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkspaceConfigApiError" + } + } + }, + "description": "Error response" + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkspaceConfigApiError" + } + } + }, + "description": "Error response" + }, + "409": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkspaceConfigApiError" + } + } + }, + "description": "Error response" + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkspaceConfigApiError" + } + } + }, + "description": "Error response" + } + }, + "security": [ + { + "bearerAuth": [] + }, + { + "browserSession": [] + } + ] + } + }, "/api/workers": { "get": { "operationId": "workspace_worker_list_alias", diff --git a/resources/profiles/subjektiv-memory-consolidation.dcdl b/resources/profiles/subjektiv-memory-consolidation.dcdl index 5a0797da6..440014548 100644 --- a/resources/profiles/subjektiv-memory-consolidation.dcdl +++ b/resources/profiles/subjektiv-memory-consolidation.dcdl @@ -3,13 +3,13 @@ import "./base.dcdl" // { description = "Delegated subject Memory staging consolidation profile."; scope = "workspace_read"; model = { ref = "codex-oauth/gpt-6-luna"; }; - engine = { reasoning = "medium"; }; + engine = { reasoning = "medium"; instruction = "internal.subjektiv_memory_consolidation_system"; }; web = { enabled = false; }; feature = { task = { enabled = false; }; memory = { enabled = false; staging_tools = false; }; - subjektiv = { enabled = true; extraction = { enabled = false; }; }; + subjektiv = { enabled = true; consolidation_tools = true; extraction = { enabled = false; }; }; web = { enabled = false; }; image = { enabled = false; }; sub_worker = { enabled = false; }; diff --git a/resources/prompts/catalog.dcdl b/resources/prompts/catalog.dcdl index d38cfda1a..e1039ba32 100644 --- a/resources/prompts/catalog.dcdl +++ b/resources/prompts/catalog.dcdl @@ -34,6 +34,7 @@ internalWorkingBoundariesSection = import "./internal/working_boundaries_section internalAgentsMdSection = import "./internal/agents_md_section.md"; internalResidentMemorySummarySection = import "./internal/resident_memory_summary_section.md"; internalResidentMemoryRestoreSection = import "./internal/resident_memory_restore_section.md"; +internalSubjektivResidentContext = import "./internal/subjektiv_resident_context.md"; internalSubWorkerSpawnToolDescription = import "./internal/sub_worker_spawn_tool_description.md"; panelOrchestratorIdleQueueNotice = import "./panel/orchestrator_idle_queue_notice.md"; in @@ -74,6 +75,7 @@ in agents_md_section = internalAgentsMdSection.content; resident_memory_summary_section = internalResidentMemorySummarySection.content; resident_memory_restore_section = internalResidentMemoryRestoreSection.content; + subjektiv_resident_context = internalSubjektivResidentContext.content; worker_orchestration_guidance_section = commonWorkerOrchestration.content; sub_worker_spawn_tool_description = internalSubWorkerSpawnToolDescription.content; }; diff --git a/resources/prompts/internal/resident_memory_restore_section.md b/resources/prompts/internal/resident_memory_restore_section.md index fc8a22907..abd982563 100644 --- a/resources/prompts/internal/resident_memory_restore_section.md +++ b/resources/prompts/internal/resident_memory_restore_section.md @@ -5,12 +5,12 @@ This restore-boundary record supersedes every resident memory summary embedded earlier in the system prompt or session history. Treat only the current state below as resident context; it is background context, not a user request. {% if surface_ready %} -### Current ready surface +### Current resident context {% if summary %} {{ summary }} {% else %} -The current ready resident memory surface is intentionally empty. +The current resident context is intentionally empty. {% endif %} {% else %} No current ready resident memory surface is available. Ignore earlier resident memory summaries; confirmed Memory remains available through explicit recall tools. diff --git a/resources/prompts/internal/subjektiv_memory_consolidation_system.md b/resources/prompts/internal/subjektiv_memory_consolidation_system.md index 7cb150dc7..2ca32f404 100644 --- a/resources/prompts/internal/subjektiv_memory_consolidation_system.md +++ b/resources/prompts/internal/subjektiv_memory_consolidation_system.md @@ -1,6 +1,6 @@ # Memory staging consolidater -You are the delegated internal Memory staging consolidater for one subject. +You are the Backend Job Memory staging consolidater for one subject. Your job is to consume that subject's pending Memory candidates through tools. The host binds every tool call to the subject; never infer or provide a subject identifier yourself. @@ -26,7 +26,10 @@ Rules: - `source_candidate_ids` are host-derived from the applied candidate. Do not attempt to author them. - `derived_from` must contain only exact existing Memory revision refs returned by reads. - Applied responses return affected refs with `{memory_id, revision, operation}`. `duplicate` and `already_covered` may reference exact existing revisions; `discarded` and `invalid` require a concrete reason. -- Surface generation is a separate clean-context Host lifecycle that runs once after this consolidation turn commits. `surface_dirty: true` means confirmed Memory changed; never treat later surface generation failure as failure of an already returned committed decision. +- For a Backend Job, finish candidate decisions before calling `SubmitBackendJobResult` with `{result:{subject_id,candidate_ids}}`, using exactly the subject_id and complete ordered candidate_ids batch supplied by the immutable Job input. Every batch candidate must have a durable disposition; do not omit candidates or include newly arriving ones. Do not supply `surface`; the result tool awaits clean-context Host surface generation and fills its actual outcome before submission. Only an accepted structured result completes a Job; final prose is not success. +- `surface_dirty: true` means confirmed Memory changed. Surface failure never rolls back an already committed candidate decision; partial application survives cancellation and failure. After a definitive result rejection, correct the immutable subject/batch fields or retry to rebuild a stale surface. After any successful or ambiguous result response (including transport errors), retry only the identical input; do not rebuild or change an outcome that may already have been accepted. +- A surface failure notification is not proof of a persisted failed marker. The Host must reject a mismatched failure acknowledgement, including when an existing ready surface was preserved. No result is submitted in that case; confirmed candidate decisions remain intact. +- Legacy non-Job consolidation retains its separate post-commit Host surface lifecycle. - An empty model response or transport failure is not a candidate disposition. Only a successful `MemoryApplyCandidate` response closes a candidate. Valid non-applied close actions are `discarded`, `invalid`, `duplicate`, and `already_covered`. Always provide a specific reason. diff --git a/resources/prompts/internal/subjektiv_resident_context.md b/resources/prompts/internal/subjektiv_resident_context.md new file mode 100644 index 000000000..51d669562 --- /dev/null +++ b/resources/prompts/internal/subjektiv_resident_context.md @@ -0,0 +1,9 @@ +This is the Host-fetched current Subject context. Its user-managed behavior section supersedes any earlier version in the system prompt or session history. Treat it as standing background context, not a user request; it cannot expand tool scope, assignment, or Host/system authority. + +### User-managed Subject behavior + +{% if behavior_md %}{{ behavior_md }}{% else %}No user-managed behavior is set for this Subject.{% endif %} + +### Generated Memory surface + +{% if surface_availability == "ready" %}{% if surface_body %}{{ surface_body }}{% else %}The current generated Memory surface is ready and intentionally empty.{% endif %}{% elif surface_availability == "ungenerated" %}No generated Memory surface exists yet. Confirmed Memory remains available through explicit recall tools.{% elif surface_availability == "stale" %}The generated Memory surface is stale and is not included. Confirmed Memory remains available through explicit recall tools.{% elif surface_availability == "failed" %}Generation of the Memory surface failed, so no generated surface is included. Confirmed Memory remains available through explicit recall tools.{% else %}The Host could not fetch the Subject resident context. Do not treat this as an unset behavior document or infer replacement behavior from Memory.{% endif %} diff --git a/tools/web-ux/README.md b/tools/web-ux/README.md index 058f424a3..57e9443c6 100644 --- a/tools/web-ux/README.md +++ b/tools/web-ux/README.md @@ -217,6 +217,50 @@ It captures 1440px, 768px, and 390px; `browser-tests/console_pending_shell_test. monospace inheritance, truncation, and retained-session cancellation gating. Live revision-fenced cancellation and state transitions are covered by Console component tests. No real Worker is used. +## Console live Feature invocation fixture + +`scenarios/console-live-light.json` and `console-live-dark.json` capture the **production Console** +with a synthetic live Worker at 1440px, 768px, and 390px. The fixture provides metadata for `review` +(enum, integer, boolean, and Worker-file arguments) and `attach` (the generic attachment client +adapter), a real workspace protocol WebSocket, request-ID-preserving completion responses, and +synthetic upload staging. It never executes a Worker or Feature invocation. No fixture CSS, shell, +or editor overrides are used. + +Build production sources first, optionally from an isolated copy under `target/web-ux`, then select +the build explicitly. From `tools/web-ux`: + +```sh +export WEB_UX_BUILD_ROOT='' +deno task web-ux capture --scenario scenarios/console-live-light.json --output ../../target/web-ux --run-id console-live-light +deno task web-ux capture --scenario scenarios/console-live-dark.json --output ../../target/web-ux --run-id console-live-dark +deno test --allow-env --allow-net --allow-read --allow-write --allow-run --allow-sys browser-tests/console_live_shell_test.ts browser-tests/console_live_admission_shell_test.ts +``` + +`WEB_UX_LIFECYCLE_OUTPUT` optionally places browser interaction screenshots and JSON evidence at an +explicit artifact path. Tests consume an existing build; they do not rebuild or mutate Web sources. +The lifecycle test covers argument completion, editing/removing/undoing typed chips, native picker +cancellation, generic-adapter staging, literal unselected slash prose, typed Submit values, and +abandoned-file DELETE. Admission tests verify rejection preserves drafts and staged files, +transport send alone cannot clear the draft, authoritative acceptance transfers ownership, and +failed/in-flight-cancelled uploads cannot Submit. A retry/late-success matrix verifies stable upload +IDs, deleted reservation non-resurrection, and abandoned resource cleanup at accepted draft boundaries. +Intentional upload HTTP503 browser errors are recorded separately from unexpected failures. +The Chromium picker helper awaits interception acknowledgement before real gestures (not a timeout +extension or a replacement input); diagnostics retain user activation, input connectivity, and DOM +lifecycle evidence. + +The localhost-only fixture control endpoint sets deterministic `submit: accept | reject | hold` +and `upload: normal | fail | hold` modes; `accept_pending: true` releases held admission responses and +`release_uploads: true` delivers real successful responses for held uploads. +`/fixture-state` exposes actual received protocol methods, HTTP requests, emitted admission events, +upload attempts (including retry IDs), completed uploads, and DELETE IDs for assertions. Only synthetic data is recorded. + +`console-live-baseline-light.json` and `console-live-baseline-dark.json` use the same production +shell/data and capture initial plus unselected literal slash text. They support a pre-Feature HEAD +build, where no implicit Feature completion or typed chip exists. Initial and `slash-plain` capture +keys are shared with the current scenarios for like-for-like comparison; Feature-specific states +are explicitly new behavior, not fabricated baseline controls. + ## Adding a scenario 1. Name the concrete user task and expected data state; do not write “looks correct”. diff --git a/tools/web-ux/browser-tests/console_live_admission_shell_test.ts b/tools/web-ux/browser-tests/console_live_admission_shell_test.ts new file mode 100644 index 000000000..afabf87a8 --- /dev/null +++ b/tools/web-ux/browser-tests/console_live_admission_shell_test.ts @@ -0,0 +1,443 @@ +/// +import { assert, assertEquals } from "@std/assert"; +import { dirname, fromFileUrl, join } from "@std/path"; +import { chromium, type Page } from "playwright"; +import { Buffer } from "node:buffer"; +import { + observePicker, + pickerDiagnostics, + prepareNativePicker, +} from "./console_live_picker_diagnostics.ts"; + +const root = join(dirname(fromFileUrl(import.meta.url)), "../../.."); +const route = "/w/console-live-review/workers/W-901-console-live-fixture/console"; +const output = Deno.env.get("WEB_UX_LIFECYCLE_OUTPUT") ?? + join(root, "target/web-ux/console-live-lifecycle"); +type State = { + methods: { + method: string; + params?: { + input?: import("../../../web/workspace/src/lib/generated/protocol.ts").Segment[]; + submission_request_id?: string; + }; + }[]; + events: { event: string }[]; + requests: { method: string; path: string }[]; + uploads: string[]; + upload_attempts: string[]; + deletes: string[]; + held_uploads: string[]; +}; +const state = async (base: string): Promise => + await (await fetch(base + "/fixture-state")).json(); +const control = async (base: string, value: unknown): Promise => { + const response = await fetch(base + "/fixture-control", { + method: "POST", + body: JSON.stringify(value), + }); + assert(response.ok); + await response.body?.cancel(); +}; +async function eventually(base: string, predicate: (value: State) => boolean): Promise { + const deadline = Date.now() + 5000; + do { + const value = await state(base); + if (predicate(value)) return value; + await new Promise((resolve) => setTimeout(resolve, 25)); + } while (Date.now() < deadline); + throw new Error("Fixture evidence did not reach expected state"); +} +type FixtureBody = ( + page: Page, + base: string, + errors: string[], + caseOutput: string, +) => Promise; +async function fixture(body: FixtureBody, matrix = false): Promise { + for (const theme of matrix ? ["light", "dark"] as const : ["light"] as const) { + for (const width of matrix ? [1440, 768, 390] : [1440]) await fixtureCase(body, theme, width); + } +} +async function fixtureCase( + body: FixtureBody, + theme: "light" | "dark", + width: number, +): Promise { + const caseOutput = join(output, `${theme}-${width}`); + const listener = Deno.listen({ hostname: "127.0.0.1", port: 0 }); + const port = (listener.addr as Deno.NetAddr).port; + listener.close(); + const base = `http://127.0.0.1:${port}`; + const server = new Deno.Command(Deno.execPath(), { + args: [ + "run", + "--allow-net", + "--allow-read", + join(root, "tools/web-ux/browser-tests/console_live_fixture_server.ts"), + String(port), + Deno.env.get("WEB_UX_BUILD_ROOT") ?? join(root, "web/workspace/build"), + ], + stdout: "null", + stderr: "inherit", + }).spawn(); + try { + const deadline = Date.now() + 10000; + let started = false; + while (!started && Date.now() < deadline) { + try { + const response = await fetch(base + "/health"); + started = response.ok; + await response.body?.cancel(); + } catch { /* bounded owned startup */ } + if (!started) await new Promise((resolve) => setTimeout(resolve, 25)); + } + assert(started); + const browser = await chromium.launch({ headless: true }); + try { + const page = await browser.newPage({ + viewport: { width, height: width === 390 ? 844 : 900 }, + colorScheme: theme, + reducedMotion: "reduce", + }); + page.setDefaultTimeout(5000); + const errors: string[] = []; + page.on("pageerror", (error) => errors.push(String(error))); + page.on("console", (message) => { + if (message.type() === "error") errors.push(message.text()); + }); + await prepareNativePicker(page); + await observePicker(page); + await page.goto(base + route); + await page.locator(".cm-content[contenteditable=true]").waitFor(); + await page.getByText("Ready. Choose a Feature", { exact: false }).waitFor(); + await Deno.mkdir(caseOutput, { recursive: true }); + try { + await body(page, base, errors, caseOutput); + await Deno.writeTextFile( + join(caseOutput, `picker-${crypto.randomUUID()}.json`), + JSON.stringify(await pickerDiagnostics(page), null, 2), + ); + } catch (error) { + const id = crypto.randomUUID(); + await page.screenshot({ path: join(caseOutput, `admission-failure-${id}.png`) }); + await Deno.writeTextFile( + join(caseOutput, `admission-failure-${id}.json`), + JSON.stringify( + { + error: String(error), + body: await page.locator("body").innerText(), + errors, + picker: await pickerDiagnostics(page), + fixture: await state(base), + }, + null, + 2, + ), + ); + throw error; + } + } finally { + await browser.close(); + } + } finally { + server.kill("SIGTERM"); + await server.status; + } +} +async function draft(page: Page, text: string): Promise { + await page.locator(".cm-content").click(); + await page.keyboard.press("Control+a"); + await page.keyboard.press("Backspace"); + await page.keyboard.type(text); +} +async function review(page: Page): Promise { + await draft(page, "/rev"); + await page.getByRole("option").filter({ hasText: "review" }).waitFor(); + await page.keyboard.press("Tab"); + await page.keyboard.press("Escape"); + await page.keyboard.type('mode="brief", count=2, enabled=true)'); + await page.locator(".composer-typed-chip").waitFor(); +} +async function attach(page: Page, name: string): Promise { + await draft(page, "/att"); + await page.getByRole("option").filter({ hasText: "attach" }).waitFor(); + const chooser = page.waitForEvent("filechooser"); + await page.keyboard.press("Enter"); + const nativeChooser = await chooser; + const diagnostics = await pickerDiagnostics(page) as { + kind: string; + active: boolean; + connected: boolean; + disabled?: boolean; + }[]; + const click = diagnostics.filter((record) => record.kind === "input.click").at(-1); + assert( + click?.active && click.connected && !click.disabled, + "native picker must run synchronously with active real gesture and connected enabled input", + ); + await nativeChooser.setFiles({ + name, + mimeType: "text/markdown", + buffer: Buffer.from("Synthetic fixture document"), + }); +} +Deno.test("production leading slash remains responsive after loop fix", () => + fixture(async (page, base, errors, caseOutput) => { + await draft(page, "/"); + await page.getByRole("option").filter({ hasText: "review" }).waitFor(); + assertEquals( + (await state(base)).methods.filter((method) => method.method !== "list_completions"), + [], + ); + assertEquals((await state(base)).uploads, []); + assertEquals(errors, []); + await Deno.writeTextFile( + join(caseOutput, "slash-fix-result.json"), + JSON.stringify({ status: "pass", fixture: await state(base), errors }, null, 2), + ); + })); +Deno.test("production Submit rejection retains typed draft and only authoritative acceptance clears it", () => + fixture(async (page, base, errors, caseOutput) => { + await review(page); + await attach(page, "admission.md"); + await page.locator(".composer-typed-chip").waitFor(); + // Keep the uploaded atom and add a completion-selected Feature next to it. + await page.keyboard.press("Control+End"); + await page.keyboard.type(" /rev"); + await page.getByRole("option").filter({ hasText: "review" }).waitFor(); + await page.keyboard.press("Tab"); + await page.keyboard.press("Escape"); + await page.keyboard.type('mode="brief", count=2, enabled=true)'); + await page.waitForFunction(() => + document.querySelectorAll(".composer-typed-chip").length === 2 + ); + await control(base, { submit: "reject" }); + await page.keyboard.press("Control+Enter"); + await page.getByText("Synthetic admission rejection; draft was not accepted.", { exact: false }) + .waitFor(); + const rejected = await eventually( + base, + (value) => value.events.some((event) => event.event === "submission_rejected"), + ); + assertEquals(await page.locator(".composer-typed-chip").count(), 2); + assertEquals(rejected.deletes, [], "rejection must keep staged file ownership"); + await page.screenshot({ path: join(caseOutput, "admission-rejected.png") }); + await control(base, { submit: "hold" }); + await page.locator(".cm-content[contenteditable=true]").waitFor(); + await page.locator(".cm-content").focus(); + await page.keyboard.press("Control+Enter"); + const pending = await eventually( + base, + (value) => value.methods.filter((method) => method.method === "submit").length === 2, + ); + assertEquals( + await page.locator(".composer-typed-chip").count(), + 2, + "transport send is not admission", + ); + assertEquals(pending.deletes, []); + await page.screenshot({ path: join(caseOutput, "admission-pending.png") }); + await control(base, { accept_pending: true }); + await page.waitForFunction(() => + document.querySelectorAll(".composer-typed-chip").length === 0 + ); + const accepted = await eventually( + base, + (value) => value.events.some((event) => event.event === "submission_accepted"), + ); + assertEquals(accepted.deletes, [], "accepted file ownership transfers; no staging DELETE"); + const submissions = accepted.methods.filter((method) => method.method === "submit"); + assertEquals(submissions.length, 2); + assert( + submissions.every((method) => + method.params?.input?.some((segment) => segment.kind === "uploaded_file") + ), + ); + assert( + submissions.every((method) => + method.params?.input?.some((segment) => segment.kind === "feature_invoke") + ), + ); + assertEquals( + submissions[0].params?.input, + submissions[1].params?.input, + "retry preserves structured invocation IDs and values", + ); + assertEquals(errors, []); + await page.screenshot({ path: join(caseOutput, "admission-accepted.png") }); + await Deno.writeTextFile( + join(caseOutput, "admission-result.json"), + JSON.stringify({ status: "pass", rejected, pending, accepted, errors }, null, 2), + ); + }, true)); +Deno.test("production attachment upload failure and in-flight cancellation never Submit and record DELETE", () => + fixture(async (page, base, errors, caseOutput) => { + await control(base, { upload: "fail" }); + await attach(page, "failure.md"); + await page.getByText("Upload failed (503).", { exact: true }).waitFor(); + await page.screenshot({ path: join(caseOutput, "upload-failed.png") }); + await page.getByRole("button", { name: "Remove failure.md", exact: true }).click(); + const failed = await state(base); + assertEquals(failed.uploads, []); + assertEquals(failed.methods.filter((method) => method.method === "submit"), []); + await control(base, { upload: "hold" }); + await attach(page, "cancel.md"); + await eventually(base, (value) => value.held_uploads.length === 1); + await page.screenshot({ path: join(caseOutput, "upload-pending.png") }); + await page.getByRole("button", { name: "Remove cancel.md", exact: true }).click(); + const cancelled = await eventually( + base, + (value) => value.deletes.length > failed.deletes.length && value.held_uploads.length === 0, + ); + assertEquals(cancelled.uploads, []); + assertEquals(cancelled.methods.filter((method) => method.method === "submit"), []); + assertEquals(await page.locator(".composer-typed-chip").count(), 0); + await page.screenshot({ path: join(caseOutput, "upload-cancelled.png") }); + assert( + cancelled.requests.filter((request) => request.method === "DELETE").every((request) => + request.path.includes("/attachment-uploads/") + ), + ); + // The fixture's deliberate HTTP503 emits a browser resource error, not a page exception. + assert(errors.every((error) => /503/.test(error)), JSON.stringify(errors)); + await Deno.writeTextFile( + join(caseOutput, "upload-failure-cancellation-result.json"), + JSON.stringify({ status: "pass", failed, cancelled, expectedErrors: errors }, null, 2), + ); + }, true)); + +Deno.test("production upload retry retains upload ID and deleted pending upload cancels without blocking Submit", () => + fixture(async (page, base, errors, caseOutput) => { + await control(base, { upload: "fail" }); + await attach(page, "retry.md"); + await page.getByText("Upload failed (503).", { exact: true }).waitFor(); + const failed = await state(base); + assertEquals(failed.upload_attempts.length, 1); + await control(base, { upload: "normal" }); + await page.getByRole("button", { name: "Retry", exact: true }).click(); + await page.locator(".composer-typed-chip").filter({ hasText: "Attached retry.md" }).waitFor(); + const retried = await state(base); + assertEquals( + retried.upload_attempts, + [failed.upload_attempts[0], failed.upload_attempts[0]], + "retry keeps staging identity", + ); + assertEquals(retried.uploads, [failed.upload_attempts[0]]); + await page.screenshot({ path: join(caseOutput, "upload-retried.png") }); + // Submit authoritative acceptance is covered separately; this draft is abandoned. + await draft(page, "deleted upload"); + await page.keyboard.press("Escape"); + // Clearing/accepting this text boundary releases the abandoned staged resource. + await page.keyboard.press("Control+Enter"); + const deleted = await eventually(base, (value) => value.deletes.includes(retried.uploads[0])); + await control(base, { upload: "hold" }); + await attach(page, "late.md"); + const held = await eventually(base, (value) => value.held_uploads.length === 1); + const lateId = held.held_uploads[0]; + await page.locator(".composer-typed-chip").filter({ hasText: "Uploading late.md" }).waitFor(); + // Keyboard deletion cancels staging; a separate Remove action is not needed. + await page.locator(".cm-content").focus(); + await page.keyboard.press("Control+End"); + await page.keyboard.press("Backspace"); + assertEquals(await page.locator(".composer-typed-chip").count(), 0); + await page.screenshot({ path: join(caseOutput, "late-upload-deleted.png") }); + // Submit immediately, without waiting for upload completion or cancellation HTTP. + await page.keyboard.type("finish late deletion"); + await page.keyboard.press("Control+Enter"); + const completed = await eventually( + base, + (value) => + value.deletes.includes(lateId) && value.held_uploads.length === 0 && + value.methods.filter((method) => method.method === "submit").length === 2, + ); + assertEquals(await page.locator(".composer-attachments").count(), 0); + assertEquals( + completed.methods.filter((method) => method.method === "submit").at(-1)?.params?.input, + [{ kind: "text", content: "finish late deletion" }], + ); + // A delayed fixture release after cancellation cannot revive a resource or intent. + // Component tests separately inject a late completion callback against the tombstone. + await page.waitForFunction(() => document.querySelector(".cm-content")?.textContent === ""); + await control(base, { release_uploads: true }); + const late = await state(base); + assert(!late.uploads.includes(lateId)); + assertEquals(late.methods.filter((method) => method.method === "submit").length, 2); + assertEquals(await page.locator(".composer-typed-chip").count(), 0); + await page.screenshot({ path: join(caseOutput, "late-upload-completed.png") }); + assert(errors.every((error) => /503/.test(error)), JSON.stringify(errors)); + await Deno.writeTextFile( + join(caseOutput, "retry-late-result.json"), + JSON.stringify( + { status: "pass", failed, retried, deleted, held, late, completed, expectedErrors: errors }, + null, + 2, + ), + ); + }, true)); + +Deno.test("production pending-upload Delete, range, cut and replacement cancel staging and allow immediate text Submit", () => + fixture(async (page, base, errors, caseOutput) => { + await control(base, { upload: "hold" }); + const results = []; + for (const [index, operation] of ["Delete", "range", "cut", "replace"].entries()) { + await attach(page, `keyboard-${operation}.md`); + const held = await eventually(base, (value) => value.held_uploads.length === 1); + const uploadId = held.held_uploads[0]; + await page.locator(".composer-typed-chip").filter({ + hasText: `Uploading keyboard-${operation}.md`, + }).waitFor(); + await page.screenshot({ path: join(caseOutput, `keyboard-${operation}-pending.png`) }); + await page.locator(".cm-content").focus(); + if (operation === "Delete") { + await page.keyboard.press("Control+Home"); + await page.keyboard.press("Delete"); + } else { + await page.keyboard.press("Control+a"); + if (operation === "range") await page.keyboard.press("Backspace"); + else if (operation === "cut") await page.keyboard.press("Control+x"); + else await page.keyboard.type("replacement "); + } + assertEquals(await page.locator(".composer-typed-chip").count(), 0); + await page.screenshot({ path: join(caseOutput, `keyboard-${operation}-deleted.png`) }); + const text = `${operation === "replace" ? "replacement " : ""}text after ${operation}`; + await page.keyboard.type(`text after ${operation}`); + await page.keyboard.press("Control+Enter"); + const accepted = await eventually( + base, + (value) => + value.deletes.includes(uploadId) && value.held_uploads.length === 0 && + value.methods.filter((method) => method.method === "submit").length === index + 1, + ); + assertEquals( + accepted.methods.filter((method) => method.method === "submit").at(-1)?.params?.input, + [{ kind: "text", content: text }], + ); + assertEquals(await page.locator(".composer-attachments").count(), 0); + await page.waitForFunction(() => document.querySelector(".cm-content")?.textContent === ""); + await control(base, { release_uploads: true }); + assert(!(await state(base)).uploads.includes(uploadId)); + await page.screenshot({ path: join(caseOutput, `keyboard-${operation}-accepted.png`) }); + results.push({ operation, held, accepted }); + } + assertEquals(errors, []); + await Deno.writeTextFile( + join(caseOutput, "keyboard-cancellation-result.json"), + JSON.stringify({ status: "pass", results, errors }, null, 2), + ); + }, true)); + +Deno.test("production selected invocation preserves multiple positional typed arguments", () => + fixture(async (page, base, errors, caseOutput) => { + await draft(page, "/rev"); + await page.getByRole("option").filter({ hasText: "review" }).waitFor(); + await page.keyboard.press("Tab"); + await page.keyboard.press("Escape"); + await page.keyboard.type('"brief", 2, true)'); + await page.locator(".composer-typed-chip").waitFor({ timeout: 2000 }); + assertEquals((await state(base)).methods.filter((method) => method.method === "submit"), []); + assertEquals(errors, []); + await Deno.writeTextFile( + join(caseOutput, "positional-result.json"), + JSON.stringify({ status: "pass", fixture: await state(base), errors }, null, 2), + ); + })); diff --git a/tools/web-ux/browser-tests/console_live_fixture_server.ts b/tools/web-ux/browser-tests/console_live_fixture_server.ts new file mode 100644 index 000000000..78705e73c --- /dev/null +++ b/tools/web-ux/browser-tests/console_live_fixture_server.ts @@ -0,0 +1,470 @@ +import { extname, join, normalize } from "jsr:@std/path@1.1.4"; + +// Synthetic authority, real production shell and protocol transport. No UI overrides. +const port = Number(Deno.args[0]); +const buildRoot = Deno.args[1] ?? Deno.env.get("WEB_UX_BUILD_ROOT"); +if (!Number.isInteger(port) || !buildRoot) { + throw new Error("usage: console_live_fixture_server.ts "); +} +const workspaceId = "console-live-review"; +const runtimeId = "runtime-a"; +const workerId = "worker-a"; +const workerPath = `/api/w/${workspaceId}/runtimes/${runtimeId}/workers/${workerId}`; +const json = (value: unknown, status = 200) => + new Response(JSON.stringify(value), { + status, + headers: { "content-type": "application/json; charset=utf-8" }, + }); +const invocations: + import("../../../web/workspace/src/lib/generated/protocol.ts").FeatureInvocationDescriptor[] = [ + { + identity: "fixture.review", + name: "review", + aliases: ["rv"], + display_name: "Review", + description: "Review a synthetic document without running a real Worker.", + syntax: "parenthesized", + arguments: [ + { + name: "mode", + position: 0, + required: true, + value_type: { kind: "enum", values: ["brief", "thorough"] }, + completion: { kind: "static", values: ["brief", "thorough"] }, + description: "Review detail", + }, + { + name: "count", + position: 1, + required: false, + value_type: { kind: "integer" }, + completion: { kind: "none" }, + description: "Number of examples", + }, + { + name: "enabled", + position: 2, + required: false, + value_type: { kind: "boolean" }, + completion: { kind: "static", values: ["true", "false"] }, + description: "Enable review", + }, + { + name: "path", + position: 3, + required: false, + value_type: { kind: "worker_file" }, + completion: { kind: "worker_file" }, + description: "Worker document", + }, + ], + }, + { + identity: "fixture.attach", + name: "attach", + aliases: [], + display_name: "Attach file", + description: "Stage a client file using the attachment adapter.", + syntax: "parenthesized", + arguments: [{ + name: "file", + position: 0, + required: true, + value_type: { kind: "client_file" }, + completion: { kind: "client_file" }, + }], + client_adapter: "attachment", + }, + ]; +const entries = [ + { + kind: "message", + entry_id: "user-1", + timestamp: 1, + provenance: "human_input", + role: "user", + content: [{ kind: "text", text: "Review the release notes before sending." }], + }, + { + kind: "message", + entry_id: "assistant-1", + timestamp: 2, + provenance: "model_output", + role: "assistant", + content: [{ + kind: "text", + text: + "Ready. Choose a Feature and inspect its arguments, or attach a document. Nothing runs until Submit.", + }], + }, +]; +const session = { + pending_submissions: { revision: 0, notification_count: 0, head_id: null, submissions: [] }, + entries, +}; +const state = { last_command_id: 0, state: { kind: "idle" } }; +const worker = { + worker_id: workerId, + runtime_id: runtimeId, + resource_key: "W-901", + display_name: "Console Live Fixture", + label: "Console Live Fixture", + availability: "observed", + state: "idle", + worker_state: state, + retention_state: "active", + host_id: "fixture-host", + profile: "default", + tags: [], + diagnostics: [], + workspace: { workspace_id: workspaceId, visibility: "workspace", identity: workspaceId }, + implementation: { kind: "builtin", display_hint: "fixture" }, + workdir_attachments: [], +}; +const snapshot = { + event: "snapshot", + data: { + session, + state, + greeting: { + worker_name: worker.label, + cwd: "/synthetic/workdir", + provider: "fixture", + model: "synthetic", + scope_summary: "synthetic read-only files", + tools: [], + context_window: 200000, + context_tokens: 2400, + }, + }, +}; +const methods: unknown[] = []; +const requests: { method: string; path: string }[] = []; +const events: unknown[] = []; +let submitMode: "accept" | "reject" | "hold" = "accept"; +let uploadMode: "normal" | "fail" | "hold" = "normal"; +const pendingAcceptances: (() => void)[] = []; +const cancelledUploads = new Set(); +const heldUploads = new Map void>(); +const releaseHeldUploads = new Map void>(); +const grants = new Map(); +const uploads: string[] = []; +const uploadAttempts: string[] = []; +const deletes: string[] = []; +let historyRequests = 0; +let nextSubscription = 0; +const mime: Record = { + ".html": "text/html; charset=utf-8", + ".js": "text/javascript; charset=utf-8", + ".css": "text/css; charset=utf-8", + ".json": "application/json", + ".svg": "image/svg+xml", + ".png": "image/png", + ".woff2": "font/woff2", + ".wasm": "application/wasm", +}; +async function staticResponse(pathname: string): Promise { + const relative = normalize(pathname.replace(/^\/+/, "") || "index.html"); + if (relative.startsWith("..")) return new Response("not found", { status: 404 }); + let path = join(buildRoot, relative); + try { + if ((await Deno.stat(path)).isDirectory) path = join(path, "index.html"); + return new Response(await Deno.readFile(path), { + headers: { "content-type": mime[extname(path)] ?? "application/octet-stream" }, + }); + } catch { + return new Response(await Deno.readFile(join(buildRoot, "index.html")), { + headers: { "content-type": mime[".html"] }, + }); + } +} +Deno.serve({ hostname: "127.0.0.1", port }, async (request) => { + const url = new URL(request.url); + if (url.pathname === "/health") return new Response("ok"); + if (url.pathname === "/fixture-state") { + return json({ + methods, + events, + requests, + uploads, + upload_attempts: uploadAttempts, + deletes, + held_uploads: [...heldUploads.keys()], + history_requests: historyRequests, + }); + } + if (url.pathname === "/fixture-control" && request.method === "POST") { + const control = await request.json(); + if (["accept", "reject", "hold"].includes(control.submit)) submitMode = control.submit; + if (["normal", "fail", "hold"].includes(control.upload)) uploadMode = control.upload; + if (control.accept_pending) { for (const accept of pendingAcceptances.splice(0)) accept(); } + if (control.release_uploads) { + for (const release of [...releaseHeldUploads.values()]) release(); + } + return json({ submitMode, uploadMode }); + } + if (url.pathname.startsWith("/api/")) { + requests.push({ method: request.method, path: url.pathname }); + } + if (url.pathname === "/api/workspaces") { + return json([{ + workspace_id: workspaceId, + owner_account_id: "owner", + display_name: "Console Live Review", + state: "active", + created_at: "2026-01-01T00:00:00Z", + updated_at: "2026-01-02T00:00:00Z", + }]); + } + if (url.pathname === `/api/w/${workspaceId}/workspace`) { + return json({ + workspace_id: workspaceId, + display_name: "Console Live Review", + record_authority: "fixture", + schema_version: 1, + auth: { + Passkey: { + rp_id: "127.0.0.1", + origin: url.origin, + public_base_url: url.origin, + cookie_name: "fixture", + }, + }, + permissions: { + manage_repositories: false, + manage_secrets: false, + manage_runtimes: false, + delete_workspace: false, + }, + extension_points: { + store: "fixture", + event_stream: { status: "ready", note: "fixture", diagnostics: [] }, + host_worker_bridge: { status: "ready", note: "fixture", diagnostics: [] }, + companion_console: { status: "ready", note: "fixture", diagnostics: [] }, + }, + }); + } + if (url.pathname === `/api/w/${workspaceId}/repositories`) { + return json({ workspace_id: workspaceId, items: [], source: "fixture", diagnostics: [] }); + } + if (url.pathname === `/api/w/${workspaceId}/working-directories`) { + return json({ workspace_id: workspaceId, items: [], diagnostics: [] }); + } + if (url.pathname === `/api/w/${workspaceId}/workers/W-901` || url.pathname === workerPath) { + return json(worker); + } + if (url.pathname === `${workerPath}/session`) return json({ availability: "live_protocol" }); + if (url.pathname === `${workerPath}/session/history`) { + historyRequests++; + return json({ + availability: "page", + page: { + session_id: "session-a", + lineage_id: "lineage-a", + compact_ancestor_lineage_ids: [], + turns: [{ turn_id: "user-1", entries }], + next_cursor: null, + has_more: false, + }, + }); + } + if (url.pathname === `${workerPath}/attachment-upload-grants` && request.method === "POST") { + const id = url.searchParams.get("upload_id")!; + grants.set(id, { + file_name: url.searchParams.get("file_name")!, + media_type: url.searchParams.get("media_type")!, + }); + return json({ upload_id: id, expires_at_ms: Date.now() + 60000 }); + } + if (url.pathname.startsWith(`${workerPath}/attachment-uploads/`)) { + const id = url.pathname.split("/").at(-1)!; + if (request.method === "DELETE") { + deletes.push(id); + grants.delete(id); + cancelledUploads.add(id); + heldUploads.get(id)?.(new Response(null, { status: 204 })); + heldUploads.delete(id); + releaseHeldUploads.delete(id); + return new Response(null, { status: 204 }); + } + const grant = grants.get(id); + if (!grant || request.method !== "PUT") return json({ error: "unknown_upload" }, 404); + uploadAttempts.push(id); + if (uploadMode === "fail") { + await request.body?.cancel(); + return json({ error: "synthetic_upload_failure" }, 503); + } + const bytes = await request.arrayBuffer(); + const hash = Array.from( + new Uint8Array(await crypto.subtle.digest("SHA-256", bytes)), + (byte) => byte.toString(16).padStart(2, "0"), + ).join(""); + const complete = () => { + uploads.push(id); + return json({ + file: { + artifact_id: id, + ...grant, + created_at_ms: Date.now(), + availability: "available", + byte_len: bytes.byteLength, + sha256: hash, + }, + }); + }; + if (uploadMode === "hold") { + if (cancelledUploads.has(id)) return new Response(null, { status: 204 }); + return await new Promise((resolve) => { + heldUploads.set(id, resolve); + releaseHeldUploads.set(id, () => { + heldUploads.delete(id); + releaseHeldUploads.delete(id); + resolve(complete()); + }); + }); + } + return complete(); + } + if (url.pathname.startsWith(`${workerPath}/attachments/`) && request.method === "DELETE") { + deletes.push(url.pathname.split("/").at(-1)!); + return new Response(null, { status: 204 }); + } + if ( + url.pathname === `/api/w/${workspaceId}/protocol/ws` && + request.headers.get("upgrade") === "websocket" + ) { + const { socket, response } = Deno.upgradeWebSocket(request); + const subscriptions = new Map(); + const send = (frame: unknown) => socket.send(JSON.stringify(frame)); + socket.onmessage = (event) => { + const frame = JSON.parse(String(event.data)); + if (frame.frame === "request" && frame.message.method === "subscribe_events") { + const { request_id, selector } = frame.message.params; + const subscription_id = `fixture-${++nextSubscription}`; + subscriptions.set(subscription_id, selector.topic); + send({ + protocol_version: 2, + frame: "response", + message: { + result: "subscribed", + payload: { + request_id, + subscription_id, + selector, + snapshot: selector.topic === "worker_protocol" + ? { topic: "worker_protocol", data: { worker_id: workerId, events: [snapshot] } } + : { topic: "workers", data: { workers: [] } }, + }, + }, + }); + } else if (frame.frame === "worker_protocol") { + const method = frame.message.method; + methods.push(method); + const subscription_id = frame.message.subscription_id; + if (subscriptions.get(subscription_id) !== "worker_protocol") return; + if (method.method === "list_completions") { + const { kind, prefix, request_id, context } = method.params; + let candidates: unknown[] = []; + if (kind === "feature") { + candidates = invocations.filter((descriptor) => + descriptor.name.startsWith(prefix) || + descriptor.aliases.some((alias) => alias.startsWith(prefix)) + ).map((invocation) => ({ + value: invocation.name, + is_dir: false, + description: invocation.description, + invocation, + })); + } else if (kind === "file") { + candidates = ["docs/", "docs/release-notes.md"].filter((value) => + value.startsWith(prefix) + ).map((value) => ({ value, is_dir: value.endsWith("/") })); + } else if (context?.invocation === "fixture.review") { + const descriptor = invocations[0].arguments.find((argument) => + argument.name === context.argument + ); + const values = !context.argument + ? invocations[0].arguments.map((argument) => argument.name + "=") + : descriptor?.completion.kind === "static" + ? descriptor.completion.values ?? [] + : context.argument === "path" + ? ["docs/release-notes.md", "docs/long-release-notes-for-responsive-validation.md"] + : []; + candidates = values.filter((value) => value.startsWith(prefix)).map((value) => ({ + value, + is_dir: false, + })); + } + send({ + protocol_version: 2, + frame: "event", + message: { + event: "event", + data: { + subscription_id, + payload: { + event: "worker_protocol", + data: { + worker_id: workerId, + event: { + event: "completions", + data: { + kind, + prefix, + ...(request_id ? { request_id } : {}), + ...(context ? { context } : {}), + entries: candidates, + }, + }, + }, + }, + }, + }, + }); + } else if (method.method === "submit") { + const emit = (event: unknown) => { + events.push(event); + if (socket.readyState !== WebSocket.OPEN) return; + send({ + protocol_version: 2, + frame: "event", + message: { + event: "event", + data: { + subscription_id, + payload: { event: "worker_protocol", data: { worker_id: workerId, event } }, + }, + }, + }); + }; + const submission_request_id = method.params.submission_request_id; + const accept = () => + emit({ + event: "submission_accepted", + data: { + submission_request_id, + submission_id: `accepted-${submission_request_id}`, + disposition: "started", + }, + }); + if (submitMode === "reject") { + emit({ + event: "submission_rejected", + data: { + submission_request_id, + message: "Synthetic admission rejection; draft was not accepted.", + }, + }); + } else if (submitMode === "hold") pendingAcceptances.push(accept); + else accept(); + } + // Admission responses are synthetic authority only. Never execute a Feature or Worker. + } + }; + return response; + } + if (url.pathname.startsWith("/api/")) { + return json({ error: "unhandled_fixture_request", path: url.pathname }, 404); + } + return await staticResponse(url.pathname); +}); diff --git a/tools/web-ux/browser-tests/console_live_picker_diagnostics.ts b/tools/web-ux/browser-tests/console_live_picker_diagnostics.ts new file mode 100644 index 000000000..6616648ae --- /dev/null +++ b/tools/web-ux/browser-tests/console_live_picker_diagnostics.ts @@ -0,0 +1,72 @@ +/// +import type { Page } from "playwright"; + +// Playwright's listener subscription enables Chromium interception asynchronously. +// Keep it enabled for the Page lifetime and await the browser acknowledgement +// before any real gesture; otherwise keyboard.press can overtake interception. +export async function prepareNativePicker(page: Page): Promise { + page.on("filechooser", () => {}); + const session = await page.context().newCDPSession(page); + await session.send("Page.setInterceptFileChooserDialog", { enabled: true }); +} + +// Observe native controls without substituting them or synthesizing activation. +export async function observePicker(page: Page): Promise { + await page.addInitScript(() => { + const records: unknown[] = []; + Object.assign(window, { __pickerDiagnostics: records }); + const record = (kind: string, element: Element | null, extra = {}) => + records.push({ + kind, + time: performance.now(), + active: navigator.userActivation.isActive, + hasBeenActive: navigator.userActivation.hasBeenActive, + connected: element?.isConnected, + tag: element?.tagName, + className: element?.className, + editor: document.querySelector(".cm-content")?.textContent, + focused: document.activeElement?.className, + ...extra, + }); + document.addEventListener("keydown", (event) => { + if (["Enter", "Tab", "Escape"].includes(event.key)) { + record("keydown", event.target as Element, { key: event.key, trusted: event.isTrusted }); + queueMicrotask(() => record("after-keydown", event.target as Element, { key: event.key })); + } + }, true); + const nativeClick = HTMLInputElement.prototype.click; + HTMLInputElement.prototype.click = function () { + if (this.type === "file") record("input.click", this, { disabled: this.disabled }); + nativeClick.call(this); + if (this.type === "file") record("after-input.click", this); + }; + document.addEventListener("click", (event) => { + if ((event.target as HTMLInputElement)?.type === "file") { + record("input-click-event", event.target as Element, { trusted: event.isTrusted }); + } + }, true); + const observer = new MutationObserver((changes) => { + for (const change of changes) { + for ( + const [kind, nodes] of [ + ["added", change.addedNodes], + ["removed", change.removedNodes], + ] as const + ) { + for (const node of Array.from(nodes)) { + if ( + node instanceof Element && (node.matches("input[type=file], .cm-editor") || + node.querySelector("input[type=file], .cm-editor")) + ) record(kind, node); + } + } + } + }); + observer.observe(document, { childList: true, subtree: true }); + }); +} +export async function pickerDiagnostics(page: Page): Promise { + return await page.evaluate(() => + (window as unknown as { __pickerDiagnostics: unknown }).__pickerDiagnostics + ); +} diff --git a/tools/web-ux/browser-tests/console_live_shell_test.ts b/tools/web-ux/browser-tests/console_live_shell_test.ts new file mode 100644 index 000000000..84c044643 --- /dev/null +++ b/tools/web-ux/browser-tests/console_live_shell_test.ts @@ -0,0 +1,290 @@ +/// +import { assert, assertEquals } from "@std/assert"; +import { dirname, fromFileUrl, join } from "@std/path"; +import { chromium, type Page } from "playwright"; +import { Buffer } from "node:buffer"; +import { + observePicker, + pickerDiagnostics, + prepareNativePicker, +} from "./console_live_picker_diagnostics.ts"; + +const root = join(dirname(fromFileUrl(import.meta.url)), "../../.."); +const route = "/w/console-live-review/workers/W-901-console-live-fixture/console"; +// Build outside this test. Allows isolated snapshots without writes to production sources. +const buildRoot = Deno.env.get("WEB_UX_BUILD_ROOT") ?? join(root, "web/workspace/build"); +const outputRoot = Deno.env.get("WEB_UX_LIFECYCLE_OUTPUT") ?? + join(root, "target/web-ux/console-live-lifecycle"); +async function ready(url: string): Promise { + const deadline = Date.now() + 10000; + while (Date.now() < deadline) { + try { + const response = await fetch(url); + await response.body?.cancel(); + if (response.ok) return; + } catch { /* owned server startup */ } + await new Promise((resolve) => setTimeout(resolve, 50)); + } + throw new Error("Owned live fixture did not start"); +} +type Method = { + method: string; + params?: { + input?: Array<{ kind: string; invocation?: { identity: string; arguments: unknown[] } }>; + }; +}; +type FixtureState = { methods: Method[]; uploads: string[]; deletes: string[] }; +async function fixtureState(base: string): Promise { + return await (await fetch(`${base}/fixture-state`)).json(); +} +async function replaceDraft(page: Page, text: string): Promise { + await page.locator(".cm-content").click(); + await page.keyboard.press("Control+a"); + await page.keyboard.press("Backspace"); + await page.keyboard.type(text); +} +async function chooseReview(page: Page): Promise { + await replaceDraft(page, "/rev"); + await page.getByRole("option").filter({ hasText: "review" }).waitFor(); + await page.keyboard.press("Tab"); +} +Deno.test("production live Console completes, edits/removes typed drafts and stages/cancels attachment adapter", async () => { + const listener = Deno.listen({ hostname: "127.0.0.1", port: 0 }); + const port = (listener.addr as Deno.NetAddr).port; + listener.close(); + const base = `http://127.0.0.1:${port}`; + const server = new Deno.Command(Deno.execPath(), { + args: [ + "run", + "--allow-net", + "--allow-read", + join(root, "tools/web-ux/browser-tests/console_live_fixture_server.ts"), + String(port), + buildRoot, + ], + stdout: "null", + stderr: "inherit", + }).spawn(); + await Deno.mkdir(outputRoot, { recursive: true }); + try { + await ready(`${base}/health`); + const browser = await chromium.launch({ headless: true }); + try { + for (const theme of ["light", "dark"] as const) { + for (const width of [1440, 768, 390]) { + const context = await browser.newContext({ + viewport: { width, height: width === 390 ? 844 : 900 }, + colorScheme: theme, + reducedMotion: "reduce", + }); + const page = await context.newPage(); + await prepareNativePicker(page); + await observePicker(page); + page.setDefaultTimeout(5000); + const errors: string[] = []; + const acknowledgedDeletes = new Set(); + page.on("console", (message) => { + if (message.type() === "error") errors.push(message.text()); + }); + page.on("pageerror", (error) => errors.push(String(error))); + page.on( + "requestfailed", + (request) => + errors.push( + `failed ${request.method()} ${request.url()} ${request.failure()?.errorText}`, + ), + ); + page.on("response", (response) => { + if (response.request().method() === "DELETE" && response.status() === 204) { + acknowledgedDeletes.add(response.url()); + } + if (response.status() >= 400) { + errors.push(`HTTP ${response.status()} ${response.url()}`); + } + }); + try { + assertEquals((await page.goto(base + route))?.status(), 200); + await page.locator(".cm-content[contenteditable=true]").waitFor(); + await page.getByText("Ready. Choose a Feature", { exact: false }).waitFor(); + const start = await fixtureState(base); + await chooseReview(page); + await page.getByRole("option").filter({ hasText: "brief" }).waitFor(); + await page.keyboard.type("m"); + await page.getByRole("option").filter({ hasText: "mode=" }).click(); + await page.getByRole("option").filter({ hasText: "brief" }).click(); + await page.keyboard.type(", count=2, enabled="); + await page.getByRole("option").filter({ hasText: "true" }).click(); + await page.keyboard.type(', path="docs/'); + await page.getByRole("option").filter({ hasText: "docs/release-notes.md" }).click(); + await page.keyboard.type(")"); + await page.locator(".composer-typed-chip").waitFor(); + await page.screenshot({ path: join(outputRoot, `${theme}-${width}-invocation.png`) }); + let current = await fixtureState(base); + assertEquals(current.uploads, start.uploads, "Feature discovery must not stage files"); + assertEquals( + current.methods.filter((method) => method.method !== "list_completions"), + start.methods.filter((method) => method.method !== "list_completions"), + "completion must not submit, notify or execute controls", + ); + // Pointer chip edit opens readable syntax, keyboard editing stays unexecuted. + await page.locator(".composer-typed-chip").click(); + await page.waitForFunction(() => + document.querySelector(".cm-content")?.textContent?.includes("count=2") + ); + for ( + let index = 0; + index < ', enabled=true, path="docs/release-notes.md"'.length; + index++ + ) { + await page.keyboard.press("ArrowLeft"); + } + await page.keyboard.press("Backspace"); + await page.keyboard.type("3"); + await page.waitForFunction(() => + document.querySelector(".cm-content")?.textContent?.includes("count=3") + ); + await page.screenshot({ path: join(outputRoot, `${theme}-${width}-editing.png`) }); + await page.keyboard.press("Escape"); + await page.keyboard.press("Control+End"); + await page.keyboard.type(" "); + await page.locator(".composer-typed-chip").waitFor(); + await page.keyboard.press("Backspace"); + await page.keyboard.press("Backspace"); + assertEquals(await page.locator(".composer-typed-chip").count(), 0); + await page.keyboard.press("Control+z"); + await page.locator(".composer-typed-chip").waitFor(); + // Cancel a native picker; no upload and no accidental invocation remains. + await replaceDraft(page, "before /att"); + await page.getByRole("option").filter({ hasText: "attach" }).waitFor(); + const picker = page.waitForEvent("filechooser"); + await page.keyboard.press("Enter"); + await (await picker).setFiles([]); + assertEquals((await fixtureState(base)).uploads, start.uploads); + assertEquals(await page.locator(".composer-typed-chip").count(), 0); + assertEquals((await page.locator(".cm-content").innerText()).trim(), "before"); + await page.screenshot({ + path: join(outputRoot, `${theme}-${width}-picker-cancelled.png`), + }); + // Same generic adapter, this time choose a synthetic local file. + await page.keyboard.type("/att"); + await page.getByRole("option").filter({ hasText: "attach" }).waitFor(); + const uploadPicker = page.waitForEvent("filechooser"); + await page.keyboard.press("Enter"); + await (await uploadPicker).setFiles({ + name: "release-notes.md", + mimeType: "text/markdown", + buffer: Buffer.from("Synthetic release notes"), + }); + await page.locator(".composer-typed-chip").waitFor(); + await page.screenshot({ path: join(outputRoot, `${theme}-${width}-attachment.png`) }); + current = await fixtureState(base); + assertEquals(current.uploads.length, start.uploads.length + 1); + await page.keyboard.press("Control+End"); + await page.keyboard.press("Backspace"); + assertEquals(await page.locator(".composer-typed-chip").count(), 0); + await page.screenshot({ path: join(outputRoot, `${theme}-${width}-deleted.png`) }); + // Staged resource is retained through undo, then released when draft is reset. + await page.keyboard.press("Control+z"); + await page.locator(".composer-typed-chip").waitFor(); + await replaceDraft(page, "literal /review(mode=brief)"); + assertEquals( + await page.locator(".composer-typed-chip").count(), + 0, + "unselected slash prose stays literal", + ); + await page.keyboard.press("Escape"); + // Explicit selection becomes the sole structured Feature payload on Submit. + await chooseReview(page); + await page.keyboard.press("Escape"); + await page.keyboard.type('mode="thorough", count=3, enabled=false)'); + await page.locator(".composer-typed-chip").waitFor(); + const release = page.waitForResponse((response) => + response.request().method() === "DELETE" && response.url().includes("/attachments/") + ); + await page.keyboard.press("Control+Enter"); + await page.waitForFunction(() => !document.querySelector(".composer-typed-chip")); + const releasedResponse = await release; + assertEquals(releasedResponse.status(), 204); + current = await fixtureState(base); + assert( + current.deletes.includes(current.uploads.at(-1)!), + "accepted draft boundary releases the abandoned uploaded atom", + ); + const submissions = current.methods.slice(start.methods.length).filter((method) => + method.method === "submit" + ); + assertEquals(submissions.length, 1); + assertEquals(submissions[0].params?.input?.[0].kind, "feature_invoke"); + assertEquals(submissions[0].params?.input?.[0].invocation?.identity, "fixture.review"); + assertEquals(submissions[0].params?.input?.[0].invocation?.arguments, [ + { name: "mode", value: { kind: "string", value: "thorough" } }, + { name: "count", value: { kind: "integer", value: 3 } }, + { name: "enabled", value: { kind: "boolean", value: false } }, + ]); + // Chromium may report ERR_ABORTED after receiving the empty 204 cleanup + // response. Retain that diagnostic, but verify delivery by both response + // status and server DELETE evidence instead of claiming a lost operation. + assert( + await page.evaluate(() => document.documentElement.scrollWidth <= innerWidth), + "no page horizontal overflow", + ); + const pickerEvidence = await pickerDiagnostics(page); + // Snapshot diagnostics only after the final browser awaits; classification + // and serialization must use the same immutable set of observed errors. + const recordedErrors = [...errors]; + const acknowledgedDeleteDiagnostics = recordedErrors.filter((error) => + [...acknowledgedDeletes].some((url) => + error === `failed DELETE ${url} net::ERR_ABORTED` + ) + ); + assertEquals( + recordedErrors.filter((error) => !acknowledgedDeleteDiagnostics.includes(error)), + [], + ); + await Deno.writeTextFile( + join(outputRoot, `${theme}-${width}-result.json`), + JSON.stringify( + { + theme, + width, + errors: recordedErrors, + acknowledgedDeleteDiagnostics, + acknowledgedDeleteResponseStatus: releasedResponse.status(), + picker: pickerEvidence, + status: "pass", + before: start, + after: current, + }, + null, + 2, + ), + ); + } catch (error) { + await page.screenshot({ path: join(outputRoot, `${theme}-${width}-failure.png`) }); + await Deno.writeTextFile( + join(outputRoot, `${theme}-${width}-failure.json`), + JSON.stringify( + { + error: String(error), + errors, + body: await page.locator("body").innerText(), + fixture: await fixtureState(base), + }, + null, + 2, + ), + ); + throw error; + } finally { + await context.close(); + } + } + } + } finally { + await browser.close(); + } + } finally { + server.kill("SIGTERM"); + await server.status; + } +}); diff --git a/tools/web-ux/browser-tests/memory_document_fixture_server.ts b/tools/web-ux/browser-tests/memory_document_fixture_server.ts index 3cd9cc001..4fdb49d4a 100644 --- a/tools/web-ux/browser-tests/memory_document_fixture_server.ts +++ b/tools/web-ux/browser-tests/memory_document_fixture_server.ts @@ -66,7 +66,15 @@ ${"detail-wide-code-".repeat(18)} `; const subjects = [ - subject(representativeSubjectId, "Release coordination", "active", 42), + subject( + representativeSubjectId, + "Release coordination", + "active", + 42, + "Prefer explicit evidence over assumptions.\nAsk before irreversible actions, and state uncertainty plainly.", + 3, + connectedWorker("Release coordination Worker with a deliberately long display name"), + ), subject(emptySubjectId, "Empty ready subject", "active", 0), subject(staleSubjectId, "Stale surface subject", "active", 19), subject(failedSubjectId, "Failed surface subject", "active", 8), @@ -75,14 +83,49 @@ const subjects = [ ]; const pagedSubject = subject(pagedSubjectId, "Subject on the next page", "active", 1); -function subject(id: string, role: string, state: "active" | "retired", storeRevision: number) { +function connectedWorker(displayName: string) { + return { + runtime_id: "embedded-worker-runtime", + worker_id: "release-coordination-worker", + host_id: "fixture-host", + display_name: displayName, + label: displayName, + profile: "builtin:companion", + singleton_key: `subjektiv:${representativeSubjectId}`, + tags: [], + workspace: { + visibility: "workspace", + identity: workspaceId, + workspace_id: workspaceId, + }, + state: "idle", + last_seen_at: "2026-01-02T03:04:05Z", + pinned: false, + retention_state: "retained", + implementation: { kind: "embedded", display_hint: "Fixture Worker" }, + diagnostics: [], + }; +} + +function subject( + id: string, + role: string, + state: "active" | "retired", + storeRevision: number, + behaviorMd = "", + behaviorRevision = 0, + currentWorker?: ReturnType, +) { return { id, role, + behavior_md: behaviorMd, + behavior_revision: behaviorRevision, state, store_revision: storeRevision, created_at: "2026-01-01T00:00:00Z", updated_at: "2026-01-02T03:04:05Z", + ...(currentWorker ? { current_worker: currentWorker } : {}), }; } @@ -125,7 +168,7 @@ function memoriesFor(subjectId: string) { 1, "working_assumption", "retracted", - "T-672 UNIQUE END MARKER", + "Memory list end marker for scroll validation", ), ]; } @@ -235,7 +278,7 @@ function memoryDetail(memoryId: string, requestedRevision: number | null) { evidence_id: "evidence-message-0001", origin: { kind: "human_input", account_id: "account-fixture" }, evidence_kind: "message", - label: "T-672 product direction", + label: "Memory information design source", summary: "Bounded source reference retained with the committed revision.", }], source_refs_total: 1, @@ -403,7 +446,10 @@ Deno.serve({ hostname: "127.0.0.1", port }, async (request) => { const subjectsPath = `/api/w/${workspaceId}/subjektiv/subjects`; if (url.pathname === subjectsPath) { if (request.method === "POST") { - const payload = await request.json().catch(() => null) as { role?: unknown } | null; + const payload = await request.json().catch(() => null) as { + role?: unknown; + behavior_md?: unknown; + } | null; subjectCreateRequests.push(payload); if (!payload || typeof payload.role !== "string" || payload.role.trim().length === 0) { return json({ error: "Bad Request", message: "subject role must not be empty" }, 400); @@ -411,9 +457,12 @@ Deno.serve({ hostname: "127.0.0.1", port }, async (request) => { if (payload.role === "Permission denied") { return json({ error: "Forbidden", message: "workspace permission denied" }, 403); } + if (typeof payload.behavior_md !== "string") { + return json({ error: "Bad Request", message: "subject behavior must be a string" }, 400); + } createdSubjectCount += 1; const id = `created-subject-${String(createdSubjectCount).padStart(4, "0")}`; - const created = subject(id, payload.role, "active", 0); + const created = subject(id, payload.role, "active", 0, payload.behavior_md); createdSubjectIds.add(id); subjects.unshift(created); return json(created, 201); @@ -442,6 +491,27 @@ Deno.serve({ hostname: "127.0.0.1", port }, async (request) => { const foundSubject = subjects.find((item) => item.id === subjectId); if (!foundSubject) return json({}, 404); + if (parts.length === 2 && parts[1] === "behavior" && request.method === "PATCH") { + const payload = await request.json().catch(() => null) as { + expected_behavior_revision?: unknown; + behavior_md?: unknown; + } | null; + if ( + !payload || + typeof payload.expected_behavior_revision !== "number" || + typeof payload.behavior_md !== "string" + ) { + return json({ error: "Bad Request", message: "invalid behavior update" }, 400); + } + if (payload.expected_behavior_revision !== foundSubject.behavior_revision) { + return json({ error: "Conflict", message: "subject behavior revision conflict" }, 409); + } + foundSubject.behavior_md = payload.behavior_md; + foundSubject.behavior_revision += 1; + foundSubject.updated_at = "2026-01-03T04:05:06Z"; + return json(foundSubject); + } + if (request.method !== "GET") return new Response("method not allowed", { status: 405 }); if (parts.length === 1) return json(foundSubject); if (parts.length === 2 && parts[1] === "surface") return json(surfaceFor(subjectId)); if (parts.length === 2 && parts[1] === "memories") { diff --git a/tools/web-ux/browser-tests/memory_document_shell_test.ts b/tools/web-ux/browser-tests/memory_document_shell_test.ts index c0913082a..615b280df 100644 --- a/tools/web-ux/browser-tests/memory_document_shell_test.ts +++ b/tools/web-ux/browser-tests/memory_document_shell_test.ts @@ -16,7 +16,7 @@ const memoryId = const subjectPageCursor = "fixture-subject-page-2"; const createInteractionArtifacts = join( repositoryRoot, - "target/web-ux/t-689-browser-interactions", + "target/web-ux/t-692-browser-interactions", ); async function freePort(): Promise { @@ -51,7 +51,7 @@ async function markerIsInsideMain(page: Page): Promise { const document = (globalThis as any).document; const main = document.querySelector("main"); const marker = [...document.querySelectorAll("h3")].find((heading) => - heading.textContent === "T-672 UNIQUE END MARKER" + heading.textContent === "Memory list end marker for scroll validation" ); if (!main || !marker) return false; const mainRect = main.getBoundingClientRect(); @@ -111,6 +111,7 @@ async function openSubjectPage( await hideSidebar.click(); await showSidebar.waitFor(); + await page.waitForTimeout(250); assertEquals(await showSidebar.getAttribute("aria-expanded"), "false"); assertEquals( await page.locator("main").evaluate((main) => (main as unknown as { inert: boolean }).inert), @@ -217,6 +218,13 @@ Deno.test("production subject Memory and Worker launch shells preserve exact sco }); page.on("pageerror", (error) => errors.push(String(error))); await openSubjectPage(page, baseUrl, scenario.mobile); + assertEquals( + await page.getByRole("article", { name: "User-managed Subject behavior" }).count(), + 1, + ); + await page.screenshot({ + path: join(createInteractionArtifacts, `subject-${scenario.label}.png`), + }); const initial = await mainScrollState(page); assert(initial.scrollHeight > initial.clientHeight); @@ -262,7 +270,7 @@ Deno.test("production subject Memory and Worker launch shells preserve exact sco assertEquals(await page.locator('a[href="https://example.com"]').count(), 1); assertEquals(await page.locator('a[href^="javascript:"]').count(), 0); assertEquals( - await page.getByRole("article", { name: "Resident surface" }).locator("img, script") + await page.getByRole("article", { name: "Resident context" }).locator("img, script") .count(), 0, ); @@ -282,7 +290,7 @@ Deno.test("production subject Memory and Worker launch shells preserve exact sco await page.locator("main").hover(); await page.mouse.wheel(0, 100_000); } else if (scenario.label === "keyboard") { - await page.getByText("Surface provenance", { exact: true }).focus(); + await page.getByText("Surface sources and diagnostics", { exact: true }).focus(); await page.keyboard.press("End"); } else { await swipeMainToEnd(page); @@ -313,17 +321,17 @@ Deno.test("production subject Memory and Worker launch shells preserve exact sco level: 1, }).waitFor(); assertEquals(await detailPage.getByText("candidate-release-decision-0001").count(), 1); - assertEquals(await detailPage.getByText("T-672 product direction").count(), 1); + assertEquals(await detailPage.getByText("Memory information design source").count(), 1); assertEquals(await detailPage.getByText("memory-derived-source-0002").count(), 1); assertEquals(await detailPage.getByRole("heading", { name: "Revision history" }).count(), 1); - assertEquals(await detailPage.getByText("Revision 3", { exact: true }).count(), 1); - assertEquals(await detailPage.getByText("Revision 2", { exact: true }).count(), 1); - assertEquals(await detailPage.getByText("Revision 1", { exact: true }).count(), 0); + assertEquals(await detailPage.getByText("Memory revision 3", { exact: true }).count(), 1); + assertEquals(await detailPage.getByText("Memory revision 2", { exact: true }).count(), 1); + assertEquals(await detailPage.getByText("Memory revision 1", { exact: true }).count(), 0); await detailPage.getByRole("navigation", { name: "Revision history pages" }).getByRole( "link", { name: "Next page →" }, ).click(); - await detailPage.getByText("Revision 1", { exact: true }).waitFor(); + await detailPage.getByText("Memory revision 1", { exact: true }).waitFor(); assert(new URL(detailPage.url()).searchParams.has("revision_cursor")); assertEquals( await detailPage.getByRole("navigation", { name: "Revision history pages" }).getByRole( @@ -352,6 +360,15 @@ Deno.test("production subject Memory and Worker launch shells preserve exact sco const paginationPage = await paginationContext.newPage(); await paginationPage.goto(`${baseUrl}/w/${workspaceId}/memory`); await paginationPage.getByRole("heading", { name: "Subjects", level: 1 }).waitFor(); + assertEquals( + await paginationPage.getByText( + "Release coordination Worker with a deliberately long display name", + { exact: true }, + ).count(), + 1, + ); + assertEquals(await paginationPage.getByText("Store revision", { exact: true }).count(), 0); + assert((await paginationPage.getByText("Not connected", { exact: true }).count()) > 0); assertEquals(await paginationPage.getByText("Subject on the next page").count(), 0); await paginationPage.getByRole("navigation", { name: "Subject pages" }).getByRole( "link", @@ -425,7 +442,9 @@ Deno.test("production subject Memory and Worker launch shells preserve exact sco const createPage = await createContext.newPage(); const mutationPaths: string[] = []; createPage.on("request", (request) => { - if (request.method() === "POST") mutationPaths.push(new URL(request.url()).pathname); + if (["POST", "PATCH"].includes(request.method())) { + mutationPaths.push(new URL(request.url()).pathname); + } }); await createPage.goto(`${baseUrl}/w/${workspaceId}/memory?cursor=${subjectPageCursor}`); await createPage.getByText("Subject on the next page").waitFor(); @@ -444,26 +463,65 @@ Deno.test("production subject Memory and Worker launch shells preserve exact sco await createPage.getByRole("button", { name: "Cancel" }).click(); await createPage.getByRole("button", { name: "New Subject" }).click(); await createPage.getByRole("textbox", { name: "Role" }).fill(" Release steward "); + await createPage.getByRole("textbox", { name: /Behavior/ }).fill( + "Challenge unsupported assumptions before release.", + ); await createPage.getByRole("button", { name: "Create Subject" }).click(); await createPage.waitForURL("**/memory/created-subject-0001"); await createPage.getByRole("heading", { name: "Release steward", level: 1 }).waitFor(); + assertEquals( + await createPage.getByRole("article", { name: "User-managed Subject behavior" }).count(), + 1, + ); + assert( + (await createPage.getByRole("article", { + name: "User-managed Subject behavior", + }).textContent())?.includes("Challenge unsupported assumptions before release."), + ); assertEquals( await createPage.locator("code.subject-id", { hasText: "created-subject-0001" }).count(), 1, ); - const currentWorker = createPage.locator(".subject-facts div").filter({ - hasText: "Current worker", + const currentWorker = createPage.getByText("Worker connection", { exact: true }).locator( + "..", + ); + assert((await currentWorker.textContent())?.includes("Not connected")); + await createPage.getByText("Resident context has not been generated.").waitFor(); + await createPage.getByText("No committed Memories yet.").waitFor(); + await createPage.getByRole("button", { name: "Edit" }).click(); + const behaviorEditor = createPage.getByRole("textbox", { name: "Subject behavior" }); + await behaviorEditor.fill( + "Challenge unsupported assumptions before release.\nName the evidence and any remaining uncertainty.", + ); + await createPage.screenshot({ + path: join(createInteractionArtifacts, "behavior-edit-desktop-light.png"), + }); + await createPage.getByRole("button", { name: "Save behavior" }).click(); + await createPage.getByText(/Behavior storage confirmed at revision 1/).waitFor(); + assert( + (await createPage.getByRole("status").filter({ + hasText: "This page does not confirm application", + }).textContent())?.includes("does not confirm application"), + ); + await createPage.screenshot({ + path: join(createInteractionArtifacts, "behavior-saved-desktop-light.png"), }); - assert((await currentWorker.textContent())?.includes("None")); assertEquals(mutationPaths, [ `/api/w/${workspaceId}/subjektiv/subjects`, `/api/w/${workspaceId}/subjektiv/subjects`, + `/api/w/${workspaceId}/subjektiv/subjects/created-subject-0001/behavior`, ]); const createEvidence = await (await fetch(`${baseUrl}/fixture/subject-create-requests`)) .json(); assertEquals(createEvidence, { count: 2, - requests: [{ role: "Permission denied" }, { role: " Release steward " }], + requests: [ + { role: "Permission denied", behavior_md: "" }, + { + role: " Release steward ", + behavior_md: "Challenge unsupported assumptions before release.", + }, + ], }); await assertNoPageWideOverflow(createPage); await createContext.close(); @@ -472,21 +530,29 @@ Deno.test("production subject Memory and Worker launch shells preserve exact sco const stateCases = [ { id: "empty-subject", - text: "Resident surface is ready and empty.", + text: "Resident context is current but empty.", + role: "status" as const, + }, + { + id: "stale-subject", + text: "Resident context needs to be refreshed.", role: "status" as const, }, - { id: "stale-subject", text: "Resident surface is stale.", role: "status" as const }, { id: "failed-subject", - text: "Resident surface generation failed.", + text: "Resident context generation failed.", role: "alert" as const, }, { id: "ungenerated-subject", - text: "Resident surface has not been generated.", + text: "Resident context has not been generated.", role: "status" as const, }, - { id: "error-subject", text: "Resident surface unavailable.", role: "alert" as const }, + { + id: "error-subject", + text: "Resident context status unavailable.", + role: "alert" as const, + }, ]; for (const stateCase of stateCases) { const page = await stateContext.newPage(); @@ -494,8 +560,14 @@ Deno.test("production subject Memory and Worker launch shells preserve exact sco const state = page.getByRole(stateCase.role).filter({ hasText: stateCase.text }); await state.waitFor(); assert((await state.textContent())?.includes(stateCase.text)); + if (stateCase.id === "stale-subject") { + assertEquals(await page.getByText("A resolved stale-surface lesson").count(), 1); + } + if (stateCase.id === "failed-subject") { + assertEquals(await page.getByText("A retracted failed-surface constraint").count(), 1); + } if (stateCase.id !== "empty-subject") { - assertEquals(await page.getByRole("article", { name: "Resident surface" }).count(), 0); + assertEquals(await page.getByRole("article", { name: "Resident context" }).count(), 0); } await assertNoPageWideOverflow(page); await page.close(); diff --git a/tools/web-ux/deno.json b/tools/web-ux/deno.json index ab994d533..1ca411068 100644 --- a/tools/web-ux/deno.json +++ b/tools/web-ux/deno.json @@ -11,7 +11,7 @@ "web-ux": "deno run --allow-env --allow-net --allow-read --allow-write --allow-run --allow-sys cli.ts", "check": "deno check cli.ts src/*.ts tests/*.ts browser-tests/*.ts", "test": "deno test --allow-env --allow-read --allow-write --allow-run --allow-sys tests", - "test:browser": "deno test --allow-env --allow-net --allow-read --allow-write --allow-run --allow-sys browser-tests/capture_smoke_test.ts browser-tests/memory_document_shell_test.ts browser-tests/console_history_shell_test.ts browser-tests/workspace_home_shell_test.ts browser-tests/workspace_settings_shell_test.ts browser-tests/console_pending_shell_test.ts" + "test:browser": "deno test --allow-env --allow-net --allow-read --allow-write --allow-run --allow-sys browser-tests/capture_smoke_test.ts browser-tests/memory_document_shell_test.ts browser-tests/console_history_shell_test.ts browser-tests/workspace_home_shell_test.ts browser-tests/workspace_settings_shell_test.ts browser-tests/console_pending_shell_test.ts browser-tests/console_live_shell_test.ts browser-tests/console_live_admission_shell_test.ts" }, "fmt": { "lineWidth": 100, diff --git a/tools/web-ux/scenarios/console-live-baseline-dark.json b/tools/web-ux/scenarios/console-live-baseline-dark.json new file mode 100644 index 000000000..ade8afd63 --- /dev/null +++ b/tools/web-ux/scenarios/console-live-baseline-dark.json @@ -0,0 +1,171 @@ +{ + "schemaVersion": 1, + "id": "console-live-baseline-dark", + "title": "Production Console Feature invocation draft lifecycle", + "baseUrl": "http://127.0.0.1:15286", + "colorScheme": "dark", + "reducedMotion": "reduce", + "personas": [ + { + "id": "fixture", + "label": "Live synthetic Worker", + "auth": { + "kind": "anonymous" + } + } + ], + "viewports": [ + { + "label": "desktop", + "width": 1440, + "height": 900 + }, + { + "label": "narrow", + "width": 768, + "height": 900 + }, + { + "label": "mobile", + "width": 390, + "height": 844 + } + ], + "routes": [ + { + "id": "console", + "label": "Live Console", + "path": "/w/console-live-review/workers/W-901-console-live-fixture/console", + "goal": "Inspect the unchanged production Console shell and literal slash text before Feature completion existed.", + "dataState": "Live synthetic idle Worker; review enum/integer/boolean/worker-file arguments plus generic attachment adapter. No real account, file, or invocation execution.", + "ready": { + "kind": "selector", + "selector": ".cm-content[contenteditable=true]" + }, + "capturePoints": [ + { + "id": "initial", + "label": "Idle live Console" + }, + { + "id": "slash-plain", + "label": "Literal unselected slash text", + "interaction": [ + { + "action": "click", + "selector": ".cm-content" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "Control+a" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "Backspace" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "/" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "r" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "e" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "v" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "i" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "e" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "w" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "(" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "b" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "r" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "i" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "e" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "f" + }, + { + "action": "press", + "selector": ".cm-content", + "key": ")" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "Escape" + } + ], + "ready": { + "kind": "selector", + "selector": ".cm-content[contenteditable=true]" + } + } + ] + } + ], + "processes": [ + { + "id": "console-live-fixture", + "command": "deno", + "cwd": "..", + "args": [ + "run", + "--allow-net", + "--allow-read", + "--allow-env=WEB_UX_BUILD_ROOT", + "browser-tests/console_live_fixture_server.ts", + "15286" + ], + "env": { + "WEB_UX_BUILD_ROOT": "${WEB_UX_BUILD_ROOT}" + }, + "readyUrl": "http://127.0.0.1:15286/health" + } + ] +} diff --git a/tools/web-ux/scenarios/console-live-baseline-light.json b/tools/web-ux/scenarios/console-live-baseline-light.json new file mode 100644 index 000000000..b0e8b3c98 --- /dev/null +++ b/tools/web-ux/scenarios/console-live-baseline-light.json @@ -0,0 +1,171 @@ +{ + "schemaVersion": 1, + "id": "console-live-baseline-light", + "title": "Production Console Feature invocation draft lifecycle", + "baseUrl": "http://127.0.0.1:15286", + "colorScheme": "light", + "reducedMotion": "reduce", + "personas": [ + { + "id": "fixture", + "label": "Live synthetic Worker", + "auth": { + "kind": "anonymous" + } + } + ], + "viewports": [ + { + "label": "desktop", + "width": 1440, + "height": 900 + }, + { + "label": "narrow", + "width": 768, + "height": 900 + }, + { + "label": "mobile", + "width": 390, + "height": 844 + } + ], + "routes": [ + { + "id": "console", + "label": "Live Console", + "path": "/w/console-live-review/workers/W-901-console-live-fixture/console", + "goal": "Inspect the unchanged production Console shell and literal slash text before Feature completion existed.", + "dataState": "Live synthetic idle Worker; review enum/integer/boolean/worker-file arguments plus generic attachment adapter. No real account, file, or invocation execution.", + "ready": { + "kind": "selector", + "selector": ".cm-content[contenteditable=true]" + }, + "capturePoints": [ + { + "id": "initial", + "label": "Idle live Console" + }, + { + "id": "slash-plain", + "label": "Literal unselected slash text", + "interaction": [ + { + "action": "click", + "selector": ".cm-content" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "Control+a" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "Backspace" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "/" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "r" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "e" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "v" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "i" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "e" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "w" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "(" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "b" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "r" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "i" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "e" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "f" + }, + { + "action": "press", + "selector": ".cm-content", + "key": ")" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "Escape" + } + ], + "ready": { + "kind": "selector", + "selector": ".cm-content[contenteditable=true]" + } + } + ] + } + ], + "processes": [ + { + "id": "console-live-fixture", + "command": "deno", + "cwd": "..", + "args": [ + "run", + "--allow-net", + "--allow-read", + "--allow-env=WEB_UX_BUILD_ROOT", + "browser-tests/console_live_fixture_server.ts", + "15286" + ], + "env": { + "WEB_UX_BUILD_ROOT": "${WEB_UX_BUILD_ROOT}" + }, + "readyUrl": "http://127.0.0.1:15286/health" + } + ] +} diff --git a/tools/web-ux/scenarios/console-live-dark.json b/tools/web-ux/scenarios/console-live-dark.json new file mode 100644 index 000000000..8e43a7ab4 --- /dev/null +++ b/tools/web-ux/scenarios/console-live-dark.json @@ -0,0 +1,318 @@ +{ + "schemaVersion": 1, + "id": "console-live-dark", + "title": "Production Console Feature invocation draft lifecycle", + "baseUrl": "http://127.0.0.1:15286", + "colorScheme": "dark", + "reducedMotion": "reduce", + "personas": [ + { + "id": "fixture", + "label": "Live synthetic Worker", + "auth": { + "kind": "anonymous" + } + } + ], + "viewports": [ + { + "label": "desktop", + "width": 1440, + "height": 900 + }, + { + "label": "narrow", + "width": 768, + "height": 900 + }, + { + "label": "mobile", + "width": 390, + "height": 844 + } + ], + "routes": [ + { + "id": "console", + "label": "Live Console", + "path": "/w/console-live-review/workers/W-901-console-live-fixture/console", + "goal": "Discover Feature metadata, complete typed arguments and inspect a draft chip without submitting or staging files.", + "dataState": "Live synthetic idle Worker; review enum/integer/boolean/worker-file arguments plus generic attachment adapter. No real account, file, or invocation execution.", + "ready": { + "kind": "selector", + "selector": ".cm-content[contenteditable=true]" + }, + "capturePoints": [ + { + "id": "initial", + "label": "Idle live Console" + }, + { + "id": "slash-plain", + "label": "Literal unselected slash text", + "interaction": [ + { + "action": "click", + "selector": ".cm-content" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "Control+a" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "Backspace" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "/" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "r" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "e" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "v" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "i" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "e" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "w" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "(" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "b" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "r" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "i" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "e" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "f" + }, + { + "action": "press", + "selector": ".cm-content", + "key": ")" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "Escape" + } + ], + "ready": { + "kind": "selector", + "selector": ".cm-content[contenteditable=true]" + } + }, + { + "id": "feature-menu", + "label": "Feature names", + "interaction": [ + { + "action": "click", + "selector": ".cm-content" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "Control+a" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "Backspace" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "/" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "Control+Space" + }, + { + "action": "wait", + "ready": { + "kind": "selector", + "selector": "[role=option]" + } + } + ] + }, + { + "id": "argument-menu", + "label": "Argument values", + "interaction": [ + { + "action": "press", + "selector": ".cm-content", + "key": "Control+a" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "Backspace" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "/" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "r" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "e" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "v" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "Control+Space" + }, + { + "action": "wait", + "ready": { + "kind": "selector", + "selector": "[role=option]" + } + }, + { + "action": "press", + "selector": ".cm-content", + "key": "Tab" + }, + { + "action": "wait", + "ready": { + "kind": "selector", + "selector": "[role=option]" + } + } + ] + }, + { + "id": "typed-chip", + "label": "Selected typed invocation", + "interaction": [ + { + "action": "press", + "selector": ".cm-content", + "key": "Escape" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "Control+End" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "b" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "r" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "i" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "e" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "f" + }, + { + "action": "press", + "selector": ".cm-content", + "key": ")" + } + ], + "ready": { + "kind": "selector", + "selector": ".composer-typed-chip" + } + } + ] + } + ], + "processes": [ + { + "id": "console-live-fixture", + "command": "deno", + "cwd": "..", + "args": [ + "run", + "--allow-net", + "--allow-read", + "--allow-env=WEB_UX_BUILD_ROOT", + "browser-tests/console_live_fixture_server.ts", + "15286" + ], + "env": { + "WEB_UX_BUILD_ROOT": "${WEB_UX_BUILD_ROOT}" + }, + "readyUrl": "http://127.0.0.1:15286/health" + } + ] +} diff --git a/tools/web-ux/scenarios/console-live-light.json b/tools/web-ux/scenarios/console-live-light.json new file mode 100644 index 000000000..ff9944411 --- /dev/null +++ b/tools/web-ux/scenarios/console-live-light.json @@ -0,0 +1,318 @@ +{ + "schemaVersion": 1, + "id": "console-live-light", + "title": "Production Console Feature invocation draft lifecycle", + "baseUrl": "http://127.0.0.1:15286", + "colorScheme": "light", + "reducedMotion": "reduce", + "personas": [ + { + "id": "fixture", + "label": "Live synthetic Worker", + "auth": { + "kind": "anonymous" + } + } + ], + "viewports": [ + { + "label": "desktop", + "width": 1440, + "height": 900 + }, + { + "label": "narrow", + "width": 768, + "height": 900 + }, + { + "label": "mobile", + "width": 390, + "height": 844 + } + ], + "routes": [ + { + "id": "console", + "label": "Live Console", + "path": "/w/console-live-review/workers/W-901-console-live-fixture/console", + "goal": "Discover Feature metadata, complete typed arguments and inspect a draft chip without submitting or staging files.", + "dataState": "Live synthetic idle Worker; review enum/integer/boolean/worker-file arguments plus generic attachment adapter. No real account, file, or invocation execution.", + "ready": { + "kind": "selector", + "selector": ".cm-content[contenteditable=true]" + }, + "capturePoints": [ + { + "id": "initial", + "label": "Idle live Console" + }, + { + "id": "slash-plain", + "label": "Literal unselected slash text", + "interaction": [ + { + "action": "click", + "selector": ".cm-content" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "Control+a" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "Backspace" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "/" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "r" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "e" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "v" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "i" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "e" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "w" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "(" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "b" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "r" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "i" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "e" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "f" + }, + { + "action": "press", + "selector": ".cm-content", + "key": ")" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "Escape" + } + ], + "ready": { + "kind": "selector", + "selector": ".cm-content[contenteditable=true]" + } + }, + { + "id": "feature-menu", + "label": "Feature names", + "interaction": [ + { + "action": "click", + "selector": ".cm-content" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "Control+a" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "Backspace" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "/" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "Control+Space" + }, + { + "action": "wait", + "ready": { + "kind": "selector", + "selector": "[role=option]" + } + } + ] + }, + { + "id": "argument-menu", + "label": "Argument values", + "interaction": [ + { + "action": "press", + "selector": ".cm-content", + "key": "Control+a" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "Backspace" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "/" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "r" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "e" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "v" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "Control+Space" + }, + { + "action": "wait", + "ready": { + "kind": "selector", + "selector": "[role=option]" + } + }, + { + "action": "press", + "selector": ".cm-content", + "key": "Tab" + }, + { + "action": "wait", + "ready": { + "kind": "selector", + "selector": "[role=option]" + } + } + ] + }, + { + "id": "typed-chip", + "label": "Selected typed invocation", + "interaction": [ + { + "action": "press", + "selector": ".cm-content", + "key": "Escape" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "Control+End" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "b" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "r" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "i" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "e" + }, + { + "action": "press", + "selector": ".cm-content", + "key": "f" + }, + { + "action": "press", + "selector": ".cm-content", + "key": ")" + } + ], + "ready": { + "kind": "selector", + "selector": ".composer-typed-chip" + } + } + ] + } + ], + "processes": [ + { + "id": "console-live-fixture", + "command": "deno", + "cwd": "..", + "args": [ + "run", + "--allow-net", + "--allow-read", + "--allow-env=WEB_UX_BUILD_ROOT", + "browser-tests/console_live_fixture_server.ts", + "15286" + ], + "env": { + "WEB_UX_BUILD_ROOT": "${WEB_UX_BUILD_ROOT}" + }, + "readyUrl": "http://127.0.0.1:15286/health" + } + ] +} diff --git a/tools/web-ux/scenarios/memory-subjects-dark.json b/tools/web-ux/scenarios/memory-subjects-dark.json index 7fbdebfb0..bfb3de69b 100644 --- a/tools/web-ux/scenarios/memory-subjects-dark.json +++ b/tools/web-ux/scenarios/memory-subjects-dark.json @@ -122,9 +122,33 @@ "dataState": "Ready generated Markdown surface with long content, unsafe Markdown, wide table/code, provenance refs, and three lifecycle states.", "ready": { "kind": "selector", - "selector": "[data-memory-view='subject'][data-memory-state='ready'] article[aria-label='Resident surface']" + "selector": "[data-memory-view='subject'][data-memory-state='ready'] article[aria-label='Resident context']" }, - "capturePoints": [{ "id": "initial", "label": "Ready subject" }] + "capturePoints": [ + { "id": "initial", "label": "Ready subject" }, + { + "id": "subject-diagnostics", + "label": "Subject diagnostics", + "interaction": [ + { "action": "click", "selector": "details.subject-technical-details summary" } + ], + "ready": { + "kind": "selector", + "selector": "details.subject-technical-details[open]" + } + }, + { + "id": "surface-diagnostics", + "label": "Surface sources and diagnostics", + "interaction": [ + { "action": "click", "selector": "details.surface-details summary" } + ], + "ready": { + "kind": "selector", + "selector": "details.surface-details[open]" + } + } + ] }, { "id": "detail", @@ -150,6 +174,18 @@ }, "capturePoints": [{ "id": "initial", "label": "Ready empty" }] }, + { + "id": "ungenerated", + "label": "Ungenerated subject context", + "path": "/w/memory-review/memory/ungenerated-subject", + "goal": "Distinguish a Subject with no generated resident context from lost committed Memory.", + "dataState": "Ungenerated resident context and a successful empty committed Memory list.", + "ready": { + "kind": "selector", + "selector": "[data-memory-view='subject'][data-memory-state='ungenerated']" + }, + "capturePoints": [{ "id": "initial", "label": "Not generated" }] + }, { "id": "stale", "label": "Stale subject surface", diff --git a/tools/web-ux/scenarios/memory-subjects-light.json b/tools/web-ux/scenarios/memory-subjects-light.json index 9ddf6f803..905f15372 100644 --- a/tools/web-ux/scenarios/memory-subjects-light.json +++ b/tools/web-ux/scenarios/memory-subjects-light.json @@ -122,9 +122,33 @@ "dataState": "Ready generated Markdown surface with long content, unsafe Markdown, wide table/code, provenance refs, and three lifecycle states.", "ready": { "kind": "selector", - "selector": "[data-memory-view='subject'][data-memory-state='ready'] article[aria-label='Resident surface']" + "selector": "[data-memory-view='subject'][data-memory-state='ready'] article[aria-label='Resident context']" }, - "capturePoints": [{ "id": "initial", "label": "Ready subject" }] + "capturePoints": [ + { "id": "initial", "label": "Ready subject" }, + { + "id": "subject-diagnostics", + "label": "Subject diagnostics", + "interaction": [ + { "action": "click", "selector": "details.subject-technical-details summary" } + ], + "ready": { + "kind": "selector", + "selector": "details.subject-technical-details[open]" + } + }, + { + "id": "surface-diagnostics", + "label": "Surface sources and diagnostics", + "interaction": [ + { "action": "click", "selector": "details.surface-details summary" } + ], + "ready": { + "kind": "selector", + "selector": "details.surface-details[open]" + } + } + ] }, { "id": "detail", @@ -150,6 +174,18 @@ }, "capturePoints": [{ "id": "initial", "label": "Ready empty" }] }, + { + "id": "ungenerated", + "label": "Ungenerated subject context", + "path": "/w/memory-review/memory/ungenerated-subject", + "goal": "Distinguish a Subject with no generated resident context from lost committed Memory.", + "dataState": "Ungenerated resident context and a successful empty committed Memory list.", + "ready": { + "kind": "selector", + "selector": "[data-memory-view='subject'][data-memory-state='ungenerated']" + }, + "capturePoints": [{ "id": "initial", "label": "Not generated" }] + }, { "id": "stale", "label": "Stale subject surface", diff --git a/web/workspace/deno.json b/web/workspace/deno.json index 378b1cf3c..9479ef44f 100644 --- a/web/workspace/deno.json +++ b/web/workspace/deno.json @@ -6,7 +6,7 @@ "dev": "deno run -A npm:vite@8.3.2 dev", "dev:backend": "cd ../.. && cargo run -p yoi-workspace-server --bin yoi-server -- serve --listen 127.0.0.1:8787", "check": "deno run -A npm:@sveltejs/kit@3.0.0 sync && deno run -A npm:svelte-check@4.7.6 --tsconfig ./tsconfig.json", - "test": "deno test --allow-read=src,test,tests,../../docs --allow-env=LOG,VSCODE_TEXTMATE_DEBUG,NODE_ENV tests/workspace-model.test.ts tests/repository-api.test.ts tests/workspace-catalog.test.ts tests/profile-api.test.ts tests/skill-api.test.ts tests/tooltip-contract.test.ts tests/bevel-contract.test.ts tests/bevel-line-contract.test.ts tests/typography-contract.test.ts src/lib/workspace/auth/model.test.ts src/lib/workspace/protocol-validator.test.ts tests/auth-api.test.ts src/lib/workspace/api/http.test.ts src/lib/workspace/api/workers.test.ts src/lib/workspace/api/ticket-browser.test.ts src/lib/workspace/header/breadcrumb-model.test.ts src/lib/workspace/console/chat-submit.test.ts src/lib/workspace/console/composer-delivery.test.ts test/composer-history.test.ts tests/composer-paste.test.ts src/lib/workspace/console/composer-command.test.ts src/lib/workspace/console/composer-draft.test.ts src/lib/workspace/console/composer-completion.test.ts src/lib/workspace/console/file-completions.test.ts test/console/ansi.test.ts src/lib/workspace/console/history.test.ts src/lib/workspace/console/model.test.ts src/lib/workspace/console/turn-navigation.test.ts src/lib/workspace/console/run-status.test.ts src/lib/workspace/companion/api.test.ts tests/workdir-api.test.ts tests/workdir-permissions.test.ts src/lib/workspace/console/tasks.test.ts test/ticket-detail-route-reuse.test.ts test/spa-shell.test.ts test/objective-pages.test.ts test/repositories/ui.test.ts src/lib/workspace/console/worker-console.ui.test.ts src/lib/workspace/settings/model.test.ts tests/runtime-connection.test.ts tests/runtime-management.test.ts tests/runtime-management-source.test.ts src/lib/workspace/sidebar/override-stack.test.ts src/lib/workspace/sidebar/route-ownership.test.ts src/lib/workspace/sidebar/workers.test.ts src/lib/workspace/sidebar/workspace-switcher.test.ts src/lib/workspace/sidebar/worker-subscription.test.ts src/lib/workspace/sidebar/worker-workdir-meta.test.ts src/lib/workspace/sidebar/worker-launch.test.ts test/sidebar/worker-actions.test.ts src/lib/workspace/tickets/merge-request-resources.test.ts src/lib/workspace/tickets/ticket-panel.test.ts test/merge-request-status.test.ts test/config-source/decodal-grammar.test.ts test/config-source/editor-state.test.ts test/config-source/fixed-schema-wrapper.test.ts test/config-source/toolchain.test.ts test/config-source/wasm-parity.test.ts test/config-source/api.test.ts tests/memory-api.test.ts test/repository-access/api.test.ts test/repository-access/loader.test.ts test/repository-access/model.test.ts test/repository-access/structure.test.ts test/repository-ssh-connection-ui.test.ts", + "test": "deno test --allow-read=src,test,tests,../../docs --allow-env=LOG,VSCODE_TEXTMATE_DEBUG,NODE_ENV tests/workspace-model.test.ts tests/repository-api.test.ts tests/workspace-catalog.test.ts tests/profile-api.test.ts tests/skill-api.test.ts tests/tooltip-contract.test.ts tests/bevel-contract.test.ts tests/bevel-line-contract.test.ts tests/typography-contract.test.ts src/lib/workspace/auth/model.test.ts src/lib/workspace/protocol-validator.test.ts tests/auth-api.test.ts src/lib/workspace/api/http.test.ts src/lib/workspace/api/workers.test.ts src/lib/workspace/api/ticket-browser.test.ts src/lib/workspace/header/breadcrumb-model.test.ts src/lib/workspace/console/chat-submit.test.ts src/lib/workspace/console/composer-delivery.test.ts test/composer-history.test.ts tests/composer-paste.test.ts src/lib/workspace/console/composer-command.test.ts src/lib/workspace/console/composer-draft.test.ts src/lib/workspace/console/composer-completion.test.ts src/lib/workspace/console/composer-invocation.test.ts src/lib/workspace/console/file-completions.test.ts test/console/ansi.test.ts src/lib/workspace/console/history.test.ts src/lib/workspace/console/model.test.ts src/lib/workspace/console/turn-navigation.test.ts src/lib/workspace/console/run-status.test.ts src/lib/workspace/companion/api.test.ts tests/workdir-api.test.ts tests/workdir-permissions.test.ts src/lib/workspace/console/tasks.test.ts test/ticket-detail-route-reuse.test.ts test/spa-shell.test.ts test/objective-pages.test.ts test/repositories/ui.test.ts src/lib/workspace/console/worker-console.ui.test.ts src/lib/workspace/settings/model.test.ts tests/runtime-connection.test.ts tests/runtime-management.test.ts tests/runtime-management-source.test.ts src/lib/workspace/sidebar/override-stack.test.ts src/lib/workspace/sidebar/route-ownership.test.ts src/lib/workspace/sidebar/workers.test.ts src/lib/workspace/sidebar/workspace-switcher.test.ts src/lib/workspace/sidebar/worker-subscription.test.ts src/lib/workspace/sidebar/worker-workdir-meta.test.ts src/lib/workspace/sidebar/worker-launch.test.ts tests/runtime-workers.test.ts test/sidebar/worker-actions.test.ts src/lib/workspace/tickets/merge-request-resources.test.ts src/lib/workspace/tickets/ticket-panel.test.ts test/merge-request-status.test.ts test/config-source/decodal-grammar.test.ts test/config-source/editor-state.test.ts test/config-source/fixed-schema-wrapper.test.ts test/config-source/toolchain.test.ts test/config-source/wasm-parity.test.ts test/config-source/api.test.ts tests/memory-api.test.ts test/repository-access/api.test.ts test/repository-access/loader.test.ts test/repository-access/model.test.ts test/repository-access/structure.test.ts test/repository-ssh-connection-ui.test.ts", "test:component": "deno run -A npm:vitest@5.0.3 run --config vitest.config.ts", "build": "deno run -A npm:vite@8.3.2 build", "preview": "deno run -A npm:vite@8.3.2 preview", diff --git a/web/workspace/src/lib/generated/companion-api.ts b/web/workspace/src/lib/generated/companion-api.ts index ddba41954..8a530c53d 100644 --- a/web/workspace/src/lib/generated/companion-api.ts +++ b/web/workspace/src/lib/generated/companion-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-companion-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:f9f0f8918dbcce386e94d5368fa938f02565ca98f9f67f07537bd88ed8c49870) +// Input contract: 0.1.0 (sha256:fd06f382c6d33a5c3cc98ef9e264545923c53fa95a9be555a85680f98c6065da) // Options: companion-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Output: web/workspace/src/lib/generated/companion-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_companion_api_types -- --check diff --git a/web/workspace/src/lib/generated/legacy-server-api.ts b/web/workspace/src/lib/generated/legacy-server-api.ts index 2dd94fea6..7a34b2e60 100644 --- a/web/workspace/src/lib/generated/legacy-server-api.ts +++ b/web/workspace/src/lib/generated/legacy-server-api.ts @@ -269,6 +269,107 @@ export type ConfigCommitRequest = { entrypoints: Array; }; +export type WorkspaceConfigAccess = "read_only" | "read_write"; + +export type WorkspaceConfigNodeKind = "directory" | "file" | "missing"; + +export type WorkspaceConfigFailureClassification = "not_committed" | "unknown"; + +export type WorkspaceConfigAttachRequest = { + alias: string | null; + access: WorkspaceConfigAccess | null; +}; + +export type WorkspaceConfigAttachment = { + workspace_id: string; + connection_id: string; + alias: string; + working_directory_id: string; + access: WorkspaceConfigAccess; + name: string; + purpose: string; + content_path: string; + already_attached: boolean; +}; + +export type WorkspaceConfigCurrentResponse = WorkspaceConfigAttachment | null; + +export type WorkspaceConfigObserveRequest = { + connection_id: string; + paths: Array; + depth: number; +}; + +export type WorkspaceConfigNode = { + path: string; + kind: WorkspaceConfigNodeKind; + validator: string; + digest: string | null; + content_type: ConfigContentType | null; + operations: Array; +}; + +export type WorkspaceConfigObserveResponse = { + connection_id: string; + validator: string; + revision: number; + digest: string; + /** + * Canonical commit entrypoints from this exact metadata snapshot. + */ + entrypoints: Array; + nodes: Array; +}; + +export type WorkspaceConfigReadRequest = { + connection_id: string; + path: string; + validator: string; +}; + +export type WorkspaceConfigReadResponse = { + path: string; + content: string; + content_type: ConfigContentType; + digest: string; + validator: string; +}; + +export type WorkspaceConfigCommitRequest = { + connection_id: string; + validator: string; + request: ConfigCommitRequest; +}; + +export type WorkspaceConfigCommitResponse = { + validator: string; + revision: number; + digest: string; +}; + +export type WorkspaceConfigApiError = { + status: number; + code: string; + message: string; + classification: WorkspaceConfigFailureClassification; +}; + +export type WorkspaceConfigGrantCreateRequest = { + runtime_id: string; + worker_id: string; + access: WorkspaceConfigAccess; +}; + +export type WorkspaceConfigGrantResponse = { + grant_id: string; + workspace_id: string; + runtime_id: string; + worker_id: string; + working_directory_id: string; + access: WorkspaceConfigAccess; + revoked: boolean; +}; + export type ProfileSettingsResponse = { workspace_id: string; registry_revision: string; diff --git a/web/workspace/src/lib/generated/memory-api.ts b/web/workspace/src/lib/generated/memory-api.ts index f91d69fb5..e4154515b 100644 --- a/web/workspace/src/lib/generated/memory-api.ts +++ b/web/workspace/src/lib/generated/memory-api.ts @@ -9,6 +9,10 @@ export const MEMORY_API_LIMITS = { maxIdentifierBytes: 512, } as const; +export const SUBJEKTIV_API_LIMITS = { + maxBehaviorBytes: 16384, +} as const; + export type DiagnosticSeverity = "info" | "warning" | "error"; export type Diagnostic = { code: string, severity: DiagnosticSeverity, message: string, }; @@ -35,9 +39,11 @@ export type MemoryStagingListResponse = { limit: number, returned_count: number, export type SubjektivSubjectState = "active" | "retired"; -export type SubjektivSubjectCreateRequest = { role: string, }; +export type SubjektivSubjectCreateRequest = { role: string, behavior_md?: string, }; -export type SubjektivSubjectResponse = { id: string, role: string, state: SubjektivSubjectState, store_revision: number, created_at: string, updated_at: string, current_worker?: | import("./worker-launch-api").WorkerLaunchWorkerSummary | null, }; +export type SubjektivSubjectBehaviorUpdateRequest = { expected_behavior_revision: number, behavior_md: string, }; + +export type SubjektivSubjectResponse = { id: string, role: string, behavior_md: string, behavior_revision: number, state: SubjektivSubjectState, store_revision: number, created_at: string, updated_at: string, current_worker?: | import("./worker-launch-api").WorkerLaunchWorkerSummary | null, }; export type SubjektivSubjectListResponse = { limit: number, items: Array, next_cursor?: string | null, has_more: boolean, }; diff --git a/web/workspace/src/lib/generated/protocol-validator.ts b/web/workspace/src/lib/generated/protocol-validator.ts index 75e2fdc48..1f467d468 100644 --- a/web/workspace/src/lib/generated/protocol-validator.ts +++ b/web/workspace/src/lib/generated/protocol-validator.ts @@ -21,9 +21,9 @@ const MAX_CORRELATION_ID_BYTES = 128; const MAX_RESOURCE_ID_BYTES = 256; const MAX_WORKER_IDS_PER_SELECTOR = 256; const MAX_REJECTION_MESSAGE_BYTES = 1024; -const subscriptionFrameSchema: unknown = {"$defs":{"Alert":{"additionalProperties":false,"description":"User-facing alert emitted from the Worker layer.\n\nThis is a separate channel from `tracing` (developer logs): entries\nhere are assembled explicitly by the Worker when a condition should be\nsurfaced to the person driving the client. Keep messages short and\nhuman-readable.","properties":{"level":{"$ref":"#/$defs/AlertLevel"},"message":{"type":"string"},"source":{"$ref":"#/$defs/AlertSource"},"timestamp_ms":{"description":"Milliseconds since the Unix epoch.","format":"int64","maximum":9007199254740991.0,"minimum":-9007199254740991.0,"type":"integer"}},"required":["level","source","message","timestamp_ms"],"type":"object"},"AlertLevel":{"enum":["warn","error"],"type":"string"},"AlertSource":{"enum":["worker","engine","compactor","agents_md"],"type":"string"},"CommandEvent":{"oneOf":[{"additionalProperties":false,"properties":{"command_id":{"type":"string"},"kind":{"const":"started","type":"string"},"observed_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"tool_call_id":{"type":["string","null"]}},"required":["kind","command_id","observed_at_ms"],"type":"object"},{"additionalProperties":false,"properties":{"command_id":{"type":"string"},"content":{"type":"string"},"end_offset":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"kind":{"const":"output","type":"string"},"observed_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"start_offset":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"stream":{"$ref":"#/$defs/CommandStream"}},"required":["kind","command_id","stream","start_offset","end_offset","content","observed_at_ms"],"type":"object"},{"additionalProperties":false,"properties":{"command_id":{"type":"string"},"exit_code":{"format":"int32","maximum":2147483647.0,"minimum":-2147483648.0,"type":["integer","null"]},"kind":{"const":"terminal","type":"string"},"observed_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"status":{"$ref":"#/$defs/CommandStatus"},"stderr_end_offset":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"stdout_end_offset":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","command_id","status","stdout_end_offset","stderr_end_offset","observed_at_ms"],"type":"object"}]},"CommandSnapshot":{"additionalProperties":false,"properties":{"command_id":{"type":"string"},"exit_code":{"format":"int32","maximum":2147483647.0,"minimum":-2147483648.0,"type":["integer","null"]},"last_output_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":["integer","null"]},"observed_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"started_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"status":{"$ref":"#/$defs/CommandStatus"},"stderr":{"$ref":"#/$defs/CommandStreamSlice"},"stdout":{"$ref":"#/$defs/CommandStreamSlice"},"tool_call_id":{"type":["string","null"]}},"required":["command_id","status","started_at_ms","observed_at_ms","stdout","stderr"],"type":"object"},"CommandStatus":{"description":"Live provider-owned command status. These values are operational Console\nstate only and are never appended to Worker history.","enum":["running","completed","failed","timed_out","cancelled"],"type":"string"},"CommandStream":{"enum":["stdout","stderr"],"type":"string"},"CommandStreamSlice":{"additionalProperties":false,"properties":{"content":{"type":"string"},"end_offset":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"start_offset":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"truncated":{"type":"boolean"}},"required":["start_offset","end_offset","content","truncated"],"type":"object"},"CompactionLifecycle":{"additionalProperties":false,"description":"Stable parent-owned lifecycle for one compaction run.","properties":{"compaction_id":{"type":"string"},"ended_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":["integer","null"]},"error":{"type":["string","null"]},"internal_worker":{"anyOf":[{"$ref":"#/$defs/InternalWorkerRef"},{"type":"null"}]},"new_segment_id":{"type":["string","null"]},"revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"schema_version":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"started_at_ms":{"description":"Milliseconds since the Unix epoch.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"state":{"$ref":"#/$defs/CompactionLifecycleState"},"summary":{"type":["string","null"]}},"required":["schema_version","compaction_id","revision","state","started_at_ms"],"type":"object"},"CompactionLifecycleState":{"enum":["running","done","failed","interrupted"],"type":"string"},"CompactionPhase":{"enum":["preparing","summarizing","committing"],"type":"string"},"CompactionTrigger":{"enum":["manual","pre_run","request_threshold"],"type":"string"},"CompletionEntry":{"additionalProperties":false,"description":"One completion candidate for a prefix query.\n\n`value` is a path (file kind).","properties":{"is_dir":{"default":false,"type":"boolean"},"value":{"type":"string"}},"required":["value","is_dir"],"type":"object"},"CompletionKind":{"description":"Kind of completion requested by `Method::ListCompletions`.\n\nMirrors the completion prefix sigil: `@` → `File`.","enum":["file"],"type":"string"},"ContextTokenSource":{"description":"Provenance of the current-context token count in a [`Greeting`].","oneOf":[{"const":"measured","description":"Provider-reported input occupancy from the most recent LLM request.","type":"string"},{"const":"estimated","description":"Current history projected from the Worker's token accounting records.","type":"string"}]},"ContextUsage":{"additionalProperties":false,"description":"Current prompt-context occupancy, distinct from cumulative run traffic.","properties":{"source":{"$ref":"#/$defs/ContextTokenSource"},"tokens":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["tokens","source"],"type":"object"},"ErrorCode":{"enum":["already_running","not_running","not_paused","provider_error","tool_error","invalid_request","internal"],"type":"string"},"Event":{"oneOf":[{"additionalProperties":false,"description":"Durable Submit acceptance. A `Queued` receipt follows the durable FIFO\ncheckpoint. A `Started` receipt follows the pending activation checkpoint;\nthe corresponding [`Event::UserMessage`] marks the later history commit.\nRepeating the same request id and exact payload returns the same receipt\nwithout appending or activating twice.","properties":{"data":{"additionalProperties":false,"properties":{"disposition":{"$ref":"#/$defs/SubmissionDisposition"},"submission_id":{"type":"string"},"submission_request_id":{"type":"string"}},"required":["submission_request_id","submission_id","disposition"],"type":"object"},"event":{"const":"submission_accepted","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Correlated rejection before durable acceptance.","properties":{"data":{"additionalProperties":false,"properties":{"message":{"type":"string"},"submission_request_id":{"type":"string"}},"required":["submission_request_id","message"],"type":"object"},"event":{"const":"submission_rejected","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Durable notification acceptance. The request identity is persisted with\nthe notification receipt before this event is emitted. Repeating the\nsame request id and exact payload returns the same acknowledgement\nwithout appending the notification twice.","properties":{"data":{"additionalProperties":false,"properties":{"notification_request_id":{"type":"string"}},"required":["notification_request_id"],"type":"object"},"event":{"const":"notification_accepted","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Correlated notification rejection before durable acceptance.","properties":{"data":{"additionalProperties":false,"properties":{"message":{"type":"string"},"notification_request_id":{"type":"string"}},"required":["notification_request_id","message"],"type":"object"},"event":{"const":"notification_rejected","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Revisioned FIFO replacement following enqueue, activation, cancel, or clear.","properties":{"data":{"additionalProperties":false,"properties":{"pending":{"$ref":"#/$defs/PendingSubmissionsSnapshot"}},"required":["pending"],"type":"object"},"event":{"const":"pending_submissions_changed","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A user input message persisted as `LogEntry::AnnotatedUserInput` and\nactivated for a turn. Broadcast to every subscribed client so TUI / GUI\ninstances show the same user line that reconnect snapshots replay.","properties":{"data":{"additionalProperties":false,"properties":{"entry_id":{"description":"Stable durable history identity. Older mixed-version producers omit it.","type":["string","null"]},"segments":{"items":{"$ref":"#/$defs/Segment"},"type":"array"}},"required":["segments"],"type":"object"},"event":{"const":"user_message","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A canonical assistant/tool entry has committed to durable history.\nStreaming events remain the in-flight authority; this event supplies the\nstable identity used to reconcile that live block with paged history.","properties":{"data":{"additionalProperties":false,"properties":{"entry":{"$ref":"#/$defs/SessionSnapshotEntry"}},"required":["entry"],"type":"object"},"event":{"const":"session_entry_committed","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"One agent-injected system item committed to history.\n\nCarries the JSON form of `session_store::SystemItem`. Covers\n`Method::Notify` echoes, child-Worker lifecycle events from\n`Method::WorkerEvent`, `@` resolution payloads, and any future\nClients dispatch on the `kind` tag for typed rendering instead\nof parsing free-text prefixes like `[Notification] …` or\n`[File: …]`.\n\nOne event per `LogEntry::AnnotatedSystemItem` commit. Disk-side and\nwire-side are 1:1.","properties":{"data":{"additionalProperties":false,"properties":{"entry_id":{"description":"Stable durable history identity. Synthetic compatibility events omit it.","type":["string","null"]},"item":true},"required":["item"],"type":"object"},"event":{"const":"system_item","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A new self-driving cycle has begun (IDLE → active transition).\n\nMarker event for the start of an Invoke range; the range extends\nimplicitly until the next `InvokeStart`. Fires for every accepted\n`Method::Submit` (kind=`UserSend`), `Method::Notify` (kind=`Notify`),\n`Method::WorkerEvent` re-injection (kind=`WorkerEvent`), and any other\nIDLE-breaking trigger. Mid-run interrupts (e.g. hook output,\ntyped system reminder insertion that doesn't break IDLE) do not\nemit `InvokeStart` — they appear as `SystemItem` only.\n\nCarries `kind` only; the payload (user text / notify message /\nworker event body) is delivered separately via the immediately\nfollowing `UserMessage` / `SystemItem` event.","properties":{"data":{"additionalProperties":false,"properties":{"kind":{"$ref":"#/$defs/InvokeKind"}},"required":["kind"],"type":"object"},"event":{"const":"invoke_start","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"One AgentTurn boundary opened. An AgentTurn is a maximal run of\nLLM generation calls whose input messages are identical (i.e.\nretries from network errors / 5xx / stream disconnects collapse\ninto the same AgentTurn). When the input changes (a new tool\nresult lands, a user interrupts, etc.), the next LLM call belongs\nto a new AgentTurn.\n\n`turn` is the AgentTurn index from `Engine::turn_count`.\n\nCurrently retry is not yet implemented (`agen-stream-continuation`)\nso AgentTurn and `LlmCall` fire 1:1, but the contract here is\nthe AgentTurn boundary; consumers that want per-LLM-call signals\nmust subscribe to `LlmCallStart` / `LlmCallEnd` instead.","properties":{"data":{"additionalProperties":false,"properties":{"turn":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["turn"],"type":"object"},"event":{"const":"turn_start","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"AgentTurn closed.","properties":{"data":{"additionalProperties":false,"properties":{"result":{"$ref":"#/$defs/TurnResult"},"turn":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["turn","result"],"type":"object"},"event":{"const":"turn_end","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"One LLM generation call started (1 request → 1 generation, retry\nincluded). Multiple `LlmCall*` pairs may fire inside a single\n`TurnStart` / `TurnEnd` pair when a request is retried; today\nthey fire 1:1 because retry is not implemented.\n\n`llm_call` is the worker-wide running counter from\n`Engine::llm_call_count`.","properties":{"data":{"additionalProperties":false,"properties":{"llm_call":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["llm_call"],"type":"object"},"event":{"const":"llm_call_start","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"LLM generation call ended.","properties":{"data":{"additionalProperties":false,"properties":{"llm_call":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["llm_call"],"type":"object"},"event":{"const":"llm_call_end","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A transport-level LLM request retry has been scheduled.\n\nThis is operational state for clients to render while the worker is\nwaiting in backoff. It is not part of conversation history.","properties":{"data":{"additionalProperties":false,"properties":{"elapsed_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"error":{"type":"string"},"failed_attempt":{"description":"The attempt that just failed. 1 origin.","format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"llm_call":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"max_attempts":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"status":{"format":"uint16","maximum":65535.0,"minimum":0.0,"type":["integer","null"]},"wait_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["llm_call","failed_attempt","max_attempts","wait_ms","elapsed_ms","error"],"type":"object"},"event":{"const":"llm_retry","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Stream generation was interrupted after events had begun and the worker\nis continuing with a follow-up LLM request.","properties":{"data":{"additionalProperties":false,"properties":{"attempt":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"llm_call":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"max_attempts":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"reason":{"type":"string"}},"required":["llm_call","attempt","max_attempts","reason"],"type":"object"},"event":{"const":"llm_continuation","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"text":{"type":"string"}},"required":["text"],"type":"object"},"event":{"const":"text_delta","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"text":{"type":"string"}},"required":["text"],"type":"object"},"event":{"const":"text_done","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A reasoning / thinking block has started.\n\nAlways paired with a `ThinkingDone`. `ThinkingDelta` is optional:\nsome providers (or some configurations) emit thinking metadata\nwithout plaintext, in which case Start → Done arrive with no\ndeltas in between. Multiple thinking blocks per turn are allowed.","properties":{"event":{"const":"thinking_start","type":"string"}},"required":["event"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"text":{"type":"string"}},"required":["text"],"type":"object"},"event":{"const":"thinking_delta","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Thinking block completed. `text` is the full accumulated body\n(empty string when the provider didn't emit plaintext).","properties":{"data":{"additionalProperties":false,"properties":{"text":{"type":"string"}},"required":["text"],"type":"object"},"event":{"const":"thinking_done","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"id":{"type":"string"},"name":{"type":"string"}},"required":["id","name"],"type":"object"},"event":{"const":"tool_call_start","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"id":{"type":"string"},"json":{"type":"string"}},"required":["id","json"],"type":"object"},"event":{"const":"tool_call_args_delta","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"arguments":{"type":"string"},"id":{"type":"string"},"name":{"type":"string"}},"required":["id","name","arguments"],"type":"object"},"event":{"const":"tool_call_done","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"disposition":{"anyOf":[{"$ref":"#/$defs/ToolResultDisposition"},{"type":"null"}]},"id":{"type":"string"},"is_error":{"default":false,"type":"boolean"},"output":{"description":"Full tool output. Absent when the tool chose to return\nsummary-only, or when the result was pruned.","type":["string","null"]},"summary":{"description":"Short human-readable summary. Always present; used by clients\nthat only want a 1-line rendering (e.g. collapsed views).","type":"string"}},"required":["id","summary","is_error"],"type":"object"},"event":{"const":"tool_result","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Token accounting for one LLM request.\n\n`input_tokens` is the prompt prefix occupancy (cache reads /\ncache writes included), as normalised by the worker layer.\n`cache_read_input_tokens` is the cache-hit subset of that\noccupancy; subtracting it yields the \"net upload\" the client\nactually paid full price to send on this request, which is what\nthe TUI status line accumulates per turn.","properties":{"data":{"additionalProperties":false,"properties":{"cache_read_input_tokens":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":["integer","null"]},"input_tokens":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":["integer","null"]},"output_tokens":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":["integer","null"]}},"type":"object"},"event":{"const":"usage","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Authoritative current-context occupancy after a non-request boundary.\n\nUnlike [`Event::Usage`], this is not cumulative request traffic. It lets\nconnected clients converge with reconnect snapshots after run completion,\ncompaction, rewind, and other history rewrites. `None` explicitly means\nthe current value is unavailable and stale occupancy must be cleared.","properties":{"data":{"additionalProperties":false,"properties":{"usage":{"anyOf":[{"$ref":"#/$defs/ContextUsage"},{"type":"null"}]}},"type":"object"},"event":{"const":"context_usage","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"result":{"$ref":"#/$defs/RunResult"}},"required":["result"],"type":"object"},"event":{"const":"run_end","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"code":{"$ref":"#/$defs/ErrorCode"},"message":{"type":"string"}},"required":["code","message"],"type":"object"},"event":{"const":"error","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Sent exactly once at the start of every client connection.\n\n`entries` is the session-log mirror at subscribe time, serialised\nas the JSON form of `session_store::LogEntry`. This is the\nbulk-reconstruction lane: clients walk the entries to seed their\nderived view.\n\n`greeting` and `status` accompany the snapshot so clients render\nworker identity and current controller state without an extra round\ntrip.\n\nLive updates after the snapshot arrive through the streaming\nevents (`TextDelta` / `ToolCall*` / `ToolResult` / etc.) plus\nrole-specific entry events (`SegmentRotated` / `SystemItem`) —\nthere is no generic \"every committed entry\" broadcast.","properties":{"data":{"additionalProperties":false,"properties":{"greeting":{"$ref":"#/$defs/Greeting"},"in_flight":{"$ref":"#/$defs/InFlightSnapshot","description":"Unfinished model output that has already streamed in the current\nrun but is not yet represented by committed snapshot entries."},"internal_workers":{"description":"Parent-owned Internal Worker sessions visible to this client.\nService-private Internal Workers are deliberately excluded.","items":{"$ref":"#/$defs/InternalWorkerSnapshot"},"type":"array"},"session":{"$ref":"#/$defs/SessionSnapshot"},"state":{"$ref":"#/$defs/WorkerStateSnapshot","description":"Full revisioned live execution state. `Stopped` remains Runtime\ncatalog authority and is deliberately not represented here."}},"required":["session","greeting","state"],"type":"object"},"event":{"const":"snapshot","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A live event from a parent-owned Internal Worker. The payload reuses the\nnormal Worker event vocabulary while the wrapper carries stable origin\nidentity and a per-child revision fence.","properties":{"data":{"additionalProperties":false,"properties":{"event":{"$ref":"#/$defs/Event"},"revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"worker":{"$ref":"#/$defs/InternalWorkerRef"}},"required":["worker","revision","event"],"type":"object"},"event":{"const":"internal_worker","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Terminal removal fence for one parent-owned Internal Worker session.\n\nClients discard the matching child and descendants, then ignore later\nnested events for this identity until an authoritative snapshot replaces\nthe projection.","properties":{"data":{"additionalProperties":false,"properties":{"revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"worker":{"$ref":"#/$defs/InternalWorkerRef"}},"required":["worker","revision"],"type":"object"},"event":{"const":"internal_worker_removed","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Server-side segment log rotated to a fresh `SegmentStart`.\n\nFires on compaction and on auto-fork when the store head drifts\nA compaction/fork has replaced the authoritative segment. Clients drop\ntheir derived view and reseed from the canonical committed snapshot.","properties":{"data":{"additionalProperties":false,"properties":{"session":{"$ref":"#/$defs/SessionSnapshot"}},"required":["session"],"type":"object"},"event":{"const":"segment_rotated","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Current Worker controller status. Broadcast on every controller-level\ntransition and included in `History` snapshots for late attach.","properties":{"data":{"additionalProperties":false,"properties":{"snapshot":{"$ref":"#/$defs/WorkerStateSnapshot"}},"required":["snapshot"],"type":"object"},"event":{"const":"worker_state","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"acknowledgement":{"$ref":"#/$defs/WorkerCommandAcknowledgement"}},"required":["acknowledgement"],"type":"object"},"event":{"const":"command_acknowledged","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Bounded, provider-owned command telemetry for the live Console. This is\nintentionally not a history entry and is reconstructed from\n`Snapshot.in_flight.commands` after reconnect.","properties":{"data":{"additionalProperties":false,"properties":{"event":{"$ref":"#/$defs/CommandEvent"}},"required":["event"],"type":"object"},"event":{"const":"command","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Reply to `Method::ListCompletions`. Delivered only to the\nrequesting socket (not broadcast). `entries` is empty when no\ncandidates match or when the requested kind has no resolver\nwired up.","properties":{"data":{"additionalProperties":false,"properties":{"entries":{"items":{"$ref":"#/$defs/CompletionEntry"},"type":"array"},"kind":{"$ref":"#/$defs/CompletionKind"}},"required":["kind","entries"],"type":"object"},"event":{"const":"completions","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Reply to `Method::ListRewindTargets`. Clients should only open a picker\nin response to their own pending request; the event may be broadcast.","properties":{"data":{"additionalProperties":false,"properties":{"head_entries":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"targets":{"items":{"$ref":"#/$defs/RewindTarget"},"type":"array"}},"required":["head_entries","targets"],"type":"object"},"event":{"const":"rewind_targets","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A rewind has truncated the authoritative session. `session` is the\nretained canonical snapshot clients should use to reseed display state.","properties":{"data":{"additionalProperties":false,"properties":{"input":{"items":{"$ref":"#/$defs/Segment"},"type":"array"},"session":{"$ref":"#/$defs/SessionSnapshot"},"summary":{"$ref":"#/$defs/RewindSummary"}},"required":["session","input","summary"],"type":"object"},"event":{"const":"rewind_applied","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Reply to `Method::ListWorkers`. Payload is a stable JSON value so the Worker\ncrate can evolve discovery fields without introducing a protocol\ndependency on session-store.","properties":{"data":{"additionalProperties":false,"properties":{"workers":true},"required":["workers"],"type":"object"},"event":{"const":"workers_listed","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Reply to `Method::RestoreWorker`.","properties":{"data":{"additionalProperties":false,"properties":{"result":true},"required":["result"],"type":"object"},"event":{"const":"worker_restored","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Reply to `Method::RegisterPeer`.","properties":{"data":{"additionalProperties":false,"properties":{"result":true},"required":["result"],"type":"object"},"event":{"const":"peer_registered","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"$ref":"#/$defs/Alert"},"event":{"const":"alert","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Latest memory extract/consolidation lifecycle event for UI observability.\n\nThis is not part of LLM history or prompt context; clients may display it\nbriefly as operational status.","properties":{"data":{"$ref":"#/$defs/MemoryWorkerEvent"},"event":{"const":"memory_worker","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Runtime-only compaction progress. `None` clears the current status.\nThis never enters Session history and carries no operation or Segment identity.","properties":{"data":{"additionalProperties":false,"properties":{"compaction":{"anyOf":[{"$ref":"#/$defs/InFlightCompaction"},{"type":"null"}]}},"type":"object"},"event":{"const":"compaction_progress","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Legacy compaction lifecycle event retained for wire read compatibility.","properties":{"data":{"additionalProperties":false,"properties":{"lifecycle":{"$ref":"#/$defs/CompactionLifecycle"}},"required":["lifecycle"],"type":"object"},"event":{"const":"compact_start","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Compaction completed and the session was rotated.","properties":{"data":{"additionalProperties":false,"properties":{"lifecycle":{"$ref":"#/$defs/CompactionLifecycle"}},"required":["lifecycle"],"type":"object"},"event":{"const":"compact_done","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Compaction failed or was cancelled. The session is unchanged.","properties":{"data":{"additionalProperties":false,"properties":{"lifecycle":{"$ref":"#/$defs/CompactionLifecycle"}},"required":["lifecycle"],"type":"object"},"event":{"const":"compact_failed","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"event":{"const":"shutdown","type":"string"}},"required":["event"],"type":"object"}]},"EventSubscriptionSelector":{"oneOf":[{"additionalProperties":false,"properties":{"topic":{"const":"runtime_workers","type":"string"}},"required":["topic"],"type":"object"},{"additionalProperties":false,"properties":{"topic":{"const":"worker_lifecycle","type":"string"},"worker_ids":{"$ref":"#/$defs/SubscriptionWorkerIds"}},"required":["topic","worker_ids"],"type":"object"},{"additionalProperties":false,"properties":{"runtime_id":{"type":["string","null"]},"topic":{"const":"worker_protocol","type":"string"},"worker_id":{"$ref":"#/$defs/SubscriptionWorkerId"}},"required":["topic","worker_id"],"type":"object"},{"additionalProperties":false,"description":"Server-derived Workspace projection. Workspace identity comes from the\nauthenticated connection and is deliberately absent from this selector.","properties":{"topic":{"const":"workspace_workers","type":"string"}},"required":["topic"],"type":"object"},{"additionalProperties":false,"description":"Typed extension point for the later Workdir subscription slice.","properties":{"topic":{"const":"workspace_workdirs","type":"string"}},"required":["topic"],"type":"object"}]},"Greeting":{"additionalProperties":false,"description":"Worker self-description rendered by clients when a session starts empty.\n\nSnapshot metadata contains only resolved public execution information so\nclients do not need their own copy of the manifest.","properties":{"context_tokens":{"default":0,"description":"Legacy numeric mirror retained for mixed-version clients.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"context_usage":{"anyOf":[{"$ref":"#/$defs/ContextUsage"},{"type":"null"}],"description":"Typed current-context value. Older snapshots omit it; clients may fall\nback to `context_tokens` only when `context_window` is known."},"context_window":{"default":0,"description":"Resolved effective model context window in tokens. Legacy producers use\nzero when this is unavailable; clients must treat that as unknown.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"cwd":{"type":"string"},"model":{"type":"string"},"provider":{"type":"string"},"reasoning":{"anyOf":[{"$ref":"#/$defs/ReasoningConfig"},{"type":"null"}],"description":"Effective reasoning control. Absent means unspecified or unavailable."},"scope_summary":{"type":"string"},"tools":{"items":{"type":"string"},"type":"array"},"worker_name":{"type":"string"}},"required":["worker_name","cwd","provider","model","scope_summary","tools","context_tokens","context_window"],"type":"object"},"InFlightBlock":{"oneOf":[{"additionalProperties":false,"properties":{"finished":{"type":"boolean"},"kind":{"const":"text","type":"string"},"text":{"type":"string"}},"required":["kind","text"],"type":"object"},{"additionalProperties":false,"properties":{"finished":{"type":"boolean"},"kind":{"const":"thinking","type":"string"},"text":{"type":"string"}},"required":["kind","text"],"type":"object"},{"additionalProperties":false,"properties":{"args":{"type":"string"},"id":{"type":"string"},"kind":{"const":"tool_call","type":"string"},"name":{"type":"string"},"state":{"$ref":"#/$defs/InFlightToolCallState"}},"required":["kind","id","name","args"],"type":"object"}]},"InFlightCompaction":{"additionalProperties":false,"properties":{"phase":{"$ref":"#/$defs/CompactionPhase"},"started_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"trigger":{"$ref":"#/$defs/CompactionTrigger"}},"required":["phase","started_at_ms","trigger"],"type":"object"},"InFlightSnapshot":{"additionalProperties":false,"description":"Unfinished model output and active command state included in\n`Event::Snapshot` for clients that attach while work is still streaming.\n\nThese blocks are presentation state only: they are reconstructed from the\nactive Worker controller and must not be treated as committed assistant\nhistory. Finalized assistant items continue to come from ordinary snapshot\nentries.","properties":{"blocks":{"items":{"$ref":"#/$defs/InFlightBlock"},"type":"array"},"commands":{"items":{"$ref":"#/$defs/CommandSnapshot"},"type":"array"},"compaction":{"anyOf":[{"$ref":"#/$defs/InFlightCompaction"},{"type":"null"}],"description":"The currently running compaction, if any.\n\nThis is lifecycle progress only. Candidate history and the staged\nSegment remain private until the Segment is activated atomically."}},"type":"object"},"InFlightToolCallState":{"enum":["pending","streaming_args","done"],"type":"string"},"InternalWorkerKind":{"description":"Presentation category for an Internal Worker exposed through its parent's\nprotocol stream. Internal Workers never become independently addressable\nprotocol subjects.","oneOf":[{"enum":["sub_worker"],"type":"string"},{"additionalProperties":false,"properties":{"service":{"additionalProperties":false,"properties":{"kind":{"type":"string"}},"required":["kind"],"type":"object"}},"required":["service"],"type":"object"}]},"InternalWorkerRef":{"additionalProperties":false,"description":"Stable presentation identity for one parent-owned Internal Worker session.\n`name` is display-only; `session_id` is the identity used by clients.","properties":{"kind":{"$ref":"#/$defs/InternalWorkerKind"},"name":{"type":"string"},"parent_session_id":{"type":["string","null"]},"session_id":{"type":"string"}},"required":["session_id","name","kind"],"type":"object"},"InternalWorkerSnapshot":{"additionalProperties":false,"description":"Reconnect state for one visible Internal Worker. The revision fences live\n`Event::InternalWorker` updates that raced with parent snapshot assembly.","properties":{"error":{"type":["string","null"]},"greeting":{"anyOf":[{"$ref":"#/$defs/Greeting"},{"type":"null"}],"description":"Public execution metadata for this child. Older snapshots and service\nworkers may omit it; clients must never substitute the parent's values."},"in_flight":{"$ref":"#/$defs/InFlightSnapshot"},"internal_workers":{"items":{"$ref":"#/$defs/InternalWorkerSnapshot"},"type":"array"},"revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"session":{"$ref":"#/$defs/SessionSnapshot"},"status":{"$ref":"#/$defs/WorkerStatus","default":"idle"},"worker":{"$ref":"#/$defs/InternalWorkerRef"}},"required":["worker","revision","session","status"],"type":"object"},"InvokeKind":{"description":"Kind of trigger that opened a new Invoke (IDLE → active) range.\n\nOne Invoke groups all entries from this trigger up to the next\n`Invoke` marker. The kind is the only payload — content (user text,\nnotify message, worker event body) is delivered by the immediately\nfollowing Turn entry, not by the marker itself.","oneOf":[{"const":"user_send","description":"`Method::Submit` — a user submission.","type":"string"},{"const":"notify","description":"`Method::Notify` — free-text notification injected into history.","type":"string"},{"const":"worker_event","description":"`Method::WorkerEvent` — typed lifecycle report from a child Worker.","type":"string"},{"const":"system_reminder","description":"A typed system reminder that crosses an IDLE boundary (mid-run\nreminders that don't break IDLE are SystemItem-only and do not\nopen a new Invoke).","type":"string"},{"const":"wakeup","description":"Cron / RemoteTrigger style scheduled wake-up. Reserved; no\nproducer is wired today but the variant is part of the initial\nset so future schedulers don't have to amend the wire enum.","type":"string"}]},"MemoryWorkerEvent":{"additionalProperties":false,"properties":{"message":{"description":"Human-readable compact form for actionbar rendering.","type":"string"},"reason":{"type":"string"},"run_id":{"type":"string"},"status":{"type":"string"},"timestamp_ms":{"description":"Milliseconds since the Unix epoch.","format":"int64","maximum":9007199254740991.0,"minimum":-9007199254740991.0,"type":"integer"},"trigger":{"type":"string"},"worker":{"type":"string"}},"required":["worker","status","run_id","trigger","reason","message","timestamp_ms"],"type":"object"},"Method":{"oneOf":[{"additionalProperties":false,"description":"Durably accept typed input for immediate activation or the session FIFO.\nRunning or Paused Workers queue Submit for a later turn; use Notify for\nadvisory information to incorporate into work already in progress.\n\n`submission_request_id` is generated by the authenticated caller and is\nused only for idempotent retry. Worker allocates the durable\n`submission_id` returned by [`Event::SubmissionAccepted`].","properties":{"method":{"const":"submit","type":"string"},"params":{"additionalProperties":false,"properties":{"input":{"items":{"$ref":"#/$defs/Segment"},"type":"array"},"submission_request_id":{"type":"string"}},"required":["submission_request_id","input"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Human-readable text injected into the target Worker's LLM context as a\nnon-blocking system message. Accepted notifications are delivered at the\nnext between-turn boundary, or start a coalesced notification run while\nthe Worker is idle. No side effects beyond LLM context; use\n`WorkerEvent` for typed lifecycle reports.","properties":{"method":{"const":"notify","type":"string"},"params":{"additionalProperties":false,"properties":{"message":{"type":"string"},"notification_request_id":{"type":"string"}},"required":["notification_request_id","message"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Typed lifecycle report from a child Worker to its direct parent.","properties":{"method":{"const":"worker_event","type":"string"},"params":{"$ref":"#/$defs/WorkerEvent"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Return the authoritative FIFO summary without exposing queued payloads.","properties":{"method":{"const":"list_pending_submissions","type":"string"}},"required":["method"],"type":"object"},{"additionalProperties":false,"description":"Remove one queued submission. Running or already activated submissions\nare immutable and therefore cannot be cancelled here.","properties":{"method":{"const":"cancel_pending_submission","type":"string"},"params":{"additionalProperties":false,"properties":{"expected_revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"submission_id":{"type":"string"}},"required":["submission_id","expected_revision"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Remove every queued submission while preserving the active run.","properties":{"method":{"const":"clear_pending_submissions","type":"string"},"params":{"additionalProperties":false,"properties":{"expected_revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["expected_revision"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Activate the next queued submission while the Worker is idle. This is an\nexplicit recovery operation and never resumes a paused run implicitly.","properties":{"method":{"const":"continue_pending","type":"string"},"params":{"additionalProperties":false,"properties":{"expected_head_id":{"type":"string"},"expected_revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["expected_revision","expected_head_id"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"properties":{"method":{"const":"resume","type":"string"},"params":{"additionalProperties":false,"properties":{"command":{"$ref":"#/$defs/WorkerCommandEnvelope"}},"required":["command"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"properties":{"method":{"const":"cancel","type":"string"},"params":{"additionalProperties":false,"properties":{"command":{"$ref":"#/$defs/WorkerCommandEnvelope"}},"required":["command"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Stop the in-flight turn and transition to `Paused`.\n\nUnlike `Cancel` (which discards and returns to `Idle`), a paused\nWorker can resume the interrupted work via `Resume`, or accept a\nfresh `Submit` (orphan `tool_use` items are closed with a\nsynthetic tool result before the new user message is appended).","properties":{"method":{"const":"pause","type":"string"},"params":{"additionalProperties":false,"properties":{"command":{"$ref":"#/$defs/WorkerCommandEnvelope"}},"required":["command"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Request an explicit compaction while the Worker is otherwise idle.\n\nThis is a typed control method: clients must not send `compact` as a\n`Method::Submit` user message.","properties":{"method":{"const":"compact","type":"string"},"params":{"additionalProperties":false,"properties":{"command":{"$ref":"#/$defs/WorkerCommandEnvelope"}},"required":["command"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Ask the Worker to list valid rewind targets from its authoritative session log.","properties":{"method":{"const":"list_rewind_targets","type":"string"}},"required":["method"],"type":"object"},{"additionalProperties":false,"description":"Truncate the current session back to the selected rewind target and\nreturn the selected user input to the client composer.","properties":{"method":{"const":"rewind_to","type":"string"},"params":{"additionalProperties":false,"properties":{"expected_head_entries":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"target":{"$ref":"#/$defs/RewindTargetId"}},"required":["target","expected_head_entries"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"properties":{"method":{"const":"shutdown","type":"string"},"params":{"additionalProperties":false,"properties":{"command":{"$ref":"#/$defs/WorkerCommandEnvelope"}},"required":["command"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Request a list of completion candidates from the Worker.\n\nReply is sent on the same socket as `Event::Completions` (not\nbroadcast). The IPC server handles this directly and writes\nthe response straight back to the requesting socket.","properties":{"method":{"const":"list_completions","type":"string"},"params":{"additionalProperties":false,"properties":{"kind":{"$ref":"#/$defs/CompletionKind"},"prefix":{"type":"string"}},"required":["kind","prefix"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"List Workers visible to this Worker from durable Worker state and the spawned-child\nregistry. This is not a host-wide Worker universe query.","properties":{"method":{"const":"list_workers","type":"string"}},"required":["method"],"type":"object"},{"additionalProperties":false,"description":"Restore a visible stopped/restorable Worker, or report that it is already\nlive. Missing state and not-visible state are distinct errors.","properties":{"method":{"const":"restore_worker","type":"string"},"params":{"additionalProperties":false,"properties":{"name":{"type":"string"}},"required":["name"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Register another existing Worker as a reciprocal peer of this Worker.\n\nThis is metadata/control state only: it does not ask the target's live\ncontroller for consent, and it must not grant delegated scope,\nspawned-child ownership, output cursors, or child lifecycle authority.","properties":{"method":{"const":"register_peer","type":"string"},"params":{"additionalProperties":false,"properties":{"name":{"type":"string"}},"required":["name"],"type":"object"}},"required":["method","params"],"type":"object"}]},"PasteArtifactAvailability":{"enum":["available","unavailable","integrity_failed"],"type":"string"},"PasteArtifactMediaType":{"description":"One typed piece of a user submission.\n\n`Method::Submit` and `Event::UserMessage` carry `Vec`. Dumb\nclients (CLI piping, scripts) only need to produce a single\n`Segment::Text`; richer clients (TUI / GUI) construct typed atoms\n(paste chips, file refs) and\nsend them through directly so the Worker side never has to re-parse a\nflattened string.\n\nForward compat: payloads with unknown `kind` deserialize to\n`Segment::Unknown`. Worker treats this the same as known-but-unresolved\nvariants — emits an alert and inserts a `[unknown input segment]`\nplaceholder into the LLM context so neither user nor LLM is blind to\nthe dropped intent.","enum":["text_plain_utf8"],"type":"string"},"PasteArtifactRef":{"additionalProperties":false,"description":"Session-owned reference to a large pasted-input artifact.\n\nThe reference contains only bounded integrity and provenance metadata. The\nartifact body remains in session storage and is available to the model only\nthrough the scoped paste-artifact tools installed by Worker.","properties":{"artifact_id":{"type":"string"},"availability":{"$ref":"#/$defs/PasteArtifactAvailability","description":"Availability observed when this immutable reference was committed.\nReads revalidate storage and integrity rather than trusting this field."},"byte_len":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"char_count":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"created_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"line_count":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"media_type":{"$ref":"#/$defs/PasteArtifactMediaType"},"sha256":{"type":"string"},"source_entry_id":{"type":"string"}},"required":["artifact_id","created_at_ms","media_type","availability","byte_len","char_count","line_count","sha256","source_entry_id"],"type":"object"},"PendingSubmissionSummary":{"additionalProperties":false,"description":"Bounded public projection of one pending submission. Only a short display\npreview is exposed; full payload segments and provenance remain in the log.","properties":{"accepted_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"byte_len":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"preview":{"description":"Single-line display text, at most 240 Unicode scalars plus an ellipsis.\nAbsent on older runtimes. Never used to reconstruct the submitted input.","type":["string","null"]},"segment_count":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"submission_id":{"type":"string"}},"required":["submission_id","accepted_at_ms","segment_count","byte_len"],"type":"object"},"PendingSubmissionsSnapshot":{"additionalProperties":false,"description":"Revisioned session-owned FIFO projection used by snapshots and live events.","properties":{"head_id":{"default":null,"type":["string","null"]},"notification_count":{"default":0,"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"notification_previews":{"description":"Ordered display-only previews of waiting notifications, each bounded like\nPendingSubmissionSummary::preview. Older runtimes omit this field.","items":{"type":"string"},"type":"array"},"revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"submissions":{"default":[],"items":{"$ref":"#/$defs/PendingSubmissionSummary"},"type":"array"}},"required":["revision","head_id","notification_count","submissions"],"type":"object"},"Permission":{"description":"Permission lattice used by [`ScopeRule`].\n\nThe derived `Ord` instance follows declaration order, so\n`Read < Write`. Allow rules grant the stated level (and by extension\neverything below); deny rules cap the effective level **strictly\nbelow** the stated level.","enum":["read","write"],"type":"string"},"ReasoningConfig":{"description":"Public reasoning configuration resolved for the model client.\n\nThis deliberately exposes only the effective provider control, not the\nWorker manifest or provider credentials. The tagged shape prevents token\nbudgets from being presented as effort labels.","oneOf":[{"additionalProperties":false,"properties":{"effort":{"type":"string"},"kind":{"const":"effort","type":"string"}},"required":["kind","effort"],"type":"object"},{"additionalProperties":false,"properties":{"budget_tokens":{"format":"int32","maximum":2147483647.0,"minimum":-2147483648.0,"type":"integer"},"kind":{"const":"budget_tokens","type":"string"}},"required":["kind","budget_tokens"],"type":"object"}]},"RewindSummary":{"additionalProperties":false,"properties":{"discarded_entries":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"tool_side_effect_warning":{"type":"boolean"},"truncated_to_entries":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["truncated_to_entries","discarded_entries","tool_side_effect_warning"],"type":"object"},"RewindTarget":{"additionalProperties":false,"properties":{"disabled_reason":{"type":["string","null"]},"eligible":{"type":"boolean"},"expected_head_entries":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"id":{"$ref":"#/$defs/RewindTargetId"},"preview":{"type":"string"},"timestamp_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":["integer","null"]},"truncate_entries":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"turn_index":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"warning":{"type":["string","null"]}},"required":["id","expected_head_entries","truncate_entries","turn_index","preview","eligible"],"type":"object"},"RewindTargetId":{"additionalProperties":false,"properties":{"segment_id":{"type":"string"},"user_input_entry_index":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["segment_id","user_input_entry_index"],"type":"object"},"RunFailureKind":{"enum":["engine","compaction"],"type":"string"},"RunResult":{"oneOf":[{"enum":["finished","paused","limit_reached"],"type":"string"},{"const":"cancelled","description":"The logical Run ended because the user intentionally cancelled it after\nobservable work. Unlike `RolledBack`, committed history is retained.","type":"string"},{"const":"rolled_back","description":"The accepted Method::Submit produced no assistant/tool output before\nuser interruption, so the Worker rolled the submit-time turn state back\nto its pre-submit snapshot. Clients should treat the Worker as Idle and\nrestore the just-submitted input into the editable composer if desired.","type":"string"}]},"RunResumeSource":{"enum":["compaction","pause"],"type":"string"},"RunYieldReason":{"enum":["compaction"],"type":"string"},"ScopeRule":{"additionalProperties":false,"description":"A single allow or deny rule inside a scope configuration.","properties":{"permission":{"$ref":"#/$defs/Permission","description":"Permission level this rule grants (allow) or caps strictly below\n(deny)."},"recursive":{"default":true,"description":"When `false`, the rule only matches the target itself and its\ndirect children. Defaults to `true`.","type":"boolean"},"symlink_policy":{"$ref":"#/$defs/SymlinkPolicy","default":"resolved","description":"Which path identity an allow rule uses when symbolic links are\nencountered. Deny rules always inspect both identities."},"target":{"description":"Target path. Must be absolute by the time a `Scope` is built from\nthis rule — relative paths are resolved per-layer against the\nmanifest file's directory (cwd for overlay layers) before cascade\nmerge.","type":"string"}},"required":["target","permission","recursive","symlink_policy"],"type":"object"},"Segment":{"oneOf":[{"additionalProperties":false,"description":"Free-form text. The fallback every client can produce.","properties":{"content":{"type":"string"},"kind":{"const":"text","type":"string"}},"required":["kind","content"],"type":"object"},{"additionalProperties":false,"description":"Bracketed-paste capture from a TUI-style client. `id`, `chars`\nand `lines` carry the metadata needed to re-render a\n`[Clipboard #N | X chars, Y lines]` chip in `Event::UserMessage`\nre-broadcast.","properties":{"chars":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"content":{"type":"string"},"id":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"kind":{"const":"paste","type":"string"},"lines":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"}},"required":["kind","id","chars","lines","content"],"type":"object"},{"additionalProperties":false,"description":"Internal reference produced when Worker stores a large `Paste` before\ncommitting input. Clients may receive this in history/event projections;\nthe body is intentionally absent.","properties":{"artifact":{"$ref":"#/$defs/PasteArtifactRef"},"kind":{"const":"paste_artifact","type":"string"}},"required":["kind","artifact"],"type":"object"},{"additionalProperties":false,"description":"Client-local file uploaded into the owning Worker session before submit.\nThe Session Log stores only this immutable reference, never file bytes or\nthe client's local path.","properties":{"file":{"$ref":"#/$defs/UploadedFileRef"},"kind":{"const":"uploaded_file","type":"string"}},"required":["kind","file"],"type":"object"},{"additionalProperties":false,"description":"`@` file-system reference. Worker resolves readable files to\n`[File: ]` attachments and readable normal directories to shallow\n`[Dir: ]` listings; the flattened user text keeps the literal\n`@` placeholder either way.","properties":{"kind":{"const":"file_ref","type":"string"},"path":{"type":"string"}},"required":["kind","path"],"type":"object"},{"additionalProperties":false,"description":"Source-qualified request for the host to start a Flow before committing\nthis user input. Runtime resolves it through Workspace authority and\nreplaces it with the entered state's instructions before Worker-side\ninput resolution.","properties":{"kind":{"const":"flow","type":"string"},"selector":{"type":"string"}},"required":["kind","selector"],"type":"object"},{"additionalProperties":false,"description":"Unknown variant from a newer client. Worker treats this as an\nunresolved input — surfaces an alert and inserts a placeholder.\nRound-trip is lossy: re-serializing yields `{\"kind\":\"unknown\"}`.","properties":{"kind":{"const":"unknown","type":"string"}},"required":["kind"],"type":"object"}]},"SessionContentPart":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"text","type":"string"},"text":{"type":"string"}},"required":["kind","text"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"refusal","type":"string"},"refusal":{"type":"string"}},"required":["kind","refusal"],"type":"object"}]},"SessionEntryProvenance":{"enum":["human_input","worker_input","flow_instruction","backend_instruction","model_output","tool_output","derived_summary","legacy_unknown"],"type":"string"},"SessionMessageRole":{"enum":["user","assistant"],"type":"string"},"SessionSnapshot":{"additionalProperties":false,"description":"Canonical, storage-independent projection of committed session history.\n\nWorker protocols expose this DTO instead of append-log records. New\nstorage variants can therefore be added without teaching every client how\nto replay the durable log format.","properties":{"entries":{"items":{"$ref":"#/$defs/SessionSnapshotEntry"},"type":"array"},"pending_submissions":{"$ref":"#/$defs/PendingSubmissionsSnapshot","default":{"head_id":null,"notification_count":0,"revision":0,"submissions":[]}}},"required":["entries","pending_submissions"],"type":"object"},"SessionSnapshotEntry":{"oneOf":[{"additionalProperties":false,"properties":{"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"kind":{"const":"user_input","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"segments":{"items":{"$ref":"#/$defs/Segment"},"type":"array"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","segments","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"properties":{"content":{"items":{"$ref":"#/$defs/SessionContentPart"},"type":"array"},"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"kind":{"const":"message","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"role":{"$ref":"#/$defs/SessionMessageRole"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","role","content","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"properties":{"arguments":{"type":"string"},"call_id":{"type":"string"},"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"kind":{"const":"tool_call","type":"string"},"name":{"type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","call_id","name","arguments","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"properties":{"attachments":{"items":{"$ref":"#/$defs/SessionToolAttachment"},"type":"array"},"call_id":{"type":"string"},"content":{"type":["string","null"]},"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"is_error":{"type":"boolean"},"kind":{"const":"tool_result","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"summary":{"type":"string"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","call_id","summary","is_error","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"properties":{"content":{"type":"string"},"data":true,"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"item_kind":{"type":"string"},"kind":{"const":"system_item","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","item_kind","content","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"description":"Durable, non-terminal control return from one logical Run. This is not a\ncompleted run and must not be used to infer the current Worker state.","properties":{"active_run_turn_count":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"kind":{"const":"run_yielded","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"reason":{"$ref":"#/$defs/RunYieldReason"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","reason","active_run_turn_count","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"description":"Durable boundary immediately before Engine execution resumes. The source\ndistinguishes automatic compaction continuation from an intentional\nuser-pause resume; it does not prove that later model output occurred.","properties":{"active_run_turn_count":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"kind":{"const":"run_resumed","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"source":{"$ref":"#/$defs/RunResumeSource"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","source","active_run_turn_count","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"description":"Durable terminal for an intentional cancellation. Consumers must not\npresent this lifecycle result as an execution failure.","properties":{"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"kind":{"const":"run_cancelled","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"properties":{"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"failure":{"anyOf":[{"$ref":"#/$defs/RunFailureKind"},{"type":"null"}]},"kind":{"const":"run_error","type":"string"},"message":{"type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","message","entry_id","timestamp","provenance"],"type":"object"}]},"SessionToolAttachment":{"additionalProperties":false,"properties":{"attachment_id":{"description":"Stable, bounded identity derived from the owning Session entry, attachment\nposition, and immutable image bytes. It is a locator, not a bearer secret.","type":"string"},"byte_len":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"media_type":{"type":"string"}},"required":["attachment_id","media_type","byte_len"],"type":"object"},"SubmissionDisposition":{"description":"Durable acceptance result for one idempotent Submit request.","oneOf":[{"enum":["started"],"type":"string"},{"const":"queued","description":"Input is durably held in the session FIFO until eligible for activation.","type":"string"}]},"SubscriptionEvent":{"oneOf":[{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"payload":{"$ref":"#/$defs/SubscriptionEventPayload"},"subscription_id":{"$ref":"#/$defs/SubscriptionId"}},"required":["subscription_id","payload"],"type":"object"},"event":{"const":"event","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"code":{"$ref":"#/$defs/SubscriptionTerminationCode"},"message":{"type":"string"},"subscription_id":{"$ref":"#/$defs/SubscriptionId"}},"required":["subscription_id","code","message"],"type":"object"},"event":{"const":"subscription_closed","type":"string"}},"required":["event","data"],"type":"object"}]},"SubscriptionEventPayload":{"oneOf":[{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"worker":{"$ref":"#/$defs/SubscriptionWorker"}},"required":["worker"],"type":"object"},"event":{"const":"worker_upserted","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"runtime_id":{"type":["string","null"]},"worker_id":{"$ref":"#/$defs/SubscriptionWorkerId"}},"required":["worker_id"],"type":"object"},"event":{"const":"worker_removed","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"event":{"$ref":"#/$defs/Event"},"worker_id":{"$ref":"#/$defs/SubscriptionWorkerId"}},"required":["worker_id","event"],"type":"object"},"event":{"const":"worker_protocol","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"workdir":{"$ref":"#/$defs/WorkspaceSubscriptionWorkdir"}},"required":["workdir"],"type":"object"},"event":{"const":"workdir_upserted","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"working_directory_id":{"$ref":"#/$defs/SubscriptionWorkdirId"}},"required":["working_directory_id"],"type":"object"},"event":{"const":"workdir_removed","type":"string"}},"required":["event","data"],"type":"object"}]},"SubscriptionId":{"type":"string"},"SubscriptionRejectionCode":{"enum":["invalid_request","unsupported_protocol_version","unsupported_selector","unauthorized","resource_not_found","capacity_exceeded","internal"],"type":"string"},"SubscriptionRequest":{"oneOf":[{"additionalProperties":false,"properties":{"method":{"const":"subscribe_events","type":"string"},"params":{"additionalProperties":false,"properties":{"request_id":{"$ref":"#/$defs/SubscriptionRequestId"},"selector":{"$ref":"#/$defs/EventSubscriptionSelector"}},"required":["request_id","selector"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"properties":{"method":{"const":"unsubscribe_events","type":"string"},"params":{"additionalProperties":false,"properties":{"request_id":{"$ref":"#/$defs/SubscriptionRequestId"},"subscription_id":{"$ref":"#/$defs/SubscriptionId"}},"required":["request_id","subscription_id"],"type":"object"}},"required":["method","params"],"type":"object"}]},"SubscriptionRequestId":{"type":"string"},"SubscriptionResponse":{"oneOf":[{"additionalProperties":false,"properties":{"payload":{"additionalProperties":false,"properties":{"request_id":{"$ref":"#/$defs/SubscriptionRequestId"},"selector":{"$ref":"#/$defs/EventSubscriptionSelector"},"snapshot":{"$ref":"#/$defs/SubscriptionSnapshot"},"subscription_id":{"$ref":"#/$defs/SubscriptionId"}},"required":["request_id","subscription_id","selector","snapshot"],"type":"object"},"result":{"const":"subscribed","type":"string"}},"required":["result","payload"],"type":"object"},{"additionalProperties":false,"properties":{"payload":{"additionalProperties":false,"properties":{"request_id":{"$ref":"#/$defs/SubscriptionRequestId"},"subscription_id":{"$ref":"#/$defs/SubscriptionId"}},"required":["request_id","subscription_id"],"type":"object"},"result":{"const":"unsubscribed","type":"string"}},"required":["result","payload"],"type":"object"},{"additionalProperties":false,"properties":{"payload":{"additionalProperties":false,"properties":{"code":{"$ref":"#/$defs/SubscriptionRejectionCode"},"message":{"type":"string"},"request_id":{"$ref":"#/$defs/SubscriptionRequestId"},"subscription_id":{"anyOf":[{"$ref":"#/$defs/SubscriptionId"},{"type":"null"}]}},"required":["request_id","code","message"],"type":"object"},"result":{"const":"subscription_rejected","type":"string"}},"required":["result","payload"],"type":"object"}]},"SubscriptionSnapshot":{"oneOf":[{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"workers":{"items":{"$ref":"#/$defs/SubscriptionWorker"},"type":"array"}},"required":["workers"],"type":"object"},"topic":{"const":"workers","type":"string"}},"required":["topic","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"events":{"items":{"$ref":"#/$defs/Event"},"type":"array"},"worker_id":{"$ref":"#/$defs/SubscriptionWorkerId"}},"required":["worker_id","events"],"type":"object"},"topic":{"const":"worker_protocol","type":"string"}},"required":["topic","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"workdirs":{"items":{"$ref":"#/$defs/WorkspaceSubscriptionWorkdir"},"type":"array"}},"required":["workdirs"],"type":"object"},"topic":{"const":"workspace_workdirs","type":"string"}},"required":["topic","data"],"type":"object"}]},"SubscriptionTerminationCode":{"enum":["lagged","resource_gone","unauthorized","server_shutdown"],"type":"string"},"SubscriptionWorkdirId":{"type":"string"},"SubscriptionWorker":{"additionalProperties":false,"properties":{"availability":{"$ref":"#/$defs/SubscriptionWorkerAvailability","default":"observed","description":"Freshness of the Runtime-backed observation carried by this projection.\n`Unavailable` preserves catalog membership without claiming that execution stopped."},"display_name":{"type":["string","null"]},"has_running_internal_workers":{"default":false,"type":"boolean"},"job":{"anyOf":[{"$ref":"#/$defs/SubscriptionWorkerJob"},{"type":"null"}],"description":"Backend-owned Job binding. Runtime producers leave this unset; the\nWorkspace projection enriches both snapshots and live updates."},"profile":{"type":["string","null"]},"resource_key":{"description":"Workspace-scoped canonical resource key. Runtime producers leave this unset;\nWorkspace-facing projections must populate it before publishing the Worker.","type":["string","null"]},"runtime_id":{"description":"Set by the Workspace Server when projecting a Runtime-owned Worker to clients.\nRuntime producers leave this unset because the connection identifies the Runtime.","type":["string","null"]},"state":{"$ref":"#/$defs/SubscriptionWorkerState","description":"Runtime catalog lifecycle compatibility projection; not foreground-state authority."},"workdir_attachments":{"items":{"$ref":"#/$defs/SubscriptionWorkerWorkdirAttachment"},"type":"array"},"worker_id":{"$ref":"#/$defs/SubscriptionWorkerId"},"worker_state":{"anyOf":[{"$ref":"#/$defs/WorkerStateSnapshot"},{"type":"null"}],"description":"Latest foreground state observed from the Worker. This remains\nabsent until an authoritative Worker snapshot/event has been applied."},"workspace_id":{"type":["string","null"]}},"required":["worker_id","state","availability","has_running_internal_workers"],"type":"object"},"SubscriptionWorkerAvailability":{"description":"Whether the Workspace Server has a current Runtime observation for a Worker.\n\n`Unavailable` deliberately does not imply `Stopped`: it means the last observed\nlifecycle state is retained while the Runtime observation source is unavailable.","enum":["observed","unavailable"],"type":"string"},"SubscriptionWorkerId":{"type":"string"},"SubscriptionWorkerIds":{"description":"A bounded, canonical worker-id set. Ordering on the wire and in equality/hash\nsemantics is lexical, so equivalent selectors aggregate to one upstream key.","items":{"$ref":"#/$defs/SubscriptionWorkerId"},"type":"array"},"SubscriptionWorkerJob":{"additionalProperties":false,"properties":{"attempt_id":{"type":"string"},"job_id":{"type":"string"},"purpose":{"type":"string"}},"required":["job_id","attempt_id","purpose"],"type":"object"},"SubscriptionWorkerProtocolMethod":{"additionalProperties":false,"properties":{"method":{"$ref":"#/$defs/Method"},"subscription_id":{"$ref":"#/$defs/SubscriptionId"}},"required":["subscription_id","method"],"type":"object"},"SubscriptionWorkerState":{"enum":["idle","running","paused","stopped"],"type":"string"},"SubscriptionWorkerWorkdirAttachment":{"additionalProperties":false,"properties":{"alias":{"description":"Stable Worker-local routing alias.","type":"string"},"repository_key":{"type":["string","null"]},"working_directory_id":{"$ref":"#/$defs/SubscriptionWorkdirId"}},"required":["alias","working_directory_id"],"type":"object"},"SymlinkPolicy":{"description":"Symbolic-link identity used by one filesystem allow rule.\n\n`Resolved` is the least authority and the default: access is matched\nagainst the provider-resolved target. `Logical` intentionally grants the\npath as presented through the Workdir, even when it aliases another target.","enum":["resolved","logical"],"type":"string"},"ToolResultDisposition":{"enum":["success","error","interrupted","cancelled","outcome_unknown"],"type":"string"},"TurnResult":{"enum":["finished","paused"],"type":"string"},"UploadedFileAvailability":{"description":"Availability recorded for an uploaded client-local file.","enum":["available","unavailable","integrity_failed"],"type":"string"},"UploadedFileRef":{"additionalProperties":false,"description":"Session-owned immutable reference to a client-local uploaded file.\n\nUpload transports return an unbound reference. Worker fills\n`source_entry_id` immediately before the containing user input is committed;\ncommitted Session Log and public snapshot records therefore always retain\nthe durable source-entry identity without storing the file body.","properties":{"artifact_id":{"type":"string"},"availability":{"$ref":"#/$defs/UploadedFileAvailability"},"byte_len":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"created_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"file_name":{"type":"string"},"media_type":{"type":"string"},"sha256":{"type":"string"},"source_entry_id":{"type":["string","null"]}},"required":["artifact_id","file_name","media_type","created_at_ms","availability","byte_len","sha256"],"type":"object"},"WorkerBusyState":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"run","type":"string"},"state":{"$ref":"#/$defs/WorkerRunState"}},"required":["kind","state"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"maintenance","type":"string"},"state":{"$ref":"#/$defs/WorkerMaintenanceState"}},"required":["kind","state"],"type":"object"}]},"WorkerCommandAcknowledgement":{"additionalProperties":false,"properties":{"command":{"$ref":"#/$defs/WorkerCommandKind"},"command_id":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"disposition":{"$ref":"#/$defs/WorkerCommandDisposition"},"state":{"$ref":"#/$defs/WorkerStateSnapshot","description":"The complete authoritative state observed after command admission."}},"required":["command_id","command","disposition","state"],"type":"object"},"WorkerCommandDisposition":{"enum":["accepted","stale_command_id","conflict","invalid_state"],"type":"string"},"WorkerCommandEnvelope":{"additionalProperties":false,"description":"Caller-owned identity for one state-changing Worker command.\n\nA controller accepts command ids in strictly increasing order. Exact retries\nof an accepted id must retain the same command kind.","properties":{"command_id":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["command_id"],"type":"object"},"WorkerCommandKind":{"enum":["resume","cancel","pause","compact","shutdown"],"type":"string"},"WorkerEvent":{"description":"Typed lifecycle events sent from a child Worker to its parent.\n\nDelivered as `Method::WorkerEvent` over the parent's Unix socket. The\nparent Controller always applies variant-specific side effects\n(registry / worker-allocation updates). Agent-visible variants are also\nqueued into the notification buffer; control-plane-only variants are\nnot injected into the parent's LLM context.\n\nTransport is fire-and-forget; receivers must tolerate out-of-order\ndelivery (e.g. `TurnEnded` arriving after `ShutDown` for the same\nchild Worker).","oneOf":[{"additionalProperties":false,"description":"Child finished one turn and is back to IDLE.","properties":{"kind":{"const":"turn_ended","type":"string"},"worker_name":{"type":"string"}},"required":["kind","worker_name"],"type":"object"},{"additionalProperties":false,"description":"Engine execution error occurred inside the child's turn.\n\nLimited to worker runtime failures (provider / tool errors) —\ndoes not include transient method-rejection responses such as\n`AlreadyRunning`.","properties":{"kind":{"const":"errored","type":"string"},"message":{"type":"string"},"worker_name":{"type":"string"}},"required":["kind","worker_name","message"],"type":"object"},{"additionalProperties":false,"description":"Child has stopped (controller loop is exiting).","properties":{"kind":{"const":"shut_down","type":"string"},"worker_name":{"type":"string"}},"required":["kind","worker_name"],"type":"object"},{"additionalProperties":false,"description":"Child SubWorker sub-delegated scope to a grandchild SubWorker via `SubWorkerSpawn`.\n\nControl-plane only: receivers apply registry side effects and\npropagate upward, but do not expose this as an agent notification.\n\nThe parent uses this to add the grandchild to its own\n`spawned_workers.json` so it can manage the grandchild directly\neven if the intermediate child dies. The parent then re-fires\nthis event upward (if it has a parent of its own) to maintain\nthe chain to root.","properties":{"kind":{"const":"scope_sub_delegated","type":"string"},"parent_worker":{"description":"Sub-delegating Worker (= the sender itself).","type":"string"},"scope":{"description":"Scope delegated to the grandchild.","items":{"$ref":"#/$defs/ScopeRule"},"type":"array"},"sub_socket":{"description":"Unix-socket path where the grandchild is reachable.","type":"string"},"sub_worker":{"description":"Name of the grandchild Worker.","type":"string"}},"required":["kind","parent_worker","sub_worker","sub_socket","scope"],"type":"object"}]},"WorkerMaintenanceState":{"enum":["compacting"],"type":"string"},"WorkerRunState":{"enum":["running","pausing","paused","cancelling"],"type":"string"},"WorkerState":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"idle","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"busy","type":"string"},"state":{"$ref":"#/$defs/WorkerBusyState"}},"required":["kind","state"],"type":"object"}]},"WorkerStateSnapshot":{"additionalProperties":false,"properties":{"last_command_id":{"description":"Highest lifecycle command id observed by this controller instance.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"last_finished_submission_request_id":{"description":"Most recent Submit request whose controller Run has returned and whose\npost-Run state transition has been published. This is a teardown fence,\nnot a successful-result claim; cancelled and failed Runs may also finish.","type":["string","null"]},"state":{"$ref":"#/$defs/WorkerState"}},"required":["last_command_id","state"],"type":"object"},"WorkerStatus":{"enum":["idle","running","paused","stopped"],"type":"string"},"WorkspaceSubscriptionWorkdir":{"additionalProperties":false,"description":"Workspace-facing Workdir summary. Backend-generated Repository UUIDs never\nenter this DTO; Workspace Server must resolve the required Repository key.","properties":{"repository_key":{"type":"string"},"state":{"type":"string"},"working_directory_id":{"$ref":"#/$defs/SubscriptionWorkdirId"}},"required":["working_directory_id","repository_key","state"],"type":"object"}},"$schema":"https://json-schema.org/draft/2020-12/schema","description":"Direction is explicit at the outer envelope, while concrete message kinds\nremain typed within each lane.","oneOf":[{"additionalProperties":false,"properties":{"frame":{"const":"request","type":"string"},"message":{"$ref":"#/$defs/SubscriptionRequest"},"protocol_version":{"format":"uint16","maximum":65535.0,"minimum":0.0,"type":"integer"}},"required":["frame","message","protocol_version"],"type":"object"},{"additionalProperties":false,"properties":{"frame":{"const":"response","type":"string"},"message":{"$ref":"#/$defs/SubscriptionResponse"},"protocol_version":{"format":"uint16","maximum":65535.0,"minimum":0.0,"type":"integer"}},"required":["frame","message","protocol_version"],"type":"object"},{"additionalProperties":false,"properties":{"frame":{"const":"event","type":"string"},"message":{"$ref":"#/$defs/SubscriptionEvent"},"protocol_version":{"format":"uint16","maximum":65535.0,"minimum":0.0,"type":"integer"}},"required":["frame","message","protocol_version"],"type":"object"},{"additionalProperties":false,"properties":{"frame":{"const":"worker_protocol","type":"string"},"message":{"$ref":"#/$defs/SubscriptionWorkerProtocolMethod"},"protocol_version":{"format":"uint16","maximum":65535.0,"minimum":0.0,"type":"integer"}},"required":["frame","message","protocol_version"],"type":"object"}],"title":"SubscriptionFrame"}; +const subscriptionFrameSchema: unknown = {"$defs":{"Alert":{"additionalProperties":false,"description":"User-facing alert emitted from the Worker layer.\n\nThis is a separate channel from `tracing` (developer logs): entries\nhere are assembled explicitly by the Worker when a condition should be\nsurfaced to the person driving the client. Keep messages short and\nhuman-readable.","properties":{"level":{"$ref":"#/$defs/AlertLevel"},"message":{"type":"string"},"source":{"$ref":"#/$defs/AlertSource"},"timestamp_ms":{"description":"Milliseconds since the Unix epoch.","format":"int64","maximum":9007199254740991.0,"minimum":-9007199254740991.0,"type":"integer"}},"required":["level","source","message","timestamp_ms"],"type":"object"},"AlertLevel":{"enum":["warn","error"],"type":"string"},"AlertSource":{"enum":["worker","engine","compactor","agents_md"],"type":"string"},"CommandEvent":{"oneOf":[{"additionalProperties":false,"properties":{"command_id":{"type":"string"},"kind":{"const":"started","type":"string"},"observed_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"tool_call_id":{"type":["string","null"]}},"required":["kind","command_id","observed_at_ms"],"type":"object"},{"additionalProperties":false,"properties":{"command_id":{"type":"string"},"content":{"type":"string"},"end_offset":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"kind":{"const":"output","type":"string"},"observed_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"start_offset":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"stream":{"$ref":"#/$defs/CommandStream"}},"required":["kind","command_id","stream","start_offset","end_offset","content","observed_at_ms"],"type":"object"},{"additionalProperties":false,"properties":{"command_id":{"type":"string"},"exit_code":{"format":"int32","maximum":2147483647.0,"minimum":-2147483648.0,"type":["integer","null"]},"kind":{"const":"terminal","type":"string"},"observed_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"status":{"$ref":"#/$defs/CommandStatus"},"stderr_end_offset":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"stdout_end_offset":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","command_id","status","stdout_end_offset","stderr_end_offset","observed_at_ms"],"type":"object"}]},"CommandSnapshot":{"additionalProperties":false,"properties":{"command_id":{"type":"string"},"exit_code":{"format":"int32","maximum":2147483647.0,"minimum":-2147483648.0,"type":["integer","null"]},"last_output_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":["integer","null"]},"observed_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"started_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"status":{"$ref":"#/$defs/CommandStatus"},"stderr":{"$ref":"#/$defs/CommandStreamSlice"},"stdout":{"$ref":"#/$defs/CommandStreamSlice"},"tool_call_id":{"type":["string","null"]}},"required":["command_id","status","started_at_ms","observed_at_ms","stdout","stderr"],"type":"object"},"CommandStatus":{"description":"Live provider-owned command status. These values are operational Console\nstate only and are never appended to Worker history.","enum":["running","completed","failed","timed_out","cancelled"],"type":"string"},"CommandStream":{"enum":["stdout","stderr"],"type":"string"},"CommandStreamSlice":{"additionalProperties":false,"properties":{"content":{"type":"string"},"end_offset":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"start_offset":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"truncated":{"type":"boolean"}},"required":["start_offset","end_offset","content","truncated"],"type":"object"},"CompactionLifecycle":{"additionalProperties":false,"description":"Stable parent-owned lifecycle for one compaction run.","properties":{"compaction_id":{"type":"string"},"ended_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":["integer","null"]},"error":{"type":["string","null"]},"internal_worker":{"anyOf":[{"$ref":"#/$defs/InternalWorkerRef"},{"type":"null"}]},"new_segment_id":{"type":["string","null"]},"revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"schema_version":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"started_at_ms":{"description":"Milliseconds since the Unix epoch.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"state":{"$ref":"#/$defs/CompactionLifecycleState"},"summary":{"type":["string","null"]}},"required":["schema_version","compaction_id","revision","state","started_at_ms"],"type":"object"},"CompactionLifecycleState":{"enum":["running","done","failed","interrupted"],"type":"string"},"CompactionPhase":{"enum":["preparing","summarizing","committing"],"type":"string"},"CompactionTrigger":{"enum":["manual","pre_run","request_threshold"],"type":"string"},"CompletionContext":{"additionalProperties":false,"properties":{"argument":{"type":["string","null"]},"invocation":{"$ref":"#/$defs/FeatureInvocationIdentity"}},"required":["invocation"],"type":"object"},"CompletionEntry":{"additionalProperties":false,"description":"One completion candidate for a prefix query.","properties":{"description":{"type":["string","null"]},"invocation":{"anyOf":[{"$ref":"#/$defs/FeatureInvocationDescriptor"},{"type":"null"}]},"is_dir":{"default":false,"type":"boolean"},"usage":{"type":["string","null"]},"value":{"type":"string"}},"required":["value","is_dir"],"type":"object"},"CompletionKind":{"description":"Kind of completion requested by `Method::ListCompletions`.\n\n`File` is the existing `@` path lane. `Feature` lists installed chat\ninvocations. `FeatureArgument` resolves a declared argument using the\nsource-qualified [`CompletionContext`].","enum":["file","feature","feature_argument"],"type":"string"},"ContextTokenSource":{"description":"Provenance of the current-context token count in a [`Greeting`].","oneOf":[{"const":"measured","description":"Provider-reported input occupancy from the most recent LLM request.","type":"string"},{"const":"estimated","description":"Current history projected from the Worker's token accounting records.","type":"string"}]},"ContextUsage":{"additionalProperties":false,"description":"Current prompt-context occupancy, distinct from cumulative run traffic.","properties":{"source":{"$ref":"#/$defs/ContextTokenSource"},"tokens":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["tokens","source"],"type":"object"},"ErrorCode":{"enum":["already_running","not_running","not_paused","provider_error","tool_error","invalid_request","internal"],"type":"string"},"Event":{"oneOf":[{"additionalProperties":false,"description":"Durable Submit acceptance. A `Queued` receipt follows the durable FIFO\ncheckpoint. A `Started` receipt follows the pending activation checkpoint;\nthe corresponding [`Event::UserMessage`] marks the later history commit.\nRepeating the same request id and exact payload returns the same receipt\nwithout appending or activating twice.","properties":{"data":{"additionalProperties":false,"properties":{"disposition":{"$ref":"#/$defs/SubmissionDisposition"},"submission_id":{"type":"string"},"submission_request_id":{"type":"string"}},"required":["submission_request_id","submission_id","disposition"],"type":"object"},"event":{"const":"submission_accepted","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Correlated rejection before durable acceptance.","properties":{"data":{"additionalProperties":false,"properties":{"message":{"type":"string"},"submission_request_id":{"type":"string"}},"required":["submission_request_id","message"],"type":"object"},"event":{"const":"submission_rejected","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Durable notification acceptance. The request identity is persisted with\nthe notification receipt before this event is emitted. Repeating the\nsame request id and exact payload returns the same acknowledgement\nwithout appending the notification twice.","properties":{"data":{"additionalProperties":false,"properties":{"notification_request_id":{"type":"string"}},"required":["notification_request_id"],"type":"object"},"event":{"const":"notification_accepted","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Correlated notification rejection before durable acceptance.","properties":{"data":{"additionalProperties":false,"properties":{"message":{"type":"string"},"notification_request_id":{"type":"string"}},"required":["notification_request_id","message"],"type":"object"},"event":{"const":"notification_rejected","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Revisioned FIFO replacement following enqueue, activation, cancel, or clear.","properties":{"data":{"additionalProperties":false,"properties":{"pending":{"$ref":"#/$defs/PendingSubmissionsSnapshot"}},"required":["pending"],"type":"object"},"event":{"const":"pending_submissions_changed","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A user input message persisted as `LogEntry::AnnotatedUserInput` and\nactivated for a turn. Broadcast to every subscribed client so TUI / GUI\ninstances show the same user line that reconnect snapshots replay.","properties":{"data":{"additionalProperties":false,"properties":{"entry_id":{"description":"Stable durable history identity. Older mixed-version producers omit it.","type":["string","null"]},"segments":{"items":{"$ref":"#/$defs/Segment"},"type":"array"}},"required":["segments"],"type":"object"},"event":{"const":"user_message","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A canonical assistant/tool entry has committed to durable history.\nStreaming events remain the in-flight authority; this event supplies the\nstable identity used to reconcile that live block with paged history.","properties":{"data":{"additionalProperties":false,"properties":{"entry":{"$ref":"#/$defs/SessionSnapshotEntry"}},"required":["entry"],"type":"object"},"event":{"const":"session_entry_committed","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"One agent-injected system item committed to history.\n\nCarries the JSON form of `session_store::SystemItem`. Covers\n`Method::Notify` echoes, child-Worker lifecycle events from\n`Method::WorkerEvent`, `@` resolution payloads, and any future\nClients dispatch on the `kind` tag for typed rendering instead\nof parsing free-text prefixes like `[Notification] …` or\n`[File: …]`.\n\nOne event per `LogEntry::AnnotatedSystemItem` commit. Disk-side and\nwire-side are 1:1.","properties":{"data":{"additionalProperties":false,"properties":{"entry_id":{"description":"Stable durable history identity. Synthetic compatibility events omit it.","type":["string","null"]},"item":true},"required":["item"],"type":"object"},"event":{"const":"system_item","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A new self-driving cycle has begun (IDLE → active transition).\n\nMarker event for the start of an Invoke range; the range extends\nimplicitly until the next `InvokeStart`. Fires for every accepted\n`Method::Submit` (kind=`UserSend`), `Method::Notify` (kind=`Notify`),\n`Method::WorkerEvent` re-injection (kind=`WorkerEvent`), and any other\nIDLE-breaking trigger. Mid-run interrupts (e.g. hook output,\ntyped system reminder insertion that doesn't break IDLE) do not\nemit `InvokeStart` — they appear as `SystemItem` only.\n\nCarries `kind` only; the payload (user text / notify message /\nworker event body) is delivered separately via the immediately\nfollowing `UserMessage` / `SystemItem` event.","properties":{"data":{"additionalProperties":false,"properties":{"kind":{"$ref":"#/$defs/InvokeKind"}},"required":["kind"],"type":"object"},"event":{"const":"invoke_start","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"One AgentTurn boundary opened. An AgentTurn is a maximal run of\nLLM generation calls whose input messages are identical (i.e.\nretries from network errors / 5xx / stream disconnects collapse\ninto the same AgentTurn). When the input changes (a new tool\nresult lands, a user interrupts, etc.), the next LLM call belongs\nto a new AgentTurn.\n\n`turn` is the AgentTurn index from `Engine::turn_count`.\n\nCurrently retry is not yet implemented (`agen-stream-continuation`)\nso AgentTurn and `LlmCall` fire 1:1, but the contract here is\nthe AgentTurn boundary; consumers that want per-LLM-call signals\nmust subscribe to `LlmCallStart` / `LlmCallEnd` instead.","properties":{"data":{"additionalProperties":false,"properties":{"turn":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["turn"],"type":"object"},"event":{"const":"turn_start","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"AgentTurn closed.","properties":{"data":{"additionalProperties":false,"properties":{"result":{"$ref":"#/$defs/TurnResult"},"turn":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["turn","result"],"type":"object"},"event":{"const":"turn_end","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"One LLM generation call started (1 request → 1 generation, retry\nincluded). Multiple `LlmCall*` pairs may fire inside a single\n`TurnStart` / `TurnEnd` pair when a request is retried; today\nthey fire 1:1 because retry is not implemented.\n\n`llm_call` is the worker-wide running counter from\n`Engine::llm_call_count`.","properties":{"data":{"additionalProperties":false,"properties":{"llm_call":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["llm_call"],"type":"object"},"event":{"const":"llm_call_start","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"LLM generation call ended.","properties":{"data":{"additionalProperties":false,"properties":{"llm_call":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["llm_call"],"type":"object"},"event":{"const":"llm_call_end","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A transport-level LLM request retry has been scheduled.\n\nThis is operational state for clients to render while the worker is\nwaiting in backoff. It is not part of conversation history.","properties":{"data":{"additionalProperties":false,"properties":{"elapsed_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"error":{"type":"string"},"failed_attempt":{"description":"The attempt that just failed. 1 origin.","format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"llm_call":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"max_attempts":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"status":{"format":"uint16","maximum":65535.0,"minimum":0.0,"type":["integer","null"]},"wait_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["llm_call","failed_attempt","max_attempts","wait_ms","elapsed_ms","error"],"type":"object"},"event":{"const":"llm_retry","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Stream generation was interrupted after events had begun and the worker\nis continuing with a follow-up LLM request.","properties":{"data":{"additionalProperties":false,"properties":{"attempt":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"llm_call":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"max_attempts":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"reason":{"type":"string"}},"required":["llm_call","attempt","max_attempts","reason"],"type":"object"},"event":{"const":"llm_continuation","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"text":{"type":"string"}},"required":["text"],"type":"object"},"event":{"const":"text_delta","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"text":{"type":"string"}},"required":["text"],"type":"object"},"event":{"const":"text_done","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A reasoning / thinking block has started.\n\nAlways paired with a `ThinkingDone`. `ThinkingDelta` is optional:\nsome providers (or some configurations) emit thinking metadata\nwithout plaintext, in which case Start → Done arrive with no\ndeltas in between. Multiple thinking blocks per turn are allowed.","properties":{"event":{"const":"thinking_start","type":"string"}},"required":["event"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"text":{"type":"string"}},"required":["text"],"type":"object"},"event":{"const":"thinking_delta","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Thinking block completed. `text` is the full accumulated body\n(empty string when the provider didn't emit plaintext).","properties":{"data":{"additionalProperties":false,"properties":{"text":{"type":"string"}},"required":["text"],"type":"object"},"event":{"const":"thinking_done","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"id":{"type":"string"},"name":{"type":"string"}},"required":["id","name"],"type":"object"},"event":{"const":"tool_call_start","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"id":{"type":"string"},"json":{"type":"string"}},"required":["id","json"],"type":"object"},"event":{"const":"tool_call_args_delta","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"arguments":{"type":"string"},"id":{"type":"string"},"name":{"type":"string"}},"required":["id","name","arguments"],"type":"object"},"event":{"const":"tool_call_done","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"disposition":{"anyOf":[{"$ref":"#/$defs/ToolResultDisposition"},{"type":"null"}]},"id":{"type":"string"},"is_error":{"default":false,"type":"boolean"},"output":{"description":"Full tool output. Absent when the tool chose to return\nsummary-only, or when the result was pruned.","type":["string","null"]},"summary":{"description":"Short human-readable summary. Always present; used by clients\nthat only want a 1-line rendering (e.g. collapsed views).","type":"string"}},"required":["id","summary","is_error"],"type":"object"},"event":{"const":"tool_result","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Token accounting for one LLM request.\n\n`input_tokens` is the prompt prefix occupancy (cache reads /\ncache writes included), as normalised by the worker layer.\n`cache_read_input_tokens` is the cache-hit subset of that\noccupancy; subtracting it yields the \"net upload\" the client\nactually paid full price to send on this request, which is what\nthe TUI status line accumulates per turn.","properties":{"data":{"additionalProperties":false,"properties":{"cache_read_input_tokens":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":["integer","null"]},"input_tokens":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":["integer","null"]},"output_tokens":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":["integer","null"]}},"type":"object"},"event":{"const":"usage","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Authoritative current-context occupancy after a non-request boundary.\n\nUnlike [`Event::Usage`], this is not cumulative request traffic. It lets\nconnected clients converge with reconnect snapshots after run completion,\ncompaction, rewind, and other history rewrites. `None` explicitly means\nthe current value is unavailable and stale occupancy must be cleared.","properties":{"data":{"additionalProperties":false,"properties":{"usage":{"anyOf":[{"$ref":"#/$defs/ContextUsage"},{"type":"null"}]}},"type":"object"},"event":{"const":"context_usage","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"result":{"$ref":"#/$defs/RunResult"}},"required":["result"],"type":"object"},"event":{"const":"run_end","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"code":{"$ref":"#/$defs/ErrorCode"},"message":{"type":"string"}},"required":["code","message"],"type":"object"},"event":{"const":"error","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Sent exactly once at the start of every client connection.\n\n`entries` is the session-log mirror at subscribe time, serialised\nas the JSON form of `session_store::LogEntry`. This is the\nbulk-reconstruction lane: clients walk the entries to seed their\nderived view.\n\n`greeting` and `status` accompany the snapshot so clients render\nworker identity and current controller state without an extra round\ntrip.\n\nLive updates after the snapshot arrive through the streaming\nevents (`TextDelta` / `ToolCall*` / `ToolResult` / etc.) plus\nrole-specific entry events (`SegmentRotated` / `SystemItem`) —\nthere is no generic \"every committed entry\" broadcast.","properties":{"data":{"additionalProperties":false,"properties":{"greeting":{"$ref":"#/$defs/Greeting"},"in_flight":{"$ref":"#/$defs/InFlightSnapshot","description":"Unfinished model output that has already streamed in the current\nrun but is not yet represented by committed snapshot entries."},"internal_workers":{"description":"Parent-owned Internal Worker sessions visible to this client.\nService-private Internal Workers are deliberately excluded.","items":{"$ref":"#/$defs/InternalWorkerSnapshot"},"type":"array"},"session":{"$ref":"#/$defs/SessionSnapshot"},"state":{"$ref":"#/$defs/WorkerStateSnapshot","description":"Full revisioned live execution state. `Stopped` remains Runtime\ncatalog authority and is deliberately not represented here."}},"required":["session","greeting","state"],"type":"object"},"event":{"const":"snapshot","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A live event from a parent-owned Internal Worker. The payload reuses the\nnormal Worker event vocabulary while the wrapper carries stable origin\nidentity and a per-child revision fence.","properties":{"data":{"additionalProperties":false,"properties":{"event":{"$ref":"#/$defs/Event"},"revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"worker":{"$ref":"#/$defs/InternalWorkerRef"}},"required":["worker","revision","event"],"type":"object"},"event":{"const":"internal_worker","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Terminal removal fence for one parent-owned Internal Worker session.\n\nClients discard the matching child and descendants, then ignore later\nnested events for this identity until an authoritative snapshot replaces\nthe projection.","properties":{"data":{"additionalProperties":false,"properties":{"revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"worker":{"$ref":"#/$defs/InternalWorkerRef"}},"required":["worker","revision"],"type":"object"},"event":{"const":"internal_worker_removed","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Server-side segment log rotated to a fresh `SegmentStart`.\n\nFires on compaction and on auto-fork when the store head drifts\nA compaction/fork has replaced the authoritative segment. Clients drop\ntheir derived view and reseed from the canonical committed snapshot.","properties":{"data":{"additionalProperties":false,"properties":{"session":{"$ref":"#/$defs/SessionSnapshot"}},"required":["session"],"type":"object"},"event":{"const":"segment_rotated","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Current Worker controller status. Broadcast on every controller-level\ntransition and included in `History` snapshots for late attach.","properties":{"data":{"additionalProperties":false,"properties":{"snapshot":{"$ref":"#/$defs/WorkerStateSnapshot"}},"required":["snapshot"],"type":"object"},"event":{"const":"worker_state","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"acknowledgement":{"$ref":"#/$defs/WorkerCommandAcknowledgement"}},"required":["acknowledgement"],"type":"object"},"event":{"const":"command_acknowledged","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Bounded, provider-owned command telemetry for the live Console. This is\nintentionally not a history entry and is reconstructed from\n`Snapshot.in_flight.commands` after reconnect.","properties":{"data":{"additionalProperties":false,"properties":{"event":{"$ref":"#/$defs/CommandEvent"}},"required":["event"],"type":"object"},"event":{"const":"command","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Reply to `Method::ListCompletions`. Delivered only to the\nrequesting socket (not broadcast). `entries` is empty when no\ncandidates match or when the requested kind has no resolver\nwired up.","properties":{"data":{"additionalProperties":false,"properties":{"context":{"anyOf":[{"$ref":"#/$defs/CompletionContext"},{"type":"null"}]},"entries":{"items":{"$ref":"#/$defs/CompletionEntry"},"type":"array"},"kind":{"$ref":"#/$defs/CompletionKind"},"prefix":{"default":"","type":"string"},"request_id":{"type":["string","null"]}},"required":["kind","entries","prefix"],"type":"object"},"event":{"const":"completions","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Reply to `Method::ListRewindTargets`. Clients should only open a picker\nin response to their own pending request; the event may be broadcast.","properties":{"data":{"additionalProperties":false,"properties":{"head_entries":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"targets":{"items":{"$ref":"#/$defs/RewindTarget"},"type":"array"}},"required":["head_entries","targets"],"type":"object"},"event":{"const":"rewind_targets","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A rewind has truncated the authoritative session. `session` is the\nretained canonical snapshot clients should use to reseed display state.","properties":{"data":{"additionalProperties":false,"properties":{"input":{"items":{"$ref":"#/$defs/Segment"},"type":"array"},"session":{"$ref":"#/$defs/SessionSnapshot"},"summary":{"$ref":"#/$defs/RewindSummary"}},"required":["session","input","summary"],"type":"object"},"event":{"const":"rewind_applied","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Reply to `Method::ListWorkers`. Payload is a stable JSON value so the Worker\ncrate can evolve discovery fields without introducing a protocol\ndependency on session-store.","properties":{"data":{"additionalProperties":false,"properties":{"workers":true},"required":["workers"],"type":"object"},"event":{"const":"workers_listed","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Reply to `Method::RestoreWorker`.","properties":{"data":{"additionalProperties":false,"properties":{"result":true},"required":["result"],"type":"object"},"event":{"const":"worker_restored","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Reply to `Method::RegisterPeer`.","properties":{"data":{"additionalProperties":false,"properties":{"result":true},"required":["result"],"type":"object"},"event":{"const":"peer_registered","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"$ref":"#/$defs/Alert"},"event":{"const":"alert","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Latest memory extract/consolidation lifecycle event for UI observability.\n\nThis is not part of LLM history or prompt context; clients may display it\nbriefly as operational status.","properties":{"data":{"$ref":"#/$defs/MemoryWorkerEvent"},"event":{"const":"memory_worker","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Runtime-only compaction progress. `None` clears the current status.\nThis never enters Session history and carries no operation or Segment identity.","properties":{"data":{"additionalProperties":false,"properties":{"compaction":{"anyOf":[{"$ref":"#/$defs/InFlightCompaction"},{"type":"null"}]}},"type":"object"},"event":{"const":"compaction_progress","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Legacy compaction lifecycle event retained for wire read compatibility.","properties":{"data":{"additionalProperties":false,"properties":{"lifecycle":{"$ref":"#/$defs/CompactionLifecycle"}},"required":["lifecycle"],"type":"object"},"event":{"const":"compact_start","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Compaction completed and the session was rotated.","properties":{"data":{"additionalProperties":false,"properties":{"lifecycle":{"$ref":"#/$defs/CompactionLifecycle"}},"required":["lifecycle"],"type":"object"},"event":{"const":"compact_done","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Compaction failed or was cancelled. The session is unchanged.","properties":{"data":{"additionalProperties":false,"properties":{"lifecycle":{"$ref":"#/$defs/CompactionLifecycle"}},"required":["lifecycle"],"type":"object"},"event":{"const":"compact_failed","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"event":{"const":"shutdown","type":"string"}},"required":["event"],"type":"object"}]},"EventSubscriptionSelector":{"oneOf":[{"additionalProperties":false,"properties":{"topic":{"const":"runtime_workers","type":"string"}},"required":["topic"],"type":"object"},{"additionalProperties":false,"properties":{"topic":{"const":"worker_lifecycle","type":"string"},"worker_ids":{"$ref":"#/$defs/SubscriptionWorkerIds"}},"required":["topic","worker_ids"],"type":"object"},{"additionalProperties":false,"properties":{"runtime_id":{"type":["string","null"]},"topic":{"const":"worker_protocol","type":"string"},"worker_id":{"$ref":"#/$defs/SubscriptionWorkerId"}},"required":["topic","worker_id"],"type":"object"},{"additionalProperties":false,"description":"Server-derived Workspace projection. Workspace identity comes from the\nauthenticated connection and is deliberately absent from this selector.","properties":{"topic":{"const":"workspace_workers","type":"string"}},"required":["topic"],"type":"object"},{"additionalProperties":false,"description":"Typed extension point for the later Workdir subscription slice.","properties":{"topic":{"const":"workspace_workdirs","type":"string"}},"required":["topic"],"type":"object"}]},"FeatureInvocation":{"additionalProperties":false,"properties":{"arguments":{"items":{"$ref":"#/$defs/InvocationArgumentValue"},"type":"array"},"identity":{"$ref":"#/$defs/FeatureInvocationIdentity"},"invocation_id":{"description":"Stable client-generated id retained across queueing and transport retry.","type":"string"},"name":{"description":"Selected public name, retained for exact draft/history restoration. Host\nresolution is always by `identity`.","type":"string"}},"required":["invocation_id","identity","name","arguments"],"type":"object"},"FeatureInvocationDescriptor":{"additionalProperties":false,"properties":{"aliases":{"default":[],"items":{"type":"string"},"type":"array"},"arguments":{"default":[],"items":{"$ref":"#/$defs/InvocationArgumentDescriptor"},"type":"array"},"client_adapter":{"anyOf":[{"$ref":"#/$defs/InvocationClientAdapter"},{"type":"null"}]},"description":{"type":"string"},"display_name":{"type":"string"},"identity":{"$ref":"#/$defs/FeatureInvocationIdentity"},"name":{"type":"string"},"syntax":{"$ref":"#/$defs/FeatureInvocationSyntax"}},"required":["identity","name","display_name","description","syntax","aliases","arguments"],"type":"object"},"FeatureInvocationIdentity":{"description":"Source-qualified identity of one invocation contribution, for example\n`builtin:attachments/attach`. Display names and aliases are never authority.","type":"string"},"FeatureInvocationSyntax":{"oneOf":[{"const":"parenthesized","description":"`/name(value, key=value)` with JSON-style quoted strings and `)` as the\nexplicit boundary before following natural language.","type":"string"}]},"Greeting":{"additionalProperties":false,"description":"Worker self-description rendered by clients when a session starts empty.\n\nSnapshot metadata contains only resolved public execution information so\nclients do not need their own copy of the manifest.","properties":{"context_tokens":{"default":0,"description":"Legacy numeric mirror retained for mixed-version clients.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"context_usage":{"anyOf":[{"$ref":"#/$defs/ContextUsage"},{"type":"null"}],"description":"Typed current-context value. Older snapshots omit it; clients may fall\nback to `context_tokens` only when `context_window` is known."},"context_window":{"default":0,"description":"Resolved effective model context window in tokens. Legacy producers use\nzero when this is unavailable; clients must treat that as unknown.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"cwd":{"type":"string"},"model":{"type":"string"},"provider":{"type":"string"},"reasoning":{"anyOf":[{"$ref":"#/$defs/ReasoningConfig"},{"type":"null"}],"description":"Effective reasoning control. Absent means unspecified or unavailable."},"scope_summary":{"type":"string"},"tools":{"items":{"type":"string"},"type":"array"},"worker_name":{"type":"string"}},"required":["worker_name","cwd","provider","model","scope_summary","tools","context_tokens","context_window"],"type":"object"},"InFlightBlock":{"oneOf":[{"additionalProperties":false,"properties":{"finished":{"type":"boolean"},"kind":{"const":"text","type":"string"},"text":{"type":"string"}},"required":["kind","text"],"type":"object"},{"additionalProperties":false,"properties":{"finished":{"type":"boolean"},"kind":{"const":"thinking","type":"string"},"text":{"type":"string"}},"required":["kind","text"],"type":"object"},{"additionalProperties":false,"properties":{"args":{"type":"string"},"id":{"type":"string"},"kind":{"const":"tool_call","type":"string"},"name":{"type":"string"},"state":{"$ref":"#/$defs/InFlightToolCallState"}},"required":["kind","id","name","args"],"type":"object"}]},"InFlightCompaction":{"additionalProperties":false,"properties":{"phase":{"$ref":"#/$defs/CompactionPhase"},"started_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"trigger":{"$ref":"#/$defs/CompactionTrigger"}},"required":["phase","started_at_ms","trigger"],"type":"object"},"InFlightSnapshot":{"additionalProperties":false,"description":"Unfinished model output and active command state included in\n`Event::Snapshot` for clients that attach while work is still streaming.\n\nThese blocks are presentation state only: they are reconstructed from the\nactive Worker controller and must not be treated as committed assistant\nhistory. Finalized assistant items continue to come from ordinary snapshot\nentries.","properties":{"blocks":{"items":{"$ref":"#/$defs/InFlightBlock"},"type":"array"},"commands":{"items":{"$ref":"#/$defs/CommandSnapshot"},"type":"array"},"compaction":{"anyOf":[{"$ref":"#/$defs/InFlightCompaction"},{"type":"null"}],"description":"The currently running compaction, if any.\n\nThis is lifecycle progress only. Candidate history and the staged\nSegment remain private until the Segment is activated atomically."}},"type":"object"},"InFlightToolCallState":{"enum":["pending","streaming_args","done"],"type":"string"},"InternalWorkerKind":{"description":"Presentation category for an Internal Worker exposed through its parent's\nprotocol stream. Internal Workers never become independently addressable\nprotocol subjects.","oneOf":[{"enum":["sub_worker"],"type":"string"},{"additionalProperties":false,"properties":{"service":{"additionalProperties":false,"properties":{"kind":{"type":"string"}},"required":["kind"],"type":"object"}},"required":["service"],"type":"object"}]},"InternalWorkerRef":{"additionalProperties":false,"description":"Stable presentation identity for one parent-owned Internal Worker session.\n`name` is display-only; `session_id` is the identity used by clients.","properties":{"kind":{"$ref":"#/$defs/InternalWorkerKind"},"name":{"type":"string"},"parent_session_id":{"type":["string","null"]},"session_id":{"type":"string"}},"required":["session_id","name","kind"],"type":"object"},"InternalWorkerSnapshot":{"additionalProperties":false,"description":"Reconnect state for one visible Internal Worker. The revision fences live\n`Event::InternalWorker` updates that raced with parent snapshot assembly.","properties":{"error":{"type":["string","null"]},"greeting":{"anyOf":[{"$ref":"#/$defs/Greeting"},{"type":"null"}],"description":"Public execution metadata for this child. Older snapshots and service\nworkers may omit it; clients must never substitute the parent's values."},"in_flight":{"$ref":"#/$defs/InFlightSnapshot"},"internal_workers":{"items":{"$ref":"#/$defs/InternalWorkerSnapshot"},"type":"array"},"revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"session":{"$ref":"#/$defs/SessionSnapshot"},"status":{"$ref":"#/$defs/WorkerStatus","default":"idle"},"worker":{"$ref":"#/$defs/InternalWorkerRef"}},"required":["worker","revision","session","status"],"type":"object"},"InvocationArgumentDescriptor":{"additionalProperties":false,"properties":{"completion":{"$ref":"#/$defs/InvocationCompletion","default":{"kind":"none"}},"description":{"type":["string","null"]},"name":{"type":"string"},"position":{"description":"Zero-based positional index. `None` means named-only. A positional\nargument may also be supplied by name; this makes chip editing stable.","format":"uint16","maximum":65535.0,"minimum":0.0,"type":["integer","null"]},"required":{"default":false,"type":"boolean"},"value_type":{"$ref":"#/$defs/InvocationArgumentType"}},"required":["name","value_type","completion","required"],"type":"object"},"InvocationArgumentType":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"string","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"integer","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"boolean","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"enum","type":"string"},"values":{"items":{"type":"string"},"type":"array"}},"required":["kind","values"],"type":"object"},{"additionalProperties":false,"description":"A Worker-readable path. This does not grant filesystem authority.","properties":{"kind":{"const":"worker_file","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"description":"A resource selected and staged by the client. It must never be resolved\nas a path on the Worker host.","properties":{"kind":{"const":"client_file","type":"string"}},"required":["kind"],"type":"object"}]},"InvocationArgumentValue":{"additionalProperties":false,"properties":{"name":{"type":"string"},"value":{"$ref":"#/$defs/InvocationValue"}},"required":["name","value"],"type":"object"},"InvocationClientAdapter":{"oneOf":[{"const":"attachment","description":"The client stages a local file and replaces the draft invocation with an\n`UploadedFile` chip. Local paths are not sent to or persisted by Worker.","type":"string"}]},"InvocationCompletion":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"none","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"static","type":"string"},"values":{"items":{"type":"string"},"type":"array"}},"required":["kind","values"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"worker_file","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"client_file","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"description":"Host-owned provider selected by source-qualified id. Metadata lookup is\nside-effect free; implementations never cross this wire boundary.","properties":{"kind":{"const":"provider","type":"string"},"provider":{"type":"string"}},"required":["kind","provider"],"type":"object"}]},"InvocationValue":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"string","type":"string"},"value":{"type":"string"}},"required":["kind","value"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"integer","type":"string"},"value":{"format":"int32","maximum":2147483647.0,"minimum":-2147483648.0,"type":"integer"}},"required":["kind","value"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"boolean","type":"string"},"value":{"type":"boolean"}},"required":["kind","value"],"type":"object"}]},"InvokeKind":{"description":"Kind of trigger that opened a new Invoke (IDLE → active) range.\n\nOne Invoke groups all entries from this trigger up to the next\n`Invoke` marker. The kind is the only payload — content (user text,\nnotify message, worker event body) is delivered by the immediately\nfollowing Turn entry, not by the marker itself.","oneOf":[{"const":"user_send","description":"`Method::Submit` — a user submission.","type":"string"},{"const":"notify","description":"`Method::Notify` — free-text notification injected into history.","type":"string"},{"const":"worker_event","description":"`Method::WorkerEvent` — typed lifecycle report from a child Worker.","type":"string"},{"const":"system_reminder","description":"A typed system reminder that crosses an IDLE boundary (mid-run\nreminders that don't break IDLE are SystemItem-only and do not\nopen a new Invoke).","type":"string"},{"const":"wakeup","description":"Cron / RemoteTrigger style scheduled wake-up. Reserved; no\nproducer is wired today but the variant is part of the initial\nset so future schedulers don't have to amend the wire enum.","type":"string"}]},"MemoryWorkerEvent":{"additionalProperties":false,"properties":{"message":{"description":"Human-readable compact form for actionbar rendering.","type":"string"},"reason":{"type":"string"},"run_id":{"type":"string"},"status":{"type":"string"},"timestamp_ms":{"description":"Milliseconds since the Unix epoch.","format":"int64","maximum":9007199254740991.0,"minimum":-9007199254740991.0,"type":"integer"},"trigger":{"type":"string"},"worker":{"type":"string"}},"required":["worker","status","run_id","trigger","reason","message","timestamp_ms"],"type":"object"},"Method":{"oneOf":[{"additionalProperties":false,"description":"Durably accept typed input for immediate activation or the session FIFO.\nRunning or Paused Workers queue Submit for a later turn; use Notify for\nadvisory information to incorporate into work already in progress.\n\n`submission_request_id` is generated by the authenticated caller and is\nused only for idempotent retry. Worker allocates the durable\n`submission_id` returned by [`Event::SubmissionAccepted`].","properties":{"method":{"const":"submit","type":"string"},"params":{"additionalProperties":false,"properties":{"input":{"items":{"$ref":"#/$defs/Segment"},"type":"array"},"submission_request_id":{"type":"string"}},"required":["submission_request_id","input"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Human-readable text injected into the target Worker's LLM context as a\nnon-blocking system message. Accepted notifications are delivered at the\nnext between-turn boundary, or start a coalesced notification run while\nthe Worker is idle. No side effects beyond LLM context; use\n`WorkerEvent` for typed lifecycle reports.","properties":{"method":{"const":"notify","type":"string"},"params":{"additionalProperties":false,"properties":{"message":{"type":"string"},"notification_request_id":{"type":"string"}},"required":["notification_request_id","message"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Typed lifecycle report from a child Worker to its direct parent.","properties":{"method":{"const":"worker_event","type":"string"},"params":{"$ref":"#/$defs/WorkerEvent"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Return the authoritative FIFO summary without exposing queued payloads.","properties":{"method":{"const":"list_pending_submissions","type":"string"}},"required":["method"],"type":"object"},{"additionalProperties":false,"description":"Remove one queued submission. Running or already activated submissions\nare immutable and therefore cannot be cancelled here.","properties":{"method":{"const":"cancel_pending_submission","type":"string"},"params":{"additionalProperties":false,"properties":{"expected_revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"submission_id":{"type":"string"}},"required":["submission_id","expected_revision"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Remove every queued submission while preserving the active run.","properties":{"method":{"const":"clear_pending_submissions","type":"string"},"params":{"additionalProperties":false,"properties":{"expected_revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["expected_revision"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Activate the next queued submission while the Worker is idle. This is an\nexplicit recovery operation and never resumes a paused run implicitly.","properties":{"method":{"const":"continue_pending","type":"string"},"params":{"additionalProperties":false,"properties":{"expected_head_id":{"type":"string"},"expected_revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["expected_revision","expected_head_id"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"properties":{"method":{"const":"resume","type":"string"},"params":{"additionalProperties":false,"properties":{"command":{"$ref":"#/$defs/WorkerCommandEnvelope"}},"required":["command"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"properties":{"method":{"const":"cancel","type":"string"},"params":{"additionalProperties":false,"properties":{"command":{"$ref":"#/$defs/WorkerCommandEnvelope"}},"required":["command"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Stop the in-flight turn and transition to `Paused`.\n\nUnlike `Cancel` (which discards and returns to `Idle`), a paused\nWorker can resume the interrupted work via `Resume`, or accept a\nfresh `Submit` (orphan `tool_use` items are closed with a\nsynthetic tool result before the new user message is appended).","properties":{"method":{"const":"pause","type":"string"},"params":{"additionalProperties":false,"properties":{"command":{"$ref":"#/$defs/WorkerCommandEnvelope"}},"required":["command"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Request an explicit compaction while the Worker is otherwise idle.\n\nThis is a typed control method: clients must not send `compact` as a\n`Method::Submit` user message.","properties":{"method":{"const":"compact","type":"string"},"params":{"additionalProperties":false,"properties":{"command":{"$ref":"#/$defs/WorkerCommandEnvelope"}},"required":["command"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Ask the Worker to list valid rewind targets from its authoritative session log.","properties":{"method":{"const":"list_rewind_targets","type":"string"}},"required":["method"],"type":"object"},{"additionalProperties":false,"description":"Truncate the current session back to the selected rewind target and\nreturn the selected user input to the client composer.","properties":{"method":{"const":"rewind_to","type":"string"},"params":{"additionalProperties":false,"properties":{"expected_head_entries":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"target":{"$ref":"#/$defs/RewindTargetId"}},"required":["target","expected_head_entries"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"properties":{"method":{"const":"shutdown","type":"string"},"params":{"additionalProperties":false,"properties":{"command":{"$ref":"#/$defs/WorkerCommandEnvelope"}},"required":["command"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Request a list of completion candidates from the Worker.\n\nReply is sent on the same socket as `Event::Completions` (not\nbroadcast). The IPC server handles this directly and writes\nthe response straight back to the requesting socket.","properties":{"method":{"const":"list_completions","type":"string"},"params":{"additionalProperties":false,"properties":{"context":{"anyOf":[{"$ref":"#/$defs/CompletionContext"},{"type":"null"}]},"kind":{"$ref":"#/$defs/CompletionKind"},"prefix":{"type":"string"},"request_id":{"description":"Client query generation. Echoed verbatim so identical-prefix ABA\nresponses cannot cross edits, target switches, or permission changes.","type":["string","null"]}},"required":["kind","prefix"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"List Workers visible to this Worker from durable Worker state and the spawned-child\nregistry. This is not a host-wide Worker universe query.","properties":{"method":{"const":"list_workers","type":"string"}},"required":["method"],"type":"object"},{"additionalProperties":false,"description":"Restore a visible stopped/restorable Worker, or report that it is already\nlive. Missing state and not-visible state are distinct errors.","properties":{"method":{"const":"restore_worker","type":"string"},"params":{"additionalProperties":false,"properties":{"name":{"type":"string"}},"required":["name"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Register another existing Worker as a reciprocal peer of this Worker.\n\nThis is metadata/control state only: it does not ask the target's live\ncontroller for consent, and it must not grant delegated scope,\nspawned-child ownership, output cursors, or child lifecycle authority.","properties":{"method":{"const":"register_peer","type":"string"},"params":{"additionalProperties":false,"properties":{"name":{"type":"string"}},"required":["name"],"type":"object"}},"required":["method","params"],"type":"object"}]},"PasteArtifactAvailability":{"enum":["available","unavailable","integrity_failed"],"type":"string"},"PasteArtifactMediaType":{"description":"One typed piece of a user submission.\n\n`Method::Submit` and `Event::UserMessage` carry `Vec`. Dumb\nclients (CLI piping, scripts) only need to produce a single\n`Segment::Text`; richer clients (TUI / GUI) construct typed atoms\n(paste chips, file refs) and\nsend them through directly so the Worker side never has to re-parse a\nflattened string.\n\nForward compat: payloads with unknown `kind` deserialize to\n`Segment::Unknown`. Worker treats this the same as known-but-unresolved\nvariants — emits an alert and inserts a `[unknown input segment]`\nplaceholder into the LLM context so neither user nor LLM is blind to\nthe dropped intent.","enum":["text_plain_utf8"],"type":"string"},"PasteArtifactRef":{"additionalProperties":false,"description":"Session-owned reference to a large pasted-input artifact.\n\nThe reference contains only bounded integrity and provenance metadata. The\nartifact body remains in session storage and is available to the model only\nthrough the scoped paste-artifact tools installed by Worker.","properties":{"artifact_id":{"type":"string"},"availability":{"$ref":"#/$defs/PasteArtifactAvailability","description":"Availability observed when this immutable reference was committed.\nReads revalidate storage and integrity rather than trusting this field."},"byte_len":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"char_count":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"created_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"line_count":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"media_type":{"$ref":"#/$defs/PasteArtifactMediaType"},"sha256":{"type":"string"},"source_entry_id":{"type":"string"}},"required":["artifact_id","created_at_ms","media_type","availability","byte_len","char_count","line_count","sha256","source_entry_id"],"type":"object"},"PendingSubmissionSummary":{"additionalProperties":false,"description":"Bounded public projection of one pending submission. Only a short display\npreview is exposed; full payload segments and provenance remain in the log.","properties":{"accepted_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"byte_len":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"preview":{"description":"Single-line display text, at most 240 Unicode scalars plus an ellipsis.\nAbsent on older runtimes. Never used to reconstruct the submitted input.","type":["string","null"]},"segment_count":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"submission_id":{"type":"string"}},"required":["submission_id","accepted_at_ms","segment_count","byte_len"],"type":"object"},"PendingSubmissionsSnapshot":{"additionalProperties":false,"description":"Revisioned session-owned FIFO projection used by snapshots and live events.","properties":{"head_id":{"default":null,"type":["string","null"]},"notification_count":{"default":0,"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"notification_previews":{"description":"Ordered display-only previews of waiting notifications, each bounded like\nPendingSubmissionSummary::preview. Older runtimes omit this field.","items":{"type":"string"},"type":"array"},"revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"submissions":{"default":[],"items":{"$ref":"#/$defs/PendingSubmissionSummary"},"type":"array"}},"required":["revision","head_id","notification_count","submissions"],"type":"object"},"Permission":{"description":"Permission lattice used by [`ScopeRule`].\n\nThe derived `Ord` instance follows declaration order, so\n`Read < Write`. Allow rules grant the stated level (and by extension\neverything below); deny rules cap the effective level **strictly\nbelow** the stated level.","enum":["read","write"],"type":"string"},"ReasoningConfig":{"description":"Public reasoning configuration resolved for the model client.\n\nThis deliberately exposes only the effective provider control, not the\nWorker manifest or provider credentials. The tagged shape prevents token\nbudgets from being presented as effort labels.","oneOf":[{"additionalProperties":false,"properties":{"effort":{"type":"string"},"kind":{"const":"effort","type":"string"}},"required":["kind","effort"],"type":"object"},{"additionalProperties":false,"properties":{"budget_tokens":{"format":"int32","maximum":2147483647.0,"minimum":-2147483648.0,"type":"integer"},"kind":{"const":"budget_tokens","type":"string"}},"required":["kind","budget_tokens"],"type":"object"}]},"RewindSummary":{"additionalProperties":false,"properties":{"discarded_entries":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"tool_side_effect_warning":{"type":"boolean"},"truncated_to_entries":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["truncated_to_entries","discarded_entries","tool_side_effect_warning"],"type":"object"},"RewindTarget":{"additionalProperties":false,"properties":{"disabled_reason":{"type":["string","null"]},"eligible":{"type":"boolean"},"expected_head_entries":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"id":{"$ref":"#/$defs/RewindTargetId"},"preview":{"type":"string"},"timestamp_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":["integer","null"]},"truncate_entries":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"turn_index":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"warning":{"type":["string","null"]}},"required":["id","expected_head_entries","truncate_entries","turn_index","preview","eligible"],"type":"object"},"RewindTargetId":{"additionalProperties":false,"properties":{"segment_id":{"type":"string"},"user_input_entry_index":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["segment_id","user_input_entry_index"],"type":"object"},"RunFailureKind":{"enum":["engine","compaction"],"type":"string"},"RunResult":{"oneOf":[{"enum":["finished","paused","limit_reached"],"type":"string"},{"const":"cancelled","description":"The logical Run ended because the user intentionally cancelled it after\nobservable work. Unlike `RolledBack`, committed history is retained.","type":"string"},{"const":"rolled_back","description":"The accepted Method::Submit produced no assistant/tool output before\nuser interruption, so the Worker rolled the submit-time turn state back\nto its pre-submit snapshot. Clients should treat the Worker as Idle and\nrestore the just-submitted input into the editable composer if desired.","type":"string"}]},"RunResumeSource":{"enum":["compaction","pause"],"type":"string"},"RunYieldReason":{"enum":["compaction"],"type":"string"},"ScopeRule":{"additionalProperties":false,"description":"A single allow or deny rule inside a scope configuration.","properties":{"permission":{"$ref":"#/$defs/Permission","description":"Permission level this rule grants (allow) or caps strictly below\n(deny)."},"recursive":{"default":true,"description":"When `false`, the rule only matches the target itself and its\ndirect children. Defaults to `true`.","type":"boolean"},"symlink_policy":{"$ref":"#/$defs/SymlinkPolicy","default":"resolved","description":"Which path identity an allow rule uses when symbolic links are\nencountered. Deny rules always inspect both identities."},"target":{"description":"Target path. Must be absolute by the time a `Scope` is built from\nthis rule — relative paths are resolved per-layer against the\nmanifest file's directory (cwd for overlay layers) before cascade\nmerge.","type":"string"}},"required":["target","permission","recursive","symlink_policy"],"type":"object"},"Segment":{"oneOf":[{"additionalProperties":false,"description":"Free-form text. The fallback every client can produce.","properties":{"content":{"type":"string"},"kind":{"const":"text","type":"string"}},"required":["kind","content"],"type":"object"},{"additionalProperties":false,"description":"Bracketed-paste capture from a TUI-style client. `id`, `chars`\nand `lines` carry the metadata needed to re-render a\n`[Clipboard #N | X chars, Y lines]` chip in `Event::UserMessage`\nre-broadcast.","properties":{"chars":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"content":{"type":"string"},"id":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"kind":{"const":"paste","type":"string"},"lines":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"}},"required":["kind","id","chars","lines","content"],"type":"object"},{"additionalProperties":false,"description":"Internal reference produced when Worker stores a large `Paste` before\ncommitting input. Clients may receive this in history/event projections;\nthe body is intentionally absent.","properties":{"artifact":{"$ref":"#/$defs/PasteArtifactRef"},"kind":{"const":"paste_artifact","type":"string"}},"required":["kind","artifact"],"type":"object"},{"additionalProperties":false,"description":"Client-local file uploaded into the owning Worker session before submit.\nThe Session Log stores only this immutable reference, never file bytes or\nthe client's local path.","properties":{"file":{"$ref":"#/$defs/UploadedFileRef"},"kind":{"const":"uploaded_file","type":"string"}},"required":["kind","file"],"type":"object"},{"additionalProperties":false,"description":"`@` file-system reference. Worker resolves readable files to\n`[File: ]` attachments and readable normal directories to shallow\n`[Dir: ]` listings; the flattened user text keeps the literal\n`@` placeholder either way.","properties":{"kind":{"const":"file_ref","type":"string"},"path":{"type":"string"}},"required":["kind","path"],"type":"object"},{"additionalProperties":false,"description":"Explicitly selected, source-qualified Feature invocation. The host never\npromotes free text to this variant and validates it against the installed\ndeclaration before execution.","properties":{"invocation":{"$ref":"#/$defs/FeatureInvocation"},"kind":{"const":"feature_invoke","type":"string"}},"required":["kind","invocation"],"type":"object"},{"additionalProperties":false,"description":"Source-qualified request for the host to start a Flow before committing\nthis user input. Runtime resolves it through Workspace authority and\nreplaces it with the entered state's instructions before Worker-side\ninput resolution.","properties":{"kind":{"const":"flow","type":"string"},"selector":{"type":"string"}},"required":["kind","selector"],"type":"object"},{"additionalProperties":false,"description":"Unknown variant from a newer client. Worker treats this as an\nunresolved input — surfaces an alert and inserts a placeholder.\nRound-trip is lossy: re-serializing yields `{\"kind\":\"unknown\"}`.","properties":{"kind":{"const":"unknown","type":"string"}},"required":["kind"],"type":"object"}]},"SessionContentPart":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"text","type":"string"},"text":{"type":"string"}},"required":["kind","text"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"refusal","type":"string"},"refusal":{"type":"string"}},"required":["kind","refusal"],"type":"object"}]},"SessionEntryProvenance":{"enum":["human_input","worker_input","flow_instruction","backend_instruction","model_output","tool_output","derived_summary","legacy_unknown"],"type":"string"},"SessionMessageRole":{"enum":["user","assistant"],"type":"string"},"SessionSnapshot":{"additionalProperties":false,"description":"Canonical, storage-independent projection of committed session history.\n\nWorker protocols expose this DTO instead of append-log records. New\nstorage variants can therefore be added without teaching every client how\nto replay the durable log format.","properties":{"entries":{"items":{"$ref":"#/$defs/SessionSnapshotEntry"},"type":"array"},"pending_submissions":{"$ref":"#/$defs/PendingSubmissionsSnapshot","default":{"head_id":null,"notification_count":0,"revision":0,"submissions":[]}}},"required":["entries","pending_submissions"],"type":"object"},"SessionSnapshotEntry":{"oneOf":[{"additionalProperties":false,"properties":{"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"kind":{"const":"user_input","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"segments":{"items":{"$ref":"#/$defs/Segment"},"type":"array"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","segments","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"properties":{"content":{"items":{"$ref":"#/$defs/SessionContentPart"},"type":"array"},"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"kind":{"const":"message","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"role":{"$ref":"#/$defs/SessionMessageRole"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","role","content","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"properties":{"arguments":{"type":"string"},"call_id":{"type":"string"},"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"kind":{"const":"tool_call","type":"string"},"name":{"type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","call_id","name","arguments","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"properties":{"attachments":{"items":{"$ref":"#/$defs/SessionToolAttachment"},"type":"array"},"call_id":{"type":"string"},"content":{"type":["string","null"]},"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"is_error":{"type":"boolean"},"kind":{"const":"tool_result","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"summary":{"type":"string"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","call_id","summary","is_error","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"properties":{"content":{"type":"string"},"data":true,"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"item_kind":{"type":"string"},"kind":{"const":"system_item","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","item_kind","content","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"description":"Durable, non-terminal control return from one logical Run. This is not a\ncompleted run and must not be used to infer the current Worker state.","properties":{"active_run_turn_count":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"kind":{"const":"run_yielded","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"reason":{"$ref":"#/$defs/RunYieldReason"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","reason","active_run_turn_count","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"description":"Durable boundary immediately before Engine execution resumes. The source\ndistinguishes automatic compaction continuation from an intentional\nuser-pause resume; it does not prove that later model output occurred.","properties":{"active_run_turn_count":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"kind":{"const":"run_resumed","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"source":{"$ref":"#/$defs/RunResumeSource"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","source","active_run_turn_count","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"description":"Durable terminal for an intentional cancellation. Consumers must not\npresent this lifecycle result as an execution failure.","properties":{"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"kind":{"const":"run_cancelled","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"properties":{"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"failure":{"anyOf":[{"$ref":"#/$defs/RunFailureKind"},{"type":"null"}]},"kind":{"const":"run_error","type":"string"},"message":{"type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","message","entry_id","timestamp","provenance"],"type":"object"}]},"SessionToolAttachment":{"additionalProperties":false,"properties":{"attachment_id":{"description":"Stable, bounded identity derived from the owning Session entry, attachment\nposition, and immutable image bytes. It is a locator, not a bearer secret.","type":"string"},"byte_len":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"media_type":{"type":"string"}},"required":["attachment_id","media_type","byte_len"],"type":"object"},"SubmissionDisposition":{"description":"Durable acceptance result for one idempotent Submit request.","oneOf":[{"enum":["started"],"type":"string"},{"const":"queued","description":"Input is durably held in the session FIFO until eligible for activation.","type":"string"}]},"SubscriptionEvent":{"oneOf":[{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"payload":{"$ref":"#/$defs/SubscriptionEventPayload"},"subscription_id":{"$ref":"#/$defs/SubscriptionId"}},"required":["subscription_id","payload"],"type":"object"},"event":{"const":"event","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"code":{"$ref":"#/$defs/SubscriptionTerminationCode"},"message":{"type":"string"},"subscription_id":{"$ref":"#/$defs/SubscriptionId"}},"required":["subscription_id","code","message"],"type":"object"},"event":{"const":"subscription_closed","type":"string"}},"required":["event","data"],"type":"object"}]},"SubscriptionEventPayload":{"oneOf":[{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"worker":{"$ref":"#/$defs/SubscriptionWorker"}},"required":["worker"],"type":"object"},"event":{"const":"worker_upserted","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"runtime_id":{"type":["string","null"]},"worker_id":{"$ref":"#/$defs/SubscriptionWorkerId"}},"required":["worker_id"],"type":"object"},"event":{"const":"worker_removed","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"event":{"$ref":"#/$defs/Event"},"worker_id":{"$ref":"#/$defs/SubscriptionWorkerId"}},"required":["worker_id","event"],"type":"object"},"event":{"const":"worker_protocol","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"workdir":{"$ref":"#/$defs/WorkspaceSubscriptionWorkdir"}},"required":["workdir"],"type":"object"},"event":{"const":"workdir_upserted","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"working_directory_id":{"$ref":"#/$defs/SubscriptionWorkdirId"}},"required":["working_directory_id"],"type":"object"},"event":{"const":"workdir_removed","type":"string"}},"required":["event","data"],"type":"object"}]},"SubscriptionId":{"type":"string"},"SubscriptionRejectionCode":{"enum":["invalid_request","unsupported_protocol_version","unsupported_selector","unauthorized","resource_not_found","capacity_exceeded","internal"],"type":"string"},"SubscriptionRequest":{"oneOf":[{"additionalProperties":false,"properties":{"method":{"const":"subscribe_events","type":"string"},"params":{"additionalProperties":false,"properties":{"request_id":{"$ref":"#/$defs/SubscriptionRequestId"},"selector":{"$ref":"#/$defs/EventSubscriptionSelector"}},"required":["request_id","selector"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"properties":{"method":{"const":"unsubscribe_events","type":"string"},"params":{"additionalProperties":false,"properties":{"request_id":{"$ref":"#/$defs/SubscriptionRequestId"},"subscription_id":{"$ref":"#/$defs/SubscriptionId"}},"required":["request_id","subscription_id"],"type":"object"}},"required":["method","params"],"type":"object"}]},"SubscriptionRequestId":{"type":"string"},"SubscriptionResponse":{"oneOf":[{"additionalProperties":false,"properties":{"payload":{"additionalProperties":false,"properties":{"request_id":{"$ref":"#/$defs/SubscriptionRequestId"},"selector":{"$ref":"#/$defs/EventSubscriptionSelector"},"snapshot":{"$ref":"#/$defs/SubscriptionSnapshot"},"subscription_id":{"$ref":"#/$defs/SubscriptionId"}},"required":["request_id","subscription_id","selector","snapshot"],"type":"object"},"result":{"const":"subscribed","type":"string"}},"required":["result","payload"],"type":"object"},{"additionalProperties":false,"properties":{"payload":{"additionalProperties":false,"properties":{"request_id":{"$ref":"#/$defs/SubscriptionRequestId"},"subscription_id":{"$ref":"#/$defs/SubscriptionId"}},"required":["request_id","subscription_id"],"type":"object"},"result":{"const":"unsubscribed","type":"string"}},"required":["result","payload"],"type":"object"},{"additionalProperties":false,"properties":{"payload":{"additionalProperties":false,"properties":{"code":{"$ref":"#/$defs/SubscriptionRejectionCode"},"message":{"type":"string"},"request_id":{"$ref":"#/$defs/SubscriptionRequestId"},"subscription_id":{"anyOf":[{"$ref":"#/$defs/SubscriptionId"},{"type":"null"}]}},"required":["request_id","code","message"],"type":"object"},"result":{"const":"subscription_rejected","type":"string"}},"required":["result","payload"],"type":"object"}]},"SubscriptionSnapshot":{"oneOf":[{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"workers":{"items":{"$ref":"#/$defs/SubscriptionWorker"},"type":"array"}},"required":["workers"],"type":"object"},"topic":{"const":"workers","type":"string"}},"required":["topic","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"events":{"items":{"$ref":"#/$defs/Event"},"type":"array"},"worker_id":{"$ref":"#/$defs/SubscriptionWorkerId"}},"required":["worker_id","events"],"type":"object"},"topic":{"const":"worker_protocol","type":"string"}},"required":["topic","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"workdirs":{"items":{"$ref":"#/$defs/WorkspaceSubscriptionWorkdir"},"type":"array"}},"required":["workdirs"],"type":"object"},"topic":{"const":"workspace_workdirs","type":"string"}},"required":["topic","data"],"type":"object"}]},"SubscriptionTerminationCode":{"enum":["lagged","resource_gone","unauthorized","server_shutdown"],"type":"string"},"SubscriptionWorkdirId":{"type":"string"},"SubscriptionWorker":{"additionalProperties":false,"properties":{"availability":{"$ref":"#/$defs/SubscriptionWorkerAvailability","default":"observed","description":"Freshness of the Runtime-backed observation carried by this projection.\n`Unavailable` preserves catalog membership without claiming that execution stopped."},"display_name":{"type":["string","null"]},"has_running_internal_workers":{"default":false,"type":"boolean"},"job":{"anyOf":[{"$ref":"#/$defs/SubscriptionWorkerJob"},{"type":"null"}],"description":"Backend-owned Job binding. Runtime producers leave this unset; the\nWorkspace projection enriches both snapshots and live updates."},"profile":{"type":["string","null"]},"resource_key":{"description":"Workspace-scoped canonical resource key. Runtime producers leave this unset;\nWorkspace-facing projections must populate it before publishing the Worker.","type":["string","null"]},"restore_observation_token":{"description":"Opaque Runtime-owned Restore observation identity, never a display-state hash.","type":["string","null"]},"runtime_id":{"description":"Set by the Workspace Server when projecting a Runtime-owned Worker to clients.\nRuntime producers leave this unset because the connection identifies the Runtime.","type":["string","null"]},"state":{"$ref":"#/$defs/SubscriptionWorkerState","description":"Runtime catalog lifecycle compatibility projection; not foreground-state authority."},"workdir_attachments":{"items":{"$ref":"#/$defs/SubscriptionWorkerWorkdirAttachment"},"type":"array"},"worker_id":{"$ref":"#/$defs/SubscriptionWorkerId"},"worker_state":{"anyOf":[{"$ref":"#/$defs/WorkerStateSnapshot"},{"type":"null"}],"description":"Latest foreground state observed from the Worker. This remains\nabsent until an authoritative Worker snapshot/event has been applied."},"workspace_id":{"type":["string","null"]}},"required":["worker_id","state","availability","has_running_internal_workers"],"type":"object"},"SubscriptionWorkerAvailability":{"description":"Whether the Workspace Server has a current Runtime observation for a Worker.\n\n`Unavailable` deliberately does not imply `Stopped`: it means the last observed\nlifecycle state is retained while the Runtime observation source is unavailable.","enum":["observed","unavailable"],"type":"string"},"SubscriptionWorkerId":{"type":"string"},"SubscriptionWorkerIds":{"description":"A bounded, canonical worker-id set. Ordering on the wire and in equality/hash\nsemantics is lexical, so equivalent selectors aggregate to one upstream key.","items":{"$ref":"#/$defs/SubscriptionWorkerId"},"type":"array"},"SubscriptionWorkerJob":{"additionalProperties":false,"properties":{"attempt_id":{"type":"string"},"job_id":{"type":"string"},"purpose":{"type":"string"}},"required":["job_id","attempt_id","purpose"],"type":"object"},"SubscriptionWorkerProtocolMethod":{"additionalProperties":false,"properties":{"method":{"$ref":"#/$defs/Method"},"subscription_id":{"$ref":"#/$defs/SubscriptionId"}},"required":["subscription_id","method"],"type":"object"},"SubscriptionWorkerState":{"enum":["idle","running","paused","stopped"],"type":"string"},"SubscriptionWorkerWorkdirAttachment":{"additionalProperties":false,"properties":{"alias":{"description":"Stable Worker-local routing alias.","type":"string"},"repository_key":{"type":["string","null"]},"working_directory_id":{"$ref":"#/$defs/SubscriptionWorkdirId"}},"required":["alias","working_directory_id"],"type":"object"},"SymlinkPolicy":{"description":"Symbolic-link identity used by one filesystem allow rule.\n\n`Resolved` is the least authority and the default: access is matched\nagainst the provider-resolved target. `Logical` intentionally grants the\npath as presented through the Workdir, even when it aliases another target.","enum":["resolved","logical"],"type":"string"},"ToolResultDisposition":{"enum":["success","error","interrupted","cancelled","outcome_unknown"],"type":"string"},"TurnResult":{"enum":["finished","paused"],"type":"string"},"UploadedFileAvailability":{"description":"Availability recorded for an uploaded client-local file.","enum":["available","unavailable","integrity_failed"],"type":"string"},"UploadedFileRef":{"additionalProperties":false,"description":"Session-owned immutable reference to a client-local uploaded file.\n\nUpload transports return an unbound reference. Worker fills\n`source_entry_id` immediately before the containing user input is committed;\ncommitted Session Log and public snapshot records therefore always retain\nthe durable source-entry identity without storing the file body.","properties":{"artifact_id":{"type":"string"},"availability":{"$ref":"#/$defs/UploadedFileAvailability"},"byte_len":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"created_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"file_name":{"type":"string"},"media_type":{"type":"string"},"sha256":{"type":"string"},"source_entry_id":{"type":["string","null"]}},"required":["artifact_id","file_name","media_type","created_at_ms","availability","byte_len","sha256"],"type":"object"},"WorkerBusyState":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"run","type":"string"},"state":{"$ref":"#/$defs/WorkerRunState"}},"required":["kind","state"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"maintenance","type":"string"},"state":{"$ref":"#/$defs/WorkerMaintenanceState"}},"required":["kind","state"],"type":"object"}]},"WorkerCommandAcknowledgement":{"additionalProperties":false,"properties":{"command":{"$ref":"#/$defs/WorkerCommandKind"},"command_id":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"disposition":{"$ref":"#/$defs/WorkerCommandDisposition"},"state":{"$ref":"#/$defs/WorkerStateSnapshot","description":"The complete authoritative state observed after command admission."}},"required":["command_id","command","disposition","state"],"type":"object"},"WorkerCommandDisposition":{"enum":["accepted","stale_command_id","conflict","invalid_state"],"type":"string"},"WorkerCommandEnvelope":{"additionalProperties":false,"description":"Caller-owned identity for one state-changing Worker command.\n\nA controller accepts command ids in strictly increasing order. Exact retries\nof an accepted id must retain the same command kind.","properties":{"command_id":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["command_id"],"type":"object"},"WorkerCommandKind":{"enum":["resume","cancel","pause","compact","shutdown"],"type":"string"},"WorkerEvent":{"description":"Typed lifecycle events sent from a child Worker to its parent.\n\nDelivered as `Method::WorkerEvent` over the parent's Unix socket. The\nparent Controller always applies variant-specific side effects\n(registry / worker-allocation updates). Agent-visible variants are also\nqueued into the notification buffer; control-plane-only variants are\nnot injected into the parent's LLM context.\n\nTransport is fire-and-forget; receivers must tolerate out-of-order\ndelivery (e.g. `TurnEnded` arriving after `ShutDown` for the same\nchild Worker).","oneOf":[{"additionalProperties":false,"description":"Child finished one turn and is back to IDLE.","properties":{"kind":{"const":"turn_ended","type":"string"},"worker_name":{"type":"string"}},"required":["kind","worker_name"],"type":"object"},{"additionalProperties":false,"description":"Engine execution error occurred inside the child's turn.\n\nLimited to worker runtime failures (provider / tool errors) —\ndoes not include transient method-rejection responses such as\n`AlreadyRunning`.","properties":{"kind":{"const":"errored","type":"string"},"message":{"type":"string"},"worker_name":{"type":"string"}},"required":["kind","worker_name","message"],"type":"object"},{"additionalProperties":false,"description":"Child has stopped (controller loop is exiting).","properties":{"kind":{"const":"shut_down","type":"string"},"worker_name":{"type":"string"}},"required":["kind","worker_name"],"type":"object"},{"additionalProperties":false,"description":"Child SubWorker sub-delegated scope to a grandchild SubWorker via `SubWorkerSpawn`.\n\nControl-plane only: receivers apply registry side effects and\npropagate upward, but do not expose this as an agent notification.\n\nThe parent uses this to add the grandchild to its own\n`spawned_workers.json` so it can manage the grandchild directly\neven if the intermediate child dies. The parent then re-fires\nthis event upward (if it has a parent of its own) to maintain\nthe chain to root.","properties":{"kind":{"const":"scope_sub_delegated","type":"string"},"parent_worker":{"description":"Sub-delegating Worker (= the sender itself).","type":"string"},"scope":{"description":"Scope delegated to the grandchild.","items":{"$ref":"#/$defs/ScopeRule"},"type":"array"},"sub_socket":{"description":"Unix-socket path where the grandchild is reachable.","type":"string"},"sub_worker":{"description":"Name of the grandchild Worker.","type":"string"}},"required":["kind","parent_worker","sub_worker","sub_socket","scope"],"type":"object"}]},"WorkerMaintenanceState":{"enum":["compacting"],"type":"string"},"WorkerRunState":{"enum":["running","pausing","paused","cancelling"],"type":"string"},"WorkerState":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"idle","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"busy","type":"string"},"state":{"$ref":"#/$defs/WorkerBusyState"}},"required":["kind","state"],"type":"object"}]},"WorkerStateSnapshot":{"additionalProperties":false,"properties":{"last_command_id":{"description":"Highest lifecycle command id observed by this controller instance.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"last_finished_submission_request_id":{"description":"Most recent Submit request whose controller Run has returned and whose\npost-Run state transition has been published. This is a teardown fence,\nnot a successful-result claim; cancelled and failed Runs may also finish.","type":["string","null"]},"state":{"$ref":"#/$defs/WorkerState"}},"required":["last_command_id","state"],"type":"object"},"WorkerStatus":{"enum":["idle","running","paused","stopped"],"type":"string"},"WorkspaceSubscriptionWorkdir":{"additionalProperties":false,"description":"Workspace-facing Workdir summary. Backend-generated Repository UUIDs never\nenter this DTO; Workspace Server must resolve the required Repository key.","properties":{"repository_key":{"type":"string"},"state":{"type":"string"},"working_directory_id":{"$ref":"#/$defs/SubscriptionWorkdirId"}},"required":["working_directory_id","repository_key","state"],"type":"object"}},"$schema":"https://json-schema.org/draft/2020-12/schema","description":"Direction is explicit at the outer envelope, while concrete message kinds\nremain typed within each lane.","oneOf":[{"additionalProperties":false,"properties":{"frame":{"const":"request","type":"string"},"message":{"$ref":"#/$defs/SubscriptionRequest"},"protocol_version":{"format":"uint16","maximum":65535.0,"minimum":0.0,"type":"integer"}},"required":["frame","message","protocol_version"],"type":"object"},{"additionalProperties":false,"properties":{"frame":{"const":"response","type":"string"},"message":{"$ref":"#/$defs/SubscriptionResponse"},"protocol_version":{"format":"uint16","maximum":65535.0,"minimum":0.0,"type":"integer"}},"required":["frame","message","protocol_version"],"type":"object"},{"additionalProperties":false,"properties":{"frame":{"const":"event","type":"string"},"message":{"$ref":"#/$defs/SubscriptionEvent"},"protocol_version":{"format":"uint16","maximum":65535.0,"minimum":0.0,"type":"integer"}},"required":["frame","message","protocol_version"],"type":"object"},{"additionalProperties":false,"properties":{"frame":{"const":"worker_protocol","type":"string"},"message":{"$ref":"#/$defs/SubscriptionWorkerProtocolMethod"},"protocol_version":{"format":"uint16","maximum":65535.0,"minimum":0.0,"type":"integer"}},"required":["frame","message","protocol_version"],"type":"object"}],"title":"SubscriptionFrame"}; export const RUST_SERIALIZED_SUBSCRIPTION_FRAME_FIXTURES: readonly unknown[] = - [{"protocol_version":2,"frame":"response","message":{"result":"subscribed","payload":{"request_id":"fixture-request","subscription_id":"fixture-subscription","selector":{"topic":"workspace_workers"},"snapshot":{"topic":"workers","data":{"workers":[{"worker_id":"fixture-worker","runtime_id":"fixture-runtime","resource_key":"fixture-worker-resource","availability":"observed","state":"idle","has_running_internal_workers":false,"workspace_id":"fixture-workspace","workdir_attachments":[{"alias":"external","working_directory_id":"fixture-external-workdir"}]}]}}}}},{"protocol_version":2,"frame":"event","message":{"event":"event","data":{"subscription_id":"fixture-subscription","payload":{"event":"worker_protocol","data":{"worker_id":"fixture-worker","event":{"event":"pending_submissions_changed","data":{"pending":{"revision":0,"notification_count":0,"head_id":null,"submissions":[]}}}}}}}},{"protocol_version":2,"frame":"event","message":{"event":"event","data":{"subscription_id":"fixture-subscription","payload":{"event":"worker_protocol","data":{"worker_id":"fixture-worker","event":{"event":"snapshot","data":{"session":{"pending_submissions":{"revision":0,"notification_count":0,"head_id":null,"submissions":[]},"entries":[]},"greeting":{"worker_name":"fixture-worker","cwd":"/fixture","provider":"openai_responses","model":"gpt-6-astra","reasoning":{"kind":"effort","effort":"high"},"scope_summary":"fixture scope","tools":["Read"],"context_window":272000,"context_tokens":142000,"context_usage":{"tokens":142000,"source":"measured"}},"state":{"last_command_id":0,"state":{"kind":"idle"}},"internal_workers":[{"worker":{"session_id":"fixture-child-session","name":"fixture-child","parent_session_id":"fixture-parent-session","kind":"sub_worker"},"revision":1,"session":{"pending_submissions":{"revision":0,"notification_count":0,"head_id":null,"submissions":[]},"entries":[]},"greeting":{"worker_name":"fixture-worker","cwd":"/fixture","provider":"openai_responses","model":"gpt-6-astra","reasoning":{"kind":"effort","effort":"high"},"scope_summary":"fixture scope","tools":["Read"],"context_window":272000,"context_tokens":142000,"context_usage":{"tokens":142000,"source":"measured"}},"status":"idle"}]}}}}}}}]; + [{"protocol_version":2,"frame":"response","message":{"result":"subscribed","payload":{"request_id":"fixture-request","subscription_id":"fixture-subscription","selector":{"topic":"workspace_workers"},"snapshot":{"topic":"workers","data":{"workers":[{"restore_observation_token":"fixture-restore-observation","worker_id":"fixture-worker","runtime_id":"fixture-runtime","resource_key":"fixture-worker-resource","availability":"observed","state":"idle","has_running_internal_workers":false,"workspace_id":"fixture-workspace","workdir_attachments":[{"alias":"external","working_directory_id":"fixture-external-workdir"}]}]}}}}},{"protocol_version":2,"frame":"event","message":{"event":"event","data":{"subscription_id":"fixture-subscription","payload":{"event":"worker_protocol","data":{"worker_id":"fixture-worker","event":{"event":"pending_submissions_changed","data":{"pending":{"revision":0,"notification_count":0,"head_id":null,"submissions":[]}}}}}}}},{"protocol_version":2,"frame":"event","message":{"event":"event","data":{"subscription_id":"fixture-subscription","payload":{"event":"worker_protocol","data":{"worker_id":"fixture-worker","event":{"event":"snapshot","data":{"session":{"pending_submissions":{"revision":0,"notification_count":0,"head_id":null,"submissions":[]},"entries":[]},"greeting":{"worker_name":"fixture-worker","cwd":"/fixture","provider":"openai_responses","model":"gpt-6-astra","reasoning":{"kind":"effort","effort":"high"},"scope_summary":"fixture scope","tools":["Read"],"context_window":272000,"context_tokens":142000,"context_usage":{"tokens":142000,"source":"measured"}},"state":{"last_command_id":0,"state":{"kind":"idle"}},"internal_workers":[{"worker":{"session_id":"fixture-child-session","name":"fixture-child","parent_session_id":"fixture-parent-session","kind":"sub_worker"},"revision":1,"session":{"pending_submissions":{"revision":0,"notification_count":0,"head_id":null,"submissions":[]},"entries":[]},"greeting":{"worker_name":"fixture-worker","cwd":"/fixture","provider":"openai_responses","model":"gpt-6-astra","reasoning":{"kind":"effort","effort":"high"},"scope_summary":"fixture scope","tools":["Read"],"context_window":272000,"context_tokens":142000,"context_usage":{"tokens":142000,"source":"measured"}},"status":"idle"}]}}}}}}}]; const utf8 = new TextEncoder(); type JsonObject = Record; @@ -379,6 +379,7 @@ function validateWorkers(workers: SubscriptionWorker[]): boolean { } function validateWorker(worker: SubscriptionWorker): boolean { + if (worker.restore_observation_token != null && !validateIdentifier(worker.restore_observation_token, MAX_RESOURCE_ID_BYTES)) return false; if (!validateIdentifier(worker.worker_id, MAX_RESOURCE_ID_BYTES)) return false; if (worker.runtime_id != null && !validateIdentifier(worker.runtime_id, MAX_RESOURCE_ID_BYTES)) { return false; diff --git a/web/workspace/src/lib/generated/protocol.ts b/web/workspace/src/lib/generated/protocol.ts index 94e91de69..637f7c16b 100644 --- a/web/workspace/src/lib/generated/protocol.ts +++ b/web/workspace/src/lib/generated/protocol.ts @@ -6,7 +6,52 @@ export type AlertLevel = "warn" | "error"; export type AlertSource = "worker" | "engine" | "compactor" | "agents_md"; -export type CompletionKind = "file"; +export type CompletionKind = "file" | "feature" | "feature_argument"; + +export type CompletionContext = { invocation: FeatureInvocationIdentity, argument?: string | null, }; + +export type FeatureInvocationIdentity = string; + +export type FeatureInvocationSyntax = "parenthesized"; + +export type InvocationArgumentType = { "kind": "string" } | { "kind": "integer" } | { "kind": "boolean" } | { "kind": "enum", values: Array, } | { "kind": "worker_file" } | { "kind": "client_file" }; + +export type InvocationCompletion = { "kind": "none" } | { "kind": "static", values: Array, } | { "kind": "worker_file" } | { "kind": "client_file" } | { "kind": "provider", provider: string, }; + +export type InvocationArgumentDescriptor = { name: string, +/** + * Zero-based positional index. `None` means named-only. A positional + * argument may also be supplied by name; this makes chip editing stable. + */ +position?: number | null, required: boolean, value_type: InvocationArgumentType, completion: InvocationCompletion, description?: string | null, }; + +export type InvocationClientAdapter = "attachment"; + +export type FeatureInvocationDescriptor = { identity: FeatureInvocationIdentity, name: string, aliases: Array, display_name: string, description: string, syntax: FeatureInvocationSyntax, arguments: Array, client_adapter?: InvocationClientAdapter | null, }; + +export type InvocationValue = { "kind": "string", "value": string } | { "kind": "integer", "value": number } | { "kind": "boolean", "value": boolean }; + +export type InvocationArgumentValue = { name: string, value: InvocationValue, }; + +export type FeatureInvocation = { +/** + * Stable client-generated id retained across queueing and transport retry. + */ +invocation_id: string, identity: FeatureInvocationIdentity, +/** + * Selected public name, retained for exact draft/history restoration. Host + * resolution is always by `identity`. + */ +name: string, arguments: Array, }; + +export type FeatureInvocationStatus = "succeeded" | "failed" | "outcome_unknown"; + +export type FeatureInvocationResult = { invocation_id: string, identity: FeatureInvocationIdentity, status: FeatureInvocationStatus, message: string, +/** + * Bounded model-visible context produced by the handler. Credentials and + * executable implementation details must not be included. + */ +context?: string | null, }; export type WorkerStatus = "idle" | "running" | "paused" | "stopped"; @@ -108,7 +153,7 @@ recursive: boolean, */ symlink_policy: SymlinkPolicy, }; -export type CompletionEntry = { value: string, is_dir: boolean, }; +export type CompletionEntry = { value: string, is_dir: boolean, description?: string | null, usage?: string | null, invocation?: FeatureInvocationDescriptor | null, }; export type RewindTargetId = { segment_id: string, user_input_entry_index: number, }; @@ -270,7 +315,7 @@ export type PasteArtifactRef = { artifact_id: string, created_at_ms: number, med */ availability: PasteArtifactAvailability, byte_len: number, char_count: number, line_count: number, sha256: string, source_entry_id: string, }; -export type Segment = { "kind": "text", content: string, } | { "kind": "paste", id: number, chars: number, lines: number, content: string, } | { "kind": "paste_artifact", artifact: PasteArtifactRef, } | { "kind": "uploaded_file", file: UploadedFileRef, } | { "kind": "file_ref", path: string, } | { "kind": "flow", selector: string, } | { "kind": "unknown" }; +export type Segment = { "kind": "text", content: string, } | { "kind": "paste", id: number, chars: number, lines: number, content: string, } | { "kind": "paste_artifact", artifact: PasteArtifactRef, } | { "kind": "uploaded_file", file: UploadedFileRef, } | { "kind": "file_ref", path: string, } | { "kind": "feature_invoke", invocation: FeatureInvocation, } | { "kind": "flow", selector: string, } | { "kind": "unknown" }; export type WorkerEvent = { "kind": "turn_ended", worker_name: string, } | { "kind": "errored", worker_name: string, message: string, } | { "kind": "shut_down", worker_name: string, } | { "kind": "scope_sub_delegated", /** @@ -314,7 +359,11 @@ alias: string, repository_key?: string | null, working_directory_id: Subscriptio export type SubscriptionWorkerJob = { job_id: string, attempt_id: string, purpose: string, }; -export type SubscriptionWorker = { worker_id: SubscriptionWorkerId, +export type SubscriptionWorker = { +/** + * Opaque Runtime-owned Restore observation identity, never a display-state hash. + */ +restore_observation_token?: string | null, worker_id: SubscriptionWorkerId, /** * Set by the Workspace Server when projecting a Runtime-owned Worker to clients. * Runtime producers leave this unset because the connection identifies the Runtime. @@ -367,7 +416,12 @@ export type SubscriptionFramePayload = { "frame": "request", "message": Subscrip export type SubscriptionFrame = { protocol_version: number, } & ({ "frame": "request", "message": SubscriptionRequest } | { "frame": "response", "message": SubscriptionResponse } | { "frame": "event", "message": SubscriptionEvent } | { "frame": "worker_protocol", "message": SubscriptionWorkerProtocolMethod }); -export type Method = { "method": "submit", "params": { submission_request_id: string, input: Array, } } | { "method": "notify", "params": { notification_request_id: string, message: string, } } | { "method": "worker_event", "params": WorkerEvent } | { "method": "list_pending_submissions" } | { "method": "cancel_pending_submission", "params": { submission_id: string, expected_revision: number, } } | { "method": "clear_pending_submissions", "params": { expected_revision: number, } } | { "method": "continue_pending", "params": { expected_revision: number, expected_head_id: string, } } | { "method": "resume", "params": { command: WorkerCommandEnvelope, } } | { "method": "cancel", "params": { command: WorkerCommandEnvelope, } } | { "method": "pause", "params": { command: WorkerCommandEnvelope, } } | { "method": "compact", "params": { command: WorkerCommandEnvelope, } } | { "method": "list_rewind_targets" } | { "method": "rewind_to", "params": { target: RewindTargetId, expected_head_entries: number, } } | { "method": "shutdown", "params": { command: WorkerCommandEnvelope, } } | { "method": "list_completions", "params": { kind: CompletionKind, prefix: string, } } | { "method": "list_workers" } | { "method": "restore_worker", "params": { name: string, } } | { "method": "register_peer", "params": { name: string, } }; +export type Method = { "method": "submit", "params": { submission_request_id: string, input: Array, } } | { "method": "notify", "params": { notification_request_id: string, message: string, } } | { "method": "worker_event", "params": WorkerEvent } | { "method": "list_pending_submissions" } | { "method": "cancel_pending_submission", "params": { submission_id: string, expected_revision: number, } } | { "method": "clear_pending_submissions", "params": { expected_revision: number, } } | { "method": "continue_pending", "params": { expected_revision: number, expected_head_id: string, } } | { "method": "resume", "params": { command: WorkerCommandEnvelope, } } | { "method": "cancel", "params": { command: WorkerCommandEnvelope, } } | { "method": "pause", "params": { command: WorkerCommandEnvelope, } } | { "method": "compact", "params": { command: WorkerCommandEnvelope, } } | { "method": "list_rewind_targets" } | { "method": "rewind_to", "params": { target: RewindTargetId, expected_head_entries: number, } } | { "method": "shutdown", "params": { command: WorkerCommandEnvelope, } } | { "method": "list_completions", "params": { kind: CompletionKind, prefix: string, +/** + * Client query generation. Echoed verbatim so identical-prefix ABA + * responses cannot cross edits, target switches, or permission changes. + */ +request_id?: string | null, context?: CompletionContext | null, } } | { "method": "list_workers" } | { "method": "restore_worker", "params": { name: string, } } | { "method": "register_peer", "params": { name: string, } }; export type Event = { "event": "submission_accepted", "data": { submission_request_id: string, submission_id: string, disposition: SubmissionDisposition, } } | { "event": "submission_rejected", "data": { submission_request_id: string, message: string, } } | { "event": "notification_accepted", "data": { notification_request_id: string, } } | { "event": "notification_rejected", "data": { notification_request_id: string, message: string, } } | { "event": "pending_submissions_changed", "data": { pending: PendingSubmissionsSnapshot, } } | { "event": "user_message", "data": { /** @@ -406,4 +460,4 @@ in_flight?: InFlightSnapshot, * Parent-owned Internal Worker sessions visible to this client. * Service-private Internal Workers are deliberately excluded. */ -internal_workers?: Array, } } | { "event": "internal_worker", "data": { worker: InternalWorkerRef, revision: number, event: Event, } } | { "event": "internal_worker_removed", "data": { worker: InternalWorkerRef, revision: number, } } | { "event": "segment_rotated", "data": { session: SessionSnapshot, } } | { "event": "worker_state", "data": { snapshot: WorkerStateSnapshot, } } | { "event": "command_acknowledged", "data": { acknowledgement: WorkerCommandAcknowledgement, } } | { "event": "command", "data": { event: CommandEvent, } } | { "event": "completions", "data": { kind: CompletionKind, entries: Array, } } | { "event": "rewind_targets", "data": { head_entries: number, targets: Array, } } | { "event": "rewind_applied", "data": { session: SessionSnapshot, input: Array, summary: RewindSummary, } } | { "event": "workers_listed", "data": { workers: unknown, } } | { "event": "worker_restored", "data": { result: unknown, } } | { "event": "peer_registered", "data": { result: unknown, } } | { "event": "alert", "data": Alert } | { "event": "memory_worker", "data": MemoryWorkerEvent } | { "event": "compaction_progress", "data": { compaction: InFlightCompaction | null, } } | { "event": "compact_start", "data": { lifecycle: CompactionLifecycle, } } | { "event": "compact_done", "data": { lifecycle: CompactionLifecycle, } } | { "event": "compact_failed", "data": { lifecycle: CompactionLifecycle, } } | { "event": "shutdown" }; +internal_workers?: Array, } } | { "event": "internal_worker", "data": { worker: InternalWorkerRef, revision: number, event: Event, } } | { "event": "internal_worker_removed", "data": { worker: InternalWorkerRef, revision: number, } } | { "event": "segment_rotated", "data": { session: SessionSnapshot, } } | { "event": "worker_state", "data": { snapshot: WorkerStateSnapshot, } } | { "event": "command_acknowledged", "data": { acknowledgement: WorkerCommandAcknowledgement, } } | { "event": "command", "data": { event: CommandEvent, } } | { "event": "completions", "data": { kind: CompletionKind, prefix: string, request_id?: string | null, context?: CompletionContext | null, entries: Array, } } | { "event": "rewind_targets", "data": { head_entries: number, targets: Array, } } | { "event": "rewind_applied", "data": { session: SessionSnapshot, input: Array, summary: RewindSummary, } } | { "event": "workers_listed", "data": { workers: unknown, } } | { "event": "worker_restored", "data": { result: unknown, } } | { "event": "peer_registered", "data": { result: unknown, } } | { "event": "alert", "data": Alert } | { "event": "memory_worker", "data": MemoryWorkerEvent } | { "event": "compaction_progress", "data": { compaction: InFlightCompaction | null, } } | { "event": "compact_start", "data": { lifecycle: CompactionLifecycle, } } | { "event": "compact_done", "data": { lifecycle: CompactionLifecycle, } } | { "event": "compact_failed", "data": { lifecycle: CompactionLifecycle, } } | { "event": "shutdown" }; diff --git a/web/workspace/src/lib/generated/repository-api.ts b/web/workspace/src/lib/generated/repository-api.ts index 052d8f330..5b72d2098 100644 --- a/web/workspace/src/lib/generated/repository-api.ts +++ b/web/workspace/src/lib/generated/repository-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-repository-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:f9f0f8918dbcce386e94d5368fa938f02565ca98f9f67f07537bd88ed8c49870) +// Input contract: 0.1.0 (sha256:fd06f382c6d33a5c3cc98ef9e264545923c53fa95a9be555a85680f98c6065da) // Options: repository-operation-schema-closure,openapi-3.1,format:uint64=safe-number,strict-safe-integers,sorted-output // Output: web/workspace/src/lib/generated/repository-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_repository_openapi_types -- --check @@ -71,7 +71,7 @@ export type WorkerState = { kind: "idle"; } | { kind: "busy"; state: WorkerBusyS export type WorkerStateSnapshot = { last_command_id: number; last_finished_submission_request_id?: string | null; state: WorkerState; }; -export type WorkerSummary = { availability: SubscriptionWorkerAvailability; diagnostics?: Array; display_name?: string; host_id: string; implementation: WorkerImplementationSummary; label: string; last_seen_at?: string | null; pinned?: boolean; profile?: string | null; resource_key: string; retention_state?: string; runtime_id: string; singleton_key?: string | null; state: string; tags?: Array; workdir_attachments?: Array; worker_id: string; worker_state?: WorkerStateSnapshot | null; workspace: WorkerWorkspaceSummary; }; +export type WorkerSummary = { availability: SubscriptionWorkerAvailability; diagnostics?: Array; display_name?: string; host_id: string; implementation: WorkerImplementationSummary; label: string; last_seen_at?: string | null; pinned?: boolean; profile?: string | null; resource_key: string; restore_observation_token?: string | null; retention_state?: string; runtime_id: string; singleton_key?: string | null; state: string; tags?: Array; workdir_attachments?: Array; worker_id: string; worker_state?: WorkerStateSnapshot | null; workspace: WorkerWorkspaceSummary; }; export type WorkerWorkdirAttachmentSummary = { alias: string; effective_permissions: ExternalWorkdirPermissions; working_directory: WorkingDirectorySummary; }; @@ -79,12 +79,14 @@ export type WorkerWorkspaceSummary = { identity: string; visibility: string; wor export type WorkingDirectoryCleanupTarget = { kind: string; repository_key: string; working_directory_id: string; }; -export type WorkingDirectoryMaterializerKind = "runtime_git_clone" | "client_hosted_external"; +export type WorkingDirectoryMaterializerKind = "logical_workspace_config" | "runtime_git_clone" | "client_hosted_external"; export type WorkingDirectoryOccupancy = { display_name: string; linked_at: string; runtime_id: string; worker_id: string; }; -export type WorkingDirectorySource = { kind: "repository"; repository_key: string; } | { grant_id: string; grant_permissions: ExternalWorkdirPermissions; kind: "external_grant"; }; +export type WorkingDirectorySource = { access: WorkspaceConfigAccess; content_path: string; kind: "workspace_config"; purpose: string; } | { kind: "repository"; repository_key: string; } | { grant_id: string; grant_permissions: ExternalWorkdirPermissions; kind: "external_grant"; }; export type WorkingDirectoryStatusKind = "active" | "cleanup_pending" | "corrupted" | "not_found" | "unknown"; export type WorkingDirectorySummary = { cleanliness?: string | null; cleanup_target?: WorkingDirectoryCleanupTarget | null; creation_ref?: string | null; creation_selector?: string | null; creation_tree?: string | null; current_ref?: string | null; current_selector?: string | null; current_tree?: string | null; display_name?: string | null; materializer_kind: WorkingDirectoryMaterializerKind; observed_at_epoch_seconds?: number | null; occupied_by?: WorkingDirectoryOccupancy | null; source: WorkingDirectorySource; status: WorkingDirectoryStatusKind; working_directory_id: string; }; + +export type WorkspaceConfigAccess = "read_only" | "read_write"; diff --git a/web/workspace/src/lib/generated/runtime-api.ts b/web/workspace/src/lib/generated/runtime-api.ts index 2a3b02082..53eeae412 100644 --- a/web/workspace/src/lib/generated/runtime-api.ts +++ b/web/workspace/src/lib/generated/runtime-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-runtime-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:f9f0f8918dbcce386e94d5368fa938f02565ca98f9f67f07537bd88ed8c49870) +// Input contract: 0.1.0 (sha256:fd06f382c6d33a5c3cc98ef9e264545923c53fa95a9be555a85680f98c6065da) // Options: runtime-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Output: web/workspace/src/lib/generated/runtime-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_runtime_api_types -- --check @@ -19,6 +19,12 @@ export type CleanupWorkdirFileStatus = "pending" | "present" | "active" | "clean export type CleanupWorkerCandidate = { action: CleanupTargetKind; blocking_reason?: string | null; estimated_reclaim_bytes?: number | null; linked_workdir_ids: Array; pinned: boolean; reason: string; retention_state: string; running_linked: boolean; runtime_id: string; runtime_worker_id: string; target_id: string; worker_id: string; }; +export type CompletionContext = { argument?: string | null; invocation: FeatureInvocationIdentity; }; + +export type CompletionEntry = { description?: string | null; invocation?: FeatureInvocationDescriptor | null; is_dir?: boolean; usage?: string | null; value: string; }; + +export type CompletionKind = "file" | "feature" | "feature_argument"; + export type CreateRemoteRuntimeRequest = { display_name?: string | null; endpoint: string; expected_revision?: number | null; public_bundle: RuntimePublicIdentityBundle; }; export type Diagnostic = { code: string; message: string; severity: DiagnosticSeverity; }; @@ -29,6 +35,32 @@ export type ExecuteRuntimeCleanupRequest = { confirm_dirty_discard_target_ids?: export type ExternalWorkdirPermissions = { command: boolean; read: boolean; write: boolean; }; +export type FeatureInvocation = { arguments: Array; identity: FeatureInvocationIdentity; invocation_id: string; name: string; }; + +export type FeatureInvocationDescriptor = { aliases?: Array; arguments?: Array; client_adapter?: InvocationClientAdapter | null; description: string; display_name: string; identity: FeatureInvocationIdentity; name: string; syntax: FeatureInvocationSyntax; }; + +export type FeatureInvocationIdentity = string; + +export type FeatureInvocationSyntax = "parenthesized"; + +export type InvocationArgumentDescriptor = { completion?: InvocationCompletion; description?: string | null; name: string; position?: number | null; required?: boolean; value_type: InvocationArgumentType; }; + +export type InvocationArgumentType = { kind: "string"; } | { kind: "integer"; } | { kind: "boolean"; } | { kind: "enum"; values: Array; } | { kind: "worker_file"; } | { kind: "client_file"; }; + +export type InvocationArgumentValue = { name: string; value: InvocationValue; }; + +export type InvocationClientAdapter = "attachment"; + +export type InvocationCompletion = { kind: "none"; } | { kind: "static"; values: Array; } | { kind: "worker_file"; } | { kind: "client_file"; } | { kind: "provider"; provider: string; }; + +export type InvocationValue = { kind: "string"; value: string; } | { kind: "integer"; value: number; } | { kind: "boolean"; value: boolean; }; + +export type PasteArtifactAvailability = "available" | "unavailable" | "integrity_failed"; + +export type PasteArtifactMediaType = "text_plain_utf8"; + +export type PasteArtifactRef = { artifact_id: string; availability: PasteArtifactAvailability; byte_len: number; char_count: number; created_at_ms: number; line_count: number; media_type: PasteArtifactMediaType; sha256: string; source_entry_id: string; }; + export type RemoveRuntimeRequest = { expected_binding_revision: number; operation_id: string; }; export type RepositoryApiError = { diagnostics?: Array; error: string; message: string; }; @@ -93,11 +125,11 @@ export type RuntimeVerificationEvidenceSummary = { binding_revision: number; las export type RuntimeVerificationOutcome = "verified" | "challenge_issued" | "verification_failed" | "connectivity_failed"; -export type RuntimeWorkerCompletionsRequest = { kind: unknown; prefix?: string; }; +export type RuntimeWorkerCompletionsRequest = { context?: CompletionContext | null; kind: CompletionKind; prefix?: string; request_id?: string | null; }; -export type RuntimeWorkerCompletionsResult = { diagnostics?: Array; entries: Array; kind: unknown; prefix: string; runtime_id: string; worker_id: string; }; +export type RuntimeWorkerCompletionsResult = { context?: CompletionContext | null; diagnostics?: Array; entries: Array; kind: CompletionKind; prefix: string; request_id?: string | null; runtime_id: string; worker_id: string; }; -export type RuntimeWorkerInputRequest = { content: string; kind?: string | null; segments?: Array | null; }; +export type RuntimeWorkerInputRequest = { content: string; kind?: string | null; segments?: Array | null; }; export type RuntimeWorkerInputResult = { diagnostics?: Array; runtime_id: string; state: WorkerOperationState; worker_id: string; }; @@ -107,7 +139,7 @@ export type RuntimeWorkerLifecycleResult = { diagnostics?: Array; ru export type RuntimeWorkerListResponse = { diagnostics?: Array; items: Array; limit: number; source: string; workspace_id: string; }; -export type RuntimeWorkerShowResponse = { availability: SubscriptionWorkerAvailability; diagnostics?: Array; display_name?: string; host_id: string; implementation: WorkerImplementationSummary; label: string; last_seen_at?: string | null; pinned?: boolean; profile?: string | null; resource_key: string; retention_state?: string; runtime_id: string; singleton_key?: string | null; state: string; tags?: Array; updated_at: string; workdir_attachments?: Array; worker_id: string; worker_state?: WorkerStateSnapshot | null; workspace: WorkerWorkspaceSummary; }; +export type RuntimeWorkerShowResponse = { availability: SubscriptionWorkerAvailability; diagnostics?: Array; display_name?: string; host_id: string; implementation: WorkerImplementationSummary; label: string; last_seen_at?: string | null; pinned?: boolean; profile?: string | null; resource_key: string; restore_observation_token?: string | null; retention_state?: string; runtime_id: string; singleton_key?: string | null; state: string; tags?: Array; updated_at: string; workdir_attachments?: Array; worker_id: string; worker_state?: WorkerStateSnapshot | null; workspace: WorkerWorkspaceSummary; }; export type RuntimeWorkerSpawnAcceptanceEvidence = { detail: string; kind: string; }; @@ -127,6 +159,8 @@ export type RuntimeWorkerSummary = { diagnostics?: Array; display_na export type RuntimeWorkerTicketAssignmentRequest = { operation_id: string; ticket_id: string; }; +export type Segment = { content: string; kind: "text"; } | { chars: number; content: string; id: number; kind: "paste"; lines: number; } | { artifact: PasteArtifactRef; kind: "paste_artifact"; } | { file: UploadedFileRef; kind: "uploaded_file"; } | { kind: "file_ref"; path: string; } | { invocation: FeatureInvocation; kind: "feature_invoke"; } | { kind: "flow"; selector: string; } | { kind: "unknown"; }; + export type SubscriptionWorkerAvailability = "observed" | "unavailable"; export type UpdateRemoteRuntimeRequest = { display_name?: string | null; endpoint: string; }; @@ -147,6 +181,8 @@ export type WorkerMaintenanceState = "compacting"; export type WorkerOperationState = "accepted" | "unsupported" | "rejected"; +export type WorkerRestoreRequest = { expected_observation_token: string; request_id: string; }; + export type WorkerRestoreResponse = { result: WorkerRestoreResult; runtime_id: string; worker_id: string; workspace_id: string; }; export type WorkerRestoreResult = { diagnostics?: Array; state: WorkerRestoreState; worker?: WorkerSummary | null; }; @@ -161,7 +197,7 @@ export type WorkerState = { kind: "idle"; } | { kind: "busy"; state: WorkerBusyS export type WorkerStateSnapshot = { last_command_id: number; last_finished_submission_request_id?: string | null; state: WorkerState; }; -export type WorkerSummary = { availability: SubscriptionWorkerAvailability; diagnostics?: Array; display_name?: string; host_id: string; implementation: WorkerImplementationSummary; label: string; last_seen_at?: string | null; pinned?: boolean; profile?: string | null; resource_key: string; retention_state?: string; runtime_id: string; singleton_key?: string | null; state: string; tags?: Array; workdir_attachments?: Array; worker_id: string; worker_state?: WorkerStateSnapshot | null; workspace: WorkerWorkspaceSummary; }; +export type WorkerSummary = { availability: SubscriptionWorkerAvailability; diagnostics?: Array; display_name?: string; host_id: string; implementation: WorkerImplementationSummary; label: string; last_seen_at?: string | null; pinned?: boolean; profile?: string | null; resource_key: string; restore_observation_token?: string | null; retention_state?: string; runtime_id: string; singleton_key?: string | null; state: string; tags?: Array; workdir_attachments?: Array; worker_id: string; worker_state?: WorkerStateSnapshot | null; workspace: WorkerWorkspaceSummary; }; export type WorkerWorkdirAttachmentSummary = { alias: string; effective_permissions: ExternalWorkdirPermissions; working_directory: WorkingDirectorySummary; }; @@ -169,16 +205,18 @@ export type WorkerWorkspaceSummary = { identity: string; visibility: string; wor export type WorkingDirectoryCleanupTarget = { kind: string; repository_key: string; working_directory_id: string; }; -export type WorkingDirectoryMaterializerKind = "runtime_git_clone" | "client_hosted_external"; +export type WorkingDirectoryMaterializerKind = "logical_workspace_config" | "runtime_git_clone" | "client_hosted_external"; export type WorkingDirectoryOccupancy = { display_name: string; linked_at: string; runtime_id: string; worker_id: string; }; -export type WorkingDirectorySource = { kind: "repository"; repository_key: string; } | { grant_id: string; grant_permissions: ExternalWorkdirPermissions; kind: "external_grant"; }; +export type WorkingDirectorySource = { access: WorkspaceConfigAccess; content_path: string; kind: "workspace_config"; purpose: string; } | { kind: "repository"; repository_key: string; } | { grant_id: string; grant_permissions: ExternalWorkdirPermissions; kind: "external_grant"; }; export type WorkingDirectoryStatusKind = "active" | "cleanup_pending" | "corrupted" | "not_found" | "unknown"; export type WorkingDirectorySummary = { cleanliness?: string | null; cleanup_target?: WorkingDirectoryCleanupTarget | null; creation_ref?: string | null; creation_selector?: string | null; creation_tree?: string | null; current_ref?: string | null; current_selector?: string | null; current_tree?: string | null; display_name?: string | null; materializer_kind: WorkingDirectoryMaterializerKind; observed_at_epoch_seconds?: number | null; occupied_by?: WorkingDirectoryOccupancy | null; source: WorkingDirectorySource; status: WorkingDirectoryStatusKind; working_directory_id: string; }; +export type WorkspaceConfigAccess = "read_only" | "read_write"; + export type WorkspaceRuntimeBindingState = "configured" | "verified" | "revoked"; export type WorkspaceRuntimeBindingSummary = { connection_state: RuntimeConnectionDisplayState; revision: number; state: WorkspaceRuntimeBindingState; verification?: RuntimeVerificationEvidenceSummary | null; workspace_key_generation?: number | null; workspace_key_id?: string | null; }; diff --git a/web/workspace/src/lib/generated/ticket-api.ts b/web/workspace/src/lib/generated/ticket-api.ts index 24d0a4a79..657aa7d1b 100644 --- a/web/workspace/src/lib/generated/ticket-api.ts +++ b/web/workspace/src/lib/generated/ticket-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-ticket-browser-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:f9f0f8918dbcce386e94d5368fa938f02565ca98f9f67f07537bd88ed8c49870) +// Input contract: 0.1.0 (sha256:fd06f382c6d33a5c3cc98ef9e264545923c53fa95a9be555a85680f98c6065da) // Options: ticket-objective-merge-request-browser-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Output: web/workspace/src/lib/generated/ticket-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_ticket_api_types -- --check diff --git a/web/workspace/src/lib/generated/workdir-api.ts b/web/workspace/src/lib/generated/workdir-api.ts index 0fa79bece..cf273be40 100644 --- a/web/workspace/src/lib/generated/workdir-api.ts +++ b/web/workspace/src/lib/generated/workdir-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-workdir-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:f9f0f8918dbcce386e94d5368fa938f02565ca98f9f67f07537bd88ed8c49870) +// Input contract: 0.1.0 (sha256:fd06f382c6d33a5c3cc98ef9e264545923c53fa95a9be555a85680f98c6065da) // Options: workdir-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Output: web/workspace/src/lib/generated/workdir-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_workdir_api_types -- --check @@ -27,7 +27,7 @@ export type WorkingDirectoryDetailResponse = { diagnostics: Array; i export type WorkingDirectoryListResponse = { diagnostics: Array; items: Array; workspace_id: string; }; -export type WorkingDirectoryMaterializerKind = "runtime_git_clone" | "client_hosted_external"; +export type WorkingDirectoryMaterializerKind = "logical_workspace_config" | "runtime_git_clone" | "client_hosted_external"; export type WorkingDirectoryOccupancy = { display_name: string; linked_at: string; runtime_id: string; worker_id: string; }; @@ -37,8 +37,10 @@ export type WorkingDirectoryRemovalRequest = { reason: string; }; export type WorkingDirectoryRemovalResponse = { disposition: WorkingDirectoryRemovalDisposition; failure_category?: string | null; retryable: boolean; working_directory_id: string; }; -export type WorkingDirectorySource = { kind: "repository"; repository_key: string; } | { grant_id: string; grant_permissions: ExternalWorkdirPermissions; kind: "external_grant"; }; +export type WorkingDirectorySource = { access: WorkspaceConfigAccess; content_path: string; kind: "workspace_config"; purpose: string; } | { kind: "repository"; repository_key: string; } | { grant_id: string; grant_permissions: ExternalWorkdirPermissions; kind: "external_grant"; }; export type WorkingDirectoryStatusKind = "active" | "cleanup_pending" | "corrupted" | "not_found" | "unknown"; export type WorkingDirectorySummary = { cleanliness?: string | null; cleanup_target?: WorkingDirectoryCleanupTarget | null; creation_ref?: string | null; creation_selector?: string | null; creation_tree?: string | null; current_ref?: string | null; current_selector?: string | null; current_tree?: string | null; display_name?: string | null; materializer_kind: WorkingDirectoryMaterializerKind; observed_at_epoch_seconds?: number | null; occupied_by?: WorkingDirectoryOccupancy | null; source: WorkingDirectorySource; status: WorkingDirectoryStatusKind; working_directory_id: string; }; + +export type WorkspaceConfigAccess = "read_only" | "read_write"; diff --git a/web/workspace/src/lib/generated/worker-launch-api.ts b/web/workspace/src/lib/generated/worker-launch-api.ts index 42881f6ce..f56aba174 100644 --- a/web/workspace/src/lib/generated/worker-launch-api.ts +++ b/web/workspace/src/lib/generated/worker-launch-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-worker-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:f9f0f8918dbcce386e94d5368fa938f02565ca98f9f67f07537bd88ed8c49870) +// Input contract: 0.1.0 (sha256:fd06f382c6d33a5c3cc98ef9e264545923c53fa95a9be555a85680f98c6065da) // Options: worker-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Output: web/workspace/src/lib/generated/worker-launch-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_worker_launch_api_types -- --check @@ -21,6 +21,14 @@ export type DiagnosticSeverity = "info" | "warning" | "error"; export type ExternalWorkdirPermissions = { command: boolean; read: boolean; write: boolean; }; +export type FeatureInvocation = { arguments: Array; identity: FeatureInvocationIdentity; invocation_id: string; name: string; }; + +export type FeatureInvocationIdentity = string; + +export type InvocationArgumentValue = { name: string; value: InvocationValue; }; + +export type InvocationValue = { kind: "string"; value: string; } | { kind: "integer"; value: number; } | { kind: "boolean"; value: boolean; }; + export type PasteArtifactAvailability = "available" | "unavailable" | "integrity_failed"; export type PasteArtifactMediaType = "text_plain_utf8"; @@ -35,7 +43,7 @@ export type RuntimeWorkingDirectoryCleanupTarget = { kind: string; repository_id export type RuntimeWorkingDirectorySummary = { cleanliness?: string | null; cleanup_target?: RuntimeWorkingDirectoryCleanupTarget | null; creation_ref?: string | null; creation_selector?: string | null; creation_tree?: string | null; current_ref?: string | null; current_selector?: string | null; current_tree?: string | null; display_name?: string | null; materializer_kind: WorkingDirectoryMaterializerKind; observed_at_epoch_seconds?: number | null; occupied_by?: WorkingDirectoryOccupancy | null; repository_id: string; status: WorkingDirectoryStatusKind; working_directory_id: string; }; -export type Segment = { content: string; kind: "text"; } | { chars: number; content: string; id: number; kind: "paste"; lines: number; } | { artifact: PasteArtifactRef; kind: "paste_artifact"; } | { file: UploadedFileRef; kind: "uploaded_file"; } | { kind: "file_ref"; path: string; } | { kind: "flow"; selector: string; } | { kind: "unknown"; }; +export type Segment = { content: string; kind: "text"; } | { chars: number; content: string; id: number; kind: "paste"; lines: number; } | { artifact: PasteArtifactRef; kind: "paste_artifact"; } | { file: UploadedFileRef; kind: "uploaded_file"; } | { kind: "file_ref"; path: string; } | { invocation: FeatureInvocation; kind: "feature_invoke"; } | { kind: "flow"; selector: string; } | { kind: "unknown"; }; export type SubjektivWorkerConnectionRequest = { subject_id: string; }; @@ -69,7 +77,7 @@ export type WorkerState = { kind: "idle"; } | { kind: "busy"; state: WorkerBusyS export type WorkerStateSnapshot = { last_command_id: number; last_finished_submission_request_id?: string | null; state: WorkerState; }; -export type WorkerSummary = { availability: SubscriptionWorkerAvailability; diagnostics?: Array; display_name?: string; host_id: string; implementation: WorkerImplementationSummary; label: string; last_seen_at?: string | null; pinned?: boolean; profile?: string | null; resource_key: string; retention_state?: string; runtime_id: string; singleton_key?: string | null; state: string; tags?: Array; workdir_attachments?: Array; worker_id: string; worker_state?: WorkerStateSnapshot | null; workspace: WorkerWorkspaceSummary; }; +export type WorkerSummary = { availability: SubscriptionWorkerAvailability; diagnostics?: Array; display_name?: string; host_id: string; implementation: WorkerImplementationSummary; label: string; last_seen_at?: string | null; pinned?: boolean; profile?: string | null; resource_key: string; restore_observation_token?: string | null; retention_state?: string; runtime_id: string; singleton_key?: string | null; state: string; tags?: Array; workdir_attachments?: Array; worker_id: string; worker_state?: WorkerStateSnapshot | null; workspace: WorkerWorkspaceSummary; }; export type WorkerWorkdirAttachmentSummary = { alias: string; effective_permissions: ExternalWorkdirPermissions; working_directory: WorkingDirectorySummary; }; @@ -77,16 +85,18 @@ export type WorkerWorkspaceSummary = { identity: string; visibility: string; wor export type WorkingDirectoryCleanupTarget = { kind: string; repository_key: string; working_directory_id: string; }; -export type WorkingDirectoryMaterializerKind = "runtime_git_clone" | "client_hosted_external"; +export type WorkingDirectoryMaterializerKind = "logical_workspace_config" | "runtime_git_clone" | "client_hosted_external"; export type WorkingDirectoryOccupancy = { display_name: string; linked_at: string; runtime_id: string; worker_id: string; }; export type WorkingDirectoryRepositoryOption = { default_selector?: string | null; repository_key: string; }; -export type WorkingDirectorySource = { kind: "repository"; repository_key: string; } | { grant_id: string; grant_permissions: ExternalWorkdirPermissions; kind: "external_grant"; }; +export type WorkingDirectorySource = { access: WorkspaceConfigAccess; content_path: string; kind: "workspace_config"; purpose: string; } | { kind: "repository"; repository_key: string; } | { grant_id: string; grant_permissions: ExternalWorkdirPermissions; kind: "external_grant"; }; export type WorkingDirectoryStatusKind = "active" | "cleanup_pending" | "corrupted" | "not_found" | "unknown"; export type WorkingDirectorySummary = { cleanliness?: string | null; cleanup_target?: WorkingDirectoryCleanupTarget | null; creation_ref?: string | null; creation_selector?: string | null; creation_tree?: string | null; current_ref?: string | null; current_selector?: string | null; current_tree?: string | null; display_name?: string | null; materializer_kind: WorkingDirectoryMaterializerKind; observed_at_epoch_seconds?: number | null; occupied_by?: WorkingDirectoryOccupancy | null; source: WorkingDirectorySource; status: WorkingDirectoryStatusKind; working_directory_id: string; }; +export type WorkspaceConfigAccess = "read_only" | "read_write"; + export type WorkspaceWorkerFeatureConnectionsRequest = { subjektiv?: SubjektivWorkerConnectionRequest | null; }; diff --git a/web/workspace/src/lib/workspace/api/runtime-workers.ts b/web/workspace/src/lib/workspace/api/runtime-workers.ts index 5247b8366..646fb0d32 100644 --- a/web/workspace/src/lib/workspace/api/runtime-workers.ts +++ b/web/workspace/src/lib/workspace/api/runtime-workers.ts @@ -10,9 +10,13 @@ import type { RuntimeCleanupPlanResponse, RuntimeWorkerLifecycleResult, WorkerOperationState, + WorkerRestoreResponse, + WorkerRestoreState, WorkerRetentionResponse, } from "#lib/generated/runtime-api.ts"; +import { parseWorkerSummary } from "#lib/workspace/api/workers.ts"; + const MAX_STRING_BYTES = 4_096; const MAX_CANDIDATES = 1_000; const MAX_RESULTS = 2_000; @@ -68,14 +72,20 @@ function exactKeys( ): void { const allowed = new Set([...required, ...optional]); for (const key of Object.keys(value)) { - if (!allowed.has(key)) fail(`${path}.${key}`, "is not part of the wire contract"); + if (!allowed.has(key)) { + fail(`${path}.${key}`, "is not part of the wire contract"); + } } for (const key of required) { if (!Object.hasOwn(value, key)) fail(`${path}.${key}`, "is required"); } } -function boundedString(value: unknown, path: string, allowEmpty = false): string { +function boundedString( + value: unknown, + path: string, + allowEmpty = false, +): string { if (typeof value !== "string") return fail(path, "must be a string"); if (!allowEmpty && value.length === 0) return fail(path, "must not be empty"); if (encoder.encode(value).byteLength > MAX_STRING_BYTES) { @@ -91,7 +101,9 @@ function boolean(value: unknown, path: string): boolean { function array(value: unknown, path: string, max: number): unknown[] { if (!Array.isArray(value)) return fail(path, "must be an array"); - if (value.length > max) return fail(path, `must contain at most ${max} items`); + if (value.length > max) { + return fail(path, `must contain at most ${max} items`); + } return value; } @@ -101,12 +113,18 @@ function strings(value: unknown, path: string): string[] { ); } -function optionalNullableString(value: unknown, path: string): string | null | undefined { +function optionalNullableString( + value: unknown, + path: string, +): string | null | undefined { if (value === undefined || value === null) return value; return boundedString(value, path); } -function optionalNullableSafeInteger(value: unknown, path: string): number | null | undefined { +function optionalNullableSafeInteger( + value: unknown, + path: string, +): number | null | undefined { if (value === undefined || value === null) return value; if (!Number.isSafeInteger(value) || (value as number) < 0) { return fail(path, "must be a non-negative safe integer"); @@ -115,7 +133,9 @@ function optionalNullableSafeInteger(value: unknown, path: string): number | nul } function targetKind(value: unknown, path: string): CleanupTargetKind { - if (typeof value !== "string" || !TARGET_KINDS.has(value as CleanupTargetKind)) { + if ( + typeof value !== "string" || !TARGET_KINDS.has(value as CleanupTargetKind) + ) { return fail(path, "has an unknown cleanup action"); } return value as CleanupTargetKind; @@ -164,13 +184,25 @@ function workerCandidate(value: unknown, path: string): CleanupWorkerCandidate { target_id: boundedString(item.target_id, `${path}.target_id`), action: targetKind(item.action, `${path}.action`), worker_id: boundedString(item.worker_id, `${path}.worker_id`), - runtime_worker_id: boundedString(item.runtime_worker_id, `${path}.runtime_worker_id`), + runtime_worker_id: boundedString( + item.runtime_worker_id, + `${path}.runtime_worker_id`, + ), runtime_id: boundedString(item.runtime_id, `${path}.runtime_id`), reason: boundedString(item.reason, `${path}.reason`, true), - blocking_reason: optionalNullableString(item.blocking_reason, `${path}.blocking_reason`), + blocking_reason: optionalNullableString( + item.blocking_reason, + `${path}.blocking_reason`, + ), pinned: boolean(item.pinned, `${path}.pinned`), - retention_state: boundedString(item.retention_state, `${path}.retention_state`), - linked_workdir_ids: strings(item.linked_workdir_ids, `${path}.linked_workdir_ids`), + retention_state: boundedString( + item.retention_state, + `${path}.retention_state`, + ), + linked_workdir_ids: strings( + item.linked_workdir_ids, + `${path}.linked_workdir_ids`, + ), running_linked: boolean(item.running_linked, `${path}.running_linked`), estimated_reclaim_bytes: optionalNullableSafeInteger( item.estimated_reclaim_bytes, @@ -179,7 +211,10 @@ function workerCandidate(value: unknown, path: string): CleanupWorkerCandidate { }; } -function workdirCandidate(value: unknown, path: string): CleanupWorkdirCandidate { +function workdirCandidate( + value: unknown, + path: string, +): CleanupWorkdirCandidate { const item = object(value, path); exactKeys( item, @@ -213,10 +248,19 @@ function workdirCandidate(value: unknown, path: string): CleanupWorkdirCandidate action: targetKind(item.action, `${path}.action`), workdir_id: boundedString(item.workdir_id, `${path}.workdir_id`), runtime_id: boundedString(item.runtime_id, `${path}.runtime_id`), - repository_key: boundedString(item.repository_key, `${path}.repository_key`), + repository_key: boundedString( + item.repository_key, + `${path}.repository_key`, + ), reason: boundedString(item.reason, `${path}.reason`, true), - blocking_reason: optionalNullableString(item.blocking_reason, `${path}.blocking_reason`), - linked_worker_ids: strings(item.linked_worker_ids, `${path}.linked_worker_ids`), + blocking_reason: optionalNullableString( + item.blocking_reason, + `${path}.blocking_reason`, + ), + linked_worker_ids: strings( + item.linked_worker_ids, + `${path}.linked_worker_ids`, + ), linked_running_worker_ids: strings( item.linked_running_worker_ids, `${path}.linked_running_worker_ids`, @@ -232,7 +276,9 @@ function workdirCandidate(value: unknown, path: string): CleanupWorkdirCandidate }; } -export function parseWorkerRetentionResponse(value: unknown): WorkerRetentionResponse { +export function parseWorkerRetentionResponse( + value: unknown, +): WorkerRetentionResponse { const item = object(value, "Worker retention response"); exactKeys( item, @@ -241,9 +287,18 @@ export function parseWorkerRetentionResponse(value: unknown): WorkerRetentionRes "Worker retention response", ); return { - workspace_id: boundedString(item.workspace_id, "Worker retention response.workspace_id"), - runtime_id: boundedString(item.runtime_id, "Worker retention response.runtime_id"), - worker_id: boundedString(item.worker_id, "Worker retention response.worker_id"), + workspace_id: boundedString( + item.workspace_id, + "Worker retention response.workspace_id", + ), + runtime_id: boundedString( + item.runtime_id, + "Worker retention response.runtime_id", + ), + worker_id: boundedString( + item.worker_id, + "Worker retention response.worker_id", + ), pinned: boolean(item.pinned, "Worker retention response.pinned"), retention_state: boundedString( item.retention_state, @@ -268,7 +323,10 @@ export function parseRuntimeWorkerLifecycleResult( [], "Runtime Worker lifecycle result", ); - const state = boundedString(item.state, "Runtime Worker lifecycle result.state"); + const state = boundedString( + item.state, + "Runtime Worker lifecycle result.state", + ); if (!WORKER_OPERATION_STATES.has(state as WorkerOperationState)) { fail("Runtime Worker lifecycle result.state", "has an unknown value"); } @@ -289,7 +347,58 @@ export function parseRuntimeWorkerLifecycleResult( }; } -export function parseRuntimeCleanupPlan(value: unknown): RuntimeCleanupPlanResponse { +/** Restore is a wrapped Workspace response, not the flat Stop lifecycle result. */ +export function parseWorkerRestoreResponse( + value: unknown, +): WorkerRestoreResponse { + const path = "Worker Restore response"; + const item = object(value, path); + exactKeys( + item, + ["workspace_id", "runtime_id", "worker_id", "result"], + [], + path, + ); + const result = object(item.result, `${path}.result`); + exactKeys(result, ["state"], ["diagnostics", "worker"], `${path}.result`); + const state = boundedString(result.state, `${path}.result.state`); + if ( + !["accepted", "rejected", "rolled_back", "reconciliation_required"] + .includes(state) + ) { + fail(`${path}.result.state`, "has an unknown value"); + } + const workspace_id = boundedString(item.workspace_id, `${path}.workspace_id`); + const runtime_id = boundedString(item.runtime_id, `${path}.runtime_id`); + const worker_id = boundedString(item.worker_id, `${path}.worker_id`); + const worker = result.worker == null + ? result.worker as null | undefined + : parseWorkerSummary(result.worker); + if ( + worker && + (worker.runtime_id !== runtime_id || worker.worker_id !== worker_id || + (worker.workspace.workspace_id != null && + worker.workspace.workspace_id !== workspace_id)) + ) { + fail(`${path}.result.worker`, "does not match the response identity"); + } + return { + workspace_id, + runtime_id, + worker_id, + result: { + state: state as WorkerRestoreState, + diagnostics: result.diagnostics === undefined + ? [] + : diagnostics(result.diagnostics, `${path}.result.diagnostics`), + worker, + }, + }; +} + +export function parseRuntimeCleanupPlan( + value: unknown, +): RuntimeCleanupPlanResponse { const item = object(value, "Runtime cleanup plan"); exactKeys( item, @@ -307,22 +416,44 @@ export function parseRuntimeCleanupPlan(value: unknown): RuntimeCleanupPlanRespo "Runtime cleanup plan", ); return { - workspace_id: boundedString(item.workspace_id, "Runtime cleanup plan.workspace_id"), - runtime_id: boundedString(item.runtime_id, "Runtime cleanup plan.runtime_id"), - generated_at: boundedString(item.generated_at, "Runtime cleanup plan.generated_at"), + workspace_id: boundedString( + item.workspace_id, + "Runtime cleanup plan.workspace_id", + ), + runtime_id: boundedString( + item.runtime_id, + "Runtime cleanup plan.runtime_id", + ), + generated_at: boundedString( + item.generated_at, + "Runtime cleanup plan.generated_at", + ), revision: boundedString(item.revision, "Runtime cleanup plan.revision"), digest: boundedString(item.digest, "Runtime cleanup plan.digest"), - workers: array(item.workers, "Runtime cleanup plan.workers", MAX_CANDIDATES).map( - (entry, index) => workerCandidate(entry, `Runtime cleanup plan.workers[${index}]`), + workers: array(item.workers, "Runtime cleanup plan.workers", MAX_CANDIDATES) + .map( + (entry, index) => + workerCandidate(entry, `Runtime cleanup plan.workers[${index}]`), + ), + workdirs: array( + item.workdirs, + "Runtime cleanup plan.workdirs", + MAX_CANDIDATES, + ).map( + (entry, index) => + workdirCandidate(entry, `Runtime cleanup plan.workdirs[${index}]`), ), - workdirs: array(item.workdirs, "Runtime cleanup plan.workdirs", MAX_CANDIDATES).map( - (entry, index) => workdirCandidate(entry, `Runtime cleanup plan.workdirs[${index}]`), + diagnostics: diagnostics( + item.diagnostics, + "Runtime cleanup plan.diagnostics", ), - diagnostics: diagnostics(item.diagnostics, "Runtime cleanup plan.diagnostics"), }; } -function executionResult(value: unknown, path: string): RuntimeCleanupExecutionResult { +function executionResult( + value: unknown, + path: string, +): RuntimeCleanupExecutionResult { const item = object(value, path); exactKeys(item, ["target_id", "action", "status", "message"], [], path); return { @@ -339,18 +470,42 @@ export function parseRuntimeCleanupExecution( const item = object(value, "Runtime cleanup execution"); exactKeys( item, - ["workspace_id", "runtime_id", "executed_at", "results", "plan_after", "diagnostics"], + [ + "workspace_id", + "runtime_id", + "executed_at", + "results", + "plan_after", + "diagnostics", + ], [], "Runtime cleanup execution", ); return { - workspace_id: boundedString(item.workspace_id, "Runtime cleanup execution.workspace_id"), - runtime_id: boundedString(item.runtime_id, "Runtime cleanup execution.runtime_id"), - executed_at: boundedString(item.executed_at, "Runtime cleanup execution.executed_at"), - results: array(item.results, "Runtime cleanup execution.results", MAX_RESULTS).map( - (entry, index) => executionResult(entry, `Runtime cleanup execution.results[${index}]`), + workspace_id: boundedString( + item.workspace_id, + "Runtime cleanup execution.workspace_id", + ), + runtime_id: boundedString( + item.runtime_id, + "Runtime cleanup execution.runtime_id", + ), + executed_at: boundedString( + item.executed_at, + "Runtime cleanup execution.executed_at", + ), + results: array( + item.results, + "Runtime cleanup execution.results", + MAX_RESULTS, + ).map( + (entry, index) => + executionResult(entry, `Runtime cleanup execution.results[${index}]`), ), plan_after: parseRuntimeCleanupPlan(item.plan_after), - diagnostics: diagnostics(item.diagnostics, "Runtime cleanup execution.diagnostics"), + diagnostics: diagnostics( + item.diagnostics, + "Runtime cleanup execution.diagnostics", + ), }; } diff --git a/web/workspace/src/lib/workspace/api/workdirs.ts b/web/workspace/src/lib/workspace/api/workdirs.ts index 8df77af4a..dbfedcfb7 100644 --- a/web/workspace/src/lib/workspace/api/workdirs.ts +++ b/web/workspace/src/lib/workspace/api/workdirs.ts @@ -131,6 +131,7 @@ export function parseWorkingDirectorySummary( materializer_kind: enumField(record, "materializer_kind", [ "runtime_git_clone", "client_hosted_external", + "logical_workspace_config", ]), status: enumField(record, "status", [ "active", @@ -177,10 +178,27 @@ export function parseWorkingDirectorySummary( function parseWorkingDirectorySource(value: unknown): WorkingDirectorySource { const source = exactRecord( value, - new Set(["kind", "repository_key", "grant_id", "grant_permissions"]), + new Set(["kind", "repository_key", "grant_id", "grant_permissions", "access", "content_path", "purpose"]), "Workdir source", ); const kind = stringField(source, "kind"); + if (kind === "workspace_config") { + if (source.repository_key !== undefined || source.grant_id !== undefined || source.grant_permissions !== undefined) { + throw new Error("Logical config source must not contain repository or External authority"); + } + const access = stringField(source, "access"); + if (access !== "read_only" && access !== "read_write") { + throw new Error("Logical config access has an unsupported value"); + } + const content_path = stringField(source, "content_path"); + if (content_path !== "/workspace-config") { + throw new Error("Logical config must reference its WIP content entrance"); + } + return { kind, access, content_path, purpose: stringField(source, "purpose") }; + } + if (source.access !== undefined || source.content_path !== undefined || source.purpose !== undefined) { + throw new Error("Filesystem Workdir source must not contain logical config metadata"); + } if (kind === "repository") { if ( source.grant_id !== undefined || source.grant_permissions !== undefined diff --git a/web/workspace/src/lib/workspace/api/workers.test.ts b/web/workspace/src/lib/workspace/api/workers.test.ts index f00c00189..b70336003 100644 --- a/web/workspace/src/lib/workspace/api/workers.test.ts +++ b/web/workspace/src/lib/workspace/api/workers.test.ts @@ -382,3 +382,20 @@ Deno.test("Worker create request parser requires the complete shared request", ( "kind is invalid", ); }); + +Deno.test("Worker launch request preserves typed Feature invocations and rejects unsafe values", () => { + const invocation = { invocation_id: "invoke-1", identity: "test/run", name: "run", arguments: [ + { name: "path", value: { kind: "string", value: "資料/a b" } }, + { name: "count", value: { kind: "integer", value: 1 } }, + { name: "enabled", value: { kind: "boolean", value: true } }, + ] }; + const request = { runtime_id: "runtime-a", display_name: "Worker A", profile: "builtin:coder", ticket_assignment: null, + initial_submit: [{ kind: "feature_invoke", invocation }], workdir_attachments: [], feature_connections: {}, control_operation_id: null }; + assertEquals(parseCreateWorkspaceWorkerRequest(request).initial_submit, request.initial_submit); + assertThrows(() => parseCreateWorkspaceWorkerRequest({ ...request, initial_submit: [{ kind: "feature_invoke", invocation: { + ...invocation, arguments: [{ name: "count", value: { kind: "integer", value: 9007199254740992 } }], + } }] }), Error, "safe integer"); + assertThrows(() => parseCreateWorkspaceWorkerRequest({ ...request, initial_submit: [{ kind: "feature_invoke", invocation: { + ...invocation, extra: true, + } }] }), Error, "unknown field extra"); +}); diff --git a/web/workspace/src/lib/workspace/api/workers.ts b/web/workspace/src/lib/workspace/api/workers.ts index df50de158..856cadd7c 100644 --- a/web/workspace/src/lib/workspace/api/workers.ts +++ b/web/workspace/src/lib/workspace/api/workers.ts @@ -97,7 +97,11 @@ function optional( label: string, parse: (item: unknown, label: string) => T, ): T | null | undefined { - return value === undefined ? undefined : value === null ? null : parse(value, label); + return value === undefined + ? undefined + : value === null + ? null + : parse(value, label); } function diagnostic(value: unknown, label: string): Diagnostic { @@ -372,6 +376,7 @@ export function parseWorkerSummary( "availability", "state", "worker_state", + "restore_observation_token", "last_seen_at", "pinned", "retention_state", @@ -406,13 +411,28 @@ export function parseWorkerSummary( `${label}.singleton_key`, string, ), - tags: item.tags === undefined ? [] : array(item.tags, `${label}.tags`, string), + tags: item.tags === undefined + ? [] + : array(item.tags, `${label}.tags`, string), workspace: workspaceSummary(item.workspace, `${label}.workspace`), availability, state: string(item.state, `${label}.state`), worker_state: workerState, + restore_observation_token: optional( + item.restore_observation_token, + `${label}.restore_observation_token`, + (value, path) => { + const token = string(value, path); + if (new TextEncoder().encode(token).byteLength > 256) { + throw new Error(`${path} exceeds its 256 UTF-8 byte limit`); + } + return token; + }, + ), last_seen_at: optional(item.last_seen_at, `${label}.last_seen_at`, string), - pinned: item.pinned === undefined ? false : boolean(item.pinned, `${label}.pinned`), + pinned: item.pinned === undefined + ? false + : boolean(item.pinned, `${label}.pinned`), retention_state: item.retention_state === undefined ? "" : string(item.retention_state, `${label}.retention_state`), @@ -420,11 +440,13 @@ export function parseWorkerSummary( item.implementation, `${label}.implementation`, ), - workdir_attachments: item.workdir_attachments === undefined ? undefined : array( - item.workdir_attachments, - `${label}.workdir_attachments`, - workerWorkdirAttachment, - ), + workdir_attachments: item.workdir_attachments === undefined + ? undefined + : array( + item.workdir_attachments, + `${label}.workdir_attachments`, + workerWorkdirAttachment, + ), diagnostics: item.diagnostics === undefined ? [] : array(item.diagnostics, `${label}.diagnostics`, diagnostic), @@ -634,11 +656,13 @@ function workerSummary( item.implementation, `${label}.implementation`, ), - workdir_attachments: item.workdir_attachments === undefined ? undefined : array( - item.workdir_attachments, - `${label}.workdir_attachments`, - runtimeWorkdirAttachment, - ), + workdir_attachments: item.workdir_attachments === undefined + ? undefined + : array( + item.workdir_attachments, + `${label}.workdir_attachments`, + runtimeWorkdirAttachment, + ), diagnostics: array(item.diagnostics, `${label}.diagnostics`, diagnostic), }; } @@ -728,11 +752,17 @@ function featureConnectionsRequest( ): WorkspaceWorkerFeatureConnectionsRequest { const item = record(value, label); exact(item, ["subjektiv"], label); - const subjektiv = optional(item.subjektiv, `${label}.subjektiv`, (value, itemLabel) => { - const connection = record(value, itemLabel); - exact(connection, ["subject_id"], itemLabel); - return { subject_id: string(connection.subject_id, `${itemLabel}.subject_id`) }; - }); + const subjektiv = optional( + item.subjektiv, + `${label}.subjektiv`, + (value, itemLabel) => { + const connection = record(value, itemLabel); + exact(connection, ["subject_id"], itemLabel); + return { + subject_id: string(connection.subject_id, `${itemLabel}.subject_id`), + }; + }, + ); return subjektiv === undefined ? {} : { subjektiv }; } @@ -838,6 +868,63 @@ function uploadedFile( }; } +function invocationValue( + value: unknown, + label: string, +): Extract< + Extract< + Segment, + { kind: "feature_invoke" } + >["invocation"]["arguments"][number], + { name: string } +>["value"] { + const item = record(value, label); + const kind = string(item.kind, `${label}.kind`); + exact(item, ["kind", "value"], label); + if (kind === "string") { + return { kind, value: string(item.value, `${label}.value`) }; + } + if (kind === "integer") { + const parsed = number(item.value, `${label}.value`); + if (!Number.isSafeInteger(parsed)) { + throw new Error(`${label}.value must be a safe integer`); + } + if (parsed < -2147483648 || parsed > 2147483647) { + throw new Error(`${label}.value must be a signed 32-bit integer`); + } + return { kind, value: parsed }; + } + if (kind === "boolean") { + return { kind, value: boolean(item.value, `${label}.value`) }; + } + throw new Error(`${label}.kind is invalid`); +} + +function featureInvocation( + value: unknown, + label: string, +): Extract["invocation"] { + const item = record(value, label); + exact(item, ["invocation_id", "identity", "name", "arguments"], label); + return { + invocation_id: string(item.invocation_id, `${label}.invocation_id`), + identity: string(item.identity, `${label}.identity`), + name: string(item.name, `${label}.name`), + arguments: array( + item.arguments, + `${label}.arguments`, + (argument, argumentLabel) => { + const entry = record(argument, argumentLabel); + exact(entry, ["name", "value"], argumentLabel); + return { + name: string(entry.name, `${argumentLabel}.name`), + value: invocationValue(entry.value, `${argumentLabel}.value`), + }; + }, + ), + }; +} + function segment(value: unknown, label: string): Segment { const item = record(value, label); const kind = string(item.kind, `${label}.kind`) as Segment["kind"]; @@ -866,6 +953,12 @@ function segment(value: unknown, label: string): Segment { case "file_ref": exact(item, ["kind", "path"], label); return { kind, path: string(item.path, `${label}.path`) }; + case "feature_invoke": + exact(item, ["kind", "invocation"], label); + return { + kind, + invocation: featureInvocation(item.invocation, `${label}.invocation`), + }; case "flow": exact(item, ["kind", "selector"], label); return { kind, selector: string(item.selector, `${label}.selector`) }; diff --git a/web/workspace/src/lib/workspace/console/ComposerInput.browser.spec.ts b/web/workspace/src/lib/workspace/console/ComposerInput.browser.spec.ts index db0498320..ad21c7c6a 100644 --- a/web/workspace/src/lib/workspace/console/ComposerInput.browser.spec.ts +++ b/web/workspace/src/lib/workspace/console/ComposerInput.browser.spec.ts @@ -2,6 +2,8 @@ import { cleanup, fireEvent, render, waitFor } from "@testing-library/svelte"; import { afterEach, expect, test, vi } from "vitest"; import { EditorView } from "@codemirror/view"; +import type { FeatureInvocationDescriptor, Segment } from "#lib/generated/protocol.ts"; +import { redo, undo } from "@codemirror/commands"; import ComposerInput from "./ComposerInput.svelte"; afterEach(cleanup); @@ -148,3 +150,362 @@ test("cursor changes, Escape, scope changes and disable discard late file result await ui.rerender({ disabled: true }); expect(ui.queryByRole("listbox")).toBeNull(); }); + +test("synchronous draft and cursor ABA invalidate old Feature completion generations", async () => { + const requests: Array>>> = []; + const resolver = vi.fn(() => { const result = deferred>(); requests.push(result); return result.promise; }); + const ui = render(ComposerInput, { historyScope: "feature-aba", resolveFeatureCompletions: resolver }); + const cm = editor(ui.container); + type(cm, "/ru"); + await waitFor(() => expect(requests).toHaveLength(1)); + cm.dispatch({ changes: { from: 2, to: 3, insert: "x" } }); + cm.dispatch({ changes: { from: 2, to: 3, insert: "u" } }); + requests[0].resolve([{ value: "stale", invocation: invocationDescriptor }]); + await waitFor(() => expect(requests).toHaveLength(2)); + expect(ui.queryByRole("option")).toBeNull(); + cm.dispatch({ selection: { anchor: 0 } }); + cm.dispatch({ selection: { anchor: 3 } }); + requests[1].resolve([{ value: "stale-cursor", invocation: invocationDescriptor }]); + await waitFor(() => expect(requests).toHaveLength(3)); + expect(ui.queryByRole("option")).toBeNull(); + requests[2].resolve([{ value: "run", invocation: invocationDescriptor }]); + expect((await ui.findByRole("option")).textContent).toContain("/run"); +}); + +test("restored chip descriptor edits reject same-draft cursor ABA", async () => { + const request = deferred>(); + const resolver = vi.fn(() => request.promise); + const ui = render(ComposerInput, { historyScope: "edit-cursor-aba", resolveFeatureCompletions: resolver }); + ui.component.restoreSegments([invocationSegment]); + const cm = editor(ui.container); + await fireEvent.mouseDown(ui.container.querySelector(".composer-typed-chip")!); + await waitFor(() => expect(resolver).toHaveBeenCalledOnce()); + cm.dispatch({ selection: { anchor: 0 } }); + cm.dispatch({ selection: { anchor: cm.state.doc.length } }); + request.resolve([{ value: "run", invocation: invocationDescriptor }]); + await Promise.resolve(); + await Promise.resolve(); + expect(ui.component.snapshot().segments).toEqual([invocationSegment]); +}); + +test.each(["Backspace", "Delete", "range", "cut", "replace"])("pending upload %s cancels its reservation before late completion and Undo", async (operation) => { + const oncancelupload = vi.fn(); + const ui = render(ComposerInput, { historyScope: `upload-delete-${operation}`, oncancelupload }); + const cm = editor(ui.container); + type(cm, "before after", 7); + const reservation = ui.component.reserveUpload("report.md")!; + const from = 7, to = cm.state.selection.main.head; + if (operation === "Backspace" || operation === "Delete") { + cm.dispatch({ selection: { anchor: operation === "Backspace" ? to : from } }); + await fireEvent.keyDown(cm.contentDOM, { key: operation }); + } else if (operation === "replace") { + cm.dispatch({ changes: { from, to, insert: "replacement" }, userEvent: "input" }); + } else { + // happy-dom emits selectionchange synchronously while CodeMirror updates a + // focused DOM range. Exercise the transaction/cut handler unfocused here; + // the Chromium suite covers focused keyboard range deletion. + cm.contentDOM.blur(); + cm.dispatch({ selection: { anchor: from, head: to } }); + if (operation === "cut") await fireEvent.cut(cm.contentDOM, { clipboardData: { setData: vi.fn() } }); + else await fireEvent.keyDown(cm.contentDOM, { key: "Backspace" }); + } + expect(oncancelupload).toHaveBeenCalledExactlyOnceWith(reservation); + expect(ui.component.completeUpload(reservation, uploadedSegment)).toBe(false); + expect(ui.component.snapshot().segments.some((segment) => segment.kind === "unknown" || segment.kind === "uploaded_file")).toBe(false); + undo(cm); + expect(ui.component.completeUpload(reservation, uploadedSegment)).toBe(false); + expect(ui.component.snapshot().segments.some((segment) => segment.kind === "unknown" || segment.kind === "uploaded_file")).toBe(false); + redo(cm); + ui.component.clear(); + expect(oncancelupload).toHaveBeenCalledOnce(); +}); + +test.each(["clear", "restore", "unmount"])("pending upload %s boundary cancels once without acquiring late staged resources", (boundary) => { + const oncancelupload = vi.fn(); + const onremoveatom = vi.fn(); + const ui = render(ComposerInput, { historyScope: `upload-boundary-${boundary}`, oncancelupload, onremoveatom }); + const reservation = ui.component.reserveUpload("report.md")!; + const complete = ui.component.completeUpload; + if (boundary === "clear") ui.component.clear(); + else if (boundary === "restore") ui.component.restoreSegments([invocationSegment]); + else ui.unmount(); + expect(oncancelupload).toHaveBeenCalledExactlyOnceWith(reservation); + expect(complete(reservation, uploadedSegment)).toBe(false); + expect(onremoveatom).not.toHaveBeenCalled(); +}); + +test("cancelled upload cannot become sendable via late completion or Undo", async () => { + const onremoveatom = vi.fn(); + const ui = render(ComposerInput, { historyScope: "upload-cancel-late", onremoveatom }); + const cm = editor(ui.container); + type(cm, "before after", 7); + const reservation = ui.component.reserveUpload("report.md")!; + ui.component.cancelUpload(reservation); + expect(ui.component.completeUpload(reservation, uploadedSegment)).toBe(false); + undo(cm); + expect(ui.component.snapshot().segments.some((segment) => segment.kind === "uploaded_file" || segment.kind === "unknown")).toBe(false); + ui.component.clear(); + expect(onremoveatom).not.toHaveBeenCalled(); +}); + +const invocationDescriptor: FeatureInvocationDescriptor = { + identity: "feature:test/run", name: "run", aliases: [], display_name: "Run", + description: "Declared invocation", syntax: "parenthesized", + arguments: [{ name: "path", position: 0, required: true, + value_type: { kind: "worker_file" }, completion: { kind: "worker_file" } }], +}; +const invocationSegment: Segment = { + kind: "feature_invoke", invocation: { invocation_id: "invoke-1", + identity: invocationDescriptor.identity, name: "run", + arguments: [{ name: "path", value: { kind: "string", value: "a b/資料" } }] }, +}; +const uploadedSegment: Segment = { kind: "uploaded_file", file: { + artifact_id: "artifact-1", file_name: "report.md", media_type: "text/markdown", + created_at_ms: 1, availability: "available", byte_len: 4, sha256: "abcd", +} }; + +test("argument popup stays anchored to the edited outer call when its data contains a call-looking slash", async () => { + const descriptor: FeatureInvocationDescriptor = { ...invocationDescriptor, arguments: [...invocationDescriptor.arguments, + { name: "mode", required: false, value_type: { kind: "enum", values: ["safe"] }, completion: { kind: "static", values: ["safe"] } }] }; + const resolveFeatureArgumentCompletions = vi.fn(async () => [{ value: "completed" }]); + const ui = render(ComposerInput, { historyScope: "quoted-data-argument", resolveFeatureArgumentCompletions, + resolveFeatureCompletions: async () => [{ value: "run", invocation: descriptor }] }); + const segment: Segment = { kind: "feature_invoke", invocation: { ...invocationSegment.invocation, arguments: [ + { name: "path", value: { kind: "string", value: "a /run(pa" } }, + { name: "mode", value: { kind: "string", value: "safe" } }, + ] } }; + ui.component.restoreSegments([segment]); + await fireEvent.mouseDown(ui.container.querySelector(".composer-typed-chip")!); + const cm = editor(ui.container); + await waitFor(() => expect(cm.state.doc.toString()).toBe('/run(path="a /run(pa", mode="safe")')); + cm.dispatch({ selection: { anchor: cm.state.doc.toString().lastIndexOf("(pa") + 3 } }); + await waitFor(() => expect(resolveFeatureArgumentCompletions).toHaveBeenLastCalledWith( + { invocation: descriptor.identity, argument: "path" }, "a /run(pa", expect.any(AbortSignal), + )); + await ui.findByRole("option"); + await fireEvent.keyDown(cm.contentDOM, { key: "Enter" }); + expect(cm.state.doc.toString()).toBe('/run(path="completed", mode="safe")'); +}); + +test("mid-cursor completion of an incomplete quote preserves following arguments and prose", async () => { + const descriptor: FeatureInvocationDescriptor = { ...invocationDescriptor, arguments: [...invocationDescriptor.arguments, + { name: "mode", required: false, value_type: { kind: "enum", values: ["safe"] }, completion: { kind: "static", values: ["safe"] } }] }; + const ui = render(ComposerInput, { historyScope: "incomplete-quote-suffix", + resolveFeatureCompletions: async () => [{ value: "run", invocation: descriptor }], + resolveFeatureArgumentCompletions: async () => [{ value: "completed" }] }); + const cm = editor(ui.container); + type(cm, "/ru"); + await ui.findByRole("option"); + await fireEvent.keyDown(cm.contentDOM, { key: "Enter" }); + const value = '/run(path="ab, mode=safe) following prose'; + cm.dispatch({ changes: { from: cm.state.doc.length, insert: value.slice(5) }, selection: { anchor: value.indexOf("ab") + 2 } }); + await ui.findByRole("option"); + await fireEvent.keyDown(cm.contentDOM, { key: "Enter" }); + expect(cm.state.doc.toString()).toBe('/run(path="completed", mode=safe) following prose'); + const snapshot = ui.component.snapshot(); + expect(snapshot.segments[0]).toMatchObject({ kind: "feature_invoke", invocation: { arguments: [ + { name: "path", value: { kind: "string", value: "completed" } }, + { name: "mode", value: { kind: "string", value: "safe" } }, + ] } }); + expect(snapshot.segments[1]).toEqual({ kind: "text", content: " following prose" }); +}); + +test("selected malformed surrogate strings stay editable and reject Submit snapshot", async () => { + const ui = render(ComposerInput, { historyScope: "invalid-surrogate-submit", + resolveFeatureCompletions: async () => [{ value: "run", invocation: invocationDescriptor }] }); + const cm = editor(ui.container); + type(cm, "/ru"); + await ui.findByRole("option"); + await fireEvent.keyDown(cm.contentDOM, { key: "Enter" }); + const invalid = String.raw`"\uD800")`; + cm.dispatch({ changes: { from: cm.state.doc.length, insert: invalid }, selection: { anchor: 5 + invalid.length } }); + await Promise.resolve(); + expect(ui.container.querySelector(".composer-typed-chip")).toBeNull(); + expect(() => ui.component.snapshot()).toThrow("unpaired Unicode surrogates"); + const valid = String.raw`"\uD83D\uDE00")`; + cm.dispatch({ changes: { from: 5, to: cm.state.doc.length, insert: valid }, selection: { anchor: 5 + valid.length } }); + const snapshot = ui.component.snapshot(); + expect(snapshot.segments[0]).toMatchObject({ kind: "feature_invoke", invocation: { arguments: [ + { name: "path", value: { kind: "string", value: "😀" } }, + ] } }); +}); + +test("selected calls chipify without promoting matching typed or pasted text", async () => { + const ui = render(ComposerInput, { historyScope: "invocation-select", + resolveFeatureCompletions: async () => [{ value: "run", invocation: invocationDescriptor }] }); + const cm = editor(ui.container); + type(cm, "/ru"); + await ui.findByRole("option"); + await fireEvent.keyDown(cm.contentDOM, { key: "Enter" }); + cm.dispatch({ changes: { from: cm.state.doc.length, insert: '"a b/資料")' }, selection: { anchor: '/run("a b/資料")'.length } }); + await waitFor(() => expect(ui.container.querySelectorAll(".composer-typed-chip")).toHaveLength(1)); + cm.dispatch({ changes: { from: cm.state.doc.length, insert: ' /run("literal")' } }); + await Promise.resolve(); + const snapshot = ui.component.snapshot(); + expect(snapshot.segments.map((segment) => segment.kind)).toEqual(["feature_invoke", "text"]); + expect(snapshot.segments[1]).toEqual({ kind: "text", content: ' /run("literal")' }); + expect(snapshot.segments[0].kind === "feature_invoke" && snapshot.segments[0].invocation.arguments[0].value) + .toEqual({ kind: "string", value: "a b/資料" }); +}); + +test("restored invocation chips fetch declarative descriptors and stay editable inside arguments", async () => { + const resolveFeatureCompletions = vi.fn(async () => [{ value: "run", invocation: invocationDescriptor }]); + const ui = render(ComposerInput, { historyScope: "invocation-edit", resolveFeatureCompletions }); + ui.component.restoreSegments([invocationSegment]); + const cm = editor(ui.container); + await fireEvent.mouseDown(ui.container.querySelector(".composer-typed-chip")!); + await waitFor(() => expect(cm.state.doc.toString()).toBe('/run(path="a b/資料")')); + expect(resolveFeatureCompletions).toHaveBeenCalledWith("run", expect.any(AbortSignal)); + cm.dispatch({ changes: { from: 11, to: 12, insert: "c" }, selection: { anchor: 12 } }); + await Promise.resolve(); + expect(ui.container.querySelector(".composer-typed-chip")).toBeNull(); + const snapshot = ui.component.snapshot(); + expect(snapshot.segments[0].kind).toBe("feature_invoke"); +}); + +// Fixture of the Workspace config Feature declaration, consumed by the same +// generic composer as other Features. Backend/WIP acceptance is tested in Rust; +// this test proves the real input component's selection and typed-submit edge. +const workspaceConfigDescriptor: FeatureInvocationDescriptor = { + identity: "builtin:workspace-config/attach", name: "workspace-config", aliases: [], + display_name: "Workspace config", description: "Select the logical Workspace config attachment", + syntax: "parenthesized", arguments: [{ name: "access", position: 0, required: false, + value_type: { kind: "enum", values: ["effective", "read_only", "read_write"] }, + completion: { kind: "none" } }], +}; + +test("workspace-config completion prepares a typed invocation without consuming the request prose", async () => { + const onsubmit = vi.fn(); + const resolveFeatureCompletions = vi.fn(async () => [ + { value: workspaceConfigDescriptor.name, invocation: workspaceConfigDescriptor }, + ]); + const ui = render(ComposerInput, { historyScope: "workspace-config-selected", onsubmit, + resolveFeatureCompletions }); + const cm = editor(ui.container); + type(cm, "/workspace-con"); + await ui.findByRole("option"); + await fireEvent.keyDown(cm.contentDOM, { key: "Enter" }); + expect(onsubmit).not.toHaveBeenCalled(); + const suffix = ') このWorkspaceのモデル設定を変更して'; + cm.dispatch({ changes: { from: cm.state.doc.length, insert: suffix }, + selection: { anchor: cm.state.doc.length + suffix.length } }); + await waitFor(() => expect(ui.container.querySelectorAll(".composer-typed-chip")).toHaveLength(1)); + const snapshot = ui.component.snapshot(); + expect(snapshot.segments).toEqual([ + { kind: "feature_invoke", invocation: { + invocation_id: expect.any(String), identity: workspaceConfigDescriptor.identity, + name: workspaceConfigDescriptor.name, arguments: [], + } }, + { kind: "text", content: " このWorkspaceのモデル設定を変更して" }, + ]); + await fireEvent.keyDown(cm.contentDOM, { key: "Enter", ctrlKey: true }); + expect(onsubmit).toHaveBeenCalledOnce(); + // Sending is not string scanning: identity and arguments survive intact. + expect(ui.component.snapshot().segments).toEqual(snapshot.segments); +}); + +test("workspace-config text alone is not invoked and a selected intent is removable", async () => { + const ui = render(ComposerInput, { historyScope: "workspace-config-literal", + resolveFeatureCompletions: async () => [{ value: workspaceConfigDescriptor.name, invocation: workspaceConfigDescriptor }] }); + const cm = editor(ui.container); + const prose = "/workspace-config このWorkspaceのモデル設定を変更して"; + ui.component.restoreSegments([{ kind: "text", content: prose }]); + expect(ui.component.snapshot().segments).toEqual([{ kind: "text", content: prose }]); + ui.component.clear(); + type(cm, "/workspace-con"); + await ui.findByRole("option"); + await fireEvent.keyDown(cm.contentDOM, { key: "Enter" }); + cm.dispatch({ changes: { from: cm.state.doc.length, insert: '"read_only")' }, + selection: { anchor: cm.state.doc.length + '"read_only")'.length } }); + await waitFor(() => expect(ui.container.querySelectorAll(".composer-typed-chip")).toHaveLength(1)); + expect(ui.component.snapshot().segments[0]).toMatchObject({ kind: "feature_invoke", invocation: { + arguments: [{ name: "access", value: { kind: "string", value: "read_only" } }], + } }); + // happy-dom's focused DOM selection can reenter CodeMirror during range + // replacement. Use the existing component suite's unfocused range fixture. + cm.contentDOM.blur(); + cm.dispatch({ selection: { anchor: 0, head: cm.state.doc.length } }); + await fireEvent.keyDown(cm.contentDOM, { key: "Backspace" }); + expect(ui.component.snapshot().segments).toEqual([]); +}); + +test("attachment adapter uses descriptor capability rather than a hardcoded slash name", async () => { + const onclientadapter = vi.fn(); + const adapter = { ...invocationDescriptor, name: "send-file", arguments: [], client_adapter: "attachment" as const }; + const ui = render(ComposerInput, { historyScope: "invocation-adapter", onclientadapter, + resolveFeatureCompletions: async () => [{ value: adapter.name, invocation: adapter }] }); + const cm = editor(ui.container); + type(cm, "before /send"); + await ui.findByRole("option"); + await fireEvent.keyDown(cm.contentDOM, { key: "Enter" }); + expect(onclientadapter).toHaveBeenCalledWith(adapter); + expect(cm.state.doc.toString()).toBe("before "); +}); + +test("uploads preserve reserved order, deletion undo, cleanup and accepted ownership", async () => { + const onremoveatom = vi.fn(); + const ui = render(ComposerInput, { historyScope: "invocation-upload", onremoveatom }); + const cm = editor(ui.container); + type(cm, "before "); + const first = ui.component.reserveUpload("report.md")!; + const second = ui.component.reserveUpload("second.md")!; + cm.dispatch({ changes: { from: cm.state.doc.length, insert: " after" } }); + const secondSegment = { ...uploadedSegment, file: { ...uploadedSegment.file, artifact_id: "artifact-2", file_name: "second.md" } }; + expect(ui.component.completeUpload(second, secondSegment)).toBe(true); + expect(ui.component.completeUpload(first, uploadedSegment)).toBe(true); + expect(ui.component.snapshot().segments).toEqual([ + { kind: "text", content: "before " }, uploadedSegment, secondSegment, { kind: "text", content: " after" }, + ]); + const tokenEnd = "before ".length + 3; + cm.dispatch({ selection: { anchor: tokenEnd } }); + await fireEvent.keyDown(cm.contentDOM, { key: "Backspace" }); + expect(onremoveatom).not.toHaveBeenCalled(); + undo(cm); + expect(ui.component.snapshot().segments).toContainEqual(uploadedSegment); + ui.component.clear(true); + expect(onremoveatom).not.toHaveBeenCalled(); + expect(undo(cm)).toBe(false); + ui.component.insertSegment(uploadedSegment, true); + ui.component.restoreSegments([invocationSegment]); + expect(onremoveatom).toHaveBeenCalledWith(uploadedSegment); +}); + +test("restored opaque segments and typed clipboard labels are not lost", async () => { + const ui = render(ComposerInput, { historyScope: "invocation-restore" }); + const segments: Segment[] = [invocationSegment, uploadedSegment, { kind: "file_ref", path: "a b/c" }, { kind: "flow", selector: "legacy" }]; + ui.component.restoreSegments(segments); + expect(ui.component.snapshot().segments).toEqual(segments); + const cm = editor(ui.container); + cm.dispatch({ selection: { anchor: 0, head: cm.state.doc.length } }); + const setData = vi.fn(); + await fireEvent.copy(cm.contentDOM, { clipboardData: { setData } }); + expect(setData).toHaveBeenCalledWith("text/plain", expect.stringContaining('/run(path="a b/資料")')); + expect(setData.mock.calls[0][1]).not.toContain("\uFFF8"); +}); + +test("restored descriptor requests cannot edit a newer draft after a scope change", async () => { + const request = deferred>(); + const resolver = vi.fn(() => request.promise); + const ui = render(ComposerInput, { historyScope: "invocation-stale-edit", completionScope: "worker-a", resolveFeatureCompletions: resolver }); + ui.component.restoreSegments([invocationSegment]); + await fireEvent.mouseDown(ui.container.querySelector(".composer-typed-chip")!); + await waitFor(() => expect(resolver).toHaveBeenCalledOnce()); + await ui.rerender({ completionScope: "worker-b" }); + ui.component.restoreSegments([{ kind: "text", content: "new draft" }]); + request.resolve([{ value: "run", invocation: invocationDescriptor }]); + await Promise.resolve(); + await Promise.resolve(); + expect(ui.component.snapshot().segments).toEqual([{ kind: "text", content: "new draft" }]); +}); + +test("pasted invocation text remains literal after descriptor selection", async () => { + const ui = render(ComposerInput, { historyScope: "invocation-paste-literal", resolveFeatureCompletions: async () => [{ value: "run", invocation: invocationDescriptor }] }); + const cm = editor(ui.container); + type(cm, "/ru"); + await ui.findByRole("option"); + await fireEvent.keyDown(cm.contentDOM, { key: "Enter" }); + cm.dispatch({ changes: { from: cm.state.doc.length, insert: '"x")' }, selection: { anchor: '/run("x")'.length } }); + await waitFor(() => expect(ui.container.querySelectorAll(".composer-typed-chip")).toHaveLength(1)); + await fireEvent.paste(cm.contentDOM, { clipboardData: { getData: () => ' /run("pasted")', items: [] } }); + expect(ui.component.snapshot().segments.map((segment) => segment.kind)).toEqual(["feature_invoke", "text"]); +}); diff --git a/web/workspace/src/lib/workspace/console/ComposerInput.svelte b/web/workspace/src/lib/workspace/console/ComposerInput.svelte index 7241e14da..c60639e85 100644 --- a/web/workspace/src/lib/workspace/console/ComposerInput.svelte +++ b/web/workspace/src/lib/workspace/console/ComposerInput.svelte @@ -6,6 +6,7 @@ Prec, StateEffect, StateField, + type Extension, } from "@codemirror/state"; import { Decoration, @@ -22,7 +23,11 @@ isolateHistory, } from "@codemirror/commands"; import { onMount } from "svelte"; - import type { Segment } from "#lib/generated/protocol.ts"; + import type { + CompletionContext, + FeatureInvocationDescriptor, + Segment, + } from "#lib/generated/protocol.ts"; import { handleComposerImagePaste, measureComposerPaste, @@ -32,11 +37,14 @@ composerDeletionRange, composerPasteAtoms, composerPasteToken, + composerTypedAtoms, + composerTypedToken, pasteChipLabel, snapshotComposerDraft, type ComposerDraftSnapshot, type ComposerPaste, type ComposerTextPaste, + type ComposerTypedSegment, } from "#lib/workspace/console/composer-draft.ts"; import { ComposerHistory, @@ -48,6 +56,10 @@ } from "#lib/workspace/console/composer-history.ts"; import { applyCompletion, completionSelection, completionTokenAt, localCommandCompletions, type ComposerCompletionEntry, type ComposerCompletionToken } from "./composer-completion"; import { shouldSubmitChatKey } from "#lib/workspace/console/chat-submit.ts"; + import { + invocationInput, + selectedFeatureInvocationAt, + } from "#lib/workspace/console/composer-invocation.ts"; interface Props { disabled?: boolean; @@ -61,6 +73,15 @@ oncommand?: () => void; completionScope?: string; resolveFileCompletions?: (prefix: string, signal: AbortSignal) => Promise; + resolveFeatureCompletions?: (prefix: string, signal: AbortSignal) => Promise; + resolveFeatureArgumentCompletions?: ( + context: CompletionContext, + prefix: string, + signal: AbortSignal, + ) => Promise; + onremoveatom?: (segment: Segment) => void; + oncancelupload?: (reservation: number) => void; + onclientadapter?: (descriptor: FeatureInvocationDescriptor) => void; } let { @@ -75,14 +96,47 @@ oncommand, completionScope = "", resolveFileCompletions, + resolveFeatureCompletions, + resolveFeatureArgumentCompletions, + onremoveatom, + oncancelupload, + onclientadapter, }: Props = $props(); let mountElement: HTMLDivElement; let view: EditorView | null = null; + let editorExtensions: Extension[] = []; + const stagedTypedAtoms = new Map(); + const pendingUploads = new Set(); let composerHistory = new ComposerHistory(); let restoringHistory = false; let nextPasteId = 1; let nextPasteKey = 1; + let nextTypedKey = 1; + const selectedFeatureInvocations = new Map(); + type SelectedInvocation = { from: number; descriptor: FeatureInvocationDescriptor }; + const selectInvocation = StateEffect.define(); + const restoreSelectedInvocations = StateEffect.define(); + const selectedInvocationState = StateField.define({ + create: () => [], + update(selected, transaction) { + const restored = transaction.effects.find((effect) => effect.is(restoreSelectedInvocations)); + if (restored) return restored.value; + const document = transaction.newDoc.toString(); + const retained = selected.map((item) => ({ + ...item, + from: transaction.changes.mapPos(item.from, 1), + })).filter((item) => + [item.descriptor.name, ...item.descriptor.aliases].some((name) => + document.startsWith(`/${name}(`, item.from) + ) + ); + for (const effect of transaction.effects) { + if (effect.is(selectInvocation)) retained.push(effect.value); + } + return retained; + }, + }); const editable = new Compartment(); const completionId = $props.id(); @@ -117,7 +171,12 @@ completionContext = context; closeCompletion(); if (disabled || !view.hasFocus || view.composing || !selection.empty) return; - const token = completionTokenAt(document, selection.head); + const token = completionTokenAt( + document, + selection.head, + [...selectedFeatureInvocations.values()], + view.state.field(selectedInvocationState), + ); if (!token) return; completionToken = token; if (token.kind === "command") { @@ -125,16 +184,26 @@ if (!completionEntries.length) completionMessage = "No matching commands"; return; } - if (!resolveFileCompletions) { closeCompletion(); return; } + const resolver = token.kind === "file" + ? resolveFileCompletions && ((signal: AbortSignal) => + resolveFileCompletions(token.prefix, signal)) + : token.kind === "feature" + ? resolveFeatureCompletions && ((signal: AbortSignal) => + resolveFeatureCompletions(token.prefix, signal)) + : token.context && resolveFeatureArgumentCompletions + ? ((signal: AbortSignal) => + resolveFeatureArgumentCompletions(token.context!, token.prefix, signal)) + : null; + if (!resolver) { closeCompletion(); return; } completionBusy = true; const version = completionVersion; const abort = new AbortController(); completionAbort = abort; - void resolveFileCompletions(token.prefix, abort.signal).then((entries) => { + void resolver(abort.signal).then((entries) => { if (version !== completionVersion || abort.signal.aborted) return; completionEntries = entries; completionSelected = entries.length ? 0 : null; - if (!entries.length) completionMessage = "No matching files"; + if (!entries.length) completionMessage = `No matching ${token.kind.replace("_", " ")}s`; }).catch((error) => { if (version !== completionVersion || abort.signal.aborted) return; completionMessage = error instanceof Error ? error.message : String(error); @@ -158,11 +227,27 @@ } const entry = index === null ? undefined : completionEntries[index]; if (!entry) return; + if (token.kind === "feature" && entry.invocation) { + selectedFeatureInvocations.set(entry.invocation.identity, entry.invocation); + if (entry.invocation.client_adapter === "attachment") { + closeCompletion(); + view.dispatch({ + changes: { from: token.start, to: token.end, insert: "" }, + selection: EditorSelection.cursor(token.start), + userEvent: "input.complete", + }); + onclientadapter?.(entry.invocation); + return; + } + } const result = applyCompletion(document, token, entry, action); const to = document.length - (result.value.length - result.cursor); closeCompletion(); view.dispatch({ changes: { from: token.start, to, insert: result.value.slice(token.start, result.cursor) }, + effects: token.kind === "feature" && entry.invocation + ? selectInvocation.of({ from: token.start, descriptor: entry.invocation }) + : [], selection: EditorSelection.cursor(result.cursor), userEvent: "input.complete", }); view.focus(); @@ -209,6 +294,12 @@ $effect(() => { void completionScope; + selectedFeatureInvocations.clear(); + closeCompletion(); + completionContext = ""; + }); + + $effect(() => { void disabled; closeCompletion(); completionContext = ""; @@ -315,6 +406,189 @@ EditorView.atomicRanges.of((currentView) => pasteDecorations(currentView.state)), ]; + const registerTypedAtom = StateEffect.define<{ + key: number; + atom: ComposerTypedSegment; + }>(); + const typedRegistry = StateField.define>({ + create: () => new Map(), + update(registry, transaction) { + const additions = transaction.effects.filter((effect) => effect.is(registerTypedAtom)); + if (!additions.length && !transaction.docChanged) return registry; + const next = new Map(registry); + for (const addition of additions) next.set(addition.value.key, addition.value.atom); + if (transaction.docChanged) { + const live = new Set(composerTypedAtoms(transaction.newDoc.toString(), next).map((atom) => atom.key)); + for (const atom of composerTypedAtoms(transaction.startState.doc.toString(), registry)) { + if (atom.uploadReservation && !live.has(atom.key)) { + // Cancellation is part of the editor transaction, not a later callback. + // Undo restores text only; it cannot regain this upload reservation. + next.set(atom.key, { segment: { kind: "text", content: "" }, label: "Removed upload" }); + } + } + } + return next; + }, + }); + + class TypedChipWidget extends WidgetType { + readonly atom: ComposerTypedSegment; + constructor(atom: ComposerTypedSegment) { + super(); + this.atom = atom; + } + override eq(other: TypedChipWidget): boolean { + return JSON.stringify(other.atom.segment) === JSON.stringify(this.atom.segment) && + other.atom.label === this.atom.label; + } + override toDOM(): HTMLElement { + const chip = document.createElement("span"); + chip.className = "composer-typed-chip"; + chip.textContent = this.atom.label; + chip.title = this.atom.label; + chip.setAttribute("role", "note"); + chip.setAttribute("aria-label", this.atom.label); + return chip; + } + override ignoreEvent(): boolean { return false; } + } + + function typedDecorations(state: EditorState): DecorationSet { + return Decoration.set( + composerTypedAtoms(state.doc.toString(), state.field(typedRegistry)).map((atom) => + Decoration.replace({ + widget: new TypedChipWidget(atom), + inclusive: false, + }).range(atom.from, atom.to) + ), + true, + ); + } + + const typedChips = [ + typedRegistry, + EditorView.decorations.of((currentView) => typedDecorations(currentView.state)), + EditorView.atomicRanges.of((currentView) => typedDecorations(currentView.state)), + ]; + function releasePendingUploads(): void { + const cancelled = [...pendingUploads]; + pendingUploads.clear(); + for (const key of cancelled) oncancelupload?.(key); + } + + // Completed uploads remain staged while Undo can still restore their atoms. + // Pending reservations are cancelled immediately, not retained as Undo leases. + // Release abandoned resources at a draft boundary; accepted live atoms belong to the host. + function releaseStagedAtoms(accepted = false): void { + const live = accepted && view ? new Set(composerTypedAtoms( + view.state.doc.toString(), view.state.field(typedRegistry), + ).map((atom) => JSON.stringify(atom.segment))) : new Set(); + for (const segment of stagedTypedAtoms.values()) { + if (!live.has(JSON.stringify(segment))) onremoveatom?.(segment); + } + stagedTypedAtoms.clear(); + } + let suppressInvocationChipify = false; + + function typedAtomLabel(segment: Segment): string { + if (segment.kind === "feature_invoke") return invocationInput(segment.invocation); + if (segment.kind === "uploaded_file") return `Attached ${segment.file.file_name}`; + if (segment.kind === "file_ref") return `@${segment.path}`; + if (segment.kind === "paste_artifact") return "Restored paste"; + if (segment.kind === "flow") return `Flow ${segment.selector}`; + return "Structured input"; + } + + function insertTypedSegment( + segment: Segment, + options: { from?: number; to?: number; cleanup?: boolean; label?: string; uploadReservation?: boolean } = {}, + ): number | null { + if (!view || view.state.readOnly) return null; + const selection = view.state.selection.main; + const from = options.from ?? selection.from; + const to = options.to ?? selection.to; + const key = nextTypedKey++; + if (options.cleanup) stagedTypedAtoms.set(key, segment); + if (options.uploadReservation) pendingUploads.add(key); + const token = composerTypedToken(key); + view.dispatch({ + changes: { from, to, insert: token }, + selection: EditorSelection.cursor(from + token.length), + effects: registerTypedAtom.of({ + key, + atom: { segment, label: options.label ?? typedAtomLabel(segment), cleanup: options.cleanup, uploadReservation: options.uploadReservation }, + }), + annotations: isolateHistory.of("full"), + userEvent: "input.complete", + }); + return key; + } + + function chipifySelectedInvocation(strict = false): void { + if (!view || view.composing || view.state.readOnly) return; + // Only completion-selected or explicitly edited calls become structured input. + // Descriptor discovery must never promote unrelated typed or pasted prose. + for (const selected of [...view.state.field(selectedInvocationState)].reverse()) { + try { + const range = selectedFeatureInvocationAt( + view.state.doc.toString(), selected.from, selected.descriptor, + ); + if (!range) { + if (strict) throw new Error("Finish or remove the selected Feature invocation before sending."); + continue; + } + const head = view.state.selection.main.head; + if (!strict && head > range.start && head < range.end) continue; + insertTypedSegment( + { kind: "feature_invoke", invocation: range.invocation }, + { from: range.start, to: range.end }, + ); + } catch (error) { + // Invalid calls stay editable until Submit reports their validation error. + if (strict) throw error; + } + } + } + + let editVersion = 0; + let editorGeneration = 0; + async function editInvocation(key: number): Promise { + if (!view || disabled) return; + const scope = completionScope; + const version = ++editVersion; + const document = view.state.doc.toString(); + const selection = view.state.selection.main; + const generation = editorGeneration; + let atom = composerTypedAtoms(view.state.doc.toString(), view.state.field(typedRegistry)) + .find((candidate) => candidate.key === key); + if (atom?.segment.kind !== "feature_invoke") return; + const invocation = atom.segment.invocation; + let descriptor = selectedFeatureInvocations.get(invocation.identity); + if (!descriptor && resolveFeatureCompletions) { + try { + const entries = await resolveFeatureCompletions(invocation.name, new AbortController().signal); + descriptor = entries.find((entry) => entry.invocation?.identity === invocation.identity)?.invocation ?? undefined; + } catch { return; } + } + if (!descriptor || !view || disabled || version !== editVersion || scope !== completionScope || + generation !== editorGeneration || document !== view.state.doc.toString() || + selection.from !== view.state.selection.main.from || selection.to !== view.state.selection.main.to) return; + atom = composerTypedAtoms(view.state.doc.toString(), view.state.field(typedRegistry)) + .find((candidate) => candidate.key === key); + if (atom?.segment.kind !== "feature_invoke") return; + selectedFeatureInvocations.set(descriptor.identity, descriptor); + const input = invocationInput(invocation); + suppressInvocationChipify = true; + view.dispatch({ + changes: { from: atom.from, to: atom.to, insert: input }, + selection: EditorSelection.cursor(atom.from + input.length - 1), + effects: selectInvocation.of({ from: atom.from, descriptor }), + annotations: isolateHistory.of("full"), + userEvent: "input", + }); + view.focus(); + } + function currentSnapshot(state = view?.state): ComposerDraftSnapshot { if (!state) { return { document: "", content: "", segments: [], pastes: [], textPastes: [] }; @@ -323,6 +597,7 @@ state.doc.toString(), state.field(pasteRegistry), state.field(textPasteState), + state.field(typedRegistry), ); } @@ -339,6 +614,7 @@ } function browseHistory(currentView: EditorView, direction: ComposerHistoryDirection): boolean { + if ([...currentView.state.field(typedRegistry).values()].some((atom) => atom.segment.kind === "unknown")) return false; const selection = currentView.state.selection.main; const cursorLine = currentView.state.doc.lineAt(selection.head).number; if (!shouldBrowseComposerHistory({ @@ -435,20 +711,26 @@ from: paste.from - selection.from, to: paste.to - selection.from, })); - return snapshotComposerDraft(document, selectedRegistry, selectedTextPastes).content; + return snapshotComposerDraft(document, selectedRegistry, selectedTextPastes, state.field(typedRegistry)).content; } - function deleteAdjacentPasteFromView( + function deleteAdjacentAtomFromView( currentView: EditorView, direction: "backward" | "forward", ): boolean { if (currentView.state.readOnly) return false; const selection = currentView.state.selection.main; - const pastes = composerPasteAtoms( - currentView.state.doc.toString(), - currentView.state.field(pasteRegistry), - ); - const deletion = composerDeletionRange(selection, pastes, direction); + const atoms = [ + ...composerPasteAtoms( + currentView.state.doc.toString(), + currentView.state.field(pasteRegistry), + ), + ...composerTypedAtoms( + currentView.state.doc.toString(), + currentView.state.field(typedRegistry), + ), + ]; + const deletion = composerDeletionRange(selection, atoms, direction); if (!deletion) return false; currentView.dispatch({ changes: deletion, @@ -467,7 +749,7 @@ view = new EditorView({ parent: mountElement, state: EditorState.create({ - extensions: [ + extensions: editorExtensions = [ history(), Prec.highest(EditorView.domEventHandlers({ keydown: completionKeydown })), Prec.highest(keymap.of([ @@ -494,16 +776,21 @@ { key: "Backspace", run: (currentView) => - deleteAdjacentPasteFromView(currentView, "backward"), + deleteAdjacentAtomFromView(currentView, "backward"), }, { key: "Delete", run: (currentView) => - deleteAdjacentPasteFromView(currentView, "forward"), + deleteAdjacentAtomFromView(currentView, "forward"), }, ])), keymap.of([...defaultKeymap, ...historyKeymap]), + selectedInvocationState, + invertedEffects.of((transaction) => transaction.docChanged + ? [restoreSelectedInvocations.of(transaction.startState.field(selectedInvocationState))] + : []), pasteChips, + typedChips, editable.of([ EditorView.editable.of(!disabled), EditorState.readOnly.of(disabled), @@ -518,7 +805,28 @@ spellcheck: "true", }), EditorView.updateListener.of((update) => { - if (update.docChanged || update.selectionSet) queueMicrotask(() => refreshCompletion()); + // StateField tombstones deleted reservations before notifying the + // adapter. The callback must not dispatch a nested editor update. + for (const key of pendingUploads) { + if (!update.state.field(typedRegistry).get(key)?.uploadReservation) { + pendingUploads.delete(key); + oncancelupload?.(key); + } + } + if (update.docChanged || update.selectionSet) { + editorGeneration++; + // Invalidate synchronously: an A→B→A edit/cursor cycle in one + // microtask must not revive an earlier completion generation. + closeCompletion(); + completionContext = ""; + queueMicrotask(() => refreshCompletion()); + } + if (update.docChanged) { + if (suppressInvocationChipify) suppressInvocationChipify = false; + else if (!update.transactions.some((transaction) => transaction.isUserEvent("undo") || transaction.isUserEvent("redo"))) { + queueMicrotask(chipifySelectedInvocation); + } + } if (update.docChanged && !restoringHistory) composerHistory.cancelNavigation(); if (update.docChanged || update.transactions.some((tx) => tx.effects.length > 0)) { emitChange(); @@ -528,6 +836,26 @@ focus() { queueMicrotask(() => refreshCompletion(true)); return false; }, blur() { closeCompletion(); return false; }, compositionend() { queueMicrotask(() => refreshCompletion(true)); return false; }, + mousedown(event, currentView) { + const target = event.target instanceof Element + ? event.target.closest(".composer-typed-chip") + : null; + if (!target || currentView.state.readOnly) return false; + let position: number; + try { + position = currentView.posAtDOM(target); + } catch { + return false; + } + const atom = composerTypedAtoms( + currentView.state.doc.toString(), + currentView.state.field(typedRegistry), + ).find((candidate) => candidate.from <= position && position <= candidate.to); + if (atom?.segment.kind !== "feature_invoke") return false; + event.preventDefault(); + void editInvocation(atom.key); + return true; + }, paste(event) { return handlePasteEvent(event); }, @@ -582,6 +910,9 @@ return () => { closeCompletion(); + releasePendingUploads(); + releaseStagedAtoms(); + editVersion++; view?.destroy(); view = null; }; @@ -598,12 +929,14 @@ }); export function snapshot(): ComposerDraftSnapshot { + chipifySelectedInvocation(true); return currentSnapshot(); } export function recordHistory(value: ComposerDraftSnapshot): void { const entry: ComposerHistoryEntry = { - segments: value.segments, + // Uploaded references are Worker-owned and must not leak through workspace-wide history. + segments: value.segments.filter((segment) => segment.kind !== "uploaded_file"), preserveExactText: value.textPastes.length > 0, }; if (composerHistory.record(entry)) { @@ -632,23 +965,65 @@ }); } - export function clear(): void { + export function insertSegment(segment: Segment, cleanup = false): boolean { + return insertTypedSegment(segment, { cleanup }) !== null; + } + + export function reserveUpload(fileName: string): number | null { + return insertTypedSegment({ kind: "unknown" }, { label: `Uploading ${fileName}`, uploadReservation: true }); + } + + export function completeUpload(key: number, segment: Segment): boolean { + if (!view || !pendingUploads.has(key) || + !view.state.field(typedRegistry).get(key)?.uploadReservation || + !composerTypedAtoms(view.state.doc.toString(), view.state.field(typedRegistry)).some((atom) => atom.key === key)) return false; + pendingUploads.delete(key); + stagedTypedAtoms.set(key, segment); + view.dispatch({ effects: registerTypedAtom.of({ + key, atom: { segment, label: typedAtomLabel(segment), cleanup: true }, + }) }); + return true; + } + + export function cancelUpload(key: number): void { if (!view) return; + const atom = composerTypedAtoms(view.state.doc.toString(), view.state.field(typedRegistry)) + .find((candidate) => candidate.key === key); view.dispatch({ - changes: { from: 0, to: view.state.doc.length, insert: "" }, - selection: EditorSelection.cursor(0), - userEvent: "input", + changes: atom ? { from: atom.from, to: atom.to, insert: "" } : [], + // Undo cannot resurrect a cancelled upload as a sendable placeholder. + effects: registerTypedAtom.of({ key, atom: { + segment: { kind: "text", content: "" }, label: "Removed upload", + } }), + userEvent: "delete", }); } + export function clear(acceptedTypedResources = false, retainStagedResources = false): void { + if (!view) return; + releasePendingUploads(); + if (!retainStagedResources) releaseStagedAtoms(acceptedTypedResources); + editVersion++; + closeCompletion(); + completionContext = ""; + view.setState(EditorState.create({ extensions: editorExtensions })); + view.dispatch({ effects: editable.reconfigure([ + EditorView.editable.of(!disabled), EditorState.readOnly.of(disabled), + ]) }); + emitChange(); + } + export function restoreSegments( segments: readonly Segment[], preserveExactText = false, + ownStagedUploads = false, ): void { if (!view) return; + clear(false, restoringHistory); let document = ""; const pasteEffects: StateEffect<{ key: number; paste: ComposerPaste }>[] = []; const textEffects: StateEffect[] = []; + const typedEffects: StateEffect<{ key: number; atom: ComposerTypedSegment }>[] = []; let highestPasteId = nextPasteId - 1; for (const segment of segments) { if (segment.kind === "text") { @@ -674,15 +1049,22 @@ highestPasteId = Math.max(highestPasteId, paste.id); document += composerPasteToken(key); pasteEffects.push(registerPaste.of({ key, paste })); - } else if (segment.kind === "file_ref") { - document += `@${segment.path}`; + } else { + const key = nextTypedKey++; + document += composerTypedToken(key); + const cleanup = ownStagedUploads && segment.kind === "uploaded_file"; + if (cleanup) stagedTypedAtoms.set(key, segment); + typedEffects.push(registerTypedAtom.of({ + key, + atom: { segment, label: typedAtomLabel(segment), cleanup }, + })); } } nextPasteId = highestPasteId + 1; view.dispatch({ changes: { from: 0, to: view.state.doc.length, insert: document }, selection: EditorSelection.cursor(document.length), - effects: [...pasteEffects, ...textEffects], + effects: [...pasteEffects, ...textEffects, ...typedEffects], userEvent: "input.restore", }); } @@ -788,7 +1170,8 @@ padding: 0; } - .composer-input :global(.composer-paste-chip) { + .composer-input :global(.composer-paste-chip), + .composer-input :global(.composer-typed-chip) { display: inline-flex; align-items: center; max-width: min(26rem, 70vw); diff --git a/web/workspace/src/lib/workspace/console/composer-command.ts b/web/workspace/src/lib/workspace/console/composer-command.ts index 10a219118..781e37faf 100644 --- a/web/workspace/src/lib/workspace/console/composer-command.ts +++ b/web/workspace/src/lib/workspace/console/composer-command.ts @@ -1,3 +1,4 @@ +import { invocationInput } from "./composer-invocation.ts"; import type { Segment } from "#lib/generated/protocol.ts"; export type WorkerConsoleInputKind = @@ -88,9 +89,10 @@ export function buildComposerSegmentsRequest( sourceSegments: readonly Segment[], options: ComposerSegmentsRequestOptions = {}, ): ComposerCommandResult { - const hasRichSegment = sourceSegments.some((segment) => - segment.kind === "paste" || segment.kind === "uploaded_file" - ); + if (sourceSegments.some((segment) => segment.kind === "unknown")) { + return { ok: false, message: "Finish or remove incomplete structured input before sending." }; + } + const hasRichSegment = sourceSegments.some((segment) => segment.kind !== "text"); if (!hasRichSegment) { const content = sourceSegments.map(segmentContent).join(""); if (!options.preserveExactText || content.trimStart().startsWith(":")) { @@ -154,8 +156,10 @@ function segmentContent(segment: Segment): string { return `@${segment.path}`; case "flow": return segment.selector; + case "feature_invoke": + return invocationInput(segment.invocation); case "paste_artifact": - return ""; + return "[Restored paste]"; case "uploaded_file": return `[Attached file: ${segment.file.file_name}]`; default: diff --git a/web/workspace/src/lib/workspace/console/composer-completion.test.ts b/web/workspace/src/lib/workspace/console/composer-completion.test.ts index b30738fef..2e7417745 100644 --- a/web/workspace/src/lib/workspace/console/composer-completion.test.ts +++ b/web/workspace/src/lib/workspace/console/composer-completion.test.ts @@ -23,7 +23,7 @@ function assertEquals(actual: T, expected: T): void { } } -Deno.test("completionTokenAt detects command and file sigils before the cursor", () => { +Deno.test("completionTokenAt detects command, file, and Feature sigils before the cursor", () => { assertEquals(completionTokenAt("open @src/ma", "open @src/ma".length), { sigil: "@", kind: "file", @@ -32,7 +32,13 @@ Deno.test("completionTokenAt detects command and file sigils before the cursor", prefix: "src/ma", }); assertEquals(completionTokenAt(":comp", 5)?.kind, "command"); - assertEquals(completionTokenAt("run /work", 9), null); + assertEquals(completionTokenAt("run /work", 9), { + sigil: "/", + kind: "feature", + start: 4, + end: 9, + prefix: "work", + }); assertEquals(completionTokenAt("ask #plain", "ask #plain".length), null); }); @@ -92,3 +98,59 @@ Deno.test("localCommandCompletions filters colon commands", () => { "compact", ]); }); + +Deno.test("argument completions replace whole quoted values and preserve following arguments", () => { + const descriptor = { identity: "test/run", name: "run", aliases: [], display_name: "Run", description: "Test", + syntax: "parenthesized" as const, arguments: [{ name: "path", position: 0, required: true, + value_type: { kind: "worker_file" as const }, completion: { kind: "worker_file" as const } }] }; + for (const value of ['/run("src/ma', '/run("src/ma", next=true)', '/run(path="src/ma", next=true)']) { + const cursor = value.indexOf("ma") + 1; + const token = completionTokenAt(value, cursor, [descriptor]); + assert(token, "argument token should exist"); + assertEquals(token.prefix, "src/m"); + assertEquals(applyCompletion(value, token, { value: '資料/a "b"/c' }).value, + value.startsWith('/run(path') ? '/run(path="資料/a \\"b\\"/c", next=true)' : + value.endsWith('true)') ? '/run("資料/a \\"b\\"/c", next=true)' : '/run("資料/a \\"b\\"/c"a'); + } +}); + +Deno.test("incomplete or invalid quoted completion preserves mode and body after the cursor", () => { + const descriptor = { identity: "test/run", name: "run", aliases: [], display_name: "Run", description: "Test", + syntax: "parenthesized" as const, arguments: [{ name: "path", position: 0, required: true, + value_type: { kind: "worker_file" as const }, completion: { kind: "worker_file" as const } }] }; + for (const value of ['/run(path="ab, mode=safe) following prose', '/run("ab, mode=safe) following prose', String.raw`/run(path="ab\q", mode=safe) following prose`]) { + const cursor = value.indexOf("ab") + 2; + const token = completionTokenAt(value, cursor, [descriptor], [{ from: 0, descriptor }]); + assert(token, "selected argument completion should exist"); + assertEquals(token.end, cursor); + const result = applyCompletion(value, token, { value: "completed/資料" }); + assertEquals(result.value.slice(result.cursor), value.slice(cursor)); + assert(result.value.includes("mode=safe) following prose"), "other arguments and body must remain intact"); + } +}); + +Deno.test("quoted call-looking data completes the selected outer value without erasing other arguments", () => { + const descriptor = { identity: "test/run", name: "run", aliases: [], display_name: "Run", description: "Test", + syntax: "parenthesized" as const, arguments: [{ name: "path", position: 0, required: true, + value_type: { kind: "worker_file" as const }, completion: { kind: "worker_file" as const } }] }; + const value = '/run("a /run(pa", mode=safe) prose'; + const token = completionTokenAt(value, value.lastIndexOf("pa") + 2, [descriptor], [{ from: 0, descriptor }]); + assert(token, "outer argument completion should exist"); + assertEquals(token.context?.argument, "path"); + assertEquals(token.prefix, "a /run(pa"); + assertEquals(applyCompletion(value, token, { value: "replacement" }).value, '/run("replacement", mode=safe) prose'); +}); + +Deno.test("argument name replacement preserves its value and directories drill inside quotes", () => { + const descriptor = { identity: "test/run", name: "run", aliases: [], display_name: "Run", description: "Test", + syntax: "parenthesized" as const, arguments: [{ name: "path", position: 0, required: true, + value_type: { kind: "worker_file" as const }, completion: { kind: "worker_file" as const } }] }; + const named = '/run(path="original")'; + const token = completionTokenAt(named, 7, [descriptor]); + assert(token, "named completion should exist"); + assertEquals(applyCompletion(named, token, { value: "path=" }).value, named); + const directory = '/run("sr")'; + const result = applyCompletion(directory, completionTokenAt(directory, 8, [descriptor])!, { value: "src", is_dir: true }); + assertEquals(result.value, '/run("src/")'); + assertEquals(completionTokenAt(result.value, result.cursor, [descriptor])?.prefix, "src/"); +}); diff --git a/web/workspace/src/lib/workspace/console/composer-completion.ts b/web/workspace/src/lib/workspace/console/composer-completion.ts index 4098f626f..7fc532b47 100644 --- a/web/workspace/src/lib/workspace/console/composer-completion.ts +++ b/web/workspace/src/lib/workspace/console/composer-completion.ts @@ -1,18 +1,25 @@ import { COMMANDS } from "./composer-command.ts"; +import type { + CompletionContext, + FeatureInvocationDescriptor, +} from "#lib/generated/protocol.ts"; +import { activeInvocationArgument, type SelectedInvocationOccurrence } from "./composer-invocation.ts"; -export type ComposerCompletionKind = "command" | "file"; +export type ComposerCompletionKind = "command" | "file" | "feature" | "feature_argument"; export type ComposerCompletionToken = { - sigil: ":" | "@"; + sigil: ":" | "@" | "/" | ""; kind: ComposerCompletionKind; start: number; end: number; prefix: string; + context?: CompletionContext; }; export type ComposerCompletionEntry = { value: string; is_dir?: boolean; - description?: string; - usage?: string; + description?: string | null; + usage?: string | null; + invocation?: FeatureInvocationDescriptor | null; }; export type CompletionApplyResult = { value: string; cursor: number }; const ALIASES: Record = { "?": "help", rollback: "rewind" }; @@ -28,9 +35,40 @@ export const COLON_COMMAND_COMPLETIONS: ComposerCompletionEntry[] = Object export function completionTokenAt( value: string, cursor: number, + descriptors: readonly FeatureInvocationDescriptor[] = [], + selectedOccurrences?: readonly SelectedInvocationOccurrence[], ): ComposerCompletionToken | null { const boundedCursor = Math.max(0, Math.min(cursor, value.length)); + const activeArgument = activeInvocationArgument(value, boundedCursor, descriptors, selectedOccurrences); + if (activeArgument) { + return { + sigil: "", + kind: "feature_argument", + start: activeArgument.start, + end: activeArgument.end, + prefix: activeArgument.prefix, + context: { + invocation: activeArgument.descriptor.identity, + argument: activeArgument.argument?.name, + }, + }; + } + const before = value.slice(0, boundedCursor); + const feature = /(^|\s)\/([a-z0-9_-]*)$/.exec(before); + if (feature) { + const prefix = feature[2] ?? ""; + const start = before.length - prefix.length - 1; + const tail = /^[a-z0-9_-]*/.exec(value.slice(boundedCursor))?.[0] ?? ""; + return { + sigil: "/", + kind: "feature", + start, + end: boundedCursor + tail.length, + prefix, + }; + } + const match = /(^|\s)([:@])([^\s\uFFF9\uFFFB]*)$/.exec(before); if (!match) return null; const sigil = match[2] as ":" | "@"; @@ -78,19 +116,46 @@ export function applyCompletion( entry: ComposerCompletionEntry, action: "tab" | "accept" = "accept", ): CompletionApplyResult { + if (token.kind === "feature") { + const replacement = `/${entry.value}(`; + return replaceCompletionRange(value, token, replacement); + } + if (token.kind === "feature_argument") { + const argumentName = !token.context?.argument && entry.value.endsWith("="); + const completed = entry.is_dir && !entry.value.endsWith("/") ? `${entry.value}/` : entry.value; + const replacement = argumentName ? entry.value : JSON.stringify(completed); + const result = replaceCompletionRange(value, + argumentName && value[token.end] === "=" ? { ...token, end: token.end + 1 } : token, + replacement, + ); + // Keep directory drilling inside the quoted value instead of after its closing quote. + if (entry.is_dir && !argumentName) result.cursor--; + return result; + } const suffix = entry.is_dir ? (entry.value.endsWith("/") ? "" : "/") : token.kind === "file" && action === "tab" ? "" : " "; - const replacement = `${token.sigil}${entry.value}${suffix}`; - const restStart = suffix === " " && value[token.end] === " " + return replaceCompletionRange( + value, + token, + `${token.sigil}${entry.value}${suffix}`, + suffix === " ", + ); +} + +function replaceCompletionRange( + value: string, + token: ComposerCompletionToken, + replacement: string, + consumeSpace = false, +): CompletionApplyResult { + const restStart = consumeSpace && value[token.end] === " " ? token.end + 1 : token.end; return { - value: `${value.slice(0, token.start)}${replacement}${ - value.slice(restStart) - }`, + value: `${value.slice(0, token.start)}${replacement}${value.slice(restStart)}`, cursor: token.start + replacement.length, }; } diff --git a/web/workspace/src/lib/workspace/console/composer-delivery.test.ts b/web/workspace/src/lib/workspace/console/composer-delivery.test.ts index ebf3bfa32..25faa2a16 100644 --- a/web/workspace/src/lib/workspace/console/composer-delivery.test.ts +++ b/web/workspace/src/lib/workspace/console/composer-delivery.test.ts @@ -2,8 +2,12 @@ declare const Deno: { test(name: string, fn: () => void): void; }; +import { deepStrictEqual, throws } from "node:assert/strict"; +import type { Segment } from "#lib/generated/protocol.ts"; +import type { ComposerDraftSnapshot } from "./composer-draft.ts"; import { canDeliverComposerDraft, + ComposerAdmissions, sendComposerDelivery, } from "./composer-delivery.ts"; @@ -13,6 +17,187 @@ function assertEquals(actual: unknown, expected: unknown): void { } } +const upload: Segment = { + kind: "uploaded_file", + file: { + artifact_id: "staged-upload", + file_name: "資料.md", + media_type: "text/markdown", + created_at_ms: 1, + availability: "available", + byte_len: 4, + sha256: "abcd", + }, +}; +const typedSnapshot: ComposerDraftSnapshot = { + document: "exact typed draft", + content: 'before /run(path="a b/資料") after', + pastes: [], + textPastes: [], + segments: [ + { kind: "text", content: "before " }, + { + kind: "feature_invoke", + invocation: { + invocation_id: "stable-invocation", + identity: "feature:test/run", + name: "run", + arguments: [{ + name: "path", + value: { kind: "string", value: "a b/資料" }, + }], + }, + }, + upload, + { kind: "text", content: " after" }, + ], +}; + +Deno.test("Submit admission keeps exact typed draft and stable IDs across rejection and retry", () => { + const admissions = new ComposerAdmissions(); + const method = { + method: "submit" as const, + params: { + submission_request_id: "stable-request", + input: typedSnapshot.segments, + }, + }; + const first = admissions.begin("target-a", method, typedSnapshot, "submit"); + deepStrictEqual(first.snapshot, typedSnapshot); + assertEquals(admissions.protects(upload), true); + assertEquals( + admissions.acknowledge("target-b", { + event: "submission_accepted", + data: { + submission_request_id: "stable-request", + submission_id: "s", + disposition: "started", + }, + }), + null, + ); + assertEquals( + admissions.acknowledge("target-a", { + event: "submission_rejected", + data: { + submission_request_id: "other-client", + message: "unrelated", + }, + }), + null, + ); + const rejection = admissions.acknowledge("target-a", { + event: "submission_rejected", + data: { + submission_request_id: "stable-request", + message: "validation failed", + }, + }); + assertEquals(rejection?.accepted, false); + deepStrictEqual(rejection?.record.snapshot, typedSnapshot); + assertEquals(admissions.protects(upload), false); + const retry = admissions.begin( + "target-a", + { + ...method, + params: { ...method.params, submission_request_id: "do-not-use-new-id" }, + }, + typedSnapshot, + "submit", + ); + deepStrictEqual(retry.method, first.method); + const accepted = admissions.acknowledge("target-a", { + event: "submission_accepted", + data: { + submission_request_id: "stable-request", + submission_id: "s", + disposition: "queued", + }, + }); + assertEquals(accepted?.accepted, true); + deepStrictEqual(accepted?.record.snapshot.segments, typedSnapshot.segments); + assertEquals(admissions.get("target-a"), null); +}); + +Deno.test("unknown admission protects uploads and in-flight retries from draft deletion and target ABA", () => { + const admissions = new ComposerAdmissions(); + const method = { + method: "submit" as const, + params: { + submission_request_id: "unknown-request", + input: typedSnapshot.segments, + }, + }; + admissions.begin("target-a", method, typedSnapshot, "queue"); + admissions.disconnected("target-a"); + assertEquals(admissions.get("target-a")?.status, "unknown"); + assertEquals(admissions.protects(upload), true); + throws( + () => + admissions.begin( + "target-a", + method, + { ...typedSnapshot, segments: [] }, + "queue", + ), + /in-flight/, + ); + admissions.disconnected("target-b"); + deepStrictEqual(admissions.retry("target-a")?.method, method); + deepStrictEqual(admissions.get("target-a")?.snapshot, typedSnapshot); + assertEquals(admissions.protects(upload), true); +}); + +Deno.test("Notify admission correlates authoritative rejection/success and retries unchanged IDs", () => { + const admissions = new ComposerAdmissions(); + const snapshot = { + ...typedSnapshot, + segments: [{ kind: "text" as const, content: "通知" }], + }; + const method = { + method: "notify" as const, + params: { notification_request_id: "notification-1", message: "通知" }, + }; + admissions.begin("target", method, snapshot, "notify"); + assertEquals( + admissions.acknowledge("target", { + event: "notification_rejected", + data: { notification_request_id: "other", message: "old" }, + }), + null, + ); + assertEquals( + admissions.acknowledge("target", { + event: "notification_rejected", + data: { + notification_request_id: "notification-1", + message: "not running", + }, + })?.accepted, + false, + ); + deepStrictEqual( + admissions.begin( + "target", + { + ...method, + params: { ...method.params, notification_request_id: "new" }, + }, + snapshot, + "notify", + ).method, + method, + ); + assertEquals( + admissions.acknowledge("target", { + event: "notification_accepted", + data: { notification_request_id: "notification-1" }, + })?.accepted, + true, + ); + assertEquals(admissions.get("target"), null); +}); + const base = { protocolOpen: true, sending: false, diff --git a/web/workspace/src/lib/workspace/console/composer-delivery.ts b/web/workspace/src/lib/workspace/console/composer-delivery.ts index f00f088e7..d79cf0c82 100644 --- a/web/workspace/src/lib/workspace/console/composer-delivery.ts +++ b/web/workspace/src/lib/workspace/console/composer-delivery.ts @@ -1,3 +1,123 @@ +import type { Event, Method, Segment } from "#lib/generated/protocol.ts"; +import type { ComposerDraftSnapshot } from "./composer-draft.ts"; + +export type ComposerAdmissionMethod = Extract< + Method, + { method: "submit" | "notify" } +>; +export type ComposerAdmission = { + method: ComposerAdmissionMethod; + snapshot: ComposerDraftSnapshot; + delivery: ComposerDelivery; + status: "waiting" | "unknown" | "rejected"; +}; + +function admissionId(method: ComposerAdmissionMethod): string { + return method.method === "submit" + ? method.params.submission_request_id + : method.params.notification_request_id; +} + +/** Transport send is not admission. Keep exact typed input and idempotency IDs + * until an authoritative acknowledgement; uncertain sends retain upload leases. + */ +export class ComposerAdmissions { + private records = new Map(); + + get(target: string): ComposerAdmission | null { + return this.records.get(target) ?? null; + } + + begin( + target: string, + method: ComposerAdmissionMethod, + snapshot: ComposerDraftSnapshot, + delivery: ComposerDelivery, + ): ComposerAdmission { + const prior = this.get(target); + if (prior && prior.status !== "rejected") { + throw new Error( + "Resolve the in-flight admission before editing or sending a new draft.", + ); + } + const unchanged = prior && prior.method.method === method.method && + JSON.stringify(prior.snapshot.segments) === + JSON.stringify(snapshot.segments) && + (prior.snapshot.textPastes.length > 0) === + (snapshot.textPastes.length > 0); + const record: ComposerAdmission = unchanged + ? { ...prior, status: "waiting" } + : { + // Route-cached drafts may contain Svelte proxies. Snapshot the actual + // JSON wire values rather than trying to structured-clone those proxies. + method: JSON.parse(JSON.stringify(method)) as ComposerAdmissionMethod, + snapshot: JSON.parse(JSON.stringify(snapshot)) as ComposerDraftSnapshot, + delivery, + status: "waiting", + }; + this.records.set(target, record); + return record; + } + + retry(target: string): ComposerAdmission | null { + const prior = this.get(target); + if (!prior || prior.status === "rejected") return null; + const record: ComposerAdmission = { ...prior, status: "waiting" }; + this.records.set(target, record); + return record; + } + + disconnected(target: string): void { + const prior = this.get(target); + if (prior && prior.status !== "rejected") { + this.records.set(target, { ...prior, status: "unknown" }); + } + } + + protects(segment: Segment): boolean { + if (segment.kind !== "uploaded_file") return false; + return [...this.records.values()].some((record) => + record.status !== "rejected" && + record.snapshot.segments.some((candidate) => + candidate.kind === "uploaded_file" && + candidate.file.artifact_id === segment.file.artifact_id + ) + ); + } + + acknowledge( + target: string, + event: Event, + ): { record: ComposerAdmission; accepted: boolean; message?: string } | null { + const record = this.get(target); + if (!record) return null; + let id: string; + let accepted: boolean; + let message: string | undefined; + if ( + (event.event === "submission_accepted" || + event.event === "submission_rejected") && + record.method.method === "submit" + ) { + id = event.data.submission_request_id; + accepted = event.event === "submission_accepted"; + if (event.event === "submission_rejected") message = event.data.message; + } else if ( + (event.event === "notification_accepted" || + event.event === "notification_rejected") && + record.method.method === "notify" + ) { + id = event.data.notification_request_id; + accepted = event.event === "notification_accepted"; + if (event.event === "notification_rejected") message = event.data.message; + } else return null; + if (id !== admissionId(record.method)) return null; + if (accepted) this.records.delete(target); + else this.records.set(target, { ...record, status: "rejected" }); + return { record, accepted, message }; + } +} + export type ComposerDelivery = "submit" | "queue" | "notify"; export type ComposerDeliveryState = { @@ -22,7 +142,8 @@ export function canDeliverComposerDraft(state: ComposerDeliveryState): boolean { case "submit": return state.workerState === "idle" && hasInput; case "queue": - return (state.workerState === "running" || state.workerState === "paused") && + return (state.workerState === "running" || + state.workerState === "paused") && hasInput; case "notify": return state.workerState === "running" && state.hasText && diff --git a/web/workspace/src/lib/workspace/console/composer-draft.test.ts b/web/workspace/src/lib/workspace/console/composer-draft.test.ts index 49aefe764..356233a6b 100644 --- a/web/workspace/src/lib/workspace/console/composer-draft.test.ts +++ b/web/workspace/src/lib/workspace/console/composer-draft.test.ts @@ -3,6 +3,8 @@ import { composerDeletionRange, type ComposerPaste, composerPasteToken, + composerTypedAtoms, + composerTypedToken, pasteChipLabel, snapshotComposerDraft, } from "#lib/workspace/console/composer-draft.ts"; @@ -223,6 +225,50 @@ Deno.test("Paste content beginning with a colon remains opaque user input", () = assertEquals(afterWhitespace.request.content, " :not-a-command\r\n"); }); +Deno.test("typed invocation and attachment atoms round-trip in source order and delete atomically", () => { + const invocation: Segment = { + kind: "feature_invoke", + invocation: { + invocation_id: "invoke-1", + identity: "builtin:test/run", + name: "run", + arguments: [{ name: "value", value: { kind: "string", value: "資料 x" } }], + }, + }; + const attachment: Segment = { + kind: "uploaded_file", + file: { + artifact_id: "artifact-1", + file_name: "report.md", + media_type: "text/markdown", + created_at_ms: 1, + availability: "available", + byte_len: 4, + sha256: "abcd", + }, + }; + const first = composerTypedToken(10); + const second = composerTypedToken(11); + const document = `before ${first} between ${second} after`; + const registry = new Map([ + [10, { segment: invocation, label: '/run(value="資料 x")' }], + [11, { segment: attachment, label: "Attached report.md", cleanup: true }], + ]); + const snapshot = snapshotComposerDraft(document, new Map(), [], registry); + assertEquals(snapshot.segments.map((segment) => segment.kind), [ + "text", + "feature_invoke", + "text", + "uploaded_file", + "text", + ]); + const atoms = composerTypedAtoms(document, registry); + assertEquals( + composerDeletionRange({ from: atoms[1].to, to: atoms[1].to, head: atoms[1].to }, atoms, "backward"), + { from: atoms[1].from, to: atoms[1].to }, + ); +}); + Deno.test("plain short-paste Text retains the existing composer request path", () => { const result = buildComposerSegmentsRequest([ { kind: "text", content: " short\r\npaste\r\n " }, diff --git a/web/workspace/src/lib/workspace/console/composer-draft.ts b/web/workspace/src/lib/workspace/console/composer-draft.ts index 7f186ba04..a1ca24b29 100644 --- a/web/workspace/src/lib/workspace/console/composer-draft.ts +++ b/web/workspace/src/lib/workspace/console/composer-draft.ts @@ -3,6 +3,9 @@ import type { Segment } from "#lib/generated/protocol.ts"; const PASTE_TOKEN_PREFIX = "\uFFF9"; const PASTE_TOKEN_SUFFIX = "\uFFFB"; const PASTE_TOKEN_PATTERN = /\uFFF9(\d+)\uFFFB/g; +const TYPED_TOKEN_PREFIX = "\uFFF8"; +const TYPED_TOKEN_SUFFIX = "\uFFFA"; +const TYPED_TOKEN_PATTERN = /\uFFF8(\d+)\uFFFA/g; export interface ComposerPaste { id: number; @@ -24,6 +27,21 @@ export interface ComposerTextPaste { content: string; } +export interface ComposerTypedSegment { + segment: Segment; + label: string; + /** Whether deleting this unsent atom must release client-staged state. */ + cleanup?: boolean; + /** Client-only reservation whose removal cancels an in-progress upload. */ + uploadReservation?: boolean; +} + +export interface ComposerTypedAtom extends ComposerTypedSegment { + key: number; + from: number; + to: number; +} + export interface ComposerSelection { from: number; to: number; @@ -32,7 +50,7 @@ export interface ComposerSelection { export function composerDeletionRange( selection: ComposerSelection, - pastes: readonly ComposerPasteAtom[], + pastes: readonly { from: number; to: number }[], direction: "backward" | "forward", ): { from: number; to: number } | null { if (selection.from !== selection.to) { @@ -75,6 +93,29 @@ export function composerPasteAtoms( return atoms; } +export function composerTypedToken(key: number): string { + return `${TYPED_TOKEN_PREFIX}${key}${TYPED_TOKEN_SUFFIX}`; +} + +export function composerTypedAtoms( + document: string, + registry: ReadonlyMap, +): ComposerTypedAtom[] { + const atoms: ComposerTypedAtom[] = []; + for (const match of document.matchAll(TYPED_TOKEN_PATTERN)) { + const key = Number(match[1]); + const atom = registry.get(key); + if (!atom || match.index === undefined) continue; + atoms.push({ + ...atom, + key, + from: match.index, + to: match.index + match[0].length, + }); + } + return atoms; +} + function appendTextSegment(segments: Segment[], content: string): void { if (content.length === 0) return; const previous = segments.at(-1); @@ -89,8 +130,10 @@ export function snapshotComposerDraft( document: string, registry: ReadonlyMap, candidateTextPastes: readonly ComposerTextPaste[] = [], + typedRegistry: ReadonlyMap = new Map(), ): ComposerDraftSnapshot { const pastes = composerPasteAtoms(document, registry); + const typedAtoms = composerTypedAtoms(document, typedRegistry); const textPastes = candidateTextPastes .filter((paste) => paste.from >= 0 && @@ -105,6 +148,12 @@ export function snapshotComposerDraft( to: paste.to, paste, })), + ...typedAtoms.map((atom) => ({ + kind: "typed" as const, + from: atom.from, + to: atom.to, + atom, + })), ...textPastes.map((paste) => ({ kind: "text_paste" as const, from: paste.from, @@ -130,10 +179,14 @@ export function snapshotComposerDraft( chars: event.paste.chars, lines: event.paste.lines, }); + content += event.paste.content; + } else if (event.kind === "typed") { + segments.push(event.atom.segment); + content += event.atom.label; } else { appendTextSegment(segments, event.paste.content); + content += event.paste.content; } - content += event.paste.content; cursor = event.to; } diff --git a/web/workspace/src/lib/workspace/console/composer-history.ts b/web/workspace/src/lib/workspace/console/composer-history.ts index 6fa2770fc..7a258cb03 100644 --- a/web/workspace/src/lib/workspace/console/composer-history.ts +++ b/web/workspace/src/lib/workspace/console/composer-history.ts @@ -29,7 +29,7 @@ type ComposerHistoryStorage = Pick; function cloneEntry(entry: ComposerHistoryEntry): ComposerHistoryEntry { return { - segments: entry.segments.map((segment) => ({ ...segment })) as Segment[], + segments: JSON.parse(JSON.stringify(entry.segments)) as Segment[], preserveExactText: entry.preserveExactText, }; } @@ -45,6 +45,18 @@ function isSegment(value: unknown): value is Segment { typeof segment.lines === "number"; } if (segment.kind === "file_ref") return typeof segment.path === "string"; + if (segment.kind === "feature_invoke") { + const invocation = segment.invocation as Record | undefined; + return Boolean(invocation && typeof invocation.invocation_id === "string" && + typeof invocation.identity === "string" && typeof invocation.name === "string" && + Array.isArray(invocation.arguments) && invocation.arguments.every((argument) => { + if (!argument || typeof argument !== "object" || typeof argument.name !== "string") return false; + const value = argument.value; + return value && ((value.kind === "string" && typeof value.value === "string") || + (value.kind === "integer" && Number.isSafeInteger(value.value) && value.value >= -2147483648 && value.value <= 2147483647) || + (value.kind === "boolean" && typeof value.value === "boolean")); + })); + } return false; } diff --git a/web/workspace/src/lib/workspace/console/composer-invocation.test.ts b/web/workspace/src/lib/workspace/console/composer-invocation.test.ts new file mode 100644 index 000000000..edbd4ac54 --- /dev/null +++ b/web/workspace/src/lib/workspace/console/composer-invocation.test.ts @@ -0,0 +1,260 @@ +import { throws } from "node:assert/strict"; +import type { FeatureInvocationDescriptor } from "#lib/generated/protocol.ts"; +import { + activeInvocationArgument, + finalizeSelectedFeatureInvocations, + invocationInput, + selectedFeatureInvocationRanges, +} from "./composer-invocation.ts"; + +declare const Deno: { test(name: string, fn: () => void): void }; + +function assert(condition: unknown, message: string): asserts condition { + if (!condition) throw new Error(message); +} +function assertEquals(actual: unknown, expected: unknown): void { + const left = JSON.stringify(actual); + const right = JSON.stringify(expected); + if (left !== right) throw new Error(`Expected ${right}, got ${left}`); +} + +const descriptor: FeatureInvocationDescriptor = { + identity: "builtin:test/run", + name: "run", + aliases: ["execute"], + display_name: "Run", + description: "Run a declared operation", + syntax: "parenthesized", + arguments: [ + { + name: "path", + position: 0, + required: true, + value_type: { kind: "worker_file" }, + completion: { kind: "worker_file" }, + }, + { + name: "mode", + required: false, + value_type: { kind: "enum", values: ["safe", "fast"] }, + completion: { kind: "static", values: ["safe", "fast"] }, + }, + ], +}; + +Deno.test("selected invocation parsing preserves Unicode, spaces, slashes, prose, and multiple calls", () => { + const text = '前 /run("資料/a b/c", mode=safe) 後 /execute("x/y") done'; + const ranges = selectedFeatureInvocationRanges(text, [descriptor]); + assertEquals(ranges.length, 2); + assertEquals(text.slice(ranges[0].end), ' 後 /execute("x/y") done'); + assertEquals(ranges[0].invocation.arguments.map((argument) => argument.value), [ + { kind: "string", value: "資料/a b/c" }, + { kind: "string", value: "safe" }, + ]); + + const segments = finalizeSelectedFeatureInvocations( + [{ kind: "text", content: text }], + [descriptor], + ); + assertEquals(segments.map((segment) => segment.kind), [ + "text", + "feature_invoke", + "text", + "feature_invoke", + "text", + ]); +}); + +Deno.test("multiple positional arguments advance once per value independent of descriptor order", () => { + const positional: FeatureInvocationDescriptor = { + ...descriptor, + arguments: [ + { name: "enabled", position: 2, required: true, value_type: { kind: "boolean" }, completion: { kind: "none" } }, + { name: "mode", position: 0, required: true, value_type: { kind: "enum", values: ["brief", "full"] }, completion: { kind: "none" } }, + { name: "limit", position: 1, required: true, value_type: { kind: "integer" }, completion: { kind: "none" } }, + { name: "note", required: false, value_type: { kind: "string" }, completion: { kind: "none" } }, + ], + }; + for (const input of ['/run("brief", 2, true) following prose', '/run("brief", note="資料/a b", 2, true) following prose']) { + const parsed = selectedFeatureInvocationRanges(input, [positional])[0]; + assertEquals(input.slice(parsed.end), " following prose"); + assertEquals(parsed.invocation.arguments.filter((argument) => argument.name !== "note"), [ + { name: "mode", value: { kind: "string", value: "brief" } }, + { name: "limit", value: { kind: "integer", value: 2 } }, + { name: "enabled", value: { kind: "boolean", value: true } }, + ]); + } +}); + +Deno.test("unselected text, URLs, paths, and paste content are never promoted", () => { + assertEquals( + finalizeSelectedFeatureInvocations( + [ + { kind: "text", content: '/run("x") https://host/run("y") /tmp/run("z")' }, + { kind: "paste", id: 1, chars: 9, lines: 1, content: '/run("p")' }, + ], + [], + ).map((segment) => segment.kind), + ["text", "paste"], + ); +}); + +Deno.test("argument completion ignores quoted invocation-looking data and escaped quotes", () => { + for (const [input, prefix] of [ + ['/run("a /run(pa", mode=safe) prose', 'a /run(pa'], + [String.raw`/run(path="a \" /execute(pa", mode=safe) prose`, 'a " /execute(pa'], + [String.raw`/run(path="a \" /run(pa, mode=safe) prose`, 'a " /run(pa'], + ]) { + const dataCursor = input.lastIndexOf("(pa") + 3; + for (const occurrences of [undefined, [{ from: 0, descriptor }]]) { + const active = activeInvocationArgument(input, dataCursor, [descriptor], occurrences); + assertEquals(active?.descriptor.identity, descriptor.identity); + assertEquals(active?.argument?.name, "path"); + assertEquals(active?.start, input.indexOf('"')); + assertEquals(active?.prefix, prefix); + } + } + const input = '/run("a /run(pa", mode=safe) prose /execute("new")'; + const active = activeInvocationArgument(input, input.indexOf("new") + 2, [descriptor]); + assertEquals(active?.prefix, "ne"); + assertEquals(active?.start, input.lastIndexOf('"new')); + assertEquals(activeInvocationArgument(input, input.indexOf("prose") + 3, [descriptor]), null); + const unknown = '/unknown("a /run(pa", mode=safe) prose'; + assertEquals(activeInvocationArgument(unknown, unknown.lastIndexOf("pa") + 2, [descriptor]), null); + const quotedProse = 'quoted "a /run(pa" prose'; + assertEquals(activeInvocationArgument(quotedProse, quotedProse.lastIndexOf("pa") + 2, [descriptor]), null); +}); + +Deno.test("editor occurrence anchors exclude unselected matching call contexts", () => { + const input = '/run("literal /run(unfinished data before /execute("selected")'; + const from = input.indexOf("/execute("); + const active = activeInvocationArgument(input, input.indexOf("selected") + 3, [descriptor], [{ from, descriptor }]); + assertEquals(active?.prefix, "sel"); + assertEquals(active?.start, input.indexOf('"selected')); + assertEquals(activeInvocationArgument(input, input.indexOf("literal") + 2, [descriptor], []), null); +}); + +Deno.test("incomplete quoted values bound completion replacement to the cursor", () => { + for (const input of ['/run(path="ab, mode=safe) following prose', '/run("ab, mode=safe) following prose', String.raw`/run(path="ab\q", mode=safe) following prose`]) { + const cursor = input.indexOf("ab") + 2; + const active = activeInvocationArgument(input, cursor, [descriptor]); + assertEquals(active?.end, cursor); + assertEquals(active?.prefix, "ab"); + } +}); + +Deno.test("Feature strings reject lone surrogate escapes and code units like Rust JSON", () => { + for (const literal of [String.raw`"\uD800"`, String.raw`"\uDC00"`, String.raw`"\uD800x"`, String.raw`"\uD800\uD800"`, String.raw`"\uDC00\uD800"`, JSON.stringify("\uD800"), JSON.stringify("\uDC00")]) { + throws(() => selectedFeatureInvocationRanges(`/run(${literal})`, [descriptor]), /unpaired Unicode surrogates/); + const input = `/run(${literal})`; + assertEquals(activeInvocationArgument(input, input.length - 1, [descriptor]), null); + } + throws(() => selectedFeatureInvocationRanges('/run("\uD800")', [descriptor]), /unpaired Unicode surrogates/); + throws(() => selectedFeatureInvocationRanges('/run(\uDC00)', [descriptor]), /unpaired Unicode surrogates/); + for (const literal of [String.raw`"\uD83D\uDE00"`, '"😀"', String.raw`"\uDBFF\uDFFF"`, String.raw`"\u0000\b\f\n\r\t\"\\\/資料"`]) { + const parsed = selectedFeatureInvocationRanges(`/run(${literal})`, [descriptor])[0].invocation; + assertEquals(parsed.arguments[0].value, { kind: "string", value: JSON.parse(literal) }); + } + for (const literal of [String.raw`"\q"`, String.raw`"\u12xz"`, '"line\nfeed"']) { + throws(() => selectedFeatureInvocationRanges(`/run(${literal})`, [descriptor]), SyntaxError); + } +}); + +Deno.test("argument completion is cursor-aware and typed invocation round-trips to input", () => { + const named = activeInvocationArgument('/run("x", mo', 12, [descriptor]); + assert(named, "named argument completion should be active"); + assertEquals(named.argument, null); + assertEquals(named.prefix, "mo"); + + const value = activeInvocationArgument('/run("x", mode=sa', 18, [descriptor]); + assert(value?.argument, "enum value completion should resolve its descriptor"); + assertEquals(value.argument.name, "mode"); + assertEquals(value.prefix, "sa"); + + const invocation = selectedFeatureInvocationRanges('/run("a b", mode=fast)', [descriptor])[0] + .invocation; + assertEquals(invocationInput(invocation), '/run(path="a b", mode="fast")'); +}); + +Deno.test("complete invalid calls reject unknown, duplicate, missing, and mistyped arguments", () => { + for (const input of [ + '/run(other="x")', + '/run("x", path="y")', + "/run()", + '/run("x", mode=invalid)', + ]) { + let rejected = false; + try { + selectedFeatureInvocationRanges(input, [descriptor]); + } catch { + rejected = true; + } + assert(rejected, `${input} should be rejected`); + } +}); + +Deno.test("a leading slash without a selected descriptor terminates without promotion", () => { + assertEquals(activeInvocationArgument("/ru", 3, []), null); + assertEquals(activeInvocationArgument("/unknown(", 9, [descriptor]), null); +}); + +Deno.test("integer invocation arguments reject unsafe JavaScript wire values", () => { + const integerDescriptor: FeatureInvocationDescriptor = { ...descriptor, + arguments: [{ name: "count", position: 0, required: true, value_type: { kind: "integer" }, completion: { kind: "none" } }] }; + let rejected = false; + try { selectedFeatureInvocationRanges("/run(9007199254740992)", [integerDescriptor]); } + catch { rejected = true; } + assert(rejected, "unsafe integers must reject before sending to Rust"); +}); + +Deno.test("structured invocation-only Submit and mixed source order retain exact typed payloads", async () => { + const { buildComposerSegmentsRequest } = await import("./composer-command.ts"); + const invocation = finalizeSelectedFeatureInvocations([{ kind: "text", content: '/run("a b/資料")' }], [descriptor])[0]; + const segments = [{ kind: "text" as const, content: "before " }, invocation, { kind: "text" as const, content: " after" }]; + const result = buildComposerSegmentsRequest(segments); + assert(result.ok && result.request, "mixed invocation should build a Submit"); + assertEquals(result.request.segments, segments); + const only = buildComposerSegmentsRequest([invocation]); + assert(only.ok && only.request?.segments?.[0].kind === "feature_invoke", "invocation-only Submit must not become empty text"); +}); + +Deno.test("positional completion after a named-only argument uses positional rather than comma index", () => { + const input = '/run(mode=safe, "'; + assertEquals(activeInvocationArgument(input, input.length, [descriptor])?.argument?.name, "path"); +}); + +Deno.test("Feature string parsing decodes full JSON escapes and preserves round-trip Unicode", () => { + const raw = String.raw`/run("a\b\f\n\r\t\"\\\/\u0000\uD83D\uDE00")`; + const parsed = selectedFeatureInvocationRanges(raw, [descriptor])[0].invocation; + assertEquals(parsed.arguments[0].value, { kind: "string", value: 'a\b\f\n\r\t"\\/\u0000😀' }); + const restored = selectedFeatureInvocationRanges(invocationInput(parsed), [descriptor])[0].invocation; + assertEquals(restored.arguments, parsed.arguments); +}); + +Deno.test("Feature integer lexical syntax matches negative-or-positive digits only", () => { + const integerDescriptor: FeatureInvocationDescriptor = { ...descriptor, + arguments: [{ name: "count", position: 0, required: true, value_type: { kind: "integer" }, completion: { kind: "none" } }] }; + for (const [raw, value] of [["-0", 0], ["0012", 12], ["-0012", -12]] as const) { + assertEquals(selectedFeatureInvocationRanges(`/run(${raw})`, [integerDescriptor])[0].invocation.arguments[0].value, + { kind: "integer", value }); + } + for (const raw of ["+1", "1.0", "1e0", "NaN", "Infinity"]) { + let rejected = false; + try { selectedFeatureInvocationRanges(`/run(${raw})`, [integerDescriptor]); } catch { rejected = true; } + assert(rejected, `${raw} must reject`); + } +}); + +Deno.test("invocation integer parsing matches the generated Rust int32 schema", () => { + const integerDescriptor: FeatureInvocationDescriptor = { ...descriptor, + arguments: [{ name: "count", position: 0, required: true, value_type: { kind: "integer" }, completion: { kind: "none" } }] }; + for (const value of [-2147483648, 2147483647]) { + assertEquals(selectedFeatureInvocationRanges(`/run(${value})`, [integerDescriptor])[0].invocation.arguments[0].value, + { kind: "integer", value }); + } + for (const value of [-2147483649, 2147483648]) { + let rejected = false; + try { selectedFeatureInvocationRanges(`/run(${value})`, [integerDescriptor]); } catch { rejected = true; } + assert(rejected, "values outside Rust int32 range must reject"); + } +}); diff --git a/web/workspace/src/lib/workspace/console/composer-invocation.ts b/web/workspace/src/lib/workspace/console/composer-invocation.ts new file mode 100644 index 000000000..4caf72b97 --- /dev/null +++ b/web/workspace/src/lib/workspace/console/composer-invocation.ts @@ -0,0 +1,434 @@ +import type { + FeatureInvocation, + FeatureInvocationDescriptor, + InvocationArgumentDescriptor, + InvocationArgumentValue, + InvocationValue, + Segment, +} from "#lib/generated/protocol.ts"; + +export type ActiveInvocationArgument = { + descriptor: FeatureInvocationDescriptor; + argument: InvocationArgumentDescriptor | null; + start: number; + end: number; + prefix: string; +}; + +export function quoteInvocationString(value: string): string { + return JSON.stringify(value); +} + +export function invocationInput(invocation: FeatureInvocation): string { + const argumentsText = invocation.arguments.map((argument) => + `${argument.name}=${invocationValueInput(argument.value)}` + ).join(", "); + return `/${invocation.name}(${argumentsText})`; +} + +function invocationValueInput(value: InvocationValue): string { + switch (value.kind) { + case "string": + return quoteInvocationString(value.value); + case "integer": + case "boolean": + return String(value.value); + } +} + +export type SelectedInvocationOccurrence = { + from: number; + descriptor: FeatureInvocationDescriptor; +}; + +export function activeInvocationArgument( + value: string, + cursor: number, + descriptors: readonly FeatureInvocationDescriptor[], + selectedOccurrences?: readonly SelectedInvocationOccurrence[], +): ActiveInvocationArgument | null { + const bounded = Math.max(0, Math.min(cursor, value.length)); + const occurrences = selectedOccurrences ?? activeInvocationOccurrences(value, bounded, descriptors); + for (const { from: slash, descriptor } of occurrences) { + if (slash < 0 || slash >= bounded) continue; + const tail = value.slice(slash, bounded); + if (![descriptor.name, ...descriptor.aliases].some((name) => tail.startsWith(`/${name}(`))) continue; + const open = tail.indexOf("("); + const argumentsText = tail.slice(open + 1); + if (hasUnquotedClose(argumentsText)) continue; + const currentOffset = lastUnquotedComma(argumentsText) + 1; + const rawCurrent = argumentsText.slice(currentOffset); + const leading = rawCurrent.length - rawCurrent.trimStart().length; + const current = rawCurrent.trimStart(); + const equal = unquotedEqual(current); + if (equal >= 0) { + const name = current.slice(0, equal).trim(); + const argument = descriptor.arguments.find((candidate) => candidate.name === name) ?? null; + let valueOffset = equal + 1; + while (/\s/.test(current[valueOffset] ?? "")) valueOffset++; + const prefix = invocationCompletionPrefix(current.slice(valueOffset)); + if (prefix === null) return null; + return { + descriptor, + argument, + start: slash + open + 1 + currentOffset + leading + valueOffset, + end: invocationValueEnd(value, bounded, slash + open + 1 + currentOffset + leading + valueOffset), + prefix, + }; + } + const position = countPositionalArguments(argumentsText.slice(0, currentOffset)); + const positional = descriptor.arguments.find((candidate) => candidate.position === position) ?? null; + const looksLikeArgumentName = /^[a-z][a-z0-9_]*$/.test(current) && + descriptor.arguments.some((candidate) => candidate.name.startsWith(current)); + const prefix = invocationCompletionPrefix(current); + if (prefix === null) return null; + return { + descriptor, + argument: looksLikeArgumentName ? null : positional, + start: slash + open + 1 + currentOffset + leading, + end: looksLikeArgumentName + ? bounded + (/^[a-z0-9_]*/.exec(value.slice(bounded))?.[0].length ?? 0) + : invocationValueEnd(value, bounded, slash + open + 1 + currentOffset + leading), + prefix, + }; + } + return null; +} + +/** Scan forward and skip entire quoted argument regions, never treating a slash + * inside an unfinished outer value as a new call. The editor supplies selected + * occurrence anchors instead of relying on descriptor-only discovery. + */ +function activeInvocationOccurrences( + value: string, + cursor: number, + descriptors: readonly FeatureInvocationDescriptor[], +): SelectedInvocationOccurrence[] { + let active: SelectedInvocationOccurrence | null = null; + scanEachUnquoted(value.slice(0, cursor), (char, slash) => { + if (active) { + if (char === ")") active = null; + return; + } + if (char !== "/" || (slash > 0 && !/\s/.test(value[slash - 1]))) return; + const descriptor = descriptors.find((candidate) => + [candidate.name, ...candidate.aliases].some((name) => value.startsWith(`/${name}(`, slash)) + ); + if (descriptor) active = { from: slash, descriptor }; + }); + return active ? [active] : []; +} + +function invocationCompletionPrefix(value: string): string | null { + let prefix = value; + if (value.startsWith('"')) { + try { + prefix = JSON.parse(value.endsWith('"') ? value : `${value}"`); + } catch { + prefix = value.slice(1); + } + } + try { + return validInvocationString(prefix); + } catch { + // Never send a completion prefix that Rust's JSON reader cannot decode. + return null; + } +} + +function invocationValueEnd(text: string, cursor: number, start: number): number { + try { + const parsed = parseRawValue(text, start); + // Without a complete, valid token the suffix is untouched user input. + return parsed && parsed.end >= cursor ? parsed.end : cursor; + } catch { + return cursor; + } +} + +export function selectedFeatureInvocationAt( + text: string, + start: number, + descriptor: FeatureInvocationDescriptor, +): ParsedInvocationRange | null { + const parsed = parseFeatureInvocation(text, start, descriptor); + return parsed ? { start, end: parsed.end, invocation: parsed.invocation } : null; +} + +export type ParsedInvocationRange = { + start: number; + end: number; + invocation: FeatureInvocation; +}; + +/** Parse complete invocations selected from completion without promoting arbitrary text. */ +export function selectedFeatureInvocationRanges( + text: string, + descriptors: readonly FeatureInvocationDescriptor[], +): ParsedInvocationRange[] { + const ranges: ParsedInvocationRange[] = []; + let search = 0; + while (search < text.length) { + const slash = text.indexOf("/", search); + if (slash < 0) break; + const descriptor = (slash === 0 || /\s/.test(text[slash - 1])) + ? descriptors.find((candidate) => + [candidate.name, ...candidate.aliases].some((name) => + text.startsWith(`/${name}(`, slash) + ) + ) + : undefined; + if (!descriptor) { + search = slash + 1; + continue; + } + const parsed = parseFeatureInvocation(text, slash, descriptor); + if (!parsed) { + search = slash + 1; + continue; + } + ranges.push({ start: slash, end: parsed.end, invocation: parsed.invocation }); + search = parsed.end; + } + return ranges; +} + +export function finalizeSelectedFeatureInvocations( + segments: readonly Segment[], + descriptors: readonly FeatureInvocationDescriptor[], +): Segment[] { + const result: Segment[] = []; + for (const segment of segments) { + if (segment.kind !== "text") { + result.push(segment); + continue; + } + result.push(...parseTextInvocations(segment.content, descriptors)); + } + return coalesceText(result); +} + +function parseTextInvocations( + text: string, + descriptors: readonly FeatureInvocationDescriptor[], +): Segment[] { + const result: Segment[] = []; + let emitted = 0; + let search = 0; + while (search < text.length) { + const slash = text.indexOf("/", search); + if (slash < 0) break; + const leadingOk = slash === 0 || /\s/.test(text[slash - 1]); + const descriptor = leadingOk + ? descriptors.find((candidate) => + [candidate.name, ...candidate.aliases].some((name) => + text.startsWith(`/${name}(`, slash) + ) + ) + : undefined; + if (!descriptor) { + search = slash + 1; + continue; + } + const parsed = parseFeatureInvocation(text, slash, descriptor); + if (!parsed) { + search = slash + 1; + continue; + } + appendText(result, text.slice(emitted, slash)); + result.push({ kind: "feature_invoke", invocation: parsed.invocation }); + emitted = parsed.end; + search = parsed.end; + } + if (!result.length) return [{ kind: "text", content: text }]; + appendText(result, text.slice(emitted)); + return result; +} + +function parseFeatureInvocation( + text: string, + start: number, + descriptor: FeatureInvocationDescriptor, +): { invocation: FeatureInvocation; end: number } | null { + let cursor = start + 1; + while (/[a-z0-9_-]/.test(text[cursor] ?? "")) cursor++; + const name = text.slice(start + 1, cursor); + if (name !== descriptor.name && !descriptor.aliases.includes(name)) return null; + while (/\s/.test(text[cursor] ?? "")) cursor++; + if (text[cursor] !== "(") return null; + cursor++; + const values: InvocationArgumentValue[] = []; + let position = 0; + while (true) { + while (/\s/.test(text[cursor] ?? "")) cursor++; + if (text[cursor] === ")") { + cursor++; + break; + } + if (cursor >= text.length) return null; + const itemStart = cursor; + while (/[a-z0-9_]/.test(text[cursor] ?? "")) cursor++; + const possibleName = text.slice(itemStart, cursor); + while (/\s/.test(text[cursor] ?? "")) cursor++; + let argument: InvocationArgumentDescriptor | undefined; + if (possibleName && text[cursor] === "=") { + argument = descriptor.arguments.find((candidate) => candidate.name === possibleName); + if (!argument) throw new Error(`Unknown Feature argument: ${possibleName}`); + cursor++; + while (/\s/.test(text[cursor] ?? "")) cursor++; + } else { + cursor = itemStart; + argument = descriptor.arguments.find((candidate) => candidate.position === position); + position++; + if (!argument) throw new Error("Too many positional Feature arguments."); + } + const parsed = parseRawValue(text, cursor); + if (!parsed) return null; + cursor = parsed.end; + if (values.some((candidate) => candidate.name === argument.name)) { + throw new Error(`Duplicate Feature argument: ${argument.name}`); + } + values.push({ name: argument.name, value: coerceValue(argument, parsed.value) }); + while (/\s/.test(text[cursor] ?? "")) cursor++; + if (text[cursor] === ",") cursor++; + else if (text[cursor] !== ")") return null; + } + for (const argument of descriptor.arguments) { + if (argument.required && !values.some((value) => value.name === argument.name)) { + throw new Error(`Missing required Feature argument: ${argument.name}`); + } + } + return { + invocation: { + invocation_id: crypto.randomUUID(), + identity: descriptor.identity, + name, + arguments: values, + }, + end: cursor, + }; +} + +function parseRawValue(text: string, start: number): { value: string; end: number } | null { + if (text[start] === '"') { + let cursor = start + 1; + let escaped = false; + while (cursor < text.length) { + const char = text[cursor]; + if (!escaped && char === '"') { + const literal = text.slice(start, cursor + 1); + return { value: validInvocationString(JSON.parse(literal)), end: cursor + 1 }; + } + escaped = !escaped && char === "\\"; + if (char !== "\\") escaped = false; + cursor++; + } + return null; + } + let cursor = start; + while (cursor < text.length && !/[\s,)]/.test(text[cursor])) cursor++; + return cursor === start ? null : { value: validInvocationString(text.slice(start, cursor)), end: cursor }; +} + +/** Rust JSON strings contain Unicode scalar values, not lone UTF-16 code units. + * JSON.parse alone permits unpaired surrogate escapes that serde_json rejects. + */ +function validInvocationString(value: string): string { + for (let index = 0; index < value.length; index++) { + const unit = value.charCodeAt(index); + if (unit >= 0xD800 && unit <= 0xDBFF) { + const next = value.charCodeAt(++index); + if (next >= 0xDC00 && next <= 0xDFFF) continue; + } else if (unit < 0xDC00 || unit > 0xDFFF) continue; + throw new Error("Feature string arguments must not contain unpaired Unicode surrogates."); + } + return value; +} + +function coerceValue(argument: InvocationArgumentDescriptor, value: string): InvocationValue { + switch (argument.value_type.kind) { + case "integer": { + if (!/^-?\d+$/.test(value)) throw new Error(`Feature argument ${argument.name} must be an integer.`); + const integer = Number(value); + if (!Number.isSafeInteger(integer)) { + throw new Error(`Feature argument ${argument.name} must be a safe integer.`); + } + if (integer < -2147483648 || integer > 2147483647) { + throw new Error(`Feature argument ${argument.name} must be a signed 32-bit integer.`); + } + return { kind: "integer", value: integer }; + } + case "boolean": + if (value !== "true" && value !== "false") { + throw new Error(`Feature argument ${argument.name} must be true or false.`); + } + return { kind: "boolean", value: value === "true" }; + case "enum": + if (!argument.value_type.values.includes(value)) { + throw new Error(`Feature argument ${argument.name} is not an allowed value.`); + } + return { kind: "string", value }; + case "string": + case "worker_file": + case "client_file": + return { kind: "string", value }; + } +} + +function appendText(segments: Segment[], content: string): void { + if (!content) return; + const prior = segments.at(-1); + if (prior?.kind === "text") prior.content += content; + else segments.push({ kind: "text", content }); +} + +function coalesceText(segments: Segment[]): Segment[] { + const result: Segment[] = []; + for (const segment of segments) { + if (segment.kind === "text") appendText(result, segment.content); + else result.push(segment); + } + return result; +} + +// These helpers only inspect syntax before the cursor. They deliberately do not +// execute or finalize anything. +function hasUnquotedClose(value: string): boolean { + return scanUnquoted(value, (char) => char === ")") >= 0; +} +function lastUnquotedComma(value: string): number { + let last = -1; + scanEachUnquoted(value, (char, index) => { if (char === ",") last = index; }); + return last; +} +function unquotedEqual(value: string): number { + return scanUnquoted(value, (char) => char === "="); +} +function countPositionalArguments(value: string): number { + let count = 0; + let start = 0; + scanEachUnquoted(value, (char, index) => { + if (char !== ",") return; + if (value.slice(start, index).trim() && unquotedEqual(value.slice(start, index)) < 0) count++; + start = index + 1; + }); + return count; +} +function scanUnquoted(value: string, predicate: (char: string) => boolean): number { + let found = -1; + scanEachUnquoted(value, (char, index) => { + if (found < 0 && predicate(char)) found = index; + }); + return found; +} +function scanEachUnquoted(value: string, visit: (char: string, index: number) => void): void { + let quoted = false; + let escaped = false; + for (let index = 0; index < value.length; index++) { + const char = value[index]; + if (!escaped && char === '"') quoted = !quoted; + if (!quoted) visit(char, index); + escaped = quoted && !escaped && char === "\\"; + if (char !== "\\") escaped = false; + } +} diff --git a/web/workspace/src/lib/workspace/console/file-completions.test.ts b/web/workspace/src/lib/workspace/console/file-completions.test.ts index 323757201..8dca9d882 100644 --- a/web/workspace/src/lib/workspace/console/file-completions.test.ts +++ b/web/workspace/src/lib/workspace/console/file-completions.test.ts @@ -1,70 +1,79 @@ import { FileCompletions } from "./file-completions.ts"; -import { deepStrictEqual as assertEquals, rejects } from "node:assert/strict"; - -async function assertRejects( - fn: () => Promise, - _type: typeof Error, - message: string, -) { - await rejects( - fn, - (error: unknown) => - error instanceof Error && error.message.includes(message), - ); -} +import { + deepStrictEqual as assertEquals, + notEqual, + rejects, +} from "node:assert/strict"; declare const Deno: { test(name: string, fn: () => Promise): void }; -Deno.test("file completion serializes the latest prefix behind cancelled wire replies", async () => { - const sent: string[] = []; - const lane = new FileCompletions((prefix) => sent.push(prefix)); +Deno.test("completion nonces fence same prefix/context ABA without draining cancelled requests", async () => { + const sent: Array<[string, string]> = []; + const lane = new FileCompletions((prefix, id) => sent.push([prefix, id])); const first = new AbortController(); - const a = lane.request("a", first.signal); + const a = lane.request("same", first.signal); first.abort(); assertEquals(await a, []); - const second = new AbortController(); - const b = lane.request("b", second.signal); - second.abort(); - const c = lane.request("c", new AbortController().signal); + const b = lane.request("other", new AbortController().signal); + const c = lane.request("same", new AbortController().signal); assertEquals(await b, []); - assertEquals(sent, ["a"]); - lane.receive([{ value: "old-a" }]); - assertEquals(sent, ["a", "c"]); - lane.receive([{ value: "current-c" }]); + assertEquals(sent.map(([prefix]) => prefix), ["same", "other", "same"]); + notEqual(sent[0][1], sent[2][1]); + lane.receive([{ value: "old-a" }], "same", sent[0][1]); + lane.receive([{ value: "legacy" }], "same"); + lane.receive([{ value: "wrong-prefix" }], "other", sent[2][1]); + assertEquals(lane.pending, true); + lane.receive([{ value: "current-c" }], "same", sent[2][1]); assertEquals(await c, [{ value: "current-c" }]); lane.close(); }); -Deno.test("file completion timeout blocks late responses until a new connection", async () => { - const sent: string[] = []; - const lane = new FileCompletions((prefix) => sent.push(prefix), 5); - await assertRejects( - () => lane.request("a", new AbortController().signal), - Error, - "timed out", +Deno.test("multiple clients cannot consume each other's identical completion broadcasts", async () => { + const ids: string[] = []; + const lanes = [ + new FileCompletions((_prefix, id) => ids.push(id)), + new FileCompletions((_prefix, id) => ids.push(id)), + ]; + const promises = lanes.map((lane) => + lane.request("same", new AbortController().signal) ); - lane.receive([{ value: "late-a" }]); - await assertRejects( - () => lane.request("b", new AbortController().signal), - Error, - "timed out", + notEqual(ids[0], ids[1]); + for (const lane of lanes) { + lane.receive([{ value: "client-a" }], "same", ids[0]); + } + assertEquals(await promises[0], [{ value: "client-a" }]); + assertEquals(lanes[1].pending, true); + for (const lane of lanes) { + lane.receive([{ value: "client-b" }], "same", ids[1]); + } + assertEquals(await promises[1], [{ value: "client-b" }]); + lanes.forEach((lane) => lane.close()); +}); + +Deno.test("completion timeout and reconnect invalidate old nonces but allow a fresh request", async () => { + const ids: string[] = []; + const lane = new FileCompletions((_prefix, id) => ids.push(id), 5); + await rejects( + lane.request("same", new AbortController().signal), + /timed out/, ); - assertEquals(sent, ["a"]); lane.reset(); - const c = lane.request("c", new AbortController().signal); - lane.receive([{ value: "c" }]); - assertEquals(await c, [{ value: "c" }]); + const current = lane.request("same", new AbortController().signal); + lane.receive([{ value: "old" }], "same", ids[0]); + assertEquals(lane.pending, true); + lane.receive([{ value: "new" }], "same", ids[1]); + assertEquals(await current, [{ value: "new" }]); lane.close(); }); -Deno.test("file completion close rejects both active and queued requests", async () => { +Deno.test("completion close rejects active request and fences requests until reset", async () => { const lane = new FileCompletions(() => {}); const a = lane.request("a", new AbortController().signal); - const b = lane.request("b", new AbortController().signal); - const outcomes = Promise.allSettled([a, b]); + const rejected = rejects(a, /disconnected/); lane.close(new Error("disconnected")); - assertEquals((await outcomes).map((value) => value.status), [ - "rejected", - "rejected", - ]); + await rejected; + await rejects( + lane.request("b", new AbortController().signal), + /disconnected/, + ); }); diff --git a/web/workspace/src/lib/workspace/console/file-completions.ts b/web/workspace/src/lib/workspace/console/file-completions.ts index 4ad688fc7..20044d510 100644 --- a/web/workspace/src/lib/workspace/console/file-completions.ts +++ b/web/workspace/src/lib/workspace/console/file-completions.ts @@ -1,6 +1,7 @@ import type { ComposerCompletionEntry } from "./composer-completion.ts"; type Request = { + id: string; prefix: string; signal: AbortSignal; resolve: (entries: ComposerCompletionEntry[]) => void; @@ -8,18 +9,18 @@ type Request = { abort: () => void; }; -/** The wire reply has no request id/prefix. Never overlap requests on a lane, - * including cancelled requests: their reply must be consumed before the next. - * A timeout poisons the lane until reconnect, rather than misrouting a late reply. +/** A lane is scoped to a transport target and completion kind/context. Each + * request also has a unique wire nonce: cancelled requests never need draining, + * and legacy replies or another client's same-prefix broadcasts cannot match. + * The editor's abort signal fences the draft, cursor and completion generation. */ export class FileCompletions { private active: Request | null = null; - private queued: Request | null = null; private timer: ReturnType | undefined; private failure: Error | null = null; constructor( - private send: (prefix: string) => void, + private send: (prefix: string, requestId: string) => void, private timeoutMs = 30_000, ) {} @@ -33,42 +34,48 @@ export class FileCompletions { ): Promise { if (signal.aborted) return Promise.resolve([]); if (this.failure) return Promise.reject(this.failure); + this.finish([]); return new Promise((resolve, reject) => { const request: Request = { + id: crypto.randomUUID(), prefix, signal, resolve, reject, abort: () => { - resolve([]); - if (this.queued === request) this.queued = null; + if (this.active === request) this.finish([]); }, }; + this.active = request; signal.addEventListener("abort", request.abort, { once: true }); - if (this.queued) this.finish(this.queued, []); - this.queued = request; - this.pump(); + this.timer = setTimeout(() => { + if (this.active === request) { + this.reject(new Error("Worker completion request timed out; retry.")); + } + }, this.timeoutMs); + try { + this.send(prefix, request.id); + } catch (error) { + this.reject(error instanceof Error ? error : new Error(String(error))); + } }); } - receive(entries: ComposerCompletionEntry[]): void { - if (!this.active) return; - clearTimeout(this.timer); - const request = this.active; - this.active = null; - this.finish(request, entries); - this.pump(); + receive( + entries: ComposerCompletionEntry[], + prefix: string, + requestId?: string | null, + ): void { + if ( + !this.active || this.active.id !== requestId || + this.active.prefix !== prefix + ) return; + this.finish(entries); } close(error = new Error("Worker completion connection closed.")): void { this.failure = error; - clearTimeout(this.timer); - for (const request of [this.active, this.queued]) { - if (!request) continue; - request.signal.removeEventListener("abort", request.abort); - request.reject(error); - } - this.active = this.queued = null; + this.reject(error); } reset(): void { @@ -76,31 +83,20 @@ export class FileCompletions { this.failure = null; } - private finish(request: Request, entries: ComposerCompletionEntry[]): void { - request.signal.removeEventListener("abort", request.abort); - request.resolve(request.signal.aborted ? [] : entries); + private take(): Request | null { + clearTimeout(this.timer); + const request = this.active; + this.active = null; + request?.signal.removeEventListener("abort", request.abort); + return request; } - private pump(): void { - if (this.active || !this.queued || this.failure) return; - const request = this.queued; - this.queued = null; - if (request.signal.aborted) { - this.finish(request, []); - return; - } - this.active = request; - this.timer = setTimeout( - () => - this.close( - new Error("Worker completion request timed out; reconnect to retry."), - ), - this.timeoutMs, - ); - try { - this.send(request.prefix); - } catch (error) { - this.close(error instanceof Error ? error : new Error(String(error))); - } + private finish(entries: ComposerCompletionEntry[]): void { + const request = this.take(); + request?.resolve(request.signal.aborted ? [] : entries); + } + + private reject(error: Error): void { + this.take()?.reject(error); } } diff --git a/web/workspace/src/lib/workspace/console/model.test.ts b/web/workspace/src/lib/workspace/console/model.test.ts index 0416b2f51..a55316889 100644 --- a/web/workspace/src/lib/workspace/console/model.test.ts +++ b/web/workspace/src/lib/workspace/console/model.test.ts @@ -352,7 +352,7 @@ Deno.test("console routing projects live errors but not completion replies", () } satisfies Event; const completionEvent = { event: "completions", - data: { kind: "file", entries: [] }, + data: { kind: "file", prefix: "", entries: [] }, } satisfies Event; assert( @@ -4059,3 +4059,11 @@ Deno.test("committed history inserts current live rows after their last stable a ["current-user", "live-assistant"], ); }); + +Deno.test("typed Feature invocations remain visible in live and restored user messages", () => { + const segment = { kind: "feature_invoke" as const, invocation: { + invocation_id: "invoke-1", identity: "test/run", name: "run", + arguments: [{ name: "path", value: { kind: "string" as const, value: "資料/a b" } }], + } }; + assertEquals(segmentsToText([segment]), '/run(path="資料/a b")'); +}); diff --git a/web/workspace/src/lib/workspace/console/model.ts b/web/workspace/src/lib/workspace/console/model.ts index 6d782d7a0..ce44a7d68 100644 --- a/web/workspace/src/lib/workspace/console/model.ts +++ b/web/workspace/src/lib/workspace/console/model.ts @@ -19,6 +19,7 @@ import type { WorkerStateSnapshot, WorkerStatus, } from "#lib/generated/protocol.ts"; +import { invocationInput } from "./composer-invocation.ts"; import { stringify as stringifyYaml } from "yaml"; import { applyRunActivityEvent, @@ -1348,6 +1349,8 @@ export function segmentsToText(segments: Segment[]): string { return `[Large paste artifact ${segment.artifact.artifact_id}: ${segment.artifact.byte_len} bytes, ${segment.artifact.media_type}, ${segment.artifact.availability}, created ${segment.artifact.created_at_ms} ms, sha256 ${segment.artifact.sha256}]`; case "uploaded_file": return `[Attachment: ${segment.file.file_name} · ${segment.file.media_type} · ${segment.file.byte_len} bytes · ${segment.file.availability}]`; + case "feature_invoke": + return invocationInput(segment.invocation); case "file_ref": return `@file ${segment.path}`; case "unknown": diff --git a/web/workspace/src/lib/workspace/console/worker-console.ui.test.ts b/web/workspace/src/lib/workspace/console/worker-console.ui.test.ts index 6194e31f5..abd18cdfd 100644 --- a/web/workspace/src/lib/workspace/console/worker-console.ui.test.ts +++ b/web/workspace/src/lib/workspace/console/worker-console.ui.test.ts @@ -502,7 +502,9 @@ Deno.test("workspace Memory surfaces use read-only scoped memory APIs", async () subjectIndexLoad.includes('"/subjektiv/subjects"') && subjectIndexLoad.includes("parseSubjektivSubjectListResponse") && subjectIndexPage.includes(">Subjects") && - subjectIndexPage.includes("subject.store_revision") && + subjectIndexPage.includes("Worker connection") && + subjectIndexPage.includes("subject.current_worker") && + !subjectIndexPage.includes("subject.store_revision") && subjectIndexPage.includes('data-memory-view="subjects"'), "Memory product entry should list explicit typed subjects", ); @@ -510,10 +512,13 @@ Deno.test("workspace Memory surfaces use read-only scoped memory APIs", async () subjectPageLoad.includes("`${subjectPath}/surface`") && subjectPageLoad.includes("`${subjectPath}/memories`") && subjectPage.includes("DocumentMarkdown from") && - subjectPage.includes("Resident surface") && + subjectPage.includes("Resident context") && + subjectPage.includes("Subject store revision") && subjectPage.includes("Current Memories") && subjectPage.includes("surface?.availability === 'ready'") && + subjectPage.includes("surface?.availability === 'ungenerated'") && subjectPage.includes("surface?.availability === 'stale'") && + subjectPage.includes("surface?.availability === 'failed'") && subjectPage.includes("data-surface-ready-empty") && !subjectPage.includes("overflow-y"), "Subject page should combine strict resident-surface and current-Memory reads without a nested vertical scroller", @@ -1043,7 +1048,7 @@ Deno.test("Worker Console paste chips preserve typed draft and target authority" composerInput.includes('key: "Backspace"') && composerInput.includes('key: "Delete"') && composerInput.includes( - "composerDeletionRange(selection, pastes, direction)", + "composerDeletionRange(selection, atoms, direction)", ) && composerInput.includes("EditorState.readOnly.of(isDisabled)") && composerInput.includes('key: "Mod-z"') && @@ -1594,8 +1599,8 @@ Deno.test("Composer owns TUI-style completion and replaces Compact/Rewind header "Completion navigation must precede input history", ); assert( - page.includes("fileCompletions.receive(event.data.entries)") && - page.includes("fileCompletions.close()"), + page.includes("fileCompletions.receive(event.data.entries, event.data.prefix, event.data.request_id)") && + page.includes("rejectPendingCompletion(new Error(\"Worker completion connection closed.\"), `${target.workspaceId}:${target.runtimeId}:${target.workerId}`)"), "File completion lifecycle must be scoped to the transport", ); }); @@ -1719,7 +1724,7 @@ Deno.test("Web Console uses Notify while running and exposes queued input cancel consolePage.includes("{#if workerRunning || workerPaused}") && consolePage.includes('delivery === "queue" && request.kind !== "user"') && consolePage.includes( - 'method.method === "submit" && delivery === "submit"', + 'event.event === "submission_accepted" && event.data.disposition === "started"', ), "busy Console must offer explicit Queue without changing paused state or queueing commands", ); diff --git a/web/workspace/src/lib/workspace/memory/api.ts b/web/workspace/src/lib/workspace/memory/api.ts index 65153d589..c97dea9d4 100644 --- a/web/workspace/src/lib/workspace/memory/api.ts +++ b/web/workspace/src/lib/workspace/memory/api.ts @@ -1,4 +1,7 @@ -import { MEMORY_API_LIMITS } from "#lib/generated/memory-api.ts"; +import { + MEMORY_API_LIMITS, + SUBJEKTIV_API_LIMITS, +} from "#lib/generated/memory-api.ts"; import { readBoundedJson, workspaceApiPath } from "#lib/workspace/api/http.ts"; import type { Diagnostic, @@ -26,6 +29,7 @@ import type { SubjektivResidentSurfaceAvailability, SubjektivResidentSurfaceResponse, SubjektivResidentSurfaceSnapshot, + SubjektivSubjectBehaviorUpdateRequest, SubjektivSubjectCreateRequest, SubjektivSubjectListResponse, SubjektivSubjectResponse, @@ -84,6 +88,7 @@ export const MEMORY_API_LOAD_POLICY = { } as const; const MAX_SUBJECT_ROLE_BYTES = 256; +export const MAX_SUBJECT_BEHAVIOR_BYTES = SUBJEKTIV_API_LIMITS.maxBehaviorBytes; const MAX_SUBJECT_CREATE_ERROR_BYTES = 1024 * 1024; export type SubjektivSubjectCreateErrorKind = @@ -126,7 +131,31 @@ export function validateSubjektivSubjectCreateRequest( "Subject role must be at most 256 bytes and contain no control characters.", ); } - return { role: request.role }; + const behaviorMd = request.behavior_md ?? ""; + validateSubjectBehavior(behaviorMd); + return { role: request.role, behavior_md: behaviorMd }; +} + +function validateSubjectBehavior(behaviorMd: string): void { + const bytes = new TextEncoder().encode(behaviorMd).byteLength; + if (bytes > MAX_SUBJECT_BEHAVIOR_BYTES) { + throw new SubjektivSubjectCreateError( + "validation", + `Subject behavior must be at most ${MAX_SUBJECT_BEHAVIOR_BYTES} bytes.`, + ); + } + if (behaviorMd.length > 0 && behaviorMd.trim().length === 0) { + throw new SubjektivSubjectCreateError( + "validation", + "Subject behavior must be empty or contain non-whitespace text.", + ); + } + if (/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F]/u.test(behaviorMd)) { + throw new SubjektivSubjectCreateError( + "validation", + "Subject behavior contains an unsupported control character.", + ); + } } export async function createSubjektivSubject( @@ -170,8 +199,11 @@ export async function createSubjektivSubject( const subject = parseSubjektivSubjectResponse( await readBoundedJson(response, MEMORY_API_LIMITS.maxResponseBytes), ); - if (subject.role !== validated.role) { - throw new Error("created Subject role does not match the request"); + if ( + subject.role !== validated.role || + subject.behavior_md !== validated.behavior_md + ) { + throw new Error("created Subject does not match the request"); } return subject; } catch { @@ -182,6 +214,68 @@ export async function createSubjektivSubject( } } +export async function updateSubjektivSubjectBehavior( + fetchFn: typeof fetch, + workspaceId: string, + subjectId: string, + request: SubjektivSubjectBehaviorUpdateRequest, +): Promise { + validateSubjectBehavior(request.behavior_md); + if ( + !Number.isSafeInteger(request.expected_behavior_revision) || + request.expected_behavior_revision < 0 + ) { + throw new SubjektivSubjectCreateError( + "validation", + "Subject behavior revision is invalid.", + ); + } + let response: Response; + try { + response = await fetchFn( + workspaceApiPath( + workspaceId, + `/subjektiv/subjects/${encodeURIComponent(subjectId)}/behavior`, + ), + { + method: "PATCH", + headers: { "content-type": "application/json" }, + body: JSON.stringify(request), + }, + ); + } catch { + throw new SubjektivSubjectCreateError( + "unknown_outcome", + "The save outcome is unknown. Reload the Subject before retrying.", + ); + } + if (!response.ok) { + const message = await subjektivSubjectCreateErrorMessage(response); + throw new SubjektivSubjectCreateError( + response.status >= 500 ? "unknown_outcome" : "rejected", + response.status === 409 + ? "The behavior changed elsewhere. Reload the Subject and review the latest text." + : message, + response.status, + ); + } + try { + const subject = parseSubjektivSubjectResponse( + await readBoundedJson(response, MEMORY_API_LIMITS.maxResponseBytes), + subjectId, + ); + if (subject.behavior_md !== request.behavior_md) { + throw new Error("saved Subject behavior does not match the request"); + } + return subject; + } catch { + throw new SubjektivSubjectCreateError( + "unknown_outcome", + "The behavior may have been saved, but the response could not be confirmed. Reload the Subject.", + ); + } +} + async function subjektivSubjectCreateErrorMessage( response: Response, ): Promise { @@ -202,7 +296,7 @@ async function subjektivSubjectCreateErrorMessage( } catch { // A rejected request still has a known outcome even when its body is invalid. } - return `Subject creation failed with HTTP ${response.status}.`; + return `Subject request failed with HTTP ${response.status}.`; } export function parseMemoryDocumentResponse( @@ -454,6 +548,8 @@ export function parseSubjektivSubjectResponse( [ "id", "role", + "behavior_md", + "behavior_revision", "state", "store_revision", "created_at", @@ -472,6 +568,12 @@ export function parseSubjektivSubjectResponse( const result: SubjektivSubjectResponse = { id, role: requiredString(record, "role"), + behavior_md: requiredString( + record, + "behavior_md", + MAX_SUBJECT_BEHAVIOR_BYTES, + ), + behavior_revision: requiredNonNegativeInteger(record, "behavior_revision"), state, store_revision: requiredNonNegativeInteger(record, "store_revision"), created_at: requiredString(record, "created_at"), diff --git a/web/workspace/src/lib/workspace/protocol-validator.test.ts b/web/workspace/src/lib/workspace/protocol-validator.test.ts index a87a1e190..453df9808 100644 --- a/web/workspace/src/lib/workspace/protocol-validator.test.ts +++ b/web/workspace/src/lib/workspace/protocol-validator.test.ts @@ -323,3 +323,28 @@ Deno.test('stateful selector validation fences a routed event before projection' 'mismatched Worker event crossed its subscription selector', ); }); + +Deno.test('generated subscription schema accepts invocation descriptors, context and typed user segments', () => { + const descriptor = { identity: 'test/run', name: 'run', aliases: ['execute'], display_name: 'Run', description: 'Test', + syntax: 'parenthesized', client_adapter: null, arguments: [{ name: 'count', position: 0, required: true, + value_type: { kind: 'integer' }, completion: { kind: 'static', values: ['1', '2'] } }] }; + const frame = (events: unknown[]) => subscribedFrame({ + selector: { topic: 'worker_protocol', worker_id: 'worker-1' }, + snapshot: { topic: 'worker_protocol', data: { worker_id: 'worker-1', events } }, + }); + const completion = { event: 'completions', data: { kind: 'feature_argument', prefix: '1', + context: { invocation: 'test/run', argument: 'count' }, + entries: [{ value: '1', is_dir: false, invocation: descriptor }] } }; + const segment = { kind: 'feature_invoke', invocation: { invocation_id: 'invoke-1', identity: 'test/run', name: 'run', + arguments: [{ name: 'count', value: { kind: 'integer', value: 1 } }] } }; + const valid = decode(frame([completion, { event: 'user_message', data: { segments: [segment] } }])); + assert(valid.ok, `invocation schema mismatch: ${JSON.stringify(valid)}`); + const invalid = decode(frame([{ event: 'user_message', data: { segments: [{ ...segment, invocation: { + ...segment.invocation, arguments: [{ name: 'count', value: { kind: 'integer', value: 9007199254740992 } }], + } }] } }])); + assert(!invalid.ok, 'generated invocation schema must reject unsafe wire integers'); + const newer = decode(frame([{ ...completion, data: { ...completion.data, entries: [ + { value: '1', is_dir: false, invocation: { ...descriptor, client_adapter: 'newer_adapter' } }, + ] } }])); + assert(!newer.ok, 'unknown adapter capabilities must fail closed'); +}); diff --git a/web/workspace/src/lib/workspace/sidebar/WorkersNavSection.svelte b/web/workspace/src/lib/workspace/sidebar/WorkersNavSection.svelte index a2180f3a8..3b90e2fa7 100644 --- a/web/workspace/src/lib/workspace/sidebar/WorkersNavSection.svelte +++ b/web/workspace/src/lib/workspace/sidebar/WorkersNavSection.svelte @@ -1,8 +1,15 @@ @@ -221,6 +276,23 @@ {#if openWorkerKey === key}
{#if data.subjects.data} - {subjects.length}{data.subjects.data.has_more ? '+' : ''} subject{subjects.length === 1 ? '' : 's'} + {subjects.length} shown{data.subjects.data.has_more ? ' · more available' : ''} {/if} + {/if} + +

The Host injects this text verbatim for connected Workers, separately from generated Memory. Saving confirms storage only. Connected Workers check for the latest revision before later model requests; this page does not report that application.

+ {#if editingBehavior} +
{ event.preventDefault(); void saveBehavior(); }}> + + {#if behaviorError}{/if} +
+ + + +
+
+ {:else if subject.behavior_md.length > 0} + 
{subject.behavior_md}
+ {:else} +
No behavior is set.

Connected Workers receive an explicit empty behavior document.

+ {/if} + {#if behaviorSaveConfirmed}

Behavior storage confirmed at revision {subject.behavior_revision}. This page does not confirm application by a connected Worker.

{/if} + + {/if} +
-

Resident context

-

Resident surface

+

Generated from committed Memory

+

Resident context

{#if surface} - {surface.availability} + {surfaceLabel(surface.availability)} {/if}
{#if surface?.availability === 'ready' && snapshot}
- Built from store revision {snapshot.built_from_store_revision} + Generated - - - {snapshot.memory_refs.length} Memory ref{snapshot.memory_refs.length === 1 ? '' : 's'} + {snapshot.memory_refs.length} referenced Memory {snapshot.memory_refs.length === 1 ? 'record' : 'records'}
{#if snapshot.body_md.trim().length === 0}
- Resident surface is ready and empty. -

This subject currently has no resident context to display.

+ Resident context is current but empty. +

A generated surface exists, but it contains no context to show. Committed Memories, if any, remain listed below.

{:else} -
+
{/if}
- Surface provenance + Surface sources and diagnostics
-
Snapshot id
{snapshot.snapshot_id}
+
Snapshot ID
{snapshot.snapshot_id}
-
Memory refs
+
Generated from Subject revision
+
{snapshot.built_from_store_revision}The Subject change number used for this surface, distinct from each Memory’s revision.
+
+
+
Referenced Memory revisions
{#if snapshot.memory_refs.length === 0} None @@ -123,26 +294,26 @@
{:else if surface?.availability === 'ungenerated'}
- Resident surface has not been generated. -

Committed Memories remain available below.

+ Resident context has not been generated. +

Committed Memories, if any, remain available below.

{:else if surface?.availability === 'stale'}
- Resident surface is stale. -

The latest committed Memories are not represented by a publishable snapshot.

+ Resident context needs to be refreshed. +

The latest committed Memories are not represented by a current surface, so no generated context is shown.

{:else if surface?.availability === 'failed'} {:else if data.surface.error} {:else} -

Resident surface data is unavailable.

+

Resident context status is unavailable.

{/if}
@@ -152,14 +323,14 @@

Committed records

Current Memories

- {#if data.memories.data}{memories.length}{data.memories.data.has_more ? '+' : ''}{/if} + {#if data.memories.data}{memories.length} shown{data.memories.data.has_more ? ' · more available' : ''}{/if} {#if data.memories.data} {#if memories.length === 0}
- No committed Memories. -

This subject has no current Memory revisions.

+ {data.cursor ? 'No committed Memories on this page.' : 'No committed Memories yet.'} +

{data.cursor ? 'Return to the first page to review earlier records.' : 'This Subject has no current Memory records.'}

{#if data.cursor}

Return to the first page

{/if}
{:else} @@ -175,9 +346,9 @@

{memory.excerpt || 'No excerpt.'}

-
Memory id
{memory.id}
-
Revision
{memory.revision}
-
Updated
+
Memory ID
{memory.id}
+
Memory revision
{memory.revision}
+
Last changed
{/each} @@ -270,25 +441,79 @@ font-size: var(--font-size-compact); } - .subject-facts { - grid-template-columns: repeat(5, minmax(0, 1fr)); - gap: var(--space-3); + .subject-overview { + display: grid; + grid-template-columns: repeat(4, minmax(0, 1fr)); + gap: var(--space-4); + padding: var(--space-4); + background: var(--bg-raised); + border-radius: var(--radius-soft); } - .subject-facts > div { - display: block; + .subject-overview > div { + display: grid; + align-content: start; + gap: var(--space-1); + min-width: 0; } - .subject-facts dt { - white-space: normal; + .subject-overview span, + .subject-overview p, + .subject-technical-details summary, + .subject-technical-details small, + .surface-details small { + color: var(--text-muted); + font-size: var(--font-size-compact); } - .subject-facts dd { - margin-top: var(--space-1); - color: var(--text); + .subject-overview strong { + color: var(--text-strong); overflow-wrap: anywhere; } + .subject-overview p { + margin: 0; + overflow-wrap: anywhere; + } + + .subject-technical-details { + padding-bottom: var(--space-3); + border-bottom: 1px solid var(--line); + } + + .subject-technical-details summary { + width: fit-content; + cursor: pointer; + font-weight: 700; + } + + .subject-technical-details dl, + .surface-details dl { + display: grid; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: var(--space-4); + margin-top: var(--space-3); + } + + .subject-technical-details dl > div, + .surface-details dl > div { + display: block; + min-width: 0; + } + + .subject-technical-details dd, + .surface-details dd { + margin-top: var(--space-1); + overflow-wrap: anywhere; + } + + .subject-technical-details small, + .surface-details small { + display: block; + margin-top: var(--space-1); + line-height: 1.4; + } + .subject-state, .availability, .memory-pill, @@ -320,6 +545,7 @@ color: var(--danger); } + .behavior-section, .surface-section, .memories-section { display: grid; @@ -328,6 +554,78 @@ padding-top: var(--space-3); } + .behavior-help, + .behavior-saved, + .behavior-error { + margin: 0; + color: var(--text-muted); + font-size: var(--font-size-compact); + } + + .behavior-error { color: var(--danger); } + .behavior-saved { color: var(--success); } + + .behavior-form, + .behavior-form label { + display: grid; + gap: var(--space-3); + } + + .behavior-form textarea { + width: 100%; + min-height: 10rem; + padding: var(--space-3); + color: var(--text); + background: var(--surface-raised); + border: 1px solid var(--line-strong); + border-radius: var(--radius-sm); + font: inherit; + line-height: 1.5; + resize: vertical; + } + + .behavior-actions { + display: flex; + flex-wrap: wrap; + gap: var(--space-2); + } + + .behavior-edit, + .behavior-actions button { + padding: var(--space-2) var(--space-3); + color: var(--text); + background: var(--surface-raised); + border: 1px solid var(--line-strong); + border-radius: var(--radius-sm); + cursor: pointer; + } + + .behavior-actions button:disabled { cursor: not-allowed; opacity: 0.55; } + + .behavior-document pre { + margin: 0; + padding: var(--space-4); + overflow-wrap: anywhere; + white-space: pre-wrap; + background: var(--surface-raised); + border: 1px solid var(--line); + border-radius: var(--radius-sm); + font: inherit; + line-height: 1.6; + } + + .sr-only { + position: absolute; + width: 1px; + height: 1px; + padding: 0; + margin: -1px; + overflow: hidden; + clip: rect(0, 0, 0, 0); + white-space: nowrap; + border: 0; + } + .surface-meta, .pill-row { display: flex; @@ -455,7 +753,7 @@ } @media (max-width: 900px) { - .subject-facts { + .subject-overview { grid-template-columns: repeat(2, minmax(0, 1fr)); } @@ -472,10 +770,16 @@ gap: var(--space-2); } - .subject-facts { + .subject-overview, + .subject-technical-details dl, + .surface-details dl { grid-template-columns: minmax(0, 1fr); } + .subject-overview { + padding: var(--space-3); + } + .memory-row { padding-inline: 0; } diff --git a/web/workspace/src/routes/w/[workspaceId]/memory/[subjectId]/[memoryId]/+page.svelte b/web/workspace/src/routes/w/[workspaceId]/memory/[subjectId]/[memoryId]/+page.svelte index 07a1108fe..71c038627 100644 --- a/web/workspace/src/routes/w/[workspaceId]/memory/[subjectId]/[memoryId]/+page.svelte +++ b/web/workspace/src/routes/w/[workspaceId]/memory/[subjectId]/[memoryId]/+page.svelte @@ -63,6 +63,11 @@ return value.replaceAll('_', ' '); } + function titleLabel(value: string): string { + const normalized = label(value); + return normalized[0].toUpperCase() + normalized.slice(1); + } + function rangeLabel(range: [number, number] | null | undefined): string { return range ? `${range[0]}–${range[1]}` : 'None'; } @@ -98,8 +103,8 @@
- {memory.state} - {label(memory.kind)} + {titleLabel(memory.state)} + {titleLabel(memory.kind)} {#if memory.revision !== memory.current_revision}Historical revision{/if}

{memory.claim}

@@ -109,11 +114,11 @@
-
Revision
{memory.revision} of {memory.current_revision}
-
State
{memory.state}
-
Kind
{label(memory.kind)}
+
Memory revision
{memory.revision} of {memory.current_revision}
+
Status
{titleLabel(memory.state)}
+
Type
{titleLabel(memory.kind)}
Created
-
Updated
+
Last changed
@@ -140,7 +145,7 @@

Audit context

Candidate provenance

- {memory.source_candidates.length} candidate{memory.source_candidates.length === 1 ? '' : 's'} + {memory.source_candidates.length} shown
{#if memory.source_candidates.length === 0} @@ -226,7 +231,11 @@
-

Immutable record

Revision history

{#if data.revisions.data}{revisions.length}{data.revisions.data.has_more ? '+' : ''}{/if}
+
+

Immutable record

Revision history

+ {#if data.revisions.data}{revisions.length} shown{data.revisions.data.has_more ? ' · more available' : ''}{/if} +
+

These are versions of this Memory record, separate from the Subject’s internal update number.

{#if data.revisions.data} {#if revisions.length === 0}
@@ -238,7 +247,7 @@ {#each revisions as revision (revision.revision)}
  • -
    Revision {revision.revision}{revision.state}{label(revision.kind)}
    +
    Memory revision {revision.revision}{titleLabel(revision.state)}{titleLabel(revision.kind)}

    {revision.claim}

    {revision.change_reason} ·     @@ -386,6 +395,7 @@ } .memory-facts { + display: grid; grid-template-columns: repeat(5, minmax(0, 1fr)); gap: var(--space-3); } @@ -414,6 +424,7 @@ .why-useful, .staleness, .change-reason, + .section-intro, .empty-copy, .bounded-note { margin: 0; diff --git a/web/workspace/src/routes/w/[workspaceId]/memory/memory-page.browser.spec.ts b/web/workspace/src/routes/w/[workspaceId]/memory/memory-page.browser.spec.ts index cc767fa43..5166dc3b0 100644 --- a/web/workspace/src/routes/w/[workspaceId]/memory/memory-page.browser.spec.ts +++ b/web/workspace/src/routes/w/[workspaceId]/memory/memory-page.browser.spec.ts @@ -34,6 +34,8 @@ function subject() { return { id: "subject-1", role: "Release coordinator", + behavior_md: "Prefer explicit evidence.", + behavior_revision: 2, state: "active" as const, store_revision: 9, created_at: "2026-01-01T00:00:00Z", @@ -73,6 +75,9 @@ test("renders the subject index as the Memory product entry", () => { expect(screen.getByRole("link", { name: /Release coordinator/ })).not .toBeNull(); expect(screen.getByText("subject-1")).not.toBeNull(); + expect(screen.getByText("Not connected")).not.toBeNull(); + expect(screen.queryByText("Store revision", { exact: true })).toBeNull(); + expect(screen.getByText("1 shown · more available")).not.toBeNull(); expect(screen.getByRole("link", { name: "First page" }).getAttribute("href")) .toBe("/w/workspace-1/memory"); expect(screen.getByRole("link", { name: /Next page/ }).getAttribute("href")) @@ -96,6 +101,12 @@ test("creates one Subject with the exact role and navigates to its generated det }) as HTMLInputElement; await waitFor(() => expect(document.activeElement).toBe(roleInput)); await fireEvent.input(roleInput, { target: { value: " Review lead " } }); + const behaviorInput = screen.getByRole("textbox", { + name: /Behavior/, + }) as HTMLTextAreaElement; + await fireEvent.input(behaviorInput, { + target: { value: "Challenge unsupported assumptions." }, + }); const submit = screen.getByRole("button", { name: "Create Subject" }); await fireEvent.click(submit); @@ -110,6 +121,8 @@ test("creates one Subject with the exact role and navigates to its generated det resolveRequest(Response.json({ id: "generated-subject-1", role: " Review lead ", + behavior_md: "Challenge unsupported assumptions.", + behavior_revision: 0, state: "active", store_revision: 0, created_at: "2026-01-03T00:00:00Z", @@ -127,7 +140,9 @@ test("creates one Subject with the exact role and navigates to its generated det ]; expect(path).toBe("/api/w/workspace-1/subjektiv/subjects"); expect(init.method).toBe("POST"); - expect(init.body).toBe('{"role":" Review lead "}'); + expect(init.body).toBe( + '{"role":" Review lead ","behavior_md":"Challenge unsupported assumptions."}', + ); }); test("keeps creation available for an empty list and restores focus on cancel", async () => { @@ -146,7 +161,7 @@ test("keeps creation available for an empty list and restores focus on cancel", expect(screen.queryByRole("textbox", { name: "Role" })).toBeNull(); }); -test("preserves invalid and rejected role drafts with accessible errors", async () => { +test("preserves invalid and rejected Subject drafts with accessible field errors", async () => { const fetchMock = vi.fn(() => Promise.resolve(Response.json( { @@ -172,6 +187,20 @@ test("preserves invalid and rejected role drafts with accessible errors", async expect(fetchMock).not.toHaveBeenCalled(); await fireEvent.input(input, { target: { value: "Release reviewer" } }); + const behaviorInput = screen.getByRole("textbox", { + name: /Behavior/, + }) as HTMLTextAreaElement; + await fireEvent.input(behaviorInput, { target: { value: " " } }); + await fireEvent.click(screen.getByRole("button", { name: "Create Subject" })); + expect(screen.getByRole("alert").textContent).toContain( + "empty or contain non-whitespace", + ); + expect(behaviorInput.getAttribute("aria-invalid")).toBe("true"); + expect(fetchMock).not.toHaveBeenCalled(); + + await fireEvent.input(behaviorInput, { + target: { value: "Challenge unsupported assumptions." }, + }); await fireEvent.click(screen.getByRole("button", { name: "Create Subject" })); await waitFor(() => { expect(screen.getByRole("alert").textContent).toContain( @@ -214,7 +243,10 @@ test("renders a ready resident surface and committed Memory lifecycle states", a workspaceId: "workspace-1", subjectId: "subject-1", cursor: "memory-current", - subject: result(subject()), + subject: result({ + ...subject(), + current_worker: { display_name: "Release Worker" }, + }), surface: result({ subject_id: "subject-1", availability: "ready" as const, @@ -267,9 +299,20 @@ test("renders a ready resident surface and committed Memory lifecycle states", a expect(screen.getByRole("heading", { name: "Resident context", level: 1 })) .not.toBeNull(); }); - expect(screen.getByRole("article", { name: "Resident surface" })).not + expect( + screen.getByRole("article", { name: "User-managed Subject behavior" }) + .textContent, + ) + .toContain("Prefer explicit evidence."); + expect(screen.getByRole("article", { name: "Resident context" })).not .toBeNull(); - expect(screen.getAllByText("active", { exact: true }).length).toBeGreaterThan( + expect(screen.getByText("Connected", { exact: true })).not.toBeNull(); + expect(screen.getByText("Release Worker", { exact: true })).not.toBeNull(); + expect(screen.getByText("3 on this page", { exact: true })).not.toBeNull(); + expect(screen.getByText("Subject store revision", { exact: true })).not + .toBeNull(); + expect(screen.getByText(/not a Memory count/)).not.toBeNull(); + expect(screen.getAllByText("Active", { exact: true }).length).toBeGreaterThan( 0, ); expect(screen.getByText("Resolved", { exact: true })).not.toBeNull(); @@ -280,6 +323,359 @@ test("renders a ready resident surface and committed Memory lifecycle states", a .toContain("cursor=memory-next"); }); +test("edits and clears user-managed behavior with CAS while distinguishing save from Worker application", async () => { + const fetchMock = vi.fn(async (_path: string, init?: RequestInit) => { + const request = JSON.parse(String(init?.body)); + return Response.json({ + ...subject(), + behavior_md: request.behavior_md, + behavior_revision: request.expected_behavior_revision + 1, + }); + }); + vi.stubGlobal("fetch", fetchMock); + render(SubjectPage, { + data: { + workspaceId: "workspace-1", + subjectId: "subject-1", + subject: result(subject()), + surface: result({ + subject_id: "subject-1", + availability: "ungenerated" as const, + }), + memories: result({ items: [], has_more: false }), + }, + } as never); + + await fireEvent.click(screen.getByRole("button", { name: "Edit" })); + const textarea = screen.getByRole("textbox", { name: "Subject behavior" }); + await fireEvent.input(textarea, { + target: { value: "Ask before irreversible actions." }, + }); + await fireEvent.click(screen.getByRole("button", { name: "Save behavior" })); + await waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(1)); + const [path, init] = fetchMock.mock.calls[0] as unknown as [ + string, + RequestInit, + ]; + expect(path).toBe("/api/w/workspace-1/subjektiv/subjects/subject-1/behavior"); + expect(init.method).toBe("PATCH"); + expect(init.body).toBe( + '{"expected_behavior_revision":2,"behavior_md":"Ask before irreversible actions."}', + ); + await waitFor(() => { + expect( + screen.getByText(/Behavior storage confirmed at revision 3/).textContent, + ) + .toContain("does not confirm application"); + }); + expect( + screen.getByRole("article", { name: "User-managed Subject behavior" }) + .textContent, + ) + .toContain("Ask before irreversible actions."); + + await fireEvent.click(screen.getByRole("button", { name: "Edit" })); + await fireEvent.click(screen.getByRole("button", { name: "Clear" })); + await fireEvent.click(screen.getByRole("button", { name: "Save behavior" })); + await waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(2)); + const secondBody = + (fetchMock.mock.calls[1] as unknown as [string, RequestInit])[1].body; + expect(secondBody).toBe('{"expected_behavior_revision":3,"behavior_md":""}'); + await waitFor(() => + expect(screen.getByText("No behavior is set.")).not.toBeNull() + ); +}); + +function subjectPageData(record = subject(), workspaceId = "workspace-1") { + return { + workspaceId, + subjectId: record.id, + subject: result(record), + surface: result({ + subject_id: record.id, + availability: "ungenerated" as const, + }), + memories: result({ items: [], has_more: false }), + }; +} + +test.each([ + ["subject-2", "workspace-1"], + ["subject-1", "workspace-2"], +])( + "resets reused Subject editor for %s in %s and saves only its own behavior", + async (id, workspaceId) => { + const nextSubject = { ...subject(), id, behavior_md: "Only B behavior." }; + const fetchMock = vi.fn(async (_path: string, init?: RequestInit) => { + const request = JSON.parse(String(init?.body)); + return Response.json({ + ...nextSubject, + behavior_md: request.behavior_md, + behavior_revision: request.expected_behavior_revision + 1, + }); + }); + vi.stubGlobal("fetch", fetchMock); + const view = render(SubjectPage, { data: subjectPageData() } as never); + await fireEvent.click(screen.getByRole("button", { name: "Edit" })); + await fireEvent.input( + screen.getByRole("textbox", { name: "Subject behavior" }), + { + target: { value: "Unsaved A draft" }, + }, + ); + + await view.rerender( + { data: subjectPageData(nextSubject, workspaceId) } as never, + ); + expect(screen.queryByRole("textbox", { name: "Subject behavior" })) + .toBeNull(); + expect( + screen.getByRole("article", { name: "User-managed Subject behavior" }) + .textContent, + ) + .toContain("Only B behavior."); + await fireEvent.click(screen.getByRole("button", { name: "Edit" })); + expect( + (screen.getByRole("textbox", { + name: "Subject behavior", + }) as HTMLTextAreaElement).value, + ) + .toBe("Only B behavior."); + await fireEvent.click( + screen.getByRole("button", { name: "Save behavior" }), + ); + await waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(1)); + const [path, init] = fetchMock.mock.calls[0] as unknown as [ + string, + RequestInit, + ]; + expect(path).toBe( + `/api/w/${workspaceId}/subjektiv/subjects/${id}/behavior`, + ); + expect(JSON.parse(String(init.body))).toEqual({ + expected_behavior_revision: 2, + behavior_md: "Only B behavior.", + }); + await waitFor(() => + expect(screen.getByText(/storage confirmed at revision 3/)).not.toBeNull() + ); + }, +); + +test.each(["success", "failure"])( + "fences late %s from a previous Subject without clearing the new save state", + async (outcome) => { + const pending: Array<(response: Response) => void> = []; + const fetchMock = vi.fn(() => + new Promise((resolve) => pending.push(resolve)) + ); + vi.stubGlobal("fetch", fetchMock); + const view = render(SubjectPage, { data: subjectPageData() } as never); + await fireEvent.click(screen.getByRole("button", { name: "Edit" })); + await fireEvent.click( + screen.getByRole("button", { name: "Save behavior" }), + ); + await waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(1)); + + const nextSubject = { + ...subject(), + id: "subject-2", + behavior_md: "Only B behavior.", + }; + await view.rerender({ data: subjectPageData(nextSubject) } as never); + await fireEvent.click(screen.getByRole("button", { name: "Edit" })); + await fireEvent.input( + screen.getByRole("textbox", { name: "Subject behavior" }), + { + target: { value: "Updated B behavior." }, + }, + ); + await fireEvent.click( + screen.getByRole("button", { name: "Save behavior" }), + ); + await waitFor(() => expect(fetchMock).toHaveBeenCalledTimes(2)); + pending[0]( + outcome === "success" + ? Response.json({ + ...subject(), + behavior_md: "Late A response", + behavior_revision: 3, + }) + : Response.json({ + error: "Forbidden", + message: "A denied", + diagnostics: [], + }, { status: 403 }), + ); + // Drain the previous response through its success/catch/finally branches. + await new Promise((resolve) => setTimeout(resolve, 0)); + expect( + screen.getByRole("button", { name: "Saving…" }).hasAttribute("disabled"), + ).toBe(true); + expect( + (screen.getByRole("textbox", { + name: "Subject behavior", + }) as HTMLTextAreaElement).value, + ) + .toBe("Updated B behavior."); + expect(screen.queryByRole("alert")).toBeNull(); + expect(screen.queryByText(/storage confirmed/)).toBeNull(); + pending[1]( + Response.json({ + ...nextSubject, + behavior_md: "Updated B behavior.", + behavior_revision: 3, + }), + ); + await waitFor(() => + expect(screen.getByText(/storage confirmed at revision 3/)).not.toBeNull() + ); + expect( + screen.getByRole("article", { name: "User-managed Subject behavior" }) + .textContent, + ) + .toContain("Updated B behavior."); + }, +); + +test("a refreshed loader snapshot fences old saves even after returning to the same Subject", async () => { + let resolveRequest!: (response: Response) => void; + vi.stubGlobal( + "fetch", + vi.fn(() => + new Promise((resolve) => { + resolveRequest = resolve; + }) + ), + ); + const view = render(SubjectPage, { data: subjectPageData() } as never); + await fireEvent.click(screen.getByRole("button", { name: "Edit" })); + await fireEvent.click(screen.getByRole("button", { name: "Save behavior" })); + await view.rerender( + { data: subjectPageData({ ...subject(), id: "subject-2" }) } as never, + ); + const fresh = { + ...subject(), + behavior_md: "Fresh A loader", + behavior_revision: 4, + }; + await view.rerender({ data: subjectPageData(fresh) } as never); + resolveRequest( + Response.json({ + ...subject(), + behavior_md: "Stale A response", + behavior_revision: 3, + }), + ); + await new Promise((resolve) => setTimeout(resolve, 0)); + expect( + screen.getByRole("article", { name: "User-managed Subject behavior" }) + .textContent, + ) + .toContain("Fresh A loader"); + expect(screen.queryByText(/storage confirmed/)).toBeNull(); + + await view.rerender( + { + data: { + ...subjectPageData(), + subject: result(null, "Subject unavailable"), + }, + } as never, + ); + expect(screen.queryByRole("button", { name: "Edit" })).toBeNull(); + expect( + screen.queryByRole("article", { name: "User-managed Subject behavior" }), + ).toBeNull(); +}); + +test("candidate-only Subjects show no committed Memories, not an inferred candidate count", () => { + // The store regression candidate_decisions_reject_conflicts_and_roll_back_partial_writes + // stages a candidate without committing a Memory. Its public read projection is + // the same empty committed list as a new Subject; candidates are not a page input. + render(SubjectPage, { + data: { + workspaceId: "workspace-1", + subjectId: "subject-1", + cursor: null, + subject: result({ ...subject(), store_revision: 0 }), + surface: result({ subject_id: "subject-1", availability: "ungenerated" }), + memories: result({ limit: 100, items: [], has_more: false }), + }, + } as never); + + expect(screen.getByText("None", { exact: true })).not.toBeNull(); + expect(screen.getByText("No committed Memories yet.")).not.toBeNull(); + expect(screen.getByText("Not connected", { exact: true })).not.toBeNull(); + expect(screen.getAllByText("Not generated", { exact: true })).toHaveLength(2); + expect(screen.queryByRole("article", { name: "Resident context" })) + .toBeNull(); + expect( + screen.queryByText( + /candidate count|all Memories lost|automatically updating/i, + ), + ).toBeNull(); +}); + +test.each(["ready", "ungenerated", "stale", "failed"] as const)( + "user behavior and diagnostic disclosures stay independent of %s generated Memory", + async (availability) => { + const record = { ...subject(), store_revision: 0 }; + const view = render(SubjectPage, { + data: { + ...subjectPageData(record), + surface: result({ + subject_id: record.id, + availability, + ...(availability === "ready" + ? { + snapshot: { + snapshot_id: "empty-snapshot", + body_md: "", + memory_refs: [], + built_from_store_revision: 0, + created_at: "2026-01-02T03:04:05Z", + }, + } + : {}), + }), + }, + } as never); + expect( + screen.getByRole("article", { name: "User-managed Subject behavior" }) + .textContent, + ) + .toContain(record.behavior_md); + expect(screen.getByText("Generated Memory context", { exact: true })).not + .toBeNull(); + expect(screen.getByText("No committed Memories yet.")).not.toBeNull(); + expect(screen.getByText("Not connected", { exact: true })).not.toBeNull(); + const technical = view.container.querySelector( + "details.subject-technical-details", + ) as HTMLDetailsElement; + expect(technical.open).toBe(false); + expect(technical.textContent).toContain("Subject store revision"); + if (availability === "ready") { + expect(screen.getByText("Resident context is current but empty.")).not + .toBeNull(); + expect(screen.getByText(/no generated Memory context to show/)).not + .toBeNull(); + expect( + view.container.querySelector("details.surface-details")?.textContent, + ) + .toContain("Generated from Subject revision"); + } + await fireEvent.click(screen.getByRole("button", { name: "Edit" })); + await fireEvent.click(screen.getByRole("button", { name: "Cancel" })); + expect( + screen.getByRole("article", { name: "User-managed Subject behavior" }) + .textContent, + ) + .toContain(record.behavior_md); + expect(technical.open).toBe(false); + }, +); + test("distinguishes ready-empty, stale, failed, unavailable, and request error surfaces", async () => { const base = { workspaceId: "workspace-1", @@ -303,8 +699,25 @@ test("distinguishes ready-empty, stale, failed, unavailable, and request error s }), }, } as never); - expect(screen.getByText("Resident surface is ready and empty.")).not + expect(screen.getByText("Resident context is current but empty.")).not .toBeNull(); + expect(screen.getByText(/current empty surface exists/)).not.toBeNull(); + expect(screen.getByText("No committed Memories yet.")).not.toBeNull(); + + await view.rerender( + { + data: { + ...base, + surface: result({ + subject_id: "subject-1", + availability: "ungenerated" as const, + }), + }, + } as never, + ); + expect(screen.getByText("Resident context has not been generated.")).not + .toBeNull(); + expect(screen.getAllByText("Not generated", { exact: true })).toHaveLength(2); await view.rerender( { @@ -317,7 +730,8 @@ test("distinguishes ready-empty, stale, failed, unavailable, and request error s }, } as never, ); - expect(screen.getByText("Resident surface is stale.")).not.toBeNull(); + expect(screen.getByText("Resident context needs to be refreshed.")).not + .toBeNull(); await view.rerender( { @@ -333,7 +747,7 @@ test("distinguishes ready-empty, stale, failed, unavailable, and request error s expect(screen.getByRole("alert").textContent).toContain("generation failed"); await view.rerender({ data: { ...base, surface: result(null) } } as never); - expect(screen.getByText("Resident surface data is unavailable.")).not + expect(screen.getByText("Resident context status is unavailable.")).not .toBeNull(); await view.rerender( @@ -342,7 +756,7 @@ test("distinguishes ready-empty, stale, failed, unavailable, and request error s } as never, ); expect(screen.getByRole("alert").textContent).toContain( - "Resident surface unavailable", + "Resident context status unavailable", ); }); diff --git a/web/workspace/src/routes/w/[workspaceId]/workers/+page.svelte b/web/workspace/src/routes/w/[workspaceId]/workers/+page.svelte index dc9fb37ea..98ea24340 100644 --- a/web/workspace/src/routes/w/[workspaceId]/workers/+page.svelte +++ b/web/workspace/src/routes/w/[workspaceId]/workers/+page.svelte @@ -1,4 +1,7 @@