fix: merge develop and reconcile schema histories
Preserve both published schema-103 histories and converge their Passkey results and Job input retention features transactionally at schema 104. Retain local Reviewer, integration-event, auth and sidebar fixes while adopting develop restore and E2E updates; regenerate API contracts. Validation: schema convergence regressions, full Server/Worker/Runtime suites, server-api and merge-request tests, Web unit/component tests, root cargo check, E2E harness tests and restore/auth target compilation, generated artifacts, fmt and naming checks. Product-process E2E not rerun; live services and databases untouched.
This commit is contained in:
@@ -11,7 +11,6 @@ default = []
|
||||
|
||||
[dependencies]
|
||||
async-trait = { workspace = true }
|
||||
clap = { version = "4.6.0", features = ["derive"] }
|
||||
agen = { workspace = true, features = ["codex"] }
|
||||
session-store = { workspace = true }
|
||||
secrets = { workspace = true }
|
||||
@@ -58,6 +57,7 @@ wip-protocol = { workspace = true }
|
||||
wip-text-view = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
tracing-subscriber.workspace = true
|
||||
axum = { workspace = true }
|
||||
workdir = { workspace = true, features = ["http-client"] }
|
||||
dotenv = "0.15.0"
|
||||
|
||||
@@ -206,26 +206,36 @@ impl AuthorizedWorkdirSession {
|
||||
.map(CheckoutSearchResult::Grep),
|
||||
};
|
||||
};
|
||||
scoped
|
||||
.authorize_scope_path(WorkdirScopeAuthorizationRequest {
|
||||
rules: vec![WorkdirToolScopeRule {
|
||||
target: WorkdirPath::root(),
|
||||
..rule.clone()
|
||||
}],
|
||||
path: relative_path(&root, operation.path())?,
|
||||
permission: WorkdirToolScopePermission::Read,
|
||||
})
|
||||
.await?;
|
||||
// Lease cwd rebases paths. Use the same provider scoped-traversal contract
|
||||
// with attachment output_root instead, retaining typed paths, cursors and
|
||||
// Grep's rendered filenames without parsing or post-filtering output.
|
||||
let mut request = CheckoutSearchRequest::new(operation);
|
||||
request.scope_layers.push(vec![rule]);
|
||||
self.session.checkout_search(request).await
|
||||
self.search_with_scope(scoped, root, rule, CheckoutSearchRequest::new(operation))
|
||||
.await
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn search_with_scope(
|
||||
&self,
|
||||
scoped: WorkdirSessionHandle,
|
||||
root: WorkdirPath,
|
||||
rule: WorkdirToolScopeRule,
|
||||
mut request: CheckoutSearchRequest,
|
||||
) -> Result<CheckoutSearchResult, WorkdirError> {
|
||||
scoped
|
||||
.authorize_scope_path(WorkdirScopeAuthorizationRequest {
|
||||
rules: vec![WorkdirToolScopeRule {
|
||||
target: WorkdirPath::root(),
|
||||
..rule.clone()
|
||||
}],
|
||||
path: relative_path(&root, request.operation.path())?,
|
||||
permission: WorkdirToolScopePermission::Read,
|
||||
})
|
||||
.await?;
|
||||
// Preserve the caller's coordinate root, cursors and all existing
|
||||
// attenuation layers. The provider applies the Host ceiling during
|
||||
// traversal, not by filtering already-rendered filenames afterward.
|
||||
request.scope_layers.push(vec![rule]);
|
||||
self.session.checkout_search(request).await
|
||||
}
|
||||
|
||||
fn authorize_command(&self) -> Result<(), WorkdirError> {
|
||||
// cwd is not a sandbox: arbitrary shell can access the entire attachment.
|
||||
self.authorize(COMMAND, &WorkdirPath::root(), true)
|
||||
@@ -261,6 +271,12 @@ impl WorkdirSession for AuthorizedWorkdirSession {
|
||||
self.session.capabilities()
|
||||
}
|
||||
|
||||
fn environment(&self) -> workdir::WorkdirSessionEnvironment {
|
||||
let mut environment = self.session.environment();
|
||||
environment.capabilities = environment.capabilities.intersection(self.capabilities());
|
||||
environment
|
||||
}
|
||||
|
||||
async fn authorize_scope_path(
|
||||
&self,
|
||||
mut request: WorkdirScopeAuthorizationRequest,
|
||||
@@ -296,8 +312,14 @@ impl WorkdirSession for AuthorizedWorkdirSession {
|
||||
&self,
|
||||
request: CheckoutSearchRequest,
|
||||
) -> Result<CheckoutSearchResult, WorkdirError> {
|
||||
self.require_legacy(READ)?;
|
||||
self.session.checkout_search(request).await
|
||||
let path = request.operation.path().clone();
|
||||
self.with_authorized(READ, &path, true, |scoped, root, rule| async move {
|
||||
match rule {
|
||||
Some(rule) => self.search_with_scope(scoped, root, rule, request).await,
|
||||
None => self.session.checkout_search(request).await,
|
||||
}
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn checkout_observe(
|
||||
@@ -490,6 +512,8 @@ mod tests {
|
||||
//! Component tests connect this adapter to the real LocalWorkdirSession.
|
||||
//! Only the Host authorization transport/policy is replaced; these are not E2E tests.
|
||||
|
||||
mod environment;
|
||||
|
||||
use super::*;
|
||||
use crate::{WorkspaceClientError, WorkspaceRequest, WorkspaceResponse};
|
||||
use server_api::authorization::ScopedPermission;
|
||||
@@ -842,18 +866,6 @@ mod tests {
|
||||
.await,
|
||||
Err(WorkdirError::Denied(_))
|
||||
));
|
||||
assert!(matches!(
|
||||
session
|
||||
.checkout_search(CheckoutSearchRequest::new(CheckoutSearchOperation::List(
|
||||
ListRequest {
|
||||
path: WorkdirPath::root(),
|
||||
after: None,
|
||||
limit: 10,
|
||||
}
|
||||
)))
|
||||
.await,
|
||||
Err(WorkdirError::Denied(_))
|
||||
));
|
||||
let rule = WorkdirToolScopeRule {
|
||||
target: WorkdirPath::root(),
|
||||
permission: WorkdirToolScopePermission::Read,
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
//! Normal lib tests connecting live Host-grant checks to real provider traversal
|
||||
//! and router cwd projection. Only the Host transport is replaced by the parent
|
||||
//! module's grant fixture; no commands, network or ambient Workspace state are used.
|
||||
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn authorization_wrapper_preserves_inner_environment_and_capability_ceiling() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
std::fs::create_dir(root.path().join("src")).unwrap();
|
||||
let provider = Arc::new(LocalWorkdirSession::materialized(
|
||||
root.path().to_path_buf(),
|
||||
root.path().to_path_buf(),
|
||||
manifest::SharedScope::new(manifest::Scope::writable(root.path()).unwrap()),
|
||||
WorkdirSessionCapabilities::READ_ONLY,
|
||||
));
|
||||
let broker = WorkdirToolBroker::at_cwd(provider, WorkdirPath::new("src").unwrap()).unwrap();
|
||||
let inner = broker.tool_session();
|
||||
let expected = inner.environment();
|
||||
let client = AuthorizationClient::new(vec![permission(READ, "src", true)]);
|
||||
let wrapped = authorized_workdir_session(inner, client.clone(), "attachment-id".into());
|
||||
assert_eq!(wrapped.environment(), expected);
|
||||
assert_eq!(wrapped.environment().cwd.as_str(), "src");
|
||||
assert_eq!(
|
||||
wrapped.environment().capabilities,
|
||||
WorkdirSessionCapabilities::READ_ONLY
|
||||
);
|
||||
assert!(
|
||||
client.requests.lock().unwrap().is_empty(),
|
||||
"environment is metadata, not a cached grant decision"
|
||||
);
|
||||
wrapped.close().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn nonroot_routed_search_retains_host_grants_output_coordinates_and_cursors() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
std::fs::create_dir(root.path().join("src")).unwrap();
|
||||
std::fs::create_dir(root.path().join("private")).unwrap();
|
||||
for name in ["a.txt", "b.txt"] {
|
||||
std::fs::write(root.path().join("src").join(name), "needle public marker").unwrap();
|
||||
}
|
||||
std::fs::write(
|
||||
root.path().join("private/secret.txt"),
|
||||
"needle private marker",
|
||||
)
|
||||
.unwrap();
|
||||
let client = AuthorizationClient::new(vec![permission(READ, "src", true)]);
|
||||
let authorized = session(&root, client.clone());
|
||||
let router = workdir::WorkdirSessionRouter::new();
|
||||
let alias = workdir::WorkdirAttachmentAlias::new("checkout").unwrap();
|
||||
router
|
||||
.attach_at_cwd(alias.clone(), authorized, WorkdirPath::new("src").unwrap())
|
||||
.unwrap();
|
||||
let routed = router.resolve(Some("checkout")).unwrap().session;
|
||||
assert_eq!(routed.environment().cwd.as_str(), "src");
|
||||
let paths = routed
|
||||
.glob(GlobRequest {
|
||||
path: WorkdirPath::root(),
|
||||
pattern: "**/*.txt".into(),
|
||||
limit: 100,
|
||||
})
|
||||
.await
|
||||
.unwrap()
|
||||
.paths;
|
||||
assert_eq!(
|
||||
paths,
|
||||
vec![
|
||||
WorkdirPath::new("a.txt").unwrap(),
|
||||
WorkdirPath::new("b.txt").unwrap()
|
||||
]
|
||||
);
|
||||
// Search output is reusable as a path without adding src/ at the caller.
|
||||
assert_eq!(
|
||||
routed.read(read(paths[0].as_str())).await.unwrap().bytes,
|
||||
b"needle public marker"
|
||||
);
|
||||
for output_mode in [
|
||||
workdir::GrepOutputMode::Content,
|
||||
workdir::GrepOutputMode::FilesWithMatches,
|
||||
workdir::GrepOutputMode::Count,
|
||||
] {
|
||||
let result = routed
|
||||
.grep(GrepRequest {
|
||||
path: WorkdirPath::root(),
|
||||
pattern: "needle".into(),
|
||||
glob: None,
|
||||
file_type: None,
|
||||
case_insensitive: false,
|
||||
before_context: 0,
|
||||
after_context: 0,
|
||||
multiline: false,
|
||||
output_mode,
|
||||
limit: 100,
|
||||
offset: 0,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let mut grep_paths = result.paths;
|
||||
grep_paths.sort_by(|left, right| left.as_str().cmp(right.as_str()));
|
||||
assert_eq!(grep_paths, paths);
|
||||
assert!(result.output.contains("a.txt"));
|
||||
assert!(!result.output.contains("src/"));
|
||||
assert!(!result.output.contains("secret"));
|
||||
}
|
||||
let first = routed
|
||||
.list(ListRequest {
|
||||
path: WorkdirPath::root(),
|
||||
after: None,
|
||||
limit: 1,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(first.entries[0].path.as_str(), "a.txt");
|
||||
assert_eq!(first.next_after.as_ref().unwrap().path.as_str(), "a.txt");
|
||||
let second = routed
|
||||
.list(ListRequest {
|
||||
path: WorkdirPath::root(),
|
||||
after: first.next_after,
|
||||
limit: 1,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(second.entries[0].path.as_str(), "b.txt");
|
||||
assert!(second.next_after.is_none());
|
||||
|
||||
// An existing caller ceiling must survive the extra Host layer. An empty
|
||||
// layer is invalid and cannot silently turn into the broader Host grant.
|
||||
let mut denied = CheckoutSearchRequest::new(CheckoutSearchOperation::Glob(GlobRequest {
|
||||
path: WorkdirPath::root(),
|
||||
pattern: "**/*.txt".into(),
|
||||
limit: 100,
|
||||
}));
|
||||
denied.scope_layers.push(Vec::new());
|
||||
assert!(routed.checkout_search(denied).await.is_err());
|
||||
assert!(client.requests.lock().unwrap().iter().all(|request| {
|
||||
matches!(&request.resource, PermissionResource::Workdir { path, .. } if path == "src" || path == "src/a.txt")
|
||||
}));
|
||||
client.revoked.store(true, Ordering::SeqCst);
|
||||
assert!(
|
||||
routed
|
||||
.glob(GlobRequest {
|
||||
path: WorkdirPath::root(),
|
||||
pattern: "**/*.txt".into(),
|
||||
limit: 100,
|
||||
})
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
router.detach(&alias).await.unwrap();
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test]
|
||||
async fn scoped_checkout_search_preserves_caller_layer_and_rejects_resolved_escape() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
std::fs::create_dir_all(root.path().join("src/nested")).unwrap();
|
||||
std::fs::create_dir(root.path().join("private")).unwrap();
|
||||
std::fs::write(root.path().join("src/public.txt"), "needle public").unwrap();
|
||||
std::fs::write(root.path().join("src/nested/hidden.txt"), "needle hidden").unwrap();
|
||||
std::fs::write(root.path().join("private/secret.txt"), "needle secret").unwrap();
|
||||
std::os::unix::fs::symlink("../private", root.path().join("src/escape")).unwrap();
|
||||
std::os::unix::fs::symlink("../private/secret.txt", root.path().join("src/alias.txt")).unwrap();
|
||||
let client = AuthorizationClient::new(vec![permission(READ, "src", true)]);
|
||||
let authorized = session(&root, client);
|
||||
let request = CheckoutSearchRequest {
|
||||
operation: CheckoutSearchOperation::Glob(GlobRequest {
|
||||
path: WorkdirPath::new("src").unwrap(),
|
||||
pattern: "**/*.txt".into(),
|
||||
limit: 100,
|
||||
}),
|
||||
output_root: WorkdirPath::new("src").unwrap(),
|
||||
scope_layers: vec![vec![WorkdirToolScopeRule {
|
||||
target: WorkdirPath::new("src").unwrap(),
|
||||
recursive: false,
|
||||
permission: WorkdirToolScopePermission::Read,
|
||||
symlink_policy: manifest::SymlinkPolicy::Resolved,
|
||||
}]],
|
||||
};
|
||||
let result = authorized.checkout_search(request).await.unwrap();
|
||||
let CheckoutSearchResult::Glob(result) = result else {
|
||||
panic!("expected Glob result")
|
||||
};
|
||||
assert_eq!(result.paths, vec![WorkdirPath::new("public.txt").unwrap()]);
|
||||
assert!(
|
||||
authorized
|
||||
.checkout_search(CheckoutSearchRequest::new(CheckoutSearchOperation::Glob(
|
||||
GlobRequest {
|
||||
path: WorkdirPath::new("private").unwrap(),
|
||||
pattern: "**/*.txt".into(),
|
||||
limit: 100,
|
||||
}
|
||||
)))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
authorized.close().await.unwrap();
|
||||
}
|
||||
+272
-28
@@ -6,9 +6,8 @@ use thiserror::Error;
|
||||
use workdir::WorkdirSessionHandle;
|
||||
|
||||
use crate::PromptCatalogSource;
|
||||
use crate::controller::{
|
||||
ShutdownReceiver, WorkerController, WorkerControllerTransport, WorkerHandle,
|
||||
};
|
||||
use crate::controller::{ShutdownReceiver, WorkerController, WorkerHandle};
|
||||
pub use crate::worker::{PreparedRestoredWorker, RestoreCandidateWorkdirCleanup};
|
||||
use crate::worker::{Worker, WorkerError, WorkerFilesystemAuthority, WorkerWorkspaceContext};
|
||||
use manifest::WorkerManifest;
|
||||
|
||||
@@ -30,7 +29,6 @@ pub struct WorkerBootstrap<St> {
|
||||
workspace_context: WorkerWorkspaceContext,
|
||||
filesystem_authority: WorkerFilesystemAuthority,
|
||||
layout: WorkerBootstrapLayout,
|
||||
transport: WorkerControllerTransport,
|
||||
model_client: Option<Box<dyn LlmClient>>,
|
||||
workdir_session: Option<WorkdirSessionHandle>,
|
||||
subjektiv_host: Option<std::sync::Arc<dyn crate::subjektiv::SubjektivHost>>,
|
||||
@@ -41,7 +39,6 @@ pub struct WorkerBootstrap<St> {
|
||||
pub struct PreparedWorker<C: LlmClient, St: Store> {
|
||||
worker: Worker<C, St>,
|
||||
layout: WorkerBootstrapLayout,
|
||||
transport: WorkerControllerTransport,
|
||||
}
|
||||
|
||||
/// Live controller returned only after Worker construction and feature
|
||||
@@ -68,7 +65,6 @@ impl<St> WorkerBootstrap<St>
|
||||
where
|
||||
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||
{
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn new(
|
||||
manifest: WorkerManifest,
|
||||
store: St,
|
||||
@@ -76,7 +72,6 @@ where
|
||||
workspace_context: WorkerWorkspaceContext,
|
||||
filesystem_authority: WorkerFilesystemAuthority,
|
||||
layout: WorkerBootstrapLayout,
|
||||
transport: WorkerControllerTransport,
|
||||
) -> Self {
|
||||
Self {
|
||||
manifest,
|
||||
@@ -85,7 +80,6 @@ where
|
||||
workspace_context,
|
||||
filesystem_authority,
|
||||
layout,
|
||||
transport,
|
||||
model_client: None,
|
||||
workdir_session: None,
|
||||
subjektiv_host: None,
|
||||
@@ -165,7 +159,39 @@ where
|
||||
)
|
||||
.map_err(WorkerBootstrapError::Worker)?;
|
||||
}
|
||||
Ok(PreparedWorker::new(worker, self.layout, self.transport))
|
||||
Ok(PreparedWorker::new(worker, self.layout))
|
||||
}
|
||||
|
||||
/// Reconstruct a saved materialized Worker without allocation, attribution
|
||||
/// repair, Feature installation, or controller exposure. The caller owns the
|
||||
/// later controller layout and must activate before starting it.
|
||||
pub async fn prepare_restored_candidate(
|
||||
self,
|
||||
worker_name: &str,
|
||||
) -> Result<PreparedRestoredWorker<St>, WorkerBootstrapError> {
|
||||
let mut candidate =
|
||||
Worker::prepare_restored_from_worker_metadata_with_context_and_model_client(
|
||||
worker_name,
|
||||
self.manifest,
|
||||
self.store,
|
||||
self.prompt_catalog,
|
||||
self.workspace_context,
|
||||
self.filesystem_authority,
|
||||
self.model_client,
|
||||
)
|
||||
.await
|
||||
.map_err(WorkerBootstrapError::Worker)?;
|
||||
if let Some(workdir_session) = self.workdir_session {
|
||||
candidate
|
||||
.worker_mut()
|
||||
.map_err(WorkerBootstrapError::Worker)?
|
||||
.bind_single_workdir_session(Some(workdir_session));
|
||||
}
|
||||
candidate
|
||||
.worker_mut()
|
||||
.map_err(WorkerBootstrapError::Worker)?
|
||||
.bind_subjektiv_host(self.subjektiv_host);
|
||||
Ok(candidate)
|
||||
}
|
||||
|
||||
pub async fn prepare_restored(
|
||||
@@ -206,7 +232,7 @@ where
|
||||
)
|
||||
.map_err(WorkerBootstrapError::Worker)?;
|
||||
}
|
||||
Ok(PreparedWorker::new(worker, self.layout, self.transport))
|
||||
Ok(PreparedWorker::new(worker, self.layout))
|
||||
}
|
||||
|
||||
pub async fn start(self) -> Result<BootstrappedWorker, WorkerBootstrapError> {
|
||||
@@ -221,16 +247,8 @@ where
|
||||
{
|
||||
/// Wrap a restored Worker in the same pre-exposure lifecycle used by fresh
|
||||
/// bootstraps.
|
||||
pub fn new(
|
||||
worker: Worker<C, St>,
|
||||
layout: WorkerBootstrapLayout,
|
||||
transport: WorkerControllerTransport,
|
||||
) -> Self {
|
||||
Self {
|
||||
worker,
|
||||
layout,
|
||||
transport,
|
||||
}
|
||||
pub fn new(worker: Worker<C, St>, layout: WorkerBootstrapLayout) -> Self {
|
||||
Self { worker, layout }
|
||||
}
|
||||
|
||||
pub fn worker(&self) -> &Worker<C, St> {
|
||||
@@ -242,7 +260,7 @@ where
|
||||
}
|
||||
|
||||
pub async fn start(self) -> Result<BootstrappedWorker, WorkerBootstrapError> {
|
||||
start_worker_controller(self.worker, self.layout, self.transport).await
|
||||
start_worker_controller(self.worker, self.layout).await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -252,22 +270,19 @@ where
|
||||
pub async fn start_worker_controller<C, St>(
|
||||
worker: Worker<C, St>,
|
||||
layout: WorkerBootstrapLayout,
|
||||
transport: WorkerControllerTransport,
|
||||
) -> Result<BootstrappedWorker, WorkerBootstrapError>
|
||||
where
|
||||
C: LlmClient + Clone + 'static,
|
||||
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||
{
|
||||
let cleanup_sessions = worker.workdir_sessions();
|
||||
let factory_owned_cleanup = worker.startup_feature_cleanup_handle().is_some();
|
||||
let controller = match layout {
|
||||
WorkerBootstrapLayout::Direct { runtime_base } => {
|
||||
WorkerController::spawn_with_transport_owned(worker, &runtime_base, transport).await
|
||||
WorkerController::spawn_owned(worker, &runtime_base).await
|
||||
}
|
||||
WorkerBootstrapLayout::RuntimeManagedRun { run_dir } => {
|
||||
WorkerController::spawn_runtime_managed_run_with_transport_owned(
|
||||
worker, &run_dir, transport,
|
||||
)
|
||||
.await
|
||||
WorkerController::spawn_runtime_managed_run_owned(worker, &run_dir).await
|
||||
}
|
||||
};
|
||||
|
||||
@@ -278,7 +293,10 @@ where
|
||||
controller_task,
|
||||
}),
|
||||
Err(source) => {
|
||||
let cleanup_failed = cleanup_sessions.close_all().await.is_err();
|
||||
// Factory-owned task scopes must settle before sessions close.
|
||||
// Cleanup remains unconfirmed here; preserve the startup source.
|
||||
let cleanup_failed =
|
||||
factory_owned_cleanup || cleanup_sessions.close_all().await.is_err();
|
||||
Err(WorkerBootstrapError::Controller {
|
||||
source,
|
||||
cleanup_failed,
|
||||
@@ -286,3 +304,229 @@ where
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Host bindings for one unpublished restored Controller. These do not expose
|
||||
/// Worker execution or permit arbitrary mutation during preparation.
|
||||
pub struct RestoredControllerBindings {
|
||||
pub backend_job: Option<crate::BackendJobExecutionBinding>,
|
||||
pub workdir_sessions: std::sync::Arc<workdir::WorkdirSessionRouter>,
|
||||
pub observation_provider:
|
||||
Option<std::sync::Arc<dyn crate::feature::builtin::WorkerObservationProvider>>,
|
||||
pub subjektiv_attached: bool,
|
||||
}
|
||||
|
||||
/// The same real receiver, snapshot sources and paired subscription retained
|
||||
/// privately across the Runtime's durable restore decision.
|
||||
pub struct UnpublishedRestoredController<St: Store> {
|
||||
initialized: Option<crate::controller::InitializedWorkerController<Box<dyn LlmClient>, St>>,
|
||||
activation: Option<crate::worker::RestoredWorkerActivation>,
|
||||
streams: Option<crate::ipc::protocol_session::WorkerProtocolSessionStreams>,
|
||||
subjektiv_attached: bool,
|
||||
effects_applied: bool,
|
||||
activation_failed: bool,
|
||||
closing: bool,
|
||||
}
|
||||
|
||||
pub struct ActivatedRestoredController {
|
||||
pub controller: BootstrappedWorker,
|
||||
pub streams: crate::ipc::protocol_session::WorkerProtocolSessionStreams,
|
||||
}
|
||||
|
||||
impl<St> PreparedRestoredWorker<St>
|
||||
where
|
||||
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||
{
|
||||
pub fn workspace_client_handle(
|
||||
&mut self,
|
||||
) -> Result<std::sync::Arc<dyn crate::WorkspaceClient>, WorkerError> {
|
||||
Ok(self.worker_mut()?.workspace_client_handle())
|
||||
}
|
||||
|
||||
pub fn retain_startup_feature_cleanup(
|
||||
&mut self,
|
||||
) -> Result<crate::WorkerStartupFeatureCleanup, WorkerError> {
|
||||
Ok(self.worker_mut()?.retain_startup_feature_cleanup())
|
||||
}
|
||||
|
||||
/// Initialize the real Controller without allocation, canonical restore
|
||||
/// writes, attribution changes, registry reclamation or input consumption.
|
||||
/// MCP discovery and Feature tasks belong to the retained startup cleanup.
|
||||
pub async fn prepare_runtime_controller(
|
||||
mut self,
|
||||
run_dir: &std::path::Path,
|
||||
bindings: RestoredControllerBindings,
|
||||
) -> Result<UnpublishedRestoredController<St>, WorkerBootstrapError> {
|
||||
let worker = self.worker_mut().map_err(WorkerBootstrapError::Worker)?;
|
||||
if let Some(binding) = bindings.backend_job {
|
||||
worker.bind_backend_job(binding).map_err(|error| {
|
||||
WorkerBootstrapError::Worker(WorkerError::FeatureLifecycle(error.into()))
|
||||
})?;
|
||||
}
|
||||
worker
|
||||
.workdir_sessions()
|
||||
.close_all()
|
||||
.await
|
||||
.map_err(|error| {
|
||||
WorkerBootstrapError::Worker(WorkerError::FeatureLifecycle(error.to_string()))
|
||||
})?;
|
||||
worker.bind_workdir_sessions(bindings.workdir_sessions);
|
||||
worker.bind_worker_observation_provider(bindings.observation_provider);
|
||||
let flow_enabled = worker.manifest().feature.flow.enabled;
|
||||
if flow_enabled {
|
||||
let report = worker
|
||||
.install_runtime_flow_transition_feature()
|
||||
.map_err(|error| {
|
||||
WorkerBootstrapError::Worker(WorkerError::FeatureLifecycle(error))
|
||||
})?;
|
||||
if report.reports.iter().any(|report| !report.installed) {
|
||||
return Err(WorkerBootstrapError::Worker(WorkerError::FeatureLifecycle(
|
||||
"Flow transition installation refused".into(),
|
||||
)));
|
||||
}
|
||||
}
|
||||
let (worker, activation) = self
|
||||
.into_controller_parts()
|
||||
.map_err(WorkerBootstrapError::Worker)?;
|
||||
let initialized = WorkerController::initialize_restored(worker, run_dir)
|
||||
.await
|
||||
.map_err(|source| WorkerBootstrapError::Controller {
|
||||
source,
|
||||
cleanup_failed: true,
|
||||
})?;
|
||||
if flow_enabled {
|
||||
initialized.handle.shared_state.enable_flow_transition();
|
||||
}
|
||||
let streams = {
|
||||
let _guard = initialized.handle.protocol_snapshot_publish_guard();
|
||||
crate::ipc::protocol_session::subscribe_worker_protocol_session(&initialized.handle)
|
||||
};
|
||||
Ok(UnpublishedRestoredController {
|
||||
initialized: Some(initialized),
|
||||
activation: Some(activation),
|
||||
streams: Some(streams),
|
||||
subjektiv_attached: bindings.subjektiv_attached,
|
||||
effects_applied: false,
|
||||
activation_failed: false,
|
||||
closing: false,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl<St> UnpublishedRestoredController<St>
|
||||
where
|
||||
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||
{
|
||||
pub fn snapshot(&self) -> &protocol::Event {
|
||||
&self
|
||||
.streams
|
||||
.as_ref()
|
||||
.expect("unpublished restore streams")
|
||||
.snapshot_event
|
||||
}
|
||||
|
||||
/// Only publication failures after confirmed effects can retry on this
|
||||
/// exact owner. Partial activation must be reconciled, never blindly rerun.
|
||||
pub fn can_retry_publication(&self) -> bool {
|
||||
self.initialized.is_some()
|
||||
&& self.effects_applied
|
||||
&& !self.activation_failed
|
||||
&& !self.closing
|
||||
}
|
||||
|
||||
/// Apply canonical restore effects after the host's durable decision, while
|
||||
/// retaining the same private Controller for subsequent publication.
|
||||
pub async fn apply_effects(&mut self) -> Result<(), WorkerBootstrapError> {
|
||||
let initialized = self
|
||||
.initialized
|
||||
.as_mut()
|
||||
.ok_or_else(|| WorkerBootstrapError::Worker(WorkerError::RestoreCandidateClosing))?;
|
||||
if self.activation_failed || self.closing {
|
||||
return Err(WorkerBootstrapError::Worker(
|
||||
WorkerError::RestoreCandidateClosing,
|
||||
));
|
||||
}
|
||||
// Preparation's read-only authorization is not a publication grant.
|
||||
// Revalidate current Host authority after commit, before effects/input,
|
||||
// including a publication retry on this same private Controller.
|
||||
initialized
|
||||
.worker
|
||||
.validate_subjektiv_body_publication()
|
||||
.map_err(WorkerBootstrapError::Worker)?;
|
||||
if !self.effects_applied {
|
||||
self.activation_failed = true;
|
||||
self.activation
|
||||
.as_ref()
|
||||
.expect("restore activation")
|
||||
.activate(&mut initialized.worker)
|
||||
.await
|
||||
.map_err(WorkerBootstrapError::Worker)?;
|
||||
initialized
|
||||
.worker
|
||||
.finalize_subjektiv_session_attribution(
|
||||
self.subjektiv_attached,
|
||||
crate::SubjektivSessionAttributionLifecycle::RestoredSession,
|
||||
)
|
||||
.map_err(WorkerBootstrapError::Worker)?;
|
||||
self.effects_applied = true;
|
||||
self.activation_failed = false;
|
||||
self.activation = None;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn allocation_cleanup(
|
||||
&self,
|
||||
) -> Option<crate::runtime::worker_allocation::ScopeAllocationCleanup> {
|
||||
self.initialized
|
||||
.as_ref()
|
||||
.and_then(|initialized| initialized.worker.restore_allocation_cleanup_handle())
|
||||
}
|
||||
|
||||
pub async fn activate(&mut self) -> Result<ActivatedRestoredController, WorkerBootstrapError> {
|
||||
self.apply_effects().await?;
|
||||
let initialized = self
|
||||
.initialized
|
||||
.as_mut()
|
||||
.expect("same initialized Controller");
|
||||
let publication = initialized.prepare_publication().await.map_err(|source| {
|
||||
WorkerBootstrapError::Controller {
|
||||
source,
|
||||
cleanup_failed: true,
|
||||
}
|
||||
})?;
|
||||
let initialized = self
|
||||
.initialized
|
||||
.take()
|
||||
.expect("same initialized Controller");
|
||||
let (handle, shutdown, controller_task) = initialized.start(publication);
|
||||
Ok(ActivatedRestoredController {
|
||||
controller: BootstrappedWorker {
|
||||
handle,
|
||||
shutdown,
|
||||
controller_task,
|
||||
},
|
||||
streams: self.streams.take().expect("same paired subscription"),
|
||||
})
|
||||
}
|
||||
|
||||
pub async fn discard(&mut self) -> Result<(), WorkerError> {
|
||||
self.closing = true;
|
||||
if let Some(initialized) = self.initialized.as_mut() {
|
||||
initialized
|
||||
.worker
|
||||
.retain_startup_feature_cleanup()
|
||||
.shutdown()
|
||||
.await
|
||||
.map_err(|error| WorkerError::FeatureLifecycle(error.to_string()))?;
|
||||
initialized
|
||||
.worker
|
||||
.workdir_sessions()
|
||||
.close_all()
|
||||
.await
|
||||
.map_err(|error| WorkerError::FeatureLifecycle(error.to_string()))?;
|
||||
}
|
||||
self.streams = None;
|
||||
self.initialized = None;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -121,6 +121,7 @@ async fn open_session(
|
||||
session_id: WorkdirSessionId::new(SESSION).unwrap(),
|
||||
workdir_id: state.session.workdir().id().clone(),
|
||||
capabilities: state.session.capabilities(),
|
||||
environment: state.session.environment(),
|
||||
})
|
||||
.into_response()
|
||||
}
|
||||
@@ -251,6 +252,10 @@ impl Drop for HttpFixture {
|
||||
|
||||
impl HttpFixture {
|
||||
async fn new(capabilities: WorkdirSessionCapabilities) -> Self {
|
||||
Self::at_cwd(capabilities, "").await
|
||||
}
|
||||
|
||||
async fn at_cwd(capabilities: WorkdirSessionCapabilities, cwd: &str) -> Self {
|
||||
let dir = TempDir::new().unwrap();
|
||||
std::fs::create_dir_all(dir.path().join("src/deep")).unwrap();
|
||||
std::fs::write(
|
||||
@@ -264,6 +269,13 @@ impl HttpFixture {
|
||||
manifest::SharedScope::new(manifest::Scope::writable(dir.path()).unwrap()),
|
||||
capabilities,
|
||||
));
|
||||
let session: WorkdirSessionHandle = if cwd.is_empty() {
|
||||
session
|
||||
} else {
|
||||
workdir::WorkdirToolBroker::at_cwd(session, WorkdirPath::new(cwd).unwrap())
|
||||
.unwrap()
|
||||
.tool_session()
|
||||
};
|
||||
let state = Arc::new(ProviderState {
|
||||
session,
|
||||
requests: Mutex::new(Vec::new()),
|
||||
@@ -349,6 +361,31 @@ impl HttpFixture {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_open_keeps_provider_environment_and_nonroot_operation_coordinates() {
|
||||
let fixture = HttpFixture::at_cwd(WorkdirSessionCapabilities::READ_ONLY, "src/deep").await;
|
||||
let remote = fixture.open(TOKEN).await.unwrap();
|
||||
assert_eq!(remote.environment(), fixture.state.session.environment());
|
||||
assert_eq!(remote.environment().cwd.as_str(), "src/deep");
|
||||
assert_eq!(
|
||||
remote.environment().capabilities,
|
||||
WorkdirSessionCapabilities::READ_ONLY
|
||||
);
|
||||
let result = remote
|
||||
.read(workdir::ReadRequest {
|
||||
path: WorkdirPath::new("a.txt").unwrap(),
|
||||
offset: 0,
|
||||
limit: 10,
|
||||
max_bytes: 4096,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(result.bytes, b"first\nneedle needle\nlast\n");
|
||||
fixture.public_output(&serde_json::to_string(&remote.environment()).unwrap());
|
||||
remote.close().await.unwrap();
|
||||
fixture.assert_endpoints();
|
||||
}
|
||||
|
||||
fn native_runtime(remote: Arc<RemoteWorkdirSession>) -> (WipRuntime, Arc<WorkdirSessionRouter>) {
|
||||
let router = Arc::new(WorkdirSessionRouter::new());
|
||||
router
|
||||
|
||||
@@ -78,6 +78,12 @@ impl WorkdirSession for HookedSession {
|
||||
self.local.capabilities()
|
||||
}
|
||||
|
||||
fn environment(&self) -> workdir::WorkdirSessionEnvironment {
|
||||
let mut environment = self.local.environment();
|
||||
environment.capabilities = environment.capabilities.intersection(self.capabilities());
|
||||
environment
|
||||
}
|
||||
|
||||
async fn authorize_scope_path(
|
||||
&self,
|
||||
request: WorkdirScopeAuthorizationRequest,
|
||||
@@ -201,11 +207,10 @@ async fn checkout_wip_file_operations_share_core_and_keep_prior_read_boundary()
|
||||
let dir = TempDir::new().unwrap();
|
||||
std::fs::write(dir.path().join("text.txt"), "旧 旧\n").unwrap();
|
||||
let router = Arc::new(WorkdirSessionRouter::new());
|
||||
let hooked = HookedSession::new(&dir);
|
||||
assert_eq!(hooked.environment(), hooked.local.environment());
|
||||
router
|
||||
.attach(
|
||||
WorkdirAttachmentAlias::new("main").unwrap(),
|
||||
HookedSession::new(&dir),
|
||||
)
|
||||
.attach(WorkdirAttachmentAlias::new("main").unwrap(), hooked)
|
||||
.unwrap();
|
||||
let provider = provider(router);
|
||||
let route = format!("{}/text.txt", checkout_root("main"));
|
||||
|
||||
+443
-720
File diff suppressed because it is too large
Load Diff
@@ -65,7 +65,6 @@ async fn prepared(
|
||||
WorkerBootstrapLayout::Direct {
|
||||
runtime_base: temp.path().join("runtime"),
|
||||
},
|
||||
WorkerControllerTransport::InProcess,
|
||||
)
|
||||
.with_model_client(client)
|
||||
.prepare()
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -4,6 +4,7 @@
|
||||
//! same descriptor-approved registry path used by feature modules. They are not
|
||||
//! an external plugin-loading surface.
|
||||
|
||||
mod backend_diagnostic;
|
||||
pub mod backend_job_result;
|
||||
pub(crate) mod bash;
|
||||
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
//! Internal adapter diagnostics are independent of public failure classifications.
|
||||
//!
|
||||
//! Only closed vocabularies, numeric status and opaque correlation tokens cross this boundary.
|
||||
//! In particular, a transport error's Display/Debug and server-controlled text are never recorded.
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum Adapter {
|
||||
Drive,
|
||||
WorkspaceConfig,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum Stage {
|
||||
Request,
|
||||
Response,
|
||||
Completion,
|
||||
Reconcile,
|
||||
Attach,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum Reason {
|
||||
Deadline,
|
||||
TaskJoin,
|
||||
TransportUnavailable,
|
||||
InvalidPath,
|
||||
Communication,
|
||||
Decode,
|
||||
RequestSize,
|
||||
ResponseSize,
|
||||
ResponseLength,
|
||||
Identity,
|
||||
InvalidMetadata,
|
||||
BackendRejected,
|
||||
AccessCeiling,
|
||||
AttachConversion,
|
||||
BackendUnknown,
|
||||
UnrecognizedResponse,
|
||||
NotCommittedSnapshot,
|
||||
IncompleteReceipt,
|
||||
Reconciled,
|
||||
}
|
||||
|
||||
/// One logical call, including its original-request reconciliation. Request identities are hashed
|
||||
/// rather than logged verbatim: callers and responses need not contain safe log text.
|
||||
#[derive(Clone)]
|
||||
pub(super) struct Diagnostic {
|
||||
adapter: Adapter,
|
||||
operation: &'static str,
|
||||
mutation: bool,
|
||||
correlation: String,
|
||||
dispatch: tracing::Dispatch,
|
||||
stage_override: Option<Stage>,
|
||||
}
|
||||
|
||||
impl Diagnostic {
|
||||
pub(super) fn new(
|
||||
adapter: Adapter,
|
||||
operation: &'static str,
|
||||
mutation: bool,
|
||||
request_id: Option<&str>,
|
||||
) -> Self {
|
||||
Self {
|
||||
adapter,
|
||||
operation,
|
||||
mutation,
|
||||
correlation: request_id
|
||||
.map(|id| {
|
||||
let hash: String = Sha256::digest(id.as_bytes())
|
||||
.iter()
|
||||
.map(|b| format!("{b:02x}"))
|
||||
.collect();
|
||||
format!("request:{hash}")
|
||||
})
|
||||
.unwrap_or_else(|| uuid::Uuid::now_v7().to_string()),
|
||||
dispatch: tracing::dispatcher::get_default(Clone::clone),
|
||||
stage_override: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn at_stage(&self, stage: Stage) -> Self {
|
||||
let mut diagnostic = self.clone();
|
||||
diagnostic.stage_override = Some(stage);
|
||||
diagnostic
|
||||
}
|
||||
|
||||
pub(super) fn record(&self, stage: Stage, reason: Reason, status: Option<u16>) {
|
||||
let stage = self.stage_override.unwrap_or(stage);
|
||||
tracing::dispatcher::with_default(&self.dispatch, || {
|
||||
tracing::warn!(
|
||||
target: "worker::backend_diagnostic",
|
||||
adapter = ?self.adapter,
|
||||
operation = self.operation,
|
||||
mutation = self.mutation,
|
||||
correlation = %self.correlation,
|
||||
stage = ?stage,
|
||||
reason = ?reason,
|
||||
status,
|
||||
"Workspace adapter diagnostic"
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
pub(super) fn transport(&self, error: &crate::worker::WorkspaceClientError) {
|
||||
use crate::worker::WorkspaceClientError;
|
||||
self.record(
|
||||
Stage::Request,
|
||||
match error {
|
||||
WorkspaceClientError::Unavailable(_) => Reason::TransportUnavailable,
|
||||
WorkspaceClientError::InvalidPath(_) => Reason::InvalidPath,
|
||||
WorkspaceClientError::Request(_) => Reason::Communication,
|
||||
},
|
||||
None,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) mod tests {
|
||||
use super::{Adapter, Diagnostic, Reason, Stage};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tracing::{Event, Metadata, Subscriber, field::Visit, span};
|
||||
|
||||
#[test]
|
||||
fn unsafe_request_identity_is_correlated_without_logging_its_value() {
|
||||
let capture = Capture::default();
|
||||
tracing::dispatcher::with_default(&capture.dispatch(), || {
|
||||
for id in [
|
||||
"secret /host/path\ncredential",
|
||||
"secret /host/path\ncredential",
|
||||
"different",
|
||||
] {
|
||||
Diagnostic::new(Adapter::Drive, "mutate", true, Some(id)).record(
|
||||
Stage::Request,
|
||||
Reason::Communication,
|
||||
None,
|
||||
);
|
||||
}
|
||||
});
|
||||
let text = capture.text();
|
||||
for secret in ["secret", "/host/path", "credential", "different"] {
|
||||
assert!(!text.contains(secret));
|
||||
}
|
||||
let correlations: Vec<_> = text
|
||||
.lines()
|
||||
.map(|line| {
|
||||
line.split("correlation=")
|
||||
.nth(1)
|
||||
.unwrap()
|
||||
.split(';')
|
||||
.next()
|
||||
.unwrap()
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(correlations.len(), 3);
|
||||
assert_eq!(correlations[0], correlations[1]);
|
||||
assert_ne!(correlations[0], correlations[2]);
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub(crate) struct Capture(pub(crate) Arc<Mutex<Vec<String>>>);
|
||||
impl Capture {
|
||||
pub(crate) fn dispatch(&self) -> tracing::Dispatch {
|
||||
tracing::Dispatch::new(self.clone())
|
||||
}
|
||||
pub(crate) fn text(&self) -> String {
|
||||
self.0.lock().unwrap().join("\n")
|
||||
}
|
||||
}
|
||||
struct Fields(String);
|
||||
impl Visit for Fields {
|
||||
fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
|
||||
use std::fmt::Write;
|
||||
write!(&mut self.0, "{}={value:?};", field.name()).unwrap();
|
||||
}
|
||||
}
|
||||
impl Subscriber for Capture {
|
||||
fn enabled(&self, metadata: &Metadata<'_>) -> bool {
|
||||
metadata.target() == "worker::backend_diagnostic"
|
||||
}
|
||||
fn new_span(&self, _: &span::Attributes<'_>) -> span::Id {
|
||||
span::Id::from_u64(1)
|
||||
}
|
||||
fn record(&self, _: &span::Id, _: &span::Record<'_>) {}
|
||||
fn record_follows_from(&self, _: &span::Id, _: &span::Id) {}
|
||||
fn event(&self, event: &Event<'_>) {
|
||||
let mut fields = Fields(String::new());
|
||||
event.record(&mut fields);
|
||||
self.0.lock().unwrap().push(fields.0);
|
||||
}
|
||||
fn enter(&self, _: &span::Id) {}
|
||||
fn exit(&self, _: &span::Id) {}
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,8 @@
|
||||
//! cases retain an unknown outcome with the original request ID. Never blindly resend with a new ID.
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
use super::super::backend_diagnostic::{Adapter, Diagnostic, Reason, Stage};
|
||||
|
||||
use crate::worker::{
|
||||
WorkspaceBinaryRequest, WorkspaceClient, WorkspaceRequest, WorkspaceRequestMethod,
|
||||
};
|
||||
@@ -23,6 +25,7 @@ const RESPONSE_LIMIT: usize = 2 * 1024 * 1024;
|
||||
#[derive(Clone)]
|
||||
pub struct DriveBackend {
|
||||
client: Arc<dyn WorkspaceClient>,
|
||||
deadline: Duration,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, thiserror::Error)]
|
||||
@@ -49,7 +52,10 @@ pub enum DriveError {
|
||||
|
||||
impl DriveBackend {
|
||||
pub fn new(client: Arc<dyn WorkspaceClient>) -> Self {
|
||||
Self { client }
|
||||
Self {
|
||||
client,
|
||||
deadline: DEADLINE,
|
||||
}
|
||||
}
|
||||
pub fn workspace(&self) -> Result<&str, DriveError> {
|
||||
self.client
|
||||
@@ -61,15 +67,28 @@ impl DriveBackend {
|
||||
Ok(format!("/api/w/{}/drive", encode(self.workspace()?)))
|
||||
}
|
||||
pub(super) fn validate_entry(&self, entry: &DriveEntry) -> Result<(), DriveError> {
|
||||
self.validate_ref(&entry.entry)?;
|
||||
self.validate_entry_at(
|
||||
entry,
|
||||
&Diagnostic::new(Adapter::Drive, "validate_entry", false, None),
|
||||
)
|
||||
}
|
||||
fn validate_entry_at(
|
||||
&self,
|
||||
entry: &DriveEntry,
|
||||
diagnostic: &Diagnostic,
|
||||
) -> Result<(), DriveError> {
|
||||
self.validate_ref(&entry.entry)
|
||||
.inspect_err(|_| diagnostic.record(Stage::Completion, Reason::Identity, None))?;
|
||||
if let Some(parent) = &entry.parent {
|
||||
self.validate_ref(parent)?;
|
||||
self.validate_ref(parent)
|
||||
.inspect_err(|_| diagnostic.record(Stage::Completion, Reason::Identity, None))?;
|
||||
}
|
||||
if (entry.last_mutation_id.len() > 128
|
||||
|| entry.last_mutation_id.chars().any(char::is_control)
|
||||
|| (entry.parent.is_some() && entry.last_mutation_id.is_empty()))
|
||||
|| entry.size.is_some_and(|n| n > DRIVE_FILE_MAX_BYTES)
|
||||
{
|
||||
diagnostic.record(Stage::Completion, Reason::InvalidMetadata, None);
|
||||
return Err(DriveError::Unavailable);
|
||||
}
|
||||
Ok(())
|
||||
@@ -90,11 +109,13 @@ impl DriveBackend {
|
||||
request: WorkspaceRequest,
|
||||
mutation: bool,
|
||||
request_id: &str,
|
||||
diagnostic: &Diagnostic,
|
||||
) -> Result<T, DriveError> {
|
||||
let client = self.client.clone();
|
||||
let deadline = self.deadline;
|
||||
let response = tokio::time::timeout(
|
||||
DEADLINE,
|
||||
tokio::task::spawn_blocking(move || client.execute_with_timeout(request, DEADLINE)),
|
||||
deadline,
|
||||
tokio::task::spawn_blocking(move || client.execute_with_timeout(request, deadline)),
|
||||
)
|
||||
.await;
|
||||
let unknown = || {
|
||||
@@ -105,10 +126,20 @@ impl DriveBackend {
|
||||
}
|
||||
};
|
||||
let response = response
|
||||
.map_err(|_| unknown())?
|
||||
.map_err(|_| unknown())?
|
||||
.map_err(|_| unknown())?;
|
||||
.map_err(|_| {
|
||||
diagnostic.record(Stage::Request, Reason::Deadline, None);
|
||||
unknown()
|
||||
})?
|
||||
.map_err(|_| {
|
||||
diagnostic.record(Stage::Request, Reason::TaskJoin, None);
|
||||
unknown()
|
||||
})?
|
||||
.map_err(|error| {
|
||||
diagnostic.transport(&error);
|
||||
unknown()
|
||||
})?;
|
||||
if response.body.len() > RESPONSE_LIMIT {
|
||||
diagnostic.record(Stage::Response, Reason::ResponseSize, None);
|
||||
return Err(if mutation {
|
||||
unknown()
|
||||
} else {
|
||||
@@ -121,42 +152,57 @@ impl DriveBackend {
|
||||
response.body.as_bytes(),
|
||||
mutation,
|
||||
request_id,
|
||||
diagnostic,
|
||||
));
|
||||
}
|
||||
serde_json::from_str(&response.body).map_err(|_| unknown())
|
||||
serde_json::from_str(&response.body).map_err(|_| {
|
||||
diagnostic.record(Stage::Response, Reason::Decode, None);
|
||||
unknown()
|
||||
})
|
||||
}
|
||||
async fn get<T: DeserializeOwned + Send + 'static, Q: Serialize>(
|
||||
&self,
|
||||
suffix: &str,
|
||||
suffix: &'static str,
|
||||
query: &Q,
|
||||
diagnostic: &Diagnostic,
|
||||
) -> Result<T, DriveError> {
|
||||
let query = query_string(query)?;
|
||||
self.json(
|
||||
WorkspaceRequest::get(format!("{}{suffix}{query}", self.base()?)),
|
||||
false,
|
||||
"",
|
||||
diagnostic,
|
||||
)
|
||||
.await
|
||||
}
|
||||
pub async fn capabilities(&self) -> Result<DriveCapabilities, DriveError> {
|
||||
self.get("/capabilities", &serde_json::json!({})).await
|
||||
self.get(
|
||||
"/capabilities",
|
||||
&serde_json::json!({}),
|
||||
&Diagnostic::new(Adapter::Drive, "capabilities", false, None),
|
||||
)
|
||||
.await
|
||||
}
|
||||
pub async fn root(&self) -> Result<DriveEntry, DriveError> {
|
||||
let diagnostic = Diagnostic::new(Adapter::Drive, "root", false, None);
|
||||
let entry: DriveEntry = self
|
||||
.json(
|
||||
WorkspaceRequest::get(format!("{}/root", self.base()?)),
|
||||
false,
|
||||
"",
|
||||
&diagnostic,
|
||||
)
|
||||
.await?;
|
||||
self.validate_entry(&entry)?;
|
||||
self.validate_entry_at(&entry, &diagnostic)?;
|
||||
if entry.kind != DriveEntryKind::Folder || entry.parent.is_some() {
|
||||
diagnostic.record(Stage::Completion, Reason::InvalidMetadata, None);
|
||||
return Err(DriveError::Unavailable);
|
||||
}
|
||||
Ok(entry)
|
||||
}
|
||||
pub async fn metadata(&self, entry: &DriveEntryRef) -> Result<DriveEntry, DriveError> {
|
||||
self.validate_ref(entry)?;
|
||||
let diagnostic = Diagnostic::new(Adapter::Drive, "metadata", false, None);
|
||||
let result: DriveEntry = self
|
||||
.get(
|
||||
"/metadata",
|
||||
@@ -164,10 +210,12 @@ impl DriveBackend {
|
||||
entry_workspace_id: entry.workspace_id.clone(),
|
||||
id: entry.node_id.clone(),
|
||||
},
|
||||
&diagnostic,
|
||||
)
|
||||
.await?;
|
||||
self.validate_entry(&result)?;
|
||||
self.validate_entry_at(&result, &diagnostic)?;
|
||||
if result.entry != *entry {
|
||||
diagnostic.record(Stage::Completion, Reason::Identity, None);
|
||||
return Err(DriveError::Unavailable);
|
||||
}
|
||||
Ok(result)
|
||||
@@ -179,6 +227,7 @@ impl DriveBackend {
|
||||
after: Option<String>,
|
||||
) -> Result<DriveListResponse, DriveError> {
|
||||
self.validate_ref(parent)?;
|
||||
let diagnostic = Diagnostic::new(Adapter::Drive, "list", false, None);
|
||||
let result: DriveListResponse = self
|
||||
.get(
|
||||
"/list",
|
||||
@@ -188,6 +237,7 @@ impl DriveBackend {
|
||||
limit,
|
||||
after,
|
||||
},
|
||||
&diagnostic,
|
||||
)
|
||||
.await?;
|
||||
self.validate_page(
|
||||
@@ -195,33 +245,43 @@ impl DriveBackend {
|
||||
limit
|
||||
.unwrap_or(DRIVE_PAGE_MAX_LIMIT)
|
||||
.min(DRIVE_PAGE_MAX_LIMIT),
|
||||
&diagnostic,
|
||||
)?;
|
||||
if result
|
||||
.entries
|
||||
.iter()
|
||||
.any(|entry| entry.parent.as_ref() != Some(parent))
|
||||
{
|
||||
diagnostic.record(Stage::Completion, Reason::Identity, None);
|
||||
return Err(DriveError::Unavailable);
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
pub async fn search(&self, query: DriveSearchQuery) -> Result<DriveListResponse, DriveError> {
|
||||
let result: DriveListResponse = self.get("/search", &query).await?;
|
||||
let diagnostic = Diagnostic::new(Adapter::Drive, "search", false, None);
|
||||
let result: DriveListResponse = self.get("/search", &query, &diagnostic).await?;
|
||||
self.validate_page(
|
||||
&result,
|
||||
query
|
||||
.limit
|
||||
.unwrap_or(DRIVE_SEARCH_MAX_LIMIT)
|
||||
.min(DRIVE_SEARCH_MAX_LIMIT),
|
||||
&diagnostic,
|
||||
)?;
|
||||
Ok(result)
|
||||
}
|
||||
fn validate_page(&self, page: &DriveListResponse, limit: u32) -> Result<(), DriveError> {
|
||||
fn validate_page(
|
||||
&self,
|
||||
page: &DriveListResponse,
|
||||
limit: u32,
|
||||
diagnostic: &Diagnostic,
|
||||
) -> Result<(), DriveError> {
|
||||
if page.entries.len() > limit as usize {
|
||||
diagnostic.record(Stage::Completion, Reason::ResponseSize, None);
|
||||
return Err(DriveError::Limit);
|
||||
}
|
||||
for entry in &page.entries {
|
||||
self.validate_entry(entry)?;
|
||||
self.validate_entry_at(entry, diagnostic)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -231,6 +291,7 @@ impl DriveBackend {
|
||||
max_bytes: u32,
|
||||
) -> Result<DriveReadTextResponse, DriveError> {
|
||||
self.validate_ref(entry)?;
|
||||
let diagnostic = Diagnostic::new(Adapter::Drive, "read_text", false, None);
|
||||
let result: DriveReadTextResponse = self
|
||||
.get(
|
||||
"/read-text",
|
||||
@@ -239,10 +300,16 @@ impl DriveBackend {
|
||||
id: entry.node_id.clone(),
|
||||
max_bytes,
|
||||
},
|
||||
&diagnostic,
|
||||
)
|
||||
.await?;
|
||||
self.validate_entry(&result.entry)?;
|
||||
if result.entry.entry != *entry || result.text.len() > max_bytes as usize {
|
||||
self.validate_entry_at(&result.entry, &diagnostic)?;
|
||||
if result.entry.entry != *entry {
|
||||
diagnostic.record(Stage::Completion, Reason::Identity, None);
|
||||
return Err(DriveError::Unavailable);
|
||||
}
|
||||
if result.text.len() > max_bytes as usize {
|
||||
diagnostic.record(Stage::Completion, Reason::ResponseSize, None);
|
||||
return Err(DriveError::Unavailable);
|
||||
}
|
||||
Ok(result)
|
||||
@@ -255,6 +322,7 @@ impl DriveBackend {
|
||||
if total as usize > limit {
|
||||
return Err(DriveError::Limit);
|
||||
}
|
||||
let diagnostic = Diagnostic::new(Adapter::Drive, "read_chunk", false, None);
|
||||
let mut bytes = Vec::with_capacity(total as usize);
|
||||
// Even an empty file gets a read bound to the observed committed mutation and current authorization.
|
||||
loop {
|
||||
@@ -274,21 +342,42 @@ impl DriveBackend {
|
||||
max_response_bytes: DRIVE_CHUNK_MAX_BYTES as usize,
|
||||
};
|
||||
let client = self.client.clone();
|
||||
let deadline = self.deadline;
|
||||
let response = tokio::time::timeout(
|
||||
DEADLINE,
|
||||
deadline,
|
||||
tokio::task::spawn_blocking(move || {
|
||||
client.execute_binary_with_timeout(request, DEADLINE)
|
||||
client.execute_binary_with_timeout(request, deadline)
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| DriveError::Unavailable)?
|
||||
.map_err(|_| DriveError::Unavailable)?
|
||||
.map_err(|_| DriveError::Unavailable)?;
|
||||
.map_err(|_| {
|
||||
diagnostic.record(Stage::Request, Reason::Deadline, None);
|
||||
DriveError::Unavailable
|
||||
})?
|
||||
.map_err(|_| {
|
||||
diagnostic.record(Stage::Request, Reason::TaskJoin, None);
|
||||
DriveError::Unavailable
|
||||
})?
|
||||
.map_err(|error| {
|
||||
diagnostic.transport(&error);
|
||||
DriveError::Unavailable
|
||||
})?;
|
||||
if response.body.len() > DRIVE_CHUNK_MAX_BYTES as usize {
|
||||
diagnostic.record(Stage::Response, Reason::ResponseSize, None);
|
||||
return Err(DriveError::Unavailable);
|
||||
}
|
||||
if !(200..300).contains(&response.status) {
|
||||
return Err(classify(response.status, &response.body, false, ""));
|
||||
return Err(classify(
|
||||
response.status,
|
||||
&response.body,
|
||||
false,
|
||||
"",
|
||||
&diagnostic,
|
||||
));
|
||||
}
|
||||
let expected = total.saturating_sub(offset).min(length) as usize;
|
||||
if response.body.len() != expected {
|
||||
diagnostic.record(Stage::Response, Reason::ResponseLength, None);
|
||||
return Err(DriveError::Unavailable);
|
||||
}
|
||||
bytes.extend_from_slice(&response.body);
|
||||
@@ -301,22 +390,40 @@ impl DriveBackend {
|
||||
&self,
|
||||
id: &str,
|
||||
result: Result<DriveMutationResponse, DriveError>,
|
||||
diagnostic: &Diagnostic,
|
||||
) -> Result<DriveMutationResponse, DriveError> {
|
||||
// A parseable success can still be an invalid completion. Classify it
|
||||
// before reconciliation so it also gets one original-request inquiry.
|
||||
let result = result.and_then(|response| {
|
||||
self.validate_completion(id, &response)
|
||||
self.validate_completion(id, &response, diagnostic)
|
||||
.map_err(|_| DriveError::OutcomeUnknown(id.into()))?;
|
||||
Ok(response)
|
||||
});
|
||||
match result {
|
||||
Err(DriveError::OutcomeUnknown(_)) => match self.status(id).await {
|
||||
Err(DriveError::OutcomeUnknown(_)) => match self
|
||||
.status_with_diagnostic(id, &diagnostic.at_stage(Stage::Reconcile))
|
||||
.await
|
||||
{
|
||||
Ok(DriveRequestStatusResponse {
|
||||
state: DriveRequestState::Committed,
|
||||
response: Some(response),
|
||||
..
|
||||
}) => Ok(response), // status validates both IDs and entry metadata
|
||||
_ => Err(DriveError::OutcomeUnknown(id.into())),
|
||||
}) => {
|
||||
diagnostic.record(Stage::Reconcile, Reason::Reconciled, None);
|
||||
Ok(response) // status validates both IDs and entry metadata
|
||||
}
|
||||
Ok(status) => {
|
||||
diagnostic.record(
|
||||
Stage::Reconcile,
|
||||
match status.state {
|
||||
DriveRequestState::Uncommitted => Reason::NotCommittedSnapshot,
|
||||
DriveRequestState::Committed => Reason::IncompleteReceipt,
|
||||
},
|
||||
None,
|
||||
);
|
||||
Err(DriveError::OutcomeUnknown(id.into()))
|
||||
}
|
||||
Err(_) => Err(DriveError::OutcomeUnknown(id.into())),
|
||||
},
|
||||
other => other,
|
||||
}
|
||||
@@ -325,12 +432,14 @@ impl DriveBackend {
|
||||
&self,
|
||||
id: &str,
|
||||
response: &DriveMutationResponse,
|
||||
diagnostic: &Diagnostic,
|
||||
) -> Result<(), DriveError> {
|
||||
if response.request_id != id {
|
||||
diagnostic.record(Stage::Completion, Reason::Identity, None);
|
||||
return Err(DriveError::Unavailable);
|
||||
}
|
||||
if let Some(entry) = &response.entry {
|
||||
self.validate_entry(entry)?;
|
||||
self.validate_entry_at(entry, diagnostic)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -338,9 +447,11 @@ impl DriveBackend {
|
||||
&self,
|
||||
request: DriveMutationRequest,
|
||||
) -> Result<DriveMutationResponse, DriveError> {
|
||||
let diagnostic = Diagnostic::new(Adapter::Drive, "mutate", true, Some(&request.request_id));
|
||||
let body = serde_json::to_string(&request)
|
||||
.map_err(|_| DriveError::Invalid("invalid mutation".into()))?;
|
||||
if body.len() > 512 * 1024 {
|
||||
diagnostic.record(Stage::Request, Reason::RequestSize, None);
|
||||
return Err(DriveError::Limit);
|
||||
}
|
||||
let result = self
|
||||
@@ -352,16 +463,20 @@ impl DriveBackend {
|
||||
),
|
||||
true,
|
||||
&request.request_id,
|
||||
&diagnostic,
|
||||
)
|
||||
.await;
|
||||
self.reconcile(&request.request_id, result).await
|
||||
self.reconcile(&request.request_id, result, &diagnostic)
|
||||
.await
|
||||
}
|
||||
pub async fn upload(
|
||||
&self,
|
||||
query: DriveUploadQuery,
|
||||
bytes: Vec<u8>,
|
||||
) -> Result<DriveMutationResponse, DriveError> {
|
||||
let diagnostic = Diagnostic::new(Adapter::Drive, "upload", true, Some(&query.request_id));
|
||||
if bytes.len() > DRIVE_FILE_MAX_BYTES as usize {
|
||||
diagnostic.record(Stage::Request, Reason::RequestSize, None);
|
||||
return Err(DriveError::Limit);
|
||||
}
|
||||
let path = format!("{}/upload{}", self.base()?, query_string(&query)?);
|
||||
@@ -372,29 +487,56 @@ impl DriveBackend {
|
||||
max_response_bytes: RESPONSE_LIMIT,
|
||||
};
|
||||
let client = self.client.clone();
|
||||
let deadline = self.deadline;
|
||||
let result = tokio::time::timeout(
|
||||
DEADLINE,
|
||||
deadline,
|
||||
tokio::task::spawn_blocking(move || {
|
||||
client.execute_binary_with_timeout(request, DEADLINE)
|
||||
client.execute_binary_with_timeout(request, deadline)
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
let result = match result {
|
||||
Ok(Ok(Ok(response))) if response.body.len() > RESPONSE_LIMIT => {
|
||||
diagnostic.record(Stage::Response, Reason::ResponseSize, None);
|
||||
Err(DriveError::OutcomeUnknown(query.request_id.clone()))
|
||||
}
|
||||
Ok(Ok(Ok(response))) if (200..300).contains(&response.status) => {
|
||||
serde_json::from_slice(&response.body)
|
||||
.map_err(|_| DriveError::OutcomeUnknown(query.request_id.clone()))
|
||||
serde_json::from_slice(&response.body).map_err(|_| {
|
||||
diagnostic.record(Stage::Response, Reason::Decode, None);
|
||||
DriveError::OutcomeUnknown(query.request_id.clone())
|
||||
})
|
||||
}
|
||||
Ok(Ok(Ok(response))) => Err(classify(
|
||||
response.status,
|
||||
&response.body,
|
||||
true,
|
||||
&query.request_id,
|
||||
&diagnostic,
|
||||
)),
|
||||
_ => Err(DriveError::OutcomeUnknown(query.request_id.clone())),
|
||||
Err(_) => {
|
||||
diagnostic.record(Stage::Request, Reason::Deadline, None);
|
||||
Err(DriveError::OutcomeUnknown(query.request_id.clone()))
|
||||
}
|
||||
Ok(Err(_)) => {
|
||||
diagnostic.record(Stage::Request, Reason::TaskJoin, None);
|
||||
Err(DriveError::OutcomeUnknown(query.request_id.clone()))
|
||||
}
|
||||
Ok(Ok(Err(error))) => {
|
||||
diagnostic.transport(&error);
|
||||
Err(DriveError::OutcomeUnknown(query.request_id.clone()))
|
||||
}
|
||||
};
|
||||
self.reconcile(&query.request_id, result).await
|
||||
self.reconcile(&query.request_id, result, &diagnostic).await
|
||||
}
|
||||
pub async fn status(&self, request_id: &str) -> Result<DriveRequestStatusResponse, DriveError> {
|
||||
let diagnostic = Diagnostic::new(Adapter::Drive, "request_status", false, Some(request_id));
|
||||
self.status_with_diagnostic(request_id, &diagnostic).await
|
||||
}
|
||||
async fn status_with_diagnostic(
|
||||
&self,
|
||||
request_id: &str,
|
||||
diagnostic: &Diagnostic,
|
||||
) -> Result<DriveRequestStatusResponse, DriveError> {
|
||||
if request_id.is_empty() || request_id.len() > 256 {
|
||||
return Err(DriveError::Invalid("invalid request ID".into()));
|
||||
}
|
||||
@@ -403,13 +545,15 @@ impl DriveBackend {
|
||||
WorkspaceRequest::get(format!("{}/requests/{}", self.base()?, encode(request_id))),
|
||||
false,
|
||||
"",
|
||||
diagnostic,
|
||||
)
|
||||
.await?;
|
||||
if response.request_id != request_id {
|
||||
diagnostic.record(Stage::Reconcile, Reason::Identity, None);
|
||||
return Err(DriveError::Unavailable);
|
||||
}
|
||||
if let Some(result) = &response.response {
|
||||
self.validate_completion(request_id, result)?;
|
||||
self.validate_completion(request_id, result, diagnostic)?;
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
@@ -463,11 +607,21 @@ fn query_string<T: Serialize>(value: &T) -> Result<String, DriveError> {
|
||||
format!("?{}", pairs.join("&"))
|
||||
})
|
||||
}
|
||||
fn classify(status: u16, body: &[u8], mutation: bool, id: &str) -> DriveError {
|
||||
fn classify(
|
||||
status: u16,
|
||||
body: &[u8],
|
||||
mutation: bool,
|
||||
id: &str,
|
||||
diagnostic: &Diagnostic,
|
||||
) -> DriveError {
|
||||
if let Ok(error) = serde_json::from_slice::<DriveApiError>(body) {
|
||||
if error.classification == DriveFailureClassification::Unknown {
|
||||
if error.classification == DriveFailureClassification::Unknown
|
||||
|| error.code == DriveApiErrorCode::OutcomeUnknown
|
||||
{
|
||||
diagnostic.record(Stage::Response, Reason::BackendUnknown, Some(status));
|
||||
return DriveError::OutcomeUnknown(id.into());
|
||||
}
|
||||
diagnostic.record(Stage::Response, Reason::BackendRejected, Some(status));
|
||||
return match error.code {
|
||||
DriveApiErrorCode::Denied => DriveError::Denied,
|
||||
DriveApiErrorCode::NotFound => DriveError::NotFound,
|
||||
@@ -478,6 +632,7 @@ fn classify(status: u16, body: &[u8], mutation: bool, id: &str) -> DriveError {
|
||||
DriveApiErrorCode::OutcomeUnknown => DriveError::OutcomeUnknown(id.into()),
|
||||
};
|
||||
}
|
||||
diagnostic.record(Stage::Response, Reason::UnrecognizedResponse, Some(status));
|
||||
if matches!(status, 401 | 403) {
|
||||
DriveError::Denied
|
||||
} else if mutation {
|
||||
@@ -486,3 +641,7 @@ fn classify(status: u16, body: &[u8], mutation: bool, id: &str) -> DriveError {
|
||||
DriveError::Unavailable
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "backend_tests.rs"]
|
||||
mod tests;
|
||||
|
||||
@@ -0,0 +1,644 @@
|
||||
//! Production Drive adapter fault injection. Only the Workspace transport boundary is substituted;
|
||||
//! retained fixture effects and original-request inquiries are inspected independently of replies.
|
||||
//! The HTTP case additionally connects the adapter to reqwest and an isolated loopback router.
|
||||
use super::*;
|
||||
use crate::feature::builtin::backend_diagnostic::{Reason, tests::Capture};
|
||||
use crate::worker::{WorkspaceBinaryResponse, WorkspaceClientError, WorkspaceResponse};
|
||||
use std::sync::{Condvar, Mutex};
|
||||
use tracing::instrument::WithSubscriber;
|
||||
|
||||
const SECRET: &str = "secret-token /private/drive credential=unsafe";
|
||||
const ID: &str = "host-request-1";
|
||||
|
||||
fn entry() -> DriveEntry {
|
||||
DriveEntry {
|
||||
entry: DriveEntryRef {
|
||||
workspace_id: "workspace".into(),
|
||||
node_id: "2".into(),
|
||||
},
|
||||
parent: Some(DriveEntryRef {
|
||||
workspace_id: "workspace".into(),
|
||||
node_id: "1".into(),
|
||||
}),
|
||||
name: SECRET.into(),
|
||||
kind: DriveEntryKind::File,
|
||||
last_mutation_id: "old-request".into(),
|
||||
size: Some(5),
|
||||
content_type: Some("text/plain".into()),
|
||||
updated_by: SECRET.into(),
|
||||
updated_at: SECRET.into(),
|
||||
latest_url: SECRET.into(),
|
||||
}
|
||||
}
|
||||
fn mutation() -> DriveMutationRequest {
|
||||
DriveMutationRequest {
|
||||
request_id: ID.into(),
|
||||
mutation: DriveMutation::UpdateText {
|
||||
id: entry().entry,
|
||||
expected_mutation_id: "old-request".into(),
|
||||
text: SECRET.into(),
|
||||
content_type: "text/plain".into(),
|
||||
},
|
||||
}
|
||||
}
|
||||
fn json(value: impl Serialize) -> WorkspaceResponse {
|
||||
WorkspaceResponse {
|
||||
status: 200,
|
||||
body: serde_json::to_string(&value).unwrap(),
|
||||
}
|
||||
}
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
enum Fault {
|
||||
Communication,
|
||||
Unavailable,
|
||||
InvalidPath,
|
||||
Decode,
|
||||
Size,
|
||||
Length,
|
||||
Identity,
|
||||
Metadata,
|
||||
Rejected,
|
||||
Unknown,
|
||||
Unrecognized,
|
||||
Incomplete,
|
||||
Panic,
|
||||
Deadline,
|
||||
None,
|
||||
}
|
||||
#[derive(Debug, Default)]
|
||||
struct State {
|
||||
mutations: usize,
|
||||
saved: Option<String>,
|
||||
requests: Vec<WorkspaceRequest>,
|
||||
binary: Vec<WorkspaceBinaryRequest>,
|
||||
released: bool,
|
||||
started: bool,
|
||||
finished: bool,
|
||||
}
|
||||
#[derive(Debug)]
|
||||
struct Client {
|
||||
fault: Fault,
|
||||
receipt: Fault,
|
||||
state: Mutex<State>,
|
||||
changed: Condvar,
|
||||
started: tokio::sync::Notify,
|
||||
}
|
||||
impl Client {
|
||||
fn new(fault: Fault, receipt: Fault) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
fault,
|
||||
receipt,
|
||||
state: Mutex::new(State::default()),
|
||||
changed: Condvar::new(),
|
||||
started: tokio::sync::Notify::new(),
|
||||
})
|
||||
}
|
||||
fn fail(
|
||||
&self,
|
||||
fault: Fault,
|
||||
success: WorkspaceResponse,
|
||||
) -> Result<WorkspaceResponse, WorkspaceClientError> {
|
||||
match fault {
|
||||
Fault::Communication => Err(WorkspaceClientError::Request(SECRET.into())),
|
||||
Fault::Unavailable => Err(WorkspaceClientError::Unavailable(SECRET.into())),
|
||||
Fault::InvalidPath => Err(WorkspaceClientError::InvalidPath(SECRET.into())),
|
||||
Fault::Panic => panic!("injected adapter task panic"),
|
||||
Fault::Decode => Ok(WorkspaceResponse {
|
||||
status: 200,
|
||||
body: SECRET.into(),
|
||||
}),
|
||||
Fault::Size => Ok(WorkspaceResponse {
|
||||
status: 200,
|
||||
body: SECRET.repeat(RESPONSE_LIMIT / SECRET.len() + 1),
|
||||
}),
|
||||
Fault::Rejected | Fault::Unknown => Ok(WorkspaceResponse {
|
||||
status: 409,
|
||||
body: serde_json::to_string(&DriveApiError {
|
||||
code: DriveApiErrorCode::Conflict,
|
||||
message: SECRET.into(),
|
||||
classification: if matches!(fault, Fault::Unknown) {
|
||||
DriveFailureClassification::Unknown
|
||||
} else {
|
||||
DriveFailureClassification::NotCommitted
|
||||
},
|
||||
})
|
||||
.unwrap(),
|
||||
}),
|
||||
Fault::Unrecognized => Ok(WorkspaceResponse {
|
||||
status: 503,
|
||||
body: SECRET.into(),
|
||||
}),
|
||||
_ => Ok(success),
|
||||
}
|
||||
}
|
||||
fn release(&self) {
|
||||
self.state.lock().unwrap().released = true;
|
||||
self.changed.notify_all();
|
||||
}
|
||||
fn wait(&self, finished: bool) {
|
||||
let (state, timeout) = self
|
||||
.changed
|
||||
.wait_timeout_while(self.state.lock().unwrap(), Duration::from_secs(5), |s| {
|
||||
if finished { !s.finished } else { !s.started }
|
||||
})
|
||||
.unwrap();
|
||||
assert!(!timeout.timed_out());
|
||||
assert!(if finished {
|
||||
state.finished
|
||||
} else {
|
||||
state.started
|
||||
});
|
||||
}
|
||||
}
|
||||
struct Release(Arc<Client>);
|
||||
impl Drop for Release {
|
||||
fn drop(&mut self) {
|
||||
self.0.release();
|
||||
}
|
||||
}
|
||||
impl WorkspaceClient for Client {
|
||||
fn workspace_id(&self) -> Option<&str> {
|
||||
Some("workspace")
|
||||
}
|
||||
fn kind(&self) -> &str {
|
||||
"drive-diagnostic-fixture"
|
||||
}
|
||||
fn is_available(&self) -> bool {
|
||||
true
|
||||
}
|
||||
fn execute(
|
||||
&self,
|
||||
request: WorkspaceRequest,
|
||||
) -> Result<WorkspaceResponse, WorkspaceClientError> {
|
||||
let is_status = request.path.contains("/requests/");
|
||||
let fault = if is_status { self.receipt } else { self.fault };
|
||||
let mutation = request.method == WorkspaceRequestMethod::Post;
|
||||
{
|
||||
let mut state = self.state.lock().unwrap();
|
||||
if mutation
|
||||
&& !matches!(
|
||||
fault,
|
||||
Fault::Rejected | Fault::InvalidPath | Fault::Unavailable
|
||||
)
|
||||
{
|
||||
let input: DriveMutationRequest =
|
||||
serde_json::from_str(request.body.as_ref().unwrap()).unwrap();
|
||||
assert_eq!(input, super::tests::mutation());
|
||||
state.mutations += 1;
|
||||
if let DriveMutation::UpdateText { text, .. } = input.mutation {
|
||||
state.saved = Some(text);
|
||||
}
|
||||
}
|
||||
state.requests.push(request);
|
||||
state.started = true;
|
||||
self.started.notify_one();
|
||||
self.changed.notify_all();
|
||||
if matches!(fault, Fault::Deadline) {
|
||||
let (_state, timeout) = self
|
||||
.changed
|
||||
.wait_timeout_while(state, Duration::from_secs(5), |s| !s.released)
|
||||
.unwrap();
|
||||
assert!(
|
||||
!timeout.timed_out(),
|
||||
"transport must be explicitly released"
|
||||
);
|
||||
}
|
||||
}
|
||||
let mut metadata = entry();
|
||||
if matches!(fault, Fault::Identity) {
|
||||
metadata.entry.node_id = "3".into();
|
||||
}
|
||||
if matches!(fault, Fault::Metadata) {
|
||||
metadata.size = Some(DRIVE_FILE_MAX_BYTES + 1);
|
||||
}
|
||||
let completion = DriveMutationResponse {
|
||||
request_id: if matches!(fault, Fault::Identity) {
|
||||
"other-request"
|
||||
} else {
|
||||
ID
|
||||
}
|
||||
.into(),
|
||||
entry: Some(metadata.clone()),
|
||||
};
|
||||
let value = if is_status {
|
||||
json(DriveRequestStatusResponse {
|
||||
request_id: if matches!(fault, Fault::Identity) {
|
||||
"other-request"
|
||||
} else {
|
||||
ID
|
||||
}
|
||||
.into(),
|
||||
state: if matches!(
|
||||
fault,
|
||||
Fault::None | Fault::Identity | Fault::Metadata | Fault::Incomplete
|
||||
) {
|
||||
DriveRequestState::Committed
|
||||
} else {
|
||||
DriveRequestState::Uncommitted
|
||||
},
|
||||
response: if matches!(fault, Fault::None | Fault::Identity | Fault::Metadata) {
|
||||
Some(completion)
|
||||
} else {
|
||||
None
|
||||
},
|
||||
})
|
||||
} else if mutation {
|
||||
json(completion)
|
||||
} else {
|
||||
json(metadata)
|
||||
};
|
||||
if !is_status {
|
||||
self.state.lock().unwrap().finished = true;
|
||||
}
|
||||
self.changed.notify_all();
|
||||
self.fail(fault, value)
|
||||
}
|
||||
fn execute_binary(
|
||||
&self,
|
||||
request: WorkspaceBinaryRequest,
|
||||
) -> Result<WorkspaceBinaryResponse, WorkspaceClientError> {
|
||||
let upload = request.method == WorkspaceRequestMethod::Put;
|
||||
{
|
||||
let mut state = self.state.lock().unwrap();
|
||||
if upload {
|
||||
state.mutations += 1;
|
||||
state.saved = Some(SECRET.into());
|
||||
}
|
||||
state.binary.push(request);
|
||||
}
|
||||
let success = if upload {
|
||||
json(DriveMutationResponse {
|
||||
request_id: ID.into(),
|
||||
entry: Some(entry()),
|
||||
})
|
||||
} else {
|
||||
WorkspaceResponse {
|
||||
status: 200,
|
||||
body: if matches!(self.fault, Fault::Length) {
|
||||
"x"
|
||||
} else {
|
||||
"12345"
|
||||
}
|
||||
.into(),
|
||||
}
|
||||
};
|
||||
self.fail(self.fault, success)
|
||||
.map(|r| WorkspaceBinaryResponse {
|
||||
status: r.status,
|
||||
body: r.body.into_bytes(),
|
||||
})
|
||||
}
|
||||
}
|
||||
fn safe(capture: &Capture, reason: Reason, public: &str) {
|
||||
let text = capture.text();
|
||||
assert!(text.contains(&format!("reason={reason:?}")), "{text}");
|
||||
assert!(text.contains("adapter=Drive"), "{text}");
|
||||
assert!(text.contains("correlation="), "{text}");
|
||||
for secret in [
|
||||
SECRET,
|
||||
"secret-token",
|
||||
"/private/drive",
|
||||
"credential=unsafe",
|
||||
] {
|
||||
assert!(!text.contains(secret), "internal diagnostic leaked content");
|
||||
assert!(!public.contains(secret), "public error leaked content");
|
||||
}
|
||||
}
|
||||
fn reason(fault: Fault) -> Reason {
|
||||
match fault {
|
||||
Fault::Communication => Reason::Communication,
|
||||
Fault::Unavailable => Reason::TransportUnavailable,
|
||||
Fault::InvalidPath => Reason::InvalidPath,
|
||||
Fault::Decode => Reason::Decode,
|
||||
Fault::Size => Reason::ResponseSize,
|
||||
Fault::Length => Reason::ResponseLength,
|
||||
Fault::Identity => Reason::Identity,
|
||||
Fault::Metadata => Reason::InvalidMetadata,
|
||||
Fault::Rejected => Reason::BackendRejected,
|
||||
Fault::Unknown => Reason::BackendUnknown,
|
||||
Fault::Unrecognized => Reason::UnrecognizedResponse,
|
||||
Fault::Panic => Reason::TaskJoin,
|
||||
Fault::Deadline => Reason::Deadline,
|
||||
Fault::Incomplete => Reason::IncompleteReceipt,
|
||||
Fault::None => panic!("no fault"),
|
||||
}
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn production_read_failures_retain_public_classification_and_safe_cause() {
|
||||
for fault in [
|
||||
Fault::Communication,
|
||||
Fault::Unavailable,
|
||||
Fault::InvalidPath,
|
||||
Fault::Decode,
|
||||
Fault::Size,
|
||||
Fault::Identity,
|
||||
Fault::Metadata,
|
||||
Fault::Rejected,
|
||||
Fault::Unknown,
|
||||
Fault::Unrecognized,
|
||||
Fault::Panic,
|
||||
] {
|
||||
let client = Client::new(fault, Fault::None);
|
||||
let backend = DriveBackend::new(client.clone());
|
||||
let capture = Capture::default();
|
||||
let error = backend
|
||||
.metadata(&entry().entry)
|
||||
.with_subscriber(capture.dispatch())
|
||||
.await
|
||||
.unwrap_err();
|
||||
match fault {
|
||||
Fault::Size => assert!(matches!(error, DriveError::Limit)),
|
||||
Fault::Rejected => assert!(matches!(error, DriveError::Conflict)),
|
||||
Fault::Unknown => assert!(matches!(error, DriveError::OutcomeUnknown(_))),
|
||||
_ => assert!(matches!(error, DriveError::Unavailable)),
|
||||
}
|
||||
safe(&capture, reason(fault), &error.to_string());
|
||||
let state = client.state.lock().unwrap();
|
||||
assert_eq!(state.requests.len(), 1);
|
||||
assert_eq!(state.mutations, 0);
|
||||
}
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn production_mutation_loss_reconciles_only_original_identity_and_never_resends() {
|
||||
for fault in [
|
||||
Fault::Communication,
|
||||
Fault::Decode,
|
||||
Fault::Size,
|
||||
Fault::Identity,
|
||||
Fault::Metadata,
|
||||
Fault::Unknown,
|
||||
Fault::Unrecognized,
|
||||
Fault::Panic,
|
||||
] {
|
||||
for receipt in [
|
||||
Fault::None,
|
||||
Fault::Identity,
|
||||
Fault::Metadata,
|
||||
Fault::Decode,
|
||||
Fault::Communication,
|
||||
Fault::Unrecognized,
|
||||
Fault::Length,
|
||||
Fault::Incomplete,
|
||||
] {
|
||||
let client = Client::new(fault, receipt);
|
||||
let backend = DriveBackend::new(client.clone());
|
||||
let capture = Capture::default();
|
||||
let result = backend
|
||||
.mutate(mutation())
|
||||
.with_subscriber(capture.dispatch())
|
||||
.await;
|
||||
if matches!(receipt, Fault::None) {
|
||||
assert_eq!(result.unwrap().request_id, ID);
|
||||
} else {
|
||||
assert!(matches!(result, Err(DriveError::OutcomeUnknown(ref id)) if id == ID));
|
||||
}
|
||||
safe(&capture, reason(fault), "");
|
||||
let text = capture.text();
|
||||
if matches!(receipt, Fault::Length | Fault::Incomplete) {
|
||||
safe(
|
||||
&capture,
|
||||
if matches!(receipt, Fault::Length) {
|
||||
Reason::NotCommittedSnapshot
|
||||
} else {
|
||||
Reason::IncompleteReceipt
|
||||
},
|
||||
"",
|
||||
);
|
||||
}
|
||||
assert!(text.contains("stage=Reconcile"), "{text}");
|
||||
let correlations: std::collections::BTreeSet<_> = text
|
||||
.lines()
|
||||
.filter_map(|l| l.split("correlation=").nth(1))
|
||||
.filter_map(|l| l.split(';').next())
|
||||
.collect();
|
||||
assert_eq!(
|
||||
correlations.len(),
|
||||
1,
|
||||
"original loss and inquiry must correlate"
|
||||
);
|
||||
assert!(
|
||||
!text.contains(ID),
|
||||
"request identity must be opaque in logs"
|
||||
);
|
||||
let state = client.state.lock().unwrap();
|
||||
assert_eq!(state.mutations, 1);
|
||||
assert_eq!(state.saved.as_deref(), Some(SECRET));
|
||||
assert_eq!(state.requests.len(), 2);
|
||||
assert_eq!(state.requests[1].method, WorkspaceRequestMethod::Get);
|
||||
assert!(state.requests[1].path.ends_with(&format!("/requests/{ID}")));
|
||||
}
|
||||
}
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn production_rejection_has_no_effect_or_status_inquiry() {
|
||||
let client = Client::new(Fault::Rejected, Fault::None);
|
||||
let backend = DriveBackend::new(client.clone());
|
||||
let capture = Capture::default();
|
||||
let error = backend
|
||||
.mutate(mutation())
|
||||
.with_subscriber(capture.dispatch())
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(error, DriveError::Conflict));
|
||||
safe(&capture, Reason::BackendRejected, &error.to_string());
|
||||
let state = client.state.lock().unwrap();
|
||||
assert_eq!(state.mutations, 0);
|
||||
assert_eq!(state.requests.len(), 1);
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn production_binary_length_transport_and_upload_loss_are_diagnosed() {
|
||||
for fault in [
|
||||
Fault::Length,
|
||||
Fault::Communication,
|
||||
Fault::Unavailable,
|
||||
Fault::Panic,
|
||||
] {
|
||||
let client = Client::new(fault, Fault::None);
|
||||
let backend = DriveBackend::new(client.clone());
|
||||
let capture = Capture::default();
|
||||
let error = backend
|
||||
.bytes(&entry(), 5)
|
||||
.with_subscriber(capture.dispatch())
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(error, DriveError::Unavailable));
|
||||
safe(&capture, reason(fault), &error.to_string());
|
||||
assert_eq!(client.state.lock().unwrap().binary.len(), 1);
|
||||
}
|
||||
for fault in [Fault::Decode, Fault::Communication, Fault::Size] {
|
||||
let client = Client::new(fault, Fault::Unrecognized);
|
||||
let backend = DriveBackend::new(client.clone());
|
||||
let capture = Capture::default();
|
||||
let query = DriveUploadQuery {
|
||||
operation: DriveUploadOperation::Update,
|
||||
request_id: ID.into(),
|
||||
entry_workspace_id: "workspace".into(),
|
||||
parent_id: None,
|
||||
name: None,
|
||||
id: Some("2".into()),
|
||||
expected_mutation_id: Some("old-request".into()),
|
||||
content_type: "text/plain".into(),
|
||||
size: 5,
|
||||
sha256: "hash".into(),
|
||||
};
|
||||
let error = backend
|
||||
.upload(query, b"12345".to_vec())
|
||||
.with_subscriber(capture.dispatch())
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(error, DriveError::OutcomeUnknown(ref id) if id == ID));
|
||||
safe(&capture, reason(fault), &error.to_string());
|
||||
let state = client.state.lock().unwrap();
|
||||
assert_eq!(state.mutations, 1);
|
||||
assert_eq!(state.binary.len(), 1);
|
||||
assert_eq!(state.requests.len(), 1);
|
||||
assert!(state.requests[0].path.ends_with(&format!("/requests/{ID}")));
|
||||
}
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn production_deadline_does_not_cancel_effect_or_infer_uncommitted_from_snapshot() {
|
||||
for mutation_call in [false, true] {
|
||||
let client = Client::new(Fault::Deadline, Fault::Unrecognized);
|
||||
let release = Release(client.clone());
|
||||
let mut backend = DriveBackend::new(client.clone());
|
||||
backend.deadline = Duration::from_secs(1);
|
||||
let capture = Capture::default();
|
||||
let request = async {
|
||||
if mutation_call {
|
||||
backend.mutate(mutation()).await.map(|_| ())
|
||||
} else {
|
||||
backend.metadata(&entry().entry).await.map(|_| ())
|
||||
}
|
||||
}
|
||||
.with_subscriber(capture.dispatch());
|
||||
tokio::pin!(request);
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = client.started.notified() => {},
|
||||
_ = &mut request => panic!("request must start before timeout"),
|
||||
}
|
||||
{
|
||||
let state = client.state.lock().unwrap();
|
||||
assert_eq!(state.mutations, usize::from(mutation_call));
|
||||
assert!(!state.released && !state.finished);
|
||||
}
|
||||
let result = request.await;
|
||||
let error = result.unwrap_err();
|
||||
assert!(if mutation_call {
|
||||
matches!(error, DriveError::OutcomeUnknown(ref id) if id == ID)
|
||||
} else {
|
||||
matches!(error, DriveError::Unavailable)
|
||||
});
|
||||
safe(&capture, Reason::Deadline, &error.to_string());
|
||||
drop(release);
|
||||
let waiting = client.clone();
|
||||
tokio::task::spawn_blocking(move || waiting.wait(true))
|
||||
.await
|
||||
.unwrap();
|
||||
let state = client.state.lock().unwrap();
|
||||
assert_eq!(state.mutations, usize::from(mutation_call));
|
||||
assert_eq!(state.saved.is_some(), mutation_call);
|
||||
assert_eq!(state.requests.len(), if mutation_call { 2 } else { 1 });
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_reply_loss_keeps_original_receipt_and_retained_effect_without_reexecution() {
|
||||
use axum::{
|
||||
Router,
|
||||
extract::State as HttpState,
|
||||
routing::{get, post},
|
||||
};
|
||||
#[derive(Default)]
|
||||
struct Stored {
|
||||
updates: usize,
|
||||
content: String,
|
||||
ids: Vec<String>,
|
||||
}
|
||||
struct Server(tokio::task::JoinHandle<()>);
|
||||
impl Drop for Server {
|
||||
fn drop(&mut self) {
|
||||
self.0.abort();
|
||||
}
|
||||
}
|
||||
let stored = Arc::new(Mutex::new(Stored::default()));
|
||||
let router = Router::new()
|
||||
.route("/api/w/workspace/drive/metadata", get(|| async { SECRET }))
|
||||
.route(
|
||||
"/api/w/workspace/drive/mutate",
|
||||
post(
|
||||
|HttpState(state): HttpState<Arc<Mutex<Stored>>>,
|
||||
body: axum::body::Bytes| async move {
|
||||
let request: DriveMutationRequest = serde_json::from_slice(&body).unwrap();
|
||||
let mut state = state.lock().unwrap();
|
||||
state.ids.push(request.request_id);
|
||||
if let DriveMutation::UpdateText {
|
||||
text,
|
||||
expected_mutation_id,
|
||||
..
|
||||
} = request.mutation
|
||||
{
|
||||
assert_eq!(expected_mutation_id, "old-request");
|
||||
state.content = text;
|
||||
state.updates += 1;
|
||||
} else {
|
||||
panic!("unexpected mutation");
|
||||
}
|
||||
// The effect is retained; only the success response is malformed.
|
||||
SECRET
|
||||
},
|
||||
),
|
||||
)
|
||||
.route(
|
||||
"/api/w/workspace/drive/requests/{id}",
|
||||
get(
|
||||
|HttpState(state): HttpState<Arc<Mutex<Stored>>>,
|
||||
axum::extract::Path(id): axum::extract::Path<String>| async move {
|
||||
let state = state.lock().unwrap();
|
||||
assert_eq!(state.ids, [id.clone()]);
|
||||
axum::Json(DriveRequestStatusResponse {
|
||||
request_id: id.clone(),
|
||||
state: DriveRequestState::Committed,
|
||||
response: Some(DriveMutationResponse {
|
||||
request_id: id,
|
||||
entry: Some(entry()),
|
||||
}),
|
||||
})
|
||||
},
|
||||
),
|
||||
)
|
||||
.with_state(stored.clone());
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = Server(tokio::spawn(async move {
|
||||
axum::serve(listener, router).await.unwrap();
|
||||
}));
|
||||
let client = Arc::new(crate::worker::TestWorkspaceHttpClient::new(
|
||||
"workspace",
|
||||
format!("http://{address}"),
|
||||
));
|
||||
let backend = DriveBackend::new(client);
|
||||
let capture = Capture::default();
|
||||
let error = backend
|
||||
.metadata(&entry().entry)
|
||||
.with_subscriber(capture.dispatch())
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(error, DriveError::Unavailable));
|
||||
safe(&capture, Reason::Decode, &error.to_string());
|
||||
let capture = Capture::default();
|
||||
let response = backend
|
||||
.mutate(mutation())
|
||||
.with_subscriber(capture.dispatch())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.request_id, ID);
|
||||
safe(&capture, Reason::Decode, "");
|
||||
assert!(capture.text().contains("reason=Reconciled"));
|
||||
let state = stored.lock().unwrap();
|
||||
assert_eq!(state.updates, 1);
|
||||
assert_eq!(state.content, SECRET);
|
||||
assert_eq!(state.ids, [ID]);
|
||||
drop(state);
|
||||
drop(server);
|
||||
}
|
||||
@@ -26,7 +26,7 @@ use flow::{
|
||||
use manifest::{Scope, WorkerManifest};
|
||||
use schemars::JsonSchema;
|
||||
use serde::Deserialize;
|
||||
use session_store::{SegmentId, SessionId, Store, collect_state};
|
||||
use session_store::{SegmentId, SessionId, Store};
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::feature::builtin::session_explore::{SessionExploreFeature, SessionExploreState};
|
||||
@@ -469,10 +469,15 @@ where
|
||||
.store
|
||||
.read_all(self.session_id, self.segment_id)
|
||||
.map_err(|error| format!("read committed parent session: {error}"))?;
|
||||
let restored = collect_state(&entries);
|
||||
Ok(SessionCapture::new(
|
||||
let history = crate::session_history::restore_history_entries(
|
||||
self.session_id,
|
||||
self.segment_id,
|
||||
&entries,
|
||||
)?;
|
||||
Ok(SessionCapture::from_complete_history(
|
||||
self.session_id.to_string(),
|
||||
self.segment_id.to_string(),
|
||||
restored.history,
|
||||
history,
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -59,6 +59,53 @@ impl JournalReadStatus {
|
||||
}
|
||||
}
|
||||
|
||||
// Only bounded classifications cross into diagnostics. Host messages/codes and
|
||||
// io::Error payloads can contain Journal text, credentials, proofs or host paths.
|
||||
// These diagnostics belong to the Feature, not persisted/model-visible input.
|
||||
enum JournalCompactionFailure {
|
||||
Connection(std::io::ErrorKind),
|
||||
SnapshotConflict,
|
||||
SnapshotInvalidArgument,
|
||||
SnapshotUnavailable,
|
||||
Serialization,
|
||||
ByteBudget,
|
||||
}
|
||||
|
||||
impl JournalCompactionFailure {
|
||||
fn snapshot(error: SubjektivHostError) -> Self {
|
||||
match error {
|
||||
SubjektivHostError::Conflict { .. } => Self::SnapshotConflict,
|
||||
SubjektivHostError::InvalidArgument(_) => Self::SnapshotInvalidArgument,
|
||||
SubjektivHostError::Unavailable(_) => Self::SnapshotUnavailable,
|
||||
}
|
||||
}
|
||||
|
||||
fn stage(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Connection(_) => "connection",
|
||||
Self::SnapshotConflict | Self::SnapshotInvalidArgument | Self::SnapshotUnavailable => {
|
||||
"snapshot"
|
||||
}
|
||||
Self::Serialization => "serialization",
|
||||
Self::ByteBudget => "byte_budget",
|
||||
}
|
||||
}
|
||||
|
||||
fn reason(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Connection(std::io::ErrorKind::PermissionDenied) => "permission_denied",
|
||||
Self::Connection(std::io::ErrorKind::InvalidInput) => "invalid_input",
|
||||
Self::Connection(std::io::ErrorKind::NotConnected) => "not_connected",
|
||||
Self::Connection(_) => "io_error",
|
||||
Self::SnapshotConflict => "host_conflict",
|
||||
Self::SnapshotInvalidArgument => "host_invalid_argument",
|
||||
Self::SnapshotUnavailable => "host_unavailable",
|
||||
Self::Serialization => "json_serialization_failed",
|
||||
Self::ByteBudget => "startup_max_bytes_exceeded",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct JournalCompactionInput {
|
||||
@@ -203,33 +250,46 @@ pub(crate) fn freeze_compaction_journal(
|
||||
.and_then(|connection| connection.host.workspace_id())
|
||||
});
|
||||
let read = match connection {
|
||||
Ok(Some(connection)) if connection.host.journal_compaction_granted() => {
|
||||
Some(connection.journal_compaction_context().map_err(|_| ()))
|
||||
}
|
||||
Ok(Some(connection)) if connection.host.journal_compaction_granted() => Some(
|
||||
connection
|
||||
.journal_compaction_context()
|
||||
.map_err(JournalCompactionFailure::snapshot),
|
||||
),
|
||||
Ok(_) => None,
|
||||
Err(_) => Some(Err(())),
|
||||
Err(error) => Some(Err(JournalCompactionFailure::Connection(error.kind()))),
|
||||
};
|
||||
let snapshot = read.as_ref().and_then(|read| read.as_ref().ok()).cloned();
|
||||
let (status, json) = match read {
|
||||
None => (Disabled, String::new()),
|
||||
let prepared = match read {
|
||||
None => Ok((Disabled, String::new())),
|
||||
Some(Ok(snapshot)) => {
|
||||
let status = if snapshot.journals.is_empty() {
|
||||
Absent
|
||||
} else {
|
||||
Available
|
||||
};
|
||||
match serde_json::to_string(&snapshot) {
|
||||
Ok(json) if json.len() <= subjektiv::journal::startup::STARTUP_MAX_BYTES => {
|
||||
(status, json)
|
||||
}
|
||||
_ => (Unavailable, String::new()),
|
||||
}
|
||||
// The current snapshot has only derived, JSON-safe fields (strings,
|
||||
// integers, vectors, enums and a UUID). There is no known serialization
|
||||
// failure fixture; retain this guard if the domain type changes.
|
||||
serde_json::to_string(&snapshot)
|
||||
.map_err(|_| JournalCompactionFailure::Serialization)
|
||||
.and_then(|json| {
|
||||
if json.len() <= subjektiv::journal::startup::STARTUP_MAX_BYTES {
|
||||
Ok((status, json))
|
||||
} else {
|
||||
Err(JournalCompactionFailure::ByteBudget)
|
||||
}
|
||||
})
|
||||
}
|
||||
Some(Err(())) => (Unavailable, String::new()),
|
||||
Some(Err(failure)) => Err(failure),
|
||||
};
|
||||
if status == Unavailable {
|
||||
tracing::warn!("Journal compaction snapshot unavailable; continuing without Journal");
|
||||
}
|
||||
let (status, json) = prepared.unwrap_or_else(|failure| {
|
||||
tracing::warn!(
|
||||
stage = failure.stage(),
|
||||
reason = failure.reason(),
|
||||
"Journal compaction snapshot unavailable; continuing without Journal"
|
||||
);
|
||||
(Unavailable, String::new())
|
||||
});
|
||||
use sha2::{Digest, Sha256};
|
||||
let snapshot_sha256: Option<String> = (!json.is_empty()).then(|| {
|
||||
Sha256::digest(json.as_bytes())
|
||||
@@ -726,12 +786,72 @@ fn tool_schema(name: &str) -> Value {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "../../../tests/common/diagnostics.rs"]
|
||||
mod diagnostic_test_support;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Mutex;
|
||||
use subjektiv::journal::JournalReceipt;
|
||||
|
||||
#[test]
|
||||
fn compaction_connection_failure_keeps_safe_reason_without_exposing_error_or_changing_input() {
|
||||
let prompts = crate::prompt::catalog::PromptCatalog::builtins_only().unwrap();
|
||||
for (kind, reason) in [
|
||||
(std::io::ErrorKind::PermissionDenied, "permission_denied"),
|
||||
(std::io::ErrorKind::InvalidInput, "invalid_input"),
|
||||
(std::io::ErrorKind::NotConnected, "not_connected"),
|
||||
(std::io::ErrorKind::Other, "io_error"),
|
||||
] {
|
||||
let logs = diagnostic_test_support::CapturedWarnings::default();
|
||||
let input = tracing::subscriber::with_default(logs.subscriber(), || {
|
||||
freeze_compaction_journal(
|
||||
Err(std::io::Error::new(
|
||||
kind,
|
||||
"PRIVATE_JOURNAL_BODY /private/host/path proof=PRIVATE_PROOF credential=PRIVATE_CREDENTIAL raw_error=PRIVATE_ERROR",
|
||||
)),
|
||||
&prompts,
|
||||
)
|
||||
.unwrap()
|
||||
});
|
||||
let log = logs.text();
|
||||
assert!(log.contains("stage=\"connection\""), "{log}");
|
||||
assert!(log.contains(&format!("reason=\"{reason}\"")), "{log}");
|
||||
assert_eq!(log.matches("continuing without Journal").count(), 1);
|
||||
assert!(!log.contains("PRIVATE_"));
|
||||
assert!(!log.contains("/private/host/path"));
|
||||
assert!(input.body().contains("Journal read status: unavailable."));
|
||||
let persisted = serde_json::to_string(&input).unwrap();
|
||||
assert!(!persisted.contains("PRIVATE_"));
|
||||
assert!(!persisted.contains("/private/host/path"));
|
||||
assert!(!persisted.contains(reason));
|
||||
let payload: JournalCompactionInput = input
|
||||
.metadata()
|
||||
.feature_payload()
|
||||
.unwrap()
|
||||
.decode("journal:compaction", 1)
|
||||
.unwrap();
|
||||
assert!(payload.snapshot.is_none());
|
||||
assert!(payload.snapshot_sha256.is_none());
|
||||
assert_eq!(
|
||||
payload.prompt_provenance.source_digest,
|
||||
prompts.projection().source_digest
|
||||
);
|
||||
assert_eq!(
|
||||
payload.prompt_provenance.projection_digest,
|
||||
prompts.projection().catalog_digest
|
||||
);
|
||||
}
|
||||
let logs = diagnostic_test_support::CapturedWarnings::default();
|
||||
let input = tracing::subscriber::with_default(logs.subscriber(), || {
|
||||
freeze_compaction_journal(Ok(None), &prompts).unwrap()
|
||||
});
|
||||
assert!(input.body().contains("Journal read status: disabled."));
|
||||
assert!(logs.text().is_empty());
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Host {
|
||||
requests: Mutex<Vec<JournalJobRequest>>,
|
||||
|
||||
@@ -226,6 +226,7 @@ pub struct WorkspaceAttachedWorkdirSession {
|
||||
target_workdir: String,
|
||||
workdir: Workdir,
|
||||
capabilities: WorkdirSessionCapabilities,
|
||||
environment: workdir::WorkdirSessionEnvironment,
|
||||
}
|
||||
|
||||
impl WorkspaceAttachedWorkdirSession {
|
||||
@@ -249,17 +250,38 @@ impl WorkspaceAttachedWorkdirSession {
|
||||
)
|
||||
}
|
||||
|
||||
/// Bind a known root-coordinate attachment. Opened provider sessions must
|
||||
/// instead pass their metadata to [`Self::handle_for_workdir_with_environment`].
|
||||
pub fn handle_for_workdir_with_capabilities(
|
||||
client: Arc<dyn WorkspaceClient>,
|
||||
target_workdir: impl Into<String>,
|
||||
workdir_id: impl Into<String>,
|
||||
capabilities: WorkdirSessionCapabilities,
|
||||
) -> WorkdirSessionHandle {
|
||||
Self::handle_for_workdir_with_environment(
|
||||
client,
|
||||
target_workdir,
|
||||
workdir_id,
|
||||
capabilities,
|
||||
workdir::WorkdirSessionEnvironment::root(capabilities),
|
||||
)
|
||||
}
|
||||
|
||||
/// Bind provider-authored coordinates without allowing metadata to expand
|
||||
/// the Backend attachment capability ceiling.
|
||||
pub fn handle_for_workdir_with_environment(
|
||||
client: Arc<dyn WorkspaceClient>,
|
||||
target_workdir: impl Into<String>,
|
||||
workdir_id: impl Into<String>,
|
||||
capabilities: WorkdirSessionCapabilities,
|
||||
environment: workdir::WorkdirSessionEnvironment,
|
||||
) -> WorkdirSessionHandle {
|
||||
Arc::new(Self {
|
||||
client,
|
||||
target_workdir: target_workdir.into(),
|
||||
workdir: Workdir::new(workdir_id),
|
||||
capabilities,
|
||||
capabilities: capabilities.intersection(environment.capabilities),
|
||||
environment,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -291,6 +313,11 @@ impl WorkspaceAttachedWorkdirSession {
|
||||
.execute(request)
|
||||
.map_err(workspace_workdir_error)?;
|
||||
if !response.is_success() {
|
||||
if let Ok(error) =
|
||||
serde_json::from_str::<server_api::WorkdirOperationApiError>(&response.body)
|
||||
{
|
||||
notify_workdir_audit_failures(&error.audit_failures);
|
||||
}
|
||||
return Err(
|
||||
serde_json::from_str::<workdir::http::WorkdirTransportError>(&response.body)
|
||||
.map(workdir::http::WorkdirTransportError::into_workdir_error)
|
||||
@@ -302,11 +329,14 @@ impl WorkspaceAttachedWorkdirSession {
|
||||
}),
|
||||
);
|
||||
}
|
||||
serde_json::from_str(&response.body).map_err(|error| {
|
||||
WorkdirError::Transport(format!(
|
||||
"failed to decode Workspace Workdir operation result: {error}"
|
||||
))
|
||||
})
|
||||
let response: workdir::http::WorkdirSessionOperationResponse =
|
||||
serde_json::from_str(&response.body).map_err(|_| {
|
||||
WorkdirError::Transport(
|
||||
"failed to decode Workspace Workdir operation result".into(),
|
||||
)
|
||||
})?;
|
||||
notify_workdir_audit_failures(&response.audit_failures);
|
||||
Ok(response.result)
|
||||
}
|
||||
|
||||
fn mismatch(expected: &str) -> WorkdirError {
|
||||
@@ -316,6 +346,27 @@ impl WorkspaceAttachedWorkdirSession {
|
||||
}
|
||||
}
|
||||
|
||||
// Notification is separate from the operation return: callers retain successful command
|
||||
// handles, checkout observations and original failures, and must not resend for audit loss.
|
||||
fn notify_workdir_audit_failures(failures: &[workdir::http::WorkdirAuditFailure]) {
|
||||
use sha2::Digest;
|
||||
for failure in failures {
|
||||
// Response strings are untrusted. Never copy them into the receiving process log.
|
||||
let operation_id_hash = sha2::Sha256::digest(failure.operation_id.as_bytes())
|
||||
.iter()
|
||||
.map(|byte| format!("{byte:02x}"))
|
||||
.collect::<String>();
|
||||
tracing::error!(
|
||||
event = "external_workdir_audit_failed",
|
||||
operation_id_hash,
|
||||
audit_stage = ?failure.stage,
|
||||
audit_reason = ?failure.reason,
|
||||
operation_outcome = ?failure.operation_outcome,
|
||||
"External Workdir audit persistence failed; operation result is independent; do not resend"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn workspace_workdir_error(error: WorkspaceClientError) -> WorkdirError {
|
||||
match error {
|
||||
WorkspaceClientError::Unavailable(message) => WorkdirError::Unavailable(message),
|
||||
@@ -333,6 +384,12 @@ impl WorkdirSession for WorkspaceAttachedWorkdirSession {
|
||||
self.capabilities
|
||||
}
|
||||
|
||||
fn environment(&self) -> workdir::WorkdirSessionEnvironment {
|
||||
let mut environment = self.environment.clone();
|
||||
environment.capabilities = environment.capabilities.intersection(self.capabilities());
|
||||
environment
|
||||
}
|
||||
|
||||
async fn authorize_scope_path(
|
||||
&self,
|
||||
request: WorkdirScopeAuthorizationRequest,
|
||||
@@ -655,11 +712,12 @@ impl WorkspaceHttpWorkdirBackend {
|
||||
let response = self.attach_response(alias, working_directory_id)?;
|
||||
if let Err(error) = self.session_router.attach(
|
||||
alias_key.clone(),
|
||||
WorkspaceAttachedWorkdirSession::handle_for_workdir_with_capabilities(
|
||||
WorkspaceAttachedWorkdirSession::handle_for_workdir_with_environment(
|
||||
self.client.clone(),
|
||||
alias,
|
||||
working_directory_id,
|
||||
response.capabilities,
|
||||
response.attachment.capabilities,
|
||||
response.environment.clone(),
|
||||
),
|
||||
) {
|
||||
let _ = self.detach(alias);
|
||||
@@ -677,9 +735,9 @@ impl WorkspaceHttpWorkdirBackend {
|
||||
&self,
|
||||
alias: &str,
|
||||
working_directory_id: &str,
|
||||
) -> Result<WorkdirAttachmentResponse, ToolError> {
|
||||
) -> Result<WorkdirSessionAttachmentResponse, ToolError> {
|
||||
let workspace_id = encode_path_segment(self.workspace_id()?);
|
||||
self.execute_json::<WorkdirAttachmentResponse>(WorkspaceRequest::json(
|
||||
self.execute_json::<WorkdirSessionAttachmentResponse>(WorkspaceRequest::json(
|
||||
WorkspaceRequestMethod::Post,
|
||||
format!("/api/w/{workspace_id}/workers/self/workdir-attachments"),
|
||||
serde_json::to_string(&WorkdirAttachRequest {
|
||||
@@ -1163,6 +1221,13 @@ struct WorkdirAttachmentResponse {
|
||||
attached: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
|
||||
struct WorkdirSessionAttachmentResponse {
|
||||
#[serde(flatten)]
|
||||
attachment: WorkdirAttachmentResponse,
|
||||
environment: workdir::WorkdirSessionEnvironment,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct WorkdirDeleteInput {
|
||||
@@ -1473,6 +1538,7 @@ mod tests {
|
||||
"alias": "checkout",
|
||||
"working_directory_id": "wd-created",
|
||||
"capabilities": {"bits": 63},
|
||||
"environment": workdir::WorkdirSessionEnvironment::root(WorkdirSessionCapabilities::ALL),
|
||||
"attached": true
|
||||
})),
|
||||
response(json!({
|
||||
@@ -1653,6 +1719,67 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn attached_session_keeps_operation_results_when_audit_persistence_failed() {
|
||||
let failure = json!({
|
||||
"operation_id":"op-safe-id", "stage":"response_received",
|
||||
"reason":"audit_store_unavailable", "operation_outcome":{"kind":"succeeded"}
|
||||
});
|
||||
let mut responses = vec![response(json!({
|
||||
"operation":"command_start", "result":"retained-command",
|
||||
"audit_failures":[failure]
|
||||
}))];
|
||||
for (code, status, outcome) in [
|
||||
("denied", 403, json!({"kind":"failed","code":"denied"})),
|
||||
("outcome_unknown", 500, json!({"kind":"unknown"})),
|
||||
("not_dispatched", 503, json!({"kind":"not_dispatched"})),
|
||||
] {
|
||||
let mut failure = failure.clone();
|
||||
failure["operation_outcome"] = outcome;
|
||||
responses.push(WorkspaceResponse { status, body: json!({
|
||||
"code":code, "message":"safe provider classification", "audit_failures":[failure]
|
||||
}).to_string() });
|
||||
}
|
||||
let client = Arc::new(RecordingWorkspaceClient::new(responses));
|
||||
let session = WorkspaceAttachedWorkdirSession::handle(client.clone(), "checkout");
|
||||
let handle = session
|
||||
.start_command(CommandRequest {
|
||||
command: "true".into(),
|
||||
timeout_secs: 1,
|
||||
transport_byte_budget: workdir::MAX_COMMAND_TRANSPORT_BYTES,
|
||||
cwd: workdir::WorkdirPath::root(),
|
||||
tool_call_id: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(handle.0, "retained-command");
|
||||
let request = workdir::CheckoutRequest {
|
||||
target: workdir::WorkdirPath::new("file.txt").unwrap(),
|
||||
validator: vec![1],
|
||||
operation: workdir::CheckoutOperation::Write {
|
||||
content: b"new".to_vec(),
|
||||
expected_hash: [0; 32],
|
||||
},
|
||||
};
|
||||
assert!(matches!(
|
||||
session.checkout_execute(request.clone()).await,
|
||||
Err(WorkdirError::Denied(_))
|
||||
));
|
||||
assert!(matches!(
|
||||
session.checkout_execute(request.clone()).await,
|
||||
Err(WorkdirError::OutcomeUnknown(_))
|
||||
));
|
||||
assert!(matches!(
|
||||
session.checkout_execute(request).await,
|
||||
Err(WorkdirError::NotDispatched)
|
||||
));
|
||||
assert_eq!(
|
||||
client.requests().len(),
|
||||
4,
|
||||
"audit loss must not trigger a resend"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn attached_session_does_not_send_worker_local_bash_spill_path() {
|
||||
let client = Arc::new(RecordingWorkspaceClient::new(vec![response(json!({
|
||||
@@ -1914,6 +2041,7 @@ mod tests {
|
||||
"alias": "checkout",
|
||||
"working_directory_id": "wd-attached",
|
||||
"capabilities": {"bits": 63},
|
||||
"environment": workdir::WorkdirSessionEnvironment::root(WorkdirSessionCapabilities::ALL),
|
||||
"attached": true
|
||||
})),
|
||||
response(json!({
|
||||
@@ -1973,16 +2101,29 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attach_propagates_provider_read_only_capabilities_to_the_alias_session() {
|
||||
#[tokio::test]
|
||||
async fn attach_projects_provider_environment_without_expanding_capability_ceiling() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
std::fs::create_dir(dir.path().join("src")).unwrap();
|
||||
let provider = Arc::new(workdir::LocalWorkdirSession::materialized(
|
||||
dir.path().to_path_buf(),
|
||||
dir.path().to_path_buf(),
|
||||
manifest::SharedScope::new(manifest::Scope::writable(dir.path()).unwrap()),
|
||||
WorkdirSessionCapabilities::ALL,
|
||||
));
|
||||
let broker =
|
||||
workdir::WorkdirToolBroker::at_cwd(provider, workdir::WorkdirPath::new("src").unwrap())
|
||||
.unwrap();
|
||||
let mut environment = broker.tool_session().environment();
|
||||
let client = Arc::new(RecordingWorkspaceClient::new(vec![response(json!({
|
||||
"workspace_id": "workspace/test",
|
||||
"alias": "external",
|
||||
"working_directory_id": "external-workdir",
|
||||
"capabilities": {"bits": 25},
|
||||
"environment": environment,
|
||||
"attached": true
|
||||
}))]));
|
||||
let backend = WorkspaceHttpWorkdirBackend::new(client);
|
||||
let backend = WorkspaceHttpWorkdirBackend::new(client.clone());
|
||||
backend
|
||||
.attach(WorkdirAttachInput {
|
||||
alias: "external".to_string(),
|
||||
@@ -1994,10 +2135,33 @@ mod tests {
|
||||
.session_router
|
||||
.session(&workdir::WorkdirAttachmentAlias::new("external").unwrap())
|
||||
.unwrap();
|
||||
environment.capabilities = WorkdirSessionCapabilities::READ_ONLY;
|
||||
assert_eq!(session.environment(), environment);
|
||||
assert_eq!(
|
||||
session.capabilities(),
|
||||
WorkdirSessionCapabilities::READ_ONLY
|
||||
);
|
||||
assert!(matches!(
|
||||
session
|
||||
.checkout_execute(workdir::CheckoutRequest {
|
||||
target: workdir::WorkdirPath::root(),
|
||||
validator: vec![1],
|
||||
operation: workdir::CheckoutOperation::Create {
|
||||
path: workdir::WorkdirPath::new("new.txt").unwrap(),
|
||||
content: b"denied".to_vec(),
|
||||
},
|
||||
})
|
||||
.await,
|
||||
Err(WorkdirError::Unsupported(
|
||||
workdir::WorkdirSessionCapability::Write
|
||||
))
|
||||
));
|
||||
assert_eq!(
|
||||
client.requests().len(),
|
||||
1,
|
||||
"denied mutation must not reach the Backend"
|
||||
);
|
||||
broker.tool_session().close().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -2007,6 +2171,7 @@ mod tests {
|
||||
"alias": "checkout",
|
||||
"working_directory_id": "wd-attached",
|
||||
"capabilities": {"bits": 63},
|
||||
"environment": workdir::WorkdirSessionEnvironment::root(WorkdirSessionCapabilities::ALL),
|
||||
"attached": true
|
||||
}))]));
|
||||
let reopen_calls = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
@@ -1253,7 +1253,7 @@ pub(crate) mod tests {
|
||||
state.generation += 1;
|
||||
let raw = json!({"alias":body["alias"], "working_directory_id":body["working_directory_id"], "connection_id":format!("connection-{}", state.generation), "capabilities":{"bits":63}});
|
||||
state.attachments.push(raw);
|
||||
json!({"workspace_id":"test-workspace", "alias":body["alias"], "working_directory_id":body["working_directory_id"], "capabilities":{"bits":63}, "attached":true})
|
||||
json!({"workspace_id":"test-workspace", "alias":body["alias"], "working_directory_id":body["working_directory_id"], "capabilities":{"bits":63}, "environment":workdir::WorkdirSessionEnvironment::root(workdir::WorkdirSessionCapabilities::ALL), "attached":true})
|
||||
}
|
||||
(WorkspaceRequestMethod::Delete, p)
|
||||
if p.starts_with("workers/self/workdir-attachments/") =>
|
||||
|
||||
@@ -456,10 +456,13 @@ impl MemoryLifecycleTask {
|
||||
.await;
|
||||
return Ok(());
|
||||
}
|
||||
let view = SessionCapture::from_history_entries(
|
||||
let view = SessionCapture::from_complete_history(
|
||||
capture.session_id.clone(),
|
||||
capture.segment_id.clone(),
|
||||
capture.history[history_start..history_end].to_vec(),
|
||||
);
|
||||
capture.history.clone(),
|
||||
)
|
||||
.select_order_range(history_start as u64..history_end as u64)
|
||||
.map_err(hook_internal)?;
|
||||
let start_entry = pointer
|
||||
.as_ref()
|
||||
.map(|pointer| pointer.processed_through_entry + 1)
|
||||
@@ -1556,11 +1559,14 @@ permission = "write"
|
||||
#[tokio::test]
|
||||
async fn run_committed_background_task_stages_non_empty_extraction_and_commits_pointer() {
|
||||
let source = capture(2, 250);
|
||||
let entry_ref =
|
||||
SessionCapture::from_history_entries(source.segment_id.clone(), source.history.clone())
|
||||
.overview()[0]
|
||||
.id
|
||||
.to_string();
|
||||
let entry_ref = SessionCapture::from_complete_history(
|
||||
source.session_id.clone(),
|
||||
source.segment_id.clone(),
|
||||
source.history.clone(),
|
||||
)
|
||||
.overview()[0]
|
||||
.id
|
||||
.to_string();
|
||||
let client = ScriptClient::new(vec![
|
||||
stage_candidate_events("stage-1", &entry_ref),
|
||||
finish_events("finish-1", 1),
|
||||
@@ -1606,14 +1612,102 @@ permission = "write"
|
||||
);
|
||||
}
|
||||
|
||||
// Real background extraction -> common fixed selection -> StageMemoryCandidate
|
||||
// -> typed backend operation and pointer write. Only model/backend transport
|
||||
// are replaced; the second extraction uses the first run's saved pointer.
|
||||
#[tokio::test]
|
||||
async fn second_extraction_preserves_original_evidence_positions_after_nonzero_pointer() {
|
||||
let first = capture(2, 250);
|
||||
let mut second = capture(4, 750);
|
||||
second.history[..2].clone_from_slice(&first.history);
|
||||
second
|
||||
.usage_history
|
||||
.insert(0, first.usage_history[0].clone());
|
||||
let selected_ref =
|
||||
crate::SessionEntryRef::from_history_entry_id(&second.history[2].annotation.entry_id)
|
||||
.to_string();
|
||||
let extension_writes = Arc::new(Mutex::new(Vec::new()));
|
||||
let workspace_client = Arc::new(RecordingWorkspaceClient::default());
|
||||
let (event_tx, _) = broadcast::channel(64);
|
||||
run_background_task(test_task(
|
||||
first,
|
||||
Box::new(ScriptClient::new(vec![
|
||||
finish_empty_events("first-empty"),
|
||||
completed_events(),
|
||||
])),
|
||||
extension_writes.clone(),
|
||||
event_tx.clone(),
|
||||
workspace_client.clone(),
|
||||
))
|
||||
.await;
|
||||
{
|
||||
let writes = extension_writes.lock().unwrap();
|
||||
assert_eq!(writes.len(), 1);
|
||||
let pointer: memory::ExtractPointerPayload =
|
||||
serde_json::from_value(writes[0].2.clone()).unwrap();
|
||||
assert_eq!(pointer.processed_through_history_len, 2);
|
||||
second
|
||||
.extensions
|
||||
.push((writes[0].1.clone(), writes[0].2.clone()));
|
||||
}
|
||||
run_background_task(test_task(
|
||||
second,
|
||||
Box::new(ScriptClient::new(vec![
|
||||
stage_candidate_events("second-stage", &selected_ref),
|
||||
finish_events("second-finish", 1),
|
||||
completed_events(),
|
||||
])),
|
||||
extension_writes.clone(),
|
||||
event_tx,
|
||||
workspace_client.clone(),
|
||||
))
|
||||
.await;
|
||||
let operations = workspace_client
|
||||
.requests
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.filter_map(|request| {
|
||||
request.body.as_deref().and_then(|body| {
|
||||
serde_json::from_str::<memory::backend::MemoryBackendOperation>(body).ok()
|
||||
})
|
||||
})
|
||||
.filter_map(|operation| match operation {
|
||||
memory::backend::MemoryBackendOperation::StageCandidate(operation) => {
|
||||
Some(operation)
|
||||
}
|
||||
_ => None,
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
assert_eq!(operations.len(), 1);
|
||||
let operation = &operations[0];
|
||||
assert_eq!(operation.source.range, [2, 3]);
|
||||
assert_eq!(operation.evidence[0].id, selected_ref);
|
||||
assert_eq!(operation.evidence[0].entry_range, Some([2, 2]));
|
||||
assert_eq!(
|
||||
operation.source_refs[0].segment_id.as_deref(),
|
||||
Some("segment-1")
|
||||
);
|
||||
assert_eq!(operation.source_refs[0].entry_range, Some([2, 2]));
|
||||
let writes = extension_writes.lock().unwrap();
|
||||
assert_eq!(writes.len(), 2);
|
||||
let pointer: memory::ExtractPointerPayload =
|
||||
serde_json::from_value(writes[1].2.clone()).unwrap();
|
||||
assert_eq!(pointer.processed_through_history_len, 4);
|
||||
assert_eq!(pointer.processed_through_entry, 3);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn subjektiv_extraction_stages_once_and_commits_its_independent_pointer() {
|
||||
let source = capture(2, 250);
|
||||
let entry_ref =
|
||||
SessionCapture::from_history_entries(source.segment_id.clone(), source.history.clone())
|
||||
.overview()[0]
|
||||
.id
|
||||
.to_string();
|
||||
let entry_ref = SessionCapture::from_complete_history(
|
||||
source.session_id.clone(),
|
||||
source.segment_id.clone(),
|
||||
source.history.clone(),
|
||||
)
|
||||
.overview()[0]
|
||||
.id
|
||||
.to_string();
|
||||
let client = ScriptClient::new(vec![
|
||||
stage_candidate_events("stage-1", &entry_ref),
|
||||
finish_events("finish-1", 1),
|
||||
@@ -1715,11 +1809,14 @@ permission = "write"
|
||||
#[tokio::test]
|
||||
async fn subjektiv_candidate_remains_staged_when_later_extraction_fails_without_pointer() {
|
||||
let source = capture(2, 250);
|
||||
let entry_ref =
|
||||
SessionCapture::from_history_entries(source.segment_id.clone(), source.history.clone())
|
||||
.overview()[0]
|
||||
.id
|
||||
.to_string();
|
||||
let entry_ref = SessionCapture::from_complete_history(
|
||||
source.session_id.clone(),
|
||||
source.segment_id.clone(),
|
||||
source.history.clone(),
|
||||
)
|
||||
.overview()[0]
|
||||
.id
|
||||
.to_string();
|
||||
let client = ScriptClient::new(vec![stage_candidate_events("stage-1", &entry_ref)]);
|
||||
let extension_writes = Arc::new(Mutex::new(Vec::new()));
|
||||
let (event_tx, _) = broadcast::channel(32);
|
||||
|
||||
@@ -510,7 +510,11 @@ mod tests {
|
||||
|
||||
fn state() -> MemoryStagingOutputState {
|
||||
MemoryStagingOutputState::new(
|
||||
SessionCapture::new("segment-1", vec![Item::user_message("durable decision")]),
|
||||
SessionCapture::new(
|
||||
"session",
|
||||
"segment-1",
|
||||
vec![Item::user_message("durable decision")],
|
||||
),
|
||||
crate::worker::marker_workspace_client(None, "test-backend"),
|
||||
SourceRef {
|
||||
segment_id: "segment-1".to_string(),
|
||||
@@ -537,6 +541,7 @@ mod tests {
|
||||
#[test]
|
||||
fn render_input_uses_session_entry_refs_and_new_tool_names() {
|
||||
let view = SessionCapture::new(
|
||||
"session",
|
||||
"segment-1",
|
||||
vec![
|
||||
Item::user_message("preference"),
|
||||
@@ -586,7 +591,11 @@ mod tests {
|
||||
async fn subjektiv_destination_sends_committed_session_without_subject_scope() {
|
||||
let client = Arc::new(RecordingSubjektivClient::default());
|
||||
let state = MemoryStagingOutputState::new_subjektiv(
|
||||
SessionCapture::new("segment-1", vec![Item::user_message("durable decision")]),
|
||||
SessionCapture::new(
|
||||
"session",
|
||||
"segment-1",
|
||||
vec![Item::user_message("durable decision")],
|
||||
),
|
||||
{
|
||||
let mut host = crate::subjektiv::test_connection(client.clone());
|
||||
host.context.session_id = "session-committed".into();
|
||||
@@ -630,4 +639,53 @@ mod tests {
|
||||
.unwrap_err();
|
||||
assert!(format!("{error:?}").contains("unknown SessionEntryRef"));
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn staging_roundtripped_mixed_sources_uses_original_evidence_ranges_not_view_order() {
|
||||
let client = Arc::new(RecordingSubjektivClient::default());
|
||||
let mut dto = SessionCapture::new(
|
||||
"session",
|
||||
"view",
|
||||
vec![
|
||||
Item::user_message("first"),
|
||||
Item::assistant_message("second"),
|
||||
],
|
||||
)
|
||||
.to_dto();
|
||||
dto.entries[0].source.segment_id = "original-a".into();
|
||||
dto.entries[0].source.entry_index = 80;
|
||||
dto.entries[1].source.segment_id = "original-b".into();
|
||||
dto.entries[1].source.entry_index = 3;
|
||||
let limits = crate::session_capture::CaptureLimits::default();
|
||||
let view = SessionCapture::decode(&dto.encode(limits).unwrap(), limits).unwrap();
|
||||
let state = MemoryStagingOutputState::new_subjektiv(
|
||||
view,
|
||||
crate::subjektiv::test_connection(client.clone()),
|
||||
SourceRef {
|
||||
segment_id: "view".into(),
|
||||
range: [0, 1],
|
||||
},
|
||||
"run".into(),
|
||||
);
|
||||
StageMemoryCandidateTool { state }.execute(
|
||||
r#"{"kind":"decision","claim":"claim","why_useful":"useful","entry_refs":["E00000000","E00000001"]}"#,
|
||||
agen::tool::ToolExecutionContext::direct(),
|
||||
).await.unwrap();
|
||||
let requests = client.requests.lock().unwrap();
|
||||
let request: server_api::SubjektivStageCandidateRequest =
|
||||
serde_json::from_str(requests[0].body.as_deref().unwrap()).unwrap();
|
||||
for (index, segment, position) in [(0, "original-a", 80), (1, "original-b", 3)] {
|
||||
assert_eq!(
|
||||
request.operation.evidence[index].entry_range,
|
||||
Some([position; 2])
|
||||
);
|
||||
assert_eq!(
|
||||
request.operation.source_refs[index].segment_id.as_deref(),
|
||||
Some(segment)
|
||||
);
|
||||
assert_eq!(
|
||||
request.operation.source_refs[index].entry_range,
|
||||
Some([position; 2])
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,14 +17,14 @@ use crate::feature::{
|
||||
ToolDeclaration,
|
||||
};
|
||||
use crate::session_capture::{
|
||||
ReadDetail, ReadOptions, ReadSelector, ReferenceKind, SearchOptions, SessionCapture,
|
||||
SessionEntryRef, ToolPart,
|
||||
CapturePurpose, RawField, ReadDetail, ReadOptions, ReadSelector, ReferenceKind, SearchOptions,
|
||||
SessionCapture, SessionEntryRef, ToolPart,
|
||||
};
|
||||
|
||||
const SHOW_OVERVIEW_DESCRIPTION: &str =
|
||||
"Show a sparse, bounded index of real user and assistant session entries.";
|
||||
const SEARCH_ENTRIES_DESCRIPTION: &str = "Search or compactly list a bounded range of committed session entries. Reasoning entries are never exposed.";
|
||||
const READ_ENTRY_DESCRIPTION: &str = "Read one committed session entry by stable SessionEntryRef. Compact mode is the default; full mode includes bounded tool input or output.";
|
||||
const READ_ENTRY_DESCRIPTION: &str = "Read one committed session entry by stable SessionEntryRef. Compact mode is the default; full mode is a bounded rendering, not raw source. Raw mode requires raw_field and supports UTF-8 byte_offset/next_offset without omission; use capture_id to bind continuation to this exact capture.";
|
||||
const DEFAULT_PAGE_LIMIT: usize = 20;
|
||||
const MAX_PAGE_LIMIT: usize = 100;
|
||||
const DEFAULT_READ_ITEMS: usize = 10;
|
||||
@@ -173,6 +173,38 @@ struct ReadEntryParams {
|
||||
mode: String,
|
||||
#[serde(default)]
|
||||
max_items: Option<usize>,
|
||||
#[serde(default)]
|
||||
raw_field: Option<RawEntryField>,
|
||||
#[serde(default)]
|
||||
part_index: Option<usize>,
|
||||
#[serde(default)]
|
||||
byte_offset: Option<usize>,
|
||||
#[serde(default)]
|
||||
max_bytes: Option<usize>,
|
||||
#[serde(default)]
|
||||
capture_id: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
enum RawEntryField {
|
||||
MessagePart,
|
||||
PublicSystem,
|
||||
ToolArguments,
|
||||
ToolSummary,
|
||||
ToolContent,
|
||||
}
|
||||
|
||||
impl RawEntryField {
|
||||
fn into_common(self, part_index: usize) -> RawField {
|
||||
match self {
|
||||
Self::MessagePart => RawField::MessagePart(part_index),
|
||||
Self::PublicSystem => RawField::PublicSystem,
|
||||
Self::ToolArguments => RawField::ToolArguments,
|
||||
Self::ToolSummary => RawField::ToolSummary,
|
||||
Self::ToolContent => RawField::ToolContent,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn default_read_mode() -> String {
|
||||
@@ -243,25 +275,20 @@ impl Tool for SearchEntriesTool {
|
||||
let from = params.from.as_deref().map(parse_entry_ref).transpose()?;
|
||||
let through = params.through.as_deref().map(parse_entry_ref).transpose()?;
|
||||
let view = self.state.view();
|
||||
if let (Some(from), Some(through)) = (&from, &through) {
|
||||
if view.source_index_for_ref(from) > view.source_index_for_ref(through) {
|
||||
return Err(ToolError::InvalidArgument(
|
||||
"SearchEntries from must not be after through".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
let limit = bounded_limit(params.limit, DEFAULT_PAGE_LIMIT, MAX_PAGE_LIMIT);
|
||||
let hits = view.search(&SearchOptions {
|
||||
query: params.query,
|
||||
kind,
|
||||
tool_part,
|
||||
tool_name: params.tool_name,
|
||||
limit: Some(limit),
|
||||
min_entry_index: None,
|
||||
from,
|
||||
through,
|
||||
offset: params.offset,
|
||||
});
|
||||
let hits = view
|
||||
.search_checked(&SearchOptions {
|
||||
query: params.query,
|
||||
kind,
|
||||
tool_part,
|
||||
tool_name: params.tool_name,
|
||||
limit: Some(limit),
|
||||
min_entry_index: None,
|
||||
from,
|
||||
through,
|
||||
offset: params.offset,
|
||||
})
|
||||
.map_err(|error| ToolError::InvalidArgument(error.to_string()))?;
|
||||
let entries = hits
|
||||
.iter()
|
||||
.map(|hit| {
|
||||
@@ -272,6 +299,7 @@ impl Tool for SearchEntriesTool {
|
||||
"tool_name": hit.tool_name,
|
||||
"label": hit.label,
|
||||
"text": hit.summary,
|
||||
"text_kind": "preview",
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
@@ -299,12 +327,62 @@ impl Tool for ReadEntryTool {
|
||||
) -> Result<ToolOutput, ToolError> {
|
||||
let params: ReadEntryParams = parse_input("ReadEntry", input_json)?;
|
||||
let entry_ref = parse_entry_ref(¶ms.entry_ref)?;
|
||||
let view = self.state.view();
|
||||
if params
|
||||
.capture_id
|
||||
.as_deref()
|
||||
.is_some_and(|id| id != view.capture_id())
|
||||
{
|
||||
return Err(ToolError::InvalidArgument(
|
||||
"capture identity does not match".into(),
|
||||
));
|
||||
}
|
||||
if params.mode == "raw" {
|
||||
let field = params
|
||||
.raw_field
|
||||
.ok_or_else(|| ToolError::InvalidArgument("raw mode requires raw_field".into()))?;
|
||||
if params.part_index.is_some() && !matches!(field, RawEntryField::MessagePart) {
|
||||
return Err(ToolError::InvalidArgument(
|
||||
"part_index is only valid for message_part".into(),
|
||||
));
|
||||
}
|
||||
let reference = view
|
||||
.scoped_ref(&entry_ref)
|
||||
.map_err(|error| ToolError::InvalidArgument(error.to_string()))?;
|
||||
let raw = view
|
||||
.read_raw(
|
||||
&reference,
|
||||
CapturePurpose::Observation,
|
||||
field.into_common(params.part_index.unwrap_or(0)),
|
||||
params.byte_offset.unwrap_or(0),
|
||||
params
|
||||
.max_bytes
|
||||
.unwrap_or(MAX_READ_BYTES)
|
||||
.clamp(1, MAX_READ_BYTES),
|
||||
)
|
||||
.map_err(|error| ToolError::InvalidArgument(error.to_string()))?;
|
||||
return json_output(
|
||||
"Read raw session source bytes.".into(),
|
||||
serde_json::json!({
|
||||
"text_kind": "raw", "raw": raw,
|
||||
}),
|
||||
);
|
||||
}
|
||||
if params.raw_field.is_some()
|
||||
|| params.part_index.is_some()
|
||||
|| params.byte_offset.is_some()
|
||||
|| params.max_bytes.is_some()
|
||||
{
|
||||
return Err(ToolError::InvalidArgument(
|
||||
"raw fields require raw mode".into(),
|
||||
));
|
||||
}
|
||||
let detail = match params.mode.as_str() {
|
||||
"compact" => ReadDetail::Compact,
|
||||
"full" => ReadDetail::Full,
|
||||
other => {
|
||||
return Err(ToolError::InvalidArgument(format!(
|
||||
"invalid ReadEntry mode {other:?}; expected compact or full"
|
||||
"invalid ReadEntry mode {other:?}; expected compact, full, or raw"
|
||||
)));
|
||||
}
|
||||
};
|
||||
@@ -347,6 +425,7 @@ impl Tool for ReadEntryTool {
|
||||
serde_json::json!({
|
||||
"entries": entries,
|
||||
"truncated": read.truncated,
|
||||
"text_kind": "rendered",
|
||||
}),
|
||||
)
|
||||
}
|
||||
@@ -408,6 +487,7 @@ mod tests {
|
||||
|
||||
fn state() -> SessionExploreState {
|
||||
SessionExploreState::new(SessionCapture::new(
|
||||
"session",
|
||||
"segment-1",
|
||||
vec![
|
||||
Item::user_message("first"),
|
||||
@@ -479,4 +559,64 @@ mod tests {
|
||||
.unwrap();
|
||||
assert!(output.content.unwrap().contains("second"));
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn search_rejects_unknown_or_reversed_capture_endpoints() {
|
||||
let search = search_entries_definition(state())().1;
|
||||
for input in [
|
||||
r#"{"from":"E99999999"}"#,
|
||||
r#"{"through":"E99999999"}"#,
|
||||
r#"{"from":"E00000003","through":"E00000000"}"#,
|
||||
] {
|
||||
assert!(
|
||||
search
|
||||
.execute(input, agen::tool::ToolExecutionContext::direct())
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn raw_tool_arguments_reassemble_without_preview_or_utf8_loss() {
|
||||
let arguments = "日本語\n\"\\".repeat(100);
|
||||
let state = SessionExploreState::new(SessionCapture::new(
|
||||
"session",
|
||||
"segment",
|
||||
vec![Item::tool_call("call", "Read", arguments.clone())],
|
||||
));
|
||||
let capture_id = state.view().capture_id().to_owned();
|
||||
let tool = read_entry_definition(state)().1;
|
||||
let mut offset = 0;
|
||||
let mut assembled = String::new();
|
||||
loop {
|
||||
let input = serde_json::json!({
|
||||
"entry_ref": "E00000000", "mode": "raw", "raw_field": "tool_arguments",
|
||||
"byte_offset": offset, "max_bytes": 17, "capture_id": capture_id,
|
||||
});
|
||||
let output = tool
|
||||
.execute(
|
||||
&input.to_string(),
|
||||
agen::tool::ToolExecutionContext::direct(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let value: serde_json::Value = serde_json::from_str(&output.content.unwrap()).unwrap();
|
||||
assert_eq!(value["text_kind"], "raw");
|
||||
let raw = &value["raw"];
|
||||
assert_eq!(raw["offset"], offset);
|
||||
let text = raw["text"].as_str().unwrap();
|
||||
assert!(text.len() <= 17);
|
||||
assembled.push_str(text);
|
||||
match raw["next_offset"].as_u64() {
|
||||
Some(next) => {
|
||||
assert!(next > offset as u64);
|
||||
offset = next as usize;
|
||||
}
|
||||
None => break,
|
||||
}
|
||||
}
|
||||
assert_eq!(assembled, arguments);
|
||||
assert!(tool.execute(r#"{"entry_ref":"E00000000","mode":"raw","raw_field":"tool_arguments","capture_id":"other"}"#,
|
||||
agen::tool::ToolExecutionContext::direct()).await.is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,13 +39,66 @@ use crate::hook::{
|
||||
HookErrorCategory, HookExecutionPolicy, HookFailurePolicy, HookPreRequestAction, PreLlmRequest,
|
||||
PreRequestContext, RunCommitted, RunCommittedContext,
|
||||
};
|
||||
use crate::session_capture::{SessionCapture, SessionEntryEvidence};
|
||||
use crate::session_capture::{CaptureBody, SessionCapture, SessionEntryEvidence};
|
||||
use crate::subjektiv::SubjektivHostConnection;
|
||||
#[cfg(test)]
|
||||
use crate::worker::WorkspaceClient;
|
||||
#[cfg(test)]
|
||||
use server_api::SubjektivMemoryBackendRequest;
|
||||
|
||||
// Pending-commit receipt interpretation belongs to Memory, not the shared capture.
|
||||
struct CommittedMemoryCall {
|
||||
call_id: String,
|
||||
name: String,
|
||||
arguments: String,
|
||||
evidence: SessionEntryEvidence,
|
||||
}
|
||||
|
||||
fn committed_pending_tool_calls(view: &SessionCapture, names: &[&str]) -> Vec<CommittedMemoryCall> {
|
||||
let pending: HashSet<_> = view
|
||||
.entries()
|
||||
.filter_map(|entry| {
|
||||
let CaptureBody::ToolResult {
|
||||
call_id,
|
||||
content: Some(content),
|
||||
is_error: false,
|
||||
..
|
||||
} = &entry.body
|
||||
else {
|
||||
return None;
|
||||
};
|
||||
(serde_json::from_str::<serde_json::Value>(content)
|
||||
.ok()
|
||||
.and_then(|value| value.get("status").cloned())
|
||||
.as_ref()
|
||||
.and_then(serde_json::Value::as_str)
|
||||
== Some("pending_commit"))
|
||||
.then_some(call_id.as_str())
|
||||
})
|
||||
.collect();
|
||||
view.entries()
|
||||
.filter_map(|entry| {
|
||||
let CaptureBody::ToolCall {
|
||||
call_id,
|
||||
name,
|
||||
arguments,
|
||||
} = &entry.body
|
||||
else {
|
||||
return None;
|
||||
};
|
||||
if !names.contains(&name.as_str()) || !pending.contains(call_id.as_str()) {
|
||||
return None;
|
||||
}
|
||||
Some(CommittedMemoryCall {
|
||||
call_id: call_id.clone(),
|
||||
name: name.clone(),
|
||||
arguments: arguments.clone(),
|
||||
evidence: view.evidence_for_tool_call(call_id)?,
|
||||
})
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
const QUERY_TOOL: &str = "SubjektivMemoryQuery";
|
||||
const READ_TOOL: &str = "SubjektivMemoryRead";
|
||||
const LIST_CHANGES_TOOL: &str = "SubjektivMemoryListChanges";
|
||||
@@ -518,14 +571,18 @@ impl SubjektivMemoryState {
|
||||
.capture()
|
||||
.map_err(|error| ToolError::ExecutionFailed(error.to_string()))?;
|
||||
let session_id = capture.session_id.clone();
|
||||
let view = SessionCapture::from_history_entries(capture.segment_id, capture.history);
|
||||
let committed = view
|
||||
.committed_pending_tool_calls(&[REMEMBER_TOOL, PROPOSE_CHANGE_TOOL])
|
||||
.into_iter()
|
||||
.any(|call| {
|
||||
receipt_id(&session_id, &call.call_id)
|
||||
.is_ok_and(|committed_receipt| committed_receipt == expected_receipt_id)
|
||||
});
|
||||
let view = SessionCapture::from_complete_history(
|
||||
capture.session_id.clone(),
|
||||
capture.segment_id,
|
||||
capture.history,
|
||||
);
|
||||
let committed =
|
||||
committed_pending_tool_calls(&view, &[REMEMBER_TOOL, PROPOSE_CHANGE_TOOL])
|
||||
.into_iter()
|
||||
.any(|call| {
|
||||
receipt_id(&session_id, &call.call_id)
|
||||
.is_ok_and(|committed_receipt| committed_receipt == expected_receipt_id)
|
||||
});
|
||||
if committed {
|
||||
status.status = SubjektivMemoryReceiptStatus::PendingCommit;
|
||||
}
|
||||
@@ -623,10 +680,13 @@ fn replay_pending_committed_memory_with_policy(
|
||||
.capture
|
||||
.capture()
|
||||
.map_err(|error| HookError::new(HookErrorCategory::Dependency, error.to_string()))?;
|
||||
let view =
|
||||
SessionCapture::from_history_entries(capture.segment_id.clone(), capture.history.clone());
|
||||
let view = SessionCapture::from_complete_history(
|
||||
capture.session_id.clone(),
|
||||
capture.segment_id.clone(),
|
||||
capture.history.clone(),
|
||||
);
|
||||
let mut failures = Vec::new();
|
||||
for call in view.committed_pending_tool_calls(&[REMEMBER_TOOL, PROPOSE_CHANGE_TOOL]) {
|
||||
for call in committed_pending_tool_calls(&view, &[REMEMBER_TOOL, PROPOSE_CHANGE_TOOL]) {
|
||||
let receipt_id = match receipt_id(&capture.session_id, &call.call_id) {
|
||||
Ok(value) => value,
|
||||
Err(error) => {
|
||||
@@ -755,8 +815,11 @@ fn resolve_entry_refs(
|
||||
capture: &crate::feature::session::CommittedSessionCapture,
|
||||
entry_refs: &[String],
|
||||
) -> Result<Vec<SessionEntryEvidence>, ToolError> {
|
||||
let view =
|
||||
SessionCapture::from_history_entries(capture.segment_id.clone(), capture.history.clone());
|
||||
let view = SessionCapture::from_complete_history(
|
||||
capture.session_id.clone(),
|
||||
capture.segment_id.clone(),
|
||||
capture.history.clone(),
|
||||
);
|
||||
let mut seen = HashSet::new();
|
||||
entry_refs
|
||||
.iter()
|
||||
|
||||
@@ -570,15 +570,8 @@ impl Tool for SearchSessionEntriesTool {
|
||||
let view = latest_view(&*self.provider, ¶ms.subject).await?;
|
||||
let from = params.from.as_deref().map(parse_entry_ref).transpose()?;
|
||||
let through = params.through.as_deref().map(parse_entry_ref).transpose()?;
|
||||
if let (Some(from), Some(through)) = (&from, &through) {
|
||||
if view.source_index_for_ref(from) > view.source_index_for_ref(through) {
|
||||
return Err(ToolError::InvalidArgument(
|
||||
"SearchSessionEntries from must not be after through".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
let entries = view
|
||||
.search(&SearchOptions {
|
||||
.search_checked(&SearchOptions {
|
||||
query: params.query,
|
||||
kind: params.kind.as_deref().map(parse_kind).transpose()?,
|
||||
tool_part: params
|
||||
@@ -593,6 +586,7 @@ impl Tool for SearchSessionEntriesTool {
|
||||
through,
|
||||
offset: params.offset,
|
||||
})
|
||||
.map_err(|error| ToolError::InvalidArgument(error.to_string()))?
|
||||
.into_iter()
|
||||
.map(|entry| {
|
||||
serde_json::json!({
|
||||
@@ -604,6 +598,7 @@ impl Tool for SearchSessionEntriesTool {
|
||||
"tool_name": entry.tool_name,
|
||||
"label": entry.label,
|
||||
"text": entry.summary,
|
||||
"text_kind": "preview",
|
||||
})
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
@@ -687,7 +682,10 @@ async fn latest_view(
|
||||
.capture_worker_session(subject)
|
||||
.await
|
||||
.map_err(tool_error)?;
|
||||
// Observation providers expose a stable subject scope rather than a raw
|
||||
// Session handle. This scope is never used to select recording material.
|
||||
Ok(SessionCapture::from_session_snapshot(
|
||||
format!("observation:{}", capture.segment_id),
|
||||
capture.segment_id,
|
||||
capture.session,
|
||||
))
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
//! undecodable mutation responses do not prove non-commit; dispatched requests are never resent.
|
||||
//! This adapter does not reimplement Backend source-persistence transactions or activation.
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use serde::{Serialize, de::DeserializeOwned};
|
||||
@@ -20,6 +21,7 @@ use super::{
|
||||
ATTACHMENT_ALIAS, CONTENT_ROOT, ConfigAccess, ConfigAttachError, ConfigAttachment, DEADLINE,
|
||||
WorkspaceConfigAttachmentBackend,
|
||||
};
|
||||
use crate::feature::builtin::backend_diagnostic::{Adapter, Diagnostic, Reason, Stage};
|
||||
use crate::wip::WipOperationError;
|
||||
use crate::worker::{
|
||||
WorkspaceClient, WorkspaceClientError, WorkspaceRequest, WorkspaceRequestMethod,
|
||||
@@ -34,11 +36,21 @@ const MAX_RESPONSE_BYTES: usize = 2 * 1024 * 1024;
|
||||
#[derive(Clone)]
|
||||
pub struct WorkspaceConfigBackend {
|
||||
client: Arc<dyn WorkspaceClient>,
|
||||
deadline: Duration,
|
||||
}
|
||||
|
||||
impl WorkspaceConfigBackend {
|
||||
pub fn new(client: Arc<dyn WorkspaceClient>) -> Self {
|
||||
Self { client }
|
||||
Self {
|
||||
client,
|
||||
deadline: DEADLINE,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn with_deadline(mut self, deadline: Duration) -> Self {
|
||||
self.deadline = deadline;
|
||||
self
|
||||
}
|
||||
|
||||
fn base(&self) -> Result<String, WipOperationError> {
|
||||
@@ -47,6 +59,7 @@ impl WorkspaceConfigBackend {
|
||||
.workspace_id()
|
||||
.filter(|id| !id.is_empty())
|
||||
.ok_or_else(|| {
|
||||
diagnostic("authority", false).record(Stage::Request, Reason::Identity, None);
|
||||
failure(
|
||||
ProtocolErrorCode::PermissionDenied,
|
||||
"Workspace config authority unavailable",
|
||||
@@ -69,11 +82,13 @@ impl WorkspaceConfigBackend {
|
||||
&self,
|
||||
request: WorkspaceRequest,
|
||||
mutation: bool,
|
||||
diagnostic: &Diagnostic,
|
||||
) -> Result<T, WipOperationError> {
|
||||
let response = self
|
||||
.client
|
||||
.execute_with_timeout(request, DEADLINE)
|
||||
.execute_with_timeout(request, self.deadline)
|
||||
.map_err(|error| {
|
||||
diagnostic.transport(&error);
|
||||
if mutation && !matches!(error, WorkspaceClientError::InvalidPath(_)) {
|
||||
unknown()
|
||||
} else {
|
||||
@@ -84,6 +99,7 @@ impl WorkspaceConfigBackend {
|
||||
}
|
||||
})?;
|
||||
if response.body.len() > MAX_RESPONSE_BYTES {
|
||||
diagnostic.record(Stage::Response, Reason::ResponseSize, Some(response.status));
|
||||
return Err(if mutation {
|
||||
unknown()
|
||||
} else {
|
||||
@@ -96,8 +112,18 @@ impl WorkspaceConfigBackend {
|
||||
if !response.is_success() {
|
||||
if let Ok(error) = serde_json::from_str::<WorkspaceConfigApiError>(&response.body) {
|
||||
if error.classification == WorkspaceConfigFailureClassification::Unknown {
|
||||
diagnostic.record(
|
||||
Stage::Response,
|
||||
Reason::BackendUnknown,
|
||||
Some(response.status),
|
||||
);
|
||||
return Err(unknown());
|
||||
}
|
||||
diagnostic.record(
|
||||
Stage::Response,
|
||||
Reason::BackendRejected,
|
||||
Some(response.status),
|
||||
);
|
||||
// Do not reflect arbitrary transport/provider text into model output.
|
||||
let code = match error.status {
|
||||
401 | 403 | 404 => ProtocolErrorCode::PermissionDenied,
|
||||
@@ -111,6 +137,11 @@ impl WorkspaceConfigBackend {
|
||||
"Workspace config request rejected without commit",
|
||||
));
|
||||
}
|
||||
diagnostic.record(
|
||||
Stage::Response,
|
||||
Reason::UnrecognizedResponse,
|
||||
Some(response.status),
|
||||
);
|
||||
return Err(if mutation {
|
||||
unknown()
|
||||
} else {
|
||||
@@ -125,6 +156,7 @@ impl WorkspaceConfigBackend {
|
||||
});
|
||||
}
|
||||
serde_json::from_str(&response.body).map_err(|_| {
|
||||
diagnostic.record(Stage::Response, Reason::Decode, Some(response.status));
|
||||
if mutation {
|
||||
unknown()
|
||||
} else {
|
||||
@@ -138,10 +170,17 @@ impl WorkspaceConfigBackend {
|
||||
|
||||
/// GET checks the current grant even when the Worker has not attached yet.
|
||||
pub fn current(&self) -> Result<Option<WorkspaceConfigAttachment>, WipOperationError> {
|
||||
self.current_diagnosed(&diagnostic("current", false))
|
||||
}
|
||||
|
||||
fn current_diagnosed(
|
||||
&self,
|
||||
diagnostic: &Diagnostic,
|
||||
) -> Result<Option<WorkspaceConfigAttachment>, WipOperationError> {
|
||||
let result: Option<WorkspaceConfigAttachment> =
|
||||
self.request(WorkspaceRequest::get(self.base()?), false)?;
|
||||
self.request(WorkspaceRequest::get(self.base()?), false, diagnostic)?;
|
||||
if let Some(attachment) = &result {
|
||||
self.validate_attachment(attachment)?;
|
||||
self.validate_attachment(attachment, diagnostic)?;
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
@@ -149,6 +188,7 @@ impl WorkspaceConfigBackend {
|
||||
fn validate_attachment(
|
||||
&self,
|
||||
attachment: &WorkspaceConfigAttachment,
|
||||
diagnostic: &Diagnostic,
|
||||
) -> Result<(), WipOperationError> {
|
||||
if Some(attachment.workspace_id.as_str()) != self.client.workspace_id()
|
||||
|| attachment.alias != ATTACHMENT_ALIAS
|
||||
@@ -160,6 +200,7 @@ impl WorkspaceConfigBackend {
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_'))
|
||||
{
|
||||
diagnostic.record(Stage::Response, Reason::Identity, None);
|
||||
return Err(failure(
|
||||
ProtocolErrorCode::PermissionDenied,
|
||||
"Invalid Workspace config attachment identity",
|
||||
@@ -173,14 +214,34 @@ impl WorkspaceConfigBackend {
|
||||
suffix: &str,
|
||||
request: &R,
|
||||
mutation: bool,
|
||||
) -> Result<T, WipOperationError> {
|
||||
let operation = match suffix {
|
||||
"/observe" => "observe",
|
||||
"/read" => "read",
|
||||
"/commit" => "commit",
|
||||
"" => "attach",
|
||||
_ => "post",
|
||||
};
|
||||
self.post_diagnosed(suffix, request, mutation, diagnostic(operation, mutation))
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn post_diagnosed<T: DeserializeOwned + Send + 'static, R: Serialize>(
|
||||
&self,
|
||||
suffix: &str,
|
||||
request: &R,
|
||||
mutation: bool,
|
||||
diagnostic: Diagnostic,
|
||||
) -> Result<T, WipOperationError> {
|
||||
let body = serde_json::to_string(request).map_err(|_| {
|
||||
diagnostic.record(Stage::Request, Reason::Decode, None);
|
||||
failure(
|
||||
ProtocolErrorCode::InvalidArguments,
|
||||
"Invalid Workspace config request",
|
||||
)
|
||||
})?;
|
||||
if body.len() > MAX_RESPONSE_BYTES {
|
||||
diagnostic.record(Stage::Request, Reason::RequestSize, None);
|
||||
return Err(failure(
|
||||
ProtocolErrorCode::ResourceLimitExceeded,
|
||||
"Workspace config request exceeds limit",
|
||||
@@ -192,12 +253,16 @@ impl WorkspaceConfigBackend {
|
||||
body,
|
||||
);
|
||||
let this = self.clone();
|
||||
let request_diagnostic = diagnostic.clone();
|
||||
tokio::time::timeout(
|
||||
DEADLINE,
|
||||
tokio::task::spawn_blocking(move || this.request(request, mutation)),
|
||||
self.deadline,
|
||||
tokio::task::spawn_blocking(move || {
|
||||
this.request(request, mutation, &request_diagnostic)
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
diagnostic.record(Stage::Completion, Reason::Deadline, None);
|
||||
if mutation {
|
||||
unknown()
|
||||
} else {
|
||||
@@ -208,6 +273,7 @@ impl WorkspaceConfigBackend {
|
||||
}
|
||||
})?
|
||||
.map_err(|_| {
|
||||
diagnostic.record(Stage::Completion, Reason::TaskJoin, None);
|
||||
if mutation {
|
||||
unknown()
|
||||
} else {
|
||||
@@ -223,18 +289,22 @@ impl WorkspaceConfigBackend {
|
||||
&self,
|
||||
) -> Result<Option<WorkspaceConfigAttachment>, WipOperationError> {
|
||||
let this = self.clone();
|
||||
let diagnostic = diagnostic("current", false);
|
||||
let request_diagnostic = diagnostic.clone();
|
||||
tokio::time::timeout(
|
||||
DEADLINE,
|
||||
tokio::task::spawn_blocking(move || this.current()),
|
||||
self.deadline,
|
||||
tokio::task::spawn_blocking(move || this.current_diagnosed(&request_diagnostic)),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
diagnostic.record(Stage::Completion, Reason::Deadline, None);
|
||||
failure(
|
||||
ProtocolErrorCode::ResourceLimitExceeded,
|
||||
"Workspace config request deadline exceeded",
|
||||
)
|
||||
})?
|
||||
.map_err(|_| {
|
||||
diagnostic.record(Stage::Completion, Reason::TaskJoin, None);
|
||||
failure(
|
||||
ProtocolErrorCode::Internal,
|
||||
"Workspace config transport unavailable",
|
||||
@@ -249,10 +319,14 @@ impl WorkspaceConfigAttachmentBackend for WorkspaceConfigBackend {
|
||||
self.client.is_available()
|
||||
}
|
||||
fn validate_access(&self, access: Option<ConfigAccess>) -> Result<(), ConfigAttachError> {
|
||||
let current = self.current().map_err(attach_error)?;
|
||||
let diagnostic = diagnostic("validate_access", false);
|
||||
let current = self
|
||||
.current_diagnosed(&diagnostic)
|
||||
.map_err(|error| attach_error(error, &diagnostic))?;
|
||||
if access == Some(ConfigAccess::ReadWrite)
|
||||
&& current.is_some_and(|a| a.access == WorkspaceConfigAccess::ReadOnly)
|
||||
{
|
||||
diagnostic.record(Stage::Attach, Reason::AccessCeiling, None);
|
||||
return Err(ConfigAttachError::Denied);
|
||||
}
|
||||
Ok(())
|
||||
@@ -271,10 +345,16 @@ impl WorkspaceConfigAttachmentBackend for WorkspaceConfigBackend {
|
||||
ConfigAccess::ReadWrite => WorkspaceConfigAccess::ReadWrite,
|
||||
}),
|
||||
};
|
||||
let attachment: WorkspaceConfigAttachment =
|
||||
self.post("", &request, true).await.map_err(attach_error)?;
|
||||
self.validate_attachment(&attachment)
|
||||
.map_err(|_| ConfigAttachError::OutcomeUnknown)?;
|
||||
let diagnostic = diagnostic("attach", true);
|
||||
let attachment: WorkspaceConfigAttachment = self
|
||||
.post_diagnosed("", &request, true, diagnostic.clone())
|
||||
.await
|
||||
.map_err(|error| attach_error(error, &diagnostic))?;
|
||||
self.validate_attachment(&attachment, &diagnostic)
|
||||
.map_err(|_| {
|
||||
diagnostic.record(Stage::Attach, Reason::Identity, None);
|
||||
ConfigAttachError::OutcomeUnknown
|
||||
})?;
|
||||
Ok(ConfigAttachment {
|
||||
connection_id: attachment.connection_id,
|
||||
access: match attachment.access {
|
||||
@@ -286,7 +366,8 @@ impl WorkspaceConfigAttachmentBackend for WorkspaceConfigBackend {
|
||||
}
|
||||
}
|
||||
|
||||
fn attach_error(error: WipOperationError) -> ConfigAttachError {
|
||||
fn attach_error(error: WipOperationError, diagnostic: &Diagnostic) -> ConfigAttachError {
|
||||
diagnostic.record(Stage::Attach, Reason::AttachConversion, None);
|
||||
match error {
|
||||
WipOperationError::OutcomeUnknown(_) => ConfigAttachError::OutcomeUnknown,
|
||||
WipOperationError::Protocol(error) => match error.code {
|
||||
@@ -298,6 +379,10 @@ fn attach_error(error: WipOperationError) -> ConfigAttachError {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn diagnostic(operation: &'static str, mutation: bool) -> Diagnostic {
|
||||
Diagnostic::new(Adapter::WorkspaceConfig, operation, mutation, None)
|
||||
}
|
||||
|
||||
pub(super) fn failure(code: ProtocolErrorCode, message: &str) -> WipOperationError {
|
||||
WipOperationError::Protocol(ProtocolError {
|
||||
code,
|
||||
@@ -307,3 +392,7 @@ pub(super) fn failure(code: ProtocolErrorCode, message: &str) -> WipOperationErr
|
||||
pub(super) fn unknown() -> WipOperationError {
|
||||
WipOperationError::OutcomeUnknown("Workspace config outcome unknown; refresh before any subsequent operation; no automatic retry".into())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "backend_tests.rs"]
|
||||
mod tests;
|
||||
|
||||
@@ -0,0 +1,580 @@
|
||||
//! Production WorkspaceConfigBackend with only WorkspaceClient substituted. The transport fixture
|
||||
//! applies effects before injected reply loss, retains its state for rereading, and gates blocking
|
||||
//! completion explicitly. These ordinary worker lib tests do not claim Backend transaction coverage.
|
||||
use super::*;
|
||||
use crate::feature::builtin::backend_diagnostic::tests::Capture;
|
||||
use crate::worker::WorkspaceResponse;
|
||||
use std::sync::{Condvar, Mutex};
|
||||
use tracing::instrument::WithSubscriber;
|
||||
|
||||
const UNSAFE: &str = "secret-token /private/config credential=unsafe";
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
enum Fault {
|
||||
Unavailable,
|
||||
InvalidPath,
|
||||
Communication,
|
||||
Decode,
|
||||
Size,
|
||||
Rejected(u16),
|
||||
Unknown,
|
||||
Unrecognized,
|
||||
Identity,
|
||||
Panic,
|
||||
Deadline,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct Effects {
|
||||
requests: usize,
|
||||
mutations: usize,
|
||||
saved: Option<String>,
|
||||
completed: bool,
|
||||
released: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct Client {
|
||||
fault: Fault,
|
||||
state: Mutex<Effects>,
|
||||
changed: Condvar,
|
||||
started: tokio::sync::Notify,
|
||||
}
|
||||
impl Client {
|
||||
fn new(fault: Fault) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
fault,
|
||||
state: Mutex::new(Effects::default()),
|
||||
changed: Condvar::new(),
|
||||
started: tokio::sync::Notify::new(),
|
||||
})
|
||||
}
|
||||
fn finish(&self) {
|
||||
self.state.lock().unwrap().completed = true;
|
||||
self.changed.notify_all();
|
||||
}
|
||||
fn release(&self) {
|
||||
self.state.lock().unwrap().released = true;
|
||||
self.changed.notify_all();
|
||||
}
|
||||
fn wait_completed(&self) {
|
||||
let (state, timeout) = self
|
||||
.changed
|
||||
.wait_timeout_while(self.state.lock().unwrap(), Duration::from_secs(5), |s| {
|
||||
!s.completed
|
||||
})
|
||||
.unwrap();
|
||||
assert!(
|
||||
!timeout.timed_out() && state.completed,
|
||||
"blocking request did not finish"
|
||||
);
|
||||
}
|
||||
}
|
||||
struct Release(Arc<Client>);
|
||||
impl Drop for Release {
|
||||
fn drop(&mut self) {
|
||||
self.0.release();
|
||||
}
|
||||
}
|
||||
|
||||
fn attachment() -> WorkspaceConfigAttachment {
|
||||
WorkspaceConfigAttachment {
|
||||
workspace_id: "workspace".into(),
|
||||
connection_id: "connection-1".into(),
|
||||
alias: ATTACHMENT_ALIAS.into(),
|
||||
content_path: CONTENT_ROOT.into(),
|
||||
working_directory_id: "grant".into(),
|
||||
access: WorkspaceConfigAccess::ReadWrite,
|
||||
name: UNSAFE.into(),
|
||||
purpose: UNSAFE.into(),
|
||||
already_attached: false,
|
||||
}
|
||||
}
|
||||
|
||||
impl WorkspaceClient for Client {
|
||||
fn workspace_id(&self) -> Option<&str> {
|
||||
Some("workspace")
|
||||
}
|
||||
fn kind(&self) -> &str {
|
||||
"diagnostic-test-boundary"
|
||||
}
|
||||
fn is_available(&self) -> bool {
|
||||
true
|
||||
}
|
||||
fn execute(
|
||||
&self,
|
||||
request: WorkspaceRequest,
|
||||
) -> Result<WorkspaceResponse, WorkspaceClientError> {
|
||||
{
|
||||
let mut state = self.state.lock().unwrap();
|
||||
state.requests += 1;
|
||||
if request.method == WorkspaceRequestMethod::Post
|
||||
&& !request.path.ends_with("/read")
|
||||
&& !matches!(
|
||||
self.fault,
|
||||
Fault::Unavailable | Fault::InvalidPath | Fault::Rejected(_)
|
||||
)
|
||||
{
|
||||
state.mutations += 1;
|
||||
state.saved = request.body.clone();
|
||||
}
|
||||
self.started.notify_one();
|
||||
if matches!(self.fault, Fault::Deadline) {
|
||||
let (_state, timeout) = self
|
||||
.changed
|
||||
.wait_timeout_while(state, Duration::from_secs(5), |s| !s.released)
|
||||
.unwrap();
|
||||
assert!(!timeout.timed_out(), "test must release blocked transport");
|
||||
}
|
||||
}
|
||||
self.finish();
|
||||
match self.fault {
|
||||
Fault::Unavailable => Err(WorkspaceClientError::Unavailable(UNSAFE.into())),
|
||||
Fault::InvalidPath => Err(WorkspaceClientError::InvalidPath(UNSAFE.into())),
|
||||
Fault::Communication => Err(WorkspaceClientError::Request(UNSAFE.into())),
|
||||
Fault::Panic => panic!("injected blocking task failure"),
|
||||
Fault::Decode => Ok(WorkspaceResponse {
|
||||
status: 200,
|
||||
body: UNSAFE.into(),
|
||||
}),
|
||||
Fault::Size => Ok(WorkspaceResponse {
|
||||
status: 200,
|
||||
body: UNSAFE.repeat(MAX_RESPONSE_BYTES / UNSAFE.len() + 1),
|
||||
}),
|
||||
Fault::Rejected(_) | Fault::Unknown => {
|
||||
let status = match self.fault {
|
||||
Fault::Rejected(status) => status,
|
||||
_ => 503,
|
||||
};
|
||||
Ok(WorkspaceResponse {
|
||||
status,
|
||||
body: serde_json::to_string(&WorkspaceConfigApiError {
|
||||
status,
|
||||
code: UNSAFE.into(),
|
||||
message: UNSAFE.into(),
|
||||
classification: if matches!(self.fault, Fault::Unknown) {
|
||||
WorkspaceConfigFailureClassification::Unknown
|
||||
} else {
|
||||
WorkspaceConfigFailureClassification::NotCommitted
|
||||
},
|
||||
})
|
||||
.unwrap(),
|
||||
})
|
||||
}
|
||||
Fault::Unrecognized => Ok(WorkspaceResponse {
|
||||
status: 503,
|
||||
body: UNSAFE.into(),
|
||||
}),
|
||||
Fault::Identity | Fault::Deadline => {
|
||||
let mut value = attachment();
|
||||
if matches!(self.fault, Fault::Identity) {
|
||||
value.workspace_id = UNSAFE.into();
|
||||
}
|
||||
Ok(WorkspaceResponse {
|
||||
status: 200,
|
||||
body: serde_json::to_string(&value).unwrap(),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_safe(capture: &Capture, reason: Reason, stage: Stage, public: &str) {
|
||||
let text = capture.text();
|
||||
assert!(
|
||||
text.contains(&format!("reason={reason:?}")),
|
||||
"missing reason: {text}"
|
||||
);
|
||||
assert!(
|
||||
text.contains(&format!("stage={stage:?}")),
|
||||
"missing stage: {text}"
|
||||
);
|
||||
assert!(text.contains("adapter=WorkspaceConfig"), "{text}");
|
||||
assert!(text.contains("correlation="), "{text}");
|
||||
for secret in [
|
||||
UNSAFE,
|
||||
"secret-token",
|
||||
"/private/config",
|
||||
"credential=unsafe",
|
||||
] {
|
||||
assert!(
|
||||
!text.contains(secret),
|
||||
"diagnostics leaked unsafe boundary data"
|
||||
);
|
||||
assert!(
|
||||
!public.contains(secret),
|
||||
"public result leaked unsafe boundary data"
|
||||
);
|
||||
}
|
||||
}
|
||||
fn public_error(
|
||||
result: Result<serde_json::Value, WipOperationError>,
|
||||
code: Option<ProtocolErrorCode>,
|
||||
) -> String {
|
||||
match (result, code) {
|
||||
(Err(WipOperationError::OutcomeUnknown(message)), None) => {
|
||||
assert!(message.contains("no automatic retry"));
|
||||
message
|
||||
}
|
||||
(Err(WipOperationError::Protocol(error)), Some(expected)) => {
|
||||
assert_eq!(error.code, expected);
|
||||
error.message
|
||||
}
|
||||
_ => panic!("unexpected public error classification"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn production_reply_failures_retain_reasons_without_retrying_mutations() {
|
||||
let cases = [
|
||||
(
|
||||
Fault::Unavailable,
|
||||
Reason::TransportUnavailable,
|
||||
Stage::Request,
|
||||
ProtocolErrorCode::Internal,
|
||||
false,
|
||||
),
|
||||
(
|
||||
Fault::InvalidPath,
|
||||
Reason::InvalidPath,
|
||||
Stage::Request,
|
||||
ProtocolErrorCode::Internal,
|
||||
false,
|
||||
),
|
||||
(
|
||||
Fault::Communication,
|
||||
Reason::Communication,
|
||||
Stage::Request,
|
||||
ProtocolErrorCode::Internal,
|
||||
true,
|
||||
),
|
||||
(
|
||||
Fault::Decode,
|
||||
Reason::Decode,
|
||||
Stage::Response,
|
||||
ProtocolErrorCode::Internal,
|
||||
true,
|
||||
),
|
||||
(
|
||||
Fault::Size,
|
||||
Reason::ResponseSize,
|
||||
Stage::Response,
|
||||
ProtocolErrorCode::ResourceLimitExceeded,
|
||||
true,
|
||||
),
|
||||
(
|
||||
Fault::Unknown,
|
||||
Reason::BackendUnknown,
|
||||
Stage::Response,
|
||||
ProtocolErrorCode::Internal,
|
||||
true,
|
||||
),
|
||||
(
|
||||
Fault::Unrecognized,
|
||||
Reason::UnrecognizedResponse,
|
||||
Stage::Response,
|
||||
ProtocolErrorCode::Internal,
|
||||
true,
|
||||
),
|
||||
(
|
||||
Fault::Panic,
|
||||
Reason::TaskJoin,
|
||||
Stage::Completion,
|
||||
ProtocolErrorCode::Internal,
|
||||
true,
|
||||
),
|
||||
];
|
||||
for (fault, reason, stage, read_code, applies) in cases {
|
||||
for mutation in [false, true] {
|
||||
let client = Client::new(fault);
|
||||
let backend = WorkspaceConfigBackend::new(client.clone());
|
||||
let capture = Capture::default();
|
||||
let result = backend
|
||||
.post::<serde_json::Value, _>(
|
||||
if mutation { "/commit" } else { "/read" },
|
||||
&serde_json::json!({"content": UNSAFE}),
|
||||
mutation,
|
||||
)
|
||||
.with_subscriber(capture.dispatch())
|
||||
.await;
|
||||
let code = if (mutation && !matches!(fault, Fault::InvalidPath))
|
||||
|| matches!(fault, Fault::Unknown)
|
||||
{
|
||||
None
|
||||
} else {
|
||||
Some(read_code)
|
||||
};
|
||||
let public = public_error(result, code);
|
||||
assert_safe(&capture, reason, stage, &public);
|
||||
let state = client.state.lock().unwrap();
|
||||
assert_eq!(state.requests, 1);
|
||||
assert_eq!(state.mutations, usize::from(mutation && applies));
|
||||
assert_eq!(state.saved.is_some(), mutation && applies);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn production_typed_rejections_preserve_public_codes_and_never_apply() {
|
||||
for (status, code) in [
|
||||
(403, ProtocolErrorCode::PermissionDenied),
|
||||
(409, ProtocolErrorCode::ValidatorMismatch),
|
||||
(413, ProtocolErrorCode::ResourceLimitExceeded),
|
||||
(422, ProtocolErrorCode::InvalidArguments),
|
||||
(500, ProtocolErrorCode::Internal),
|
||||
] {
|
||||
for mutation in [false, true] {
|
||||
let client = Client::new(Fault::Rejected(status));
|
||||
let backend = WorkspaceConfigBackend::new(client.clone());
|
||||
let capture = Capture::default();
|
||||
let public = public_error(
|
||||
backend
|
||||
.post::<serde_json::Value, _>(
|
||||
if mutation { "/commit" } else { "/read" },
|
||||
&(),
|
||||
mutation,
|
||||
)
|
||||
.with_subscriber(capture.dispatch())
|
||||
.await,
|
||||
Some(code),
|
||||
);
|
||||
assert_safe(&capture, Reason::BackendRejected, Stage::Response, &public);
|
||||
assert!(capture.text().contains(&format!("status={status}")));
|
||||
let state = client.state.lock().unwrap();
|
||||
assert_eq!(state.requests, 1);
|
||||
assert_eq!(state.mutations, 0);
|
||||
assert!(state.saved.is_none());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn production_deadline_is_unknown_after_effect_and_blocking_completion_is_not_retried() {
|
||||
for mutation in [false, true] {
|
||||
let client = Client::new(Fault::Deadline);
|
||||
let release = Release(client.clone());
|
||||
let backend =
|
||||
WorkspaceConfigBackend::new(client.clone()).with_deadline(Duration::from_secs(1));
|
||||
let capture = Capture::default();
|
||||
let body = serde_json::json!({"content": UNSAFE});
|
||||
let request = backend
|
||||
.post::<serde_json::Value, _>(
|
||||
if mutation { "/commit" } else { "/read" },
|
||||
&body,
|
||||
mutation,
|
||||
)
|
||||
.with_subscriber(capture.dispatch());
|
||||
tokio::pin!(request);
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = client.started.notified() => {},
|
||||
_ = &mut request => panic!("request must start and apply its effect before timeout is observed"),
|
||||
}
|
||||
{
|
||||
let state = client.state.lock().unwrap();
|
||||
assert_eq!(state.mutations, usize::from(mutation));
|
||||
assert_eq!(state.saved.is_some(), mutation);
|
||||
assert!(!state.completed && !state.released);
|
||||
}
|
||||
let result = request.await;
|
||||
let public = public_error(
|
||||
result,
|
||||
if mutation {
|
||||
None
|
||||
} else {
|
||||
Some(ProtocolErrorCode::ResourceLimitExceeded)
|
||||
},
|
||||
);
|
||||
assert_safe(&capture, Reason::Deadline, Stage::Completion, &public);
|
||||
drop(release);
|
||||
let completed = client.clone();
|
||||
tokio::task::spawn_blocking(move || completed.wait_completed())
|
||||
.await
|
||||
.unwrap();
|
||||
let state = client.state.lock().unwrap();
|
||||
assert_eq!(state.requests, 1);
|
||||
assert_eq!(state.mutations, usize::from(mutation));
|
||||
assert_eq!(state.saved.is_some(), mutation);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn production_current_identity_and_task_loss_remain_safe() {
|
||||
for (fault, reason, stage, code) in [
|
||||
(
|
||||
Fault::Identity,
|
||||
Reason::Identity,
|
||||
Stage::Response,
|
||||
ProtocolErrorCode::PermissionDenied,
|
||||
),
|
||||
(
|
||||
Fault::Panic,
|
||||
Reason::TaskJoin,
|
||||
Stage::Completion,
|
||||
ProtocolErrorCode::Internal,
|
||||
),
|
||||
(
|
||||
Fault::Decode,
|
||||
Reason::Decode,
|
||||
Stage::Response,
|
||||
ProtocolErrorCode::Internal,
|
||||
),
|
||||
] {
|
||||
let client = Client::new(fault);
|
||||
let backend = WorkspaceConfigBackend::new(client.clone());
|
||||
let capture = Capture::default();
|
||||
let result = backend
|
||||
.current_async()
|
||||
.with_subscriber(capture.dispatch())
|
||||
.await;
|
||||
let message = match result {
|
||||
Err(WipOperationError::Protocol(error)) => {
|
||||
assert_eq!(error.code, code);
|
||||
error.message
|
||||
}
|
||||
_ => panic!("unexpected current classification"),
|
||||
};
|
||||
assert_safe(&capture, reason, stage, &message);
|
||||
assert_eq!(client.state.lock().unwrap().mutations, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn production_attach_conversion_retains_original_reason_and_effect_count() {
|
||||
for (fault, reason, expected, mutations) in [
|
||||
(
|
||||
Fault::Communication,
|
||||
Reason::Communication,
|
||||
ConfigAttachError::OutcomeUnknown,
|
||||
1,
|
||||
),
|
||||
(
|
||||
Fault::Identity,
|
||||
Reason::Identity,
|
||||
ConfigAttachError::OutcomeUnknown,
|
||||
1,
|
||||
),
|
||||
(
|
||||
Fault::Rejected(403),
|
||||
Reason::BackendRejected,
|
||||
ConfigAttachError::Denied,
|
||||
0,
|
||||
),
|
||||
(
|
||||
Fault::Rejected(422),
|
||||
Reason::BackendRejected,
|
||||
ConfigAttachError::InvalidRequest,
|
||||
0,
|
||||
),
|
||||
(
|
||||
Fault::Rejected(500),
|
||||
Reason::BackendRejected,
|
||||
ConfigAttachError::Unavailable,
|
||||
0,
|
||||
),
|
||||
] {
|
||||
let client = Client::new(fault);
|
||||
let backend = WorkspaceConfigBackend::new(client.clone());
|
||||
let capture = Capture::default();
|
||||
assert_eq!(
|
||||
backend
|
||||
.attach("host-key-not-logged", None)
|
||||
.with_subscriber(capture.dispatch())
|
||||
.await,
|
||||
Err(expected)
|
||||
);
|
||||
assert_safe(&capture, reason, Stage::Attach, expected.message());
|
||||
let text = capture.text();
|
||||
if !matches!(fault, Fault::Identity) {
|
||||
assert!(text.contains("reason=AttachConversion"), "{text}");
|
||||
}
|
||||
assert!(!text.contains("host-key-not-logged"));
|
||||
let correlations: std::collections::BTreeSet<_> = text
|
||||
.lines()
|
||||
.filter_map(|line| line.split("correlation=").nth(1))
|
||||
.filter_map(|tail| tail.split(';').next())
|
||||
.collect();
|
||||
assert_eq!(
|
||||
correlations.len(),
|
||||
1,
|
||||
"request and attach conversion must correlate"
|
||||
);
|
||||
let state = client.state.lock().unwrap();
|
||||
assert_eq!(state.requests, 1);
|
||||
assert_eq!(state.mutations, mutations);
|
||||
assert_eq!(state.saved.is_some(), mutations == 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn production_oversized_request_is_rejected_before_dispatch() {
|
||||
let client = Client::new(Fault::Communication);
|
||||
let backend = WorkspaceConfigBackend::new(client.clone());
|
||||
let capture = Capture::default();
|
||||
let public = public_error(
|
||||
backend
|
||||
.post::<serde_json::Value, _>("/commit", &"x".repeat(MAX_RESPONSE_BYTES), true)
|
||||
.with_subscriber(capture.dispatch())
|
||||
.await,
|
||||
Some(ProtocolErrorCode::ResourceLimitExceeded),
|
||||
);
|
||||
assert_safe(&capture, Reason::RequestSize, Stage::Request, &public);
|
||||
let state = client.state.lock().unwrap();
|
||||
assert_eq!(state.requests, 0);
|
||||
assert_eq!(state.mutations, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn production_current_deadline_remains_bounded_and_has_no_mutation() {
|
||||
let client = Client::new(Fault::Deadline);
|
||||
let release = Release(client.clone());
|
||||
let backend = WorkspaceConfigBackend::new(client.clone()).with_deadline(Duration::from_secs(1));
|
||||
let capture = Capture::default();
|
||||
let request = backend.current_async().with_subscriber(capture.dispatch());
|
||||
tokio::pin!(request);
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = client.started.notified() => {},
|
||||
_ = &mut request => panic!("current request must enter the gate before timeout is observed"),
|
||||
}
|
||||
assert_eq!(client.state.lock().unwrap().mutations, 0);
|
||||
let error = request.await;
|
||||
let message = match error {
|
||||
Err(WipOperationError::Protocol(error)) => {
|
||||
assert_eq!(error.code, ProtocolErrorCode::ResourceLimitExceeded);
|
||||
error.message
|
||||
}
|
||||
_ => panic!("current deadline must remain a public resource-limit error"),
|
||||
};
|
||||
assert_safe(&capture, Reason::Deadline, Stage::Completion, &message);
|
||||
drop(release);
|
||||
let completed = client.clone();
|
||||
tokio::task::spawn_blocking(move || completed.wait_completed())
|
||||
.await
|
||||
.unwrap();
|
||||
let state = client.state.lock().unwrap();
|
||||
assert_eq!(state.requests, 1);
|
||||
assert_eq!(state.mutations, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn production_request_encoding_failure_is_safe_and_never_dispatched() {
|
||||
struct Unencodable;
|
||||
impl Serialize for Unencodable {
|
||||
fn serialize<S: serde::Serializer>(&self, _: S) -> Result<S::Ok, S::Error> {
|
||||
Err(serde::ser::Error::custom(UNSAFE))
|
||||
}
|
||||
}
|
||||
let client = Client::new(Fault::Communication);
|
||||
let backend = WorkspaceConfigBackend::new(client.clone());
|
||||
let capture = Capture::default();
|
||||
let result = backend
|
||||
.post::<serde_json::Value, _>("/commit", &Unencodable, true)
|
||||
.with_subscriber(capture.dispatch())
|
||||
.await;
|
||||
let message = public_error(result, Some(ProtocolErrorCode::InvalidArguments));
|
||||
assert_safe(&capture, Reason::Decode, Stage::Request, &message);
|
||||
let state = client.state.lock().unwrap();
|
||||
assert_eq!(state.requests, 0);
|
||||
assert_eq!(state.mutations, 0);
|
||||
}
|
||||
@@ -36,9 +36,11 @@ use wip_protocol::{
|
||||
};
|
||||
|
||||
use super::backend::{
|
||||
MAX_DEPTH, MAX_NODES, MAX_PATH_BYTES, MAX_TEXT_BYTES, WorkspaceConfigBackend, failure, unknown,
|
||||
MAX_DEPTH, MAX_NODES, MAX_PATH_BYTES, MAX_TEXT_BYTES, WorkspaceConfigBackend, diagnostic,
|
||||
failure, unknown,
|
||||
};
|
||||
use super::{CONTENT_ROOT, WorkspaceConfigFeature};
|
||||
use crate::feature::builtin::backend_diagnostic::{Diagnostic, Reason, Stage};
|
||||
use crate::wip::{
|
||||
WipCallContext, WipOperationError, WipOperationHandler, WipOperationOutput, WipProjection,
|
||||
WipProjectionKind, WipSubtreeProvider, contextual_reference, json_to_wip,
|
||||
@@ -138,6 +140,23 @@ async fn observe(
|
||||
connection: &str,
|
||||
paths: Vec<String>,
|
||||
depth: u32,
|
||||
) -> Result<WorkspaceConfigObserveResponse, WipOperationError> {
|
||||
observe_diagnosed(
|
||||
backend,
|
||||
connection,
|
||||
paths,
|
||||
depth,
|
||||
diagnostic("observe", false),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn observe_diagnosed(
|
||||
backend: &WorkspaceConfigBackend,
|
||||
connection: &str,
|
||||
paths: Vec<String>,
|
||||
depth: u32,
|
||||
diagnostic: Diagnostic,
|
||||
) -> Result<WorkspaceConfigObserveResponse, WipOperationError> {
|
||||
if paths.len() > MAX_NODES || depth > MAX_DEPTH {
|
||||
return Err(invalid());
|
||||
@@ -148,7 +167,7 @@ async fn observe(
|
||||
}
|
||||
}
|
||||
let result: WorkspaceConfigObserveResponse = backend
|
||||
.post(
|
||||
.post_diagnosed(
|
||||
"/observe",
|
||||
&WorkspaceConfigObserveRequest {
|
||||
connection_id: connection.into(),
|
||||
@@ -156,6 +175,7 @@ async fn observe(
|
||||
depth,
|
||||
},
|
||||
false,
|
||||
diagnostic.clone(),
|
||||
)
|
||||
.await?;
|
||||
if result.connection_id != connection
|
||||
@@ -164,24 +184,37 @@ async fn observe(
|
||||
|| result.validator.len() > 4096
|
||||
|| result.digest.is_empty()
|
||||
{
|
||||
diagnostic.record(
|
||||
Stage::Response,
|
||||
if result.connection_id != connection {
|
||||
Reason::Identity
|
||||
} else {
|
||||
Reason::InvalidMetadata
|
||||
},
|
||||
None,
|
||||
);
|
||||
return Err(failure(
|
||||
ProtocolErrorCode::Internal,
|
||||
"Invalid Workspace config metadata",
|
||||
));
|
||||
}
|
||||
if result.entrypoints.is_empty() || result.entrypoints.len() > MAX_NODES {
|
||||
diagnostic.record(Stage::Response, Reason::InvalidMetadata, None);
|
||||
return Err(failure(
|
||||
ProtocolErrorCode::Internal,
|
||||
"Invalid Workspace config entrypoints",
|
||||
));
|
||||
}
|
||||
for path in &result.entrypoints {
|
||||
validate_source(path)?;
|
||||
validate_source(path)
|
||||
.inspect_err(|_| diagnostic.record(Stage::Response, Reason::InvalidMetadata, None))?;
|
||||
}
|
||||
let mut seen = BTreeSet::new();
|
||||
for node in &result.nodes {
|
||||
if !node.path.is_empty() {
|
||||
validate_source(&node.path)?;
|
||||
validate_source(&node.path).inspect_err(|_| {
|
||||
diagnostic.record(Stage::Response, Reason::InvalidMetadata, None)
|
||||
})?;
|
||||
}
|
||||
if !seen.insert(&node.path)
|
||||
|| node.validator.is_empty()
|
||||
@@ -198,6 +231,7 @@ async fn observe(
|
||||
suffix.is_some_and(|s| s.split('/').count() <= depth as usize)
|
||||
})
|
||||
{
|
||||
diagnostic.record(Stage::Response, Reason::InvalidMetadata, None);
|
||||
return Err(failure(
|
||||
ProtocolErrorCode::Internal,
|
||||
"Invalid Workspace config metadata scope",
|
||||
@@ -490,9 +524,10 @@ fn content_type(value: Option<&Value>) -> Result<ConfigContentType, WipOperation
|
||||
|
||||
impl ConfigHandler {
|
||||
async fn read(&self) -> Result<WorkspaceConfigReadResponse, WipOperationError> {
|
||||
let diagnostic = diagnostic("read", false);
|
||||
let response: WorkspaceConfigReadResponse = self
|
||||
.backend
|
||||
.post(
|
||||
.post_diagnosed(
|
||||
"/read",
|
||||
&WorkspaceConfigReadRequest {
|
||||
connection_id: self.attachment.connection_id.clone(),
|
||||
@@ -500,6 +535,7 @@ impl ConfigHandler {
|
||||
validator: self.node.validator.clone(),
|
||||
},
|
||||
false,
|
||||
diagnostic.clone(),
|
||||
)
|
||||
.await?;
|
||||
if response.path != self.node.path
|
||||
@@ -507,12 +543,14 @@ impl ConfigHandler {
|
||||
|| Some(&response.digest) != self.node.digest.as_ref()
|
||||
|| Some(&response.content_type) != self.node.content_type.as_ref()
|
||||
{
|
||||
diagnostic.record(Stage::Response, Reason::Identity, None);
|
||||
return Err(failure(
|
||||
ProtocolErrorCode::Internal,
|
||||
"Invalid Workspace config read response",
|
||||
));
|
||||
}
|
||||
if response.content.len() > MAX_TEXT_BYTES {
|
||||
diagnostic.record(Stage::Response, Reason::ResponseSize, None);
|
||||
return Err(failure(
|
||||
ProtocolErrorCode::ResourceLimitExceeded,
|
||||
"Workspace config source exceeds text limit",
|
||||
@@ -630,12 +668,17 @@ impl WipOperationHandler for ConfigHandler {
|
||||
context: WipCallContext,
|
||||
) -> Result<WipOperationOutput, WipOperationError> {
|
||||
let dispatched = std::sync::atomic::AtomicBool::new(false);
|
||||
let diagnostic = diagnostic(
|
||||
"operation",
|
||||
!matches!(operation, "read" | "read_lines" | "search_text"),
|
||||
);
|
||||
tokio::time::timeout(
|
||||
super::DEADLINE,
|
||||
self.execute(operation, arguments, context, &dispatched),
|
||||
self.execute(operation, arguments, context, &dispatched, &diagnostic),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
diagnostic.record(Stage::Completion, Reason::Deadline, None);
|
||||
if dispatched.load(std::sync::atomic::Ordering::Acquire) {
|
||||
unknown()
|
||||
} else {
|
||||
@@ -655,6 +698,7 @@ impl ConfigHandler {
|
||||
arguments: &BTreeMap<String, Value>,
|
||||
context: WipCallContext,
|
||||
dispatched: &std::sync::atomic::AtomicBool,
|
||||
operation_diagnostic: &Diagnostic,
|
||||
) -> Result<WipOperationOutput, WipOperationError> {
|
||||
if !self.operations.iter().any(|name| name == operation) {
|
||||
return Err(failure(
|
||||
@@ -798,9 +842,10 @@ impl ConfigHandler {
|
||||
};
|
||||
let changes = self.canonical_changes(changes).await?;
|
||||
dispatched.store(true, std::sync::atomic::Ordering::Release);
|
||||
let diagnostic = operation_diagnostic.clone();
|
||||
let result: WorkspaceConfigCommitResponse = self
|
||||
.backend
|
||||
.post(
|
||||
.post_diagnosed(
|
||||
"/commit",
|
||||
&WorkspaceConfigCommitRequest {
|
||||
connection_id: self.attachment.connection_id.clone(),
|
||||
@@ -812,28 +857,37 @@ impl ConfigHandler {
|
||||
},
|
||||
},
|
||||
true,
|
||||
diagnostic.clone(),
|
||||
)
|
||||
.await?;
|
||||
// Commit response supplies ROOT state. Obtain the exact post-commit PATH
|
||||
// validator, never substitute the root validator for a file validator.
|
||||
let updated = observe(
|
||||
let updated = observe_diagnosed(
|
||||
&self.backend,
|
||||
&self.attachment.connection_id,
|
||||
vec![self.node.path.clone()],
|
||||
0,
|
||||
diagnostic.at_stage(Stage::Reconcile),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| unknown())?;
|
||||
if updated.digest != result.digest || updated.validator != result.validator {
|
||||
diagnostic.record(Stage::Reconcile, Reason::Identity, None);
|
||||
return Err(unknown());
|
||||
}
|
||||
let node = updated
|
||||
.nodes
|
||||
.iter()
|
||||
.find(|n| n.path == self.node.path)
|
||||
.ok_or_else(unknown)?;
|
||||
.ok_or_else(|| {
|
||||
diagnostic.record(Stage::Reconcile, Reason::InvalidMetadata, None);
|
||||
unknown()
|
||||
})?;
|
||||
let value = json_to_wip(&json!({"digest": result.digest,
|
||||
"activation": "Saved to canonical Workspace configuration; existing Worker manifests are not rewritten"})).map_err(|_| unknown())?;
|
||||
"activation": "Saved to canonical Workspace configuration; existing Worker manifests are not rewritten"})).map_err(|_| {
|
||||
diagnostic.record(Stage::Completion, Reason::Decode, None);
|
||||
unknown()
|
||||
})?;
|
||||
Ok(WipOperationOutput::native_with_validator(
|
||||
value,
|
||||
object_validator(
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
//! Worker-side E2E across real Feature invocation, WIP Client/Host and typed
|
||||
//! Workspace transport. Canonical persistence/projection tests live in Backend;
|
||||
//! this deterministic router models its published auth/CAS/response contract.
|
||||
//! Worker-side integration across real Feature invocation, WIP Client/Host and the production
|
||||
//! WorkspaceConfigBackend. Only WorkspaceClient transport is substituted by a deterministic router
|
||||
//! retaining applied effects and modeling the published auth/CAS/response contract. These ordinary
|
||||
//! worker lib tests do not replace canonical persistence/projection tests owned by Backend.
|
||||
use super::*;
|
||||
use crate::feature::FeatureRegistryBuilder;
|
||||
use crate::feature::builtin::backend_diagnostic::tests::Capture;
|
||||
use crate::hook::HookRegistryBuilder;
|
||||
use crate::wip::{WipMountRegistry, WipRuntime};
|
||||
use crate::worker::{
|
||||
@@ -16,6 +18,7 @@ use serde_json::{Value as Json, json};
|
||||
use server_api::*;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tracing::instrument::WithSubscriber;
|
||||
|
||||
#[derive(Debug)]
|
||||
struct State {
|
||||
@@ -26,6 +29,10 @@ struct State {
|
||||
entries: BTreeMap<String, (ConfigContentType, String)>,
|
||||
commits: Vec<WorkspaceConfigCommitRequest>,
|
||||
requests: Vec<WorkspaceRequest>,
|
||||
applied_commits: usize,
|
||||
malformed_read_reply: Option<&'static str>,
|
||||
malformed_observe_reply: Option<&'static str>,
|
||||
lose_postcommit_observe_reply: bool,
|
||||
fail_commit: bool,
|
||||
fail_current: bool,
|
||||
lose_commit_reply: bool,
|
||||
@@ -58,6 +65,10 @@ impl Router {
|
||||
]),
|
||||
commits: Vec::new(),
|
||||
requests: Vec::new(),
|
||||
applied_commits: 0,
|
||||
malformed_read_reply: None,
|
||||
malformed_observe_reply: None,
|
||||
lose_postcommit_observe_reply: false,
|
||||
fail_commit: false,
|
||||
fail_current: false,
|
||||
lose_commit_reply: false,
|
||||
@@ -222,6 +233,11 @@ impl WorkspaceClient for Router {
|
||||
}
|
||||
match suffix {
|
||||
"/observe" => {
|
||||
if state.lose_postcommit_observe_reply && state.applied_commits > 0 {
|
||||
return Err(WorkspaceClientError::Request(
|
||||
"secret-token /private/host/config".into(),
|
||||
));
|
||||
}
|
||||
let request: WorkspaceConfigObserveRequest = serde_json::from_value(input).unwrap();
|
||||
assert!(request.depth <= 8 && request.paths.len() <= 256);
|
||||
let mut kinds =
|
||||
@@ -268,13 +284,23 @@ impl WorkspaceClient for Router {
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
Ok(response(&WorkspaceConfigObserveResponse {
|
||||
let mut result = WorkspaceConfigObserveResponse {
|
||||
connection_id: state.active.as_ref().unwrap().connection_id.clone(),
|
||||
validator: state.validator(""),
|
||||
digest: state.tree_digest(),
|
||||
entrypoints: vec!["main.dcdl".into()],
|
||||
nodes,
|
||||
}))
|
||||
};
|
||||
match state.malformed_observe_reply {
|
||||
Some("connection") => {
|
||||
result.connection_id = "secret-token /private/host/config".into()
|
||||
}
|
||||
Some("entrypoints") => result.entrypoints.clear(),
|
||||
Some("scope") => result.nodes[0].path = "outside/path.dcdl".into(),
|
||||
None => {}
|
||||
Some(_) => unreachable!(),
|
||||
}
|
||||
Ok(response(&result))
|
||||
}
|
||||
"/read" => {
|
||||
let request: WorkspaceConfigReadRequest = serde_json::from_value(input).unwrap();
|
||||
@@ -290,13 +316,20 @@ impl WorkspaceClient for Router {
|
||||
WorkspaceConfigFailureClassification::NotCommitted,
|
||||
));
|
||||
};
|
||||
Ok(response(&WorkspaceConfigReadResponse {
|
||||
let mut result = WorkspaceConfigReadResponse {
|
||||
path: request.path.clone(),
|
||||
content: content.clone(),
|
||||
content_type: *content_type,
|
||||
digest: digest(content),
|
||||
validator: state.validator(&request.path),
|
||||
}))
|
||||
};
|
||||
match state.malformed_read_reply {
|
||||
Some("identity") => result.path = "secret-token /private/host/config".into(),
|
||||
Some("size") => result.content = "x".repeat(256 * 1024 + 1),
|
||||
None => {}
|
||||
Some(_) => unreachable!(),
|
||||
}
|
||||
Ok(response(&result))
|
||||
}
|
||||
"/commit" => {
|
||||
let request: WorkspaceConfigCommitRequest = serde_json::from_value(input).unwrap();
|
||||
@@ -403,6 +436,7 @@ impl WorkspaceClient for Router {
|
||||
));
|
||||
}
|
||||
state.entries = entries;
|
||||
state.applied_commits += 1;
|
||||
if state.lose_commit_reply {
|
||||
state.lose_commit_reply = false;
|
||||
return Err(WorkspaceClientError::Request(
|
||||
@@ -797,14 +831,20 @@ async fn post_commit_observation_mismatch_reports_unknown_without_replaying_writ
|
||||
let path = "/workspace-config/main.dcdl";
|
||||
observe_interface(&runtime, path, true).await;
|
||||
client.state.lock().unwrap().mismatched_commit_reply = Some(field);
|
||||
let capture = Capture::default();
|
||||
let error = call(&runtime, path, "write", json!({"content":"saved"}))
|
||||
.with_subscriber(capture.dispatch())
|
||||
.await
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(error.contains("unknown"), "{field}: {error}");
|
||||
assert!(capture.text().contains("stage=Reconcile"));
|
||||
assert!(capture.text().contains("reason=Identity"));
|
||||
assert!(!capture.text().contains("saved"));
|
||||
let state = client.state.lock().unwrap();
|
||||
assert_eq!(state.commits.len(), 1);
|
||||
assert_eq!(state.entries["main.dcdl"].1, "saved");
|
||||
assert_eq!(state.applied_commits, 1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1155,3 +1195,110 @@ async fn workspace_config_backend_validators_reject_path_and_connection_transpla
|
||||
assert!(result.content.unwrap().contains("model = updated"));
|
||||
assert_eq!(client.state.lock().unwrap().commits.len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn production_read_validation_retains_reason_without_committing() {
|
||||
for (fault, reason) in [("identity", "Identity"), ("size", "ResponseSize")] {
|
||||
let client = Router::new();
|
||||
let (feature, runtime) = runtime(client.clone());
|
||||
feature.attach("initial", None).await.unwrap();
|
||||
let path = "/workspace-config/main.dcdl";
|
||||
observe_interface(&runtime, path, true).await;
|
||||
client.state.lock().unwrap().malformed_read_reply = Some(fault);
|
||||
let capture = Capture::default();
|
||||
let error = call(&runtime, path, "read", json!({}))
|
||||
.with_subscriber(capture.dispatch())
|
||||
.await
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
let text = capture.text();
|
||||
assert!(text.contains(&format!("reason={reason}")), "{text}");
|
||||
assert!(text.contains("stage=Response"));
|
||||
assert!(!error.contains("unknown"));
|
||||
assert!(!text.contains("secret-token") && !text.contains("/private/host"));
|
||||
assert!(!error.contains("secret-token") && !error.contains("/private/host"));
|
||||
let state = client.state.lock().unwrap();
|
||||
assert_eq!(state.applied_commits, 0);
|
||||
assert_eq!(state.entries["main.dcdl"].1, "model = old");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn production_observation_validation_retains_reason_without_committing() {
|
||||
for (fault, reason) in [
|
||||
("connection", "Identity"),
|
||||
("entrypoints", "InvalidMetadata"),
|
||||
("scope", "InvalidMetadata"),
|
||||
] {
|
||||
let client = Router::new();
|
||||
let (feature, runtime) = runtime(client.clone());
|
||||
feature.attach("initial", None).await.unwrap();
|
||||
client.state.lock().unwrap().malformed_observe_reply = Some(fault);
|
||||
let capture = Capture::default();
|
||||
let error = runtime
|
||||
.tree("/workspace-config/main.dcdl".into(), 0, true)
|
||||
.with_subscriber(capture.dispatch())
|
||||
.await
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
let text = capture.text();
|
||||
assert!(text.contains(&format!("reason={reason}")), "{text}");
|
||||
assert!(text.contains("stage=Response"));
|
||||
assert!(!text.contains("secret-token") && !text.contains("/private/host"));
|
||||
assert!(!error.contains("secret-token") && !error.contains("/private/host"));
|
||||
assert_eq!(client.state.lock().unwrap().applied_commits, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn production_postcommit_transport_loss_retains_cause_and_unknown_without_retry() {
|
||||
let client = Router::new();
|
||||
let (feature, runtime) = runtime(client.clone());
|
||||
feature.attach("initial", None).await.unwrap();
|
||||
let path = "/workspace-config/main.dcdl";
|
||||
observe_interface(&runtime, path, true).await;
|
||||
client.state.lock().unwrap().lose_postcommit_observe_reply = true;
|
||||
let capture = Capture::default();
|
||||
let error = call(
|
||||
&runtime,
|
||||
path,
|
||||
"write",
|
||||
json!({"content":"saved-before-observation-loss"}),
|
||||
)
|
||||
.with_subscriber(capture.dispatch())
|
||||
.await
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(error.contains("unknown"));
|
||||
let text = capture.text();
|
||||
assert!(
|
||||
text.lines().any(
|
||||
|event| event.contains("reason=Communication") && event.contains("stage=Reconcile")
|
||||
),
|
||||
"{text}"
|
||||
);
|
||||
assert!(
|
||||
!text.contains("reason=UnrecognizedResponse"),
|
||||
"must not invent a response-decode cause for transport loss"
|
||||
);
|
||||
assert!(!text.contains("secret-token") && !text.contains("/private/host"));
|
||||
assert!(!text.contains("saved-before-observation-loss"));
|
||||
assert!(!error.contains("secret-token") && !error.contains("/private/host"));
|
||||
let correlations: BTreeSet<_> = text
|
||||
.lines()
|
||||
.filter_map(|line| line.split("correlation=").nth(1))
|
||||
.filter_map(|tail| tail.split(';').next())
|
||||
.collect();
|
||||
assert_eq!(
|
||||
correlations.len(),
|
||||
1,
|
||||
"lost postcommit observation must correlate to original commit"
|
||||
);
|
||||
let state = client.state.lock().unwrap();
|
||||
assert_eq!(state.commits.len(), 1);
|
||||
assert_eq!(state.applied_commits, 1);
|
||||
assert_eq!(
|
||||
state.entries["main.dcdl"].1,
|
||||
"saved-before-observation-loss"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -161,7 +161,14 @@ impl WipOperationHandler for AttachHandler {
|
||||
"attachment_path": format!("/workdir-attachments/{}", encode_identity(&result.connection_id)),
|
||||
"content_path": CONTENT_ROOT,
|
||||
"already_attached": result.already_attached,
|
||||
})).map_err(|_| WipOperationError::OutcomeUnknown("Completed config attachment response unavailable".into()))?;
|
||||
})).map_err(|_| {
|
||||
super::backend::diagnostic("attach_output", true).record(
|
||||
crate::feature::builtin::backend_diagnostic::Stage::Attach,
|
||||
crate::feature::builtin::backend_diagnostic::Reason::Decode,
|
||||
None,
|
||||
);
|
||||
WipOperationError::OutcomeUnknown("Completed config attachment response unavailable".into())
|
||||
})?;
|
||||
Ok(WipOperationOutput::native(value))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
//! the regular registry decides Tool-name collisions.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::io;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -25,6 +26,7 @@ use mcp::stdio::{
|
||||
};
|
||||
use serde::Serialize;
|
||||
use serde_json::{Map, Value};
|
||||
use tokio::sync::Mutex;
|
||||
|
||||
use super::{
|
||||
FeatureDescriptor, FeatureDiagnostic, FeatureInstallContext, FeatureInstallError,
|
||||
@@ -52,8 +54,88 @@ const MAX_LIST_ITEMS: usize = 128;
|
||||
const MAX_RESOURCE_CONTENTS: usize = 16;
|
||||
const MAX_PROMPT_MESSAGES: usize = 32;
|
||||
|
||||
/// Owns discovery clients until process reaping is confirmed. Retain a clone in
|
||||
/// startup cleanup *before* awaiting discovery. Stop/cancel discovery before
|
||||
/// calling `shutdown`; the shared lock serializes discovery and cleanup, not
|
||||
/// cancellation of an in-flight discovery task.
|
||||
///
|
||||
/// Error or cancellation leaves the actual clients here for another shutdown
|
||||
/// attempt. Dropping the final owner only provides best-effort process killing.
|
||||
#[derive(Clone, Default)]
|
||||
pub struct McpDiscoveryCleanup {
|
||||
clients: Arc<Mutex<Vec<McpStdioClient>>>,
|
||||
}
|
||||
|
||||
impl McpDiscoveryCleanup {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
|
||||
/// Try every retained client once, removing only confirmed shutdowns. An
|
||||
/// error is not a cleanup receipt; retry this same owner, not discovery.
|
||||
pub async fn shutdown(&self) -> io::Result<()> {
|
||||
let mut clients = self.clients.lock().await;
|
||||
let mut first_error = None;
|
||||
let mut index = 0;
|
||||
while index < clients.len() {
|
||||
match clients[index].shutdown().await {
|
||||
Ok(_) => {
|
||||
clients.remove(index);
|
||||
}
|
||||
Err(err) => {
|
||||
first_error.get_or_insert_with(|| discovery_shutdown_error(&err));
|
||||
index += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
match first_error {
|
||||
Some(err) => Err(err),
|
||||
None => Ok(()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn discovery_shutdown_error(err: &McpClientError) -> io::Error {
|
||||
io::Error::other(bounded_diagnostic(format!(
|
||||
"MCP discovery shutdown unconfirmed; client retained for cleanup retry: {err}"
|
||||
)))
|
||||
}
|
||||
|
||||
/// Restore preparation must fail if discovery cannot confirm process shutdown.
|
||||
/// The caller retains `cleanup` before this future is polled, including during
|
||||
/// initialization. Ordinary discovery failures remain isolated contributions
|
||||
/// only after their client has been reaped.
|
||||
pub async fn discover_stdio_tool_feature_with_cleanup(
|
||||
config: &McpConfig,
|
||||
workspace_root: &Path,
|
||||
cleanup: &McpDiscoveryCleanup,
|
||||
) -> Result<Option<McpStdioToolFeature>, io::Error> {
|
||||
if config.stdio_servers.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
let mut feature = McpStdioToolFeature::new();
|
||||
for server in &config.stdio_servers {
|
||||
match resolve_stdio_server(server, workspace_root, None) {
|
||||
Ok(spec) => {
|
||||
feature.add_contribution(
|
||||
discover_server_tools_with_cleanup(spec, cleanup, McpStdioLimits::default())
|
||||
.await?,
|
||||
);
|
||||
}
|
||||
Err(err) => {
|
||||
feature.add_diagnostic(FeatureDiagnostic::error(bounded_diagnostic(format!(
|
||||
"failed to resolve MCP stdio server `{}`: {err}",
|
||||
server.name
|
||||
))))
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Some(feature))
|
||||
}
|
||||
|
||||
/// Discover enabled MCP stdio server tools and return a single feature module
|
||||
/// containing startup contributions for normal ToolRegistry installation.
|
||||
/// Callers needing retryable startup cleanup must use the cleanup-aware API.
|
||||
pub async fn discover_stdio_tool_feature(
|
||||
config: &McpConfig,
|
||||
workspace_root: &Path,
|
||||
@@ -81,21 +163,66 @@ pub async fn discover_stdio_tool_feature(
|
||||
}
|
||||
|
||||
async fn discover_server_tools(spec: McpStdioServerSpec) -> ProtocolProviderContribution {
|
||||
let declaration = provider_declaration(&spec.name, None);
|
||||
let cleanup = McpDiscoveryCleanup::new();
|
||||
match discover_server_tools_with_cleanup(spec, &cleanup, McpStdioLimits::default()).await {
|
||||
Ok(contribution) => contribution,
|
||||
Err(err) => {
|
||||
ProtocolProviderContribution::failed(declaration, bounded_diagnostic(err.to_string()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn discover_server_tools_with_cleanup(
|
||||
spec: McpStdioServerSpec,
|
||||
cleanup: &McpDiscoveryCleanup,
|
||||
limits: McpStdioLimits,
|
||||
) -> io::Result<ProtocolProviderContribution> {
|
||||
let declaration = provider_declaration(&spec.name, None);
|
||||
let mut clients = cleanup.clients.lock().await;
|
||||
if !clients.is_empty() {
|
||||
return Err(io::Error::other(
|
||||
"MCP discovery cleanup is pending; shut down the retained owner before retrying discovery",
|
||||
));
|
||||
}
|
||||
let client = match McpStdioClient::spawn_uninitialized(spec.clone(), limits) {
|
||||
Ok(client) => client,
|
||||
Err(err) => {
|
||||
// The synchronous spawn API only errors before a process exists;
|
||||
// pipe/handshake failures retain a client and use the shutdown path.
|
||||
return Ok(ProtocolProviderContribution::failed(
|
||||
declaration,
|
||||
bounded_diagnostic(err.to_string()),
|
||||
));
|
||||
}
|
||||
};
|
||||
// Spawn is synchronous: the owner holds the client before the first await
|
||||
// that can leave a running process on initialization failure/cancellation.
|
||||
clients.push(client);
|
||||
let client = clients.last_mut().expect("retained discovery client");
|
||||
let contribution = match client.initialize_with_timeout().await {
|
||||
Ok(()) => discover_connected_server_tools(spec, client).await,
|
||||
Err(err) => {
|
||||
ProtocolProviderContribution::failed(declaration, bounded_diagnostic(err.to_string()))
|
||||
}
|
||||
};
|
||||
client
|
||||
.shutdown()
|
||||
.await
|
||||
.map_err(|err| discovery_shutdown_error(&err))?;
|
||||
drop(clients.pop());
|
||||
Ok(contribution)
|
||||
}
|
||||
|
||||
async fn discover_connected_server_tools(
|
||||
spec: McpStdioServerSpec,
|
||||
client: &mut McpStdioClient,
|
||||
) -> ProtocolProviderContribution {
|
||||
let declaration = provider_declaration(&spec.name, None);
|
||||
let mut contribution = ProtocolProviderContribution::ready(declaration.clone());
|
||||
let server_namespace = sanitize_segment(&spec.name);
|
||||
let execution_spec = spec.clone();
|
||||
|
||||
let mut client = match McpStdioClient::connect(spec, McpStdioLimits::default()).await {
|
||||
Ok(client) => client,
|
||||
Err(err) => {
|
||||
return ProtocolProviderContribution::failed(
|
||||
declaration,
|
||||
bounded_diagnostic(err.to_string()),
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
let (server_version, has_tools, has_resources, has_prompts) = client
|
||||
.initialize_result()
|
||||
.map(|result| {
|
||||
@@ -133,18 +260,10 @@ async fn discover_server_tools(spec: McpStdioServerSpec) -> ProtocolProviderCont
|
||||
{
|
||||
Ok(list) => list,
|
||||
Err(err) => {
|
||||
let mut failed = ProtocolProviderContribution::failed(
|
||||
return ProtocolProviderContribution::failed(
|
||||
declaration,
|
||||
bounded_diagnostic(err.to_string()),
|
||||
);
|
||||
if let Err(shutdown_err) = client.shutdown().await {
|
||||
failed = failed.with_diagnostic(FeatureDiagnostic::warning(
|
||||
bounded_diagnostic(format!(
|
||||
"MCP server shutdown after discovery failure failed: {shutdown_err}"
|
||||
)),
|
||||
));
|
||||
}
|
||||
return failed;
|
||||
}
|
||||
};
|
||||
let list_changes = client.snapshot_list_changes().await;
|
||||
@@ -166,20 +285,12 @@ async fn discover_server_tools(spec: McpStdioServerSpec) -> ProtocolProviderCont
|
||||
{
|
||||
Ok(list) => list,
|
||||
Err(err) => {
|
||||
let mut failed = ProtocolProviderContribution::failed(
|
||||
return ProtocolProviderContribution::failed(
|
||||
declaration,
|
||||
bounded_diagnostic(format!(
|
||||
"MCP server sent notifications/tools/list_changed during tool discovery and refresh failed: {err}"
|
||||
)),
|
||||
);
|
||||
if let Err(shutdown_err) = client.shutdown().await {
|
||||
failed = failed.with_diagnostic(FeatureDiagnostic::warning(
|
||||
bounded_diagnostic(format!(
|
||||
"MCP server shutdown after discovery refresh failure failed: {shutdown_err}"
|
||||
)),
|
||||
));
|
||||
}
|
||||
return failed;
|
||||
}
|
||||
};
|
||||
let refresh_changes = client.snapshot_list_changes().await;
|
||||
@@ -197,13 +308,6 @@ async fn discover_server_tools(spec: McpStdioServerSpec) -> ProtocolProviderCont
|
||||
} else {
|
||||
None
|
||||
};
|
||||
let shutdown_result = client.shutdown().await;
|
||||
if let Err(err) = shutdown_result {
|
||||
contribution =
|
||||
contribution.with_diagnostic(FeatureDiagnostic::warning(bounded_diagnostic(format!(
|
||||
"MCP server shutdown after capability discovery failed: {err}"
|
||||
))));
|
||||
}
|
||||
|
||||
if let Some(list) = list {
|
||||
contribution = normalize_listed_tools(
|
||||
@@ -1819,6 +1923,140 @@ read exit_notification || true
|
||||
McpStdioServerSpec::new("shell-mock", "/bin/sh").args(["-c".to_string(), script])
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
fn discovery_config(spec: McpStdioServerSpec) -> McpConfig {
|
||||
McpConfig {
|
||||
stdio_servers: vec![manifest::McpStdioServerConfig {
|
||||
name: spec.name,
|
||||
command: spec.command,
|
||||
args: spec.args,
|
||||
cwd: None,
|
||||
env: Default::default(),
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn cancelled_discovery_retains_initializing_client_in_cleanup_clone() {
|
||||
use std::future::{Future, poll_fn};
|
||||
use std::task::Poll;
|
||||
use std::time::Duration;
|
||||
|
||||
let config = discovery_config(
|
||||
McpStdioServerSpec::new("initializing", "/bin/sh")
|
||||
.args(["-c", "while read line; do :; done"]),
|
||||
);
|
||||
let cleanup = McpDiscoveryCleanup::new();
|
||||
let retained = cleanup.clone();
|
||||
let mut discovery = Box::pin(discover_stdio_tool_feature_with_cleanup(
|
||||
&config,
|
||||
Path::new("."),
|
||||
&cleanup,
|
||||
));
|
||||
poll_fn(|cx| {
|
||||
assert!(discovery.as_mut().poll(cx).is_pending());
|
||||
Poll::Ready(())
|
||||
})
|
||||
.await;
|
||||
drop(discovery);
|
||||
drop(cleanup);
|
||||
assert_eq!(retained.clients.lock().await.len(), 1);
|
||||
let retry_error =
|
||||
discover_stdio_tool_feature_with_cleanup(&config, Path::new("."), &retained)
|
||||
.await
|
||||
.err()
|
||||
.expect("discovery must not replace an unreaped client");
|
||||
assert!(retry_error.to_string().contains("cleanup is pending"));
|
||||
assert_eq!(retained.clients.lock().await.len(), 1);
|
||||
tokio::time::timeout(Duration::from_secs(3), retained.shutdown())
|
||||
.await
|
||||
.expect("cleanup bounded")
|
||||
.expect("initializing process reaped");
|
||||
assert!(retained.clients.lock().await.is_empty());
|
||||
retained
|
||||
.shutdown()
|
||||
.await
|
||||
.expect("confirmed cleanup repeatable");
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn cancelled_owner_shutdown_keeps_client_for_retry() {
|
||||
use std::future::{Future, poll_fn};
|
||||
use std::task::Poll;
|
||||
use std::time::Duration;
|
||||
|
||||
let cleanup = McpDiscoveryCleanup::new();
|
||||
cleanup.clients.lock().await.push(
|
||||
McpStdioClient::spawn_uninitialized(
|
||||
McpStdioServerSpec::new("shutdown-pending", "/bin/sh")
|
||||
.args(["-c", "while :; do :; done"]),
|
||||
McpStdioLimits {
|
||||
shutdown_timeout: Duration::from_millis(50),
|
||||
kill_timeout: Duration::from_millis(100),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
.expect("local process spawned"),
|
||||
);
|
||||
let mut shutdown = Box::pin(cleanup.shutdown());
|
||||
poll_fn(|cx| {
|
||||
assert!(shutdown.as_mut().poll(cx).is_pending());
|
||||
Poll::Ready(())
|
||||
})
|
||||
.await;
|
||||
drop(shutdown);
|
||||
assert_eq!(cleanup.clients.lock().await.len(), 1);
|
||||
tokio::time::timeout(Duration::from_secs(2), cleanup.shutdown())
|
||||
.await
|
||||
.expect("retry bounded")
|
||||
.expect("retained client shutdown");
|
||||
assert!(cleanup.clients.lock().await.is_empty());
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn cleanup_aware_discovery_reaps_success_and_initialize_failure() {
|
||||
let cleanup = McpDiscoveryCleanup::new();
|
||||
let config = discovery_config(shell_capability_server(r#"{"resources":{}}"#));
|
||||
let feature = discover_stdio_tool_feature_with_cleanup(&config, Path::new("."), &cleanup)
|
||||
.await
|
||||
.expect("discovery cleanup confirmed")
|
||||
.expect("configured feature");
|
||||
assert!(
|
||||
feature.contributions[0]
|
||||
.tools
|
||||
.iter()
|
||||
.any(|tool| { tool.name() == "Mcp_shell_mock_resources_read" })
|
||||
);
|
||||
assert!(cleanup.clients.lock().await.is_empty());
|
||||
|
||||
let spec = McpStdioServerSpec::new("init-rejected", "/bin/sh").args([
|
||||
"-c",
|
||||
r#"read init || exit 1
|
||||
printf '%s\n' '{"jsonrpc":"2.0","id":1,"error":{"code":-32000,"message":"initialization rejected"}}'
|
||||
while read line; do :; done
|
||||
"#,
|
||||
]);
|
||||
let config = discovery_config(spec);
|
||||
let feature = discover_stdio_tool_feature_with_cleanup(&config, Path::new("."), &cleanup)
|
||||
.await
|
||||
.expect("failure isolated only after cleanup")
|
||||
.expect("failed contribution preserved");
|
||||
assert!(feature.contributions[0].tools.is_empty());
|
||||
assert!(
|
||||
feature.contributions[0]
|
||||
.diagnostics
|
||||
.iter()
|
||||
.any(|diagnostic| {
|
||||
diagnostic.severity == FeatureDiagnosticSeverity::Error
|
||||
&& diagnostic.message.contains("initialization rejected")
|
||||
})
|
||||
);
|
||||
assert!(cleanup.clients.lock().await.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn mcp_resource_prompt_operation_tools_are_namespaced_and_untrusted() {
|
||||
let declaration = provider_declaration("demo", Some("1.0.0"));
|
||||
|
||||
@@ -1,57 +1,6 @@
|
||||
//! `WorkerEvent` send / receive helpers.
|
||||
//!
|
||||
//! This module owns the parent-facing lifecycle-event primitive
|
||||
//! (`WorkerEvent`) that children fire upward on turn-end / error /
|
||||
//! shutdown / scope-sub-delegation. Three responsibilities live here:
|
||||
//!
|
||||
//! - **Send** a `Method::WorkerEvent` to the parent socket, fire-and-forget,
|
||||
//! logging failures without blocking the child.
|
||||
//! - **Render** agent-visible variants into human-readable strings for the
|
||||
//! parent's notification buffer. Control-plane-only variants may still have
|
||||
//! a renderer for diagnostics, but receive-side classification keeps them
|
||||
//! out of LLM history/context.
|
||||
//! - **Apply side effects** on the parent (registry / worker-allocation
|
||||
//! updates) so that the receive path is idempotent and tolerant of
|
||||
//! out-of-order delivery.
|
||||
//!
|
||||
//! Transport is fire-and-forget — the ticket's decision is that
|
||||
//! callbacks are an optimisation and `ListWorkers` + `reclaim_stale` are
|
||||
//! the real fallback. This module is allowed to drop events on the
|
||||
//! floor (with a warn log) rather than retry.
|
||||
//!
|
||||
//! `apply_event_side_effects` takes its dependencies (registry, scope
|
||||
//! lock path, self identity) by reference so the caller owns lifetime
|
||||
//! and locking concerns.
|
||||
//! Agent-visible rendering of typed Internal SubWorker lifecycle events.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use protocol::{Method, WorkerEvent};
|
||||
|
||||
use crate::spawn::comm_tools::connect_and_send;
|
||||
use crate::spawn::registry::SpawnedWorkerRegistry;
|
||||
|
||||
/// Connect to `socket`, send a single `Method::WorkerEvent(event)`, and
|
||||
/// return. Used by children to report up to their parent.
|
||||
///
|
||||
/// This is a synchronous helper — callers that want fire-and-forget
|
||||
/// semantics should wrap the call in `tokio::spawn` themselves.
|
||||
pub async fn send_worker_event(socket: &Path, event: WorkerEvent) -> std::io::Result<()> {
|
||||
connect_and_send(socket, &Method::WorkerEvent(event)).await
|
||||
}
|
||||
|
||||
/// Spawn a fire-and-forget task that sends `event` to `socket`. If
|
||||
/// `socket` is `None`, no send happens (top-level Workers have no parent).
|
||||
/// Any send failure is logged at warn level but otherwise ignored —
|
||||
/// the parent is treated as best-effort.
|
||||
pub fn fire_and_forget(socket: Option<PathBuf>, event: WorkerEvent) {
|
||||
let Some(socket) = socket else { return };
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = send_worker_event(&socket, event).await {
|
||||
tracing::warn!(error = %e, socket = %socket.display(), "WorkerEvent send failed");
|
||||
}
|
||||
});
|
||||
}
|
||||
use protocol::WorkerEvent;
|
||||
|
||||
/// Render a variant into a one-line human-readable string.
|
||||
///
|
||||
@@ -83,16 +32,3 @@ pub fn render_event(event: &WorkerEvent) -> String {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Legacy process callback events have no SubWorker registry authority.
|
||||
///
|
||||
/// Internal SubWorker lifecycle is applied directly through typed session handles. A callback from
|
||||
/// an externally adopted Worker may still be rendered for diagnostics, but it cannot add/remove
|
||||
/// Internal children or transfer filesystem authority.
|
||||
pub async fn apply_event_side_effects(
|
||||
_event: &WorkerEvent,
|
||||
_registry: &Arc<SpawnedWorkerRegistry>,
|
||||
_self_name: &str,
|
||||
_self_parent_socket: &Option<PathBuf>,
|
||||
) {
|
||||
}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
pub mod alerter;
|
||||
pub mod event;
|
||||
pub mod protocol_session;
|
||||
pub mod server;
|
||||
|
||||
pub(crate) mod interceptor;
|
||||
pub(crate) mod notify_buffer;
|
||||
|
||||
@@ -1,224 +0,0 @@
|
||||
use std::io;
|
||||
use std::io::ErrorKind;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use protocol::stream::{JsonLineReader, JsonLineWriter};
|
||||
use tokio::net::UnixListener;
|
||||
use tokio::sync::oneshot;
|
||||
use tokio::task::{JoinHandle, JoinSet};
|
||||
|
||||
use crate::controller::WorkerHandle;
|
||||
use crate::ipc::protocol_session::{
|
||||
dispatch_worker_protocol_method, live_log_entry_event, subscribe_worker_protocol_session,
|
||||
};
|
||||
use protocol::{ErrorCode, Event};
|
||||
|
||||
/// Unix socket server for Worker Protocol.
|
||||
///
|
||||
/// Listens on the Worker's runtime directory socket path.
|
||||
/// Each client connection gets bidirectional JSONL:
|
||||
/// - Client writes Method lines → forwarded to WorkerController
|
||||
/// - Worker events → written as Event lines to all connected clients
|
||||
pub struct SocketServer {
|
||||
accept_task: Option<JoinHandle<()>>,
|
||||
shutdown: Option<oneshot::Sender<()>>,
|
||||
path: PathBuf,
|
||||
}
|
||||
|
||||
impl SocketServer {
|
||||
/// Start listening on the WorkerHandle's socket path.
|
||||
pub async fn start(handle: &WorkerHandle) -> Result<Self, io::Error> {
|
||||
let path = handle.runtime_dir.socket_path();
|
||||
|
||||
// Remove stale socket file if it exists
|
||||
let _ = tokio::fs::remove_file(&path).await;
|
||||
|
||||
let listener = UnixListener::bind(&path)?;
|
||||
let handle = handle.clone();
|
||||
let (shutdown_tx, mut shutdown_rx) = oneshot::channel();
|
||||
|
||||
let accept_task = tokio::spawn(async move {
|
||||
let mut connections = JoinSet::new();
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = &mut shutdown_rx => break,
|
||||
accepted = listener.accept() => match accepted {
|
||||
Ok((stream, _)) => {
|
||||
let handle = handle.clone();
|
||||
connections.spawn(handle_connection(stream, handle));
|
||||
}
|
||||
Err(_) => break,
|
||||
},
|
||||
completed = connections.join_next(), if !connections.is_empty() => {
|
||||
let _ = completed;
|
||||
}
|
||||
}
|
||||
}
|
||||
connections.shutdown().await;
|
||||
});
|
||||
|
||||
Ok(Self {
|
||||
accept_task: Some(accept_task),
|
||||
shutdown: Some(shutdown_tx),
|
||||
path,
|
||||
})
|
||||
}
|
||||
|
||||
/// Stop accepting connections and join the server task. Dropping its
|
||||
/// `JoinSet` cancels every active connection task before this returns.
|
||||
pub async fn shutdown(mut self) {
|
||||
if let Some(shutdown) = self.shutdown.take() {
|
||||
let _ = shutdown.send(());
|
||||
}
|
||||
if let Some(task) = self.accept_task.take() {
|
||||
let _ = task.await;
|
||||
}
|
||||
let _ = tokio::fs::remove_file(&self.path).await;
|
||||
}
|
||||
|
||||
/// The socket file path.
|
||||
pub fn path(&self) -> &std::path::Path {
|
||||
&self.path
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for SocketServer {
|
||||
fn drop(&mut self) {
|
||||
if let Some(shutdown) = self.shutdown.take() {
|
||||
let _ = shutdown.send(());
|
||||
}
|
||||
let _ = self.accept_task.take();
|
||||
let _ = std::fs::remove_file(&self.path);
|
||||
}
|
||||
}
|
||||
|
||||
fn is_peer_disconnect_read_error(error: &io::Error) -> bool {
|
||||
matches!(
|
||||
error.kind(),
|
||||
ErrorKind::ConnectionReset
|
||||
| ErrorKind::ConnectionAborted
|
||||
| ErrorKind::BrokenPipe
|
||||
| ErrorKind::UnexpectedEof
|
||||
)
|
||||
}
|
||||
|
||||
async fn handle_connection(stream: tokio::net::UnixStream, handle: WorkerHandle) {
|
||||
let (reader, writer) = stream.into_split();
|
||||
let mut reader = JsonLineReader::new(reader);
|
||||
let mut writer = JsonLineWriter::new(writer);
|
||||
|
||||
// Hold the in-flight stream lock while taking the session-log mirror
|
||||
// snapshot. `LogEntry::AnnotatedAssistantItem` is mirror-only for live clients,
|
||||
// so a finalized assistant block must be observed either as an already
|
||||
// committed entry or as the still-present in-flight block. This lock
|
||||
// order matches `append_entry` (in-flight clear before sink publish) and
|
||||
// keeps the snapshot/live boundary gap-free.
|
||||
let mut streams = subscribe_worker_protocol_session(&handle);
|
||||
for alert in streams.alert_snapshot {
|
||||
if writer.write_event(&Event::Alert(alert)).await.is_err() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Send the typed snapshot up front so late attachers can
|
||||
// reconstruct view state without an extra round trip.
|
||||
if writer.write_event(&streams.snapshot_event).await.is_err() {
|
||||
return;
|
||||
}
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
// Live session-log entries → dispatched as the role-specific
|
||||
// wire events. `SegmentLogSink` only broadcasts committed log
|
||||
// entries with live UI meaning; `UserInput` travels this lane so
|
||||
// the visible user line is ordered with `SegmentStart` rotation.
|
||||
entry = streams.log_entries.recv() => {
|
||||
match entry {
|
||||
Ok(entry) => {
|
||||
if let Some(event) = live_log_entry_event(entry) {
|
||||
if writer.write_event(&event).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {
|
||||
// Slow client fell behind the broadcast buffer.
|
||||
// Drop the connection so the next reconnect
|
||||
// re-seeds the prefix via subscribe_with_snapshot.
|
||||
break;
|
||||
}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Closed) => break,
|
||||
}
|
||||
}
|
||||
// Broadcast events → this client
|
||||
event = streams.events.recv() => {
|
||||
match event {
|
||||
Ok(event) => {
|
||||
let Ok(event) = handle.alerter.freshen_event(event) else {
|
||||
// The queued Alert was evicted; reconnect must rebuild
|
||||
// the bounded prefix rather than forward a stale anchor.
|
||||
break;
|
||||
};
|
||||
if writer.write_event(&event).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
// Client methods → handle or forward to controller
|
||||
method = reader.next_method() => {
|
||||
match method {
|
||||
Ok(Some(method)) => {
|
||||
if let Some(response) = dispatch_worker_protocol_method(&handle, method).await
|
||||
&& writer.write_event(&response).await.is_err()
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok(None) => break,
|
||||
Err(e) if is_peer_disconnect_read_error(&e) => break,
|
||||
Err(e) => {
|
||||
if writer
|
||||
.write(&Event::Error {
|
||||
code: ErrorCode::InvalidRequest,
|
||||
message: format!("invalid method: {e}"),
|
||||
})
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn peer_disconnect_read_errors_are_connection_close() {
|
||||
for kind in [
|
||||
ErrorKind::ConnectionReset,
|
||||
ErrorKind::ConnectionAborted,
|
||||
ErrorKind::BrokenPipe,
|
||||
ErrorKind::UnexpectedEof,
|
||||
] {
|
||||
let error = io::Error::new(kind, "peer disconnected");
|
||||
assert!(
|
||||
is_peer_disconnect_read_error(&error),
|
||||
"{kind:?} should be treated as a normal peer disconnect"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_data_is_not_peer_disconnect() {
|
||||
let error = io::Error::new(ErrorKind::InvalidData, "malformed method");
|
||||
assert!(!is_peer_disconnect_read_error(&error));
|
||||
}
|
||||
}
|
||||
@@ -13,7 +13,6 @@
|
||||
mod result;
|
||||
pub use result::{JobResultFeature, JobResultFinalizer, JobResultSink, SUBMIT_JOB_RESULT_TOOL};
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -154,7 +153,7 @@ fn prepare_job_from_registry_with_grant(
|
||||
)?;
|
||||
let mut system_prompt = template.render(&SystemPromptContext {
|
||||
now: chrono::Utc::now(),
|
||||
cwd: Cow::Borrowed("(no filesystem)"),
|
||||
workdir_sessions: &workdir::WorkdirSessionRouter::new(),
|
||||
language: &manifest.engine.language,
|
||||
scope: &scope,
|
||||
tool_names,
|
||||
|
||||
@@ -179,11 +179,11 @@ async fn builtin_journal_profile_installs_only_bound_tools_and_finalizes_saved_s
|
||||
call(SUBMIT_JOB_RESULT_TOOL, json!({"result":{}})),
|
||||
prose(),
|
||||
]);
|
||||
// This runner/Feature fixture has no selected sources; descriptor delivery gates
|
||||
// for nonempty ranges are owned and tested by the Subjektiv service.
|
||||
// This runner/Feature fixture has no selected sources; actual body-bearing input,
|
||||
// trusted delivery and checkpoint gates are covered by the Host integration suites.
|
||||
let mut journal_request = request("builtin:subjektiv-journal-recording");
|
||||
journal_request.input = json!({"selected_ranges":[],"origins":[]});
|
||||
journal_request.instruction = "Read immutable selected_ranges through ReadJournalText Source before recording; descriptors are not source bodies.".into();
|
||||
journal_request.input = json!({"selected_ranges":[],"capture":{"entries":[]}});
|
||||
journal_request.instruction = "Record only the immutable canonical bodies in this Job input. Do not read original Sessions through ReadJournalText Source.".into();
|
||||
let job = prepare_job_from_registry_with_grant(
|
||||
&journal_request,
|
||||
root.path(),
|
||||
@@ -249,7 +249,7 @@ async fn builtin_journal_profile_installs_only_bound_tools_and_finalizes_saved_s
|
||||
"ReadJournalText",
|
||||
"Source",
|
||||
"content_token",
|
||||
"Conflict",
|
||||
"input digest",
|
||||
] {
|
||||
assert!(
|
||||
prompt.contains(contract),
|
||||
|
||||
@@ -7,8 +7,6 @@ pub mod bootstrap;
|
||||
pub mod checkout;
|
||||
pub mod compact;
|
||||
pub mod controller;
|
||||
pub mod discovery;
|
||||
pub mod entrypoint;
|
||||
pub mod feature;
|
||||
mod file_operation;
|
||||
pub mod fs_view;
|
||||
@@ -19,9 +17,12 @@ pub mod job;
|
||||
mod metadata_lifecycle;
|
||||
pub mod model_client;
|
||||
mod paste_artifact_tool;
|
||||
pub mod profile_manifest;
|
||||
pub mod prompt;
|
||||
#[cfg(test)]
|
||||
#[path = "../../../tests/support/restore_environment.rs"]
|
||||
mod restore_test_environment;
|
||||
pub mod runtime;
|
||||
pub mod runtime_command;
|
||||
pub mod segment_log_sink;
|
||||
mod session_capture;
|
||||
mod session_history;
|
||||
@@ -37,6 +38,7 @@ pub mod wip;
|
||||
|
||||
mod internal_worker;
|
||||
mod interrupt_prep;
|
||||
mod peer;
|
||||
mod permission;
|
||||
mod worker;
|
||||
|
||||
@@ -45,10 +47,9 @@ pub use bootstrap::{
|
||||
WorkerBootstrapLayout, start_worker_controller,
|
||||
};
|
||||
pub use compact::token_counter::{EstimateSource, SplitPoint, TokenEstimate};
|
||||
pub use controller::{ShutdownReceiver, WorkerController, WorkerControllerTransport, WorkerHandle};
|
||||
pub use controller::{ShutdownReceiver, WorkerController, WorkerHandle};
|
||||
pub use hook::{Hook, HookEventKind, HookRegistryBuilder};
|
||||
pub use ipc::alerter::Alerter;
|
||||
pub use ipc::server::SocketServer;
|
||||
pub use manifest::{
|
||||
AuthRef, ModelManifest, SchemeKind, Scope, WorkerManifest, WorkerManifestConfig,
|
||||
WorkerMetaConfig,
|
||||
@@ -73,8 +74,8 @@ pub use shared_state::WorkerSharedState;
|
||||
pub use worker::{
|
||||
LocalWorkingDirectory, ReviewerChildWorkspaceClient, ReviewerContext,
|
||||
SubjektivSessionAttributionLifecycle, Worker, WorkerError, WorkerFilesystemAuthority,
|
||||
WorkerRunResult, WorkerWorkspaceContext, WorkspaceBinaryRequest, WorkspaceBinaryResponse,
|
||||
WorkspaceClient, WorkspaceClientError, WorkspaceId, WorkspaceIdError,
|
||||
WorkerRunResult, WorkerStartupFeatureCleanup, WorkerWorkspaceContext, WorkspaceBinaryRequest,
|
||||
WorkspaceBinaryResponse, WorkspaceClient, WorkspaceClientError, WorkspaceId, WorkspaceIdError,
|
||||
WorkspacePromptCatalogResolution, WorkspaceRequest, WorkspaceRequestMethod, WorkspaceResponse,
|
||||
WorkspaceServerOperation, WorkspaceWorkerDiscoveryRequest, apply_worker_manifest,
|
||||
marker_workspace_client, unavailable_workspace_client,
|
||||
|
||||
@@ -0,0 +1,186 @@
|
||||
//! Reciprocal peer metadata registration, independent of discovery and transport.
|
||||
//!
|
||||
//! Registering a peer does not start a Worker or grant filesystem, observation, or
|
||||
//! messaging authority. The two metadata writes are not a transaction: a failed
|
||||
//! reciprocal write triggers best-effort restoration of the source's prior peers.
|
||||
|
||||
use serde::Serialize;
|
||||
use session_store::{WorkerMetadataStore, WorkerStoreError, validate_worker_name};
|
||||
|
||||
#[derive(Debug, Serialize, PartialEq, Eq)]
|
||||
pub(crate) struct PeerRegistrationResult {
|
||||
pub source: String,
|
||||
pub peer: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub(crate) enum PeerRegistrationError {
|
||||
#[error("worker state missing for `{worker_name}`")]
|
||||
StateMissing { worker_name: String },
|
||||
#[error("worker `{worker_name}` cannot be registered as a peer of itself")]
|
||||
SelfPeer { worker_name: String },
|
||||
#[error("worker `{worker_name}` does not exist")]
|
||||
MissingWorker { worker_name: String },
|
||||
#[error("worker store error: {0}")]
|
||||
WorkerStore(#[from] WorkerStoreError),
|
||||
}
|
||||
|
||||
pub(crate) fn register_peer(
|
||||
store: &impl WorkerMetadataStore,
|
||||
source: &str,
|
||||
peer: &str,
|
||||
) -> Result<PeerRegistrationResult, PeerRegistrationError> {
|
||||
ensure_existing_peer(store, source, peer)?.ok_or_else(|| PeerRegistrationError::MissingWorker {
|
||||
worker_name: peer.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
fn ensure_existing_peer(
|
||||
store: &impl WorkerMetadataStore,
|
||||
source: &str,
|
||||
peer: &str,
|
||||
) -> Result<Option<PeerRegistrationResult>, PeerRegistrationError> {
|
||||
validate_worker_name(peer)?;
|
||||
if peer == source {
|
||||
return Err(PeerRegistrationError::SelfPeer {
|
||||
worker_name: peer.to_string(),
|
||||
});
|
||||
}
|
||||
let prior_source_peers = store
|
||||
.read_by_name(source)?
|
||||
.ok_or_else(|| PeerRegistrationError::StateMissing {
|
||||
worker_name: source.to_string(),
|
||||
})?
|
||||
.peers;
|
||||
if store.read_by_name(peer)?.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
store.add_peer(source, peer)?;
|
||||
if let Err(error) = store.add_peer(peer, source) {
|
||||
let _ = store.set_peers(source, prior_source_peers);
|
||||
return Err(PeerRegistrationError::WorkerStore(error));
|
||||
}
|
||||
Ok(Some(PeerRegistrationResult {
|
||||
source: source.to_string(),
|
||||
peer: peer.to_string(),
|
||||
}))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use session_store::{FsWorkerStore, WorkerMetadata};
|
||||
|
||||
#[test]
|
||||
fn register_peer_persists_reciprocal_metadata_without_duplicates() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let store = FsWorkerStore::new(root.path()).unwrap();
|
||||
store.write(&WorkerMetadata::new("source", None)).unwrap();
|
||||
store.write(&WorkerMetadata::new("target", None)).unwrap();
|
||||
for _ in 0..2 {
|
||||
let result = register_peer(&store, "source", "target").unwrap();
|
||||
assert_eq!(
|
||||
serde_json::to_value(result).unwrap(),
|
||||
serde_json::json!({
|
||||
"source": "source", "peer": "target",
|
||||
})
|
||||
);
|
||||
}
|
||||
// Reopen the real store rather than asserting only the write calls.
|
||||
let persisted = FsWorkerStore::new(root.path()).unwrap();
|
||||
let source = persisted.read_by_name("source").unwrap().unwrap();
|
||||
let target = persisted.read_by_name("target").unwrap().unwrap();
|
||||
assert_eq!(source.peers.len(), 1);
|
||||
assert_eq!(source.peers[0].worker_name, "target");
|
||||
assert_eq!(target.peers.len(), 1);
|
||||
assert_eq!(target.peers[0].worker_name, "source");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_self_or_missing_peer_leaves_metadata_unchanged() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let store = FsWorkerStore::new(root.path()).unwrap();
|
||||
store.write(&WorkerMetadata::new("source", None)).unwrap();
|
||||
assert!(matches!(
|
||||
register_peer(&store, "source", "source"),
|
||||
Err(PeerRegistrationError::SelfPeer { .. })
|
||||
));
|
||||
assert!(matches!(
|
||||
register_peer(&store, "source", "missing"),
|
||||
Err(PeerRegistrationError::MissingWorker { .. })
|
||||
));
|
||||
assert!(matches!(
|
||||
register_peer(&store, "source", "../escape"),
|
||||
Err(PeerRegistrationError::WorkerStore(_))
|
||||
));
|
||||
assert!(matches!(
|
||||
register_peer(&store, "missing-source", "source"),
|
||||
Err(PeerRegistrationError::StateMissing { .. })
|
||||
));
|
||||
assert!(
|
||||
store
|
||||
.read_by_name("source")
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.peers
|
||||
.is_empty()
|
||||
);
|
||||
assert_eq!(store.list_names().unwrap(), vec!["source"]);
|
||||
}
|
||||
|
||||
struct FailTargetPeerStore(FsWorkerStore);
|
||||
|
||||
impl WorkerMetadataStore for FailTargetPeerStore {
|
||||
fn write(&self, metadata: &WorkerMetadata) -> Result<(), WorkerStoreError> {
|
||||
if metadata.worker_name == "target" && !metadata.peers.is_empty() {
|
||||
return Err(WorkerStoreError::Io(std::io::Error::other(
|
||||
"target write failed",
|
||||
)));
|
||||
}
|
||||
self.0.write(metadata)
|
||||
}
|
||||
|
||||
fn read_by_name(&self, name: &str) -> Result<Option<WorkerMetadata>, WorkerStoreError> {
|
||||
self.0.read_by_name(name)
|
||||
}
|
||||
|
||||
fn list_names(&self) -> Result<Vec<String>, WorkerStoreError> {
|
||||
self.0.list_names()
|
||||
}
|
||||
|
||||
fn delete_by_name(&self, name: &str) -> Result<(), WorkerStoreError> {
|
||||
self.0.delete_by_name(name)
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reciprocal_write_failure_restores_exact_prior_source_peers() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let inner = FsWorkerStore::new(root.path()).unwrap();
|
||||
for name in ["source", "target", "existing"] {
|
||||
inner.write(&WorkerMetadata::new(name, None)).unwrap();
|
||||
}
|
||||
inner.add_peer("source", "existing").unwrap();
|
||||
for had_target in [false, true] {
|
||||
if had_target {
|
||||
inner.add_peer("source", "target").unwrap();
|
||||
}
|
||||
let before = inner.read_by_name("source").unwrap().unwrap().peers;
|
||||
let store = FailTargetPeerStore(inner.clone());
|
||||
assert!(matches!(
|
||||
register_peer(&store, "source", "target"),
|
||||
Err(PeerRegistrationError::WorkerStore(_))
|
||||
));
|
||||
assert_eq!(inner.read_by_name("source").unwrap().unwrap().peers, before);
|
||||
assert!(
|
||||
inner
|
||||
.read_by_name("target")
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.peers
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
//! Runtime manifest preparation from Backend-resolved profiles.
|
||||
//!
|
||||
//! This module has no process entrypoint or local profile discovery authority.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use crate::PromptCatalogSource;
|
||||
use manifest::{Permission, ScopeConfig, ScopeRule, WorkerManifest};
|
||||
|
||||
pub fn resolve_runtime_profile_manifest_from_manifest(
|
||||
mut manifest: WorkerManifest,
|
||||
workspace_root: &Path,
|
||||
worker_name: &str,
|
||||
) -> Result<(WorkerManifest, PromptCatalogSource), String> {
|
||||
if manifest.worker.name.is_empty() {
|
||||
manifest.worker.name = worker_name.to_string();
|
||||
}
|
||||
apply_profile_launch_policy(&mut manifest, workspace_root);
|
||||
// Do not run plugin discovery here: runtime-created Workers receive their
|
||||
// resolved manifest/profile archive from Backend authority, not by scanning
|
||||
// materialized workdir-local plugin stores.
|
||||
Ok((manifest, PromptCatalogSource::builtins_only()))
|
||||
}
|
||||
|
||||
pub fn resolve_runtime_profile_manifest_from_manifest_without_filesystem(
|
||||
mut manifest: WorkerManifest,
|
||||
_workspace_root: &Path,
|
||||
worker_name: &str,
|
||||
) -> Result<(WorkerManifest, PromptCatalogSource), String> {
|
||||
if manifest.worker.name.is_empty() {
|
||||
manifest.worker.name = worker_name.to_string();
|
||||
}
|
||||
manifest.scope = ScopeConfig::default();
|
||||
manifest.delegation_scope = ScopeConfig::default();
|
||||
// Same as the filesystem-capable runtime path: no local discovery.
|
||||
Ok((manifest, PromptCatalogSource::builtins_only()))
|
||||
}
|
||||
|
||||
fn read_rule(target: PathBuf) -> ScopeRule {
|
||||
ScopeRule {
|
||||
target,
|
||||
permission: Permission::Read,
|
||||
recursive: true,
|
||||
symlink_policy: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn write_rule(target: PathBuf) -> ScopeRule {
|
||||
ScopeRule {
|
||||
target,
|
||||
permission: Permission::Write,
|
||||
recursive: true,
|
||||
symlink_policy: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn workspace_scope(
|
||||
workspace_root: &Path,
|
||||
permission: Permission,
|
||||
deny_write: &[PathBuf],
|
||||
) -> ScopeConfig {
|
||||
let allow_rule = ScopeRule {
|
||||
target: workspace_root.to_path_buf(),
|
||||
permission,
|
||||
recursive: true,
|
||||
symlink_policy: Default::default(),
|
||||
};
|
||||
let deny = deny_write
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(write_rule)
|
||||
.collect::<Vec<_>>();
|
||||
ScopeConfig {
|
||||
allow: vec![allow_rule],
|
||||
deny,
|
||||
}
|
||||
}
|
||||
|
||||
fn workspace_worktree_delegation(workspace_root: &Path) -> ScopeConfig {
|
||||
ScopeConfig {
|
||||
allow: vec![
|
||||
read_rule(workspace_root.to_path_buf()),
|
||||
write_rule(workspace_root.join(".worktree")),
|
||||
],
|
||||
deny: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn append_missing_rules(target: &mut Vec<ScopeRule>, defaults: Vec<ScopeRule>) {
|
||||
for rule in defaults {
|
||||
if !target.contains(&rule) {
|
||||
target.push(rule);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_scope_launch_defaults(scope: &mut ScopeConfig, defaults: ScopeConfig) {
|
||||
// Profile resolution has already applied explicit profile/workspace override scope rules.
|
||||
// Launch policy contributes runtime defaults on top rather than replacing those grants.
|
||||
append_missing_rules(&mut scope.allow, defaults.allow);
|
||||
append_missing_rules(&mut scope.deny, defaults.deny);
|
||||
}
|
||||
|
||||
fn apply_profile_launch_policy(manifest: &mut WorkerManifest, workspace_root: &Path) {
|
||||
let worktree_root = workspace_root.join(".worktree");
|
||||
let default_scope = workspace_scope(
|
||||
workspace_root,
|
||||
Permission::Write,
|
||||
std::slice::from_ref(&worktree_root),
|
||||
);
|
||||
apply_scope_launch_defaults(&mut manifest.scope, default_scope);
|
||||
manifest.delegation_scope = workspace_worktree_delegation(workspace_root);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn manifest() -> WorkerManifest {
|
||||
let mut config = manifest::WorkerManifestConfig::resolution_defaults();
|
||||
config.worker.name = Some("resolved".into());
|
||||
WorkerManifest::try_from(config).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_launch_preserves_explicit_scope_without_repository_discovery() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
std::fs::create_dir(root.path().join(".yoi")).unwrap();
|
||||
std::fs::write(root.path().join(".yoi/manifest.toml"), "invalid TOML").unwrap();
|
||||
let mut original = manifest();
|
||||
let explicit = read_rule(root.path().join("reference"));
|
||||
original.scope.allow.push(explicit.clone());
|
||||
let (resolved, _) =
|
||||
resolve_runtime_profile_manifest_from_manifest(original, root.path(), "fallback")
|
||||
.unwrap();
|
||||
assert_eq!(resolved.worker.name, "resolved");
|
||||
assert!(resolved.scope.allow.contains(&explicit));
|
||||
assert!(
|
||||
resolved
|
||||
.scope
|
||||
.allow
|
||||
.contains(&write_rule(root.path().to_path_buf()))
|
||||
);
|
||||
assert!(
|
||||
resolved
|
||||
.scope
|
||||
.deny
|
||||
.contains(&write_rule(root.path().join(".worktree")))
|
||||
);
|
||||
assert_eq!(
|
||||
resolved.delegation_scope.allow,
|
||||
vec![
|
||||
read_rule(root.path().to_path_buf()),
|
||||
write_rule(root.path().join(".worktree")),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_without_filesystem_clears_scope_and_delegation_grants() {
|
||||
let mut original = manifest();
|
||||
original.worker.name.clear();
|
||||
original
|
||||
.scope
|
||||
.allow
|
||||
.push(write_rule(PathBuf::from("/work")));
|
||||
original
|
||||
.delegation_scope
|
||||
.allow
|
||||
.push(write_rule(PathBuf::from("/work")));
|
||||
let (resolved, _) = resolve_runtime_profile_manifest_from_manifest_without_filesystem(
|
||||
original,
|
||||
Path::new("/unused"),
|
||||
"runtime-worker",
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(resolved.worker.name, "runtime-worker");
|
||||
assert!(resolved.scope.allow.is_empty());
|
||||
assert!(resolved.scope.deny.is_empty());
|
||||
assert!(resolved.delegation_scope.allow.is_empty());
|
||||
assert!(resolved.delegation_scope.deny.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -7,13 +7,12 @@
|
||||
//! eagerly syntax-checks it at Worker construction. The final system
|
||||
//! prompt is materialised exactly once just before the first LLM turn:
|
||||
//! the rendered body is appended with a fixed trailing section carrying
|
||||
//! the Worker's `Scope` summary, (if present) the project's `AGENTS.md`
|
||||
//! contents, resident memory sections, and conditional Worker-orchestration
|
||||
//! guidance, then the whole string is handed to the Engine via
|
||||
//! `set_system_prompt`. Subsequent turns and compactions reuse that
|
||||
//! the Worker's `Scope` summary, actual Workdir session environments, (if
|
||||
//! present) the project's `AGENTS.md` contents, resident memory sections, and
|
||||
//! conditional Worker-orchestration guidance, then the whole string is handed
|
||||
//! to the Engine via `set_system_prompt`. Subsequent turns and compactions reuse that
|
||||
//! materialised string verbatim.
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -105,7 +104,8 @@ impl std::fmt::Debug for SystemPromptTemplate {
|
||||
/// templates cannot drop them on the floor.
|
||||
pub struct SystemPromptContext<'a> {
|
||||
pub now: DateTime<Utc>,
|
||||
pub cwd: Cow<'a, str>,
|
||||
/// The same session router used by the Worker's filesystem tools.
|
||||
pub workdir_sessions: &'a workdir::WorkdirSessionRouter,
|
||||
/// Language policy exposed to instruction templates as `{{ language }}`.
|
||||
pub language: &'a str,
|
||||
pub scope: &'a Scope,
|
||||
@@ -140,7 +140,14 @@ impl<'a> SystemPromptContext<'a> {
|
||||
"datetime".into(),
|
||||
Value::from(self.now.to_rfc3339_opts(SecondsFormat::Secs, true)),
|
||||
);
|
||||
root.insert("cwd".into(), Value::from(self.cwd.as_ref()));
|
||||
// A singular template cwd is meaningful only with one attachment. Never
|
||||
// substitute the local Profile/Memory discovery directory for a session cwd.
|
||||
let cwd = self
|
||||
.workdir_sessions
|
||||
.only_session()
|
||||
.map(|session| display_workdir_path(&session.environment().cwd).to_owned())
|
||||
.unwrap_or_else(|| "no single Workdir session cwd; see execution environment".into());
|
||||
root.insert("cwd".into(), Value::from(cwd));
|
||||
root.insert("language".into(), Value::from(self.language));
|
||||
root.insert(
|
||||
"tools".into(),
|
||||
@@ -255,6 +262,70 @@ fn exact_prompt_name(reference: &str) -> Option<String> {
|
||||
}
|
||||
}
|
||||
|
||||
fn display_workdir_path(path: &workdir::WorkdirPath) -> &str {
|
||||
if path.as_str().is_empty() {
|
||||
"."
|
||||
} else {
|
||||
path.as_str()
|
||||
}
|
||||
}
|
||||
|
||||
/// Project provider-owned session metadata, rather than reconstructing authority
|
||||
/// from a manifest Scope or the directory used to discover Profiles and Memory.
|
||||
pub(crate) fn workdir_environment_section(
|
||||
router: &workdir::WorkdirSessionRouter,
|
||||
) -> Result<String, SystemPromptError> {
|
||||
use workdir::WorkdirSessionCapability::{Command, Edit, Glob, Grep, Read, Write};
|
||||
|
||||
let mut environments = Vec::new();
|
||||
for alias in router.aliases() {
|
||||
let resolved = router
|
||||
.resolve(Some(alias.as_str()))
|
||||
.map_err(|error| SystemPromptError::Render(error.to_string()))?;
|
||||
let environment = resolved.session.environment();
|
||||
let capabilities: Vec<_> = [
|
||||
(Read, "read"),
|
||||
(Write, "write"),
|
||||
(Edit, "edit"),
|
||||
(Glob, "glob"),
|
||||
(Grep, "grep"),
|
||||
(Command, "command"),
|
||||
]
|
||||
.into_iter()
|
||||
.filter_map(|(capability, name)| {
|
||||
environment
|
||||
.capabilities
|
||||
.supports(capability)
|
||||
.then_some(name)
|
||||
})
|
||||
.collect();
|
||||
environments.push(serde_json::json!({
|
||||
"alias": alias.as_str(),
|
||||
"cwd": display_workdir_path(&environment.cwd),
|
||||
"scope": environment.scope,
|
||||
"restrictions": environment.restrictions,
|
||||
"capabilities": capabilities,
|
||||
"command": environment.capabilities.supports(Command),
|
||||
}));
|
||||
}
|
||||
let json = serde_json::to_string_pretty(&environments)
|
||||
.map_err(|error| SystemPromptError::Render(error.to_string()))?;
|
||||
Ok(format!(
|
||||
"## Workdir execution environment\n\n\
|
||||
Select a Workdir by its alias (`target_workdir`); omission is allowed only with one attachment. \
|
||||
Relative filesystem operation paths are resolved from that selected session's cwd, not its Workdir root. \
|
||||
Paths returned by Glob/Grep use the same cwd-relative coordinates and can be passed unchanged to other filesystem tools. \
|
||||
Bash starts at the session cwd; an explicit relative cwd is resolved from it, and shell cd does not change the session cwd.\n\n\
|
||||
In the metadata below, cwd and relative scope targets are Workdir-root-relative (`.` or an empty path denotes the root); explicit absolute provider scope targets stay absolute. \
|
||||
A null scope means the provider's full available scope, not unrestricted host access; an empty scope grants no paths. \
|
||||
Restrictions retain additional provider or ancestor ceilings, including deny rules. \
|
||||
Capabilities and command availability are session ceilings, not new grants. \
|
||||
Resolved targets remain subject to scope, symlink and command authorization. \
|
||||
SubWorker scope declarations remain Workdir-root-relative; other Scope declarations and Profile/Memory roots retain their own meaning and are not rebased by session cwd.\n\n\
|
||||
```json\n{json}\n```"
|
||||
))
|
||||
}
|
||||
|
||||
/// Build the final system prompt by appending the fixed trailing
|
||||
/// section to `body`. The Rust side owns the layout (blank-line
|
||||
/// separators, trailing-whitespace trim); each section's header + body
|
||||
@@ -276,8 +347,12 @@ fn append_trailing_section(
|
||||
}
|
||||
out.push('\n');
|
||||
|
||||
let boundaries = prompts.working_boundaries_section(&scope.summary())?;
|
||||
out.push_str(boundaries.trim_end_matches(&['\n', ' '][..]));
|
||||
if ctx.workdir_sessions.is_empty() {
|
||||
let boundaries = prompts.working_boundaries_section(&scope.summary())?;
|
||||
out.push_str(boundaries.trim_end_matches(&['\n', ' '][..]));
|
||||
out.push('\n');
|
||||
}
|
||||
out.push_str(&workdir_environment_section(ctx.workdir_sessions)?);
|
||||
out.push('\n');
|
||||
if let Some(agents) = agents_md {
|
||||
out.push('\n');
|
||||
@@ -392,7 +467,7 @@ mod tests {
|
||||
now: DateTime::parse_from_rfc3339("2026-10-08T00:00:00Z")
|
||||
.unwrap()
|
||||
.with_timezone(&Utc),
|
||||
cwd: Cow::Borrowed("."),
|
||||
workdir_sessions: &workdir::WorkdirSessionRouter::new(),
|
||||
language: "en",
|
||||
scope: &scope,
|
||||
tool_names: metas.iter().map(|meta| meta.name.clone()).collect(),
|
||||
|
||||
@@ -1,17 +1,12 @@
|
||||
use std::io;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use manifest::{ScopeRule, paths};
|
||||
use manifest::ScopeRule;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::fs;
|
||||
|
||||
use crate::shared_state::WorkerSharedState;
|
||||
|
||||
/// One spawned-child record mirrored to `spawned_workers.json`.
|
||||
///
|
||||
/// Written by the spawner after registry changes so runtime-local tools
|
||||
/// have a materialised snapshot. Durable restore uses Worker state metadata;
|
||||
/// this file is not the authoritative source.
|
||||
/// Legacy process-child projection used only by persisted scope reclamation.
|
||||
/// Socket fields retain the stored format; they are never connected to.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SpawnedWorkerRecord {
|
||||
/// Spawned Worker's identity.
|
||||
@@ -25,56 +20,33 @@ pub struct SpawnedWorkerRecord {
|
||||
pub callback_address: PathBuf,
|
||||
}
|
||||
|
||||
/// Manages the Worker's runtime directory on tmpfs.
|
||||
/// Artifact directory retained by an in-process Controller.
|
||||
///
|
||||
/// ```text
|
||||
/// <runtime_dir>/{worker_name}/
|
||||
/// ├── pid
|
||||
/// ├── status.json
|
||||
/// ├── manifest.toml
|
||||
/// ├── history.json
|
||||
/// └── sock (created by socket listener, not by RuntimeDir)
|
||||
/// ```
|
||||
///
|
||||
/// `<runtime_dir>` is resolved via [`manifest::paths::runtime_dir`].
|
||||
/// Files are written atomically (write tmp → rename).
|
||||
/// The directory is removed on drop.
|
||||
/// Worker liveness and configuration belong to the Host, not filesystem projections.
|
||||
/// Persistent run directories and pre-existing data are never removed on drop.
|
||||
pub struct RuntimeDir {
|
||||
path: PathBuf,
|
||||
write_legacy_snapshots: bool,
|
||||
preserve_on_drop: bool,
|
||||
socket_file_name: &'static str,
|
||||
}
|
||||
|
||||
impl RuntimeDir {
|
||||
/// Create the runtime directory and write the PID file.
|
||||
/// Create an ephemeral directory without PID, status, manifest or socket files.
|
||||
pub async fn create(base: &Path, worker_name: &str) -> Result<Self, io::Error> {
|
||||
let path = base.join(worker_name);
|
||||
fs::create_dir_all(&path).await?;
|
||||
|
||||
let pid = std::process::id().to_string();
|
||||
fs::write(path.join("pid"), pid.as_bytes()).await?;
|
||||
|
||||
Ok(Self {
|
||||
path,
|
||||
write_legacy_snapshots: true,
|
||||
preserve_on_drop: false,
|
||||
socket_file_name: "sock",
|
||||
})
|
||||
Self::create_transient(base, worker_name).await
|
||||
}
|
||||
|
||||
/// Create an ephemeral Runtime-owned artifact directory.
|
||||
///
|
||||
/// Runtime-managed Workers keep their status in the owning Runtime and do
|
||||
/// not materialize legacy pid/status/manifest projections.
|
||||
/// Preserve pre-existing directories rather than claiming their contents for cleanup.
|
||||
pub async fn create_transient(base: &Path, worker_name: &str) -> Result<Self, io::Error> {
|
||||
fs::create_dir_all(base).await?;
|
||||
let path = base.join(worker_name);
|
||||
fs::create_dir_all(&path).await?;
|
||||
let preserve_on_drop = match fs::create_dir(&path).await {
|
||||
Ok(()) => false,
|
||||
Err(error) if error.kind() == io::ErrorKind::AlreadyExists && path.is_dir() => true,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
Ok(Self {
|
||||
path,
|
||||
write_legacy_snapshots: false,
|
||||
preserve_on_drop: false,
|
||||
socket_file_name: "sock",
|
||||
preserve_on_drop,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -100,39 +72,11 @@ impl RuntimeDir {
|
||||
std::fs::File::open(parent)?.sync_all()?;
|
||||
Ok(Self {
|
||||
path: path.to_path_buf(),
|
||||
write_legacy_snapshots: false,
|
||||
preserve_on_drop: true,
|
||||
socket_file_name: "worker.sock",
|
||||
})
|
||||
}
|
||||
|
||||
/// Create in the default base directory resolved via
|
||||
/// [`manifest::paths::runtime_dir`].
|
||||
pub async fn create_default(worker_name: &str) -> Result<Self, io::Error> {
|
||||
let base = default_base()?;
|
||||
Self::create(&base, worker_name).await
|
||||
}
|
||||
|
||||
/// Write status.json atomically.
|
||||
pub async fn write_status(&self, state: &WorkerSharedState) -> Result<(), io::Error> {
|
||||
if !self.write_legacy_snapshots {
|
||||
return Ok(());
|
||||
}
|
||||
let content = state.status_json();
|
||||
atomic_write(&self.path.join("status.json"), content.as_bytes()).await
|
||||
}
|
||||
|
||||
/// Write manifest.toml (typically once at startup).
|
||||
pub async fn write_manifest(&self, toml: &str) -> Result<(), io::Error> {
|
||||
if !self.write_legacy_snapshots {
|
||||
return Ok(());
|
||||
}
|
||||
atomic_write(&self.path.join("manifest.toml"), toml.as_bytes()).await
|
||||
}
|
||||
|
||||
/// Write `spawned_workers.json` atomically. The entries are the full
|
||||
/// set of spawned children known to this Worker — callers pass the
|
||||
/// replacement list, no incremental merge.
|
||||
/// Rewrite the legacy projection during persisted process-child reclamation.
|
||||
pub async fn write_spawned_workers(
|
||||
&self,
|
||||
records: &[SpawnedWorkerRecord],
|
||||
@@ -145,26 +89,10 @@ impl RuntimeDir {
|
||||
pub fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
|
||||
/// Path where the Unix socket should be created. External callers
|
||||
/// that only know the worker name (e.g. the TUI's attach flow)
|
||||
/// predict the same path via [`manifest::paths::worker_socket_path`].
|
||||
pub fn socket_path(&self) -> PathBuf {
|
||||
self.path.join(self.socket_file_name)
|
||||
}
|
||||
|
||||
pub async fn close_socket(&self) -> Result<(), io::Error> {
|
||||
match fs::remove_file(self.socket_path()).await {
|
||||
Ok(()) => Ok(()),
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()),
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RuntimeDir {
|
||||
fn drop(&mut self) {
|
||||
let _ = std::fs::remove_file(self.socket_path());
|
||||
if !self.preserve_on_drop {
|
||||
let _ = std::fs::remove_dir_all(&self.path);
|
||||
}
|
||||
@@ -179,156 +107,72 @@ async fn atomic_write(target: &Path, content: &[u8]) -> Result<(), io::Error> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Resolve the default base directory for runtime data.
|
||||
///
|
||||
/// Thin wrapper over [`manifest::paths::runtime_dir`] that converts a
|
||||
/// missing-env situation into an `io::Error`.
|
||||
pub fn default_base() -> Result<PathBuf, io::Error> {
|
||||
paths::runtime_dir().ok_or_else(|| {
|
||||
io::Error::new(
|
||||
io::ErrorKind::NotFound,
|
||||
"could not resolve runtime directory (no YOI_HOME / \
|
||||
YOI_RUNTIME_DIR / XDG_RUNTIME_DIR / HOME)",
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::shared_state::WorkerSharedState;
|
||||
|
||||
fn test_state() -> WorkerSharedState {
|
||||
WorkerSharedState::new(
|
||||
"test-worker".into(),
|
||||
session_store::new_segment_id(),
|
||||
"[engine]\nname = \"test-worker\"".into(),
|
||||
protocol::Greeting {
|
||||
worker_name: "test-worker".into(),
|
||||
cwd: "/tmp".into(),
|
||||
provider: "anthropic".into(),
|
||||
model: "claude".into(),
|
||||
scope_summary: String::new(),
|
||||
tools: Vec::new(),
|
||||
context_window: 200_000,
|
||||
context_tokens: 0,
|
||||
reasoning: None,
|
||||
context_usage: None,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn creates_directory_and_pid() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let rt = RuntimeDir::create(tmp.path(), "my-worker").await.unwrap();
|
||||
|
||||
assert!(rt.path().join("pid").exists());
|
||||
let pid = std::fs::read_to_string(rt.path().join("pid")).unwrap();
|
||||
assert_eq!(pid, std::process::id().to_string());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn write_status_creates_file() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let rt = RuntimeDir::create(tmp.path(), "my-worker").await.unwrap();
|
||||
let state = test_state();
|
||||
|
||||
rt.write_status(&state).await.unwrap();
|
||||
|
||||
let content = std::fs::read_to_string(rt.path().join("status.json")).unwrap();
|
||||
let parsed: serde_json::Value = serde_json::from_str(&content).unwrap();
|
||||
assert_eq!(parsed["state"], "idle");
|
||||
assert_eq!(parsed["worker_name"], "test-worker");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn write_status_reflects_changes() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let rt = RuntimeDir::create(tmp.path(), "my-worker").await.unwrap();
|
||||
let state = test_state();
|
||||
|
||||
state.transition(protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
|
||||
protocol::WorkerRunState::Running,
|
||||
)));
|
||||
rt.write_status(&state).await.unwrap();
|
||||
|
||||
let content = std::fs::read_to_string(rt.path().join("status.json")).unwrap();
|
||||
let parsed: serde_json::Value = serde_json::from_str(&content).unwrap();
|
||||
assert_eq!(parsed["state"], "running");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn write_manifest_creates_file() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let rt = RuntimeDir::create(tmp.path(), "my-worker").await.unwrap();
|
||||
|
||||
rt.write_manifest("[engine]\nname = \"test\"")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let content = std::fs::read_to_string(rt.path().join("manifest.toml")).unwrap();
|
||||
assert_eq!(content, "[engine]\nname = \"test\"");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn transient_directory_does_not_write_liveness_snapshots() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let rt = RuntimeDir::create_transient(tmp.path(), "runtime-worker")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
rt.write_status(&test_state()).await.unwrap();
|
||||
rt.write_manifest("[worker]\nname = \"runtime-worker\"")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(!rt.path().join("pid").exists());
|
||||
assert!(!rt.path().join("status.json").exists());
|
||||
assert!(!rt.path().join("manifest.toml").exists());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn write_spawned_workers_creates_file() {
|
||||
use manifest::{Permission, ScopeRule};
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let rt = RuntimeDir::create(tmp.path(), "my-worker").await.unwrap();
|
||||
|
||||
let records = vec![SpawnedWorkerRecord {
|
||||
worker_name: "child".into(),
|
||||
socket_path: "/run/yoi/child/sock".into(),
|
||||
scope_delegated: vec![ScopeRule {
|
||||
target: "/tmp/work".into(),
|
||||
permission: Permission::Write,
|
||||
recursive: true,
|
||||
symlink_policy: Default::default(),
|
||||
}],
|
||||
callback_address: "/run/yoi/my-worker/sock".into(),
|
||||
}];
|
||||
rt.write_spawned_workers(&records).await.unwrap();
|
||||
|
||||
let content = std::fs::read_to_string(rt.path().join("spawned_workers.json")).unwrap();
|
||||
let parsed: Vec<SpawnedWorkerRecord> = serde_json::from_str(&content).unwrap();
|
||||
assert_eq!(parsed.len(), 1);
|
||||
assert_eq!(parsed[0].worker_name, "child");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn socket_path() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let rt = RuntimeDir::create(tmp.path(), "my-worker").await.unwrap();
|
||||
assert_eq!(rt.socket_path(), rt.path().join("sock"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn drop_removes_directory() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let dir_path;
|
||||
async fn transient_controller_directory_has_no_liveness_projections_and_is_released() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let path;
|
||||
{
|
||||
let rt = RuntimeDir::create(tmp.path(), "my-worker").await.unwrap();
|
||||
dir_path = rt.path().to_owned();
|
||||
assert!(dir_path.exists());
|
||||
let runtime = RuntimeDir::create(root.path(), "worker").await.unwrap();
|
||||
path = runtime.path().to_owned();
|
||||
assert_eq!(std::fs::read_dir(&path).unwrap().count(), 0);
|
||||
}
|
||||
assert!(!dir_path.exists());
|
||||
assert!(!path.exists());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn preexisting_data_is_not_overwritten_or_removed_on_drop() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let path = root.path().join("worker");
|
||||
std::fs::create_dir(&path).unwrap();
|
||||
std::fs::write(path.join("pid"), "retained legacy data").unwrap();
|
||||
{
|
||||
let _runtime = RuntimeDir::create(root.path(), "worker").await.unwrap();
|
||||
}
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(path.join("pid")).unwrap(),
|
||||
"retained legacy data"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn persistent_run_preserves_logs_and_rejects_directory_reuse() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let path = root.path().join("runs/1");
|
||||
{
|
||||
let _runtime = RuntimeDir::create_worker_run(&path).await.unwrap();
|
||||
for name in ["artifacts", "spawned", "worker.out.log", "worker.err.log"] {
|
||||
assert!(path.join(name).exists(), "missing {name}");
|
||||
}
|
||||
}
|
||||
assert!(path.exists());
|
||||
let error = RuntimeDir::create_worker_run(&path).await.err().unwrap();
|
||||
assert_eq!(error.kind(), io::ErrorKind::AlreadyExists);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_reclamation_projection_retains_its_stored_format() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let runtime = RuntimeDir::create(root.path(), "worker").await.unwrap();
|
||||
let record: SpawnedWorkerRecord = serde_json::from_value(serde_json::json!({
|
||||
"worker_name": "legacy-child", "socket_path": "/old/child/sock",
|
||||
"scope_delegated": [], "callback_address": "/old/parent/sock",
|
||||
}))
|
||||
.unwrap();
|
||||
runtime.write_spawned_workers(&[record]).await.unwrap();
|
||||
let value: serde_json::Value = serde_json::from_slice(
|
||||
&std::fs::read(runtime.path().join("spawned_workers.json")).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(value[0]["socket_path"], "/old/child/sock");
|
||||
assert_eq!(value[0]["callback_address"], "/old/parent/sock");
|
||||
runtime.write_spawned_workers(&[]).await.unwrap();
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(runtime.path().join("spawned_workers.json")).unwrap(),
|
||||
"[]"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,8 +21,9 @@ pub use error::ScopeLockError;
|
||||
#[cfg(test)]
|
||||
pub(crate) use lifecycle::install_top_level_at_for_test;
|
||||
pub use lifecycle::{
|
||||
ScopeAllocationGuard, SegmentActivationCommitError, SegmentActivationGuard, SegmentLockInfo,
|
||||
adopt_allocation, install_top_level, install_top_level_with_deny, lookup_segment,
|
||||
ScopeAllocationCleanup, ScopeAllocationGuard, SegmentActivationCommitError,
|
||||
SegmentActivationGuard, SegmentLockInfo, adopt_allocation, install_top_level,
|
||||
install_top_level_with_deny, lookup_segment,
|
||||
};
|
||||
pub use mutate::{
|
||||
delegate_scope, reclaim_delegated_scope, reclaim_stale, reclaim_stale_with, register_worker,
|
||||
|
||||
@@ -11,6 +11,14 @@ use session_store::SegmentId;
|
||||
pub enum ScopeLockError {
|
||||
#[error("I/O error on workers.json: {0}")]
|
||||
Io(#[from] io::Error),
|
||||
#[error("worker `{worker_name}` allocation still has a live guard")]
|
||||
AllocationGuardAttached { worker_name: String },
|
||||
#[error("allocation acquisition was published but its save failed: {source}")]
|
||||
AcquisitionSaveFailed {
|
||||
#[source]
|
||||
source: io::Error,
|
||||
cleanup: super::lifecycle::ScopeAllocationCleanup,
|
||||
},
|
||||
#[error("worker `{0}` is already allocated")]
|
||||
DuplicateWorkerName(String),
|
||||
#[error("requested scope `{}` conflicts with worker allocation `{competitor}` rule `{}`", .rule.target.display(), .competitor_rule.target.display())]
|
||||
@@ -46,3 +54,15 @@ pub enum ScopeLockError {
|
||||
actual: Option<SegmentId>,
|
||||
},
|
||||
}
|
||||
|
||||
impl ScopeLockError {
|
||||
/// Retain the release authority for an owned acquisition whose table rename
|
||||
/// succeeded before save returned an error. Refusals and unpublished saves
|
||||
/// do not confer ownership of any preexisting allocation.
|
||||
pub fn allocation_cleanup_handle(&self) -> Option<super::lifecycle::ScopeAllocationCleanup> {
|
||||
match self {
|
||||
Self::AcquisitionSaveFailed { cleanup, .. } => Some(cleanup.clone()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
//! up on drop.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use manifest::ScopeRule;
|
||||
use session_store::SegmentId;
|
||||
@@ -11,13 +12,82 @@ use super::error::ScopeLockError;
|
||||
use super::mutate::release_worker;
|
||||
use super::table::{LockFileGuard, default_allocation_path};
|
||||
|
||||
/// Owned allocation: on drop, opens the lock file and releases this
|
||||
/// Worker's entry. The guard keeps only the name + lock-file path; it
|
||||
/// does not hold the `flock` for the Worker's lifetime.
|
||||
/// Owned allocation shared with any retained cleanup handles. Only the final
|
||||
/// owner's drop attempts best-effort release; no `flock` is held for its lifetime.
|
||||
#[derive(Debug)]
|
||||
pub struct ScopeAllocationGuard {
|
||||
owner: Arc<ScopeAllocationOwner>,
|
||||
}
|
||||
|
||||
/// Retains an allocation's release authority even after its Worker is consumed
|
||||
/// or dropped. A successful controller transfer should drop this handle without
|
||||
/// explicitly releasing the live Worker's allocation.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct ScopeAllocationCleanup {
|
||||
owner: Arc<ScopeAllocationOwner>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct ScopeAllocationOwner {
|
||||
worker_name: String,
|
||||
lock_path: PathBuf,
|
||||
state: Mutex<ScopeAllocationOwnerState>,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct ScopeAllocationOwnerState {
|
||||
guard_attached: bool,
|
||||
released: bool,
|
||||
}
|
||||
|
||||
impl ScopeAllocationOwner {
|
||||
fn release(&self) -> Result<(), ScopeLockError> {
|
||||
let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner());
|
||||
if state.guard_attached {
|
||||
return Err(ScopeLockError::AllocationGuardAttached {
|
||||
worker_name: self.worker_name.clone(),
|
||||
});
|
||||
}
|
||||
if state.released {
|
||||
return Ok(());
|
||||
}
|
||||
let mut guard = LockFileGuard::open(&self.lock_path)?;
|
||||
if guard.data().find(&self.worker_name).is_some() {
|
||||
release_worker(&mut guard, &self.worker_name)?;
|
||||
} else {
|
||||
// A previous save may have removed the entry but failed after rename.
|
||||
// Complete the durable save before reporting release as confirmed.
|
||||
guard.save()?;
|
||||
}
|
||||
state.released = true;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ScopeAllocationOwner {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.release();
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ScopeAllocationGuard {
|
||||
fn drop(&mut self) {
|
||||
self.owner
|
||||
.state
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner())
|
||||
.guard_attached = false;
|
||||
}
|
||||
}
|
||||
|
||||
impl ScopeAllocationCleanup {
|
||||
/// Release only after the factory has returned an error and no live
|
||||
/// Controller can still use this allocation. An attached guard rejects
|
||||
/// release. Success is idempotent across all clones; failure retains the
|
||||
/// authority for an explicit retry.
|
||||
pub fn release(&self) -> Result<(), ScopeLockError> {
|
||||
self.owner.release()
|
||||
}
|
||||
}
|
||||
|
||||
/// Exclusive authority to activate a replacement Segment for one allocated Worker.
|
||||
@@ -56,12 +126,58 @@ impl SegmentActivationCommitError {
|
||||
}
|
||||
|
||||
impl ScopeAllocationGuard {
|
||||
fn new(worker_name: String, lock_path: PathBuf) -> Self {
|
||||
Self {
|
||||
owner: Arc::new(ScopeAllocationOwner {
|
||||
worker_name,
|
||||
lock_path,
|
||||
state: Mutex::new(ScopeAllocationOwnerState {
|
||||
guard_attached: true,
|
||||
released: false,
|
||||
}),
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn commit_acquisition(self, table: &mut LockFileGuard) -> Result<Self, ScopeLockError> {
|
||||
let mut published = false;
|
||||
match table.save_with_publication(&mut published) {
|
||||
Ok(()) => Ok(self),
|
||||
Err(source) => {
|
||||
let cleanup = if published {
|
||||
Some(self.cleanup_handle())
|
||||
} else {
|
||||
// A staged entry is not ours until rename. Disarm before
|
||||
// detaching so Drop cannot remove a preexisting allocation
|
||||
// (or reopen the table lock still held by this caller).
|
||||
self.owner
|
||||
.state
|
||||
.lock()
|
||||
.unwrap_or_else(|error| error.into_inner())
|
||||
.released = true;
|
||||
None
|
||||
};
|
||||
drop(self);
|
||||
match cleanup {
|
||||
Some(cleanup) => Err(ScopeLockError::AcquisitionSaveFailed { source, cleanup }),
|
||||
None => Err(ScopeLockError::Io(source)),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn worker_name(&self) -> &str {
|
||||
&self.worker_name
|
||||
&self.owner.worker_name
|
||||
}
|
||||
|
||||
pub fn lock_path(&self) -> &Path {
|
||||
&self.lock_path
|
||||
&self.owner.lock_path
|
||||
}
|
||||
|
||||
pub fn cleanup_handle(&self) -> ScopeAllocationCleanup {
|
||||
ScopeAllocationCleanup {
|
||||
owner: Arc::clone(&self.owner),
|
||||
}
|
||||
}
|
||||
|
||||
/// Lock the machine-wide table and validate a Segment replacement before
|
||||
@@ -71,24 +187,25 @@ impl ScopeAllocationGuard {
|
||||
expected: SegmentId,
|
||||
replacement: SegmentId,
|
||||
) -> Result<SegmentActivationGuard, ScopeLockError> {
|
||||
let guard = LockFileGuard::open(&self.lock_path)?;
|
||||
let guard = LockFileGuard::open(self.lock_path())?;
|
||||
let actual = guard
|
||||
.data()
|
||||
.allocations
|
||||
.iter()
|
||||
.find(|allocation| allocation.worker_name == self.worker_name)
|
||||
.find(|allocation| allocation.worker_name == self.worker_name())
|
||||
.and_then(|allocation| allocation.segment_id);
|
||||
|
||||
if actual != Some(expected) {
|
||||
return Err(ScopeLockError::SegmentChanged {
|
||||
worker_name: self.worker_name.clone(),
|
||||
worker_name: self.worker_name().to_owned(),
|
||||
expected,
|
||||
actual,
|
||||
});
|
||||
}
|
||||
|
||||
if let Some(existing) = guard.data().allocations.iter().find(|allocation| {
|
||||
allocation.segment_id == Some(replacement) && allocation.worker_name != self.worker_name
|
||||
allocation.segment_id == Some(replacement)
|
||||
&& allocation.worker_name != self.worker_name()
|
||||
}) {
|
||||
return Err(ScopeLockError::SegmentConflict {
|
||||
segment_id: replacement,
|
||||
@@ -99,7 +216,7 @@ impl ScopeAllocationGuard {
|
||||
|
||||
Ok(SegmentActivationGuard {
|
||||
guard,
|
||||
worker_name: self.worker_name.clone(),
|
||||
worker_name: self.worker_name().to_owned(),
|
||||
expected,
|
||||
replacement,
|
||||
})
|
||||
@@ -145,14 +262,6 @@ impl SegmentActivationGuard {
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ScopeAllocationGuard {
|
||||
fn drop(&mut self) {
|
||||
if let Ok(mut guard) = LockFileGuard::open(&self.lock_path) {
|
||||
let _ = release_worker(&mut guard, &self.worker_name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Open the default lock file, register a top-level Worker, and return a
|
||||
/// guard that will release the allocation on drop.
|
||||
pub fn install_top_level(
|
||||
@@ -185,7 +294,7 @@ pub fn install_top_level_with_deny(
|
||||
) -> Result<ScopeAllocationGuard, ScopeLockError> {
|
||||
let lock_path = default_allocation_path()?;
|
||||
let mut guard = LockFileGuard::open(&lock_path)?;
|
||||
super::mutate::register_worker_with_deny(
|
||||
super::mutate::prepare_worker_with_deny(
|
||||
&mut guard,
|
||||
worker_name.clone(),
|
||||
pid,
|
||||
@@ -194,10 +303,7 @@ pub fn install_top_level_with_deny(
|
||||
scope_deny,
|
||||
segment_id,
|
||||
)?;
|
||||
Ok(ScopeAllocationGuard {
|
||||
worker_name,
|
||||
lock_path,
|
||||
})
|
||||
ScopeAllocationGuard::new(worker_name, lock_path).commit_acquisition(&mut guard)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -210,7 +316,7 @@ pub(crate) fn install_top_level_at_for_test(
|
||||
segment_id: SegmentId,
|
||||
) -> Result<ScopeAllocationGuard, ScopeLockError> {
|
||||
let mut guard = LockFileGuard::open(&lock_path)?;
|
||||
super::mutate::register_worker_with_deny(
|
||||
super::mutate::prepare_worker_with_deny(
|
||||
&mut guard,
|
||||
worker_name.clone(),
|
||||
pid,
|
||||
@@ -219,10 +325,7 @@ pub(crate) fn install_top_level_at_for_test(
|
||||
Vec::new(),
|
||||
segment_id,
|
||||
)?;
|
||||
Ok(ScopeAllocationGuard {
|
||||
worker_name,
|
||||
lock_path,
|
||||
})
|
||||
ScopeAllocationGuard::new(worker_name, lock_path).commit_acquisition(&mut guard)
|
||||
}
|
||||
|
||||
/// Take ownership of an existing allocation that was pre-registered by
|
||||
@@ -240,6 +343,15 @@ pub fn adopt_allocation(
|
||||
segment_id: SegmentId,
|
||||
) -> Result<ScopeAllocationGuard, ScopeLockError> {
|
||||
let lock_path = default_allocation_path()?;
|
||||
adopt_allocation_at(lock_path, worker_name, new_pid, segment_id)
|
||||
}
|
||||
|
||||
fn adopt_allocation_at(
|
||||
lock_path: PathBuf,
|
||||
worker_name: String,
|
||||
new_pid: u32,
|
||||
segment_id: SegmentId,
|
||||
) -> Result<ScopeAllocationGuard, ScopeLockError> {
|
||||
let mut guard = LockFileGuard::open(&lock_path)?;
|
||||
let alloc = guard
|
||||
.data_mut()
|
||||
@@ -247,11 +359,7 @@ pub fn adopt_allocation(
|
||||
.ok_or_else(|| ScopeLockError::UnknownWorker(worker_name.clone()))?;
|
||||
alloc.pid = new_pid;
|
||||
alloc.segment_id = Some(segment_id);
|
||||
guard.save()?;
|
||||
Ok(ScopeAllocationGuard {
|
||||
worker_name,
|
||||
lock_path,
|
||||
})
|
||||
ScopeAllocationGuard::new(worker_name, lock_path).commit_acquisition(&mut guard)
|
||||
}
|
||||
|
||||
/// Information about a Worker that currently holds an allocation for a
|
||||
@@ -329,6 +437,388 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retained_cleanup_handle_defers_release_until_final_owner_drop() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let lock_path = dir.path().join("workers.json");
|
||||
let guard = install_top_level_at_for_test(
|
||||
lock_path.clone(),
|
||||
"retained".into(),
|
||||
std::process::id(),
|
||||
sock("retained"),
|
||||
Vec::new(),
|
||||
sid(),
|
||||
)
|
||||
.unwrap();
|
||||
let cleanup = guard.cleanup_handle();
|
||||
let retained = cleanup.clone();
|
||||
|
||||
drop(guard);
|
||||
drop(cleanup);
|
||||
{
|
||||
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||
assert!(table.data().find("retained").is_some());
|
||||
}
|
||||
drop(retained);
|
||||
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||
assert!(table.data().find("retained").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn dropping_cleanup_handle_after_transfer_preserves_live_guard() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let lock_path = dir.path().join("workers.json");
|
||||
let segment = sid();
|
||||
let guard = install_top_level_at_for_test(
|
||||
lock_path.clone(),
|
||||
"transferred".into(),
|
||||
std::process::id(),
|
||||
sock("transferred"),
|
||||
Vec::new(),
|
||||
segment,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
drop(guard.cleanup_handle());
|
||||
// A successful controller transfer leaves the guard usable, including
|
||||
// the existing activation protocol, without retaining a cleanup handle.
|
||||
let replacement = sid();
|
||||
guard
|
||||
.begin_segment_activation(segment, replacement)
|
||||
.unwrap()
|
||||
.commit()
|
||||
.unwrap();
|
||||
{
|
||||
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||
assert_eq!(
|
||||
table.data().find("transferred").unwrap().segment_id,
|
||||
Some(replacement)
|
||||
);
|
||||
}
|
||||
drop(guard);
|
||||
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||
assert!(table.data().find("transferred").is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cleanup_rejects_live_guard_then_releases_only_detached_allocation() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let lock_path = dir.path().join("workers.json");
|
||||
let guard = install_top_level_at_for_test(
|
||||
lock_path.clone(),
|
||||
"live".into(),
|
||||
std::process::id(),
|
||||
sock("live"),
|
||||
Vec::new(),
|
||||
sid(),
|
||||
)
|
||||
.unwrap();
|
||||
let _competitor = install_top_level_at_for_test(
|
||||
lock_path.clone(),
|
||||
"competitor".into(),
|
||||
std::process::id(),
|
||||
sock("competitor"),
|
||||
Vec::new(),
|
||||
sid(),
|
||||
)
|
||||
.unwrap();
|
||||
let cleanup = guard.cleanup_handle();
|
||||
assert!(matches!(
|
||||
cleanup.release(),
|
||||
Err(ScopeLockError::AllocationGuardAttached { worker_name }) if worker_name == "live"
|
||||
));
|
||||
drop(cleanup.clone());
|
||||
{
|
||||
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||
assert!(table.data().find("live").is_some());
|
||||
assert!(table.data().find("competitor").is_some());
|
||||
}
|
||||
|
||||
drop(guard);
|
||||
cleanup.release().unwrap();
|
||||
cleanup.clone().release().unwrap();
|
||||
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||
assert!(table.data().find("live").is_none());
|
||||
assert!(table.data().find("competitor").is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn published_install_failure_retains_cleanup_across_error_drop_and_retry() {
|
||||
use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test};
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let lock_path = dir.path().join("workers.json");
|
||||
let _competitor = install_top_level_at_for_test(
|
||||
lock_path.clone(),
|
||||
"competitor".into(),
|
||||
std::process::id(),
|
||||
sock("competitor"),
|
||||
Vec::new(),
|
||||
sid(),
|
||||
)
|
||||
.unwrap();
|
||||
let segment = sid();
|
||||
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::AfterRename);
|
||||
let error = install_top_level_at_for_test(
|
||||
lock_path.clone(),
|
||||
"failed".into(),
|
||||
std::process::id(),
|
||||
sock("failed"),
|
||||
Vec::new(),
|
||||
segment,
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(matches!(
|
||||
&error,
|
||||
ScopeLockError::AcquisitionSaveFailed { .. }
|
||||
));
|
||||
let cleanup = error.allocation_cleanup_handle().unwrap();
|
||||
let retry = error.allocation_cleanup_handle().unwrap();
|
||||
drop(error);
|
||||
{
|
||||
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||
assert_eq!(
|
||||
table.data().find("failed").unwrap().segment_id,
|
||||
Some(segment)
|
||||
);
|
||||
assert!(table.data().find("competitor").is_some());
|
||||
}
|
||||
|
||||
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::BeforeRename);
|
||||
assert!(matches!(cleanup.release(), Err(ScopeLockError::Io(_))));
|
||||
drop(cleanup);
|
||||
{
|
||||
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||
assert!(table.data().find("failed").is_some());
|
||||
}
|
||||
retry.release().unwrap();
|
||||
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||
assert!(table.data().find("failed").is_none());
|
||||
assert!(table.data().find("competitor").is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn refused_install_has_no_cleanup_authority_over_existing_allocation() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let lock_path = dir.path().join("workers.json");
|
||||
let segment = sid();
|
||||
let _competitor = install_top_level_at_for_test(
|
||||
lock_path.clone(),
|
||||
"competitor".into(),
|
||||
std::process::id(),
|
||||
sock("competitor"),
|
||||
vec![write_rule("/shared", true)],
|
||||
segment,
|
||||
)
|
||||
.unwrap();
|
||||
let cases = [
|
||||
("competitor", sid(), Vec::new()),
|
||||
("other", segment, Vec::new()),
|
||||
("other", sid(), vec![write_rule("/shared", true)]),
|
||||
];
|
||||
for (name, requested_segment, scope) in cases {
|
||||
let error = install_top_level_at_for_test(
|
||||
lock_path.clone(),
|
||||
name.into(),
|
||||
std::process::id(),
|
||||
sock("refused"),
|
||||
scope,
|
||||
requested_segment,
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(matches!(
|
||||
&error,
|
||||
ScopeLockError::DuplicateWorkerName(_)
|
||||
| ScopeLockError::SegmentConflict { .. }
|
||||
| ScopeLockError::WriteConflict { .. }
|
||||
));
|
||||
assert!(error.allocation_cleanup_handle().is_none());
|
||||
drop(error);
|
||||
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||
assert_eq!(table.data().allocations.len(), 1);
|
||||
assert_eq!(
|
||||
table.data().find("competitor").unwrap().socket,
|
||||
sock("competitor")
|
||||
);
|
||||
assert_eq!(
|
||||
table.data().find("competitor").unwrap().segment_id,
|
||||
Some(segment)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unpublished_install_failure_has_no_cleanup_authority() {
|
||||
use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test};
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let lock_path = dir.path().join("workers.json");
|
||||
let _competitor = install_top_level_at_for_test(
|
||||
lock_path.clone(),
|
||||
"competitor".into(),
|
||||
std::process::id(),
|
||||
sock("competitor"),
|
||||
Vec::new(),
|
||||
sid(),
|
||||
)
|
||||
.unwrap();
|
||||
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::BeforeRename);
|
||||
let error = install_top_level_at_for_test(
|
||||
lock_path.clone(),
|
||||
"failed".into(),
|
||||
std::process::id(),
|
||||
sock("failed"),
|
||||
Vec::new(),
|
||||
sid(),
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(matches!(&error, ScopeLockError::Io(_)));
|
||||
assert!(error.allocation_cleanup_handle().is_none());
|
||||
drop(error);
|
||||
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||
assert!(table.data().find("failed").is_none());
|
||||
assert!(table.data().find("competitor").is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn published_adoption_failure_retains_cleanup_for_retry() {
|
||||
use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test};
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let lock_path = dir.path().join("workers.json");
|
||||
{
|
||||
let mut table = LockFileGuard::open(&lock_path).unwrap();
|
||||
delegate_placeholder(&mut table, "child", std::process::id());
|
||||
delegate_placeholder(&mut table, "competitor", std::process::id());
|
||||
}
|
||||
let segment = sid();
|
||||
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::AfterRename);
|
||||
let error = adopt_allocation_at(
|
||||
lock_path.clone(),
|
||||
"child".into(),
|
||||
std::process::id(),
|
||||
segment,
|
||||
)
|
||||
.unwrap_err();
|
||||
let cleanup = error.allocation_cleanup_handle().unwrap();
|
||||
drop(error);
|
||||
{
|
||||
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||
assert_eq!(
|
||||
table.data().find("child").unwrap().segment_id,
|
||||
Some(segment)
|
||||
);
|
||||
}
|
||||
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::AfterRename);
|
||||
assert!(matches!(cleanup.release(), Err(ScopeLockError::Io(_))));
|
||||
{
|
||||
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||
assert!(table.data().find("child").is_none());
|
||||
assert!(table.data().find("competitor").is_some());
|
||||
}
|
||||
cleanup.release().unwrap();
|
||||
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||
assert!(table.data().find("child").is_none());
|
||||
assert!(table.data().find("competitor").is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unpublished_adoption_failure_preserves_placeholder_without_cleanup_authority() {
|
||||
use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test};
|
||||
|
||||
let dir = TempDir::new().unwrap();
|
||||
let lock_path = dir.path().join("workers.json");
|
||||
let placeholder_pid = std::process::id();
|
||||
{
|
||||
let mut table = LockFileGuard::open(&lock_path).unwrap();
|
||||
delegate_placeholder(&mut table, "child", placeholder_pid);
|
||||
}
|
||||
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::BeforeRename);
|
||||
let error = adopt_allocation_at(
|
||||
lock_path.clone(),
|
||||
"child".into(),
|
||||
placeholder_pid.wrapping_add(1),
|
||||
sid(),
|
||||
)
|
||||
.unwrap_err();
|
||||
assert!(matches!(&error, ScopeLockError::Io(_)));
|
||||
assert!(error.allocation_cleanup_handle().is_none());
|
||||
drop(error);
|
||||
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||
let allocation = table.data().find("child").unwrap();
|
||||
assert_eq!(allocation.pid, placeholder_pid);
|
||||
assert_eq!(allocation.segment_id, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failed_cleanup_release_retains_authority_for_retry() {
|
||||
use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test};
|
||||
|
||||
for point in [SaveFaultPoint::BeforeRename, SaveFaultPoint::AfterRename] {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let lock_path = dir.path().join("workers.json");
|
||||
let guard = install_top_level_at_for_test(
|
||||
lock_path.clone(),
|
||||
"failed".into(),
|
||||
std::process::id(),
|
||||
sock("failed"),
|
||||
vec![read_rule("/shared", true)],
|
||||
sid(),
|
||||
)
|
||||
.unwrap();
|
||||
let _other = install_top_level_at_for_test(
|
||||
lock_path.clone(),
|
||||
"other".into(),
|
||||
std::process::id(),
|
||||
sock("other"),
|
||||
vec![read_rule("/shared", true)],
|
||||
sid(),
|
||||
)
|
||||
.unwrap();
|
||||
let cleanup = guard.cleanup_handle();
|
||||
let retry = cleanup.clone();
|
||||
drop(guard);
|
||||
fail_next_save_at_for_test(&lock_path, point);
|
||||
|
||||
assert!(matches!(cleanup.release(), Err(ScopeLockError::Io(_))));
|
||||
drop(cleanup);
|
||||
{
|
||||
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||
assert_eq!(
|
||||
table.data().find("failed").is_some(),
|
||||
point == SaveFaultPoint::BeforeRename
|
||||
);
|
||||
assert!(table.data().find("other").is_some());
|
||||
}
|
||||
retry.release().unwrap();
|
||||
{
|
||||
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||
assert!(table.data().find("failed").is_none());
|
||||
assert!(table.data().find("other").is_some());
|
||||
}
|
||||
|
||||
// Success disarms every clone and final Drop, even if the name is
|
||||
// subsequently allocated to a different Worker.
|
||||
let _replacement = install_top_level_at_for_test(
|
||||
lock_path.clone(),
|
||||
"failed".into(),
|
||||
std::process::id(),
|
||||
sock("replacement"),
|
||||
vec![read_rule("/shared", true)],
|
||||
sid(),
|
||||
)
|
||||
.unwrap();
|
||||
retry.clone().release().unwrap();
|
||||
drop(retry);
|
||||
let table = LockFileGuard::open(&lock_path).unwrap();
|
||||
assert_eq!(
|
||||
table.data().find("failed").unwrap().socket,
|
||||
sock("replacement")
|
||||
);
|
||||
assert!(table.data().find("other").is_some());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn adopt_allocation_rewrites_pid_and_releases_on_drop() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
@@ -358,6 +848,7 @@ mod tests {
|
||||
let dir = TempDir::new().unwrap();
|
||||
let _sandbox = RuntimeDirSandbox::new(dir.path());
|
||||
let err = adopt_allocation("ghost".into(), 42, sid()).unwrap_err();
|
||||
assert!(err.allocation_cleanup_handle().is_none());
|
||||
assert!(matches!(err, ScopeLockError::UnknownWorker(ref n) if n == "ghost"));
|
||||
}
|
||||
|
||||
|
||||
@@ -57,6 +57,30 @@ pub fn register_worker_with_deny(
|
||||
scope_allow: Vec<ScopeRule>,
|
||||
scope_deny: Vec<ScopeRule>,
|
||||
segment_id: SegmentId,
|
||||
) -> Result<(), ScopeLockError> {
|
||||
prepare_worker_with_deny(
|
||||
guard,
|
||||
worker_name,
|
||||
pid,
|
||||
socket,
|
||||
scope_allow,
|
||||
scope_deny,
|
||||
segment_id,
|
||||
)?;
|
||||
guard.save()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Admit and stage a registration without publishing it. Owned installation
|
||||
/// constructs its cleanup authority after admission and before committing.
|
||||
pub(super) fn prepare_worker_with_deny(
|
||||
guard: &mut LockFileGuard,
|
||||
worker_name: String,
|
||||
pid: u32,
|
||||
socket: PathBuf,
|
||||
scope_allow: Vec<ScopeRule>,
|
||||
scope_deny: Vec<ScopeRule>,
|
||||
segment_id: SegmentId,
|
||||
) -> Result<(), ScopeLockError> {
|
||||
reclaim_stale(guard);
|
||||
if guard.data().find(&worker_name).is_some() {
|
||||
@@ -101,7 +125,6 @@ pub fn register_worker_with_deny(
|
||||
delegated_from: None,
|
||||
segment_id: Some(segment_id),
|
||||
});
|
||||
guard.save()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -72,7 +72,8 @@ pub struct Allocation {
|
||||
pub worker_name: String,
|
||||
/// Owning process. Checked with `kill(pid, 0)` for stale detection.
|
||||
pub pid: u32,
|
||||
/// Worker's Unix socket path.
|
||||
/// Legacy stored endpoint hint. Empty means the Host owns connectivity.
|
||||
/// Allocation admission and stale reclamation use identity/PID, never this path.
|
||||
pub socket: PathBuf,
|
||||
/// Allow rules granted to this Worker (write + read).
|
||||
pub scope_allow: Vec<ScopeRule>,
|
||||
@@ -245,6 +246,14 @@ impl LockFileGuard {
|
||||
/// Persist with atomic replacement while the separate allocation lock stays
|
||||
/// held. A crash exposes either the old complete table or the new one.
|
||||
pub fn save(&mut self) -> io::Result<()> {
|
||||
let mut published = false;
|
||||
self.save_with_publication(&mut published)
|
||||
}
|
||||
|
||||
/// Report this save's rename while the caller still holds the table lock.
|
||||
/// In-memory mutation alone is not evidence that an acquisition took place.
|
||||
pub(super) fn save_with_publication(&mut self, published: &mut bool) -> io::Result<()> {
|
||||
*published = false;
|
||||
let json = serde_json::to_vec_pretty(&self.data).map_err(io::Error::other)?;
|
||||
let mut temp_os = self.data_path.as_os_str().to_owned();
|
||||
temp_os.push(".tmp");
|
||||
@@ -264,6 +273,7 @@ impl LockFileGuard {
|
||||
));
|
||||
}
|
||||
fs::rename(&temp_path, &self.data_path)?;
|
||||
*published = true;
|
||||
#[cfg(test)]
|
||||
if take_save_fault(&self.data_path, SaveFaultPoint::AfterRename) {
|
||||
return Err(io::Error::other(
|
||||
@@ -338,6 +348,51 @@ mod tests {
|
||||
assert_eq!(guard.data().allocations[0].worker_name, "a");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_legacy_socket_roundtrips_without_path_resolution_or_cleanup() {
|
||||
let root = TempDir::new().unwrap();
|
||||
let path = root.path().join("workers.json");
|
||||
let retained = root.path().join("retained-data");
|
||||
fs::write(&retained, "do not touch").unwrap();
|
||||
let segment = sid();
|
||||
{
|
||||
let mut guard = open_empty(&path);
|
||||
register_worker(
|
||||
&mut guard,
|
||||
"in-process".into(),
|
||||
std::process::id(),
|
||||
PathBuf::new(),
|
||||
Vec::new(),
|
||||
segment,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
let stored: serde_json::Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap();
|
||||
assert_eq!(stored["allocations"][0]["socket"], "");
|
||||
let mut guard = LockFileGuard::open(&path).unwrap();
|
||||
assert!(guard.data().allocations[0].socket.as_os_str().is_empty());
|
||||
super::super::reclaim_stale_with(&mut guard, |pid| {
|
||||
assert_eq!(pid, std::process::id());
|
||||
true
|
||||
});
|
||||
assert_eq!(
|
||||
guard.data().allocations.len(),
|
||||
1,
|
||||
"missing endpoint is not stale ownership"
|
||||
);
|
||||
assert!(guard.data().find_by_segment(segment).is_some());
|
||||
super::super::reclaim_stale_with(&mut guard, |_| false);
|
||||
drop(guard);
|
||||
assert!(
|
||||
LockFileGuard::open(&path)
|
||||
.unwrap()
|
||||
.data()
|
||||
.allocations
|
||||
.is_empty()
|
||||
);
|
||||
assert_eq!(fs::read_to_string(retained).unwrap(), "do not touch");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn atomic_save_reopens_complete_table_at_both_rename_boundaries() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
|
||||
@@ -1,165 +0,0 @@
|
||||
use std::ffi::OsString;
|
||||
use std::fmt;
|
||||
use std::io;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
const POD_RUNTIME_COMMAND_ENV: &str = "YOI_POD_RUNTIME_COMMAND";
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct WorkerRuntimeCommand {
|
||||
pub program: PathBuf,
|
||||
pub prefix_args: Vec<OsString>,
|
||||
}
|
||||
|
||||
impl WorkerRuntimeCommand {
|
||||
pub fn new(program: impl Into<PathBuf>, prefix_args: Vec<OsString>) -> Self {
|
||||
Self {
|
||||
program: program.into(),
|
||||
prefix_args,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn for_current_exe() -> io::Result<Self> {
|
||||
Ok(Self::for_executable(std::env::current_exe()?))
|
||||
}
|
||||
|
||||
pub fn for_executable(program: impl Into<PathBuf>) -> Self {
|
||||
Self::new(program, vec![OsString::from("worker")])
|
||||
}
|
||||
|
||||
/// Resolve the Worker runtime command used for subprocess launches.
|
||||
///
|
||||
/// The default launch path is always the current `yoi` executable plus
|
||||
/// the unified `worker` prefix argument. During development, a non-empty
|
||||
/// `YOI_POD_RUNTIME_COMMAND` value replaces only the executable path;
|
||||
/// the `worker` prefix is still added here and the env value is not parsed as a
|
||||
/// shell command.
|
||||
pub fn resolve() -> io::Result<Self> {
|
||||
Self::resolve_from_env_value(
|
||||
std::env::var_os(POD_RUNTIME_COMMAND_ENV),
|
||||
std::env::current_exe,
|
||||
)
|
||||
}
|
||||
|
||||
fn resolve_from_env_value<F>(
|
||||
override_program: Option<OsString>,
|
||||
current_exe: F,
|
||||
) -> io::Result<Self>
|
||||
where
|
||||
F: FnOnce() -> io::Result<PathBuf>,
|
||||
{
|
||||
if let Some(program) = override_program.filter(|program| !program.as_os_str().is_empty()) {
|
||||
return Ok(Self::for_executable(program));
|
||||
}
|
||||
|
||||
Ok(Self::for_executable(current_exe()?))
|
||||
}
|
||||
|
||||
pub fn program(&self) -> &Path {
|
||||
&self.program
|
||||
}
|
||||
|
||||
pub fn prefix_args(&self) -> &[OsString] {
|
||||
&self.prefix_args
|
||||
}
|
||||
|
||||
pub fn argv_with<I, S>(&self, args: I) -> Vec<OsString>
|
||||
where
|
||||
I: IntoIterator<Item = S>,
|
||||
S: Into<OsString>,
|
||||
{
|
||||
let mut argv = self.prefix_args.clone();
|
||||
argv.extend(args.into_iter().map(Into::into));
|
||||
argv
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for WorkerRuntimeCommand {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{}", self.program.display())?;
|
||||
for arg in &self.prefix_args {
|
||||
write!(f, " {}", arg.to_string_lossy())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn yoi_binary_defaults_to_worker_prefix() {
|
||||
let command = WorkerRuntimeCommand::for_executable("/opt/yoi/bin/yoi");
|
||||
|
||||
assert_eq!(command.program(), Path::new("/opt/yoi/bin/yoi"));
|
||||
assert_eq!(command.prefix_args(), [OsString::from("worker")]);
|
||||
assert_eq!(
|
||||
command.argv_with(["--worker", "agent"]),
|
||||
vec!["worker", "--worker", "agent"]
|
||||
.into_iter()
|
||||
.map(OsString::from)
|
||||
.collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn any_runtime_executable_gets_worker_prefix() {
|
||||
let command = WorkerRuntimeCommand::for_executable("/opt/yoi/bin/custom-runtime");
|
||||
|
||||
assert_eq!(command.program(), Path::new("/opt/yoi/bin/custom-runtime"));
|
||||
assert_eq!(command.prefix_args(), [OsString::from("worker")]);
|
||||
assert_eq!(
|
||||
command.argv_with(["--worker", "agent"]),
|
||||
vec!["worker", "--worker", "agent"]
|
||||
.into_iter()
|
||||
.map(OsString::from)
|
||||
.collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_uses_current_exe_when_override_is_unset() {
|
||||
let command = WorkerRuntimeCommand::resolve_from_env_value(None, || {
|
||||
Ok(PathBuf::from("/opt/yoi/bin/yoi"))
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
command,
|
||||
WorkerRuntimeCommand::for_executable("/opt/yoi/bin/yoi")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_uses_current_exe_when_override_is_empty() {
|
||||
let command = WorkerRuntimeCommand::resolve_from_env_value(Some(OsString::new()), || {
|
||||
Ok(PathBuf::from("/opt/yoi/bin/yoi"))
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
command,
|
||||
WorkerRuntimeCommand::for_executable("/opt/yoi/bin/yoi")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_override_replaces_only_program_and_keeps_worker_prefix() {
|
||||
let command = WorkerRuntimeCommand::resolve_from_env_value(
|
||||
Some(OsString::from("/tmp/rebuilt yoi")),
|
||||
|| panic!("override must not inspect current_exe"),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(command.program(), Path::new("/tmp/rebuilt yoi"));
|
||||
assert_eq!(command.prefix_args(), [OsString::from("worker")]);
|
||||
assert_eq!(
|
||||
command.argv_with(["--worker", "agent"]),
|
||||
vec!["worker", "--worker", "agent"]
|
||||
.into_iter()
|
||||
.map(OsString::from)
|
||||
.collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,181 +0,0 @@
|
||||
//! Socket communication retained for the legacy top-level Worker callback protocol.
|
||||
//!
|
||||
//! Direct Internal SubWorker lifecycle is exposed through `worker.control` and the
|
||||
//! canonical Worker tools, not a second SubWorker-specific tool family.
|
||||
|
||||
use std::path::Path;
|
||||
use std::time::Duration;
|
||||
|
||||
use protocol::stream::{JsonLineReader, JsonLineWriter};
|
||||
use protocol::{Event, Method};
|
||||
use tokio::net::UnixStream;
|
||||
|
||||
/// Timeout applied to each socket-level operation — connect, write,
|
||||
/// read. Kept short so a stuck child doesn't block the spawner's turn.
|
||||
const SOCKET_OP_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
|
||||
/// Connect with a timeout, drain the server's connect-time snapshot,
|
||||
/// write one `Method` line, flush, and close.
|
||||
///
|
||||
/// The Worker socket protocol sends replayed alerts and an initial
|
||||
/// `Event::Snapshot` before it starts reading client methods. Send-only
|
||||
/// callers must consume that prefix; otherwise a large snapshot can block
|
||||
/// the server's writer before it reaches the method-read branch. Any
|
||||
/// socket error maps to an `io::Error`; the caller decides whether to
|
||||
/// surface it to the LLM or treat it as "worker stopped".
|
||||
pub(crate) async fn connect_and_send(socket: &Path, method: &Method) -> std::io::Result<()> {
|
||||
let stream = tokio::time::timeout(SOCKET_OP_TIMEOUT, UnixStream::connect(socket))
|
||||
.await
|
||||
.map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "connect timed out"))??;
|
||||
let (r, w) = stream.into_split();
|
||||
let mut reader = JsonLineReader::new(r);
|
||||
let mut writer = JsonLineWriter::new(w);
|
||||
|
||||
drain_initial_snapshot(&mut reader).await?;
|
||||
|
||||
tokio::time::timeout(SOCKET_OP_TIMEOUT, writer.write(method))
|
||||
.await
|
||||
.map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "write timed out"))??;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn drain_initial_snapshot<R>(reader: &mut JsonLineReader<R>) -> std::io::Result<()>
|
||||
where
|
||||
R: tokio::io::AsyncBufRead + Unpin,
|
||||
{
|
||||
loop {
|
||||
let event = tokio::time::timeout(SOCKET_OP_TIMEOUT, reader.next::<Event>())
|
||||
.await
|
||||
.map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "read timed out"))??;
|
||||
match event {
|
||||
Some(Event::Snapshot { .. }) => return Ok(()),
|
||||
Some(_) => continue,
|
||||
None => {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::UnexpectedEof,
|
||||
"worker closed connection before Snapshot event",
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
use protocol::{Alert, AlertLevel, AlertSource, Greeting, WorkerEvent, WorkerStatus};
|
||||
use tempfile::TempDir;
|
||||
use tokio::net::UnixListener;
|
||||
use tokio::task::JoinHandle;
|
||||
|
||||
fn snapshot(entries: Vec<serde_json::Value>) -> Event {
|
||||
Event::Snapshot {
|
||||
session: protocol::SessionSnapshot {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries: entries
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(index, value)| protocol::SessionSnapshotEntry {
|
||||
use_policy: None,
|
||||
entry_id: format!("test-{index}"),
|
||||
timestamp: index as u64,
|
||||
provenance: protocol::SessionEntryProvenance::LegacyUnknown,
|
||||
derived_from: Vec::new(),
|
||||
data: protocol::SessionSnapshotEntryData::RunError {
|
||||
message: value.to_string(),
|
||||
failure: None,
|
||||
},
|
||||
})
|
||||
.collect(),
|
||||
},
|
||||
greeting: Greeting {
|
||||
worker_name: "server".into(),
|
||||
cwd: "/tmp".into(),
|
||||
provider: "test".into(),
|
||||
model: "test".into(),
|
||||
scope_summary: String::new(),
|
||||
tools: Vec::new(),
|
||||
context_window: 200_000,
|
||||
context_tokens: 0,
|
||||
reasoning: None,
|
||||
context_usage: None,
|
||||
},
|
||||
state: WorkerStatus::Idle.into(),
|
||||
in_flight: Default::default(),
|
||||
internal_workers: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn serve_initial_events_then_method(
|
||||
listener: UnixListener,
|
||||
events: Vec<Event>,
|
||||
) -> JoinHandle<Option<Method>> {
|
||||
tokio::spawn(async move {
|
||||
let (stream, _) = listener.accept().await.ok()?;
|
||||
let (r, w) = stream.into_split();
|
||||
let mut reader = JsonLineReader::new(r);
|
||||
let mut writer = JsonLineWriter::new(w);
|
||||
for event in events {
|
||||
writer.write(&event).await.ok()?;
|
||||
}
|
||||
reader.next::<Method>().await.ok().flatten()
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn connect_and_send_drains_initial_alert_and_snapshot_before_method() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let socket = tmp.path().join("worker.sock");
|
||||
let listener = UnixListener::bind(&socket).unwrap();
|
||||
let received = serve_initial_events_then_method(
|
||||
listener,
|
||||
vec![
|
||||
Event::Alert(Alert {
|
||||
id: None,
|
||||
position: None,
|
||||
level: AlertLevel::Warn,
|
||||
source: AlertSource::Worker,
|
||||
message: "replayed alert".into(),
|
||||
timestamp_ms: 0,
|
||||
}),
|
||||
snapshot(Vec::new()),
|
||||
],
|
||||
);
|
||||
|
||||
let method = Method::Shutdown {
|
||||
command: protocol::WorkerCommandEnvelope::new(1),
|
||||
};
|
||||
connect_and_send(&socket, &method).await.unwrap();
|
||||
|
||||
let method = received.await.unwrap().expect("expected method");
|
||||
assert!(matches!(method, Method::Shutdown { .. }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn connect_and_send_delivers_method_after_large_initial_snapshot() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let socket = tmp.path().join("worker.sock");
|
||||
let listener = UnixListener::bind(&socket).unwrap();
|
||||
let large_payload = "x".repeat(2 * 1024 * 1024);
|
||||
let received = serve_initial_events_then_method(
|
||||
listener,
|
||||
vec![snapshot(vec![
|
||||
serde_json::json!({ "payload": large_payload }),
|
||||
])],
|
||||
);
|
||||
let expected = Method::WorkerEvent(WorkerEvent::TurnEnded {
|
||||
worker_name: "child".into(),
|
||||
});
|
||||
|
||||
connect_and_send(&socket, &expected).await.unwrap();
|
||||
|
||||
let method = received.await.unwrap().expect("expected method");
|
||||
match method {
|
||||
Method::WorkerEvent(WorkerEvent::TurnEnded { worker_name }) => {
|
||||
assert_eq!(worker_name, "child")
|
||||
}
|
||||
other => panic!("expected TurnEnded WorkerEvent, got {other:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,3 @@
|
||||
pub mod comm_tools;
|
||||
mod published;
|
||||
pub mod registry;
|
||||
pub mod tool;
|
||||
|
||||
@@ -340,6 +340,84 @@ pub struct SpawnedWorkerRegistryLoad {
|
||||
pub reclaimed_unreachable: bool,
|
||||
}
|
||||
|
||||
/// Read-only restore preparation using the registry that will be published by the Host.
|
||||
/// The Host must retain its lifecycle exclusion from preparation through application;
|
||||
/// the metadata reread detects source drift, but is not an atomic store transaction.
|
||||
/// Dropping an unapplied plan does not reclaim legacy child state.
|
||||
pub(crate) struct PreparedSpawnedWorkerRegistryLoad<St> {
|
||||
pub(crate) registry: Arc<SpawnedWorkerRegistry>,
|
||||
/// Projected outcome, not evidence that reclamation has already happened.
|
||||
pub(crate) reclaimed_unreachable: bool,
|
||||
runtime_dir: Arc<RuntimeDir>,
|
||||
store: St,
|
||||
worker_name: String,
|
||||
parent_scope: Option<SharedScope>,
|
||||
persisted_children: Vec<WorkerSpawnedChild>,
|
||||
valid_records: Vec<SpawnedWorkerRecord>,
|
||||
legacy_projection_exists: bool,
|
||||
}
|
||||
|
||||
impl<St> PreparedSpawnedWorkerRegistryLoad<St>
|
||||
where
|
||||
St: WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||
{
|
||||
/// Apply only after all other fallible tool initialization, before socket publication.
|
||||
/// Effects retain their original order: projection, canonical reclaim history, then
|
||||
/// allocation/scope. An error may follow committed effects; no rollback is attempted.
|
||||
pub(crate) async fn apply(&mut self) -> io::Result<SpawnedWorkerRegistryLoad> {
|
||||
let metadata = self
|
||||
.store
|
||||
.read_by_name(&self.worker_name)
|
||||
.map_err(store_error_to_io)?;
|
||||
let reclaimed: Vec<_> = self
|
||||
.persisted_children
|
||||
.iter()
|
||||
.map(reclaimed_child_from_metadata)
|
||||
.collect();
|
||||
// A prior apply may have committed the canonical reclaim and then failed
|
||||
// in allocation/scope cleanup. Under the host's same lifecycle lock,
|
||||
// confirm its exact retained facts rather than appending them again.
|
||||
let already_reclaimed = !reclaimed.is_empty()
|
||||
&& metadata.as_ref().is_some_and(|metadata| {
|
||||
metadata.spawned_children.is_empty()
|
||||
&& metadata.reclaimed_children.ends_with(&reclaimed)
|
||||
});
|
||||
if !already_reclaimed
|
||||
&& metadata
|
||||
.as_ref()
|
||||
.map(|metadata| metadata.spawned_children.as_slice())
|
||||
.unwrap_or_default()
|
||||
!= self.persisted_children.as_slice()
|
||||
{
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidData,
|
||||
"spawned-child metadata changed after registry preparation",
|
||||
));
|
||||
}
|
||||
|
||||
// Runtime projection is migration input only; the normal Internal registry is never
|
||||
// materialized into spawned_workers.json. Only existence was captured, not its contents.
|
||||
if !self.persisted_children.is_empty() || self.legacy_projection_exists {
|
||||
self.runtime_dir.write_spawned_workers(&[]).await?;
|
||||
}
|
||||
if !reclaimed.is_empty() && !already_reclaimed {
|
||||
self.store
|
||||
.reclaim_spawned_children(&self.worker_name, reclaimed)
|
||||
.map_err(store_error_to_io)?;
|
||||
}
|
||||
for record in &self.valid_records {
|
||||
reclaim_record(&self.worker_name, self.parent_scope.as_ref(), record)?;
|
||||
}
|
||||
|
||||
let reclaimed_unreachable = self.reclaimed_unreachable;
|
||||
self.reclaimed_unreachable = false;
|
||||
Ok(SpawnedWorkerRegistryLoad {
|
||||
registry: self.registry.clone(),
|
||||
reclaimed_unreachable,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl SpawnedWorkerRegistry {
|
||||
pub(crate) fn bash_output(&self) -> tools::bash_output::BashOutputView {
|
||||
self.bash_output.lock().unwrap().clone()
|
||||
@@ -426,6 +504,23 @@ impl SpawnedWorkerRegistry {
|
||||
worker_name: String,
|
||||
parent_scope: Option<SharedScope>,
|
||||
) -> io::Result<SpawnedWorkerRegistryLoad>
|
||||
where
|
||||
St: WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||
{
|
||||
Self::prepare_from_worker_state_with_reclaim(runtime_dir, store, worker_name, parent_scope)?
|
||||
.apply()
|
||||
.await
|
||||
}
|
||||
|
||||
/// Capture legacy child facts without starting children or changing metadata,
|
||||
/// projection, allocation, or scope. Tool initialization must clone this registry,
|
||||
/// not construct a replacement, and defer reclamation to the owned plan's `apply`.
|
||||
pub(crate) fn prepare_from_worker_state_with_reclaim<St>(
|
||||
runtime_dir: Arc<RuntimeDir>,
|
||||
store: St,
|
||||
worker_name: String,
|
||||
parent_scope: Option<SharedScope>,
|
||||
) -> io::Result<PreparedSpawnedWorkerRegistryLoad<St>>
|
||||
where
|
||||
St: WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||
{
|
||||
@@ -442,54 +537,33 @@ impl SpawnedWorkerRegistry {
|
||||
Ok(record) => {
|
||||
warn!(
|
||||
worker = %record.worker_name,
|
||||
"reclaiming legacy persisted process SubWorker during Internal session restore"
|
||||
"captured legacy persisted process SubWorker for reclamation during Internal session restore"
|
||||
);
|
||||
valid_records.push(record);
|
||||
}
|
||||
Err(error) => warn!(
|
||||
error = %error,
|
||||
worker = %child.worker_name,
|
||||
"clearing corrupt legacy persisted process SubWorker record"
|
||||
"captured corrupt legacy persisted process SubWorker record for clearing"
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
// Runtime projection is migration input only; the normal Internal registry is never
|
||||
// materialized into spawned_workers.json.
|
||||
let legacy_projection_exists = runtime_dir.path().join("spawned_workers.json").exists();
|
||||
if !persisted_children.is_empty() || legacy_projection_exists {
|
||||
runtime_dir.write_spawned_workers(&[]).await?;
|
||||
}
|
||||
if !persisted_children.is_empty() {
|
||||
let reclaimed = persisted_children
|
||||
.iter()
|
||||
.map(reclaimed_child_from_metadata)
|
||||
.collect();
|
||||
store
|
||||
.reclaim_spawned_children(&worker_name, reclaimed)
|
||||
.map_err(store_error_to_io)?;
|
||||
}
|
||||
for record in &valid_records {
|
||||
reclaim_record(&worker_name, parent_scope.as_ref(), record)?;
|
||||
}
|
||||
|
||||
Ok(SpawnedWorkerRegistryLoad {
|
||||
registry: Arc::new(Self {
|
||||
internal_records: std::sync::Mutex::new(Vec::new()),
|
||||
service_records: std::sync::Mutex::new(Vec::new()),
|
||||
#[cfg(test)]
|
||||
fail_service_stops: std::sync::Mutex::new(HashMap::new()),
|
||||
internal_names: std::sync::Mutex::new(HashSet::new()),
|
||||
internal_shutting_down: AtomicBool::new(false),
|
||||
pending_internal_spawns: AtomicUsize::new(0),
|
||||
pending_internal_notify: Notify::new(),
|
||||
internal_spawn_cleanup_failed: AtomicBool::new(false),
|
||||
parent_scope,
|
||||
bash_output: Mutex::new(tools::bash_output::BashOutputView::new()),
|
||||
parent_protocol: Mutex::new(None),
|
||||
protocol_publish_lock: Arc::new(Mutex::new(())),
|
||||
}),
|
||||
let registry = match &parent_scope {
|
||||
Some(scope) => Self::new_internal(worker_name.clone(), scope.clone()),
|
||||
None => Self::new(runtime_dir.clone()),
|
||||
};
|
||||
Ok(PreparedSpawnedWorkerRegistryLoad {
|
||||
registry,
|
||||
reclaimed_unreachable: !persisted_children.is_empty(),
|
||||
runtime_dir,
|
||||
store,
|
||||
worker_name,
|
||||
parent_scope,
|
||||
persisted_children,
|
||||
valid_records,
|
||||
legacy_projection_exists,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1140,6 +1214,10 @@ impl Drop for SpawnedWorkerRegistry {
|
||||
.filter(|record| !record.scope_reclaimed.load(Ordering::Acquire))
|
||||
.flat_map(delegated_write_rules)
|
||||
.collect::<Vec<_>>();
|
||||
// Empty prepared registries own no delegations, including when abandoned or rejected.
|
||||
if write_rules.is_empty() {
|
||||
return;
|
||||
}
|
||||
let _ = parent_scope.update(|current| current.with_removed_deny_rules(write_rules));
|
||||
}
|
||||
}
|
||||
@@ -1165,19 +1243,18 @@ fn reclaim_record(
|
||||
parent_scope: Option<&SharedScope>,
|
||||
record: &SpawnedWorkerRecord,
|
||||
) -> io::Result<()> {
|
||||
if let Ok(path) = worker_allocation::default_allocation_path() {
|
||||
if let Ok(mut guard) = worker_allocation::LockFileGuard::open(&path) {
|
||||
match worker_allocation::reclaim_delegated_scope(
|
||||
&mut guard,
|
||||
parent_name,
|
||||
&record.worker_name,
|
||||
&record.scope_delegated,
|
||||
) {
|
||||
Ok(()) | Err(worker_allocation::ScopeLockError::UnknownWorker(_)) => {}
|
||||
Err(error) => return Err(io::Error::other(error)),
|
||||
}
|
||||
}
|
||||
let path = worker_allocation::default_allocation_path().map_err(io::Error::other)?;
|
||||
let mut guard = worker_allocation::LockFileGuard::open(&path).map_err(io::Error::other)?;
|
||||
match worker_allocation::reclaim_delegated_scope(
|
||||
&mut guard,
|
||||
parent_name,
|
||||
&record.worker_name,
|
||||
&record.scope_delegated,
|
||||
) {
|
||||
Ok(()) | Err(worker_allocation::ScopeLockError::UnknownWorker(_)) => {}
|
||||
Err(error) => return Err(io::Error::other(error)),
|
||||
}
|
||||
drop(guard);
|
||||
if let Some(parent_scope) = parent_scope {
|
||||
let write_rules = record
|
||||
.scope_delegated
|
||||
@@ -1240,10 +1317,18 @@ fn store_error_to_io(error: WorkerStoreError) -> io::Error {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
//! Legacy restore cases connect the registry to canonical aggregate storage,
|
||||
//! runtime projections, and temporary SharedScope rules. Successful apply uses
|
||||
//! a corrupt legacy record to exercise durable clearing without consulting the
|
||||
//! process-global legacy allocation file; valid-record scope reclaim is also
|
||||
//! covered by the existing external restore test.
|
||||
|
||||
use std::path::PathBuf;
|
||||
use std::time::Duration;
|
||||
|
||||
use manifest::{Scope, ScopeConfig};
|
||||
use session_store::LogEntry;
|
||||
use session_store::{LogEntry, WorkerAggregateStore, WorkerMetadata, WorkerSpawnedScopeRule};
|
||||
use tempfile::TempDir;
|
||||
|
||||
use super::*;
|
||||
use crate::internal_worker::{
|
||||
@@ -1251,6 +1336,249 @@ mod tests {
|
||||
test_internal_worker_session_with_greeting,
|
||||
};
|
||||
|
||||
struct LegacyRestoreFixture {
|
||||
runtime_dir: Arc<RuntimeDir>,
|
||||
store: WorkerAggregateStore,
|
||||
parent_scope: SharedScope,
|
||||
scope_root: PathBuf,
|
||||
metadata_path: PathBuf,
|
||||
// Keep all storage and scope paths alive until the other fixture fields are dropped.
|
||||
_root: TempDir,
|
||||
}
|
||||
|
||||
impl LegacyRestoreFixture {
|
||||
async fn new(permission: &str) -> Self {
|
||||
let root = TempDir::new().unwrap();
|
||||
let scope_root = root.path().join("scope");
|
||||
std::fs::create_dir(&scope_root).unwrap();
|
||||
let aggregate = root.path().join("aggregate");
|
||||
let store = WorkerAggregateStore::new(&aggregate, "parent").unwrap();
|
||||
let runtime_dir = Arc::new(
|
||||
RuntimeDir::create(&root.path().join("runtime"), "parent")
|
||||
.await
|
||||
.unwrap(),
|
||||
);
|
||||
let mut metadata =
|
||||
WorkerMetadata::new("parent", None).with_workspace_id("restore-test-workspace");
|
||||
metadata.spawned_children.push(WorkerSpawnedChild {
|
||||
worker_name: "legacy-child".into(),
|
||||
socket_path: root.path().join("legacy.sock"),
|
||||
callback_address: runtime_dir.path().join("sock"),
|
||||
scope_delegated: vec![WorkerSpawnedScopeRule {
|
||||
target: scope_root.clone(),
|
||||
permission: permission.into(),
|
||||
recursive: true,
|
||||
symlink_policy: Default::default(),
|
||||
}],
|
||||
});
|
||||
store.write(&metadata).unwrap();
|
||||
let write_rule = ScopeRule {
|
||||
target: scope_root.clone(),
|
||||
permission: Permission::Write,
|
||||
recursive: true,
|
||||
symlink_policy: Default::default(),
|
||||
};
|
||||
let parent_scope = SharedScope::new(
|
||||
Scope::from_config(&ScopeConfig {
|
||||
allow: vec![write_rule.clone()],
|
||||
deny: vec![write_rule],
|
||||
})
|
||||
.unwrap(),
|
||||
);
|
||||
Self {
|
||||
runtime_dir,
|
||||
store,
|
||||
parent_scope,
|
||||
scope_root,
|
||||
metadata_path: aggregate.join("metadata.json"),
|
||||
_root: root,
|
||||
}
|
||||
}
|
||||
|
||||
fn prepare(&self) -> PreparedSpawnedWorkerRegistryLoad<WorkerAggregateStore> {
|
||||
SpawnedWorkerRegistry::prepare_from_worker_state_with_reclaim(
|
||||
self.runtime_dir.clone(),
|
||||
self.store.clone(),
|
||||
"parent".into(),
|
||||
Some(self.parent_scope.clone()),
|
||||
)
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn projection_path(&self) -> PathBuf {
|
||||
self.runtime_dir.path().join("spawned_workers.json")
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn registry_preparation_and_abandonment_leave_legacy_state_unchanged() {
|
||||
for projection_exists in [false, true] {
|
||||
let fixture = LegacyRestoreFixture::new("write").await;
|
||||
if projection_exists {
|
||||
// Projection contents are deliberately not decoded during preparation.
|
||||
std::fs::write(fixture.projection_path(), b"obsolete projection").unwrap();
|
||||
}
|
||||
let metadata_before = std::fs::read(&fixture.metadata_path).unwrap();
|
||||
let projection_before = std::fs::read(fixture.projection_path()).ok();
|
||||
let scope_before = fixture.parent_scope.snapshot();
|
||||
assert!(!scope_before.is_writable(&fixture.scope_root));
|
||||
|
||||
let prepared = fixture.prepare();
|
||||
|
||||
assert!(prepared.reclaimed_unreachable);
|
||||
assert!(prepared.registry.list_internal().is_empty());
|
||||
assert!(prepared.registry.internal_worker_snapshots().is_empty());
|
||||
assert_eq!(
|
||||
std::fs::read(&fixture.metadata_path).unwrap(),
|
||||
metadata_before
|
||||
);
|
||||
assert_eq!(
|
||||
std::fs::read(fixture.projection_path()).ok(),
|
||||
projection_before
|
||||
);
|
||||
assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot()));
|
||||
assert!(!fixture.runtime_dir.path().join("sock").exists());
|
||||
drop(prepared);
|
||||
assert_eq!(
|
||||
std::fs::read(&fixture.metadata_path).unwrap(),
|
||||
metadata_before
|
||||
);
|
||||
assert_eq!(
|
||||
std::fs::read(fixture.projection_path()).ok(),
|
||||
projection_before
|
||||
);
|
||||
assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot()));
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn prepared_apply_clears_canonical_children_and_projection_on_same_registry() {
|
||||
let fixture = LegacyRestoreFixture::new("unsupported-legacy-permission").await;
|
||||
std::fs::write(fixture.projection_path(), b"obsolete projection").unwrap();
|
||||
let scope_before = fixture.parent_scope.snapshot();
|
||||
let mut prepared = fixture.prepare();
|
||||
let shared_registry = prepared.registry.clone();
|
||||
// Unrelated metadata may advance while tools are initialized.
|
||||
fixture.store.add_peer("parent", "new-peer").unwrap();
|
||||
let mut expected = fixture.store.read_by_name("parent").unwrap().unwrap();
|
||||
expected.reclaimed_children.extend(
|
||||
expected
|
||||
.spawned_children
|
||||
.iter()
|
||||
.map(reclaimed_child_from_metadata),
|
||||
);
|
||||
expected.spawned_children.clear();
|
||||
|
||||
let loaded = prepared.apply().await.unwrap();
|
||||
|
||||
assert!(loaded.reclaimed_unreachable);
|
||||
assert!(Arc::ptr_eq(&shared_registry, &loaded.registry));
|
||||
assert!(shared_registry.list_internal().is_empty());
|
||||
assert_eq!(
|
||||
fixture.store.read_by_name("parent").unwrap(),
|
||||
Some(expected)
|
||||
);
|
||||
let projection: serde_json::Value =
|
||||
serde_json::from_slice(&std::fs::read(fixture.projection_path()).unwrap()).unwrap();
|
||||
assert_eq!(projection, serde_json::json!([]));
|
||||
// Corrupt records are audited and cleared, not interpreted as scope grants.
|
||||
assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot()));
|
||||
assert!(
|
||||
!fixture
|
||||
.parent_scope
|
||||
.snapshot()
|
||||
.is_writable(&fixture.scope_root)
|
||||
);
|
||||
assert!(!fixture.runtime_dir.path().join("sock").exists());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn prepared_apply_rejects_child_source_drift_before_any_effects() {
|
||||
for drift in ["scope", "socket", "removal", "metadata-deletion"] {
|
||||
let fixture = LegacyRestoreFixture::new("write").await;
|
||||
std::fs::write(fixture.projection_path(), b"obsolete projection").unwrap();
|
||||
let scope_before = fixture.parent_scope.snapshot();
|
||||
let mut prepared = fixture.prepare();
|
||||
if drift == "metadata-deletion" {
|
||||
fixture.store.delete_by_name("parent").unwrap();
|
||||
} else {
|
||||
fixture
|
||||
.store
|
||||
.update_by_name("parent", |metadata| match drift {
|
||||
"scope" => {
|
||||
metadata.spawned_children[0].scope_delegated[0].recursive = false
|
||||
}
|
||||
"socket" => {
|
||||
metadata.spawned_children[0].socket_path =
|
||||
fixture.runtime_dir.path().join("replacement.sock");
|
||||
}
|
||||
"removal" => metadata.spawned_children.clear(),
|
||||
_ => unreachable!(),
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
let metadata_before = std::fs::read(&fixture.metadata_path).ok();
|
||||
let projection_before = std::fs::read(fixture.projection_path()).unwrap();
|
||||
|
||||
let error = prepared
|
||||
.apply()
|
||||
.await
|
||||
.err()
|
||||
.expect("source drift must fail");
|
||||
|
||||
assert_eq!(error.kind(), io::ErrorKind::InvalidData);
|
||||
assert_eq!(std::fs::read(&fixture.metadata_path).ok(), metadata_before);
|
||||
assert_eq!(
|
||||
std::fs::read(fixture.projection_path()).unwrap(),
|
||||
projection_before
|
||||
);
|
||||
assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot()));
|
||||
assert!(!fixture.runtime_dir.path().join("sock").exists());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn prepared_apply_clears_projection_only_without_reporting_child_reclaim() {
|
||||
let fixture = LegacyRestoreFixture::new("write").await;
|
||||
fixture
|
||||
.store
|
||||
.set_spawned_children("parent", Vec::new())
|
||||
.unwrap();
|
||||
std::fs::write(fixture.projection_path(), b"obsolete projection").unwrap();
|
||||
let metadata_before = std::fs::read(&fixture.metadata_path).unwrap();
|
||||
let mut prepared = fixture.prepare();
|
||||
assert!(!prepared.reclaimed_unreachable);
|
||||
|
||||
let loaded = prepared.apply().await.unwrap();
|
||||
|
||||
assert!(!loaded.reclaimed_unreachable);
|
||||
assert_eq!(
|
||||
std::fs::read(&fixture.metadata_path).unwrap(),
|
||||
metadata_before
|
||||
);
|
||||
let projection: serde_json::Value =
|
||||
serde_json::from_slice(&std::fs::read(fixture.projection_path()).unwrap()).unwrap();
|
||||
assert_eq!(projection, serde_json::json!([]));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn prepared_apply_projection_failure_keeps_canonical_children_and_scope() {
|
||||
let fixture = LegacyRestoreFixture::new("write").await;
|
||||
std::fs::create_dir(fixture.projection_path()).unwrap();
|
||||
let metadata_before = std::fs::read(&fixture.metadata_path).unwrap();
|
||||
let scope_before = fixture.parent_scope.snapshot();
|
||||
let mut prepared = fixture.prepare();
|
||||
|
||||
assert!(prepared.apply().await.is_err());
|
||||
|
||||
assert_eq!(
|
||||
std::fs::read(&fixture.metadata_path).unwrap(),
|
||||
metadata_before
|
||||
);
|
||||
assert!(fixture.projection_path().is_dir());
|
||||
assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot()));
|
||||
}
|
||||
|
||||
fn registry() -> Arc<SpawnedWorkerRegistry> {
|
||||
let scope = Scope::from_config(&ScopeConfig {
|
||||
allow: vec![ScopeRule {
|
||||
|
||||
@@ -1108,6 +1108,8 @@ fn sub_worker_spawn_tool_impl(
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
mod environment;
|
||||
|
||||
use super::*;
|
||||
use manifest::{DelegationScope, Permission, Scope, SharedScope};
|
||||
use std::pin::Pin;
|
||||
|
||||
@@ -0,0 +1,629 @@
|
||||
//! Connection tests for model-visible environments and actual routed file operations.
|
||||
//! Runs the production SubWorker spawn/broker/Worker/prompt/tool path with temporary
|
||||
//! LocalWorkdirSession providers; only the external LLM is scripted. Selected by
|
||||
//! the normal worker lib suite, without network credentials or live Workspace data.
|
||||
|
||||
use super::*;
|
||||
use agen::Item;
|
||||
use serde_json::{Value, json};
|
||||
use workdir::{WorkdirSessionCapability, WorkdirSessionRouter};
|
||||
|
||||
const TASK: &str = "Read marker.txt from each attached Workdir.";
|
||||
|
||||
fn prompt_environments(prompt: &str) -> Vec<Value> {
|
||||
let section = prompt
|
||||
.split_once("## Workdir execution environment\n")
|
||||
.expect("Worker must supply the execution environment independently of task text")
|
||||
.1;
|
||||
let json = section.split_once("```json\n").unwrap().1;
|
||||
serde_json::from_str(json.split_once("\n```").unwrap().0).unwrap()
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct EnvironmentClient {
|
||||
requests: Arc<Mutex<Vec<Request>>>,
|
||||
read_issued: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl LlmClient for EnvironmentClient {
|
||||
fn clone_boxed(&self) -> Box<dyn LlmClient> {
|
||||
Box::new(self.clone())
|
||||
}
|
||||
|
||||
async fn stream(
|
||||
&self,
|
||||
request: Request,
|
||||
) -> Result<Pin<Box<dyn Stream<Item = Result<LlmEvent, ClientError>> + Send>>, ClientError>
|
||||
{
|
||||
let environments = prompt_environments(request.system_prompt.as_deref().unwrap());
|
||||
self.requests.lock().unwrap().push(request);
|
||||
let mut events = Vec::new();
|
||||
if !self.read_issued.swap(true, Ordering::SeqCst) {
|
||||
// Use only the model-visible aliases and the same relative path in
|
||||
// every session. No test-side cwd prefix is passed to the Read tool.
|
||||
for (index, environment) in environments.iter().enumerate() {
|
||||
let alias = environment["alias"].as_str().unwrap();
|
||||
events.extend([
|
||||
LlmEvent::tool_use_start(index, format!("read-{alias}"), "Read"),
|
||||
LlmEvent::tool_input_delta(
|
||||
index,
|
||||
json!({"target_workdir": alias, "file_path": "marker.txt"}).to_string(),
|
||||
),
|
||||
LlmEvent::tool_use_stop(index),
|
||||
]);
|
||||
}
|
||||
} else {
|
||||
events.extend([
|
||||
LlmEvent::text_block_start(0),
|
||||
LlmEvent::text_delta(0, "read complete"),
|
||||
LlmEvent::text_block_stop(0, None),
|
||||
]);
|
||||
}
|
||||
events.push(LlmEvent::Status(StatusEvent {
|
||||
status: ResponseStatus::Completed,
|
||||
}));
|
||||
Ok(Box::pin(futures::stream::iter(events.into_iter().map(Ok))))
|
||||
}
|
||||
}
|
||||
|
||||
struct EnvironmentFixture {
|
||||
runtime: TempDir,
|
||||
root: PathBuf,
|
||||
manifest: WorkerManifest,
|
||||
profiles: AvailableProfiles,
|
||||
registry: Arc<SpawnedWorkerRegistry>,
|
||||
sessions: Arc<WorkdirSessionRouter>,
|
||||
}
|
||||
|
||||
impl EnvironmentFixture {
|
||||
fn new(aliases: &[(&str, &str)]) -> Self {
|
||||
let runtime = TempDir::new().unwrap();
|
||||
let root = runtime.path().join("profile-root");
|
||||
// Profile discovery stays at this root, deliberately outside every
|
||||
// attached Workdir. Session cwd must never redirect it into work/nested.
|
||||
let profiles = write_project_profile_registry(&root, None, &[]);
|
||||
let mut manifest = parent_manifest(&root, None);
|
||||
manifest.feature.sub_worker.enabled = true;
|
||||
let registry = SpawnedWorkerRegistry::new_internal(
|
||||
"parent".into(),
|
||||
SharedScope::new(Scope::from_config(&manifest.scope).unwrap()),
|
||||
);
|
||||
let sessions = Arc::new(WorkdirSessionRouter::new());
|
||||
for (alias, cwd) in aliases {
|
||||
let checkout = runtime.path().join(alias);
|
||||
for path in ["", "work", "work/nested", "work/nested/deep", "other"] {
|
||||
std::fs::create_dir_all(checkout.join(path)).unwrap();
|
||||
std::fs::write(checkout.join(path).join("marker.txt"), marker(alias, path))
|
||||
.unwrap();
|
||||
}
|
||||
let scope = SharedScope::new(Scope::writable(&checkout).unwrap());
|
||||
let session = Arc::new(workdir::LocalWorkdirSession::materialized_bound(
|
||||
workdir::Workdir::new(format!("fixture-{alias}")),
|
||||
checkout.clone(),
|
||||
checkout.join(cwd),
|
||||
scope,
|
||||
workdir::WorkdirSessionCapabilities::ALL,
|
||||
));
|
||||
sessions
|
||||
.attach_at_cwd(
|
||||
workdir::WorkdirAttachmentAlias::new(*alias).unwrap(),
|
||||
session,
|
||||
workdir::WorkdirPath::new(cwd).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
Self {
|
||||
runtime,
|
||||
root,
|
||||
manifest,
|
||||
profiles,
|
||||
registry,
|
||||
sessions,
|
||||
}
|
||||
}
|
||||
|
||||
async fn spawn(
|
||||
&self,
|
||||
spawner: &str,
|
||||
registry: Arc<SpawnedWorkerRegistry>,
|
||||
broker: workdir::WorkdirToolBrokerRouter,
|
||||
input: Value,
|
||||
) -> (
|
||||
crate::spawn::registry::InternalSpawnedWorkerRecord,
|
||||
EnvironmentClient,
|
||||
) {
|
||||
assert_eq!(input["task"], TASK);
|
||||
let client = EnvironmentClient::default();
|
||||
let mut manifest = self.manifest.clone();
|
||||
manifest.worker.name = spawner.into();
|
||||
let tool = SubWorkerSpawnTool::new(
|
||||
spawner.into(),
|
||||
crate::worker::WorkerWorkspaceContext::local_filesystem(None),
|
||||
ParentNotificationTarget::Durable(Arc::new(|_| {})),
|
||||
self.runtime.path().to_path_buf(),
|
||||
self.root.clone(),
|
||||
Some(broker),
|
||||
registry.clone(),
|
||||
manifest,
|
||||
PromptCatalogSource::builtins_only(),
|
||||
self.profiles.clone(),
|
||||
)
|
||||
.with_internal_client(Box::new(client.clone()));
|
||||
tool.execute(
|
||||
&input.to_string(),
|
||||
agen::tool::ToolExecutionContext::direct(),
|
||||
)
|
||||
.await
|
||||
.expect("spawn through production delegation and Worker construction");
|
||||
let record = registry
|
||||
.get_internal(input["name"].as_str().unwrap())
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
tokio::time::timeout(Duration::from_secs(5), record.session.wait_until_idle())
|
||||
.await
|
||||
.expect("scripted read turn must terminate"),
|
||||
InternalWorkerSessionStatus::Idle,
|
||||
);
|
||||
(record, client)
|
||||
}
|
||||
}
|
||||
|
||||
fn marker(alias: &str, cwd: &str) -> String {
|
||||
format!(
|
||||
"{alias}:{}:session-marker",
|
||||
if cwd.is_empty() { "." } else { cwd }
|
||||
)
|
||||
}
|
||||
|
||||
fn assert_environment_and_reads(
|
||||
router: &WorkdirSessionRouter,
|
||||
client: &EnvironmentClient,
|
||||
expected: &[(&str, &str, bool)],
|
||||
) -> String {
|
||||
let requests = client.requests.lock().unwrap();
|
||||
assert!(
|
||||
requests.len() >= 2,
|
||||
"Read must execute and return to the model"
|
||||
);
|
||||
let prompt = requests[0].system_prompt.as_deref().unwrap();
|
||||
assert!(
|
||||
!prompt.contains("## Working boundaries"),
|
||||
"legacy Scope must not override the connected session environment"
|
||||
);
|
||||
let environments = prompt_environments(prompt);
|
||||
assert_eq!(environments.len(), expected.len());
|
||||
assert_eq!(router.len(), expected.len());
|
||||
for (alias, cwd, command) in expected {
|
||||
let environment = router.resolve(Some(alias)).unwrap().session.environment();
|
||||
assert_eq!(environment.cwd.as_str(), *cwd);
|
||||
assert_eq!(
|
||||
environment
|
||||
.capabilities
|
||||
.supports(WorkdirSessionCapability::Command),
|
||||
*command
|
||||
);
|
||||
let projected = environments
|
||||
.iter()
|
||||
.find(|env| env["alias"] == *alias)
|
||||
.unwrap();
|
||||
assert_eq!(projected["cwd"], if cwd.is_empty() { "." } else { *cwd });
|
||||
assert_eq!(
|
||||
projected["scope"],
|
||||
serde_json::to_value(&environment.scope).unwrap()
|
||||
);
|
||||
assert_eq!(
|
||||
projected["restrictions"],
|
||||
serde_json::to_value(&environment.restrictions).unwrap()
|
||||
);
|
||||
assert_eq!(projected["command"], *command);
|
||||
for (capability, name) in [
|
||||
(WorkdirSessionCapability::Read, "read"),
|
||||
(WorkdirSessionCapability::Write, "write"),
|
||||
(WorkdirSessionCapability::Edit, "edit"),
|
||||
(WorkdirSessionCapability::Glob, "glob"),
|
||||
(WorkdirSessionCapability::Grep, "grep"),
|
||||
(WorkdirSessionCapability::Command, "command"),
|
||||
] {
|
||||
assert_eq!(
|
||||
projected["capabilities"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|value| value == name),
|
||||
environment.capabilities.supports(capability),
|
||||
);
|
||||
}
|
||||
let expected_marker = marker(alias, cwd);
|
||||
let observed_read = requests
|
||||
.iter()
|
||||
.flat_map(|request| &request.items)
|
||||
.any(|item| {
|
||||
matches!(item, Item::ToolResult { call_id, content: Some(content), .. }
|
||||
if call_id == &format!("read-{alias}") && content.contains(&expected_marker))
|
||||
});
|
||||
assert!(
|
||||
observed_read,
|
||||
"{alias}: relative Read did not return {expected_marker}"
|
||||
);
|
||||
}
|
||||
prompt.to_owned()
|
||||
}
|
||||
|
||||
fn assert_retained_environment(
|
||||
record: &crate::spawn::registry::InternalSpawnedWorkerRecord,
|
||||
prompt: &str,
|
||||
) {
|
||||
let retained = record.session.entries().iter().any(|entry| {
|
||||
matches!(entry, session_store::LogEntry::AnnotatedSegmentStart { system_prompt: Some(saved), .. }
|
||||
if saved == prompt)
|
||||
});
|
||||
assert!(
|
||||
retained,
|
||||
"the model-visible environment must also be retained in session history"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn spawn_root_and_nonroot_cwd_are_projected_without_task_annotations() {
|
||||
for (name, cwd, command, permission) in [
|
||||
("root-child", "", false, "read"),
|
||||
("cwd-child", "work", true, "write"),
|
||||
] {
|
||||
let fixture = EnvironmentFixture::new(&[("checkout", "")]);
|
||||
let (record, client) = fixture
|
||||
.spawn(
|
||||
"parent",
|
||||
fixture.registry.clone(),
|
||||
workdir::WorkdirToolBrokerRouter::new(fixture.sessions.clone()),
|
||||
json!({
|
||||
"name": name, "task": TASK, "cwd": if cwd.is_empty() { "." } else { cwd },
|
||||
"scope": [{"target": ".", "permission": permission}],
|
||||
"command": command,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
let router = record.workdir_tool_scope.session_router();
|
||||
let prompt = assert_environment_and_reads(&router, &client, &[("checkout", cwd, command)]);
|
||||
assert_retained_environment(&record, &prompt);
|
||||
assert_eq!(
|
||||
record.installed_tools.iter().any(|tool| tool == "Bash"),
|
||||
command
|
||||
);
|
||||
fixture.registry.remove_internal(name).await.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn spawn_multiple_aliases_and_nested_child_keep_real_session_environment() {
|
||||
let fixture = EnvironmentFixture::new(&[("alpha", ""), ("beta", "")]);
|
||||
let (child, child_client) = fixture
|
||||
.spawn(
|
||||
"parent",
|
||||
fixture.registry.clone(),
|
||||
workdir::WorkdirToolBrokerRouter::new(fixture.sessions.clone()),
|
||||
json!({
|
||||
"name": "child", "task": TASK, "cwd": "work", "command": true,
|
||||
"scope": [
|
||||
{"target_workdir": "alpha", "target": ".", "permission": "write"},
|
||||
{"target_workdir": "beta", "target": ".", "permission": "write"},
|
||||
],
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
let child_router = child.workdir_tool_scope.session_router();
|
||||
let child_prompt = assert_environment_and_reads(
|
||||
&child_router,
|
||||
&child_client,
|
||||
&[("alpha", "work", true), ("beta", "work", true)],
|
||||
);
|
||||
assert_retained_environment(&child, &child_prompt);
|
||||
assert!(
|
||||
child
|
||||
.installed_tools
|
||||
.iter()
|
||||
.any(|tool| tool == "SubWorkerSpawn")
|
||||
);
|
||||
|
||||
// Reuse the actual child's registry and attenuated broker. The nested tool
|
||||
// receives the scripted LLM at the same boundary as the first spawn; session
|
||||
// construction, routing, delegation and prompt materialization remain real.
|
||||
let (nested, nested_client) = fixture
|
||||
.spawn(
|
||||
"child",
|
||||
child.child_registry.clone(),
|
||||
child.workdir_tool_scope.broker_router(),
|
||||
json!({
|
||||
"name": "nested", "task": TASK, "cwd": "work/nested",
|
||||
"scope": [
|
||||
{"target_workdir": "alpha", "target": "work/nested", "permission": "read"},
|
||||
{"target_workdir": "beta", "target": "work/nested", "permission": "read"},
|
||||
],
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
let nested_router = nested.workdir_tool_scope.session_router();
|
||||
let nested_prompt = assert_environment_and_reads(
|
||||
&nested_router,
|
||||
&nested_client,
|
||||
&[
|
||||
("alpha", "work/nested", false),
|
||||
("beta", "work/nested", false),
|
||||
],
|
||||
);
|
||||
assert_retained_environment(&nested, &nested_prompt);
|
||||
assert!(!nested.installed_tools.iter().any(|tool| tool == "Bash"));
|
||||
for alias in ["alpha", "beta"] {
|
||||
let session = nested_router.resolve(Some(alias)).unwrap().session;
|
||||
let environment = session.environment();
|
||||
let scope = environment
|
||||
.scope
|
||||
.expect("nested session must remain scoped");
|
||||
assert_eq!(scope.len(), 1);
|
||||
assert_eq!(scope[0].target.as_str(), "work/nested");
|
||||
assert_eq!(
|
||||
scope[0].permission,
|
||||
workdir::WorkdirToolScopePermission::Read
|
||||
);
|
||||
assert!(
|
||||
session
|
||||
.read(workdir::ReadRequest {
|
||||
path: workdir::WorkdirPath::new_scoped(
|
||||
fixture
|
||||
.runtime
|
||||
.path()
|
||||
.join(alias)
|
||||
.join("marker.txt")
|
||||
.display()
|
||||
.to_string(),
|
||||
)
|
||||
.unwrap(),
|
||||
offset: 0,
|
||||
limit: 10,
|
||||
max_bytes: 1024,
|
||||
})
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
// Grandchild cwd and reduced command authority must not rewrite its parent.
|
||||
assert_environment_and_reads(
|
||||
&child_router,
|
||||
&child_client,
|
||||
&[("alpha", "work", true), ("beta", "work", true)],
|
||||
);
|
||||
child
|
||||
.child_registry
|
||||
.remove_internal("nested")
|
||||
.await
|
||||
.unwrap();
|
||||
fixture.registry.remove_internal("child").await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn spawn_multiple_aliases_keep_root_scope_independent_of_parent_session_cwds() {
|
||||
let fixture = EnvironmentFixture::new(&[("alpha", "work"), ("beta", "other")]);
|
||||
let (record, client) = fixture
|
||||
.spawn(
|
||||
"parent",
|
||||
fixture.registry.clone(),
|
||||
workdir::WorkdirToolBrokerRouter::new(fixture.sessions.clone()),
|
||||
json!({
|
||||
"name": "child", "task": TASK, "cwd": "work/nested",
|
||||
"scope": [
|
||||
{"target_workdir": "alpha", "target": "work/nested", "permission": "read"},
|
||||
{"target_workdir": "beta", "target": "work/nested", "permission": "read"},
|
||||
],
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
let prompt = assert_environment_and_reads(
|
||||
&record.workdir_tool_scope.session_router(),
|
||||
&client,
|
||||
&[
|
||||
("alpha", "work/nested", false),
|
||||
("beta", "work/nested", false),
|
||||
],
|
||||
);
|
||||
assert_retained_environment(&record, &prompt);
|
||||
for (alias, cwd) in [("alpha", "work"), ("beta", "other")] {
|
||||
assert_eq!(
|
||||
fixture
|
||||
.sessions
|
||||
.resolve(Some(alias))
|
||||
.unwrap()
|
||||
.session
|
||||
.environment()
|
||||
.cwd
|
||||
.as_str(),
|
||||
cwd
|
||||
);
|
||||
}
|
||||
fixture.registry.remove_internal("child").await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn nonroot_startup_then_spawn_and_nested_child_keep_root_relative_scope() {
|
||||
let fixture = EnvironmentFixture::new(&[("workdir", "")]);
|
||||
let checkout = fixture.runtime.path().join("workdir");
|
||||
let client = EnvironmentClient::default();
|
||||
let mut manifest = fixture.manifest.clone();
|
||||
manifest.scope.allow = vec![abs_rule(&checkout, Permission::Write)];
|
||||
let mut worker =
|
||||
Worker::<Box<dyn LlmClient>, EphemeralSessionStore>::from_internal_manifest_with_context(
|
||||
manifest,
|
||||
EphemeralSessionStore::default(),
|
||||
PromptCatalogSource::builtins_only(),
|
||||
crate::worker::WorkerWorkspaceContext::local_filesystem(None),
|
||||
WorkerFilesystemAuthority::local(checkout.clone(), checkout.join("work")),
|
||||
Some(Box::new(client.clone())),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
// Do not bind a test router: exercise workdir_sessions_from_authority via
|
||||
// the real constructor and the normal controller's broker selection.
|
||||
let startup = worker.workdir_sessions();
|
||||
register_worker_tools(
|
||||
&mut worker,
|
||||
fixture.runtime.path().to_path_buf(),
|
||||
fixture.registry.clone(),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(5), worker.run_text(TASK))
|
||||
.await
|
||||
.expect("startup read turn must terminate")
|
||||
.unwrap();
|
||||
assert_environment_and_reads(&startup, &client, &[("workdir", "work", true)]);
|
||||
|
||||
let (child, child_client) = fixture
|
||||
.spawn(
|
||||
"parent",
|
||||
fixture.registry.clone(),
|
||||
workdir::WorkdirToolBrokerRouter::new(startup.clone()),
|
||||
json!({
|
||||
"name": "child", "task": TASK, "cwd": "work/nested",
|
||||
"scope": [{"target": "work/nested", "permission": "read"}],
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
let child_router = child.workdir_tool_scope.session_router();
|
||||
let child_prompt = assert_environment_and_reads(
|
||||
&child_router,
|
||||
&child_client,
|
||||
&[("workdir", "work/nested", false)],
|
||||
);
|
||||
assert_retained_environment(&child, &child_prompt);
|
||||
let (nested, nested_client) = fixture
|
||||
.spawn(
|
||||
"child",
|
||||
child.child_registry.clone(),
|
||||
child.workdir_tool_scope.broker_router(),
|
||||
json!({
|
||||
"name": "nested", "task": TASK, "cwd": "work/nested/deep",
|
||||
"scope": [{"target": "work/nested/deep", "permission": "read"}],
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
let nested_router = nested.workdir_tool_scope.session_router();
|
||||
let nested_prompt = assert_environment_and_reads(
|
||||
&nested_router,
|
||||
&nested_client,
|
||||
&[("workdir", "work/nested/deep", false)],
|
||||
);
|
||||
assert_retained_environment(&nested, &nested_prompt);
|
||||
for (router, target) in [
|
||||
(&child_router, "work/nested"),
|
||||
(&nested_router, "work/nested/deep"),
|
||||
] {
|
||||
let environment = router.resolve(None).unwrap().session.environment();
|
||||
let rules = environment.scope.unwrap();
|
||||
assert_eq!(rules.len(), 1);
|
||||
assert_eq!(rules[0].target.as_str(), target);
|
||||
}
|
||||
assert_eq!(
|
||||
startup
|
||||
.resolve(None)
|
||||
.unwrap()
|
||||
.session
|
||||
.environment()
|
||||
.cwd
|
||||
.as_str(),
|
||||
"work"
|
||||
);
|
||||
let alias = workdir::WorkdirAttachmentAlias::new("workdir").unwrap();
|
||||
startup.begin_detach(&alias).await.unwrap();
|
||||
// Both the parent tool view and descendant root-coordinate broker source
|
||||
// must share this fence, rather than bypassing it through a cached broker.
|
||||
for router in [&startup, &child_router, &nested_router] {
|
||||
let session = router.resolve(None).unwrap().session;
|
||||
assert!(
|
||||
session
|
||||
.read(workdir::ReadRequest {
|
||||
path: workdir::WorkdirPath::new("marker.txt").unwrap(),
|
||||
offset: 0,
|
||||
limit: 10,
|
||||
max_bytes: 1024,
|
||||
})
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
startup.cancel_detach(&alias);
|
||||
child
|
||||
.child_registry
|
||||
.remove_internal("nested")
|
||||
.await
|
||||
.unwrap();
|
||||
fixture.registry.remove_internal("child").await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn startup_rejects_cwd_outside_workdir_root_without_running_the_model() {
|
||||
let fixture = EnvironmentFixture::new(&[("workdir", "")]);
|
||||
let client = EnvironmentClient::default();
|
||||
let mut manifest = fixture.manifest.clone();
|
||||
manifest.scope.allow = vec![abs_rule(fixture.runtime.path(), Permission::Write)];
|
||||
let error =
|
||||
Worker::<Box<dyn LlmClient>, EphemeralSessionStore>::from_internal_manifest_with_context(
|
||||
manifest,
|
||||
EphemeralSessionStore::default(),
|
||||
PromptCatalogSource::builtins_only(),
|
||||
crate::worker::WorkerWorkspaceContext::local_filesystem(None),
|
||||
WorkerFilesystemAuthority::local(
|
||||
fixture.runtime.path().join("workdir"),
|
||||
fixture.root.clone(),
|
||||
),
|
||||
Some(Box::new(client.clone())),
|
||||
)
|
||||
.await
|
||||
.err()
|
||||
.expect("a session cwd must be representable within its Workdir root");
|
||||
assert!(matches!(error, crate::WorkerError::InvalidCwd { .. }));
|
||||
assert!(client.requests.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn worker_multiple_aliases_project_distinct_session_cwds_not_discovery_root() {
|
||||
for prompt_override in [None, Some("Feature-owned instruction body")] {
|
||||
let fixture = EnvironmentFixture::new(&[("alpha", "work"), ("beta", "other")]);
|
||||
let client = EnvironmentClient::default();
|
||||
let mut worker = Worker::<Box<dyn LlmClient>, EphemeralSessionStore>::from_internal_manifest_with_context(
|
||||
fixture.manifest.clone(),
|
||||
EphemeralSessionStore::default(),
|
||||
PromptCatalogSource::builtins_only(),
|
||||
crate::worker::WorkerWorkspaceContext::local_filesystem(None),
|
||||
WorkerFilesystemAuthority::None,
|
||||
Some(Box::new(client.clone())),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
worker.bind_workdir_sessions(fixture.sessions.clone());
|
||||
register_worker_tools(
|
||||
&mut worker,
|
||||
fixture.runtime.path().to_path_buf(),
|
||||
fixture.registry.clone(),
|
||||
None,
|
||||
Some(workdir::WorkdirToolBrokerRouter::new(
|
||||
fixture.sessions.clone(),
|
||||
)),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
worker.install_system_prompt_contribution(None, prompt_override.map(str::to_owned));
|
||||
tokio::time::timeout(Duration::from_secs(5), worker.run_text(TASK))
|
||||
.await
|
||||
.expect("scripted worker turn must terminate")
|
||||
.unwrap();
|
||||
let prompt = assert_environment_and_reads(
|
||||
&fixture.sessions,
|
||||
&client,
|
||||
&[("alpha", "work", true), ("beta", "other", true)],
|
||||
);
|
||||
if let Some(body) = prompt_override {
|
||||
assert!(prompt.starts_with(body));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -678,8 +678,8 @@ pub(crate) fn test_connection(client: Arc<dyn WorkspaceClient>) -> SubjektivHost
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::{
|
||||
Method, WorkerBootstrap, WorkerBootstrapLayout, WorkerControllerTransport,
|
||||
WorkerFilesystemAuthority, WorkerManifest, WorkerWorkspaceContext,
|
||||
Method, WorkerBootstrap, WorkerBootstrapLayout, WorkerFilesystemAuthority, WorkerManifest,
|
||||
WorkerWorkspaceContext,
|
||||
};
|
||||
use agen::llm_client::event::{Event as LlmEvent, ResponseStatus, StatusEvent};
|
||||
use agen::llm_client::{ClientError, LlmClient, Request};
|
||||
@@ -944,7 +944,6 @@ enabled = true
|
||||
WorkerBootstrapLayout::Direct {
|
||||
runtime_base: temp.path().join("runtime"),
|
||||
},
|
||||
WorkerControllerTransport::InProcess,
|
||||
)
|
||||
.with_model_client(TaskScriptClient::default())
|
||||
.with_subjektiv_host(host)
|
||||
@@ -1137,7 +1136,6 @@ enabled = true
|
||||
WorkerBootstrapLayout::Direct {
|
||||
runtime_base: temp.path().join("runtime"),
|
||||
},
|
||||
WorkerControllerTransport::InProcess,
|
||||
)
|
||||
.with_model_client(ScriptedClient(tx))
|
||||
.with_subjektiv_host(host.clone())
|
||||
@@ -1318,7 +1316,6 @@ enabled = true
|
||||
WorkerBootstrapLayout::Direct {
|
||||
runtime_base: temp.path().join("runtime"),
|
||||
},
|
||||
WorkerControllerTransport::InProcess,
|
||||
)
|
||||
.with_model_client(ScriptedClient(tx))
|
||||
.with_subjektiv_host(host.clone())
|
||||
@@ -1377,7 +1374,6 @@ enabled = true
|
||||
WorkerBootstrapLayout::Direct {
|
||||
runtime_base: temp.path().join("runtime"),
|
||||
},
|
||||
WorkerControllerTransport::InProcess,
|
||||
)
|
||||
.with_model_client(ScriptedClient(tx));
|
||||
let bootstrap = if attach {
|
||||
|
||||
+1918
-240
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,33 @@
|
||||
use std::io::{self, Write};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
/// Per-test warning capture; never installs a global subscriber or writes fixture secrets.
|
||||
#[derive(Clone, Default)]
|
||||
pub struct CapturedWarnings(Arc<Mutex<Vec<u8>>>);
|
||||
|
||||
impl CapturedWarnings {
|
||||
pub fn subscriber(&self) -> impl tracing::Subscriber + Send + Sync + 'static {
|
||||
let output = self.clone();
|
||||
tracing_subscriber::fmt()
|
||||
.with_writer(move || output.clone())
|
||||
.without_time()
|
||||
.with_ansi(false)
|
||||
.with_max_level(tracing::Level::WARN)
|
||||
.finish()
|
||||
}
|
||||
|
||||
pub fn text(&self) -> String {
|
||||
String::from_utf8(self.0.lock().unwrap().clone()).unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for CapturedWarnings {
|
||||
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
||||
self.0.lock().unwrap().extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -26,6 +26,9 @@ use tokio::sync::broadcast;
|
||||
|
||||
use worker::{Worker, WorkerController, WorkerError};
|
||||
|
||||
#[path = "common/diagnostics.rs"]
|
||||
mod diagnostics;
|
||||
|
||||
type TestStore = CombinedStore<FsStore, FsWorkerStore>;
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -2876,7 +2879,7 @@ const COMPACTION_HOST_WORKSPACE_ID: &str = "journal-transport-host-workspace";
|
||||
// Host authorization/selection or live-model semantic continuation quality.
|
||||
struct CompactionJournalHost {
|
||||
granted: bool,
|
||||
unavailable: bool,
|
||||
failure: Option<&'static str>,
|
||||
snapshot: Mutex<subjektiv::journal::startup::JournalStartupSnapshot>,
|
||||
reads: Mutex<Vec<worker::subjektiv::SubjektivHostContext>>,
|
||||
startup_reads: AtomicUsize,
|
||||
@@ -2886,7 +2889,7 @@ impl CompactionJournalHost {
|
||||
fn new(snapshot: subjektiv::journal::startup::JournalStartupSnapshot) -> Self {
|
||||
Self {
|
||||
granted: true,
|
||||
unavailable: false,
|
||||
failure: None,
|
||||
snapshot: Mutex::new(snapshot),
|
||||
reads: Mutex::new(vec![]),
|
||||
startup_reads: AtomicUsize::new(0),
|
||||
@@ -2917,10 +2920,18 @@ impl worker::subjektiv::SubjektivHost for CompactionJournalHost {
|
||||
worker::subjektiv::SubjektivHostError,
|
||||
> {
|
||||
self.reads.lock().unwrap().push(context.clone());
|
||||
if self.unavailable {
|
||||
return Err(worker::subjektiv::SubjektivHostError::Unavailable(
|
||||
"injected Journal read failure".into(),
|
||||
));
|
||||
if let Some(failure) = self.failure {
|
||||
use worker::subjektiv::SubjektivHostError;
|
||||
let secret = "PRIVATE_JOURNAL_BODY /private/host/path proof=PRIVATE_PROOF credential=PRIVATE_CREDENTIAL raw_error=PRIVATE_ERROR";
|
||||
return Err(match failure {
|
||||
"unavailable" => SubjektivHostError::Unavailable(secret.into()),
|
||||
"invalid-argument" => SubjektivHostError::InvalidArgument(secret.into()),
|
||||
"conflict" => SubjektivHostError::Conflict {
|
||||
code: secret.into(),
|
||||
message: secret.into(),
|
||||
},
|
||||
_ => panic!("unknown failure fixture"),
|
||||
});
|
||||
}
|
||||
Ok(self.snapshot.lock().unwrap().clone())
|
||||
}
|
||||
@@ -3466,26 +3477,37 @@ async fn journal_three_compactions_freeze_one_snapshot_preserve_full_summary_tai
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn journal_absent_disabled_unavailable_and_over_byte_budget_keep_conversation_compaction() {
|
||||
for case in [
|
||||
"absent",
|
||||
"disabled",
|
||||
"unavailable",
|
||||
"over-byte-budget",
|
||||
"no-host",
|
||||
] {
|
||||
async fn journal_empty_disabled_and_available_compaction_preserve_inputs_without_failure_warnings()
|
||||
{
|
||||
assert_journal_compaction_cases(&["absent", "available", "disabled", "no-host"]).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn journal_snapshot_failures_keep_safe_diagnostics_and_continue_compaction() {
|
||||
assert_journal_compaction_cases(&["unavailable", "invalid-argument", "conflict"]).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn journal_over_byte_budget_keeps_safe_diagnostic_and_continues_compaction() {
|
||||
assert_journal_compaction_cases(&["over-byte-budget"]).await;
|
||||
}
|
||||
|
||||
async fn assert_journal_compaction_cases(cases: &[&'static str]) {
|
||||
for &case in cases {
|
||||
let mut snapshot = compaction_snapshot(0);
|
||||
if case == "absent" {
|
||||
snapshot.journals.clear();
|
||||
}
|
||||
if case == "over-byte-budget" {
|
||||
snapshot.journals[0].current_position =
|
||||
"x".repeat(subjektiv::journal::startup::STARTUP_MAX_BYTES);
|
||||
"PRIVATE_JOURNAL_BODY".repeat(subjektiv::journal::startup::STARTUP_MAX_BYTES);
|
||||
}
|
||||
let expected_json = (case == "absent").then(|| serde_json::to_string(&snapshot).unwrap());
|
||||
let expected_json = matches!(case, "absent" | "available")
|
||||
.then(|| serde_json::to_string(&snapshot).unwrap());
|
||||
let mut host = CompactionJournalHost::new(snapshot);
|
||||
host.granted = case != "disabled";
|
||||
host.unavailable = case == "unavailable";
|
||||
host.failure =
|
||||
matches!(case, "unavailable" | "invalid-argument" | "conflict").then_some(case);
|
||||
let host = Arc::new(host);
|
||||
let client = MockClient::new(vec![
|
||||
single_text_events("seed"),
|
||||
@@ -3517,7 +3539,51 @@ async fn journal_absent_disabled_unavailable_and_over_byte_budget_keep_conversat
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
fixture.worker.compact(0).await.unwrap();
|
||||
// Scope the subscriber to every poll, not a thread-local guard across await.
|
||||
// This observes the real Feature warning while compacting into FsStore.
|
||||
let logs = diagnostics::CapturedWarnings::default();
|
||||
use tracing::instrument::WithSubscriber;
|
||||
fixture
|
||||
.worker
|
||||
.compact(0)
|
||||
.with_subscriber(logs.subscriber())
|
||||
.await
|
||||
.unwrap();
|
||||
let log = logs.text();
|
||||
let failure = match case {
|
||||
"unavailable" => Some(("snapshot", "host_unavailable")),
|
||||
"invalid-argument" => Some(("snapshot", "host_invalid_argument")),
|
||||
"conflict" => Some(("snapshot", "host_conflict")),
|
||||
"over-byte-budget" => Some(("byte_budget", "startup_max_bytes_exceeded")),
|
||||
_ => None,
|
||||
};
|
||||
if let Some((stage, reason)) = failure {
|
||||
assert!(log.contains(&format!("stage=\"{stage}\"")), "{case}: {log}");
|
||||
assert!(
|
||||
log.contains(&format!("reason=\"{reason}\"")),
|
||||
"{case}: {log}"
|
||||
);
|
||||
assert_eq!(
|
||||
log.matches("continuing without Journal").count(),
|
||||
1,
|
||||
"{case}"
|
||||
);
|
||||
} else {
|
||||
assert!(
|
||||
!log.contains("Journal compaction snapshot unavailable"),
|
||||
"{case}: {log}"
|
||||
);
|
||||
}
|
||||
for secret in [
|
||||
"PRIVATE_",
|
||||
"/private/host/path",
|
||||
"JOURNAL_POSITION_0",
|
||||
"JOURNAL_BODY_0",
|
||||
"state-token-0",
|
||||
"fixed-event-reference-0",
|
||||
] {
|
||||
assert!(!log.contains(secret), "{case}: diagnostic leaked {secret}");
|
||||
}
|
||||
let input = first_compactor_input(&observed.requests()[1]);
|
||||
let body = if matches!(case, "disabled" | "no-host") {
|
||||
assert_eq!(
|
||||
@@ -3532,8 +3598,8 @@ async fn journal_absent_disabled_unavailable_and_over_byte_budget_keep_conversat
|
||||
String::new()
|
||||
} else {
|
||||
let body = journal_body(&fixture.worker);
|
||||
let expected = if case == "absent" {
|
||||
"absent"
|
||||
let expected = if matches!(case, "absent" | "available") {
|
||||
case
|
||||
} else {
|
||||
"unavailable"
|
||||
};
|
||||
@@ -3548,6 +3614,71 @@ async fn journal_absent_disabled_unavailable_and_over_byte_budget_keep_conversat
|
||||
&fixture.worker.prompts().load_full(),
|
||||
Some(COMPACTION_HOST_WORKSPACE_ID),
|
||||
);
|
||||
let seed = replacement_seed(&fixture.worker);
|
||||
let metadata = seed[1].metadata.input_metadata.as_ref().unwrap();
|
||||
assert_eq!(
|
||||
metadata.usage().recording_material,
|
||||
session_store::InputEvidenceUse::Exclude
|
||||
);
|
||||
assert_eq!(
|
||||
metadata.usage().memory_evidence,
|
||||
session_store::InputEvidenceUse::Exclude
|
||||
);
|
||||
assert_eq!(
|
||||
metadata.usage().observation,
|
||||
session_store::InputObservationUse::Hidden
|
||||
);
|
||||
let payload: serde_json::Value = metadata
|
||||
.feature_payload()
|
||||
.unwrap()
|
||||
.decode("journal:compaction", 1)
|
||||
.unwrap();
|
||||
assert_eq!(payload["status"], expected);
|
||||
assert_eq!(
|
||||
payload["prompt_provenance"]["workspace_id"],
|
||||
COMPACTION_HOST_WORKSPACE_ID
|
||||
);
|
||||
let projection = fixture.worker.prompts().load_full().projection().clone();
|
||||
assert_eq!(
|
||||
payload["prompt_provenance"]["source_digest"],
|
||||
projection.source_digest
|
||||
);
|
||||
assert_eq!(
|
||||
payload["prompt_provenance"]["projection_digest"],
|
||||
projection.catalog_digest
|
||||
);
|
||||
if let Some(json) = expected_json.as_deref() {
|
||||
use sha2::{Digest, Sha256};
|
||||
assert_eq!(
|
||||
payload["snapshot"],
|
||||
serde_json::from_str::<serde_json::Value>(json).unwrap()
|
||||
);
|
||||
assert_eq!(
|
||||
payload["snapshot_sha256"],
|
||||
Sha256::digest(json.as_bytes())
|
||||
.iter()
|
||||
.map(|byte| format!("{byte:02x}"))
|
||||
.collect::<String>()
|
||||
);
|
||||
} else {
|
||||
assert!(payload["snapshot"].is_null());
|
||||
assert!(payload["snapshot_sha256"].is_null());
|
||||
}
|
||||
// Feature diagnostics do not become model input or a new metadata contract.
|
||||
let saved = serde_json::to_string(&seed[1]).unwrap();
|
||||
for forbidden in [
|
||||
"PRIVATE_",
|
||||
"/private/host/path",
|
||||
"host_unavailable",
|
||||
"host_conflict",
|
||||
"host_invalid_argument",
|
||||
"startup_max_bytes_exceeded",
|
||||
] {
|
||||
assert!(
|
||||
!saved.contains(forbidden),
|
||||
"{case}: saved input leaked {forbidden}"
|
||||
);
|
||||
}
|
||||
body
|
||||
};
|
||||
assert_full_initial_summary(&input, &inherited);
|
||||
@@ -3559,8 +3690,8 @@ async fn journal_absent_disabled_unavailable_and_over_byte_budget_keep_conversat
|
||||
.is_some_and(|text| text.contains("PREVIOUS_JOURNAL_DELEGATED_CONTEXT"))),
|
||||
"fallback retains only the newly frozen reference, not an old snapshot as latest"
|
||||
);
|
||||
assert!(!body.contains("JOURNAL_POSITION_0"));
|
||||
assert!(!body.contains("journal-reference-0"));
|
||||
assert_eq!(body.contains("JOURNAL_POSITION_0"), case == "available");
|
||||
assert_eq!(body.contains("journal-reference-0"), case == "available");
|
||||
assert_eq!(
|
||||
host.reads.lock().unwrap().len(),
|
||||
usize::from(!matches!(case, "disabled" | "no-host")),
|
||||
|
||||
@@ -1,3 +1,4 @@
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::pin::Pin;
|
||||
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
@@ -150,6 +151,19 @@ impl LlmClient for MockClient {
|
||||
// Helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[path = "../../../tests/support/restore_environment.rs"]
|
||||
mod restore_test_environment;
|
||||
use restore_test_environment::require_restore_test_environment;
|
||||
|
||||
fn owned_relative_runtime_base() -> (tempfile::TempDir, PathBuf) {
|
||||
require_restore_test_environment();
|
||||
let temp = tempfile::tempdir_in(".").unwrap();
|
||||
// The relative reference and the TempDir name the same owned tree, without
|
||||
// putting the checkout prefix into sockaddr_un or changing process cwd.
|
||||
let runtime_base = Path::new(temp.path().file_name().unwrap()).join("runtime");
|
||||
(temp, runtime_base)
|
||||
}
|
||||
|
||||
fn simple_text_events() -> Vec<LlmEvent> {
|
||||
vec![
|
||||
LlmEvent::text_block_start(0),
|
||||
@@ -295,6 +309,7 @@ async fn make_worker_with_manifest_and_workspace_context(
|
||||
manifest: WorkerManifest,
|
||||
workspace_context: WorkerWorkspaceContext,
|
||||
) -> (Worker<MockClient, TestStore>, std::path::PathBuf) {
|
||||
require_restore_test_environment();
|
||||
let store_tmp = tempfile::tempdir().unwrap();
|
||||
let store = CombinedStore::new(
|
||||
FsStore::new(store_tmp.path()).unwrap(),
|
||||
@@ -320,18 +335,21 @@ async fn make_worker_with_manifest_and_workspace_context(
|
||||
}
|
||||
|
||||
async fn spawn_controller(worker: Worker<MockClient, TestStore>) -> WorkerHandle {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let runtime_base = tmp.path().to_owned();
|
||||
std::mem::forget(tmp);
|
||||
let (handle, _shutdown_rx) = WorkerController::spawn(worker, &runtime_base)
|
||||
let (tmp, runtime_base) = owned_relative_runtime_base();
|
||||
let (handle, shutdown_rx) = WorkerController::spawn(worker, &runtime_base)
|
||||
.await
|
||||
.unwrap();
|
||||
// Retain ownership until controller shutdown (or test-runtime teardown).
|
||||
tokio::spawn(async move {
|
||||
let _ = shutdown_rx.await;
|
||||
drop(tmp);
|
||||
});
|
||||
handle
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn canonical_worker_handle_uploads_and_submits_attachment() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let (root, runtime_base) = owned_relative_runtime_base();
|
||||
let manifest = WorkerManifest::from_toml(MANIFEST_TOML).unwrap();
|
||||
let session = session_store::WorkerSessionStore::new(root.path().join("session")).unwrap();
|
||||
let store = CombinedStore::new(
|
||||
@@ -353,7 +371,7 @@ async fn canonical_worker_handle_uploads_and_submits_attachment() {
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let (handle, shutdown_rx) = WorkerController::spawn(worker, &root.path().join("runtime"))
|
||||
let (handle, shutdown_rx) = WorkerController::spawn(worker, &runtime_base)
|
||||
.await
|
||||
.unwrap();
|
||||
let mut rx = handle.subscribe();
|
||||
@@ -762,6 +780,227 @@ async fn controller_startup_failure_closes_bound_workdir_session() {
|
||||
));
|
||||
}
|
||||
|
||||
mod registry_startup_failure {
|
||||
//! Connect real controller preparation, registry source fences and session cleanup.
|
||||
//! The Workdir telemetry boundary substitutes a concurrent metadata change at subscription;
|
||||
//! the real registry must reject drift before starting its prepared observer.
|
||||
use super::*;
|
||||
use session_store::WorkerMetadataStore;
|
||||
use tokio::sync::broadcast;
|
||||
use workdir::WorkdirSession;
|
||||
|
||||
struct ObservedSession {
|
||||
local: LocalWorkdirSession,
|
||||
metadata_store: TestStore,
|
||||
worker_name: String,
|
||||
events: broadcast::Sender<workdir::CommandEvent>,
|
||||
subscriptions: AtomicUsize,
|
||||
receivers_at_close: Mutex<Vec<usize>>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for ObservedSession {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("ObservedSession")
|
||||
.field("worker_name", &self.worker_name)
|
||||
.finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl WorkdirSession for ObservedSession {
|
||||
fn workdir(&self) -> &Workdir {
|
||||
self.local.workdir()
|
||||
}
|
||||
|
||||
fn capabilities(&self) -> WorkdirSessionCapabilities {
|
||||
self.local.capabilities()
|
||||
}
|
||||
|
||||
async fn stat(
|
||||
&self,
|
||||
request: workdir::StatRequest,
|
||||
) -> Result<workdir::StatResult, WorkdirError> {
|
||||
self.local.stat(request).await
|
||||
}
|
||||
|
||||
async fn read(
|
||||
&self,
|
||||
request: workdir::ReadRequest,
|
||||
) -> Result<workdir::ReadResult, WorkdirError> {
|
||||
self.local.read(request).await
|
||||
}
|
||||
|
||||
async fn write(
|
||||
&self,
|
||||
request: workdir::WriteRequest,
|
||||
) -> Result<workdir::WriteResult, WorkdirError> {
|
||||
self.local.write(request).await
|
||||
}
|
||||
|
||||
async fn edit(
|
||||
&self,
|
||||
request: workdir::EditRequest,
|
||||
) -> Result<workdir::EditResult, WorkdirError> {
|
||||
self.local.edit(request).await
|
||||
}
|
||||
|
||||
async fn list(
|
||||
&self,
|
||||
request: workdir::ListRequest,
|
||||
) -> Result<workdir::ListResult, WorkdirError> {
|
||||
self.local.list(request).await
|
||||
}
|
||||
|
||||
async fn glob(
|
||||
&self,
|
||||
request: workdir::GlobRequest,
|
||||
) -> Result<workdir::GlobResult, WorkdirError> {
|
||||
self.local.glob(request).await
|
||||
}
|
||||
|
||||
async fn grep(
|
||||
&self,
|
||||
request: workdir::GrepRequest,
|
||||
) -> Result<workdir::GrepResult, WorkdirError> {
|
||||
self.local.grep(request).await
|
||||
}
|
||||
|
||||
async fn start_command(
|
||||
&self,
|
||||
request: CommandRequest,
|
||||
) -> Result<workdir::CommandHandle, WorkdirError> {
|
||||
self.local.start_command(request).await
|
||||
}
|
||||
|
||||
async fn command_status(
|
||||
&self,
|
||||
handle: workdir::CommandHandle,
|
||||
) -> Result<workdir::CommandStatus, WorkdirError> {
|
||||
self.local.command_status(handle).await
|
||||
}
|
||||
|
||||
async fn command_output(
|
||||
&self,
|
||||
request: CommandOutputRequest,
|
||||
) -> Result<workdir::CommandOutput, WorkdirError> {
|
||||
self.local.command_output(request).await
|
||||
}
|
||||
|
||||
async fn cancel_command(&self, handle: workdir::CommandHandle) -> Result<(), WorkdirError> {
|
||||
self.local.cancel_command(handle).await
|
||||
}
|
||||
|
||||
fn subscribe_command_events(&self) -> Option<broadcast::Receiver<workdir::CommandEvent>> {
|
||||
// Interleave a canonical change after read-only registry preparation.
|
||||
let mut metadata = self
|
||||
.metadata_store
|
||||
.read_by_name(&self.worker_name)
|
||||
.unwrap()
|
||||
.unwrap_or_else(|| session_store::WorkerMetadata::new(&self.worker_name, None));
|
||||
metadata
|
||||
.spawned_children
|
||||
.push(session_store::WorkerSpawnedChild {
|
||||
worker_name: "legacy-drift".into(),
|
||||
socket_path: PathBuf::new(),
|
||||
callback_address: PathBuf::new(),
|
||||
scope_delegated: vec![],
|
||||
});
|
||||
self.metadata_store.write(&metadata).unwrap();
|
||||
self.subscriptions.fetch_add(1, Ordering::SeqCst);
|
||||
Some(self.events.subscribe())
|
||||
}
|
||||
|
||||
async fn close(&self) -> Result<(), WorkdirError> {
|
||||
// Record before any await: aborting without joining must not pass by getting a
|
||||
// scheduler turn during the provider's asynchronous close. The sender stays alive
|
||||
// even after close, so channel closure cannot accidentally dispose of the observer.
|
||||
self.receivers_at_close
|
||||
.lock()
|
||||
.unwrap()
|
||||
.push(self.events.receiver_count());
|
||||
self.local.close().await
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn registry_drift_drops_prepared_observer_before_closing_session() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let run_dir = root.path().join("runs/1");
|
||||
let client = MockClient::new(simple_text_events());
|
||||
let manifest = WorkerManifest::from_toml(MANIFEST_TOML).unwrap();
|
||||
let store = CombinedStore::new(
|
||||
FsStore::new(root.path().join("sessions")).unwrap(),
|
||||
FsWorkerStore::new(root.path().join("workers")).unwrap(),
|
||||
);
|
||||
let engine =
|
||||
Engine::<_, agen::state::Mutable, worker::SessionHistoryMetadata>::new_annotated(
|
||||
client.clone(),
|
||||
);
|
||||
let mut worker = Worker::new(
|
||||
manifest,
|
||||
engine,
|
||||
store.clone(),
|
||||
WorkerWorkspaceContext::local_filesystem(None),
|
||||
WorkerFilesystemAuthority::local(root.path().into(), root.path().into()),
|
||||
manifest::Scope::writable(root.path()).unwrap(),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
let (events, unused_receiver) = broadcast::channel(8);
|
||||
drop(unused_receiver);
|
||||
let session = Arc::new(ObservedSession {
|
||||
local: LocalWorkdirSession::materialized_bound(
|
||||
Workdir::new("registry-startup-failure"),
|
||||
root.path().into(),
|
||||
root.path().into(),
|
||||
worker.scope().clone(),
|
||||
WorkdirSessionCapabilities::ALL,
|
||||
),
|
||||
metadata_store: store,
|
||||
worker_name: worker.manifest().worker.name.clone(),
|
||||
events,
|
||||
subscriptions: AtomicUsize::new(0),
|
||||
receivers_at_close: Mutex::new(Vec::new()),
|
||||
});
|
||||
worker.bind_single_workdir_session(Some(session.clone()));
|
||||
let sessions = worker.workdir_sessions();
|
||||
|
||||
let result = tokio::time::timeout(
|
||||
std::time::Duration::from_secs(5),
|
||||
WorkerController::spawn_runtime_managed_run(worker, &run_dir),
|
||||
)
|
||||
.await
|
||||
.expect("registry rejection and cleanup must settle");
|
||||
let error = match result {
|
||||
Err(error) => error,
|
||||
Ok(_) => panic!("registry source drift must reject controller startup"),
|
||||
};
|
||||
assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
|
||||
assert_eq!(session.subscriptions.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(*session.receivers_at_close.lock().unwrap(), vec![0]);
|
||||
assert_eq!(session.events.receiver_count(), 0);
|
||||
assert!(sessions.aliases().is_empty());
|
||||
// No controller retains the Worker/Engine after the failed startup.
|
||||
assert_eq!(Arc::strong_count(&client.call_count), 1);
|
||||
assert_eq!(client.call_count.load(Ordering::SeqCst), 0);
|
||||
assert!(!run_dir.join("worker.sock").exists());
|
||||
assert!(!run_dir.join("status.json").exists());
|
||||
assert!(!run_dir.join("manifest.toml").exists());
|
||||
assert!(matches!(
|
||||
session
|
||||
.start_command(CommandRequest {
|
||||
command: "printf unreachable".to_owned(),
|
||||
timeout_secs: 5,
|
||||
transport_byte_budget: workdir::MAX_COMMAND_TRANSPORT_BYTES,
|
||||
cwd: workdir::WorkdirPath::root(),
|
||||
tool_call_id: None,
|
||||
})
|
||||
.await,
|
||||
Err(ref error) if matches!(error.classification_source(), WorkdirError::Unavailable(_))
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
async fn wait_for_status(handle: &WorkerHandle, status: WorkerStatus) {
|
||||
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
|
||||
loop {
|
||||
@@ -1561,11 +1800,12 @@ permission = "write"
|
||||
"#;
|
||||
let client = MockClient::new(simple_text_events());
|
||||
let worker = make_worker_with_pwd_and_manifest(client, manifest).await.0;
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let result = WorkerController::spawn(worker, tmp.path()).await;
|
||||
let (_tmp, runtime_base) = owned_relative_runtime_base();
|
||||
let result = WorkerController::spawn(worker, &runtime_base).await;
|
||||
assert!(
|
||||
result.is_ok(),
|
||||
"feature exposure must not imply delegation authority"
|
||||
"feature exposure must not imply delegation authority: {:?}",
|
||||
result.as_ref().err()
|
||||
);
|
||||
}
|
||||
|
||||
@@ -1738,42 +1978,24 @@ async fn snapshot_includes_user_input_for_in_flight_turn() {
|
||||
// The Running event is the in-flight visibility fence: the committed
|
||||
// annotated input must already be available to an immediately attaching
|
||||
// subscriber rather than racing behind this status transition.
|
||||
let stream = tokio::net::UnixStream::connect(handle.runtime_dir.socket_path())
|
||||
.await
|
||||
.unwrap();
|
||||
let (reader, _writer) = stream.into_split();
|
||||
let mut reader = protocol::stream::JsonLineReader::new(reader);
|
||||
|
||||
loop {
|
||||
let event = reader.next::<Event>().await.unwrap().unwrap();
|
||||
match event {
|
||||
Event::Snapshot { session, .. } => {
|
||||
let found = session.entries.iter().any(|entry| match &entry.data {
|
||||
protocol::SessionSnapshotEntryData::UserInput { segments } => {
|
||||
protocol::Segment::flatten_to_text(segments) == "hello in-flight"
|
||||
}
|
||||
protocol::SessionSnapshotEntryData::Message {
|
||||
role: protocol::SessionMessageRole::User,
|
||||
content,
|
||||
} => content.iter().any(|part| {
|
||||
matches!(
|
||||
part,
|
||||
protocol::SessionContentPart::Text { text }
|
||||
if text == "hello in-flight"
|
||||
)
|
||||
}),
|
||||
_ => false,
|
||||
});
|
||||
assert!(
|
||||
found,
|
||||
"snapshot must carry the in-flight UserInput entry: {session:?}"
|
||||
);
|
||||
return;
|
||||
let Event::Snapshot { session, .. } = handle.snapshot_event() else {
|
||||
panic!("expected snapshot");
|
||||
};
|
||||
assert!(
|
||||
session.entries.iter().any(|entry| match &entry.data {
|
||||
protocol::SessionSnapshotEntryData::UserInput { segments } => {
|
||||
protocol::Segment::flatten_to_text(segments) == "hello in-flight"
|
||||
}
|
||||
Event::Alert(_) => continue,
|
||||
other => panic!("expected Snapshot first, got {other:?}"),
|
||||
}
|
||||
}
|
||||
protocol::SessionSnapshotEntryData::Message {
|
||||
role: protocol::SessionMessageRole::User,
|
||||
content,
|
||||
} => content.iter().any(|part| matches!(part,
|
||||
protocol::SessionContentPart::Text { text } if text == "hello in-flight"
|
||||
)),
|
||||
_ => false,
|
||||
}),
|
||||
"snapshot must carry the in-flight UserInput entry: {session:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1855,24 +2077,10 @@ async fn attach_snapshot_includes_current_status() {
|
||||
.unwrap();
|
||||
wait_for_status(&handle, WorkerStatus::Running).await;
|
||||
|
||||
let stream = tokio::net::UnixStream::connect(handle.runtime_dir.socket_path())
|
||||
.await
|
||||
.unwrap();
|
||||
let (reader, _writer) = stream.into_split();
|
||||
let mut reader = protocol::stream::JsonLineReader::new(reader);
|
||||
|
||||
// First event after connect is the snapshot — it carries the current status.
|
||||
loop {
|
||||
let event = reader.next::<Event>().await.unwrap().unwrap();
|
||||
match event {
|
||||
Event::Snapshot { state, .. } => {
|
||||
assert_eq!(state.catalog_status(), WorkerStatus::Running);
|
||||
return;
|
||||
}
|
||||
Event::Alert(_) => continue,
|
||||
other => panic!("expected Snapshot, got {other:?}"),
|
||||
}
|
||||
}
|
||||
let Event::Snapshot { state, .. } = handle.snapshot_event() else {
|
||||
panic!("expected snapshot");
|
||||
};
|
||||
assert_eq!(state.catalog_status(), WorkerStatus::Running);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -2928,7 +3136,7 @@ async fn status_json_reflects_worker_name() {
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Socket transport tests
|
||||
// Controller lifecycle tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
@@ -2966,283 +3174,148 @@ async fn shutdown_closes_method_admission_before_terminal_confirmation() {
|
||||
shutdown_rx.await.unwrap();
|
||||
}
|
||||
|
||||
/// Uses the real Controller and metadata store; only the model client is replaced.
|
||||
#[tokio::test]
|
||||
async fn shutdown_joins_socket_server_with_active_connection() {
|
||||
use tokio::net::UnixStream;
|
||||
|
||||
async fn register_peer_returns_existing_response_and_persists_reciprocal_metadata() {
|
||||
use session_store::{WorkerMetadata, WorkerMetadataStore};
|
||||
let worker = make_worker(MockClient::new(simple_text_events())).await;
|
||||
let runtime_base = tempfile::tempdir().unwrap();
|
||||
let (handle, shutdown_rx) = WorkerController::spawn(worker, runtime_base.path())
|
||||
.await
|
||||
let store = worker.store().clone();
|
||||
store
|
||||
.write(&WorkerMetadata::new("test-worker", None))
|
||||
.unwrap();
|
||||
let socket_path = handle.runtime_dir.socket_path();
|
||||
let _connection = UnixStream::connect(&socket_path).await.unwrap();
|
||||
|
||||
store
|
||||
.write(&WorkerMetadata::new("peer-worker", None))
|
||||
.unwrap();
|
||||
let handle = spawn_controller(worker).await;
|
||||
let mut events = handle.subscribe();
|
||||
handle
|
||||
.send(Method::Shutdown {
|
||||
command: worker_command(&handle),
|
||||
.send(Method::RegisterPeer {
|
||||
name: "peer-worker".into(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), shutdown_rx)
|
||||
assert!(
|
||||
drain_until(
|
||||
&mut events,
|
||||
std::time::Duration::from_secs(2),
|
||||
|event| matches!(
|
||||
event,
|
||||
Event::PeerRegistered { result } if result == &serde_json::json!({
|
||||
"source": "test-worker", "peer": "peer-worker",
|
||||
})
|
||||
)
|
||||
)
|
||||
.await
|
||||
.expect("controller should join its socket tasks")
|
||||
.expect("controller shutdown signal should remain open");
|
||||
assert!(!socket_path.exists());
|
||||
);
|
||||
assert_eq!(
|
||||
store.read_by_name("test-worker").unwrap().unwrap().peers[0].worker_name,
|
||||
"peer-worker"
|
||||
);
|
||||
assert_eq!(
|
||||
store.read_by_name("peer-worker").unwrap().unwrap().peers[0].worker_name,
|
||||
"test-worker"
|
||||
);
|
||||
assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn socket_run_receives_events() {
|
||||
use protocol::stream::{JsonLineReader, JsonLineWriter};
|
||||
use tokio::net::UnixStream;
|
||||
|
||||
let client = MockClient::new(simple_text_events());
|
||||
let worker = make_worker(client).await;
|
||||
async fn register_peer_missing_target_is_rejected_without_starting_a_worker() {
|
||||
use session_store::{WorkerMetadata, WorkerMetadataStore};
|
||||
let worker = make_worker(MockClient::new(simple_text_events())).await;
|
||||
let store = worker.store().clone();
|
||||
store
|
||||
.write(&WorkerMetadata::new("test-worker", None))
|
||||
.unwrap();
|
||||
let handle = spawn_controller(worker).await;
|
||||
|
||||
// Give the socket server a moment to bind
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
|
||||
let sock_path = handle.runtime_dir.socket_path();
|
||||
let stream = UnixStream::connect(&sock_path).await.unwrap();
|
||||
let (reader, writer) = stream.into_split();
|
||||
let mut reader = JsonLineReader::new(reader);
|
||||
let mut writer = JsonLineWriter::new(writer);
|
||||
|
||||
// Send run method via socket
|
||||
writer
|
||||
.write(&Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"Hello",
|
||||
))
|
||||
let mut events = handle.subscribe();
|
||||
handle
|
||||
.send(Method::RegisterPeer {
|
||||
name: "missing-peer".into(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Collect events
|
||||
let mut saw_turn_start = false;
|
||||
let mut saw_text_delta = false;
|
||||
let mut saw_turn_end = false;
|
||||
|
||||
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
|
||||
loop {
|
||||
tokio::select! {
|
||||
event = reader.next::<Event>() => {
|
||||
match event {
|
||||
Ok(Some(Event::TurnStart { .. })) => saw_turn_start = true,
|
||||
Ok(Some(Event::TextDelta { .. })) => saw_text_delta = true,
|
||||
Ok(Some(Event::TurnEnd { .. })) => {
|
||||
saw_turn_end = true;
|
||||
break;
|
||||
}
|
||||
Ok(None) | Err(_) => break,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
_ = tokio::time::sleep_until(deadline) => break,
|
||||
}
|
||||
}
|
||||
|
||||
assert!(saw_turn_start, "should see turn_start via socket");
|
||||
assert!(saw_text_delta, "should see text_delta via socket");
|
||||
assert!(saw_turn_end, "should see turn_end via socket");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn socket_worker_event_turn_ended_while_idle_auto_starts_turn() {
|
||||
use protocol::stream::{JsonLineReader, JsonLineWriter};
|
||||
use tokio::net::UnixStream;
|
||||
|
||||
let client = MockClient::new(simple_text_events());
|
||||
let worker = make_worker(client).await;
|
||||
let handle = spawn_controller(worker).await;
|
||||
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
|
||||
let sock_path = handle.runtime_dir.socket_path();
|
||||
let stream = UnixStream::connect(&sock_path).await.unwrap();
|
||||
let (reader, writer) = stream.into_split();
|
||||
let mut reader = JsonLineReader::new(reader);
|
||||
let mut writer = JsonLineWriter::new(writer);
|
||||
|
||||
writer
|
||||
.write(&Method::WorkerEvent(protocol::WorkerEvent::TurnEnded {
|
||||
worker_name: "child".into(),
|
||||
}))
|
||||
assert!(
|
||||
drain_until(
|
||||
&mut events,
|
||||
std::time::Duration::from_secs(2),
|
||||
|event| matches!(
|
||||
event,
|
||||
Event::Error { code: worker::ErrorCode::InvalidRequest, message }
|
||||
if message.contains("missing-peer") && message.contains("does not exist")
|
||||
)
|
||||
)
|
||||
.await
|
||||
);
|
||||
assert!(store.read_by_name("missing-peer").unwrap().is_none());
|
||||
assert!(
|
||||
store
|
||||
.read_by_name("test-worker")
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.peers
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_process_discovery_methods_require_host_or_runtime_authority() {
|
||||
let worker = make_worker(MockClient::new(simple_text_events())).await;
|
||||
let handle = spawn_controller(worker).await;
|
||||
let mut events = handle.subscribe();
|
||||
for method in [
|
||||
Method::ListWorkers,
|
||||
Method::RestoreWorker {
|
||||
name: "old-worker".into(),
|
||||
},
|
||||
] {
|
||||
handle.send(method).await.unwrap();
|
||||
assert!(
|
||||
drain_until(
|
||||
&mut events,
|
||||
std::time::Duration::from_secs(2),
|
||||
|event| matches!(
|
||||
event,
|
||||
Event::Error { code: worker::ErrorCode::InvalidRequest, message }
|
||||
if message.contains("Host or Runtime")
|
||||
)
|
||||
)
|
||||
.await
|
||||
);
|
||||
}
|
||||
assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn all_controller_launch_modes_leave_external_transport_to_the_host() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
for mode in ["direct", "runtime", "run"] {
|
||||
let worker = make_worker(MockClient::new(simple_text_events())).await;
|
||||
let base = root.path().join(mode);
|
||||
let (handle, shutdown) = match mode {
|
||||
"direct" => WorkerController::spawn(worker, &base).await,
|
||||
"runtime" => WorkerController::spawn_runtime_managed(worker, &base).await,
|
||||
"run" => {
|
||||
WorkerController::spawn_runtime_managed_run(worker, &base.join("runs/1")).await
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
.unwrap();
|
||||
|
||||
let mut saw_worker_event_echo = false;
|
||||
let mut saw_turn_start = false;
|
||||
let mut saw_turn_end = false;
|
||||
|
||||
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
|
||||
// The SystemItem and TurnEnd events arrive through independent
|
||||
// broadcast lanes (sink fan-out vs `working_event_tx`), so their relative
|
||||
// order on the wire is non-deterministic. Keep reading until both
|
||||
// are observed (or the deadline trips), rather than breaking on
|
||||
// the first TurnEnd.
|
||||
loop {
|
||||
if saw_worker_event_echo && saw_turn_end {
|
||||
break;
|
||||
}
|
||||
tokio::select! {
|
||||
event = reader.next::<Event>() => {
|
||||
match event {
|
||||
Ok(Some(Event::SystemItem { ref item, .. }))
|
||||
if item.get("kind").and_then(|k| k.as_str()) == Some("worker_event")
|
||||
&& item
|
||||
.pointer("/presentation/event/worker_name")
|
||||
.and_then(|v| v.as_str()) == Some("child") =>
|
||||
{
|
||||
saw_worker_event_echo = true;
|
||||
}
|
||||
Ok(Some(Event::TurnStart { .. })) => saw_turn_start = true,
|
||||
Ok(Some(Event::TurnEnd { .. })) => {
|
||||
saw_turn_end = true;
|
||||
}
|
||||
Ok(None) | Err(_) => break,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
_ = tokio::time::sleep_until(deadline) => break,
|
||||
}
|
||||
}
|
||||
|
||||
assert!(
|
||||
saw_worker_event_echo,
|
||||
"WorkerEvent::TurnEnded via socket should be echoed as Event::SystemItem(WorkerEvent)"
|
||||
);
|
||||
assert!(
|
||||
saw_turn_start,
|
||||
"WorkerEvent::TurnEnded via socket should auto-start a turn"
|
||||
);
|
||||
assert!(
|
||||
saw_turn_end,
|
||||
"auto-triggered turn should reach turn_end via socket"
|
||||
);
|
||||
}
|
||||
|
||||
async fn socket_error_after_method_line(
|
||||
handle: &WorkerHandle,
|
||||
line: &[u8],
|
||||
) -> (worker::ErrorCode, String) {
|
||||
use protocol::stream::JsonLineReader;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use tokio::net::UnixStream;
|
||||
|
||||
let sock_path = handle.runtime_dir.socket_path();
|
||||
let stream = UnixStream::connect(&sock_path).await.unwrap();
|
||||
let (reader, mut writer) = stream.into_split();
|
||||
let mut reader = JsonLineReader::new(reader);
|
||||
|
||||
writer.write_all(line).await.unwrap();
|
||||
|
||||
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(1);
|
||||
loop {
|
||||
tokio::select! {
|
||||
event = reader.next::<Event>() => {
|
||||
match event {
|
||||
Ok(Some(Event::Error { code, message })) => return (code, message),
|
||||
Ok(Some(_)) => {}
|
||||
Ok(None) => panic!("socket closed before invalid-method error"),
|
||||
Err(e) => panic!("socket read failed before invalid-method error: {e}"),
|
||||
}
|
||||
}
|
||||
_ = tokio::time::sleep_until(deadline) => {
|
||||
panic!("timed out waiting for invalid-method error")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn socket_schema_invalid_method_returns_error() {
|
||||
let client = MockClient::new(simple_text_events());
|
||||
let worker = make_worker(client).await;
|
||||
let handle = spawn_controller(worker).await;
|
||||
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
|
||||
let (code, message) = socket_error_after_method_line(&handle, b"{\"bad\":\"json\"}\n").await;
|
||||
|
||||
assert_eq!(code, worker::ErrorCode::InvalidRequest);
|
||||
assert!(
|
||||
message.contains("invalid method"),
|
||||
"expected invalid-method diagnostic, got: {message}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn socket_malformed_method_returns_error() {
|
||||
let client = MockClient::new(simple_text_events());
|
||||
let worker = make_worker(client).await;
|
||||
let handle = spawn_controller(worker).await;
|
||||
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
|
||||
let (code, message) = socket_error_after_method_line(&handle, b"{not-json}\n").await;
|
||||
|
||||
assert_eq!(code, worker::ErrorCode::InvalidRequest);
|
||||
assert!(
|
||||
message.contains("invalid method"),
|
||||
"expected invalid-method diagnostic, got: {message}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn socket_peer_close_without_method_does_not_broadcast_error() {
|
||||
use protocol::stream::JsonLineReader;
|
||||
use tokio::net::UnixStream;
|
||||
|
||||
let client = MockClient::new(simple_text_events());
|
||||
let worker = make_worker(client).await;
|
||||
let handle = spawn_controller(worker).await;
|
||||
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
|
||||
let mut broadcast_rx = handle.subscribe();
|
||||
let sock_path = handle.runtime_dir.socket_path();
|
||||
let stream = UnixStream::connect(&sock_path).await.unwrap();
|
||||
let (reader, writer) = stream.into_split();
|
||||
let mut reader = JsonLineReader::new(reader);
|
||||
|
||||
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(1);
|
||||
loop {
|
||||
tokio::select! {
|
||||
event = reader.next::<Event>() => {
|
||||
match event {
|
||||
Ok(Some(Event::Snapshot { .. })) => break,
|
||||
Ok(Some(_)) => {}
|
||||
Ok(None) => panic!("socket closed before connect-time snapshot"),
|
||||
Err(e) => panic!("socket read failed before connect-time snapshot: {e}"),
|
||||
}
|
||||
}
|
||||
_ = tokio::time::sleep_until(deadline) => {
|
||||
panic!("timed out waiting for connect-time snapshot")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
drop(writer);
|
||||
drop(reader);
|
||||
|
||||
let deadline = tokio::time::Instant::now() + std::time::Duration::from_millis(200);
|
||||
loop {
|
||||
tokio::select! {
|
||||
event = broadcast_rx.recv() => {
|
||||
match event {
|
||||
Ok(Event::Error { code, message }) => {
|
||||
panic!("peer close without Method broadcast error {code:?}: {message}")
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
|
||||
panic!("broadcast receiver lagged while checking peer close: {n}")
|
||||
}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Closed) => break,
|
||||
}
|
||||
}
|
||||
_ = tokio::time::sleep_until(deadline) => break,
|
||||
for legacy in ["pid", "status.json", "manifest.toml", "sock", "worker.sock"] {
|
||||
assert!(
|
||||
!handle.runtime_dir.path().join(legacy).exists(),
|
||||
"{mode} created {legacy}"
|
||||
);
|
||||
}
|
||||
handle
|
||||
.send(Method::Shutdown {
|
||||
command: worker_command(&handle),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), shutdown)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4006,27 +4079,15 @@ fn item_text_contains(item: &Item, needle: &str) -> bool {
|
||||
}
|
||||
|
||||
async fn snapshot_contains_user_input(handle: &WorkerHandle, needle: &str) -> bool {
|
||||
let stream = tokio::net::UnixStream::connect(handle.runtime_dir.socket_path())
|
||||
.await
|
||||
.unwrap();
|
||||
let (reader, _writer) = stream.into_split();
|
||||
let mut reader = protocol::stream::JsonLineReader::new(reader);
|
||||
|
||||
loop {
|
||||
let event = reader.next::<Event>().await.unwrap().unwrap();
|
||||
match event {
|
||||
Event::Snapshot { session, .. } => {
|
||||
return session.entries.into_iter().any(|entry| match entry.data {
|
||||
protocol::SessionSnapshotEntryData::UserInput { segments } => {
|
||||
protocol::Segment::flatten_to_text(&segments).contains(needle)
|
||||
}
|
||||
_ => false,
|
||||
});
|
||||
}
|
||||
Event::Alert(_) => continue,
|
||||
other => panic!("expected Snapshot first, got {other:?}"),
|
||||
let Event::Snapshot { session, .. } = handle.snapshot_event() else {
|
||||
panic!("expected snapshot");
|
||||
};
|
||||
session.entries.into_iter().any(|entry| match entry.data {
|
||||
protocol::SessionSnapshotEntryData::UserInput { segments } => {
|
||||
protocol::Segment::flatten_to_text(&segments).contains(needle)
|
||||
}
|
||||
}
|
||||
_ => false,
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -187,7 +187,8 @@ async fn materialise_on_first_turn_populates_worker() {
|
||||
.expect("system prompt materialised")
|
||||
.to_string();
|
||||
assert!(rendered.contains("date="));
|
||||
assert!(rendered.contains(&pwd.display().to_string()));
|
||||
assert!(rendered.contains("## Workdir execution environment"));
|
||||
assert!(!rendered.contains(&pwd.display().to_string()));
|
||||
assert!(rendered.starts_with("date="));
|
||||
}
|
||||
|
||||
@@ -206,7 +207,8 @@ async fn session_start_state_captures_rendered_prompt() {
|
||||
LogEntry::AnnotatedSegmentStart { system_prompt, .. } => {
|
||||
let sp = system_prompt.as_deref().expect("system prompt set");
|
||||
assert!(sp.starts_with("hello"));
|
||||
assert!(sp.contains(&pwd.display().to_string()));
|
||||
assert!(sp.contains("## Workdir execution environment"));
|
||||
assert!(!sp.contains(&pwd.display().to_string()));
|
||||
}
|
||||
other => panic!("expected AnnotatedSegmentStart as first entry, got {other:?}"),
|
||||
}
|
||||
|
||||
@@ -1,15 +1,10 @@
|
||||
//! Legacy process callback events are diagnostics only after Internal SubWorker migration.
|
||||
//! Agent-visible Internal SubWorker event summaries.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use protocol::{Permission, ScopeRule, WorkerEvent};
|
||||
use tempfile::TempDir;
|
||||
use worker::ipc::event::{apply_event_side_effects, render_event};
|
||||
use worker::runtime::dir::RuntimeDir;
|
||||
use worker::spawn::registry::SpawnedWorkerRegistry;
|
||||
use protocol::WorkerEvent;
|
||||
use worker::ipc::event::render_event;
|
||||
|
||||
#[test]
|
||||
fn render_event_keeps_bounded_legacy_diagnostics() {
|
||||
fn render_event_identifies_child_and_failure() {
|
||||
let rendered = render_event(&WorkerEvent::Errored {
|
||||
worker_name: "legacy-child".into(),
|
||||
message: "boom".into(),
|
||||
@@ -17,30 +12,3 @@ fn render_event_keeps_bounded_legacy_diagnostics() {
|
||||
assert!(rendered.contains("legacy-child"));
|
||||
assert!(rendered.contains("boom"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_callback_cannot_register_process_subworker_authority() {
|
||||
let runtime_base = TempDir::new().unwrap();
|
||||
let runtime_dir = Arc::new(
|
||||
RuntimeDir::create(runtime_base.path(), "parent")
|
||||
.await
|
||||
.unwrap(),
|
||||
);
|
||||
let registry = SpawnedWorkerRegistry::new(runtime_dir.clone());
|
||||
let scope_root = TempDir::new().unwrap();
|
||||
let event = WorkerEvent::ScopeSubDelegated {
|
||||
parent_worker: "legacy-parent".into(),
|
||||
sub_worker: "legacy-child".into(),
|
||||
sub_socket: "/tmp/legacy-child.sock".into(),
|
||||
scope: vec![ScopeRule {
|
||||
target: scope_root.path().to_path_buf(),
|
||||
permission: Permission::Write,
|
||||
recursive: true,
|
||||
symlink_policy: Default::default(),
|
||||
}],
|
||||
};
|
||||
|
||||
apply_event_side_effects(&event, ®istry, "parent", &None).await;
|
||||
|
||||
assert!(!runtime_dir.path().join("spawned_workers.json").exists());
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user