fix: merge develop and reconcile schema histories

Preserve both published schema-103 histories and converge their Passkey results and Job input retention features transactionally at schema 104. Retain local Reviewer, integration-event, auth and sidebar fixes while adopting develop restore and E2E updates; regenerate API contracts.

Validation: schema convergence regressions, full Server/Worker/Runtime suites, server-api and merge-request tests, Web unit/component tests, root cargo check, E2E harness tests and restore/auth target compilation, generated artifacts, fmt and naming checks. Product-process E2E not rerun; live services and databases untouched.
This commit is contained in:
2026-10-28 04:01:34 +09:00
218 changed files with 48985 additions and 11155 deletions
+1 -1
View File
@@ -11,7 +11,6 @@ default = []
[dependencies]
async-trait = { workspace = true }
clap = { version = "4.6.0", features = ["derive"] }
agen = { workspace = true, features = ["codex"] }
session-store = { workspace = true }
secrets = { workspace = true }
@@ -58,6 +57,7 @@ wip-protocol = { workspace = true }
wip-text-view = { workspace = true }
[dev-dependencies]
tracing-subscriber.workspace = true
axum = { workspace = true }
workdir = { workspace = true, features = ["http-client"] }
dotenv = "0.15.0"
+42 -30
View File
@@ -206,26 +206,36 @@ impl AuthorizedWorkdirSession {
.map(CheckoutSearchResult::Grep),
};
};
scoped
.authorize_scope_path(WorkdirScopeAuthorizationRequest {
rules: vec![WorkdirToolScopeRule {
target: WorkdirPath::root(),
..rule.clone()
}],
path: relative_path(&root, operation.path())?,
permission: WorkdirToolScopePermission::Read,
})
.await?;
// Lease cwd rebases paths. Use the same provider scoped-traversal contract
// with attachment output_root instead, retaining typed paths, cursors and
// Grep's rendered filenames without parsing or post-filtering output.
let mut request = CheckoutSearchRequest::new(operation);
request.scope_layers.push(vec![rule]);
self.session.checkout_search(request).await
self.search_with_scope(scoped, root, rule, CheckoutSearchRequest::new(operation))
.await
})
.await
}
async fn search_with_scope(
&self,
scoped: WorkdirSessionHandle,
root: WorkdirPath,
rule: WorkdirToolScopeRule,
mut request: CheckoutSearchRequest,
) -> Result<CheckoutSearchResult, WorkdirError> {
scoped
.authorize_scope_path(WorkdirScopeAuthorizationRequest {
rules: vec![WorkdirToolScopeRule {
target: WorkdirPath::root(),
..rule.clone()
}],
path: relative_path(&root, request.operation.path())?,
permission: WorkdirToolScopePermission::Read,
})
.await?;
// Preserve the caller's coordinate root, cursors and all existing
// attenuation layers. The provider applies the Host ceiling during
// traversal, not by filtering already-rendered filenames afterward.
request.scope_layers.push(vec![rule]);
self.session.checkout_search(request).await
}
fn authorize_command(&self) -> Result<(), WorkdirError> {
// cwd is not a sandbox: arbitrary shell can access the entire attachment.
self.authorize(COMMAND, &WorkdirPath::root(), true)
@@ -261,6 +271,12 @@ impl WorkdirSession for AuthorizedWorkdirSession {
self.session.capabilities()
}
fn environment(&self) -> workdir::WorkdirSessionEnvironment {
let mut environment = self.session.environment();
environment.capabilities = environment.capabilities.intersection(self.capabilities());
environment
}
async fn authorize_scope_path(
&self,
mut request: WorkdirScopeAuthorizationRequest,
@@ -296,8 +312,14 @@ impl WorkdirSession for AuthorizedWorkdirSession {
&self,
request: CheckoutSearchRequest,
) -> Result<CheckoutSearchResult, WorkdirError> {
self.require_legacy(READ)?;
self.session.checkout_search(request).await
let path = request.operation.path().clone();
self.with_authorized(READ, &path, true, |scoped, root, rule| async move {
match rule {
Some(rule) => self.search_with_scope(scoped, root, rule, request).await,
None => self.session.checkout_search(request).await,
}
})
.await
}
async fn checkout_observe(
@@ -490,6 +512,8 @@ mod tests {
//! Component tests connect this adapter to the real LocalWorkdirSession.
//! Only the Host authorization transport/policy is replaced; these are not E2E tests.
mod environment;
use super::*;
use crate::{WorkspaceClientError, WorkspaceRequest, WorkspaceResponse};
use server_api::authorization::ScopedPermission;
@@ -842,18 +866,6 @@ mod tests {
.await,
Err(WorkdirError::Denied(_))
));
assert!(matches!(
session
.checkout_search(CheckoutSearchRequest::new(CheckoutSearchOperation::List(
ListRequest {
path: WorkdirPath::root(),
after: None,
limit: 10,
}
)))
.await,
Err(WorkdirError::Denied(_))
));
let rule = WorkdirToolScopeRule {
target: WorkdirPath::root(),
permission: WorkdirToolScopePermission::Read,
@@ -0,0 +1,198 @@
//! Normal lib tests connecting live Host-grant checks to real provider traversal
//! and router cwd projection. Only the Host transport is replaced by the parent
//! module's grant fixture; no commands, network or ambient Workspace state are used.
use super::*;
#[tokio::test]
async fn authorization_wrapper_preserves_inner_environment_and_capability_ceiling() {
let root = tempfile::tempdir().unwrap();
std::fs::create_dir(root.path().join("src")).unwrap();
let provider = Arc::new(LocalWorkdirSession::materialized(
root.path().to_path_buf(),
root.path().to_path_buf(),
manifest::SharedScope::new(manifest::Scope::writable(root.path()).unwrap()),
WorkdirSessionCapabilities::READ_ONLY,
));
let broker = WorkdirToolBroker::at_cwd(provider, WorkdirPath::new("src").unwrap()).unwrap();
let inner = broker.tool_session();
let expected = inner.environment();
let client = AuthorizationClient::new(vec![permission(READ, "src", true)]);
let wrapped = authorized_workdir_session(inner, client.clone(), "attachment-id".into());
assert_eq!(wrapped.environment(), expected);
assert_eq!(wrapped.environment().cwd.as_str(), "src");
assert_eq!(
wrapped.environment().capabilities,
WorkdirSessionCapabilities::READ_ONLY
);
assert!(
client.requests.lock().unwrap().is_empty(),
"environment is metadata, not a cached grant decision"
);
wrapped.close().await.unwrap();
}
#[tokio::test]
async fn nonroot_routed_search_retains_host_grants_output_coordinates_and_cursors() {
let root = tempfile::tempdir().unwrap();
std::fs::create_dir(root.path().join("src")).unwrap();
std::fs::create_dir(root.path().join("private")).unwrap();
for name in ["a.txt", "b.txt"] {
std::fs::write(root.path().join("src").join(name), "needle public marker").unwrap();
}
std::fs::write(
root.path().join("private/secret.txt"),
"needle private marker",
)
.unwrap();
let client = AuthorizationClient::new(vec![permission(READ, "src", true)]);
let authorized = session(&root, client.clone());
let router = workdir::WorkdirSessionRouter::new();
let alias = workdir::WorkdirAttachmentAlias::new("checkout").unwrap();
router
.attach_at_cwd(alias.clone(), authorized, WorkdirPath::new("src").unwrap())
.unwrap();
let routed = router.resolve(Some("checkout")).unwrap().session;
assert_eq!(routed.environment().cwd.as_str(), "src");
let paths = routed
.glob(GlobRequest {
path: WorkdirPath::root(),
pattern: "**/*.txt".into(),
limit: 100,
})
.await
.unwrap()
.paths;
assert_eq!(
paths,
vec![
WorkdirPath::new("a.txt").unwrap(),
WorkdirPath::new("b.txt").unwrap()
]
);
// Search output is reusable as a path without adding src/ at the caller.
assert_eq!(
routed.read(read(paths[0].as_str())).await.unwrap().bytes,
b"needle public marker"
);
for output_mode in [
workdir::GrepOutputMode::Content,
workdir::GrepOutputMode::FilesWithMatches,
workdir::GrepOutputMode::Count,
] {
let result = routed
.grep(GrepRequest {
path: WorkdirPath::root(),
pattern: "needle".into(),
glob: None,
file_type: None,
case_insensitive: false,
before_context: 0,
after_context: 0,
multiline: false,
output_mode,
limit: 100,
offset: 0,
})
.await
.unwrap();
let mut grep_paths = result.paths;
grep_paths.sort_by(|left, right| left.as_str().cmp(right.as_str()));
assert_eq!(grep_paths, paths);
assert!(result.output.contains("a.txt"));
assert!(!result.output.contains("src/"));
assert!(!result.output.contains("secret"));
}
let first = routed
.list(ListRequest {
path: WorkdirPath::root(),
after: None,
limit: 1,
})
.await
.unwrap();
assert_eq!(first.entries[0].path.as_str(), "a.txt");
assert_eq!(first.next_after.as_ref().unwrap().path.as_str(), "a.txt");
let second = routed
.list(ListRequest {
path: WorkdirPath::root(),
after: first.next_after,
limit: 1,
})
.await
.unwrap();
assert_eq!(second.entries[0].path.as_str(), "b.txt");
assert!(second.next_after.is_none());
// An existing caller ceiling must survive the extra Host layer. An empty
// layer is invalid and cannot silently turn into the broader Host grant.
let mut denied = CheckoutSearchRequest::new(CheckoutSearchOperation::Glob(GlobRequest {
path: WorkdirPath::root(),
pattern: "**/*.txt".into(),
limit: 100,
}));
denied.scope_layers.push(Vec::new());
assert!(routed.checkout_search(denied).await.is_err());
assert!(client.requests.lock().unwrap().iter().all(|request| {
matches!(&request.resource, PermissionResource::Workdir { path, .. } if path == "src" || path == "src/a.txt")
}));
client.revoked.store(true, Ordering::SeqCst);
assert!(
routed
.glob(GlobRequest {
path: WorkdirPath::root(),
pattern: "**/*.txt".into(),
limit: 100,
})
.await
.is_err()
);
router.detach(&alias).await.unwrap();
}
#[cfg(target_os = "linux")]
#[tokio::test]
async fn scoped_checkout_search_preserves_caller_layer_and_rejects_resolved_escape() {
let root = tempfile::tempdir().unwrap();
std::fs::create_dir_all(root.path().join("src/nested")).unwrap();
std::fs::create_dir(root.path().join("private")).unwrap();
std::fs::write(root.path().join("src/public.txt"), "needle public").unwrap();
std::fs::write(root.path().join("src/nested/hidden.txt"), "needle hidden").unwrap();
std::fs::write(root.path().join("private/secret.txt"), "needle secret").unwrap();
std::os::unix::fs::symlink("../private", root.path().join("src/escape")).unwrap();
std::os::unix::fs::symlink("../private/secret.txt", root.path().join("src/alias.txt")).unwrap();
let client = AuthorizationClient::new(vec![permission(READ, "src", true)]);
let authorized = session(&root, client);
let request = CheckoutSearchRequest {
operation: CheckoutSearchOperation::Glob(GlobRequest {
path: WorkdirPath::new("src").unwrap(),
pattern: "**/*.txt".into(),
limit: 100,
}),
output_root: WorkdirPath::new("src").unwrap(),
scope_layers: vec![vec![WorkdirToolScopeRule {
target: WorkdirPath::new("src").unwrap(),
recursive: false,
permission: WorkdirToolScopePermission::Read,
symlink_policy: manifest::SymlinkPolicy::Resolved,
}]],
};
let result = authorized.checkout_search(request).await.unwrap();
let CheckoutSearchResult::Glob(result) = result else {
panic!("expected Glob result")
};
assert_eq!(result.paths, vec![WorkdirPath::new("public.txt").unwrap()]);
assert!(
authorized
.checkout_search(CheckoutSearchRequest::new(CheckoutSearchOperation::Glob(
GlobRequest {
path: WorkdirPath::new("private").unwrap(),
pattern: "**/*.txt".into(),
limit: 100,
}
)))
.await
.is_err()
);
authorized.close().await.unwrap();
}
+272 -28
View File
@@ -6,9 +6,8 @@ use thiserror::Error;
use workdir::WorkdirSessionHandle;
use crate::PromptCatalogSource;
use crate::controller::{
ShutdownReceiver, WorkerController, WorkerControllerTransport, WorkerHandle,
};
use crate::controller::{ShutdownReceiver, WorkerController, WorkerHandle};
pub use crate::worker::{PreparedRestoredWorker, RestoreCandidateWorkdirCleanup};
use crate::worker::{Worker, WorkerError, WorkerFilesystemAuthority, WorkerWorkspaceContext};
use manifest::WorkerManifest;
@@ -30,7 +29,6 @@ pub struct WorkerBootstrap<St> {
workspace_context: WorkerWorkspaceContext,
filesystem_authority: WorkerFilesystemAuthority,
layout: WorkerBootstrapLayout,
transport: WorkerControllerTransport,
model_client: Option<Box<dyn LlmClient>>,
workdir_session: Option<WorkdirSessionHandle>,
subjektiv_host: Option<std::sync::Arc<dyn crate::subjektiv::SubjektivHost>>,
@@ -41,7 +39,6 @@ pub struct WorkerBootstrap<St> {
pub struct PreparedWorker<C: LlmClient, St: Store> {
worker: Worker<C, St>,
layout: WorkerBootstrapLayout,
transport: WorkerControllerTransport,
}
/// Live controller returned only after Worker construction and feature
@@ -68,7 +65,6 @@ impl<St> WorkerBootstrap<St>
where
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
{
#[allow(clippy::too_many_arguments)]
pub fn new(
manifest: WorkerManifest,
store: St,
@@ -76,7 +72,6 @@ where
workspace_context: WorkerWorkspaceContext,
filesystem_authority: WorkerFilesystemAuthority,
layout: WorkerBootstrapLayout,
transport: WorkerControllerTransport,
) -> Self {
Self {
manifest,
@@ -85,7 +80,6 @@ where
workspace_context,
filesystem_authority,
layout,
transport,
model_client: None,
workdir_session: None,
subjektiv_host: None,
@@ -165,7 +159,39 @@ where
)
.map_err(WorkerBootstrapError::Worker)?;
}
Ok(PreparedWorker::new(worker, self.layout, self.transport))
Ok(PreparedWorker::new(worker, self.layout))
}
/// Reconstruct a saved materialized Worker without allocation, attribution
/// repair, Feature installation, or controller exposure. The caller owns the
/// later controller layout and must activate before starting it.
pub async fn prepare_restored_candidate(
self,
worker_name: &str,
) -> Result<PreparedRestoredWorker<St>, WorkerBootstrapError> {
let mut candidate =
Worker::prepare_restored_from_worker_metadata_with_context_and_model_client(
worker_name,
self.manifest,
self.store,
self.prompt_catalog,
self.workspace_context,
self.filesystem_authority,
self.model_client,
)
.await
.map_err(WorkerBootstrapError::Worker)?;
if let Some(workdir_session) = self.workdir_session {
candidate
.worker_mut()
.map_err(WorkerBootstrapError::Worker)?
.bind_single_workdir_session(Some(workdir_session));
}
candidate
.worker_mut()
.map_err(WorkerBootstrapError::Worker)?
.bind_subjektiv_host(self.subjektiv_host);
Ok(candidate)
}
pub async fn prepare_restored(
@@ -206,7 +232,7 @@ where
)
.map_err(WorkerBootstrapError::Worker)?;
}
Ok(PreparedWorker::new(worker, self.layout, self.transport))
Ok(PreparedWorker::new(worker, self.layout))
}
pub async fn start(self) -> Result<BootstrappedWorker, WorkerBootstrapError> {
@@ -221,16 +247,8 @@ where
{
/// Wrap a restored Worker in the same pre-exposure lifecycle used by fresh
/// bootstraps.
pub fn new(
worker: Worker<C, St>,
layout: WorkerBootstrapLayout,
transport: WorkerControllerTransport,
) -> Self {
Self {
worker,
layout,
transport,
}
pub fn new(worker: Worker<C, St>, layout: WorkerBootstrapLayout) -> Self {
Self { worker, layout }
}
pub fn worker(&self) -> &Worker<C, St> {
@@ -242,7 +260,7 @@ where
}
pub async fn start(self) -> Result<BootstrappedWorker, WorkerBootstrapError> {
start_worker_controller(self.worker, self.layout, self.transport).await
start_worker_controller(self.worker, self.layout).await
}
}
@@ -252,22 +270,19 @@ where
pub async fn start_worker_controller<C, St>(
worker: Worker<C, St>,
layout: WorkerBootstrapLayout,
transport: WorkerControllerTransport,
) -> Result<BootstrappedWorker, WorkerBootstrapError>
where
C: LlmClient + Clone + 'static,
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
{
let cleanup_sessions = worker.workdir_sessions();
let factory_owned_cleanup = worker.startup_feature_cleanup_handle().is_some();
let controller = match layout {
WorkerBootstrapLayout::Direct { runtime_base } => {
WorkerController::spawn_with_transport_owned(worker, &runtime_base, transport).await
WorkerController::spawn_owned(worker, &runtime_base).await
}
WorkerBootstrapLayout::RuntimeManagedRun { run_dir } => {
WorkerController::spawn_runtime_managed_run_with_transport_owned(
worker, &run_dir, transport,
)
.await
WorkerController::spawn_runtime_managed_run_owned(worker, &run_dir).await
}
};
@@ -278,7 +293,10 @@ where
controller_task,
}),
Err(source) => {
let cleanup_failed = cleanup_sessions.close_all().await.is_err();
// Factory-owned task scopes must settle before sessions close.
// Cleanup remains unconfirmed here; preserve the startup source.
let cleanup_failed =
factory_owned_cleanup || cleanup_sessions.close_all().await.is_err();
Err(WorkerBootstrapError::Controller {
source,
cleanup_failed,
@@ -286,3 +304,229 @@ where
}
}
}
/// Host bindings for one unpublished restored Controller. These do not expose
/// Worker execution or permit arbitrary mutation during preparation.
pub struct RestoredControllerBindings {
pub backend_job: Option<crate::BackendJobExecutionBinding>,
pub workdir_sessions: std::sync::Arc<workdir::WorkdirSessionRouter>,
pub observation_provider:
Option<std::sync::Arc<dyn crate::feature::builtin::WorkerObservationProvider>>,
pub subjektiv_attached: bool,
}
/// The same real receiver, snapshot sources and paired subscription retained
/// privately across the Runtime's durable restore decision.
pub struct UnpublishedRestoredController<St: Store> {
initialized: Option<crate::controller::InitializedWorkerController<Box<dyn LlmClient>, St>>,
activation: Option<crate::worker::RestoredWorkerActivation>,
streams: Option<crate::ipc::protocol_session::WorkerProtocolSessionStreams>,
subjektiv_attached: bool,
effects_applied: bool,
activation_failed: bool,
closing: bool,
}
pub struct ActivatedRestoredController {
pub controller: BootstrappedWorker,
pub streams: crate::ipc::protocol_session::WorkerProtocolSessionStreams,
}
impl<St> PreparedRestoredWorker<St>
where
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
{
pub fn workspace_client_handle(
&mut self,
) -> Result<std::sync::Arc<dyn crate::WorkspaceClient>, WorkerError> {
Ok(self.worker_mut()?.workspace_client_handle())
}
pub fn retain_startup_feature_cleanup(
&mut self,
) -> Result<crate::WorkerStartupFeatureCleanup, WorkerError> {
Ok(self.worker_mut()?.retain_startup_feature_cleanup())
}
/// Initialize the real Controller without allocation, canonical restore
/// writes, attribution changes, registry reclamation or input consumption.
/// MCP discovery and Feature tasks belong to the retained startup cleanup.
pub async fn prepare_runtime_controller(
mut self,
run_dir: &std::path::Path,
bindings: RestoredControllerBindings,
) -> Result<UnpublishedRestoredController<St>, WorkerBootstrapError> {
let worker = self.worker_mut().map_err(WorkerBootstrapError::Worker)?;
if let Some(binding) = bindings.backend_job {
worker.bind_backend_job(binding).map_err(|error| {
WorkerBootstrapError::Worker(WorkerError::FeatureLifecycle(error.into()))
})?;
}
worker
.workdir_sessions()
.close_all()
.await
.map_err(|error| {
WorkerBootstrapError::Worker(WorkerError::FeatureLifecycle(error.to_string()))
})?;
worker.bind_workdir_sessions(bindings.workdir_sessions);
worker.bind_worker_observation_provider(bindings.observation_provider);
let flow_enabled = worker.manifest().feature.flow.enabled;
if flow_enabled {
let report = worker
.install_runtime_flow_transition_feature()
.map_err(|error| {
WorkerBootstrapError::Worker(WorkerError::FeatureLifecycle(error))
})?;
if report.reports.iter().any(|report| !report.installed) {
return Err(WorkerBootstrapError::Worker(WorkerError::FeatureLifecycle(
"Flow transition installation refused".into(),
)));
}
}
let (worker, activation) = self
.into_controller_parts()
.map_err(WorkerBootstrapError::Worker)?;
let initialized = WorkerController::initialize_restored(worker, run_dir)
.await
.map_err(|source| WorkerBootstrapError::Controller {
source,
cleanup_failed: true,
})?;
if flow_enabled {
initialized.handle.shared_state.enable_flow_transition();
}
let streams = {
let _guard = initialized.handle.protocol_snapshot_publish_guard();
crate::ipc::protocol_session::subscribe_worker_protocol_session(&initialized.handle)
};
Ok(UnpublishedRestoredController {
initialized: Some(initialized),
activation: Some(activation),
streams: Some(streams),
subjektiv_attached: bindings.subjektiv_attached,
effects_applied: false,
activation_failed: false,
closing: false,
})
}
}
impl<St> UnpublishedRestoredController<St>
where
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
{
pub fn snapshot(&self) -> &protocol::Event {
&self
.streams
.as_ref()
.expect("unpublished restore streams")
.snapshot_event
}
/// Only publication failures after confirmed effects can retry on this
/// exact owner. Partial activation must be reconciled, never blindly rerun.
pub fn can_retry_publication(&self) -> bool {
self.initialized.is_some()
&& self.effects_applied
&& !self.activation_failed
&& !self.closing
}
/// Apply canonical restore effects after the host's durable decision, while
/// retaining the same private Controller for subsequent publication.
pub async fn apply_effects(&mut self) -> Result<(), WorkerBootstrapError> {
let initialized = self
.initialized
.as_mut()
.ok_or_else(|| WorkerBootstrapError::Worker(WorkerError::RestoreCandidateClosing))?;
if self.activation_failed || self.closing {
return Err(WorkerBootstrapError::Worker(
WorkerError::RestoreCandidateClosing,
));
}
// Preparation's read-only authorization is not a publication grant.
// Revalidate current Host authority after commit, before effects/input,
// including a publication retry on this same private Controller.
initialized
.worker
.validate_subjektiv_body_publication()
.map_err(WorkerBootstrapError::Worker)?;
if !self.effects_applied {
self.activation_failed = true;
self.activation
.as_ref()
.expect("restore activation")
.activate(&mut initialized.worker)
.await
.map_err(WorkerBootstrapError::Worker)?;
initialized
.worker
.finalize_subjektiv_session_attribution(
self.subjektiv_attached,
crate::SubjektivSessionAttributionLifecycle::RestoredSession,
)
.map_err(WorkerBootstrapError::Worker)?;
self.effects_applied = true;
self.activation_failed = false;
self.activation = None;
}
Ok(())
}
pub fn allocation_cleanup(
&self,
) -> Option<crate::runtime::worker_allocation::ScopeAllocationCleanup> {
self.initialized
.as_ref()
.and_then(|initialized| initialized.worker.restore_allocation_cleanup_handle())
}
pub async fn activate(&mut self) -> Result<ActivatedRestoredController, WorkerBootstrapError> {
self.apply_effects().await?;
let initialized = self
.initialized
.as_mut()
.expect("same initialized Controller");
let publication = initialized.prepare_publication().await.map_err(|source| {
WorkerBootstrapError::Controller {
source,
cleanup_failed: true,
}
})?;
let initialized = self
.initialized
.take()
.expect("same initialized Controller");
let (handle, shutdown, controller_task) = initialized.start(publication);
Ok(ActivatedRestoredController {
controller: BootstrappedWorker {
handle,
shutdown,
controller_task,
},
streams: self.streams.take().expect("same paired subscription"),
})
}
pub async fn discard(&mut self) -> Result<(), WorkerError> {
self.closing = true;
if let Some(initialized) = self.initialized.as_mut() {
initialized
.worker
.retain_startup_feature_cleanup()
.shutdown()
.await
.map_err(|error| WorkerError::FeatureLifecycle(error.to_string()))?;
initialized
.worker
.workdir_sessions()
.close_all()
.await
.map_err(|error| WorkerError::FeatureLifecycle(error.to_string()))?;
}
self.streams = None;
self.initialized = None;
Ok(())
}
}
+37
View File
@@ -121,6 +121,7 @@ async fn open_session(
session_id: WorkdirSessionId::new(SESSION).unwrap(),
workdir_id: state.session.workdir().id().clone(),
capabilities: state.session.capabilities(),
environment: state.session.environment(),
})
.into_response()
}
@@ -251,6 +252,10 @@ impl Drop for HttpFixture {
impl HttpFixture {
async fn new(capabilities: WorkdirSessionCapabilities) -> Self {
Self::at_cwd(capabilities, "").await
}
async fn at_cwd(capabilities: WorkdirSessionCapabilities, cwd: &str) -> Self {
let dir = TempDir::new().unwrap();
std::fs::create_dir_all(dir.path().join("src/deep")).unwrap();
std::fs::write(
@@ -264,6 +269,13 @@ impl HttpFixture {
manifest::SharedScope::new(manifest::Scope::writable(dir.path()).unwrap()),
capabilities,
));
let session: WorkdirSessionHandle = if cwd.is_empty() {
session
} else {
workdir::WorkdirToolBroker::at_cwd(session, WorkdirPath::new(cwd).unwrap())
.unwrap()
.tool_session()
};
let state = Arc::new(ProviderState {
session,
requests: Mutex::new(Vec::new()),
@@ -349,6 +361,31 @@ impl HttpFixture {
}
}
#[tokio::test]
async fn remote_open_keeps_provider_environment_and_nonroot_operation_coordinates() {
let fixture = HttpFixture::at_cwd(WorkdirSessionCapabilities::READ_ONLY, "src/deep").await;
let remote = fixture.open(TOKEN).await.unwrap();
assert_eq!(remote.environment(), fixture.state.session.environment());
assert_eq!(remote.environment().cwd.as_str(), "src/deep");
assert_eq!(
remote.environment().capabilities,
WorkdirSessionCapabilities::READ_ONLY
);
let result = remote
.read(workdir::ReadRequest {
path: WorkdirPath::new("a.txt").unwrap(),
offset: 0,
limit: 10,
max_bytes: 4096,
})
.await
.unwrap();
assert_eq!(result.bytes, b"first\nneedle needle\nlast\n");
fixture.public_output(&serde_json::to_string(&remote.environment()).unwrap());
remote.close().await.unwrap();
fixture.assert_endpoints();
}
fn native_runtime(remote: Arc<RemoteWorkdirSession>) -> (WipRuntime, Arc<WorkdirSessionRouter>) {
let router = Arc::new(WorkdirSessionRouter::new());
router
+9 -4
View File
@@ -78,6 +78,12 @@ impl WorkdirSession for HookedSession {
self.local.capabilities()
}
fn environment(&self) -> workdir::WorkdirSessionEnvironment {
let mut environment = self.local.environment();
environment.capabilities = environment.capabilities.intersection(self.capabilities());
environment
}
async fn authorize_scope_path(
&self,
request: WorkdirScopeAuthorizationRequest,
@@ -201,11 +207,10 @@ async fn checkout_wip_file_operations_share_core_and_keep_prior_read_boundary()
let dir = TempDir::new().unwrap();
std::fs::write(dir.path().join("text.txt"), "旧 旧\n").unwrap();
let router = Arc::new(WorkdirSessionRouter::new());
let hooked = HookedSession::new(&dir);
assert_eq!(hooked.environment(), hooked.local.environment());
router
.attach(
WorkdirAttachmentAlias::new("main").unwrap(),
HookedSession::new(&dir),
)
.attach(WorkdirAttachmentAlias::new("main").unwrap(), hooked)
.unwrap();
let provider = provider(router);
let route = format!("{}/text.txt", checkout_root("main"));
File diff suppressed because it is too large Load Diff
@@ -65,7 +65,6 @@ async fn prepared(
WorkerBootstrapLayout::Direct {
runtime_base: temp.path().join("runtime"),
},
WorkerControllerTransport::InProcess,
)
.with_model_client(client)
.prepare()
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+1
View File
@@ -4,6 +4,7 @@
//! same descriptor-approved registry path used by feature modules. They are not
//! an external plugin-loading surface.
mod backend_diagnostic;
pub mod backend_job_result;
pub(crate) mod bash;
@@ -0,0 +1,195 @@
//! Internal adapter diagnostics are independent of public failure classifications.
//!
//! Only closed vocabularies, numeric status and opaque correlation tokens cross this boundary.
//! In particular, a transport error's Display/Debug and server-controlled text are never recorded.
use sha2::{Digest, Sha256};
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) enum Adapter {
Drive,
WorkspaceConfig,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) enum Stage {
Request,
Response,
Completion,
Reconcile,
Attach,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(super) enum Reason {
Deadline,
TaskJoin,
TransportUnavailable,
InvalidPath,
Communication,
Decode,
RequestSize,
ResponseSize,
ResponseLength,
Identity,
InvalidMetadata,
BackendRejected,
AccessCeiling,
AttachConversion,
BackendUnknown,
UnrecognizedResponse,
NotCommittedSnapshot,
IncompleteReceipt,
Reconciled,
}
/// One logical call, including its original-request reconciliation. Request identities are hashed
/// rather than logged verbatim: callers and responses need not contain safe log text.
#[derive(Clone)]
pub(super) struct Diagnostic {
adapter: Adapter,
operation: &'static str,
mutation: bool,
correlation: String,
dispatch: tracing::Dispatch,
stage_override: Option<Stage>,
}
impl Diagnostic {
pub(super) fn new(
adapter: Adapter,
operation: &'static str,
mutation: bool,
request_id: Option<&str>,
) -> Self {
Self {
adapter,
operation,
mutation,
correlation: request_id
.map(|id| {
let hash: String = Sha256::digest(id.as_bytes())
.iter()
.map(|b| format!("{b:02x}"))
.collect();
format!("request:{hash}")
})
.unwrap_or_else(|| uuid::Uuid::now_v7().to_string()),
dispatch: tracing::dispatcher::get_default(Clone::clone),
stage_override: None,
}
}
pub(super) fn at_stage(&self, stage: Stage) -> Self {
let mut diagnostic = self.clone();
diagnostic.stage_override = Some(stage);
diagnostic
}
pub(super) fn record(&self, stage: Stage, reason: Reason, status: Option<u16>) {
let stage = self.stage_override.unwrap_or(stage);
tracing::dispatcher::with_default(&self.dispatch, || {
tracing::warn!(
target: "worker::backend_diagnostic",
adapter = ?self.adapter,
operation = self.operation,
mutation = self.mutation,
correlation = %self.correlation,
stage = ?stage,
reason = ?reason,
status,
"Workspace adapter diagnostic"
);
});
}
pub(super) fn transport(&self, error: &crate::worker::WorkspaceClientError) {
use crate::worker::WorkspaceClientError;
self.record(
Stage::Request,
match error {
WorkspaceClientError::Unavailable(_) => Reason::TransportUnavailable,
WorkspaceClientError::InvalidPath(_) => Reason::InvalidPath,
WorkspaceClientError::Request(_) => Reason::Communication,
},
None,
);
}
}
#[cfg(test)]
pub(super) mod tests {
use super::{Adapter, Diagnostic, Reason, Stage};
use std::sync::{Arc, Mutex};
use tracing::{Event, Metadata, Subscriber, field::Visit, span};
#[test]
fn unsafe_request_identity_is_correlated_without_logging_its_value() {
let capture = Capture::default();
tracing::dispatcher::with_default(&capture.dispatch(), || {
for id in [
"secret /host/path\ncredential",
"secret /host/path\ncredential",
"different",
] {
Diagnostic::new(Adapter::Drive, "mutate", true, Some(id)).record(
Stage::Request,
Reason::Communication,
None,
);
}
});
let text = capture.text();
for secret in ["secret", "/host/path", "credential", "different"] {
assert!(!text.contains(secret));
}
let correlations: Vec<_> = text
.lines()
.map(|line| {
line.split("correlation=")
.nth(1)
.unwrap()
.split(';')
.next()
.unwrap()
})
.collect();
assert_eq!(correlations.len(), 3);
assert_eq!(correlations[0], correlations[1]);
assert_ne!(correlations[0], correlations[2]);
}
#[derive(Clone, Default)]
pub(crate) struct Capture(pub(crate) Arc<Mutex<Vec<String>>>);
impl Capture {
pub(crate) fn dispatch(&self) -> tracing::Dispatch {
tracing::Dispatch::new(self.clone())
}
pub(crate) fn text(&self) -> String {
self.0.lock().unwrap().join("\n")
}
}
struct Fields(String);
impl Visit for Fields {
fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
use std::fmt::Write;
write!(&mut self.0, "{}={value:?};", field.name()).unwrap();
}
}
impl Subscriber for Capture {
fn enabled(&self, metadata: &Metadata<'_>) -> bool {
metadata.target() == "worker::backend_diagnostic"
}
fn new_span(&self, _: &span::Attributes<'_>) -> span::Id {
span::Id::from_u64(1)
}
fn record(&self, _: &span::Id, _: &span::Record<'_>) {}
fn record_follows_from(&self, _: &span::Id, _: &span::Id) {}
fn event(&self, event: &Event<'_>) {
let mut fields = Fields(String::new());
event.record(&mut fields);
self.0.lock().unwrap().push(fields.0);
}
fn enter(&self, _: &span::Id) {}
fn exit(&self, _: &span::Id) {}
}
}
@@ -11,6 +11,8 @@
//! cases retain an unknown outcome with the original request ID. Never blindly resend with a new ID.
use std::{sync::Arc, time::Duration};
use super::super::backend_diagnostic::{Adapter, Diagnostic, Reason, Stage};
use crate::worker::{
WorkspaceBinaryRequest, WorkspaceClient, WorkspaceRequest, WorkspaceRequestMethod,
};
@@ -23,6 +25,7 @@ const RESPONSE_LIMIT: usize = 2 * 1024 * 1024;
#[derive(Clone)]
pub struct DriveBackend {
client: Arc<dyn WorkspaceClient>,
deadline: Duration,
}
#[derive(Debug, Clone, thiserror::Error)]
@@ -49,7 +52,10 @@ pub enum DriveError {
impl DriveBackend {
pub fn new(client: Arc<dyn WorkspaceClient>) -> Self {
Self { client }
Self {
client,
deadline: DEADLINE,
}
}
pub fn workspace(&self) -> Result<&str, DriveError> {
self.client
@@ -61,15 +67,28 @@ impl DriveBackend {
Ok(format!("/api/w/{}/drive", encode(self.workspace()?)))
}
pub(super) fn validate_entry(&self, entry: &DriveEntry) -> Result<(), DriveError> {
self.validate_ref(&entry.entry)?;
self.validate_entry_at(
entry,
&Diagnostic::new(Adapter::Drive, "validate_entry", false, None),
)
}
fn validate_entry_at(
&self,
entry: &DriveEntry,
diagnostic: &Diagnostic,
) -> Result<(), DriveError> {
self.validate_ref(&entry.entry)
.inspect_err(|_| diagnostic.record(Stage::Completion, Reason::Identity, None))?;
if let Some(parent) = &entry.parent {
self.validate_ref(parent)?;
self.validate_ref(parent)
.inspect_err(|_| diagnostic.record(Stage::Completion, Reason::Identity, None))?;
}
if (entry.last_mutation_id.len() > 128
|| entry.last_mutation_id.chars().any(char::is_control)
|| (entry.parent.is_some() && entry.last_mutation_id.is_empty()))
|| entry.size.is_some_and(|n| n > DRIVE_FILE_MAX_BYTES)
{
diagnostic.record(Stage::Completion, Reason::InvalidMetadata, None);
return Err(DriveError::Unavailable);
}
Ok(())
@@ -90,11 +109,13 @@ impl DriveBackend {
request: WorkspaceRequest,
mutation: bool,
request_id: &str,
diagnostic: &Diagnostic,
) -> Result<T, DriveError> {
let client = self.client.clone();
let deadline = self.deadline;
let response = tokio::time::timeout(
DEADLINE,
tokio::task::spawn_blocking(move || client.execute_with_timeout(request, DEADLINE)),
deadline,
tokio::task::spawn_blocking(move || client.execute_with_timeout(request, deadline)),
)
.await;
let unknown = || {
@@ -105,10 +126,20 @@ impl DriveBackend {
}
};
let response = response
.map_err(|_| unknown())?
.map_err(|_| unknown())?
.map_err(|_| unknown())?;
.map_err(|_| {
diagnostic.record(Stage::Request, Reason::Deadline, None);
unknown()
})?
.map_err(|_| {
diagnostic.record(Stage::Request, Reason::TaskJoin, None);
unknown()
})?
.map_err(|error| {
diagnostic.transport(&error);
unknown()
})?;
if response.body.len() > RESPONSE_LIMIT {
diagnostic.record(Stage::Response, Reason::ResponseSize, None);
return Err(if mutation {
unknown()
} else {
@@ -121,42 +152,57 @@ impl DriveBackend {
response.body.as_bytes(),
mutation,
request_id,
diagnostic,
));
}
serde_json::from_str(&response.body).map_err(|_| unknown())
serde_json::from_str(&response.body).map_err(|_| {
diagnostic.record(Stage::Response, Reason::Decode, None);
unknown()
})
}
async fn get<T: DeserializeOwned + Send + 'static, Q: Serialize>(
&self,
suffix: &str,
suffix: &'static str,
query: &Q,
diagnostic: &Diagnostic,
) -> Result<T, DriveError> {
let query = query_string(query)?;
self.json(
WorkspaceRequest::get(format!("{}{suffix}{query}", self.base()?)),
false,
"",
diagnostic,
)
.await
}
pub async fn capabilities(&self) -> Result<DriveCapabilities, DriveError> {
self.get("/capabilities", &serde_json::json!({})).await
self.get(
"/capabilities",
&serde_json::json!({}),
&Diagnostic::new(Adapter::Drive, "capabilities", false, None),
)
.await
}
pub async fn root(&self) -> Result<DriveEntry, DriveError> {
let diagnostic = Diagnostic::new(Adapter::Drive, "root", false, None);
let entry: DriveEntry = self
.json(
WorkspaceRequest::get(format!("{}/root", self.base()?)),
false,
"",
&diagnostic,
)
.await?;
self.validate_entry(&entry)?;
self.validate_entry_at(&entry, &diagnostic)?;
if entry.kind != DriveEntryKind::Folder || entry.parent.is_some() {
diagnostic.record(Stage::Completion, Reason::InvalidMetadata, None);
return Err(DriveError::Unavailable);
}
Ok(entry)
}
pub async fn metadata(&self, entry: &DriveEntryRef) -> Result<DriveEntry, DriveError> {
self.validate_ref(entry)?;
let diagnostic = Diagnostic::new(Adapter::Drive, "metadata", false, None);
let result: DriveEntry = self
.get(
"/metadata",
@@ -164,10 +210,12 @@ impl DriveBackend {
entry_workspace_id: entry.workspace_id.clone(),
id: entry.node_id.clone(),
},
&diagnostic,
)
.await?;
self.validate_entry(&result)?;
self.validate_entry_at(&result, &diagnostic)?;
if result.entry != *entry {
diagnostic.record(Stage::Completion, Reason::Identity, None);
return Err(DriveError::Unavailable);
}
Ok(result)
@@ -179,6 +227,7 @@ impl DriveBackend {
after: Option<String>,
) -> Result<DriveListResponse, DriveError> {
self.validate_ref(parent)?;
let diagnostic = Diagnostic::new(Adapter::Drive, "list", false, None);
let result: DriveListResponse = self
.get(
"/list",
@@ -188,6 +237,7 @@ impl DriveBackend {
limit,
after,
},
&diagnostic,
)
.await?;
self.validate_page(
@@ -195,33 +245,43 @@ impl DriveBackend {
limit
.unwrap_or(DRIVE_PAGE_MAX_LIMIT)
.min(DRIVE_PAGE_MAX_LIMIT),
&diagnostic,
)?;
if result
.entries
.iter()
.any(|entry| entry.parent.as_ref() != Some(parent))
{
diagnostic.record(Stage::Completion, Reason::Identity, None);
return Err(DriveError::Unavailable);
}
Ok(result)
}
pub async fn search(&self, query: DriveSearchQuery) -> Result<DriveListResponse, DriveError> {
let result: DriveListResponse = self.get("/search", &query).await?;
let diagnostic = Diagnostic::new(Adapter::Drive, "search", false, None);
let result: DriveListResponse = self.get("/search", &query, &diagnostic).await?;
self.validate_page(
&result,
query
.limit
.unwrap_or(DRIVE_SEARCH_MAX_LIMIT)
.min(DRIVE_SEARCH_MAX_LIMIT),
&diagnostic,
)?;
Ok(result)
}
fn validate_page(&self, page: &DriveListResponse, limit: u32) -> Result<(), DriveError> {
fn validate_page(
&self,
page: &DriveListResponse,
limit: u32,
diagnostic: &Diagnostic,
) -> Result<(), DriveError> {
if page.entries.len() > limit as usize {
diagnostic.record(Stage::Completion, Reason::ResponseSize, None);
return Err(DriveError::Limit);
}
for entry in &page.entries {
self.validate_entry(entry)?;
self.validate_entry_at(entry, diagnostic)?;
}
Ok(())
}
@@ -231,6 +291,7 @@ impl DriveBackend {
max_bytes: u32,
) -> Result<DriveReadTextResponse, DriveError> {
self.validate_ref(entry)?;
let diagnostic = Diagnostic::new(Adapter::Drive, "read_text", false, None);
let result: DriveReadTextResponse = self
.get(
"/read-text",
@@ -239,10 +300,16 @@ impl DriveBackend {
id: entry.node_id.clone(),
max_bytes,
},
&diagnostic,
)
.await?;
self.validate_entry(&result.entry)?;
if result.entry.entry != *entry || result.text.len() > max_bytes as usize {
self.validate_entry_at(&result.entry, &diagnostic)?;
if result.entry.entry != *entry {
diagnostic.record(Stage::Completion, Reason::Identity, None);
return Err(DriveError::Unavailable);
}
if result.text.len() > max_bytes as usize {
diagnostic.record(Stage::Completion, Reason::ResponseSize, None);
return Err(DriveError::Unavailable);
}
Ok(result)
@@ -255,6 +322,7 @@ impl DriveBackend {
if total as usize > limit {
return Err(DriveError::Limit);
}
let diagnostic = Diagnostic::new(Adapter::Drive, "read_chunk", false, None);
let mut bytes = Vec::with_capacity(total as usize);
// Even an empty file gets a read bound to the observed committed mutation and current authorization.
loop {
@@ -274,21 +342,42 @@ impl DriveBackend {
max_response_bytes: DRIVE_CHUNK_MAX_BYTES as usize,
};
let client = self.client.clone();
let deadline = self.deadline;
let response = tokio::time::timeout(
DEADLINE,
deadline,
tokio::task::spawn_blocking(move || {
client.execute_binary_with_timeout(request, DEADLINE)
client.execute_binary_with_timeout(request, deadline)
}),
)
.await
.map_err(|_| DriveError::Unavailable)?
.map_err(|_| DriveError::Unavailable)?
.map_err(|_| DriveError::Unavailable)?;
.map_err(|_| {
diagnostic.record(Stage::Request, Reason::Deadline, None);
DriveError::Unavailable
})?
.map_err(|_| {
diagnostic.record(Stage::Request, Reason::TaskJoin, None);
DriveError::Unavailable
})?
.map_err(|error| {
diagnostic.transport(&error);
DriveError::Unavailable
})?;
if response.body.len() > DRIVE_CHUNK_MAX_BYTES as usize {
diagnostic.record(Stage::Response, Reason::ResponseSize, None);
return Err(DriveError::Unavailable);
}
if !(200..300).contains(&response.status) {
return Err(classify(response.status, &response.body, false, ""));
return Err(classify(
response.status,
&response.body,
false,
"",
&diagnostic,
));
}
let expected = total.saturating_sub(offset).min(length) as usize;
if response.body.len() != expected {
diagnostic.record(Stage::Response, Reason::ResponseLength, None);
return Err(DriveError::Unavailable);
}
bytes.extend_from_slice(&response.body);
@@ -301,22 +390,40 @@ impl DriveBackend {
&self,
id: &str,
result: Result<DriveMutationResponse, DriveError>,
diagnostic: &Diagnostic,
) -> Result<DriveMutationResponse, DriveError> {
// A parseable success can still be an invalid completion. Classify it
// before reconciliation so it also gets one original-request inquiry.
let result = result.and_then(|response| {
self.validate_completion(id, &response)
self.validate_completion(id, &response, diagnostic)
.map_err(|_| DriveError::OutcomeUnknown(id.into()))?;
Ok(response)
});
match result {
Err(DriveError::OutcomeUnknown(_)) => match self.status(id).await {
Err(DriveError::OutcomeUnknown(_)) => match self
.status_with_diagnostic(id, &diagnostic.at_stage(Stage::Reconcile))
.await
{
Ok(DriveRequestStatusResponse {
state: DriveRequestState::Committed,
response: Some(response),
..
}) => Ok(response), // status validates both IDs and entry metadata
_ => Err(DriveError::OutcomeUnknown(id.into())),
}) => {
diagnostic.record(Stage::Reconcile, Reason::Reconciled, None);
Ok(response) // status validates both IDs and entry metadata
}
Ok(status) => {
diagnostic.record(
Stage::Reconcile,
match status.state {
DriveRequestState::Uncommitted => Reason::NotCommittedSnapshot,
DriveRequestState::Committed => Reason::IncompleteReceipt,
},
None,
);
Err(DriveError::OutcomeUnknown(id.into()))
}
Err(_) => Err(DriveError::OutcomeUnknown(id.into())),
},
other => other,
}
@@ -325,12 +432,14 @@ impl DriveBackend {
&self,
id: &str,
response: &DriveMutationResponse,
diagnostic: &Diagnostic,
) -> Result<(), DriveError> {
if response.request_id != id {
diagnostic.record(Stage::Completion, Reason::Identity, None);
return Err(DriveError::Unavailable);
}
if let Some(entry) = &response.entry {
self.validate_entry(entry)?;
self.validate_entry_at(entry, diagnostic)?;
}
Ok(())
}
@@ -338,9 +447,11 @@ impl DriveBackend {
&self,
request: DriveMutationRequest,
) -> Result<DriveMutationResponse, DriveError> {
let diagnostic = Diagnostic::new(Adapter::Drive, "mutate", true, Some(&request.request_id));
let body = serde_json::to_string(&request)
.map_err(|_| DriveError::Invalid("invalid mutation".into()))?;
if body.len() > 512 * 1024 {
diagnostic.record(Stage::Request, Reason::RequestSize, None);
return Err(DriveError::Limit);
}
let result = self
@@ -352,16 +463,20 @@ impl DriveBackend {
),
true,
&request.request_id,
&diagnostic,
)
.await;
self.reconcile(&request.request_id, result).await
self.reconcile(&request.request_id, result, &diagnostic)
.await
}
pub async fn upload(
&self,
query: DriveUploadQuery,
bytes: Vec<u8>,
) -> Result<DriveMutationResponse, DriveError> {
let diagnostic = Diagnostic::new(Adapter::Drive, "upload", true, Some(&query.request_id));
if bytes.len() > DRIVE_FILE_MAX_BYTES as usize {
diagnostic.record(Stage::Request, Reason::RequestSize, None);
return Err(DriveError::Limit);
}
let path = format!("{}/upload{}", self.base()?, query_string(&query)?);
@@ -372,29 +487,56 @@ impl DriveBackend {
max_response_bytes: RESPONSE_LIMIT,
};
let client = self.client.clone();
let deadline = self.deadline;
let result = tokio::time::timeout(
DEADLINE,
deadline,
tokio::task::spawn_blocking(move || {
client.execute_binary_with_timeout(request, DEADLINE)
client.execute_binary_with_timeout(request, deadline)
}),
)
.await;
let result = match result {
Ok(Ok(Ok(response))) if response.body.len() > RESPONSE_LIMIT => {
diagnostic.record(Stage::Response, Reason::ResponseSize, None);
Err(DriveError::OutcomeUnknown(query.request_id.clone()))
}
Ok(Ok(Ok(response))) if (200..300).contains(&response.status) => {
serde_json::from_slice(&response.body)
.map_err(|_| DriveError::OutcomeUnknown(query.request_id.clone()))
serde_json::from_slice(&response.body).map_err(|_| {
diagnostic.record(Stage::Response, Reason::Decode, None);
DriveError::OutcomeUnknown(query.request_id.clone())
})
}
Ok(Ok(Ok(response))) => Err(classify(
response.status,
&response.body,
true,
&query.request_id,
&diagnostic,
)),
_ => Err(DriveError::OutcomeUnknown(query.request_id.clone())),
Err(_) => {
diagnostic.record(Stage::Request, Reason::Deadline, None);
Err(DriveError::OutcomeUnknown(query.request_id.clone()))
}
Ok(Err(_)) => {
diagnostic.record(Stage::Request, Reason::TaskJoin, None);
Err(DriveError::OutcomeUnknown(query.request_id.clone()))
}
Ok(Ok(Err(error))) => {
diagnostic.transport(&error);
Err(DriveError::OutcomeUnknown(query.request_id.clone()))
}
};
self.reconcile(&query.request_id, result).await
self.reconcile(&query.request_id, result, &diagnostic).await
}
pub async fn status(&self, request_id: &str) -> Result<DriveRequestStatusResponse, DriveError> {
let diagnostic = Diagnostic::new(Adapter::Drive, "request_status", false, Some(request_id));
self.status_with_diagnostic(request_id, &diagnostic).await
}
async fn status_with_diagnostic(
&self,
request_id: &str,
diagnostic: &Diagnostic,
) -> Result<DriveRequestStatusResponse, DriveError> {
if request_id.is_empty() || request_id.len() > 256 {
return Err(DriveError::Invalid("invalid request ID".into()));
}
@@ -403,13 +545,15 @@ impl DriveBackend {
WorkspaceRequest::get(format!("{}/requests/{}", self.base()?, encode(request_id))),
false,
"",
diagnostic,
)
.await?;
if response.request_id != request_id {
diagnostic.record(Stage::Reconcile, Reason::Identity, None);
return Err(DriveError::Unavailable);
}
if let Some(result) = &response.response {
self.validate_completion(request_id, result)?;
self.validate_completion(request_id, result, diagnostic)?;
}
Ok(response)
}
@@ -463,11 +607,21 @@ fn query_string<T: Serialize>(value: &T) -> Result<String, DriveError> {
format!("?{}", pairs.join("&"))
})
}
fn classify(status: u16, body: &[u8], mutation: bool, id: &str) -> DriveError {
fn classify(
status: u16,
body: &[u8],
mutation: bool,
id: &str,
diagnostic: &Diagnostic,
) -> DriveError {
if let Ok(error) = serde_json::from_slice::<DriveApiError>(body) {
if error.classification == DriveFailureClassification::Unknown {
if error.classification == DriveFailureClassification::Unknown
|| error.code == DriveApiErrorCode::OutcomeUnknown
{
diagnostic.record(Stage::Response, Reason::BackendUnknown, Some(status));
return DriveError::OutcomeUnknown(id.into());
}
diagnostic.record(Stage::Response, Reason::BackendRejected, Some(status));
return match error.code {
DriveApiErrorCode::Denied => DriveError::Denied,
DriveApiErrorCode::NotFound => DriveError::NotFound,
@@ -478,6 +632,7 @@ fn classify(status: u16, body: &[u8], mutation: bool, id: &str) -> DriveError {
DriveApiErrorCode::OutcomeUnknown => DriveError::OutcomeUnknown(id.into()),
};
}
diagnostic.record(Stage::Response, Reason::UnrecognizedResponse, Some(status));
if matches!(status, 401 | 403) {
DriveError::Denied
} else if mutation {
@@ -486,3 +641,7 @@ fn classify(status: u16, body: &[u8], mutation: bool, id: &str) -> DriveError {
DriveError::Unavailable
}
}
#[cfg(test)]
#[path = "backend_tests.rs"]
mod tests;
@@ -0,0 +1,644 @@
//! Production Drive adapter fault injection. Only the Workspace transport boundary is substituted;
//! retained fixture effects and original-request inquiries are inspected independently of replies.
//! The HTTP case additionally connects the adapter to reqwest and an isolated loopback router.
use super::*;
use crate::feature::builtin::backend_diagnostic::{Reason, tests::Capture};
use crate::worker::{WorkspaceBinaryResponse, WorkspaceClientError, WorkspaceResponse};
use std::sync::{Condvar, Mutex};
use tracing::instrument::WithSubscriber;
const SECRET: &str = "secret-token /private/drive credential=unsafe";
const ID: &str = "host-request-1";
fn entry() -> DriveEntry {
DriveEntry {
entry: DriveEntryRef {
workspace_id: "workspace".into(),
node_id: "2".into(),
},
parent: Some(DriveEntryRef {
workspace_id: "workspace".into(),
node_id: "1".into(),
}),
name: SECRET.into(),
kind: DriveEntryKind::File,
last_mutation_id: "old-request".into(),
size: Some(5),
content_type: Some("text/plain".into()),
updated_by: SECRET.into(),
updated_at: SECRET.into(),
latest_url: SECRET.into(),
}
}
fn mutation() -> DriveMutationRequest {
DriveMutationRequest {
request_id: ID.into(),
mutation: DriveMutation::UpdateText {
id: entry().entry,
expected_mutation_id: "old-request".into(),
text: SECRET.into(),
content_type: "text/plain".into(),
},
}
}
fn json(value: impl Serialize) -> WorkspaceResponse {
WorkspaceResponse {
status: 200,
body: serde_json::to_string(&value).unwrap(),
}
}
#[derive(Clone, Copy, Debug)]
enum Fault {
Communication,
Unavailable,
InvalidPath,
Decode,
Size,
Length,
Identity,
Metadata,
Rejected,
Unknown,
Unrecognized,
Incomplete,
Panic,
Deadline,
None,
}
#[derive(Debug, Default)]
struct State {
mutations: usize,
saved: Option<String>,
requests: Vec<WorkspaceRequest>,
binary: Vec<WorkspaceBinaryRequest>,
released: bool,
started: bool,
finished: bool,
}
#[derive(Debug)]
struct Client {
fault: Fault,
receipt: Fault,
state: Mutex<State>,
changed: Condvar,
started: tokio::sync::Notify,
}
impl Client {
fn new(fault: Fault, receipt: Fault) -> Arc<Self> {
Arc::new(Self {
fault,
receipt,
state: Mutex::new(State::default()),
changed: Condvar::new(),
started: tokio::sync::Notify::new(),
})
}
fn fail(
&self,
fault: Fault,
success: WorkspaceResponse,
) -> Result<WorkspaceResponse, WorkspaceClientError> {
match fault {
Fault::Communication => Err(WorkspaceClientError::Request(SECRET.into())),
Fault::Unavailable => Err(WorkspaceClientError::Unavailable(SECRET.into())),
Fault::InvalidPath => Err(WorkspaceClientError::InvalidPath(SECRET.into())),
Fault::Panic => panic!("injected adapter task panic"),
Fault::Decode => Ok(WorkspaceResponse {
status: 200,
body: SECRET.into(),
}),
Fault::Size => Ok(WorkspaceResponse {
status: 200,
body: SECRET.repeat(RESPONSE_LIMIT / SECRET.len() + 1),
}),
Fault::Rejected | Fault::Unknown => Ok(WorkspaceResponse {
status: 409,
body: serde_json::to_string(&DriveApiError {
code: DriveApiErrorCode::Conflict,
message: SECRET.into(),
classification: if matches!(fault, Fault::Unknown) {
DriveFailureClassification::Unknown
} else {
DriveFailureClassification::NotCommitted
},
})
.unwrap(),
}),
Fault::Unrecognized => Ok(WorkspaceResponse {
status: 503,
body: SECRET.into(),
}),
_ => Ok(success),
}
}
fn release(&self) {
self.state.lock().unwrap().released = true;
self.changed.notify_all();
}
fn wait(&self, finished: bool) {
let (state, timeout) = self
.changed
.wait_timeout_while(self.state.lock().unwrap(), Duration::from_secs(5), |s| {
if finished { !s.finished } else { !s.started }
})
.unwrap();
assert!(!timeout.timed_out());
assert!(if finished {
state.finished
} else {
state.started
});
}
}
struct Release(Arc<Client>);
impl Drop for Release {
fn drop(&mut self) {
self.0.release();
}
}
impl WorkspaceClient for Client {
fn workspace_id(&self) -> Option<&str> {
Some("workspace")
}
fn kind(&self) -> &str {
"drive-diagnostic-fixture"
}
fn is_available(&self) -> bool {
true
}
fn execute(
&self,
request: WorkspaceRequest,
) -> Result<WorkspaceResponse, WorkspaceClientError> {
let is_status = request.path.contains("/requests/");
let fault = if is_status { self.receipt } else { self.fault };
let mutation = request.method == WorkspaceRequestMethod::Post;
{
let mut state = self.state.lock().unwrap();
if mutation
&& !matches!(
fault,
Fault::Rejected | Fault::InvalidPath | Fault::Unavailable
)
{
let input: DriveMutationRequest =
serde_json::from_str(request.body.as_ref().unwrap()).unwrap();
assert_eq!(input, super::tests::mutation());
state.mutations += 1;
if let DriveMutation::UpdateText { text, .. } = input.mutation {
state.saved = Some(text);
}
}
state.requests.push(request);
state.started = true;
self.started.notify_one();
self.changed.notify_all();
if matches!(fault, Fault::Deadline) {
let (_state, timeout) = self
.changed
.wait_timeout_while(state, Duration::from_secs(5), |s| !s.released)
.unwrap();
assert!(
!timeout.timed_out(),
"transport must be explicitly released"
);
}
}
let mut metadata = entry();
if matches!(fault, Fault::Identity) {
metadata.entry.node_id = "3".into();
}
if matches!(fault, Fault::Metadata) {
metadata.size = Some(DRIVE_FILE_MAX_BYTES + 1);
}
let completion = DriveMutationResponse {
request_id: if matches!(fault, Fault::Identity) {
"other-request"
} else {
ID
}
.into(),
entry: Some(metadata.clone()),
};
let value = if is_status {
json(DriveRequestStatusResponse {
request_id: if matches!(fault, Fault::Identity) {
"other-request"
} else {
ID
}
.into(),
state: if matches!(
fault,
Fault::None | Fault::Identity | Fault::Metadata | Fault::Incomplete
) {
DriveRequestState::Committed
} else {
DriveRequestState::Uncommitted
},
response: if matches!(fault, Fault::None | Fault::Identity | Fault::Metadata) {
Some(completion)
} else {
None
},
})
} else if mutation {
json(completion)
} else {
json(metadata)
};
if !is_status {
self.state.lock().unwrap().finished = true;
}
self.changed.notify_all();
self.fail(fault, value)
}
fn execute_binary(
&self,
request: WorkspaceBinaryRequest,
) -> Result<WorkspaceBinaryResponse, WorkspaceClientError> {
let upload = request.method == WorkspaceRequestMethod::Put;
{
let mut state = self.state.lock().unwrap();
if upload {
state.mutations += 1;
state.saved = Some(SECRET.into());
}
state.binary.push(request);
}
let success = if upload {
json(DriveMutationResponse {
request_id: ID.into(),
entry: Some(entry()),
})
} else {
WorkspaceResponse {
status: 200,
body: if matches!(self.fault, Fault::Length) {
"x"
} else {
"12345"
}
.into(),
}
};
self.fail(self.fault, success)
.map(|r| WorkspaceBinaryResponse {
status: r.status,
body: r.body.into_bytes(),
})
}
}
fn safe(capture: &Capture, reason: Reason, public: &str) {
let text = capture.text();
assert!(text.contains(&format!("reason={reason:?}")), "{text}");
assert!(text.contains("adapter=Drive"), "{text}");
assert!(text.contains("correlation="), "{text}");
for secret in [
SECRET,
"secret-token",
"/private/drive",
"credential=unsafe",
] {
assert!(!text.contains(secret), "internal diagnostic leaked content");
assert!(!public.contains(secret), "public error leaked content");
}
}
fn reason(fault: Fault) -> Reason {
match fault {
Fault::Communication => Reason::Communication,
Fault::Unavailable => Reason::TransportUnavailable,
Fault::InvalidPath => Reason::InvalidPath,
Fault::Decode => Reason::Decode,
Fault::Size => Reason::ResponseSize,
Fault::Length => Reason::ResponseLength,
Fault::Identity => Reason::Identity,
Fault::Metadata => Reason::InvalidMetadata,
Fault::Rejected => Reason::BackendRejected,
Fault::Unknown => Reason::BackendUnknown,
Fault::Unrecognized => Reason::UnrecognizedResponse,
Fault::Panic => Reason::TaskJoin,
Fault::Deadline => Reason::Deadline,
Fault::Incomplete => Reason::IncompleteReceipt,
Fault::None => panic!("no fault"),
}
}
#[tokio::test]
async fn production_read_failures_retain_public_classification_and_safe_cause() {
for fault in [
Fault::Communication,
Fault::Unavailable,
Fault::InvalidPath,
Fault::Decode,
Fault::Size,
Fault::Identity,
Fault::Metadata,
Fault::Rejected,
Fault::Unknown,
Fault::Unrecognized,
Fault::Panic,
] {
let client = Client::new(fault, Fault::None);
let backend = DriveBackend::new(client.clone());
let capture = Capture::default();
let error = backend
.metadata(&entry().entry)
.with_subscriber(capture.dispatch())
.await
.unwrap_err();
match fault {
Fault::Size => assert!(matches!(error, DriveError::Limit)),
Fault::Rejected => assert!(matches!(error, DriveError::Conflict)),
Fault::Unknown => assert!(matches!(error, DriveError::OutcomeUnknown(_))),
_ => assert!(matches!(error, DriveError::Unavailable)),
}
safe(&capture, reason(fault), &error.to_string());
let state = client.state.lock().unwrap();
assert_eq!(state.requests.len(), 1);
assert_eq!(state.mutations, 0);
}
}
#[tokio::test]
async fn production_mutation_loss_reconciles_only_original_identity_and_never_resends() {
for fault in [
Fault::Communication,
Fault::Decode,
Fault::Size,
Fault::Identity,
Fault::Metadata,
Fault::Unknown,
Fault::Unrecognized,
Fault::Panic,
] {
for receipt in [
Fault::None,
Fault::Identity,
Fault::Metadata,
Fault::Decode,
Fault::Communication,
Fault::Unrecognized,
Fault::Length,
Fault::Incomplete,
] {
let client = Client::new(fault, receipt);
let backend = DriveBackend::new(client.clone());
let capture = Capture::default();
let result = backend
.mutate(mutation())
.with_subscriber(capture.dispatch())
.await;
if matches!(receipt, Fault::None) {
assert_eq!(result.unwrap().request_id, ID);
} else {
assert!(matches!(result, Err(DriveError::OutcomeUnknown(ref id)) if id == ID));
}
safe(&capture, reason(fault), "");
let text = capture.text();
if matches!(receipt, Fault::Length | Fault::Incomplete) {
safe(
&capture,
if matches!(receipt, Fault::Length) {
Reason::NotCommittedSnapshot
} else {
Reason::IncompleteReceipt
},
"",
);
}
assert!(text.contains("stage=Reconcile"), "{text}");
let correlations: std::collections::BTreeSet<_> = text
.lines()
.filter_map(|l| l.split("correlation=").nth(1))
.filter_map(|l| l.split(';').next())
.collect();
assert_eq!(
correlations.len(),
1,
"original loss and inquiry must correlate"
);
assert!(
!text.contains(ID),
"request identity must be opaque in logs"
);
let state = client.state.lock().unwrap();
assert_eq!(state.mutations, 1);
assert_eq!(state.saved.as_deref(), Some(SECRET));
assert_eq!(state.requests.len(), 2);
assert_eq!(state.requests[1].method, WorkspaceRequestMethod::Get);
assert!(state.requests[1].path.ends_with(&format!("/requests/{ID}")));
}
}
}
#[tokio::test]
async fn production_rejection_has_no_effect_or_status_inquiry() {
let client = Client::new(Fault::Rejected, Fault::None);
let backend = DriveBackend::new(client.clone());
let capture = Capture::default();
let error = backend
.mutate(mutation())
.with_subscriber(capture.dispatch())
.await
.unwrap_err();
assert!(matches!(error, DriveError::Conflict));
safe(&capture, Reason::BackendRejected, &error.to_string());
let state = client.state.lock().unwrap();
assert_eq!(state.mutations, 0);
assert_eq!(state.requests.len(), 1);
}
#[tokio::test]
async fn production_binary_length_transport_and_upload_loss_are_diagnosed() {
for fault in [
Fault::Length,
Fault::Communication,
Fault::Unavailable,
Fault::Panic,
] {
let client = Client::new(fault, Fault::None);
let backend = DriveBackend::new(client.clone());
let capture = Capture::default();
let error = backend
.bytes(&entry(), 5)
.with_subscriber(capture.dispatch())
.await
.unwrap_err();
assert!(matches!(error, DriveError::Unavailable));
safe(&capture, reason(fault), &error.to_string());
assert_eq!(client.state.lock().unwrap().binary.len(), 1);
}
for fault in [Fault::Decode, Fault::Communication, Fault::Size] {
let client = Client::new(fault, Fault::Unrecognized);
let backend = DriveBackend::new(client.clone());
let capture = Capture::default();
let query = DriveUploadQuery {
operation: DriveUploadOperation::Update,
request_id: ID.into(),
entry_workspace_id: "workspace".into(),
parent_id: None,
name: None,
id: Some("2".into()),
expected_mutation_id: Some("old-request".into()),
content_type: "text/plain".into(),
size: 5,
sha256: "hash".into(),
};
let error = backend
.upload(query, b"12345".to_vec())
.with_subscriber(capture.dispatch())
.await
.unwrap_err();
assert!(matches!(error, DriveError::OutcomeUnknown(ref id) if id == ID));
safe(&capture, reason(fault), &error.to_string());
let state = client.state.lock().unwrap();
assert_eq!(state.mutations, 1);
assert_eq!(state.binary.len(), 1);
assert_eq!(state.requests.len(), 1);
assert!(state.requests[0].path.ends_with(&format!("/requests/{ID}")));
}
}
#[tokio::test]
async fn production_deadline_does_not_cancel_effect_or_infer_uncommitted_from_snapshot() {
for mutation_call in [false, true] {
let client = Client::new(Fault::Deadline, Fault::Unrecognized);
let release = Release(client.clone());
let mut backend = DriveBackend::new(client.clone());
backend.deadline = Duration::from_secs(1);
let capture = Capture::default();
let request = async {
if mutation_call {
backend.mutate(mutation()).await.map(|_| ())
} else {
backend.metadata(&entry().entry).await.map(|_| ())
}
}
.with_subscriber(capture.dispatch());
tokio::pin!(request);
tokio::select! {
biased;
_ = client.started.notified() => {},
_ = &mut request => panic!("request must start before timeout"),
}
{
let state = client.state.lock().unwrap();
assert_eq!(state.mutations, usize::from(mutation_call));
assert!(!state.released && !state.finished);
}
let result = request.await;
let error = result.unwrap_err();
assert!(if mutation_call {
matches!(error, DriveError::OutcomeUnknown(ref id) if id == ID)
} else {
matches!(error, DriveError::Unavailable)
});
safe(&capture, Reason::Deadline, &error.to_string());
drop(release);
let waiting = client.clone();
tokio::task::spawn_blocking(move || waiting.wait(true))
.await
.unwrap();
let state = client.state.lock().unwrap();
assert_eq!(state.mutations, usize::from(mutation_call));
assert_eq!(state.saved.is_some(), mutation_call);
assert_eq!(state.requests.len(), if mutation_call { 2 } else { 1 });
}
}
#[tokio::test]
async fn http_reply_loss_keeps_original_receipt_and_retained_effect_without_reexecution() {
use axum::{
Router,
extract::State as HttpState,
routing::{get, post},
};
#[derive(Default)]
struct Stored {
updates: usize,
content: String,
ids: Vec<String>,
}
struct Server(tokio::task::JoinHandle<()>);
impl Drop for Server {
fn drop(&mut self) {
self.0.abort();
}
}
let stored = Arc::new(Mutex::new(Stored::default()));
let router = Router::new()
.route("/api/w/workspace/drive/metadata", get(|| async { SECRET }))
.route(
"/api/w/workspace/drive/mutate",
post(
|HttpState(state): HttpState<Arc<Mutex<Stored>>>,
body: axum::body::Bytes| async move {
let request: DriveMutationRequest = serde_json::from_slice(&body).unwrap();
let mut state = state.lock().unwrap();
state.ids.push(request.request_id);
if let DriveMutation::UpdateText {
text,
expected_mutation_id,
..
} = request.mutation
{
assert_eq!(expected_mutation_id, "old-request");
state.content = text;
state.updates += 1;
} else {
panic!("unexpected mutation");
}
// The effect is retained; only the success response is malformed.
SECRET
},
),
)
.route(
"/api/w/workspace/drive/requests/{id}",
get(
|HttpState(state): HttpState<Arc<Mutex<Stored>>>,
axum::extract::Path(id): axum::extract::Path<String>| async move {
let state = state.lock().unwrap();
assert_eq!(state.ids, [id.clone()]);
axum::Json(DriveRequestStatusResponse {
request_id: id.clone(),
state: DriveRequestState::Committed,
response: Some(DriveMutationResponse {
request_id: id,
entry: Some(entry()),
}),
})
},
),
)
.with_state(stored.clone());
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let address = listener.local_addr().unwrap();
let server = Server(tokio::spawn(async move {
axum::serve(listener, router).await.unwrap();
}));
let client = Arc::new(crate::worker::TestWorkspaceHttpClient::new(
"workspace",
format!("http://{address}"),
));
let backend = DriveBackend::new(client);
let capture = Capture::default();
let error = backend
.metadata(&entry().entry)
.with_subscriber(capture.dispatch())
.await
.unwrap_err();
assert!(matches!(error, DriveError::Unavailable));
safe(&capture, Reason::Decode, &error.to_string());
let capture = Capture::default();
let response = backend
.mutate(mutation())
.with_subscriber(capture.dispatch())
.await
.unwrap();
assert_eq!(response.request_id, ID);
safe(&capture, Reason::Decode, "");
assert!(capture.text().contains("reason=Reconciled"));
let state = stored.lock().unwrap();
assert_eq!(state.updates, 1);
assert_eq!(state.content, SECRET);
assert_eq!(state.ids, [ID]);
drop(state);
drop(server);
}
@@ -26,7 +26,7 @@ use flow::{
use manifest::{Scope, WorkerManifest};
use schemars::JsonSchema;
use serde::Deserialize;
use session_store::{SegmentId, SessionId, Store, collect_state};
use session_store::{SegmentId, SessionId, Store};
use uuid::Uuid;
use crate::feature::builtin::session_explore::{SessionExploreFeature, SessionExploreState};
@@ -469,10 +469,15 @@ where
.store
.read_all(self.session_id, self.segment_id)
.map_err(|error| format!("read committed parent session: {error}"))?;
let restored = collect_state(&entries);
Ok(SessionCapture::new(
let history = crate::session_history::restore_history_entries(
self.session_id,
self.segment_id,
&entries,
)?;
Ok(SessionCapture::from_complete_history(
self.session_id.to_string(),
self.segment_id.to_string(),
restored.history,
history,
))
}
}
+136 -16
View File
@@ -59,6 +59,53 @@ impl JournalReadStatus {
}
}
// Only bounded classifications cross into diagnostics. Host messages/codes and
// io::Error payloads can contain Journal text, credentials, proofs or host paths.
// These diagnostics belong to the Feature, not persisted/model-visible input.
enum JournalCompactionFailure {
Connection(std::io::ErrorKind),
SnapshotConflict,
SnapshotInvalidArgument,
SnapshotUnavailable,
Serialization,
ByteBudget,
}
impl JournalCompactionFailure {
fn snapshot(error: SubjektivHostError) -> Self {
match error {
SubjektivHostError::Conflict { .. } => Self::SnapshotConflict,
SubjektivHostError::InvalidArgument(_) => Self::SnapshotInvalidArgument,
SubjektivHostError::Unavailable(_) => Self::SnapshotUnavailable,
}
}
fn stage(&self) -> &'static str {
match self {
Self::Connection(_) => "connection",
Self::SnapshotConflict | Self::SnapshotInvalidArgument | Self::SnapshotUnavailable => {
"snapshot"
}
Self::Serialization => "serialization",
Self::ByteBudget => "byte_budget",
}
}
fn reason(&self) -> &'static str {
match self {
Self::Connection(std::io::ErrorKind::PermissionDenied) => "permission_denied",
Self::Connection(std::io::ErrorKind::InvalidInput) => "invalid_input",
Self::Connection(std::io::ErrorKind::NotConnected) => "not_connected",
Self::Connection(_) => "io_error",
Self::SnapshotConflict => "host_conflict",
Self::SnapshotInvalidArgument => "host_invalid_argument",
Self::SnapshotUnavailable => "host_unavailable",
Self::Serialization => "json_serialization_failed",
Self::ByteBudget => "startup_max_bytes_exceeded",
}
}
}
#[derive(Debug, Clone, Serialize, Deserialize)]
#[serde(deny_unknown_fields)]
struct JournalCompactionInput {
@@ -203,33 +250,46 @@ pub(crate) fn freeze_compaction_journal(
.and_then(|connection| connection.host.workspace_id())
});
let read = match connection {
Ok(Some(connection)) if connection.host.journal_compaction_granted() => {
Some(connection.journal_compaction_context().map_err(|_| ()))
}
Ok(Some(connection)) if connection.host.journal_compaction_granted() => Some(
connection
.journal_compaction_context()
.map_err(JournalCompactionFailure::snapshot),
),
Ok(_) => None,
Err(_) => Some(Err(())),
Err(error) => Some(Err(JournalCompactionFailure::Connection(error.kind()))),
};
let snapshot = read.as_ref().and_then(|read| read.as_ref().ok()).cloned();
let (status, json) = match read {
None => (Disabled, String::new()),
let prepared = match read {
None => Ok((Disabled, String::new())),
Some(Ok(snapshot)) => {
let status = if snapshot.journals.is_empty() {
Absent
} else {
Available
};
match serde_json::to_string(&snapshot) {
Ok(json) if json.len() <= subjektiv::journal::startup::STARTUP_MAX_BYTES => {
(status, json)
}
_ => (Unavailable, String::new()),
}
// The current snapshot has only derived, JSON-safe fields (strings,
// integers, vectors, enums and a UUID). There is no known serialization
// failure fixture; retain this guard if the domain type changes.
serde_json::to_string(&snapshot)
.map_err(|_| JournalCompactionFailure::Serialization)
.and_then(|json| {
if json.len() <= subjektiv::journal::startup::STARTUP_MAX_BYTES {
Ok((status, json))
} else {
Err(JournalCompactionFailure::ByteBudget)
}
})
}
Some(Err(())) => (Unavailable, String::new()),
Some(Err(failure)) => Err(failure),
};
if status == Unavailable {
tracing::warn!("Journal compaction snapshot unavailable; continuing without Journal");
}
let (status, json) = prepared.unwrap_or_else(|failure| {
tracing::warn!(
stage = failure.stage(),
reason = failure.reason(),
"Journal compaction snapshot unavailable; continuing without Journal"
);
(Unavailable, String::new())
});
use sha2::{Digest, Sha256};
let snapshot_sha256: Option<String> = (!json.is_empty()).then(|| {
Sha256::digest(json.as_bytes())
@@ -726,12 +786,72 @@ fn tool_schema(name: &str) -> Value {
}
}
#[cfg(test)]
#[path = "../../../tests/common/diagnostics.rs"]
mod diagnostic_test_support;
#[cfg(test)]
mod tests {
use super::*;
use std::sync::Mutex;
use subjektiv::journal::JournalReceipt;
#[test]
fn compaction_connection_failure_keeps_safe_reason_without_exposing_error_or_changing_input() {
let prompts = crate::prompt::catalog::PromptCatalog::builtins_only().unwrap();
for (kind, reason) in [
(std::io::ErrorKind::PermissionDenied, "permission_denied"),
(std::io::ErrorKind::InvalidInput, "invalid_input"),
(std::io::ErrorKind::NotConnected, "not_connected"),
(std::io::ErrorKind::Other, "io_error"),
] {
let logs = diagnostic_test_support::CapturedWarnings::default();
let input = tracing::subscriber::with_default(logs.subscriber(), || {
freeze_compaction_journal(
Err(std::io::Error::new(
kind,
"PRIVATE_JOURNAL_BODY /private/host/path proof=PRIVATE_PROOF credential=PRIVATE_CREDENTIAL raw_error=PRIVATE_ERROR",
)),
&prompts,
)
.unwrap()
});
let log = logs.text();
assert!(log.contains("stage=\"connection\""), "{log}");
assert!(log.contains(&format!("reason=\"{reason}\"")), "{log}");
assert_eq!(log.matches("continuing without Journal").count(), 1);
assert!(!log.contains("PRIVATE_"));
assert!(!log.contains("/private/host/path"));
assert!(input.body().contains("Journal read status: unavailable."));
let persisted = serde_json::to_string(&input).unwrap();
assert!(!persisted.contains("PRIVATE_"));
assert!(!persisted.contains("/private/host/path"));
assert!(!persisted.contains(reason));
let payload: JournalCompactionInput = input
.metadata()
.feature_payload()
.unwrap()
.decode("journal:compaction", 1)
.unwrap();
assert!(payload.snapshot.is_none());
assert!(payload.snapshot_sha256.is_none());
assert_eq!(
payload.prompt_provenance.source_digest,
prompts.projection().source_digest
);
assert_eq!(
payload.prompt_provenance.projection_digest,
prompts.projection().catalog_digest
);
}
let logs = diagnostic_test_support::CapturedWarnings::default();
let input = tracing::subscriber::with_default(logs.subscriber(), || {
freeze_compaction_journal(Ok(None), &prompts).unwrap()
});
assert!(input.body().contains("Journal read status: disabled."));
assert!(logs.text().is_empty());
}
#[derive(Default)]
struct Host {
requests: Mutex<Vec<JournalJobRequest>>,
@@ -226,6 +226,7 @@ pub struct WorkspaceAttachedWorkdirSession {
target_workdir: String,
workdir: Workdir,
capabilities: WorkdirSessionCapabilities,
environment: workdir::WorkdirSessionEnvironment,
}
impl WorkspaceAttachedWorkdirSession {
@@ -249,17 +250,38 @@ impl WorkspaceAttachedWorkdirSession {
)
}
/// Bind a known root-coordinate attachment. Opened provider sessions must
/// instead pass their metadata to [`Self::handle_for_workdir_with_environment`].
pub fn handle_for_workdir_with_capabilities(
client: Arc<dyn WorkspaceClient>,
target_workdir: impl Into<String>,
workdir_id: impl Into<String>,
capabilities: WorkdirSessionCapabilities,
) -> WorkdirSessionHandle {
Self::handle_for_workdir_with_environment(
client,
target_workdir,
workdir_id,
capabilities,
workdir::WorkdirSessionEnvironment::root(capabilities),
)
}
/// Bind provider-authored coordinates without allowing metadata to expand
/// the Backend attachment capability ceiling.
pub fn handle_for_workdir_with_environment(
client: Arc<dyn WorkspaceClient>,
target_workdir: impl Into<String>,
workdir_id: impl Into<String>,
capabilities: WorkdirSessionCapabilities,
environment: workdir::WorkdirSessionEnvironment,
) -> WorkdirSessionHandle {
Arc::new(Self {
client,
target_workdir: target_workdir.into(),
workdir: Workdir::new(workdir_id),
capabilities,
capabilities: capabilities.intersection(environment.capabilities),
environment,
})
}
@@ -291,6 +313,11 @@ impl WorkspaceAttachedWorkdirSession {
.execute(request)
.map_err(workspace_workdir_error)?;
if !response.is_success() {
if let Ok(error) =
serde_json::from_str::<server_api::WorkdirOperationApiError>(&response.body)
{
notify_workdir_audit_failures(&error.audit_failures);
}
return Err(
serde_json::from_str::<workdir::http::WorkdirTransportError>(&response.body)
.map(workdir::http::WorkdirTransportError::into_workdir_error)
@@ -302,11 +329,14 @@ impl WorkspaceAttachedWorkdirSession {
}),
);
}
serde_json::from_str(&response.body).map_err(|error| {
WorkdirError::Transport(format!(
"failed to decode Workspace Workdir operation result: {error}"
))
})
let response: workdir::http::WorkdirSessionOperationResponse =
serde_json::from_str(&response.body).map_err(|_| {
WorkdirError::Transport(
"failed to decode Workspace Workdir operation result".into(),
)
})?;
notify_workdir_audit_failures(&response.audit_failures);
Ok(response.result)
}
fn mismatch(expected: &str) -> WorkdirError {
@@ -316,6 +346,27 @@ impl WorkspaceAttachedWorkdirSession {
}
}
// Notification is separate from the operation return: callers retain successful command
// handles, checkout observations and original failures, and must not resend for audit loss.
fn notify_workdir_audit_failures(failures: &[workdir::http::WorkdirAuditFailure]) {
use sha2::Digest;
for failure in failures {
// Response strings are untrusted. Never copy them into the receiving process log.
let operation_id_hash = sha2::Sha256::digest(failure.operation_id.as_bytes())
.iter()
.map(|byte| format!("{byte:02x}"))
.collect::<String>();
tracing::error!(
event = "external_workdir_audit_failed",
operation_id_hash,
audit_stage = ?failure.stage,
audit_reason = ?failure.reason,
operation_outcome = ?failure.operation_outcome,
"External Workdir audit persistence failed; operation result is independent; do not resend"
);
}
}
fn workspace_workdir_error(error: WorkspaceClientError) -> WorkdirError {
match error {
WorkspaceClientError::Unavailable(message) => WorkdirError::Unavailable(message),
@@ -333,6 +384,12 @@ impl WorkdirSession for WorkspaceAttachedWorkdirSession {
self.capabilities
}
fn environment(&self) -> workdir::WorkdirSessionEnvironment {
let mut environment = self.environment.clone();
environment.capabilities = environment.capabilities.intersection(self.capabilities());
environment
}
async fn authorize_scope_path(
&self,
request: WorkdirScopeAuthorizationRequest,
@@ -655,11 +712,12 @@ impl WorkspaceHttpWorkdirBackend {
let response = self.attach_response(alias, working_directory_id)?;
if let Err(error) = self.session_router.attach(
alias_key.clone(),
WorkspaceAttachedWorkdirSession::handle_for_workdir_with_capabilities(
WorkspaceAttachedWorkdirSession::handle_for_workdir_with_environment(
self.client.clone(),
alias,
working_directory_id,
response.capabilities,
response.attachment.capabilities,
response.environment.clone(),
),
) {
let _ = self.detach(alias);
@@ -677,9 +735,9 @@ impl WorkspaceHttpWorkdirBackend {
&self,
alias: &str,
working_directory_id: &str,
) -> Result<WorkdirAttachmentResponse, ToolError> {
) -> Result<WorkdirSessionAttachmentResponse, ToolError> {
let workspace_id = encode_path_segment(self.workspace_id()?);
self.execute_json::<WorkdirAttachmentResponse>(WorkspaceRequest::json(
self.execute_json::<WorkdirSessionAttachmentResponse>(WorkspaceRequest::json(
WorkspaceRequestMethod::Post,
format!("/api/w/{workspace_id}/workers/self/workdir-attachments"),
serde_json::to_string(&WorkdirAttachRequest {
@@ -1163,6 +1221,13 @@ struct WorkdirAttachmentResponse {
attached: bool,
}
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
struct WorkdirSessionAttachmentResponse {
#[serde(flatten)]
attachment: WorkdirAttachmentResponse,
environment: workdir::WorkdirSessionEnvironment,
}
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
struct WorkdirDeleteInput {
@@ -1473,6 +1538,7 @@ mod tests {
"alias": "checkout",
"working_directory_id": "wd-created",
"capabilities": {"bits": 63},
"environment": workdir::WorkdirSessionEnvironment::root(WorkdirSessionCapabilities::ALL),
"attached": true
})),
response(json!({
@@ -1653,6 +1719,67 @@ mod tests {
}
}
#[tokio::test]
async fn attached_session_keeps_operation_results_when_audit_persistence_failed() {
let failure = json!({
"operation_id":"op-safe-id", "stage":"response_received",
"reason":"audit_store_unavailable", "operation_outcome":{"kind":"succeeded"}
});
let mut responses = vec![response(json!({
"operation":"command_start", "result":"retained-command",
"audit_failures":[failure]
}))];
for (code, status, outcome) in [
("denied", 403, json!({"kind":"failed","code":"denied"})),
("outcome_unknown", 500, json!({"kind":"unknown"})),
("not_dispatched", 503, json!({"kind":"not_dispatched"})),
] {
let mut failure = failure.clone();
failure["operation_outcome"] = outcome;
responses.push(WorkspaceResponse { status, body: json!({
"code":code, "message":"safe provider classification", "audit_failures":[failure]
}).to_string() });
}
let client = Arc::new(RecordingWorkspaceClient::new(responses));
let session = WorkspaceAttachedWorkdirSession::handle(client.clone(), "checkout");
let handle = session
.start_command(CommandRequest {
command: "true".into(),
timeout_secs: 1,
transport_byte_budget: workdir::MAX_COMMAND_TRANSPORT_BYTES,
cwd: workdir::WorkdirPath::root(),
tool_call_id: None,
})
.await
.unwrap();
assert_eq!(handle.0, "retained-command");
let request = workdir::CheckoutRequest {
target: workdir::WorkdirPath::new("file.txt").unwrap(),
validator: vec![1],
operation: workdir::CheckoutOperation::Write {
content: b"new".to_vec(),
expected_hash: [0; 32],
},
};
assert!(matches!(
session.checkout_execute(request.clone()).await,
Err(WorkdirError::Denied(_))
));
assert!(matches!(
session.checkout_execute(request.clone()).await,
Err(WorkdirError::OutcomeUnknown(_))
));
assert!(matches!(
session.checkout_execute(request).await,
Err(WorkdirError::NotDispatched)
));
assert_eq!(
client.requests().len(),
4,
"audit loss must not trigger a resend"
);
}
#[tokio::test]
async fn attached_session_does_not_send_worker_local_bash_spill_path() {
let client = Arc::new(RecordingWorkspaceClient::new(vec![response(json!({
@@ -1914,6 +2041,7 @@ mod tests {
"alias": "checkout",
"working_directory_id": "wd-attached",
"capabilities": {"bits": 63},
"environment": workdir::WorkdirSessionEnvironment::root(WorkdirSessionCapabilities::ALL),
"attached": true
})),
response(json!({
@@ -1973,16 +2101,29 @@ mod tests {
);
}
#[test]
fn attach_propagates_provider_read_only_capabilities_to_the_alias_session() {
#[tokio::test]
async fn attach_projects_provider_environment_without_expanding_capability_ceiling() {
let dir = tempfile::tempdir().unwrap();
std::fs::create_dir(dir.path().join("src")).unwrap();
let provider = Arc::new(workdir::LocalWorkdirSession::materialized(
dir.path().to_path_buf(),
dir.path().to_path_buf(),
manifest::SharedScope::new(manifest::Scope::writable(dir.path()).unwrap()),
WorkdirSessionCapabilities::ALL,
));
let broker =
workdir::WorkdirToolBroker::at_cwd(provider, workdir::WorkdirPath::new("src").unwrap())
.unwrap();
let mut environment = broker.tool_session().environment();
let client = Arc::new(RecordingWorkspaceClient::new(vec![response(json!({
"workspace_id": "workspace/test",
"alias": "external",
"working_directory_id": "external-workdir",
"capabilities": {"bits": 25},
"environment": environment,
"attached": true
}))]));
let backend = WorkspaceHttpWorkdirBackend::new(client);
let backend = WorkspaceHttpWorkdirBackend::new(client.clone());
backend
.attach(WorkdirAttachInput {
alias: "external".to_string(),
@@ -1994,10 +2135,33 @@ mod tests {
.session_router
.session(&workdir::WorkdirAttachmentAlias::new("external").unwrap())
.unwrap();
environment.capabilities = WorkdirSessionCapabilities::READ_ONLY;
assert_eq!(session.environment(), environment);
assert_eq!(
session.capabilities(),
WorkdirSessionCapabilities::READ_ONLY
);
assert!(matches!(
session
.checkout_execute(workdir::CheckoutRequest {
target: workdir::WorkdirPath::root(),
validator: vec![1],
operation: workdir::CheckoutOperation::Create {
path: workdir::WorkdirPath::new("new.txt").unwrap(),
content: b"denied".to_vec(),
},
})
.await,
Err(WorkdirError::Unsupported(
workdir::WorkdirSessionCapability::Write
))
));
assert_eq!(
client.requests().len(),
1,
"denied mutation must not reach the Backend"
);
broker.tool_session().close().await.unwrap();
}
#[tokio::test]
@@ -2007,6 +2171,7 @@ mod tests {
"alias": "checkout",
"working_directory_id": "wd-attached",
"capabilities": {"bits": 63},
"environment": workdir::WorkdirSessionEnvironment::root(WorkdirSessionCapabilities::ALL),
"attached": true
}))]));
let reopen_calls = Arc::new(AtomicUsize::new(0));
@@ -1253,7 +1253,7 @@ pub(crate) mod tests {
state.generation += 1;
let raw = json!({"alias":body["alias"], "working_directory_id":body["working_directory_id"], "connection_id":format!("connection-{}", state.generation), "capabilities":{"bits":63}});
state.attachments.push(raw);
json!({"workspace_id":"test-workspace", "alias":body["alias"], "working_directory_id":body["working_directory_id"], "capabilities":{"bits":63}, "attached":true})
json!({"workspace_id":"test-workspace", "alias":body["alias"], "working_directory_id":body["working_directory_id"], "capabilities":{"bits":63}, "environment":workdir::WorkdirSessionEnvironment::root(workdir::WorkdirSessionCapabilities::ALL), "attached":true})
}
(WorkspaceRequestMethod::Delete, p)
if p.starts_with("workers/self/workdir-attachments/") =>
@@ -456,10 +456,13 @@ impl MemoryLifecycleTask {
.await;
return Ok(());
}
let view = SessionCapture::from_history_entries(
let view = SessionCapture::from_complete_history(
capture.session_id.clone(),
capture.segment_id.clone(),
capture.history[history_start..history_end].to_vec(),
);
capture.history.clone(),
)
.select_order_range(history_start as u64..history_end as u64)
.map_err(hook_internal)?;
let start_entry = pointer
.as_ref()
.map(|pointer| pointer.processed_through_entry + 1)
@@ -1556,11 +1559,14 @@ permission = "write"
#[tokio::test]
async fn run_committed_background_task_stages_non_empty_extraction_and_commits_pointer() {
let source = capture(2, 250);
let entry_ref =
SessionCapture::from_history_entries(source.segment_id.clone(), source.history.clone())
.overview()[0]
.id
.to_string();
let entry_ref = SessionCapture::from_complete_history(
source.session_id.clone(),
source.segment_id.clone(),
source.history.clone(),
)
.overview()[0]
.id
.to_string();
let client = ScriptClient::new(vec![
stage_candidate_events("stage-1", &entry_ref),
finish_events("finish-1", 1),
@@ -1606,14 +1612,102 @@ permission = "write"
);
}
// Real background extraction -> common fixed selection -> StageMemoryCandidate
// -> typed backend operation and pointer write. Only model/backend transport
// are replaced; the second extraction uses the first run's saved pointer.
#[tokio::test]
async fn second_extraction_preserves_original_evidence_positions_after_nonzero_pointer() {
let first = capture(2, 250);
let mut second = capture(4, 750);
second.history[..2].clone_from_slice(&first.history);
second
.usage_history
.insert(0, first.usage_history[0].clone());
let selected_ref =
crate::SessionEntryRef::from_history_entry_id(&second.history[2].annotation.entry_id)
.to_string();
let extension_writes = Arc::new(Mutex::new(Vec::new()));
let workspace_client = Arc::new(RecordingWorkspaceClient::default());
let (event_tx, _) = broadcast::channel(64);
run_background_task(test_task(
first,
Box::new(ScriptClient::new(vec![
finish_empty_events("first-empty"),
completed_events(),
])),
extension_writes.clone(),
event_tx.clone(),
workspace_client.clone(),
))
.await;
{
let writes = extension_writes.lock().unwrap();
assert_eq!(writes.len(), 1);
let pointer: memory::ExtractPointerPayload =
serde_json::from_value(writes[0].2.clone()).unwrap();
assert_eq!(pointer.processed_through_history_len, 2);
second
.extensions
.push((writes[0].1.clone(), writes[0].2.clone()));
}
run_background_task(test_task(
second,
Box::new(ScriptClient::new(vec![
stage_candidate_events("second-stage", &selected_ref),
finish_events("second-finish", 1),
completed_events(),
])),
extension_writes.clone(),
event_tx,
workspace_client.clone(),
))
.await;
let operations = workspace_client
.requests
.lock()
.unwrap()
.iter()
.filter_map(|request| {
request.body.as_deref().and_then(|body| {
serde_json::from_str::<memory::backend::MemoryBackendOperation>(body).ok()
})
})
.filter_map(|operation| match operation {
memory::backend::MemoryBackendOperation::StageCandidate(operation) => {
Some(operation)
}
_ => None,
})
.collect::<Vec<_>>();
assert_eq!(operations.len(), 1);
let operation = &operations[0];
assert_eq!(operation.source.range, [2, 3]);
assert_eq!(operation.evidence[0].id, selected_ref);
assert_eq!(operation.evidence[0].entry_range, Some([2, 2]));
assert_eq!(
operation.source_refs[0].segment_id.as_deref(),
Some("segment-1")
);
assert_eq!(operation.source_refs[0].entry_range, Some([2, 2]));
let writes = extension_writes.lock().unwrap();
assert_eq!(writes.len(), 2);
let pointer: memory::ExtractPointerPayload =
serde_json::from_value(writes[1].2.clone()).unwrap();
assert_eq!(pointer.processed_through_history_len, 4);
assert_eq!(pointer.processed_through_entry, 3);
}
#[tokio::test]
async fn subjektiv_extraction_stages_once_and_commits_its_independent_pointer() {
let source = capture(2, 250);
let entry_ref =
SessionCapture::from_history_entries(source.segment_id.clone(), source.history.clone())
.overview()[0]
.id
.to_string();
let entry_ref = SessionCapture::from_complete_history(
source.session_id.clone(),
source.segment_id.clone(),
source.history.clone(),
)
.overview()[0]
.id
.to_string();
let client = ScriptClient::new(vec![
stage_candidate_events("stage-1", &entry_ref),
finish_events("finish-1", 1),
@@ -1715,11 +1809,14 @@ permission = "write"
#[tokio::test]
async fn subjektiv_candidate_remains_staged_when_later_extraction_fails_without_pointer() {
let source = capture(2, 250);
let entry_ref =
SessionCapture::from_history_entries(source.segment_id.clone(), source.history.clone())
.overview()[0]
.id
.to_string();
let entry_ref = SessionCapture::from_complete_history(
source.session_id.clone(),
source.segment_id.clone(),
source.history.clone(),
)
.overview()[0]
.id
.to_string();
let client = ScriptClient::new(vec![stage_candidate_events("stage-1", &entry_ref)]);
let extension_writes = Arc::new(Mutex::new(Vec::new()));
let (event_tx, _) = broadcast::channel(32);
@@ -510,7 +510,11 @@ mod tests {
fn state() -> MemoryStagingOutputState {
MemoryStagingOutputState::new(
SessionCapture::new("segment-1", vec![Item::user_message("durable decision")]),
SessionCapture::new(
"session",
"segment-1",
vec![Item::user_message("durable decision")],
),
crate::worker::marker_workspace_client(None, "test-backend"),
SourceRef {
segment_id: "segment-1".to_string(),
@@ -537,6 +541,7 @@ mod tests {
#[test]
fn render_input_uses_session_entry_refs_and_new_tool_names() {
let view = SessionCapture::new(
"session",
"segment-1",
vec![
Item::user_message("preference"),
@@ -586,7 +591,11 @@ mod tests {
async fn subjektiv_destination_sends_committed_session_without_subject_scope() {
let client = Arc::new(RecordingSubjektivClient::default());
let state = MemoryStagingOutputState::new_subjektiv(
SessionCapture::new("segment-1", vec![Item::user_message("durable decision")]),
SessionCapture::new(
"session",
"segment-1",
vec![Item::user_message("durable decision")],
),
{
let mut host = crate::subjektiv::test_connection(client.clone());
host.context.session_id = "session-committed".into();
@@ -630,4 +639,53 @@ mod tests {
.unwrap_err();
assert!(format!("{error:?}").contains("unknown SessionEntryRef"));
}
#[tokio::test]
async fn staging_roundtripped_mixed_sources_uses_original_evidence_ranges_not_view_order() {
let client = Arc::new(RecordingSubjektivClient::default());
let mut dto = SessionCapture::new(
"session",
"view",
vec![
Item::user_message("first"),
Item::assistant_message("second"),
],
)
.to_dto();
dto.entries[0].source.segment_id = "original-a".into();
dto.entries[0].source.entry_index = 80;
dto.entries[1].source.segment_id = "original-b".into();
dto.entries[1].source.entry_index = 3;
let limits = crate::session_capture::CaptureLimits::default();
let view = SessionCapture::decode(&dto.encode(limits).unwrap(), limits).unwrap();
let state = MemoryStagingOutputState::new_subjektiv(
view,
crate::subjektiv::test_connection(client.clone()),
SourceRef {
segment_id: "view".into(),
range: [0, 1],
},
"run".into(),
);
StageMemoryCandidateTool { state }.execute(
r#"{"kind":"decision","claim":"claim","why_useful":"useful","entry_refs":["E00000000","E00000001"]}"#,
agen::tool::ToolExecutionContext::direct(),
).await.unwrap();
let requests = client.requests.lock().unwrap();
let request: server_api::SubjektivStageCandidateRequest =
serde_json::from_str(requests[0].body.as_deref().unwrap()).unwrap();
for (index, segment, position) in [(0, "original-a", 80), (1, "original-b", 3)] {
assert_eq!(
request.operation.evidence[index].entry_range,
Some([position; 2])
);
assert_eq!(
request.operation.source_refs[index].segment_id.as_deref(),
Some(segment)
);
assert_eq!(
request.operation.source_refs[index].entry_range,
Some([position; 2])
);
}
}
}
@@ -17,14 +17,14 @@ use crate::feature::{
ToolDeclaration,
};
use crate::session_capture::{
ReadDetail, ReadOptions, ReadSelector, ReferenceKind, SearchOptions, SessionCapture,
SessionEntryRef, ToolPart,
CapturePurpose, RawField, ReadDetail, ReadOptions, ReadSelector, ReferenceKind, SearchOptions,
SessionCapture, SessionEntryRef, ToolPart,
};
const SHOW_OVERVIEW_DESCRIPTION: &str =
"Show a sparse, bounded index of real user and assistant session entries.";
const SEARCH_ENTRIES_DESCRIPTION: &str = "Search or compactly list a bounded range of committed session entries. Reasoning entries are never exposed.";
const READ_ENTRY_DESCRIPTION: &str = "Read one committed session entry by stable SessionEntryRef. Compact mode is the default; full mode includes bounded tool input or output.";
const READ_ENTRY_DESCRIPTION: &str = "Read one committed session entry by stable SessionEntryRef. Compact mode is the default; full mode is a bounded rendering, not raw source. Raw mode requires raw_field and supports UTF-8 byte_offset/next_offset without omission; use capture_id to bind continuation to this exact capture.";
const DEFAULT_PAGE_LIMIT: usize = 20;
const MAX_PAGE_LIMIT: usize = 100;
const DEFAULT_READ_ITEMS: usize = 10;
@@ -173,6 +173,38 @@ struct ReadEntryParams {
mode: String,
#[serde(default)]
max_items: Option<usize>,
#[serde(default)]
raw_field: Option<RawEntryField>,
#[serde(default)]
part_index: Option<usize>,
#[serde(default)]
byte_offset: Option<usize>,
#[serde(default)]
max_bytes: Option<usize>,
#[serde(default)]
capture_id: Option<String>,
}
#[derive(Debug, Deserialize, JsonSchema)]
#[serde(rename_all = "snake_case")]
enum RawEntryField {
MessagePart,
PublicSystem,
ToolArguments,
ToolSummary,
ToolContent,
}
impl RawEntryField {
fn into_common(self, part_index: usize) -> RawField {
match self {
Self::MessagePart => RawField::MessagePart(part_index),
Self::PublicSystem => RawField::PublicSystem,
Self::ToolArguments => RawField::ToolArguments,
Self::ToolSummary => RawField::ToolSummary,
Self::ToolContent => RawField::ToolContent,
}
}
}
fn default_read_mode() -> String {
@@ -243,25 +275,20 @@ impl Tool for SearchEntriesTool {
let from = params.from.as_deref().map(parse_entry_ref).transpose()?;
let through = params.through.as_deref().map(parse_entry_ref).transpose()?;
let view = self.state.view();
if let (Some(from), Some(through)) = (&from, &through) {
if view.source_index_for_ref(from) > view.source_index_for_ref(through) {
return Err(ToolError::InvalidArgument(
"SearchEntries from must not be after through".to_string(),
));
}
}
let limit = bounded_limit(params.limit, DEFAULT_PAGE_LIMIT, MAX_PAGE_LIMIT);
let hits = view.search(&SearchOptions {
query: params.query,
kind,
tool_part,
tool_name: params.tool_name,
limit: Some(limit),
min_entry_index: None,
from,
through,
offset: params.offset,
});
let hits = view
.search_checked(&SearchOptions {
query: params.query,
kind,
tool_part,
tool_name: params.tool_name,
limit: Some(limit),
min_entry_index: None,
from,
through,
offset: params.offset,
})
.map_err(|error| ToolError::InvalidArgument(error.to_string()))?;
let entries = hits
.iter()
.map(|hit| {
@@ -272,6 +299,7 @@ impl Tool for SearchEntriesTool {
"tool_name": hit.tool_name,
"label": hit.label,
"text": hit.summary,
"text_kind": "preview",
})
})
.collect::<Vec<_>>();
@@ -299,12 +327,62 @@ impl Tool for ReadEntryTool {
) -> Result<ToolOutput, ToolError> {
let params: ReadEntryParams = parse_input("ReadEntry", input_json)?;
let entry_ref = parse_entry_ref(&params.entry_ref)?;
let view = self.state.view();
if params
.capture_id
.as_deref()
.is_some_and(|id| id != view.capture_id())
{
return Err(ToolError::InvalidArgument(
"capture identity does not match".into(),
));
}
if params.mode == "raw" {
let field = params
.raw_field
.ok_or_else(|| ToolError::InvalidArgument("raw mode requires raw_field".into()))?;
if params.part_index.is_some() && !matches!(field, RawEntryField::MessagePart) {
return Err(ToolError::InvalidArgument(
"part_index is only valid for message_part".into(),
));
}
let reference = view
.scoped_ref(&entry_ref)
.map_err(|error| ToolError::InvalidArgument(error.to_string()))?;
let raw = view
.read_raw(
&reference,
CapturePurpose::Observation,
field.into_common(params.part_index.unwrap_or(0)),
params.byte_offset.unwrap_or(0),
params
.max_bytes
.unwrap_or(MAX_READ_BYTES)
.clamp(1, MAX_READ_BYTES),
)
.map_err(|error| ToolError::InvalidArgument(error.to_string()))?;
return json_output(
"Read raw session source bytes.".into(),
serde_json::json!({
"text_kind": "raw", "raw": raw,
}),
);
}
if params.raw_field.is_some()
|| params.part_index.is_some()
|| params.byte_offset.is_some()
|| params.max_bytes.is_some()
{
return Err(ToolError::InvalidArgument(
"raw fields require raw mode".into(),
));
}
let detail = match params.mode.as_str() {
"compact" => ReadDetail::Compact,
"full" => ReadDetail::Full,
other => {
return Err(ToolError::InvalidArgument(format!(
"invalid ReadEntry mode {other:?}; expected compact or full"
"invalid ReadEntry mode {other:?}; expected compact, full, or raw"
)));
}
};
@@ -347,6 +425,7 @@ impl Tool for ReadEntryTool {
serde_json::json!({
"entries": entries,
"truncated": read.truncated,
"text_kind": "rendered",
}),
)
}
@@ -408,6 +487,7 @@ mod tests {
fn state() -> SessionExploreState {
SessionExploreState::new(SessionCapture::new(
"session",
"segment-1",
vec![
Item::user_message("first"),
@@ -479,4 +559,64 @@ mod tests {
.unwrap();
assert!(output.content.unwrap().contains("second"));
}
#[tokio::test]
async fn search_rejects_unknown_or_reversed_capture_endpoints() {
let search = search_entries_definition(state())().1;
for input in [
r#"{"from":"E99999999"}"#,
r#"{"through":"E99999999"}"#,
r#"{"from":"E00000003","through":"E00000000"}"#,
] {
assert!(
search
.execute(input, agen::tool::ToolExecutionContext::direct())
.await
.is_err()
);
}
}
#[tokio::test]
async fn raw_tool_arguments_reassemble_without_preview_or_utf8_loss() {
let arguments = "日本語\n\"\\".repeat(100);
let state = SessionExploreState::new(SessionCapture::new(
"session",
"segment",
vec![Item::tool_call("call", "Read", arguments.clone())],
));
let capture_id = state.view().capture_id().to_owned();
let tool = read_entry_definition(state)().1;
let mut offset = 0;
let mut assembled = String::new();
loop {
let input = serde_json::json!({
"entry_ref": "E00000000", "mode": "raw", "raw_field": "tool_arguments",
"byte_offset": offset, "max_bytes": 17, "capture_id": capture_id,
});
let output = tool
.execute(
&input.to_string(),
agen::tool::ToolExecutionContext::direct(),
)
.await
.unwrap();
let value: serde_json::Value = serde_json::from_str(&output.content.unwrap()).unwrap();
assert_eq!(value["text_kind"], "raw");
let raw = &value["raw"];
assert_eq!(raw["offset"], offset);
let text = raw["text"].as_str().unwrap();
assert!(text.len() <= 17);
assembled.push_str(text);
match raw["next_offset"].as_u64() {
Some(next) => {
assert!(next > offset as u64);
offset = next as usize;
}
None => break,
}
}
assert_eq!(assembled, arguments);
assert!(tool.execute(r#"{"entry_ref":"E00000000","mode":"raw","raw_field":"tool_arguments","capture_id":"other"}"#,
agen::tool::ToolExecutionContext::direct()).await.is_err());
}
}
@@ -39,13 +39,66 @@ use crate::hook::{
HookErrorCategory, HookExecutionPolicy, HookFailurePolicy, HookPreRequestAction, PreLlmRequest,
PreRequestContext, RunCommitted, RunCommittedContext,
};
use crate::session_capture::{SessionCapture, SessionEntryEvidence};
use crate::session_capture::{CaptureBody, SessionCapture, SessionEntryEvidence};
use crate::subjektiv::SubjektivHostConnection;
#[cfg(test)]
use crate::worker::WorkspaceClient;
#[cfg(test)]
use server_api::SubjektivMemoryBackendRequest;
// Pending-commit receipt interpretation belongs to Memory, not the shared capture.
struct CommittedMemoryCall {
call_id: String,
name: String,
arguments: String,
evidence: SessionEntryEvidence,
}
fn committed_pending_tool_calls(view: &SessionCapture, names: &[&str]) -> Vec<CommittedMemoryCall> {
let pending: HashSet<_> = view
.entries()
.filter_map(|entry| {
let CaptureBody::ToolResult {
call_id,
content: Some(content),
is_error: false,
..
} = &entry.body
else {
return None;
};
(serde_json::from_str::<serde_json::Value>(content)
.ok()
.and_then(|value| value.get("status").cloned())
.as_ref()
.and_then(serde_json::Value::as_str)
== Some("pending_commit"))
.then_some(call_id.as_str())
})
.collect();
view.entries()
.filter_map(|entry| {
let CaptureBody::ToolCall {
call_id,
name,
arguments,
} = &entry.body
else {
return None;
};
if !names.contains(&name.as_str()) || !pending.contains(call_id.as_str()) {
return None;
}
Some(CommittedMemoryCall {
call_id: call_id.clone(),
name: name.clone(),
arguments: arguments.clone(),
evidence: view.evidence_for_tool_call(call_id)?,
})
})
.collect()
}
const QUERY_TOOL: &str = "SubjektivMemoryQuery";
const READ_TOOL: &str = "SubjektivMemoryRead";
const LIST_CHANGES_TOOL: &str = "SubjektivMemoryListChanges";
@@ -518,14 +571,18 @@ impl SubjektivMemoryState {
.capture()
.map_err(|error| ToolError::ExecutionFailed(error.to_string()))?;
let session_id = capture.session_id.clone();
let view = SessionCapture::from_history_entries(capture.segment_id, capture.history);
let committed = view
.committed_pending_tool_calls(&[REMEMBER_TOOL, PROPOSE_CHANGE_TOOL])
.into_iter()
.any(|call| {
receipt_id(&session_id, &call.call_id)
.is_ok_and(|committed_receipt| committed_receipt == expected_receipt_id)
});
let view = SessionCapture::from_complete_history(
capture.session_id.clone(),
capture.segment_id,
capture.history,
);
let committed =
committed_pending_tool_calls(&view, &[REMEMBER_TOOL, PROPOSE_CHANGE_TOOL])
.into_iter()
.any(|call| {
receipt_id(&session_id, &call.call_id)
.is_ok_and(|committed_receipt| committed_receipt == expected_receipt_id)
});
if committed {
status.status = SubjektivMemoryReceiptStatus::PendingCommit;
}
@@ -623,10 +680,13 @@ fn replay_pending_committed_memory_with_policy(
.capture
.capture()
.map_err(|error| HookError::new(HookErrorCategory::Dependency, error.to_string()))?;
let view =
SessionCapture::from_history_entries(capture.segment_id.clone(), capture.history.clone());
let view = SessionCapture::from_complete_history(
capture.session_id.clone(),
capture.segment_id.clone(),
capture.history.clone(),
);
let mut failures = Vec::new();
for call in view.committed_pending_tool_calls(&[REMEMBER_TOOL, PROPOSE_CHANGE_TOOL]) {
for call in committed_pending_tool_calls(&view, &[REMEMBER_TOOL, PROPOSE_CHANGE_TOOL]) {
let receipt_id = match receipt_id(&capture.session_id, &call.call_id) {
Ok(value) => value,
Err(error) => {
@@ -755,8 +815,11 @@ fn resolve_entry_refs(
capture: &crate::feature::session::CommittedSessionCapture,
entry_refs: &[String],
) -> Result<Vec<SessionEntryEvidence>, ToolError> {
let view =
SessionCapture::from_history_entries(capture.segment_id.clone(), capture.history.clone());
let view = SessionCapture::from_complete_history(
capture.session_id.clone(),
capture.segment_id.clone(),
capture.history.clone(),
);
let mut seen = HashSet::new();
entry_refs
.iter()
@@ -570,15 +570,8 @@ impl Tool for SearchSessionEntriesTool {
let view = latest_view(&*self.provider, &params.subject).await?;
let from = params.from.as_deref().map(parse_entry_ref).transpose()?;
let through = params.through.as_deref().map(parse_entry_ref).transpose()?;
if let (Some(from), Some(through)) = (&from, &through) {
if view.source_index_for_ref(from) > view.source_index_for_ref(through) {
return Err(ToolError::InvalidArgument(
"SearchSessionEntries from must not be after through".to_string(),
));
}
}
let entries = view
.search(&SearchOptions {
.search_checked(&SearchOptions {
query: params.query,
kind: params.kind.as_deref().map(parse_kind).transpose()?,
tool_part: params
@@ -593,6 +586,7 @@ impl Tool for SearchSessionEntriesTool {
through,
offset: params.offset,
})
.map_err(|error| ToolError::InvalidArgument(error.to_string()))?
.into_iter()
.map(|entry| {
serde_json::json!({
@@ -604,6 +598,7 @@ impl Tool for SearchSessionEntriesTool {
"tool_name": entry.tool_name,
"label": entry.label,
"text": entry.summary,
"text_kind": "preview",
})
})
.collect::<Vec<_>>();
@@ -687,7 +682,10 @@ async fn latest_view(
.capture_worker_session(subject)
.await
.map_err(tool_error)?;
// Observation providers expose a stable subject scope rather than a raw
// Session handle. This scope is never used to select recording material.
Ok(SessionCapture::from_session_snapshot(
format!("observation:{}", capture.segment_id),
capture.segment_id,
capture.session,
))
@@ -7,6 +7,7 @@
//! undecodable mutation responses do not prove non-commit; dispatched requests are never resent.
//! This adapter does not reimplement Backend source-persistence transactions or activation.
use std::sync::Arc;
use std::time::Duration;
use async_trait::async_trait;
use serde::{Serialize, de::DeserializeOwned};
@@ -20,6 +21,7 @@ use super::{
ATTACHMENT_ALIAS, CONTENT_ROOT, ConfigAccess, ConfigAttachError, ConfigAttachment, DEADLINE,
WorkspaceConfigAttachmentBackend,
};
use crate::feature::builtin::backend_diagnostic::{Adapter, Diagnostic, Reason, Stage};
use crate::wip::WipOperationError;
use crate::worker::{
WorkspaceClient, WorkspaceClientError, WorkspaceRequest, WorkspaceRequestMethod,
@@ -34,11 +36,21 @@ const MAX_RESPONSE_BYTES: usize = 2 * 1024 * 1024;
#[derive(Clone)]
pub struct WorkspaceConfigBackend {
client: Arc<dyn WorkspaceClient>,
deadline: Duration,
}
impl WorkspaceConfigBackend {
pub fn new(client: Arc<dyn WorkspaceClient>) -> Self {
Self { client }
Self {
client,
deadline: DEADLINE,
}
}
#[cfg(test)]
pub(super) fn with_deadline(mut self, deadline: Duration) -> Self {
self.deadline = deadline;
self
}
fn base(&self) -> Result<String, WipOperationError> {
@@ -47,6 +59,7 @@ impl WorkspaceConfigBackend {
.workspace_id()
.filter(|id| !id.is_empty())
.ok_or_else(|| {
diagnostic("authority", false).record(Stage::Request, Reason::Identity, None);
failure(
ProtocolErrorCode::PermissionDenied,
"Workspace config authority unavailable",
@@ -69,11 +82,13 @@ impl WorkspaceConfigBackend {
&self,
request: WorkspaceRequest,
mutation: bool,
diagnostic: &Diagnostic,
) -> Result<T, WipOperationError> {
let response = self
.client
.execute_with_timeout(request, DEADLINE)
.execute_with_timeout(request, self.deadline)
.map_err(|error| {
diagnostic.transport(&error);
if mutation && !matches!(error, WorkspaceClientError::InvalidPath(_)) {
unknown()
} else {
@@ -84,6 +99,7 @@ impl WorkspaceConfigBackend {
}
})?;
if response.body.len() > MAX_RESPONSE_BYTES {
diagnostic.record(Stage::Response, Reason::ResponseSize, Some(response.status));
return Err(if mutation {
unknown()
} else {
@@ -96,8 +112,18 @@ impl WorkspaceConfigBackend {
if !response.is_success() {
if let Ok(error) = serde_json::from_str::<WorkspaceConfigApiError>(&response.body) {
if error.classification == WorkspaceConfigFailureClassification::Unknown {
diagnostic.record(
Stage::Response,
Reason::BackendUnknown,
Some(response.status),
);
return Err(unknown());
}
diagnostic.record(
Stage::Response,
Reason::BackendRejected,
Some(response.status),
);
// Do not reflect arbitrary transport/provider text into model output.
let code = match error.status {
401 | 403 | 404 => ProtocolErrorCode::PermissionDenied,
@@ -111,6 +137,11 @@ impl WorkspaceConfigBackend {
"Workspace config request rejected without commit",
));
}
diagnostic.record(
Stage::Response,
Reason::UnrecognizedResponse,
Some(response.status),
);
return Err(if mutation {
unknown()
} else {
@@ -125,6 +156,7 @@ impl WorkspaceConfigBackend {
});
}
serde_json::from_str(&response.body).map_err(|_| {
diagnostic.record(Stage::Response, Reason::Decode, Some(response.status));
if mutation {
unknown()
} else {
@@ -138,10 +170,17 @@ impl WorkspaceConfigBackend {
/// GET checks the current grant even when the Worker has not attached yet.
pub fn current(&self) -> Result<Option<WorkspaceConfigAttachment>, WipOperationError> {
self.current_diagnosed(&diagnostic("current", false))
}
fn current_diagnosed(
&self,
diagnostic: &Diagnostic,
) -> Result<Option<WorkspaceConfigAttachment>, WipOperationError> {
let result: Option<WorkspaceConfigAttachment> =
self.request(WorkspaceRequest::get(self.base()?), false)?;
self.request(WorkspaceRequest::get(self.base()?), false, diagnostic)?;
if let Some(attachment) = &result {
self.validate_attachment(attachment)?;
self.validate_attachment(attachment, diagnostic)?;
}
Ok(result)
}
@@ -149,6 +188,7 @@ impl WorkspaceConfigBackend {
fn validate_attachment(
&self,
attachment: &WorkspaceConfigAttachment,
diagnostic: &Diagnostic,
) -> Result<(), WipOperationError> {
if Some(attachment.workspace_id.as_str()) != self.client.workspace_id()
|| attachment.alias != ATTACHMENT_ALIAS
@@ -160,6 +200,7 @@ impl WorkspaceConfigBackend {
.bytes()
.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_'))
{
diagnostic.record(Stage::Response, Reason::Identity, None);
return Err(failure(
ProtocolErrorCode::PermissionDenied,
"Invalid Workspace config attachment identity",
@@ -173,14 +214,34 @@ impl WorkspaceConfigBackend {
suffix: &str,
request: &R,
mutation: bool,
) -> Result<T, WipOperationError> {
let operation = match suffix {
"/observe" => "observe",
"/read" => "read",
"/commit" => "commit",
"" => "attach",
_ => "post",
};
self.post_diagnosed(suffix, request, mutation, diagnostic(operation, mutation))
.await
}
pub(super) async fn post_diagnosed<T: DeserializeOwned + Send + 'static, R: Serialize>(
&self,
suffix: &str,
request: &R,
mutation: bool,
diagnostic: Diagnostic,
) -> Result<T, WipOperationError> {
let body = serde_json::to_string(request).map_err(|_| {
diagnostic.record(Stage::Request, Reason::Decode, None);
failure(
ProtocolErrorCode::InvalidArguments,
"Invalid Workspace config request",
)
})?;
if body.len() > MAX_RESPONSE_BYTES {
diagnostic.record(Stage::Request, Reason::RequestSize, None);
return Err(failure(
ProtocolErrorCode::ResourceLimitExceeded,
"Workspace config request exceeds limit",
@@ -192,12 +253,16 @@ impl WorkspaceConfigBackend {
body,
);
let this = self.clone();
let request_diagnostic = diagnostic.clone();
tokio::time::timeout(
DEADLINE,
tokio::task::spawn_blocking(move || this.request(request, mutation)),
self.deadline,
tokio::task::spawn_blocking(move || {
this.request(request, mutation, &request_diagnostic)
}),
)
.await
.map_err(|_| {
diagnostic.record(Stage::Completion, Reason::Deadline, None);
if mutation {
unknown()
} else {
@@ -208,6 +273,7 @@ impl WorkspaceConfigBackend {
}
})?
.map_err(|_| {
diagnostic.record(Stage::Completion, Reason::TaskJoin, None);
if mutation {
unknown()
} else {
@@ -223,18 +289,22 @@ impl WorkspaceConfigBackend {
&self,
) -> Result<Option<WorkspaceConfigAttachment>, WipOperationError> {
let this = self.clone();
let diagnostic = diagnostic("current", false);
let request_diagnostic = diagnostic.clone();
tokio::time::timeout(
DEADLINE,
tokio::task::spawn_blocking(move || this.current()),
self.deadline,
tokio::task::spawn_blocking(move || this.current_diagnosed(&request_diagnostic)),
)
.await
.map_err(|_| {
diagnostic.record(Stage::Completion, Reason::Deadline, None);
failure(
ProtocolErrorCode::ResourceLimitExceeded,
"Workspace config request deadline exceeded",
)
})?
.map_err(|_| {
diagnostic.record(Stage::Completion, Reason::TaskJoin, None);
failure(
ProtocolErrorCode::Internal,
"Workspace config transport unavailable",
@@ -249,10 +319,14 @@ impl WorkspaceConfigAttachmentBackend for WorkspaceConfigBackend {
self.client.is_available()
}
fn validate_access(&self, access: Option<ConfigAccess>) -> Result<(), ConfigAttachError> {
let current = self.current().map_err(attach_error)?;
let diagnostic = diagnostic("validate_access", false);
let current = self
.current_diagnosed(&diagnostic)
.map_err(|error| attach_error(error, &diagnostic))?;
if access == Some(ConfigAccess::ReadWrite)
&& current.is_some_and(|a| a.access == WorkspaceConfigAccess::ReadOnly)
{
diagnostic.record(Stage::Attach, Reason::AccessCeiling, None);
return Err(ConfigAttachError::Denied);
}
Ok(())
@@ -271,10 +345,16 @@ impl WorkspaceConfigAttachmentBackend for WorkspaceConfigBackend {
ConfigAccess::ReadWrite => WorkspaceConfigAccess::ReadWrite,
}),
};
let attachment: WorkspaceConfigAttachment =
self.post("", &request, true).await.map_err(attach_error)?;
self.validate_attachment(&attachment)
.map_err(|_| ConfigAttachError::OutcomeUnknown)?;
let diagnostic = diagnostic("attach", true);
let attachment: WorkspaceConfigAttachment = self
.post_diagnosed("", &request, true, diagnostic.clone())
.await
.map_err(|error| attach_error(error, &diagnostic))?;
self.validate_attachment(&attachment, &diagnostic)
.map_err(|_| {
diagnostic.record(Stage::Attach, Reason::Identity, None);
ConfigAttachError::OutcomeUnknown
})?;
Ok(ConfigAttachment {
connection_id: attachment.connection_id,
access: match attachment.access {
@@ -286,7 +366,8 @@ impl WorkspaceConfigAttachmentBackend for WorkspaceConfigBackend {
}
}
fn attach_error(error: WipOperationError) -> ConfigAttachError {
fn attach_error(error: WipOperationError, diagnostic: &Diagnostic) -> ConfigAttachError {
diagnostic.record(Stage::Attach, Reason::AttachConversion, None);
match error {
WipOperationError::OutcomeUnknown(_) => ConfigAttachError::OutcomeUnknown,
WipOperationError::Protocol(error) => match error.code {
@@ -298,6 +379,10 @@ fn attach_error(error: WipOperationError) -> ConfigAttachError {
}
}
pub(super) fn diagnostic(operation: &'static str, mutation: bool) -> Diagnostic {
Diagnostic::new(Adapter::WorkspaceConfig, operation, mutation, None)
}
pub(super) fn failure(code: ProtocolErrorCode, message: &str) -> WipOperationError {
WipOperationError::Protocol(ProtocolError {
code,
@@ -307,3 +392,7 @@ pub(super) fn failure(code: ProtocolErrorCode, message: &str) -> WipOperationErr
pub(super) fn unknown() -> WipOperationError {
WipOperationError::OutcomeUnknown("Workspace config outcome unknown; refresh before any subsequent operation; no automatic retry".into())
}
#[cfg(test)]
#[path = "backend_tests.rs"]
mod tests;
@@ -0,0 +1,580 @@
//! Production WorkspaceConfigBackend with only WorkspaceClient substituted. The transport fixture
//! applies effects before injected reply loss, retains its state for rereading, and gates blocking
//! completion explicitly. These ordinary worker lib tests do not claim Backend transaction coverage.
use super::*;
use crate::feature::builtin::backend_diagnostic::tests::Capture;
use crate::worker::WorkspaceResponse;
use std::sync::{Condvar, Mutex};
use tracing::instrument::WithSubscriber;
const UNSAFE: &str = "secret-token /private/config credential=unsafe";
#[derive(Clone, Copy, Debug)]
enum Fault {
Unavailable,
InvalidPath,
Communication,
Decode,
Size,
Rejected(u16),
Unknown,
Unrecognized,
Identity,
Panic,
Deadline,
}
#[derive(Debug, Default)]
struct Effects {
requests: usize,
mutations: usize,
saved: Option<String>,
completed: bool,
released: bool,
}
#[derive(Debug)]
struct Client {
fault: Fault,
state: Mutex<Effects>,
changed: Condvar,
started: tokio::sync::Notify,
}
impl Client {
fn new(fault: Fault) -> Arc<Self> {
Arc::new(Self {
fault,
state: Mutex::new(Effects::default()),
changed: Condvar::new(),
started: tokio::sync::Notify::new(),
})
}
fn finish(&self) {
self.state.lock().unwrap().completed = true;
self.changed.notify_all();
}
fn release(&self) {
self.state.lock().unwrap().released = true;
self.changed.notify_all();
}
fn wait_completed(&self) {
let (state, timeout) = self
.changed
.wait_timeout_while(self.state.lock().unwrap(), Duration::from_secs(5), |s| {
!s.completed
})
.unwrap();
assert!(
!timeout.timed_out() && state.completed,
"blocking request did not finish"
);
}
}
struct Release(Arc<Client>);
impl Drop for Release {
fn drop(&mut self) {
self.0.release();
}
}
fn attachment() -> WorkspaceConfigAttachment {
WorkspaceConfigAttachment {
workspace_id: "workspace".into(),
connection_id: "connection-1".into(),
alias: ATTACHMENT_ALIAS.into(),
content_path: CONTENT_ROOT.into(),
working_directory_id: "grant".into(),
access: WorkspaceConfigAccess::ReadWrite,
name: UNSAFE.into(),
purpose: UNSAFE.into(),
already_attached: false,
}
}
impl WorkspaceClient for Client {
fn workspace_id(&self) -> Option<&str> {
Some("workspace")
}
fn kind(&self) -> &str {
"diagnostic-test-boundary"
}
fn is_available(&self) -> bool {
true
}
fn execute(
&self,
request: WorkspaceRequest,
) -> Result<WorkspaceResponse, WorkspaceClientError> {
{
let mut state = self.state.lock().unwrap();
state.requests += 1;
if request.method == WorkspaceRequestMethod::Post
&& !request.path.ends_with("/read")
&& !matches!(
self.fault,
Fault::Unavailable | Fault::InvalidPath | Fault::Rejected(_)
)
{
state.mutations += 1;
state.saved = request.body.clone();
}
self.started.notify_one();
if matches!(self.fault, Fault::Deadline) {
let (_state, timeout) = self
.changed
.wait_timeout_while(state, Duration::from_secs(5), |s| !s.released)
.unwrap();
assert!(!timeout.timed_out(), "test must release blocked transport");
}
}
self.finish();
match self.fault {
Fault::Unavailable => Err(WorkspaceClientError::Unavailable(UNSAFE.into())),
Fault::InvalidPath => Err(WorkspaceClientError::InvalidPath(UNSAFE.into())),
Fault::Communication => Err(WorkspaceClientError::Request(UNSAFE.into())),
Fault::Panic => panic!("injected blocking task failure"),
Fault::Decode => Ok(WorkspaceResponse {
status: 200,
body: UNSAFE.into(),
}),
Fault::Size => Ok(WorkspaceResponse {
status: 200,
body: UNSAFE.repeat(MAX_RESPONSE_BYTES / UNSAFE.len() + 1),
}),
Fault::Rejected(_) | Fault::Unknown => {
let status = match self.fault {
Fault::Rejected(status) => status,
_ => 503,
};
Ok(WorkspaceResponse {
status,
body: serde_json::to_string(&WorkspaceConfigApiError {
status,
code: UNSAFE.into(),
message: UNSAFE.into(),
classification: if matches!(self.fault, Fault::Unknown) {
WorkspaceConfigFailureClassification::Unknown
} else {
WorkspaceConfigFailureClassification::NotCommitted
},
})
.unwrap(),
})
}
Fault::Unrecognized => Ok(WorkspaceResponse {
status: 503,
body: UNSAFE.into(),
}),
Fault::Identity | Fault::Deadline => {
let mut value = attachment();
if matches!(self.fault, Fault::Identity) {
value.workspace_id = UNSAFE.into();
}
Ok(WorkspaceResponse {
status: 200,
body: serde_json::to_string(&value).unwrap(),
})
}
}
}
}
fn assert_safe(capture: &Capture, reason: Reason, stage: Stage, public: &str) {
let text = capture.text();
assert!(
text.contains(&format!("reason={reason:?}")),
"missing reason: {text}"
);
assert!(
text.contains(&format!("stage={stage:?}")),
"missing stage: {text}"
);
assert!(text.contains("adapter=WorkspaceConfig"), "{text}");
assert!(text.contains("correlation="), "{text}");
for secret in [
UNSAFE,
"secret-token",
"/private/config",
"credential=unsafe",
] {
assert!(
!text.contains(secret),
"diagnostics leaked unsafe boundary data"
);
assert!(
!public.contains(secret),
"public result leaked unsafe boundary data"
);
}
}
fn public_error(
result: Result<serde_json::Value, WipOperationError>,
code: Option<ProtocolErrorCode>,
) -> String {
match (result, code) {
(Err(WipOperationError::OutcomeUnknown(message)), None) => {
assert!(message.contains("no automatic retry"));
message
}
(Err(WipOperationError::Protocol(error)), Some(expected)) => {
assert_eq!(error.code, expected);
error.message
}
_ => panic!("unexpected public error classification"),
}
}
#[tokio::test]
async fn production_reply_failures_retain_reasons_without_retrying_mutations() {
let cases = [
(
Fault::Unavailable,
Reason::TransportUnavailable,
Stage::Request,
ProtocolErrorCode::Internal,
false,
),
(
Fault::InvalidPath,
Reason::InvalidPath,
Stage::Request,
ProtocolErrorCode::Internal,
false,
),
(
Fault::Communication,
Reason::Communication,
Stage::Request,
ProtocolErrorCode::Internal,
true,
),
(
Fault::Decode,
Reason::Decode,
Stage::Response,
ProtocolErrorCode::Internal,
true,
),
(
Fault::Size,
Reason::ResponseSize,
Stage::Response,
ProtocolErrorCode::ResourceLimitExceeded,
true,
),
(
Fault::Unknown,
Reason::BackendUnknown,
Stage::Response,
ProtocolErrorCode::Internal,
true,
),
(
Fault::Unrecognized,
Reason::UnrecognizedResponse,
Stage::Response,
ProtocolErrorCode::Internal,
true,
),
(
Fault::Panic,
Reason::TaskJoin,
Stage::Completion,
ProtocolErrorCode::Internal,
true,
),
];
for (fault, reason, stage, read_code, applies) in cases {
for mutation in [false, true] {
let client = Client::new(fault);
let backend = WorkspaceConfigBackend::new(client.clone());
let capture = Capture::default();
let result = backend
.post::<serde_json::Value, _>(
if mutation { "/commit" } else { "/read" },
&serde_json::json!({"content": UNSAFE}),
mutation,
)
.with_subscriber(capture.dispatch())
.await;
let code = if (mutation && !matches!(fault, Fault::InvalidPath))
|| matches!(fault, Fault::Unknown)
{
None
} else {
Some(read_code)
};
let public = public_error(result, code);
assert_safe(&capture, reason, stage, &public);
let state = client.state.lock().unwrap();
assert_eq!(state.requests, 1);
assert_eq!(state.mutations, usize::from(mutation && applies));
assert_eq!(state.saved.is_some(), mutation && applies);
}
}
}
#[tokio::test]
async fn production_typed_rejections_preserve_public_codes_and_never_apply() {
for (status, code) in [
(403, ProtocolErrorCode::PermissionDenied),
(409, ProtocolErrorCode::ValidatorMismatch),
(413, ProtocolErrorCode::ResourceLimitExceeded),
(422, ProtocolErrorCode::InvalidArguments),
(500, ProtocolErrorCode::Internal),
] {
for mutation in [false, true] {
let client = Client::new(Fault::Rejected(status));
let backend = WorkspaceConfigBackend::new(client.clone());
let capture = Capture::default();
let public = public_error(
backend
.post::<serde_json::Value, _>(
if mutation { "/commit" } else { "/read" },
&(),
mutation,
)
.with_subscriber(capture.dispatch())
.await,
Some(code),
);
assert_safe(&capture, Reason::BackendRejected, Stage::Response, &public);
assert!(capture.text().contains(&format!("status={status}")));
let state = client.state.lock().unwrap();
assert_eq!(state.requests, 1);
assert_eq!(state.mutations, 0);
assert!(state.saved.is_none());
}
}
}
#[tokio::test]
async fn production_deadline_is_unknown_after_effect_and_blocking_completion_is_not_retried() {
for mutation in [false, true] {
let client = Client::new(Fault::Deadline);
let release = Release(client.clone());
let backend =
WorkspaceConfigBackend::new(client.clone()).with_deadline(Duration::from_secs(1));
let capture = Capture::default();
let body = serde_json::json!({"content": UNSAFE});
let request = backend
.post::<serde_json::Value, _>(
if mutation { "/commit" } else { "/read" },
&body,
mutation,
)
.with_subscriber(capture.dispatch());
tokio::pin!(request);
tokio::select! {
biased;
_ = client.started.notified() => {},
_ = &mut request => panic!("request must start and apply its effect before timeout is observed"),
}
{
let state = client.state.lock().unwrap();
assert_eq!(state.mutations, usize::from(mutation));
assert_eq!(state.saved.is_some(), mutation);
assert!(!state.completed && !state.released);
}
let result = request.await;
let public = public_error(
result,
if mutation {
None
} else {
Some(ProtocolErrorCode::ResourceLimitExceeded)
},
);
assert_safe(&capture, Reason::Deadline, Stage::Completion, &public);
drop(release);
let completed = client.clone();
tokio::task::spawn_blocking(move || completed.wait_completed())
.await
.unwrap();
let state = client.state.lock().unwrap();
assert_eq!(state.requests, 1);
assert_eq!(state.mutations, usize::from(mutation));
assert_eq!(state.saved.is_some(), mutation);
}
}
#[tokio::test]
async fn production_current_identity_and_task_loss_remain_safe() {
for (fault, reason, stage, code) in [
(
Fault::Identity,
Reason::Identity,
Stage::Response,
ProtocolErrorCode::PermissionDenied,
),
(
Fault::Panic,
Reason::TaskJoin,
Stage::Completion,
ProtocolErrorCode::Internal,
),
(
Fault::Decode,
Reason::Decode,
Stage::Response,
ProtocolErrorCode::Internal,
),
] {
let client = Client::new(fault);
let backend = WorkspaceConfigBackend::new(client.clone());
let capture = Capture::default();
let result = backend
.current_async()
.with_subscriber(capture.dispatch())
.await;
let message = match result {
Err(WipOperationError::Protocol(error)) => {
assert_eq!(error.code, code);
error.message
}
_ => panic!("unexpected current classification"),
};
assert_safe(&capture, reason, stage, &message);
assert_eq!(client.state.lock().unwrap().mutations, 0);
}
}
#[tokio::test]
async fn production_attach_conversion_retains_original_reason_and_effect_count() {
for (fault, reason, expected, mutations) in [
(
Fault::Communication,
Reason::Communication,
ConfigAttachError::OutcomeUnknown,
1,
),
(
Fault::Identity,
Reason::Identity,
ConfigAttachError::OutcomeUnknown,
1,
),
(
Fault::Rejected(403),
Reason::BackendRejected,
ConfigAttachError::Denied,
0,
),
(
Fault::Rejected(422),
Reason::BackendRejected,
ConfigAttachError::InvalidRequest,
0,
),
(
Fault::Rejected(500),
Reason::BackendRejected,
ConfigAttachError::Unavailable,
0,
),
] {
let client = Client::new(fault);
let backend = WorkspaceConfigBackend::new(client.clone());
let capture = Capture::default();
assert_eq!(
backend
.attach("host-key-not-logged", None)
.with_subscriber(capture.dispatch())
.await,
Err(expected)
);
assert_safe(&capture, reason, Stage::Attach, expected.message());
let text = capture.text();
if !matches!(fault, Fault::Identity) {
assert!(text.contains("reason=AttachConversion"), "{text}");
}
assert!(!text.contains("host-key-not-logged"));
let correlations: std::collections::BTreeSet<_> = text
.lines()
.filter_map(|line| line.split("correlation=").nth(1))
.filter_map(|tail| tail.split(';').next())
.collect();
assert_eq!(
correlations.len(),
1,
"request and attach conversion must correlate"
);
let state = client.state.lock().unwrap();
assert_eq!(state.requests, 1);
assert_eq!(state.mutations, mutations);
assert_eq!(state.saved.is_some(), mutations == 1);
}
}
#[tokio::test]
async fn production_oversized_request_is_rejected_before_dispatch() {
let client = Client::new(Fault::Communication);
let backend = WorkspaceConfigBackend::new(client.clone());
let capture = Capture::default();
let public = public_error(
backend
.post::<serde_json::Value, _>("/commit", &"x".repeat(MAX_RESPONSE_BYTES), true)
.with_subscriber(capture.dispatch())
.await,
Some(ProtocolErrorCode::ResourceLimitExceeded),
);
assert_safe(&capture, Reason::RequestSize, Stage::Request, &public);
let state = client.state.lock().unwrap();
assert_eq!(state.requests, 0);
assert_eq!(state.mutations, 0);
}
#[tokio::test]
async fn production_current_deadline_remains_bounded_and_has_no_mutation() {
let client = Client::new(Fault::Deadline);
let release = Release(client.clone());
let backend = WorkspaceConfigBackend::new(client.clone()).with_deadline(Duration::from_secs(1));
let capture = Capture::default();
let request = backend.current_async().with_subscriber(capture.dispatch());
tokio::pin!(request);
tokio::select! {
biased;
_ = client.started.notified() => {},
_ = &mut request => panic!("current request must enter the gate before timeout is observed"),
}
assert_eq!(client.state.lock().unwrap().mutations, 0);
let error = request.await;
let message = match error {
Err(WipOperationError::Protocol(error)) => {
assert_eq!(error.code, ProtocolErrorCode::ResourceLimitExceeded);
error.message
}
_ => panic!("current deadline must remain a public resource-limit error"),
};
assert_safe(&capture, Reason::Deadline, Stage::Completion, &message);
drop(release);
let completed = client.clone();
tokio::task::spawn_blocking(move || completed.wait_completed())
.await
.unwrap();
let state = client.state.lock().unwrap();
assert_eq!(state.requests, 1);
assert_eq!(state.mutations, 0);
}
#[tokio::test]
async fn production_request_encoding_failure_is_safe_and_never_dispatched() {
struct Unencodable;
impl Serialize for Unencodable {
fn serialize<S: serde::Serializer>(&self, _: S) -> Result<S::Ok, S::Error> {
Err(serde::ser::Error::custom(UNSAFE))
}
}
let client = Client::new(Fault::Communication);
let backend = WorkspaceConfigBackend::new(client.clone());
let capture = Capture::default();
let result = backend
.post::<serde_json::Value, _>("/commit", &Unencodable, true)
.with_subscriber(capture.dispatch())
.await;
let message = public_error(result, Some(ProtocolErrorCode::InvalidArguments));
assert_safe(&capture, Reason::Decode, Stage::Request, &message);
let state = client.state.lock().unwrap();
assert_eq!(state.requests, 0);
assert_eq!(state.mutations, 0);
}
@@ -36,9 +36,11 @@ use wip_protocol::{
};
use super::backend::{
MAX_DEPTH, MAX_NODES, MAX_PATH_BYTES, MAX_TEXT_BYTES, WorkspaceConfigBackend, failure, unknown,
MAX_DEPTH, MAX_NODES, MAX_PATH_BYTES, MAX_TEXT_BYTES, WorkspaceConfigBackend, diagnostic,
failure, unknown,
};
use super::{CONTENT_ROOT, WorkspaceConfigFeature};
use crate::feature::builtin::backend_diagnostic::{Diagnostic, Reason, Stage};
use crate::wip::{
WipCallContext, WipOperationError, WipOperationHandler, WipOperationOutput, WipProjection,
WipProjectionKind, WipSubtreeProvider, contextual_reference, json_to_wip,
@@ -138,6 +140,23 @@ async fn observe(
connection: &str,
paths: Vec<String>,
depth: u32,
) -> Result<WorkspaceConfigObserveResponse, WipOperationError> {
observe_diagnosed(
backend,
connection,
paths,
depth,
diagnostic("observe", false),
)
.await
}
async fn observe_diagnosed(
backend: &WorkspaceConfigBackend,
connection: &str,
paths: Vec<String>,
depth: u32,
diagnostic: Diagnostic,
) -> Result<WorkspaceConfigObserveResponse, WipOperationError> {
if paths.len() > MAX_NODES || depth > MAX_DEPTH {
return Err(invalid());
@@ -148,7 +167,7 @@ async fn observe(
}
}
let result: WorkspaceConfigObserveResponse = backend
.post(
.post_diagnosed(
"/observe",
&WorkspaceConfigObserveRequest {
connection_id: connection.into(),
@@ -156,6 +175,7 @@ async fn observe(
depth,
},
false,
diagnostic.clone(),
)
.await?;
if result.connection_id != connection
@@ -164,24 +184,37 @@ async fn observe(
|| result.validator.len() > 4096
|| result.digest.is_empty()
{
diagnostic.record(
Stage::Response,
if result.connection_id != connection {
Reason::Identity
} else {
Reason::InvalidMetadata
},
None,
);
return Err(failure(
ProtocolErrorCode::Internal,
"Invalid Workspace config metadata",
));
}
if result.entrypoints.is_empty() || result.entrypoints.len() > MAX_NODES {
diagnostic.record(Stage::Response, Reason::InvalidMetadata, None);
return Err(failure(
ProtocolErrorCode::Internal,
"Invalid Workspace config entrypoints",
));
}
for path in &result.entrypoints {
validate_source(path)?;
validate_source(path)
.inspect_err(|_| diagnostic.record(Stage::Response, Reason::InvalidMetadata, None))?;
}
let mut seen = BTreeSet::new();
for node in &result.nodes {
if !node.path.is_empty() {
validate_source(&node.path)?;
validate_source(&node.path).inspect_err(|_| {
diagnostic.record(Stage::Response, Reason::InvalidMetadata, None)
})?;
}
if !seen.insert(&node.path)
|| node.validator.is_empty()
@@ -198,6 +231,7 @@ async fn observe(
suffix.is_some_and(|s| s.split('/').count() <= depth as usize)
})
{
diagnostic.record(Stage::Response, Reason::InvalidMetadata, None);
return Err(failure(
ProtocolErrorCode::Internal,
"Invalid Workspace config metadata scope",
@@ -490,9 +524,10 @@ fn content_type(value: Option<&Value>) -> Result<ConfigContentType, WipOperation
impl ConfigHandler {
async fn read(&self) -> Result<WorkspaceConfigReadResponse, WipOperationError> {
let diagnostic = diagnostic("read", false);
let response: WorkspaceConfigReadResponse = self
.backend
.post(
.post_diagnosed(
"/read",
&WorkspaceConfigReadRequest {
connection_id: self.attachment.connection_id.clone(),
@@ -500,6 +535,7 @@ impl ConfigHandler {
validator: self.node.validator.clone(),
},
false,
diagnostic.clone(),
)
.await?;
if response.path != self.node.path
@@ -507,12 +543,14 @@ impl ConfigHandler {
|| Some(&response.digest) != self.node.digest.as_ref()
|| Some(&response.content_type) != self.node.content_type.as_ref()
{
diagnostic.record(Stage::Response, Reason::Identity, None);
return Err(failure(
ProtocolErrorCode::Internal,
"Invalid Workspace config read response",
));
}
if response.content.len() > MAX_TEXT_BYTES {
diagnostic.record(Stage::Response, Reason::ResponseSize, None);
return Err(failure(
ProtocolErrorCode::ResourceLimitExceeded,
"Workspace config source exceeds text limit",
@@ -630,12 +668,17 @@ impl WipOperationHandler for ConfigHandler {
context: WipCallContext,
) -> Result<WipOperationOutput, WipOperationError> {
let dispatched = std::sync::atomic::AtomicBool::new(false);
let diagnostic = diagnostic(
"operation",
!matches!(operation, "read" | "read_lines" | "search_text"),
);
tokio::time::timeout(
super::DEADLINE,
self.execute(operation, arguments, context, &dispatched),
self.execute(operation, arguments, context, &dispatched, &diagnostic),
)
.await
.map_err(|_| {
diagnostic.record(Stage::Completion, Reason::Deadline, None);
if dispatched.load(std::sync::atomic::Ordering::Acquire) {
unknown()
} else {
@@ -655,6 +698,7 @@ impl ConfigHandler {
arguments: &BTreeMap<String, Value>,
context: WipCallContext,
dispatched: &std::sync::atomic::AtomicBool,
operation_diagnostic: &Diagnostic,
) -> Result<WipOperationOutput, WipOperationError> {
if !self.operations.iter().any(|name| name == operation) {
return Err(failure(
@@ -798,9 +842,10 @@ impl ConfigHandler {
};
let changes = self.canonical_changes(changes).await?;
dispatched.store(true, std::sync::atomic::Ordering::Release);
let diagnostic = operation_diagnostic.clone();
let result: WorkspaceConfigCommitResponse = self
.backend
.post(
.post_diagnosed(
"/commit",
&WorkspaceConfigCommitRequest {
connection_id: self.attachment.connection_id.clone(),
@@ -812,28 +857,37 @@ impl ConfigHandler {
},
},
true,
diagnostic.clone(),
)
.await?;
// Commit response supplies ROOT state. Obtain the exact post-commit PATH
// validator, never substitute the root validator for a file validator.
let updated = observe(
let updated = observe_diagnosed(
&self.backend,
&self.attachment.connection_id,
vec![self.node.path.clone()],
0,
diagnostic.at_stage(Stage::Reconcile),
)
.await
.map_err(|_| unknown())?;
if updated.digest != result.digest || updated.validator != result.validator {
diagnostic.record(Stage::Reconcile, Reason::Identity, None);
return Err(unknown());
}
let node = updated
.nodes
.iter()
.find(|n| n.path == self.node.path)
.ok_or_else(unknown)?;
.ok_or_else(|| {
diagnostic.record(Stage::Reconcile, Reason::InvalidMetadata, None);
unknown()
})?;
let value = json_to_wip(&json!({"digest": result.digest,
"activation": "Saved to canonical Workspace configuration; existing Worker manifests are not rewritten"})).map_err(|_| unknown())?;
"activation": "Saved to canonical Workspace configuration; existing Worker manifests are not rewritten"})).map_err(|_| {
diagnostic.record(Stage::Completion, Reason::Decode, None);
unknown()
})?;
Ok(WipOperationOutput::native_with_validator(
value,
object_validator(
@@ -1,8 +1,10 @@
//! Worker-side E2E across real Feature invocation, WIP Client/Host and typed
//! Workspace transport. Canonical persistence/projection tests live in Backend;
//! this deterministic router models its published auth/CAS/response contract.
//! Worker-side integration across real Feature invocation, WIP Client/Host and the production
//! WorkspaceConfigBackend. Only WorkspaceClient transport is substituted by a deterministic router
//! retaining applied effects and modeling the published auth/CAS/response contract. These ordinary
//! worker lib tests do not replace canonical persistence/projection tests owned by Backend.
use super::*;
use crate::feature::FeatureRegistryBuilder;
use crate::feature::builtin::backend_diagnostic::tests::Capture;
use crate::hook::HookRegistryBuilder;
use crate::wip::{WipMountRegistry, WipRuntime};
use crate::worker::{
@@ -16,6 +18,7 @@ use serde_json::{Value as Json, json};
use server_api::*;
use std::collections::{BTreeMap, BTreeSet};
use std::sync::{Arc, Mutex};
use tracing::instrument::WithSubscriber;
#[derive(Debug)]
struct State {
@@ -26,6 +29,10 @@ struct State {
entries: BTreeMap<String, (ConfigContentType, String)>,
commits: Vec<WorkspaceConfigCommitRequest>,
requests: Vec<WorkspaceRequest>,
applied_commits: usize,
malformed_read_reply: Option<&'static str>,
malformed_observe_reply: Option<&'static str>,
lose_postcommit_observe_reply: bool,
fail_commit: bool,
fail_current: bool,
lose_commit_reply: bool,
@@ -58,6 +65,10 @@ impl Router {
]),
commits: Vec::new(),
requests: Vec::new(),
applied_commits: 0,
malformed_read_reply: None,
malformed_observe_reply: None,
lose_postcommit_observe_reply: false,
fail_commit: false,
fail_current: false,
lose_commit_reply: false,
@@ -222,6 +233,11 @@ impl WorkspaceClient for Router {
}
match suffix {
"/observe" => {
if state.lose_postcommit_observe_reply && state.applied_commits > 0 {
return Err(WorkspaceClientError::Request(
"secret-token /private/host/config".into(),
));
}
let request: WorkspaceConfigObserveRequest = serde_json::from_value(input).unwrap();
assert!(request.depth <= 8 && request.paths.len() <= 256);
let mut kinds =
@@ -268,13 +284,23 @@ impl WorkspaceClient for Router {
)
})
.collect();
Ok(response(&WorkspaceConfigObserveResponse {
let mut result = WorkspaceConfigObserveResponse {
connection_id: state.active.as_ref().unwrap().connection_id.clone(),
validator: state.validator(""),
digest: state.tree_digest(),
entrypoints: vec!["main.dcdl".into()],
nodes,
}))
};
match state.malformed_observe_reply {
Some("connection") => {
result.connection_id = "secret-token /private/host/config".into()
}
Some("entrypoints") => result.entrypoints.clear(),
Some("scope") => result.nodes[0].path = "outside/path.dcdl".into(),
None => {}
Some(_) => unreachable!(),
}
Ok(response(&result))
}
"/read" => {
let request: WorkspaceConfigReadRequest = serde_json::from_value(input).unwrap();
@@ -290,13 +316,20 @@ impl WorkspaceClient for Router {
WorkspaceConfigFailureClassification::NotCommitted,
));
};
Ok(response(&WorkspaceConfigReadResponse {
let mut result = WorkspaceConfigReadResponse {
path: request.path.clone(),
content: content.clone(),
content_type: *content_type,
digest: digest(content),
validator: state.validator(&request.path),
}))
};
match state.malformed_read_reply {
Some("identity") => result.path = "secret-token /private/host/config".into(),
Some("size") => result.content = "x".repeat(256 * 1024 + 1),
None => {}
Some(_) => unreachable!(),
}
Ok(response(&result))
}
"/commit" => {
let request: WorkspaceConfigCommitRequest = serde_json::from_value(input).unwrap();
@@ -403,6 +436,7 @@ impl WorkspaceClient for Router {
));
}
state.entries = entries;
state.applied_commits += 1;
if state.lose_commit_reply {
state.lose_commit_reply = false;
return Err(WorkspaceClientError::Request(
@@ -797,14 +831,20 @@ async fn post_commit_observation_mismatch_reports_unknown_without_replaying_writ
let path = "/workspace-config/main.dcdl";
observe_interface(&runtime, path, true).await;
client.state.lock().unwrap().mismatched_commit_reply = Some(field);
let capture = Capture::default();
let error = call(&runtime, path, "write", json!({"content":"saved"}))
.with_subscriber(capture.dispatch())
.await
.unwrap_err()
.to_string();
assert!(error.contains("unknown"), "{field}: {error}");
assert!(capture.text().contains("stage=Reconcile"));
assert!(capture.text().contains("reason=Identity"));
assert!(!capture.text().contains("saved"));
let state = client.state.lock().unwrap();
assert_eq!(state.commits.len(), 1);
assert_eq!(state.entries["main.dcdl"].1, "saved");
assert_eq!(state.applied_commits, 1);
}
}
@@ -1155,3 +1195,110 @@ async fn workspace_config_backend_validators_reject_path_and_connection_transpla
assert!(result.content.unwrap().contains("model = updated"));
assert_eq!(client.state.lock().unwrap().commits.len(), 1);
}
#[tokio::test]
async fn production_read_validation_retains_reason_without_committing() {
for (fault, reason) in [("identity", "Identity"), ("size", "ResponseSize")] {
let client = Router::new();
let (feature, runtime) = runtime(client.clone());
feature.attach("initial", None).await.unwrap();
let path = "/workspace-config/main.dcdl";
observe_interface(&runtime, path, true).await;
client.state.lock().unwrap().malformed_read_reply = Some(fault);
let capture = Capture::default();
let error = call(&runtime, path, "read", json!({}))
.with_subscriber(capture.dispatch())
.await
.unwrap_err()
.to_string();
let text = capture.text();
assert!(text.contains(&format!("reason={reason}")), "{text}");
assert!(text.contains("stage=Response"));
assert!(!error.contains("unknown"));
assert!(!text.contains("secret-token") && !text.contains("/private/host"));
assert!(!error.contains("secret-token") && !error.contains("/private/host"));
let state = client.state.lock().unwrap();
assert_eq!(state.applied_commits, 0);
assert_eq!(state.entries["main.dcdl"].1, "model = old");
}
}
#[tokio::test]
async fn production_observation_validation_retains_reason_without_committing() {
for (fault, reason) in [
("connection", "Identity"),
("entrypoints", "InvalidMetadata"),
("scope", "InvalidMetadata"),
] {
let client = Router::new();
let (feature, runtime) = runtime(client.clone());
feature.attach("initial", None).await.unwrap();
client.state.lock().unwrap().malformed_observe_reply = Some(fault);
let capture = Capture::default();
let error = runtime
.tree("/workspace-config/main.dcdl".into(), 0, true)
.with_subscriber(capture.dispatch())
.await
.unwrap_err()
.to_string();
let text = capture.text();
assert!(text.contains(&format!("reason={reason}")), "{text}");
assert!(text.contains("stage=Response"));
assert!(!text.contains("secret-token") && !text.contains("/private/host"));
assert!(!error.contains("secret-token") && !error.contains("/private/host"));
assert_eq!(client.state.lock().unwrap().applied_commits, 0);
}
}
#[tokio::test]
async fn production_postcommit_transport_loss_retains_cause_and_unknown_without_retry() {
let client = Router::new();
let (feature, runtime) = runtime(client.clone());
feature.attach("initial", None).await.unwrap();
let path = "/workspace-config/main.dcdl";
observe_interface(&runtime, path, true).await;
client.state.lock().unwrap().lose_postcommit_observe_reply = true;
let capture = Capture::default();
let error = call(
&runtime,
path,
"write",
json!({"content":"saved-before-observation-loss"}),
)
.with_subscriber(capture.dispatch())
.await
.unwrap_err()
.to_string();
assert!(error.contains("unknown"));
let text = capture.text();
assert!(
text.lines().any(
|event| event.contains("reason=Communication") && event.contains("stage=Reconcile")
),
"{text}"
);
assert!(
!text.contains("reason=UnrecognizedResponse"),
"must not invent a response-decode cause for transport loss"
);
assert!(!text.contains("secret-token") && !text.contains("/private/host"));
assert!(!text.contains("saved-before-observation-loss"));
assert!(!error.contains("secret-token") && !error.contains("/private/host"));
let correlations: BTreeSet<_> = text
.lines()
.filter_map(|line| line.split("correlation=").nth(1))
.filter_map(|tail| tail.split(';').next())
.collect();
assert_eq!(
correlations.len(),
1,
"lost postcommit observation must correlate to original commit"
);
let state = client.state.lock().unwrap();
assert_eq!(state.commits.len(), 1);
assert_eq!(state.applied_commits, 1);
assert_eq!(
state.entries["main.dcdl"].1,
"saved-before-observation-loss"
);
}
@@ -161,7 +161,14 @@ impl WipOperationHandler for AttachHandler {
"attachment_path": format!("/workdir-attachments/{}", encode_identity(&result.connection_id)),
"content_path": CONTENT_ROOT,
"already_attached": result.already_attached,
})).map_err(|_| WipOperationError::OutcomeUnknown("Completed config attachment response unavailable".into()))?;
})).map_err(|_| {
super::backend::diagnostic("attach_output", true).record(
crate::feature::builtin::backend_diagnostic::Stage::Attach,
crate::feature::builtin::backend_diagnostic::Reason::Decode,
None,
);
WipOperationError::OutcomeUnknown("Completed config attachment response unavailable".into())
})?;
Ok(WipOperationOutput::native(value))
}
}
+273 -35
View File
@@ -8,6 +8,7 @@
//! the regular registry decides Tool-name collisions.
use std::collections::BTreeMap;
use std::io;
use std::path::Path;
use std::sync::Arc;
@@ -25,6 +26,7 @@ use mcp::stdio::{
};
use serde::Serialize;
use serde_json::{Map, Value};
use tokio::sync::Mutex;
use super::{
FeatureDescriptor, FeatureDiagnostic, FeatureInstallContext, FeatureInstallError,
@@ -52,8 +54,88 @@ const MAX_LIST_ITEMS: usize = 128;
const MAX_RESOURCE_CONTENTS: usize = 16;
const MAX_PROMPT_MESSAGES: usize = 32;
/// Owns discovery clients until process reaping is confirmed. Retain a clone in
/// startup cleanup *before* awaiting discovery. Stop/cancel discovery before
/// calling `shutdown`; the shared lock serializes discovery and cleanup, not
/// cancellation of an in-flight discovery task.
///
/// Error or cancellation leaves the actual clients here for another shutdown
/// attempt. Dropping the final owner only provides best-effort process killing.
#[derive(Clone, Default)]
pub struct McpDiscoveryCleanup {
clients: Arc<Mutex<Vec<McpStdioClient>>>,
}
impl McpDiscoveryCleanup {
pub fn new() -> Self {
Self::default()
}
/// Try every retained client once, removing only confirmed shutdowns. An
/// error is not a cleanup receipt; retry this same owner, not discovery.
pub async fn shutdown(&self) -> io::Result<()> {
let mut clients = self.clients.lock().await;
let mut first_error = None;
let mut index = 0;
while index < clients.len() {
match clients[index].shutdown().await {
Ok(_) => {
clients.remove(index);
}
Err(err) => {
first_error.get_or_insert_with(|| discovery_shutdown_error(&err));
index += 1;
}
}
}
match first_error {
Some(err) => Err(err),
None => Ok(()),
}
}
}
fn discovery_shutdown_error(err: &McpClientError) -> io::Error {
io::Error::other(bounded_diagnostic(format!(
"MCP discovery shutdown unconfirmed; client retained for cleanup retry: {err}"
)))
}
/// Restore preparation must fail if discovery cannot confirm process shutdown.
/// The caller retains `cleanup` before this future is polled, including during
/// initialization. Ordinary discovery failures remain isolated contributions
/// only after their client has been reaped.
pub async fn discover_stdio_tool_feature_with_cleanup(
config: &McpConfig,
workspace_root: &Path,
cleanup: &McpDiscoveryCleanup,
) -> Result<Option<McpStdioToolFeature>, io::Error> {
if config.stdio_servers.is_empty() {
return Ok(None);
}
let mut feature = McpStdioToolFeature::new();
for server in &config.stdio_servers {
match resolve_stdio_server(server, workspace_root, None) {
Ok(spec) => {
feature.add_contribution(
discover_server_tools_with_cleanup(spec, cleanup, McpStdioLimits::default())
.await?,
);
}
Err(err) => {
feature.add_diagnostic(FeatureDiagnostic::error(bounded_diagnostic(format!(
"failed to resolve MCP stdio server `{}`: {err}",
server.name
))))
}
}
}
Ok(Some(feature))
}
/// Discover enabled MCP stdio server tools and return a single feature module
/// containing startup contributions for normal ToolRegistry installation.
/// Callers needing retryable startup cleanup must use the cleanup-aware API.
pub async fn discover_stdio_tool_feature(
config: &McpConfig,
workspace_root: &Path,
@@ -81,21 +163,66 @@ pub async fn discover_stdio_tool_feature(
}
async fn discover_server_tools(spec: McpStdioServerSpec) -> ProtocolProviderContribution {
let declaration = provider_declaration(&spec.name, None);
let cleanup = McpDiscoveryCleanup::new();
match discover_server_tools_with_cleanup(spec, &cleanup, McpStdioLimits::default()).await {
Ok(contribution) => contribution,
Err(err) => {
ProtocolProviderContribution::failed(declaration, bounded_diagnostic(err.to_string()))
}
}
}
async fn discover_server_tools_with_cleanup(
spec: McpStdioServerSpec,
cleanup: &McpDiscoveryCleanup,
limits: McpStdioLimits,
) -> io::Result<ProtocolProviderContribution> {
let declaration = provider_declaration(&spec.name, None);
let mut clients = cleanup.clients.lock().await;
if !clients.is_empty() {
return Err(io::Error::other(
"MCP discovery cleanup is pending; shut down the retained owner before retrying discovery",
));
}
let client = match McpStdioClient::spawn_uninitialized(spec.clone(), limits) {
Ok(client) => client,
Err(err) => {
// The synchronous spawn API only errors before a process exists;
// pipe/handshake failures retain a client and use the shutdown path.
return Ok(ProtocolProviderContribution::failed(
declaration,
bounded_diagnostic(err.to_string()),
));
}
};
// Spawn is synchronous: the owner holds the client before the first await
// that can leave a running process on initialization failure/cancellation.
clients.push(client);
let client = clients.last_mut().expect("retained discovery client");
let contribution = match client.initialize_with_timeout().await {
Ok(()) => discover_connected_server_tools(spec, client).await,
Err(err) => {
ProtocolProviderContribution::failed(declaration, bounded_diagnostic(err.to_string()))
}
};
client
.shutdown()
.await
.map_err(|err| discovery_shutdown_error(&err))?;
drop(clients.pop());
Ok(contribution)
}
async fn discover_connected_server_tools(
spec: McpStdioServerSpec,
client: &mut McpStdioClient,
) -> ProtocolProviderContribution {
let declaration = provider_declaration(&spec.name, None);
let mut contribution = ProtocolProviderContribution::ready(declaration.clone());
let server_namespace = sanitize_segment(&spec.name);
let execution_spec = spec.clone();
let mut client = match McpStdioClient::connect(spec, McpStdioLimits::default()).await {
Ok(client) => client,
Err(err) => {
return ProtocolProviderContribution::failed(
declaration,
bounded_diagnostic(err.to_string()),
);
}
};
let (server_version, has_tools, has_resources, has_prompts) = client
.initialize_result()
.map(|result| {
@@ -133,18 +260,10 @@ async fn discover_server_tools(spec: McpStdioServerSpec) -> ProtocolProviderCont
{
Ok(list) => list,
Err(err) => {
let mut failed = ProtocolProviderContribution::failed(
return ProtocolProviderContribution::failed(
declaration,
bounded_diagnostic(err.to_string()),
);
if let Err(shutdown_err) = client.shutdown().await {
failed = failed.with_diagnostic(FeatureDiagnostic::warning(
bounded_diagnostic(format!(
"MCP server shutdown after discovery failure failed: {shutdown_err}"
)),
));
}
return failed;
}
};
let list_changes = client.snapshot_list_changes().await;
@@ -166,20 +285,12 @@ async fn discover_server_tools(spec: McpStdioServerSpec) -> ProtocolProviderCont
{
Ok(list) => list,
Err(err) => {
let mut failed = ProtocolProviderContribution::failed(
return ProtocolProviderContribution::failed(
declaration,
bounded_diagnostic(format!(
"MCP server sent notifications/tools/list_changed during tool discovery and refresh failed: {err}"
)),
);
if let Err(shutdown_err) = client.shutdown().await {
failed = failed.with_diagnostic(FeatureDiagnostic::warning(
bounded_diagnostic(format!(
"MCP server shutdown after discovery refresh failure failed: {shutdown_err}"
)),
));
}
return failed;
}
};
let refresh_changes = client.snapshot_list_changes().await;
@@ -197,13 +308,6 @@ async fn discover_server_tools(spec: McpStdioServerSpec) -> ProtocolProviderCont
} else {
None
};
let shutdown_result = client.shutdown().await;
if let Err(err) = shutdown_result {
contribution =
contribution.with_diagnostic(FeatureDiagnostic::warning(bounded_diagnostic(format!(
"MCP server shutdown after capability discovery failed: {err}"
))));
}
if let Some(list) = list {
contribution = normalize_listed_tools(
@@ -1819,6 +1923,140 @@ read exit_notification || true
McpStdioServerSpec::new("shell-mock", "/bin/sh").args(["-c".to_string(), script])
}
#[cfg(unix)]
fn discovery_config(spec: McpStdioServerSpec) -> McpConfig {
McpConfig {
stdio_servers: vec![manifest::McpStdioServerConfig {
name: spec.name,
command: spec.command,
args: spec.args,
cwd: None,
env: Default::default(),
}],
}
}
#[cfg(unix)]
#[tokio::test]
async fn cancelled_discovery_retains_initializing_client_in_cleanup_clone() {
use std::future::{Future, poll_fn};
use std::task::Poll;
use std::time::Duration;
let config = discovery_config(
McpStdioServerSpec::new("initializing", "/bin/sh")
.args(["-c", "while read line; do :; done"]),
);
let cleanup = McpDiscoveryCleanup::new();
let retained = cleanup.clone();
let mut discovery = Box::pin(discover_stdio_tool_feature_with_cleanup(
&config,
Path::new("."),
&cleanup,
));
poll_fn(|cx| {
assert!(discovery.as_mut().poll(cx).is_pending());
Poll::Ready(())
})
.await;
drop(discovery);
drop(cleanup);
assert_eq!(retained.clients.lock().await.len(), 1);
let retry_error =
discover_stdio_tool_feature_with_cleanup(&config, Path::new("."), &retained)
.await
.err()
.expect("discovery must not replace an unreaped client");
assert!(retry_error.to_string().contains("cleanup is pending"));
assert_eq!(retained.clients.lock().await.len(), 1);
tokio::time::timeout(Duration::from_secs(3), retained.shutdown())
.await
.expect("cleanup bounded")
.expect("initializing process reaped");
assert!(retained.clients.lock().await.is_empty());
retained
.shutdown()
.await
.expect("confirmed cleanup repeatable");
}
#[cfg(unix)]
#[tokio::test]
async fn cancelled_owner_shutdown_keeps_client_for_retry() {
use std::future::{Future, poll_fn};
use std::task::Poll;
use std::time::Duration;
let cleanup = McpDiscoveryCleanup::new();
cleanup.clients.lock().await.push(
McpStdioClient::spawn_uninitialized(
McpStdioServerSpec::new("shutdown-pending", "/bin/sh")
.args(["-c", "while :; do :; done"]),
McpStdioLimits {
shutdown_timeout: Duration::from_millis(50),
kill_timeout: Duration::from_millis(100),
..Default::default()
},
)
.expect("local process spawned"),
);
let mut shutdown = Box::pin(cleanup.shutdown());
poll_fn(|cx| {
assert!(shutdown.as_mut().poll(cx).is_pending());
Poll::Ready(())
})
.await;
drop(shutdown);
assert_eq!(cleanup.clients.lock().await.len(), 1);
tokio::time::timeout(Duration::from_secs(2), cleanup.shutdown())
.await
.expect("retry bounded")
.expect("retained client shutdown");
assert!(cleanup.clients.lock().await.is_empty());
}
#[cfg(unix)]
#[tokio::test]
async fn cleanup_aware_discovery_reaps_success_and_initialize_failure() {
let cleanup = McpDiscoveryCleanup::new();
let config = discovery_config(shell_capability_server(r#"{"resources":{}}"#));
let feature = discover_stdio_tool_feature_with_cleanup(&config, Path::new("."), &cleanup)
.await
.expect("discovery cleanup confirmed")
.expect("configured feature");
assert!(
feature.contributions[0]
.tools
.iter()
.any(|tool| { tool.name() == "Mcp_shell_mock_resources_read" })
);
assert!(cleanup.clients.lock().await.is_empty());
let spec = McpStdioServerSpec::new("init-rejected", "/bin/sh").args([
"-c",
r#"read init || exit 1
printf '%s\n' '{"jsonrpc":"2.0","id":1,"error":{"code":-32000,"message":"initialization rejected"}}'
while read line; do :; done
"#,
]);
let config = discovery_config(spec);
let feature = discover_stdio_tool_feature_with_cleanup(&config, Path::new("."), &cleanup)
.await
.expect("failure isolated only after cleanup")
.expect("failed contribution preserved");
assert!(feature.contributions[0].tools.is_empty());
assert!(
feature.contributions[0]
.diagnostics
.iter()
.any(|diagnostic| {
diagnostic.severity == FeatureDiagnosticSeverity::Error
&& diagnostic.message.contains("initialization rejected")
})
);
assert!(cleanup.clients.lock().await.is_empty());
}
#[test]
fn mcp_resource_prompt_operation_tools_are_namespaced_and_untrusted() {
let declaration = provider_declaration("demo", Some("1.0.0"));
+2 -66
View File
@@ -1,57 +1,6 @@
//! `WorkerEvent` send / receive helpers.
//!
//! This module owns the parent-facing lifecycle-event primitive
//! (`WorkerEvent`) that children fire upward on turn-end / error /
//! shutdown / scope-sub-delegation. Three responsibilities live here:
//!
//! - **Send** a `Method::WorkerEvent` to the parent socket, fire-and-forget,
//! logging failures without blocking the child.
//! - **Render** agent-visible variants into human-readable strings for the
//! parent's notification buffer. Control-plane-only variants may still have
//! a renderer for diagnostics, but receive-side classification keeps them
//! out of LLM history/context.
//! - **Apply side effects** on the parent (registry / worker-allocation
//! updates) so that the receive path is idempotent and tolerant of
//! out-of-order delivery.
//!
//! Transport is fire-and-forget — the ticket's decision is that
//! callbacks are an optimisation and `ListWorkers` + `reclaim_stale` are
//! the real fallback. This module is allowed to drop events on the
//! floor (with a warn log) rather than retry.
//!
//! `apply_event_side_effects` takes its dependencies (registry, scope
//! lock path, self identity) by reference so the caller owns lifetime
//! and locking concerns.
//! Agent-visible rendering of typed Internal SubWorker lifecycle events.
use std::path::{Path, PathBuf};
use std::sync::Arc;
use protocol::{Method, WorkerEvent};
use crate::spawn::comm_tools::connect_and_send;
use crate::spawn::registry::SpawnedWorkerRegistry;
/// Connect to `socket`, send a single `Method::WorkerEvent(event)`, and
/// return. Used by children to report up to their parent.
///
/// This is a synchronous helper — callers that want fire-and-forget
/// semantics should wrap the call in `tokio::spawn` themselves.
pub async fn send_worker_event(socket: &Path, event: WorkerEvent) -> std::io::Result<()> {
connect_and_send(socket, &Method::WorkerEvent(event)).await
}
/// Spawn a fire-and-forget task that sends `event` to `socket`. If
/// `socket` is `None`, no send happens (top-level Workers have no parent).
/// Any send failure is logged at warn level but otherwise ignored —
/// the parent is treated as best-effort.
pub fn fire_and_forget(socket: Option<PathBuf>, event: WorkerEvent) {
let Some(socket) = socket else { return };
tokio::spawn(async move {
if let Err(e) = send_worker_event(&socket, event).await {
tracing::warn!(error = %e, socket = %socket.display(), "WorkerEvent send failed");
}
});
}
use protocol::WorkerEvent;
/// Render a variant into a one-line human-readable string.
///
@@ -83,16 +32,3 @@ pub fn render_event(event: &WorkerEvent) -> String {
}
}
}
/// Legacy process callback events have no SubWorker registry authority.
///
/// Internal SubWorker lifecycle is applied directly through typed session handles. A callback from
/// an externally adopted Worker may still be rendered for diagnostics, but it cannot add/remove
/// Internal children or transfer filesystem authority.
pub async fn apply_event_side_effects(
_event: &WorkerEvent,
_registry: &Arc<SpawnedWorkerRegistry>,
_self_name: &str,
_self_parent_socket: &Option<PathBuf>,
) {
}
-1
View File
@@ -1,7 +1,6 @@
pub mod alerter;
pub mod event;
pub mod protocol_session;
pub mod server;
pub(crate) mod interceptor;
pub(crate) mod notify_buffer;
-224
View File
@@ -1,224 +0,0 @@
use std::io;
use std::io::ErrorKind;
use std::path::PathBuf;
use protocol::stream::{JsonLineReader, JsonLineWriter};
use tokio::net::UnixListener;
use tokio::sync::oneshot;
use tokio::task::{JoinHandle, JoinSet};
use crate::controller::WorkerHandle;
use crate::ipc::protocol_session::{
dispatch_worker_protocol_method, live_log_entry_event, subscribe_worker_protocol_session,
};
use protocol::{ErrorCode, Event};
/// Unix socket server for Worker Protocol.
///
/// Listens on the Worker's runtime directory socket path.
/// Each client connection gets bidirectional JSONL:
/// - Client writes Method lines → forwarded to WorkerController
/// - Worker events → written as Event lines to all connected clients
pub struct SocketServer {
accept_task: Option<JoinHandle<()>>,
shutdown: Option<oneshot::Sender<()>>,
path: PathBuf,
}
impl SocketServer {
/// Start listening on the WorkerHandle's socket path.
pub async fn start(handle: &WorkerHandle) -> Result<Self, io::Error> {
let path = handle.runtime_dir.socket_path();
// Remove stale socket file if it exists
let _ = tokio::fs::remove_file(&path).await;
let listener = UnixListener::bind(&path)?;
let handle = handle.clone();
let (shutdown_tx, mut shutdown_rx) = oneshot::channel();
let accept_task = tokio::spawn(async move {
let mut connections = JoinSet::new();
loop {
tokio::select! {
_ = &mut shutdown_rx => break,
accepted = listener.accept() => match accepted {
Ok((stream, _)) => {
let handle = handle.clone();
connections.spawn(handle_connection(stream, handle));
}
Err(_) => break,
},
completed = connections.join_next(), if !connections.is_empty() => {
let _ = completed;
}
}
}
connections.shutdown().await;
});
Ok(Self {
accept_task: Some(accept_task),
shutdown: Some(shutdown_tx),
path,
})
}
/// Stop accepting connections and join the server task. Dropping its
/// `JoinSet` cancels every active connection task before this returns.
pub async fn shutdown(mut self) {
if let Some(shutdown) = self.shutdown.take() {
let _ = shutdown.send(());
}
if let Some(task) = self.accept_task.take() {
let _ = task.await;
}
let _ = tokio::fs::remove_file(&self.path).await;
}
/// The socket file path.
pub fn path(&self) -> &std::path::Path {
&self.path
}
}
impl Drop for SocketServer {
fn drop(&mut self) {
if let Some(shutdown) = self.shutdown.take() {
let _ = shutdown.send(());
}
let _ = self.accept_task.take();
let _ = std::fs::remove_file(&self.path);
}
}
fn is_peer_disconnect_read_error(error: &io::Error) -> bool {
matches!(
error.kind(),
ErrorKind::ConnectionReset
| ErrorKind::ConnectionAborted
| ErrorKind::BrokenPipe
| ErrorKind::UnexpectedEof
)
}
async fn handle_connection(stream: tokio::net::UnixStream, handle: WorkerHandle) {
let (reader, writer) = stream.into_split();
let mut reader = JsonLineReader::new(reader);
let mut writer = JsonLineWriter::new(writer);
// Hold the in-flight stream lock while taking the session-log mirror
// snapshot. `LogEntry::AnnotatedAssistantItem` is mirror-only for live clients,
// so a finalized assistant block must be observed either as an already
// committed entry or as the still-present in-flight block. This lock
// order matches `append_entry` (in-flight clear before sink publish) and
// keeps the snapshot/live boundary gap-free.
let mut streams = subscribe_worker_protocol_session(&handle);
for alert in streams.alert_snapshot {
if writer.write_event(&Event::Alert(alert)).await.is_err() {
return;
}
}
// Send the typed snapshot up front so late attachers can
// reconstruct view state without an extra round trip.
if writer.write_event(&streams.snapshot_event).await.is_err() {
return;
}
loop {
tokio::select! {
// Live session-log entries → dispatched as the role-specific
// wire events. `SegmentLogSink` only broadcasts committed log
// entries with live UI meaning; `UserInput` travels this lane so
// the visible user line is ordered with `SegmentStart` rotation.
entry = streams.log_entries.recv() => {
match entry {
Ok(entry) => {
if let Some(event) = live_log_entry_event(entry) {
if writer.write_event(&event).await.is_err() {
break;
}
}
}
Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {
// Slow client fell behind the broadcast buffer.
// Drop the connection so the next reconnect
// re-seeds the prefix via subscribe_with_snapshot.
break;
}
Err(tokio::sync::broadcast::error::RecvError::Closed) => break,
}
}
// Broadcast events → this client
event = streams.events.recv() => {
match event {
Ok(event) => {
let Ok(event) = handle.alerter.freshen_event(event) else {
// The queued Alert was evicted; reconnect must rebuild
// the bounded prefix rather than forward a stale anchor.
break;
};
if writer.write_event(&event).await.is_err() {
break;
}
}
Err(_) => break,
}
}
// Client methods → handle or forward to controller
method = reader.next_method() => {
match method {
Ok(Some(method)) => {
if let Some(response) = dispatch_worker_protocol_method(&handle, method).await
&& writer.write_event(&response).await.is_err()
{
break;
}
}
Ok(None) => break,
Err(e) if is_peer_disconnect_read_error(&e) => break,
Err(e) => {
if writer
.write(&Event::Error {
code: ErrorCode::InvalidRequest,
message: format!("invalid method: {e}"),
})
.await
.is_err()
{
break;
}
}
}
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn peer_disconnect_read_errors_are_connection_close() {
for kind in [
ErrorKind::ConnectionReset,
ErrorKind::ConnectionAborted,
ErrorKind::BrokenPipe,
ErrorKind::UnexpectedEof,
] {
let error = io::Error::new(kind, "peer disconnected");
assert!(
is_peer_disconnect_read_error(&error),
"{kind:?} should be treated as a normal peer disconnect"
);
}
}
#[test]
fn invalid_data_is_not_peer_disconnect() {
let error = io::Error::new(ErrorKind::InvalidData, "malformed method");
assert!(!is_peer_disconnect_read_error(&error));
}
}
+1 -2
View File
@@ -13,7 +13,6 @@
mod result;
pub use result::{JobResultFeature, JobResultFinalizer, JobResultSink, SUBMIT_JOB_RESULT_TOOL};
use std::borrow::Cow;
use std::path::Path;
use std::sync::Arc;
@@ -154,7 +153,7 @@ fn prepare_job_from_registry_with_grant(
)?;
let mut system_prompt = template.render(&SystemPromptContext {
now: chrono::Utc::now(),
cwd: Cow::Borrowed("(no filesystem)"),
workdir_sessions: &workdir::WorkdirSessionRouter::new(),
language: &manifest.engine.language,
scope: &scope,
tool_names,
+5 -5
View File
@@ -179,11 +179,11 @@ async fn builtin_journal_profile_installs_only_bound_tools_and_finalizes_saved_s
call(SUBMIT_JOB_RESULT_TOOL, json!({"result":{}})),
prose(),
]);
// This runner/Feature fixture has no selected sources; descriptor delivery gates
// for nonempty ranges are owned and tested by the Subjektiv service.
// This runner/Feature fixture has no selected sources; actual body-bearing input,
// trusted delivery and checkpoint gates are covered by the Host integration suites.
let mut journal_request = request("builtin:subjektiv-journal-recording");
journal_request.input = json!({"selected_ranges":[],"origins":[]});
journal_request.instruction = "Read immutable selected_ranges through ReadJournalText Source before recording; descriptors are not source bodies.".into();
journal_request.input = json!({"selected_ranges":[],"capture":{"entries":[]}});
journal_request.instruction = "Record only the immutable canonical bodies in this Job input. Do not read original Sessions through ReadJournalText Source.".into();
let job = prepare_job_from_registry_with_grant(
&journal_request,
root.path(),
@@ -249,7 +249,7 @@ async fn builtin_journal_profile_installs_only_bound_tools_and_finalizes_saved_s
"ReadJournalText",
"Source",
"content_token",
"Conflict",
"input digest",
] {
assert!(
prompt.contains(contract),
+8 -7
View File
@@ -7,8 +7,6 @@ pub mod bootstrap;
pub mod checkout;
pub mod compact;
pub mod controller;
pub mod discovery;
pub mod entrypoint;
pub mod feature;
mod file_operation;
pub mod fs_view;
@@ -19,9 +17,12 @@ pub mod job;
mod metadata_lifecycle;
pub mod model_client;
mod paste_artifact_tool;
pub mod profile_manifest;
pub mod prompt;
#[cfg(test)]
#[path = "../../../tests/support/restore_environment.rs"]
mod restore_test_environment;
pub mod runtime;
pub mod runtime_command;
pub mod segment_log_sink;
mod session_capture;
mod session_history;
@@ -37,6 +38,7 @@ pub mod wip;
mod internal_worker;
mod interrupt_prep;
mod peer;
mod permission;
mod worker;
@@ -45,10 +47,9 @@ pub use bootstrap::{
WorkerBootstrapLayout, start_worker_controller,
};
pub use compact::token_counter::{EstimateSource, SplitPoint, TokenEstimate};
pub use controller::{ShutdownReceiver, WorkerController, WorkerControllerTransport, WorkerHandle};
pub use controller::{ShutdownReceiver, WorkerController, WorkerHandle};
pub use hook::{Hook, HookEventKind, HookRegistryBuilder};
pub use ipc::alerter::Alerter;
pub use ipc::server::SocketServer;
pub use manifest::{
AuthRef, ModelManifest, SchemeKind, Scope, WorkerManifest, WorkerManifestConfig,
WorkerMetaConfig,
@@ -73,8 +74,8 @@ pub use shared_state::WorkerSharedState;
pub use worker::{
LocalWorkingDirectory, ReviewerChildWorkspaceClient, ReviewerContext,
SubjektivSessionAttributionLifecycle, Worker, WorkerError, WorkerFilesystemAuthority,
WorkerRunResult, WorkerWorkspaceContext, WorkspaceBinaryRequest, WorkspaceBinaryResponse,
WorkspaceClient, WorkspaceClientError, WorkspaceId, WorkspaceIdError,
WorkerRunResult, WorkerStartupFeatureCleanup, WorkerWorkspaceContext, WorkspaceBinaryRequest,
WorkspaceBinaryResponse, WorkspaceClient, WorkspaceClientError, WorkspaceId, WorkspaceIdError,
WorkspacePromptCatalogResolution, WorkspaceRequest, WorkspaceRequestMethod, WorkspaceResponse,
WorkspaceServerOperation, WorkspaceWorkerDiscoveryRequest, apply_worker_manifest,
marker_workspace_client, unavailable_workspace_client,
+186
View File
@@ -0,0 +1,186 @@
//! Reciprocal peer metadata registration, independent of discovery and transport.
//!
//! Registering a peer does not start a Worker or grant filesystem, observation, or
//! messaging authority. The two metadata writes are not a transaction: a failed
//! reciprocal write triggers best-effort restoration of the source's prior peers.
use serde::Serialize;
use session_store::{WorkerMetadataStore, WorkerStoreError, validate_worker_name};
#[derive(Debug, Serialize, PartialEq, Eq)]
pub(crate) struct PeerRegistrationResult {
pub source: String,
pub peer: String,
}
#[derive(Debug, thiserror::Error)]
pub(crate) enum PeerRegistrationError {
#[error("worker state missing for `{worker_name}`")]
StateMissing { worker_name: String },
#[error("worker `{worker_name}` cannot be registered as a peer of itself")]
SelfPeer { worker_name: String },
#[error("worker `{worker_name}` does not exist")]
MissingWorker { worker_name: String },
#[error("worker store error: {0}")]
WorkerStore(#[from] WorkerStoreError),
}
pub(crate) fn register_peer(
store: &impl WorkerMetadataStore,
source: &str,
peer: &str,
) -> Result<PeerRegistrationResult, PeerRegistrationError> {
ensure_existing_peer(store, source, peer)?.ok_or_else(|| PeerRegistrationError::MissingWorker {
worker_name: peer.to_string(),
})
}
fn ensure_existing_peer(
store: &impl WorkerMetadataStore,
source: &str,
peer: &str,
) -> Result<Option<PeerRegistrationResult>, PeerRegistrationError> {
validate_worker_name(peer)?;
if peer == source {
return Err(PeerRegistrationError::SelfPeer {
worker_name: peer.to_string(),
});
}
let prior_source_peers = store
.read_by_name(source)?
.ok_or_else(|| PeerRegistrationError::StateMissing {
worker_name: source.to_string(),
})?
.peers;
if store.read_by_name(peer)?.is_none() {
return Ok(None);
}
store.add_peer(source, peer)?;
if let Err(error) = store.add_peer(peer, source) {
let _ = store.set_peers(source, prior_source_peers);
return Err(PeerRegistrationError::WorkerStore(error));
}
Ok(Some(PeerRegistrationResult {
source: source.to_string(),
peer: peer.to_string(),
}))
}
#[cfg(test)]
mod tests {
use super::*;
use session_store::{FsWorkerStore, WorkerMetadata};
#[test]
fn register_peer_persists_reciprocal_metadata_without_duplicates() {
let root = tempfile::tempdir().unwrap();
let store = FsWorkerStore::new(root.path()).unwrap();
store.write(&WorkerMetadata::new("source", None)).unwrap();
store.write(&WorkerMetadata::new("target", None)).unwrap();
for _ in 0..2 {
let result = register_peer(&store, "source", "target").unwrap();
assert_eq!(
serde_json::to_value(result).unwrap(),
serde_json::json!({
"source": "source", "peer": "target",
})
);
}
// Reopen the real store rather than asserting only the write calls.
let persisted = FsWorkerStore::new(root.path()).unwrap();
let source = persisted.read_by_name("source").unwrap().unwrap();
let target = persisted.read_by_name("target").unwrap().unwrap();
assert_eq!(source.peers.len(), 1);
assert_eq!(source.peers[0].worker_name, "target");
assert_eq!(target.peers.len(), 1);
assert_eq!(target.peers[0].worker_name, "source");
}
#[test]
fn invalid_self_or_missing_peer_leaves_metadata_unchanged() {
let root = tempfile::tempdir().unwrap();
let store = FsWorkerStore::new(root.path()).unwrap();
store.write(&WorkerMetadata::new("source", None)).unwrap();
assert!(matches!(
register_peer(&store, "source", "source"),
Err(PeerRegistrationError::SelfPeer { .. })
));
assert!(matches!(
register_peer(&store, "source", "missing"),
Err(PeerRegistrationError::MissingWorker { .. })
));
assert!(matches!(
register_peer(&store, "source", "../escape"),
Err(PeerRegistrationError::WorkerStore(_))
));
assert!(matches!(
register_peer(&store, "missing-source", "source"),
Err(PeerRegistrationError::StateMissing { .. })
));
assert!(
store
.read_by_name("source")
.unwrap()
.unwrap()
.peers
.is_empty()
);
assert_eq!(store.list_names().unwrap(), vec!["source"]);
}
struct FailTargetPeerStore(FsWorkerStore);
impl WorkerMetadataStore for FailTargetPeerStore {
fn write(&self, metadata: &WorkerMetadata) -> Result<(), WorkerStoreError> {
if metadata.worker_name == "target" && !metadata.peers.is_empty() {
return Err(WorkerStoreError::Io(std::io::Error::other(
"target write failed",
)));
}
self.0.write(metadata)
}
fn read_by_name(&self, name: &str) -> Result<Option<WorkerMetadata>, WorkerStoreError> {
self.0.read_by_name(name)
}
fn list_names(&self) -> Result<Vec<String>, WorkerStoreError> {
self.0.list_names()
}
fn delete_by_name(&self, name: &str) -> Result<(), WorkerStoreError> {
self.0.delete_by_name(name)
}
}
#[test]
fn reciprocal_write_failure_restores_exact_prior_source_peers() {
let root = tempfile::tempdir().unwrap();
let inner = FsWorkerStore::new(root.path()).unwrap();
for name in ["source", "target", "existing"] {
inner.write(&WorkerMetadata::new(name, None)).unwrap();
}
inner.add_peer("source", "existing").unwrap();
for had_target in [false, true] {
if had_target {
inner.add_peer("source", "target").unwrap();
}
let before = inner.read_by_name("source").unwrap().unwrap().peers;
let store = FailTargetPeerStore(inner.clone());
assert!(matches!(
register_peer(&store, "source", "target"),
Err(PeerRegistrationError::WorkerStore(_))
));
assert_eq!(inner.read_by_name("source").unwrap().unwrap().peers, before);
assert!(
inner
.read_by_name("target")
.unwrap()
.unwrap()
.peers
.is_empty()
);
}
}
}
+183
View File
@@ -0,0 +1,183 @@
//! Runtime manifest preparation from Backend-resolved profiles.
//!
//! This module has no process entrypoint or local profile discovery authority.
use std::path::{Path, PathBuf};
use crate::PromptCatalogSource;
use manifest::{Permission, ScopeConfig, ScopeRule, WorkerManifest};
pub fn resolve_runtime_profile_manifest_from_manifest(
mut manifest: WorkerManifest,
workspace_root: &Path,
worker_name: &str,
) -> Result<(WorkerManifest, PromptCatalogSource), String> {
if manifest.worker.name.is_empty() {
manifest.worker.name = worker_name.to_string();
}
apply_profile_launch_policy(&mut manifest, workspace_root);
// Do not run plugin discovery here: runtime-created Workers receive their
// resolved manifest/profile archive from Backend authority, not by scanning
// materialized workdir-local plugin stores.
Ok((manifest, PromptCatalogSource::builtins_only()))
}
pub fn resolve_runtime_profile_manifest_from_manifest_without_filesystem(
mut manifest: WorkerManifest,
_workspace_root: &Path,
worker_name: &str,
) -> Result<(WorkerManifest, PromptCatalogSource), String> {
if manifest.worker.name.is_empty() {
manifest.worker.name = worker_name.to_string();
}
manifest.scope = ScopeConfig::default();
manifest.delegation_scope = ScopeConfig::default();
// Same as the filesystem-capable runtime path: no local discovery.
Ok((manifest, PromptCatalogSource::builtins_only()))
}
fn read_rule(target: PathBuf) -> ScopeRule {
ScopeRule {
target,
permission: Permission::Read,
recursive: true,
symlink_policy: Default::default(),
}
}
fn write_rule(target: PathBuf) -> ScopeRule {
ScopeRule {
target,
permission: Permission::Write,
recursive: true,
symlink_policy: Default::default(),
}
}
fn workspace_scope(
workspace_root: &Path,
permission: Permission,
deny_write: &[PathBuf],
) -> ScopeConfig {
let allow_rule = ScopeRule {
target: workspace_root.to_path_buf(),
permission,
recursive: true,
symlink_policy: Default::default(),
};
let deny = deny_write
.iter()
.cloned()
.map(write_rule)
.collect::<Vec<_>>();
ScopeConfig {
allow: vec![allow_rule],
deny,
}
}
fn workspace_worktree_delegation(workspace_root: &Path) -> ScopeConfig {
ScopeConfig {
allow: vec![
read_rule(workspace_root.to_path_buf()),
write_rule(workspace_root.join(".worktree")),
],
deny: Vec::new(),
}
}
fn append_missing_rules(target: &mut Vec<ScopeRule>, defaults: Vec<ScopeRule>) {
for rule in defaults {
if !target.contains(&rule) {
target.push(rule);
}
}
}
fn apply_scope_launch_defaults(scope: &mut ScopeConfig, defaults: ScopeConfig) {
// Profile resolution has already applied explicit profile/workspace override scope rules.
// Launch policy contributes runtime defaults on top rather than replacing those grants.
append_missing_rules(&mut scope.allow, defaults.allow);
append_missing_rules(&mut scope.deny, defaults.deny);
}
fn apply_profile_launch_policy(manifest: &mut WorkerManifest, workspace_root: &Path) {
let worktree_root = workspace_root.join(".worktree");
let default_scope = workspace_scope(
workspace_root,
Permission::Write,
std::slice::from_ref(&worktree_root),
);
apply_scope_launch_defaults(&mut manifest.scope, default_scope);
manifest.delegation_scope = workspace_worktree_delegation(workspace_root);
}
#[cfg(test)]
mod tests {
use super::*;
fn manifest() -> WorkerManifest {
let mut config = manifest::WorkerManifestConfig::resolution_defaults();
config.worker.name = Some("resolved".into());
WorkerManifest::try_from(config).unwrap()
}
#[test]
fn runtime_launch_preserves_explicit_scope_without_repository_discovery() {
let root = tempfile::tempdir().unwrap();
std::fs::create_dir(root.path().join(".yoi")).unwrap();
std::fs::write(root.path().join(".yoi/manifest.toml"), "invalid TOML").unwrap();
let mut original = manifest();
let explicit = read_rule(root.path().join("reference"));
original.scope.allow.push(explicit.clone());
let (resolved, _) =
resolve_runtime_profile_manifest_from_manifest(original, root.path(), "fallback")
.unwrap();
assert_eq!(resolved.worker.name, "resolved");
assert!(resolved.scope.allow.contains(&explicit));
assert!(
resolved
.scope
.allow
.contains(&write_rule(root.path().to_path_buf()))
);
assert!(
resolved
.scope
.deny
.contains(&write_rule(root.path().join(".worktree")))
);
assert_eq!(
resolved.delegation_scope.allow,
vec![
read_rule(root.path().to_path_buf()),
write_rule(root.path().join(".worktree")),
]
);
}
#[test]
fn runtime_without_filesystem_clears_scope_and_delegation_grants() {
let mut original = manifest();
original.worker.name.clear();
original
.scope
.allow
.push(write_rule(PathBuf::from("/work")));
original
.delegation_scope
.allow
.push(write_rule(PathBuf::from("/work")));
let (resolved, _) = resolve_runtime_profile_manifest_from_manifest_without_filesystem(
original,
Path::new("/unused"),
"runtime-worker",
)
.unwrap();
assert_eq!(resolved.worker.name, "runtime-worker");
assert!(resolved.scope.allow.is_empty());
assert!(resolved.scope.deny.is_empty());
assert!(resolved.delegation_scope.allow.is_empty());
assert!(resolved.delegation_scope.deny.is_empty());
}
}
+85 -10
View File
@@ -7,13 +7,12 @@
//! eagerly syntax-checks it at Worker construction. The final system
//! prompt is materialised exactly once just before the first LLM turn:
//! the rendered body is appended with a fixed trailing section carrying
//! the Worker's `Scope` summary, (if present) the project's `AGENTS.md`
//! contents, resident memory sections, and conditional Worker-orchestration
//! guidance, then the whole string is handed to the Engine via
//! `set_system_prompt`. Subsequent turns and compactions reuse that
//! the Worker's `Scope` summary, actual Workdir session environments, (if
//! present) the project's `AGENTS.md` contents, resident memory sections, and
//! conditional Worker-orchestration guidance, then the whole string is handed
//! to the Engine via `set_system_prompt`. Subsequent turns and compactions reuse that
//! materialised string verbatim.
use std::borrow::Cow;
use std::collections::BTreeMap;
use std::sync::Arc;
@@ -105,7 +104,8 @@ impl std::fmt::Debug for SystemPromptTemplate {
/// templates cannot drop them on the floor.
pub struct SystemPromptContext<'a> {
pub now: DateTime<Utc>,
pub cwd: Cow<'a, str>,
/// The same session router used by the Worker's filesystem tools.
pub workdir_sessions: &'a workdir::WorkdirSessionRouter,
/// Language policy exposed to instruction templates as `{{ language }}`.
pub language: &'a str,
pub scope: &'a Scope,
@@ -140,7 +140,14 @@ impl<'a> SystemPromptContext<'a> {
"datetime".into(),
Value::from(self.now.to_rfc3339_opts(SecondsFormat::Secs, true)),
);
root.insert("cwd".into(), Value::from(self.cwd.as_ref()));
// A singular template cwd is meaningful only with one attachment. Never
// substitute the local Profile/Memory discovery directory for a session cwd.
let cwd = self
.workdir_sessions
.only_session()
.map(|session| display_workdir_path(&session.environment().cwd).to_owned())
.unwrap_or_else(|| "no single Workdir session cwd; see execution environment".into());
root.insert("cwd".into(), Value::from(cwd));
root.insert("language".into(), Value::from(self.language));
root.insert(
"tools".into(),
@@ -255,6 +262,70 @@ fn exact_prompt_name(reference: &str) -> Option<String> {
}
}
fn display_workdir_path(path: &workdir::WorkdirPath) -> &str {
if path.as_str().is_empty() {
"."
} else {
path.as_str()
}
}
/// Project provider-owned session metadata, rather than reconstructing authority
/// from a manifest Scope or the directory used to discover Profiles and Memory.
pub(crate) fn workdir_environment_section(
router: &workdir::WorkdirSessionRouter,
) -> Result<String, SystemPromptError> {
use workdir::WorkdirSessionCapability::{Command, Edit, Glob, Grep, Read, Write};
let mut environments = Vec::new();
for alias in router.aliases() {
let resolved = router
.resolve(Some(alias.as_str()))
.map_err(|error| SystemPromptError::Render(error.to_string()))?;
let environment = resolved.session.environment();
let capabilities: Vec<_> = [
(Read, "read"),
(Write, "write"),
(Edit, "edit"),
(Glob, "glob"),
(Grep, "grep"),
(Command, "command"),
]
.into_iter()
.filter_map(|(capability, name)| {
environment
.capabilities
.supports(capability)
.then_some(name)
})
.collect();
environments.push(serde_json::json!({
"alias": alias.as_str(),
"cwd": display_workdir_path(&environment.cwd),
"scope": environment.scope,
"restrictions": environment.restrictions,
"capabilities": capabilities,
"command": environment.capabilities.supports(Command),
}));
}
let json = serde_json::to_string_pretty(&environments)
.map_err(|error| SystemPromptError::Render(error.to_string()))?;
Ok(format!(
"## Workdir execution environment\n\n\
Select a Workdir by its alias (`target_workdir`); omission is allowed only with one attachment. \
Relative filesystem operation paths are resolved from that selected session's cwd, not its Workdir root. \
Paths returned by Glob/Grep use the same cwd-relative coordinates and can be passed unchanged to other filesystem tools. \
Bash starts at the session cwd; an explicit relative cwd is resolved from it, and shell cd does not change the session cwd.\n\n\
In the metadata below, cwd and relative scope targets are Workdir-root-relative (`.` or an empty path denotes the root); explicit absolute provider scope targets stay absolute. \
A null scope means the provider's full available scope, not unrestricted host access; an empty scope grants no paths. \
Restrictions retain additional provider or ancestor ceilings, including deny rules. \
Capabilities and command availability are session ceilings, not new grants. \
Resolved targets remain subject to scope, symlink and command authorization. \
SubWorker scope declarations remain Workdir-root-relative; other Scope declarations and Profile/Memory roots retain their own meaning and are not rebased by session cwd.\n\n\
```json\n{json}\n```"
))
}
/// Build the final system prompt by appending the fixed trailing
/// section to `body`. The Rust side owns the layout (blank-line
/// separators, trailing-whitespace trim); each section's header + body
@@ -276,8 +347,12 @@ fn append_trailing_section(
}
out.push('\n');
let boundaries = prompts.working_boundaries_section(&scope.summary())?;
out.push_str(boundaries.trim_end_matches(&['\n', ' '][..]));
if ctx.workdir_sessions.is_empty() {
let boundaries = prompts.working_boundaries_section(&scope.summary())?;
out.push_str(boundaries.trim_end_matches(&['\n', ' '][..]));
out.push('\n');
}
out.push_str(&workdir_environment_section(ctx.workdir_sessions)?);
out.push('\n');
if let Some(agents) = agents_md {
out.push('\n');
@@ -392,7 +467,7 @@ mod tests {
now: DateTime::parse_from_rfc3339("2026-10-08T00:00:00Z")
.unwrap()
.with_timezone(&Utc),
cwd: Cow::Borrowed("."),
workdir_sessions: &workdir::WorkdirSessionRouter::new(),
language: "en",
scope: &scope,
tool_names: metas.iter().map(|meta| meta.name.clone()).collect(),
+77 -233
View File
@@ -1,17 +1,12 @@
use std::io;
use std::path::{Path, PathBuf};
use manifest::{ScopeRule, paths};
use manifest::ScopeRule;
use serde::{Deserialize, Serialize};
use tokio::fs;
use crate::shared_state::WorkerSharedState;
/// One spawned-child record mirrored to `spawned_workers.json`.
///
/// Written by the spawner after registry changes so runtime-local tools
/// have a materialised snapshot. Durable restore uses Worker state metadata;
/// this file is not the authoritative source.
/// Legacy process-child projection used only by persisted scope reclamation.
/// Socket fields retain the stored format; they are never connected to.
#[derive(Debug, Clone, Serialize, Deserialize)]
pub struct SpawnedWorkerRecord {
/// Spawned Worker's identity.
@@ -25,56 +20,33 @@ pub struct SpawnedWorkerRecord {
pub callback_address: PathBuf,
}
/// Manages the Worker's runtime directory on tmpfs.
/// Artifact directory retained by an in-process Controller.
///
/// ```text
/// <runtime_dir>/{worker_name}/
/// ├── pid
/// ├── status.json
/// ├── manifest.toml
/// ├── history.json
/// └── sock (created by socket listener, not by RuntimeDir)
/// ```
///
/// `<runtime_dir>` is resolved via [`manifest::paths::runtime_dir`].
/// Files are written atomically (write tmp → rename).
/// The directory is removed on drop.
/// Worker liveness and configuration belong to the Host, not filesystem projections.
/// Persistent run directories and pre-existing data are never removed on drop.
pub struct RuntimeDir {
path: PathBuf,
write_legacy_snapshots: bool,
preserve_on_drop: bool,
socket_file_name: &'static str,
}
impl RuntimeDir {
/// Create the runtime directory and write the PID file.
/// Create an ephemeral directory without PID, status, manifest or socket files.
pub async fn create(base: &Path, worker_name: &str) -> Result<Self, io::Error> {
let path = base.join(worker_name);
fs::create_dir_all(&path).await?;
let pid = std::process::id().to_string();
fs::write(path.join("pid"), pid.as_bytes()).await?;
Ok(Self {
path,
write_legacy_snapshots: true,
preserve_on_drop: false,
socket_file_name: "sock",
})
Self::create_transient(base, worker_name).await
}
/// Create an ephemeral Runtime-owned artifact directory.
///
/// Runtime-managed Workers keep their status in the owning Runtime and do
/// not materialize legacy pid/status/manifest projections.
/// Preserve pre-existing directories rather than claiming their contents for cleanup.
pub async fn create_transient(base: &Path, worker_name: &str) -> Result<Self, io::Error> {
fs::create_dir_all(base).await?;
let path = base.join(worker_name);
fs::create_dir_all(&path).await?;
let preserve_on_drop = match fs::create_dir(&path).await {
Ok(()) => false,
Err(error) if error.kind() == io::ErrorKind::AlreadyExists && path.is_dir() => true,
Err(error) => return Err(error),
};
Ok(Self {
path,
write_legacy_snapshots: false,
preserve_on_drop: false,
socket_file_name: "sock",
preserve_on_drop,
})
}
@@ -100,39 +72,11 @@ impl RuntimeDir {
std::fs::File::open(parent)?.sync_all()?;
Ok(Self {
path: path.to_path_buf(),
write_legacy_snapshots: false,
preserve_on_drop: true,
socket_file_name: "worker.sock",
})
}
/// Create in the default base directory resolved via
/// [`manifest::paths::runtime_dir`].
pub async fn create_default(worker_name: &str) -> Result<Self, io::Error> {
let base = default_base()?;
Self::create(&base, worker_name).await
}
/// Write status.json atomically.
pub async fn write_status(&self, state: &WorkerSharedState) -> Result<(), io::Error> {
if !self.write_legacy_snapshots {
return Ok(());
}
let content = state.status_json();
atomic_write(&self.path.join("status.json"), content.as_bytes()).await
}
/// Write manifest.toml (typically once at startup).
pub async fn write_manifest(&self, toml: &str) -> Result<(), io::Error> {
if !self.write_legacy_snapshots {
return Ok(());
}
atomic_write(&self.path.join("manifest.toml"), toml.as_bytes()).await
}
/// Write `spawned_workers.json` atomically. The entries are the full
/// set of spawned children known to this Worker — callers pass the
/// replacement list, no incremental merge.
/// Rewrite the legacy projection during persisted process-child reclamation.
pub async fn write_spawned_workers(
&self,
records: &[SpawnedWorkerRecord],
@@ -145,26 +89,10 @@ impl RuntimeDir {
pub fn path(&self) -> &Path {
&self.path
}
/// Path where the Unix socket should be created. External callers
/// that only know the worker name (e.g. the TUI's attach flow)
/// predict the same path via [`manifest::paths::worker_socket_path`].
pub fn socket_path(&self) -> PathBuf {
self.path.join(self.socket_file_name)
}
pub async fn close_socket(&self) -> Result<(), io::Error> {
match fs::remove_file(self.socket_path()).await {
Ok(()) => Ok(()),
Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()),
Err(error) => Err(error),
}
}
}
impl Drop for RuntimeDir {
fn drop(&mut self) {
let _ = std::fs::remove_file(self.socket_path());
if !self.preserve_on_drop {
let _ = std::fs::remove_dir_all(&self.path);
}
@@ -179,156 +107,72 @@ async fn atomic_write(target: &Path, content: &[u8]) -> Result<(), io::Error> {
Ok(())
}
/// Resolve the default base directory for runtime data.
///
/// Thin wrapper over [`manifest::paths::runtime_dir`] that converts a
/// missing-env situation into an `io::Error`.
pub fn default_base() -> Result<PathBuf, io::Error> {
paths::runtime_dir().ok_or_else(|| {
io::Error::new(
io::ErrorKind::NotFound,
"could not resolve runtime directory (no YOI_HOME / \
YOI_RUNTIME_DIR / XDG_RUNTIME_DIR / HOME)",
)
})
}
#[cfg(test)]
mod tests {
use super::*;
use crate::shared_state::WorkerSharedState;
fn test_state() -> WorkerSharedState {
WorkerSharedState::new(
"test-worker".into(),
session_store::new_segment_id(),
"[engine]\nname = \"test-worker\"".into(),
protocol::Greeting {
worker_name: "test-worker".into(),
cwd: "/tmp".into(),
provider: "anthropic".into(),
model: "claude".into(),
scope_summary: String::new(),
tools: Vec::new(),
context_window: 200_000,
context_tokens: 0,
reasoning: None,
context_usage: None,
},
)
}
#[tokio::test]
async fn creates_directory_and_pid() {
let tmp = tempfile::tempdir().unwrap();
let rt = RuntimeDir::create(tmp.path(), "my-worker").await.unwrap();
assert!(rt.path().join("pid").exists());
let pid = std::fs::read_to_string(rt.path().join("pid")).unwrap();
assert_eq!(pid, std::process::id().to_string());
}
#[tokio::test]
async fn write_status_creates_file() {
let tmp = tempfile::tempdir().unwrap();
let rt = RuntimeDir::create(tmp.path(), "my-worker").await.unwrap();
let state = test_state();
rt.write_status(&state).await.unwrap();
let content = std::fs::read_to_string(rt.path().join("status.json")).unwrap();
let parsed: serde_json::Value = serde_json::from_str(&content).unwrap();
assert_eq!(parsed["state"], "idle");
assert_eq!(parsed["worker_name"], "test-worker");
}
#[tokio::test]
async fn write_status_reflects_changes() {
let tmp = tempfile::tempdir().unwrap();
let rt = RuntimeDir::create(tmp.path(), "my-worker").await.unwrap();
let state = test_state();
state.transition(protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
protocol::WorkerRunState::Running,
)));
rt.write_status(&state).await.unwrap();
let content = std::fs::read_to_string(rt.path().join("status.json")).unwrap();
let parsed: serde_json::Value = serde_json::from_str(&content).unwrap();
assert_eq!(parsed["state"], "running");
}
#[tokio::test]
async fn write_manifest_creates_file() {
let tmp = tempfile::tempdir().unwrap();
let rt = RuntimeDir::create(tmp.path(), "my-worker").await.unwrap();
rt.write_manifest("[engine]\nname = \"test\"")
.await
.unwrap();
let content = std::fs::read_to_string(rt.path().join("manifest.toml")).unwrap();
assert_eq!(content, "[engine]\nname = \"test\"");
}
#[tokio::test]
async fn transient_directory_does_not_write_liveness_snapshots() {
let tmp = tempfile::tempdir().unwrap();
let rt = RuntimeDir::create_transient(tmp.path(), "runtime-worker")
.await
.unwrap();
rt.write_status(&test_state()).await.unwrap();
rt.write_manifest("[worker]\nname = \"runtime-worker\"")
.await
.unwrap();
assert!(!rt.path().join("pid").exists());
assert!(!rt.path().join("status.json").exists());
assert!(!rt.path().join("manifest.toml").exists());
}
#[tokio::test]
async fn write_spawned_workers_creates_file() {
use manifest::{Permission, ScopeRule};
let tmp = tempfile::tempdir().unwrap();
let rt = RuntimeDir::create(tmp.path(), "my-worker").await.unwrap();
let records = vec![SpawnedWorkerRecord {
worker_name: "child".into(),
socket_path: "/run/yoi/child/sock".into(),
scope_delegated: vec![ScopeRule {
target: "/tmp/work".into(),
permission: Permission::Write,
recursive: true,
symlink_policy: Default::default(),
}],
callback_address: "/run/yoi/my-worker/sock".into(),
}];
rt.write_spawned_workers(&records).await.unwrap();
let content = std::fs::read_to_string(rt.path().join("spawned_workers.json")).unwrap();
let parsed: Vec<SpawnedWorkerRecord> = serde_json::from_str(&content).unwrap();
assert_eq!(parsed.len(), 1);
assert_eq!(parsed[0].worker_name, "child");
}
#[tokio::test]
async fn socket_path() {
let tmp = tempfile::tempdir().unwrap();
let rt = RuntimeDir::create(tmp.path(), "my-worker").await.unwrap();
assert_eq!(rt.socket_path(), rt.path().join("sock"));
}
#[tokio::test]
async fn drop_removes_directory() {
let tmp = tempfile::tempdir().unwrap();
let dir_path;
async fn transient_controller_directory_has_no_liveness_projections_and_is_released() {
let root = tempfile::tempdir().unwrap();
let path;
{
let rt = RuntimeDir::create(tmp.path(), "my-worker").await.unwrap();
dir_path = rt.path().to_owned();
assert!(dir_path.exists());
let runtime = RuntimeDir::create(root.path(), "worker").await.unwrap();
path = runtime.path().to_owned();
assert_eq!(std::fs::read_dir(&path).unwrap().count(), 0);
}
assert!(!dir_path.exists());
assert!(!path.exists());
}
#[tokio::test]
async fn preexisting_data_is_not_overwritten_or_removed_on_drop() {
let root = tempfile::tempdir().unwrap();
let path = root.path().join("worker");
std::fs::create_dir(&path).unwrap();
std::fs::write(path.join("pid"), "retained legacy data").unwrap();
{
let _runtime = RuntimeDir::create(root.path(), "worker").await.unwrap();
}
assert_eq!(
std::fs::read_to_string(path.join("pid")).unwrap(),
"retained legacy data"
);
}
#[tokio::test]
async fn persistent_run_preserves_logs_and_rejects_directory_reuse() {
let root = tempfile::tempdir().unwrap();
let path = root.path().join("runs/1");
{
let _runtime = RuntimeDir::create_worker_run(&path).await.unwrap();
for name in ["artifacts", "spawned", "worker.out.log", "worker.err.log"] {
assert!(path.join(name).exists(), "missing {name}");
}
}
assert!(path.exists());
let error = RuntimeDir::create_worker_run(&path).await.err().unwrap();
assert_eq!(error.kind(), io::ErrorKind::AlreadyExists);
}
#[tokio::test]
async fn legacy_reclamation_projection_retains_its_stored_format() {
let root = tempfile::tempdir().unwrap();
let runtime = RuntimeDir::create(root.path(), "worker").await.unwrap();
let record: SpawnedWorkerRecord = serde_json::from_value(serde_json::json!({
"worker_name": "legacy-child", "socket_path": "/old/child/sock",
"scope_delegated": [], "callback_address": "/old/parent/sock",
}))
.unwrap();
runtime.write_spawned_workers(&[record]).await.unwrap();
let value: serde_json::Value = serde_json::from_slice(
&std::fs::read(runtime.path().join("spawned_workers.json")).unwrap(),
)
.unwrap();
assert_eq!(value[0]["socket_path"], "/old/child/sock");
assert_eq!(value[0]["callback_address"], "/old/parent/sock");
runtime.write_spawned_workers(&[]).await.unwrap();
assert_eq!(
std::fs::read_to_string(runtime.path().join("spawned_workers.json")).unwrap(),
"[]"
);
}
}
@@ -21,8 +21,9 @@ pub use error::ScopeLockError;
#[cfg(test)]
pub(crate) use lifecycle::install_top_level_at_for_test;
pub use lifecycle::{
ScopeAllocationGuard, SegmentActivationCommitError, SegmentActivationGuard, SegmentLockInfo,
adopt_allocation, install_top_level, install_top_level_with_deny, lookup_segment,
ScopeAllocationCleanup, ScopeAllocationGuard, SegmentActivationCommitError,
SegmentActivationGuard, SegmentLockInfo, adopt_allocation, install_top_level,
install_top_level_with_deny, lookup_segment,
};
pub use mutate::{
delegate_scope, reclaim_delegated_scope, reclaim_stale, reclaim_stale_with, register_worker,
@@ -11,6 +11,14 @@ use session_store::SegmentId;
pub enum ScopeLockError {
#[error("I/O error on workers.json: {0}")]
Io(#[from] io::Error),
#[error("worker `{worker_name}` allocation still has a live guard")]
AllocationGuardAttached { worker_name: String },
#[error("allocation acquisition was published but its save failed: {source}")]
AcquisitionSaveFailed {
#[source]
source: io::Error,
cleanup: super::lifecycle::ScopeAllocationCleanup,
},
#[error("worker `{0}` is already allocated")]
DuplicateWorkerName(String),
#[error("requested scope `{}` conflicts with worker allocation `{competitor}` rule `{}`", .rule.target.display(), .competitor_rule.target.display())]
@@ -46,3 +54,15 @@ pub enum ScopeLockError {
actual: Option<SegmentId>,
},
}
impl ScopeLockError {
/// Retain the release authority for an owned acquisition whose table rename
/// succeeded before save returned an error. Refusals and unpublished saves
/// do not confer ownership of any preexisting allocation.
pub fn allocation_cleanup_handle(&self) -> Option<super::lifecycle::ScopeAllocationCleanup> {
match self {
Self::AcquisitionSaveFailed { cleanup, .. } => Some(cleanup.clone()),
_ => None,
}
}
}
@@ -3,6 +3,7 @@
//! up on drop.
use std::path::{Path, PathBuf};
use std::sync::{Arc, Mutex};
use manifest::ScopeRule;
use session_store::SegmentId;
@@ -11,13 +12,82 @@ use super::error::ScopeLockError;
use super::mutate::release_worker;
use super::table::{LockFileGuard, default_allocation_path};
/// Owned allocation: on drop, opens the lock file and releases this
/// Worker's entry. The guard keeps only the name + lock-file path; it
/// does not hold the `flock` for the Worker's lifetime.
/// Owned allocation shared with any retained cleanup handles. Only the final
/// owner's drop attempts best-effort release; no `flock` is held for its lifetime.
#[derive(Debug)]
pub struct ScopeAllocationGuard {
owner: Arc<ScopeAllocationOwner>,
}
/// Retains an allocation's release authority even after its Worker is consumed
/// or dropped. A successful controller transfer should drop this handle without
/// explicitly releasing the live Worker's allocation.
#[derive(Debug, Clone)]
pub struct ScopeAllocationCleanup {
owner: Arc<ScopeAllocationOwner>,
}
#[derive(Debug)]
struct ScopeAllocationOwner {
worker_name: String,
lock_path: PathBuf,
state: Mutex<ScopeAllocationOwnerState>,
}
#[derive(Debug)]
struct ScopeAllocationOwnerState {
guard_attached: bool,
released: bool,
}
impl ScopeAllocationOwner {
fn release(&self) -> Result<(), ScopeLockError> {
let mut state = self.state.lock().unwrap_or_else(|error| error.into_inner());
if state.guard_attached {
return Err(ScopeLockError::AllocationGuardAttached {
worker_name: self.worker_name.clone(),
});
}
if state.released {
return Ok(());
}
let mut guard = LockFileGuard::open(&self.lock_path)?;
if guard.data().find(&self.worker_name).is_some() {
release_worker(&mut guard, &self.worker_name)?;
} else {
// A previous save may have removed the entry but failed after rename.
// Complete the durable save before reporting release as confirmed.
guard.save()?;
}
state.released = true;
Ok(())
}
}
impl Drop for ScopeAllocationOwner {
fn drop(&mut self) {
let _ = self.release();
}
}
impl Drop for ScopeAllocationGuard {
fn drop(&mut self) {
self.owner
.state
.lock()
.unwrap_or_else(|error| error.into_inner())
.guard_attached = false;
}
}
impl ScopeAllocationCleanup {
/// Release only after the factory has returned an error and no live
/// Controller can still use this allocation. An attached guard rejects
/// release. Success is idempotent across all clones; failure retains the
/// authority for an explicit retry.
pub fn release(&self) -> Result<(), ScopeLockError> {
self.owner.release()
}
}
/// Exclusive authority to activate a replacement Segment for one allocated Worker.
@@ -56,12 +126,58 @@ impl SegmentActivationCommitError {
}
impl ScopeAllocationGuard {
fn new(worker_name: String, lock_path: PathBuf) -> Self {
Self {
owner: Arc::new(ScopeAllocationOwner {
worker_name,
lock_path,
state: Mutex::new(ScopeAllocationOwnerState {
guard_attached: true,
released: false,
}),
}),
}
}
fn commit_acquisition(self, table: &mut LockFileGuard) -> Result<Self, ScopeLockError> {
let mut published = false;
match table.save_with_publication(&mut published) {
Ok(()) => Ok(self),
Err(source) => {
let cleanup = if published {
Some(self.cleanup_handle())
} else {
// A staged entry is not ours until rename. Disarm before
// detaching so Drop cannot remove a preexisting allocation
// (or reopen the table lock still held by this caller).
self.owner
.state
.lock()
.unwrap_or_else(|error| error.into_inner())
.released = true;
None
};
drop(self);
match cleanup {
Some(cleanup) => Err(ScopeLockError::AcquisitionSaveFailed { source, cleanup }),
None => Err(ScopeLockError::Io(source)),
}
}
}
}
pub fn worker_name(&self) -> &str {
&self.worker_name
&self.owner.worker_name
}
pub fn lock_path(&self) -> &Path {
&self.lock_path
&self.owner.lock_path
}
pub fn cleanup_handle(&self) -> ScopeAllocationCleanup {
ScopeAllocationCleanup {
owner: Arc::clone(&self.owner),
}
}
/// Lock the machine-wide table and validate a Segment replacement before
@@ -71,24 +187,25 @@ impl ScopeAllocationGuard {
expected: SegmentId,
replacement: SegmentId,
) -> Result<SegmentActivationGuard, ScopeLockError> {
let guard = LockFileGuard::open(&self.lock_path)?;
let guard = LockFileGuard::open(self.lock_path())?;
let actual = guard
.data()
.allocations
.iter()
.find(|allocation| allocation.worker_name == self.worker_name)
.find(|allocation| allocation.worker_name == self.worker_name())
.and_then(|allocation| allocation.segment_id);
if actual != Some(expected) {
return Err(ScopeLockError::SegmentChanged {
worker_name: self.worker_name.clone(),
worker_name: self.worker_name().to_owned(),
expected,
actual,
});
}
if let Some(existing) = guard.data().allocations.iter().find(|allocation| {
allocation.segment_id == Some(replacement) && allocation.worker_name != self.worker_name
allocation.segment_id == Some(replacement)
&& allocation.worker_name != self.worker_name()
}) {
return Err(ScopeLockError::SegmentConflict {
segment_id: replacement,
@@ -99,7 +216,7 @@ impl ScopeAllocationGuard {
Ok(SegmentActivationGuard {
guard,
worker_name: self.worker_name.clone(),
worker_name: self.worker_name().to_owned(),
expected,
replacement,
})
@@ -145,14 +262,6 @@ impl SegmentActivationGuard {
}
}
impl Drop for ScopeAllocationGuard {
fn drop(&mut self) {
if let Ok(mut guard) = LockFileGuard::open(&self.lock_path) {
let _ = release_worker(&mut guard, &self.worker_name);
}
}
}
/// Open the default lock file, register a top-level Worker, and return a
/// guard that will release the allocation on drop.
pub fn install_top_level(
@@ -185,7 +294,7 @@ pub fn install_top_level_with_deny(
) -> Result<ScopeAllocationGuard, ScopeLockError> {
let lock_path = default_allocation_path()?;
let mut guard = LockFileGuard::open(&lock_path)?;
super::mutate::register_worker_with_deny(
super::mutate::prepare_worker_with_deny(
&mut guard,
worker_name.clone(),
pid,
@@ -194,10 +303,7 @@ pub fn install_top_level_with_deny(
scope_deny,
segment_id,
)?;
Ok(ScopeAllocationGuard {
worker_name,
lock_path,
})
ScopeAllocationGuard::new(worker_name, lock_path).commit_acquisition(&mut guard)
}
#[cfg(test)]
@@ -210,7 +316,7 @@ pub(crate) fn install_top_level_at_for_test(
segment_id: SegmentId,
) -> Result<ScopeAllocationGuard, ScopeLockError> {
let mut guard = LockFileGuard::open(&lock_path)?;
super::mutate::register_worker_with_deny(
super::mutate::prepare_worker_with_deny(
&mut guard,
worker_name.clone(),
pid,
@@ -219,10 +325,7 @@ pub(crate) fn install_top_level_at_for_test(
Vec::new(),
segment_id,
)?;
Ok(ScopeAllocationGuard {
worker_name,
lock_path,
})
ScopeAllocationGuard::new(worker_name, lock_path).commit_acquisition(&mut guard)
}
/// Take ownership of an existing allocation that was pre-registered by
@@ -240,6 +343,15 @@ pub fn adopt_allocation(
segment_id: SegmentId,
) -> Result<ScopeAllocationGuard, ScopeLockError> {
let lock_path = default_allocation_path()?;
adopt_allocation_at(lock_path, worker_name, new_pid, segment_id)
}
fn adopt_allocation_at(
lock_path: PathBuf,
worker_name: String,
new_pid: u32,
segment_id: SegmentId,
) -> Result<ScopeAllocationGuard, ScopeLockError> {
let mut guard = LockFileGuard::open(&lock_path)?;
let alloc = guard
.data_mut()
@@ -247,11 +359,7 @@ pub fn adopt_allocation(
.ok_or_else(|| ScopeLockError::UnknownWorker(worker_name.clone()))?;
alloc.pid = new_pid;
alloc.segment_id = Some(segment_id);
guard.save()?;
Ok(ScopeAllocationGuard {
worker_name,
lock_path,
})
ScopeAllocationGuard::new(worker_name, lock_path).commit_acquisition(&mut guard)
}
/// Information about a Worker that currently holds an allocation for a
@@ -329,6 +437,388 @@ mod tests {
}
}
#[test]
fn retained_cleanup_handle_defers_release_until_final_owner_drop() {
let dir = TempDir::new().unwrap();
let lock_path = dir.path().join("workers.json");
let guard = install_top_level_at_for_test(
lock_path.clone(),
"retained".into(),
std::process::id(),
sock("retained"),
Vec::new(),
sid(),
)
.unwrap();
let cleanup = guard.cleanup_handle();
let retained = cleanup.clone();
drop(guard);
drop(cleanup);
{
let table = LockFileGuard::open(&lock_path).unwrap();
assert!(table.data().find("retained").is_some());
}
drop(retained);
let table = LockFileGuard::open(&lock_path).unwrap();
assert!(table.data().find("retained").is_none());
}
#[test]
fn dropping_cleanup_handle_after_transfer_preserves_live_guard() {
let dir = TempDir::new().unwrap();
let lock_path = dir.path().join("workers.json");
let segment = sid();
let guard = install_top_level_at_for_test(
lock_path.clone(),
"transferred".into(),
std::process::id(),
sock("transferred"),
Vec::new(),
segment,
)
.unwrap();
drop(guard.cleanup_handle());
// A successful controller transfer leaves the guard usable, including
// the existing activation protocol, without retaining a cleanup handle.
let replacement = sid();
guard
.begin_segment_activation(segment, replacement)
.unwrap()
.commit()
.unwrap();
{
let table = LockFileGuard::open(&lock_path).unwrap();
assert_eq!(
table.data().find("transferred").unwrap().segment_id,
Some(replacement)
);
}
drop(guard);
let table = LockFileGuard::open(&lock_path).unwrap();
assert!(table.data().find("transferred").is_none());
}
#[test]
fn cleanup_rejects_live_guard_then_releases_only_detached_allocation() {
let dir = TempDir::new().unwrap();
let lock_path = dir.path().join("workers.json");
let guard = install_top_level_at_for_test(
lock_path.clone(),
"live".into(),
std::process::id(),
sock("live"),
Vec::new(),
sid(),
)
.unwrap();
let _competitor = install_top_level_at_for_test(
lock_path.clone(),
"competitor".into(),
std::process::id(),
sock("competitor"),
Vec::new(),
sid(),
)
.unwrap();
let cleanup = guard.cleanup_handle();
assert!(matches!(
cleanup.release(),
Err(ScopeLockError::AllocationGuardAttached { worker_name }) if worker_name == "live"
));
drop(cleanup.clone());
{
let table = LockFileGuard::open(&lock_path).unwrap();
assert!(table.data().find("live").is_some());
assert!(table.data().find("competitor").is_some());
}
drop(guard);
cleanup.release().unwrap();
cleanup.clone().release().unwrap();
let table = LockFileGuard::open(&lock_path).unwrap();
assert!(table.data().find("live").is_none());
assert!(table.data().find("competitor").is_some());
}
#[test]
fn published_install_failure_retains_cleanup_across_error_drop_and_retry() {
use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test};
let dir = TempDir::new().unwrap();
let lock_path = dir.path().join("workers.json");
let _competitor = install_top_level_at_for_test(
lock_path.clone(),
"competitor".into(),
std::process::id(),
sock("competitor"),
Vec::new(),
sid(),
)
.unwrap();
let segment = sid();
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::AfterRename);
let error = install_top_level_at_for_test(
lock_path.clone(),
"failed".into(),
std::process::id(),
sock("failed"),
Vec::new(),
segment,
)
.unwrap_err();
assert!(matches!(
&error,
ScopeLockError::AcquisitionSaveFailed { .. }
));
let cleanup = error.allocation_cleanup_handle().unwrap();
let retry = error.allocation_cleanup_handle().unwrap();
drop(error);
{
let table = LockFileGuard::open(&lock_path).unwrap();
assert_eq!(
table.data().find("failed").unwrap().segment_id,
Some(segment)
);
assert!(table.data().find("competitor").is_some());
}
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::BeforeRename);
assert!(matches!(cleanup.release(), Err(ScopeLockError::Io(_))));
drop(cleanup);
{
let table = LockFileGuard::open(&lock_path).unwrap();
assert!(table.data().find("failed").is_some());
}
retry.release().unwrap();
let table = LockFileGuard::open(&lock_path).unwrap();
assert!(table.data().find("failed").is_none());
assert!(table.data().find("competitor").is_some());
}
#[test]
fn refused_install_has_no_cleanup_authority_over_existing_allocation() {
let dir = TempDir::new().unwrap();
let lock_path = dir.path().join("workers.json");
let segment = sid();
let _competitor = install_top_level_at_for_test(
lock_path.clone(),
"competitor".into(),
std::process::id(),
sock("competitor"),
vec![write_rule("/shared", true)],
segment,
)
.unwrap();
let cases = [
("competitor", sid(), Vec::new()),
("other", segment, Vec::new()),
("other", sid(), vec![write_rule("/shared", true)]),
];
for (name, requested_segment, scope) in cases {
let error = install_top_level_at_for_test(
lock_path.clone(),
name.into(),
std::process::id(),
sock("refused"),
scope,
requested_segment,
)
.unwrap_err();
assert!(matches!(
&error,
ScopeLockError::DuplicateWorkerName(_)
| ScopeLockError::SegmentConflict { .. }
| ScopeLockError::WriteConflict { .. }
));
assert!(error.allocation_cleanup_handle().is_none());
drop(error);
let table = LockFileGuard::open(&lock_path).unwrap();
assert_eq!(table.data().allocations.len(), 1);
assert_eq!(
table.data().find("competitor").unwrap().socket,
sock("competitor")
);
assert_eq!(
table.data().find("competitor").unwrap().segment_id,
Some(segment)
);
}
}
#[test]
fn unpublished_install_failure_has_no_cleanup_authority() {
use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test};
let dir = TempDir::new().unwrap();
let lock_path = dir.path().join("workers.json");
let _competitor = install_top_level_at_for_test(
lock_path.clone(),
"competitor".into(),
std::process::id(),
sock("competitor"),
Vec::new(),
sid(),
)
.unwrap();
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::BeforeRename);
let error = install_top_level_at_for_test(
lock_path.clone(),
"failed".into(),
std::process::id(),
sock("failed"),
Vec::new(),
sid(),
)
.unwrap_err();
assert!(matches!(&error, ScopeLockError::Io(_)));
assert!(error.allocation_cleanup_handle().is_none());
drop(error);
let table = LockFileGuard::open(&lock_path).unwrap();
assert!(table.data().find("failed").is_none());
assert!(table.data().find("competitor").is_some());
}
#[test]
fn published_adoption_failure_retains_cleanup_for_retry() {
use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test};
let dir = TempDir::new().unwrap();
let lock_path = dir.path().join("workers.json");
{
let mut table = LockFileGuard::open(&lock_path).unwrap();
delegate_placeholder(&mut table, "child", std::process::id());
delegate_placeholder(&mut table, "competitor", std::process::id());
}
let segment = sid();
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::AfterRename);
let error = adopt_allocation_at(
lock_path.clone(),
"child".into(),
std::process::id(),
segment,
)
.unwrap_err();
let cleanup = error.allocation_cleanup_handle().unwrap();
drop(error);
{
let table = LockFileGuard::open(&lock_path).unwrap();
assert_eq!(
table.data().find("child").unwrap().segment_id,
Some(segment)
);
}
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::AfterRename);
assert!(matches!(cleanup.release(), Err(ScopeLockError::Io(_))));
{
let table = LockFileGuard::open(&lock_path).unwrap();
assert!(table.data().find("child").is_none());
assert!(table.data().find("competitor").is_some());
}
cleanup.release().unwrap();
let table = LockFileGuard::open(&lock_path).unwrap();
assert!(table.data().find("child").is_none());
assert!(table.data().find("competitor").is_some());
}
#[test]
fn unpublished_adoption_failure_preserves_placeholder_without_cleanup_authority() {
use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test};
let dir = TempDir::new().unwrap();
let lock_path = dir.path().join("workers.json");
let placeholder_pid = std::process::id();
{
let mut table = LockFileGuard::open(&lock_path).unwrap();
delegate_placeholder(&mut table, "child", placeholder_pid);
}
fail_next_save_at_for_test(&lock_path, SaveFaultPoint::BeforeRename);
let error = adopt_allocation_at(
lock_path.clone(),
"child".into(),
placeholder_pid.wrapping_add(1),
sid(),
)
.unwrap_err();
assert!(matches!(&error, ScopeLockError::Io(_)));
assert!(error.allocation_cleanup_handle().is_none());
drop(error);
let table = LockFileGuard::open(&lock_path).unwrap();
let allocation = table.data().find("child").unwrap();
assert_eq!(allocation.pid, placeholder_pid);
assert_eq!(allocation.segment_id, None);
}
#[test]
fn failed_cleanup_release_retains_authority_for_retry() {
use super::super::table::{SaveFaultPoint, fail_next_save_at_for_test};
for point in [SaveFaultPoint::BeforeRename, SaveFaultPoint::AfterRename] {
let dir = TempDir::new().unwrap();
let lock_path = dir.path().join("workers.json");
let guard = install_top_level_at_for_test(
lock_path.clone(),
"failed".into(),
std::process::id(),
sock("failed"),
vec![read_rule("/shared", true)],
sid(),
)
.unwrap();
let _other = install_top_level_at_for_test(
lock_path.clone(),
"other".into(),
std::process::id(),
sock("other"),
vec![read_rule("/shared", true)],
sid(),
)
.unwrap();
let cleanup = guard.cleanup_handle();
let retry = cleanup.clone();
drop(guard);
fail_next_save_at_for_test(&lock_path, point);
assert!(matches!(cleanup.release(), Err(ScopeLockError::Io(_))));
drop(cleanup);
{
let table = LockFileGuard::open(&lock_path).unwrap();
assert_eq!(
table.data().find("failed").is_some(),
point == SaveFaultPoint::BeforeRename
);
assert!(table.data().find("other").is_some());
}
retry.release().unwrap();
{
let table = LockFileGuard::open(&lock_path).unwrap();
assert!(table.data().find("failed").is_none());
assert!(table.data().find("other").is_some());
}
// Success disarms every clone and final Drop, even if the name is
// subsequently allocated to a different Worker.
let _replacement = install_top_level_at_for_test(
lock_path.clone(),
"failed".into(),
std::process::id(),
sock("replacement"),
vec![read_rule("/shared", true)],
sid(),
)
.unwrap();
retry.clone().release().unwrap();
drop(retry);
let table = LockFileGuard::open(&lock_path).unwrap();
assert_eq!(
table.data().find("failed").unwrap().socket,
sock("replacement")
);
assert!(table.data().find("other").is_some());
}
}
#[test]
fn adopt_allocation_rewrites_pid_and_releases_on_drop() {
let dir = TempDir::new().unwrap();
@@ -358,6 +848,7 @@ mod tests {
let dir = TempDir::new().unwrap();
let _sandbox = RuntimeDirSandbox::new(dir.path());
let err = adopt_allocation("ghost".into(), 42, sid()).unwrap_err();
assert!(err.allocation_cleanup_handle().is_none());
assert!(matches!(err, ScopeLockError::UnknownWorker(ref n) if n == "ghost"));
}
@@ -57,6 +57,30 @@ pub fn register_worker_with_deny(
scope_allow: Vec<ScopeRule>,
scope_deny: Vec<ScopeRule>,
segment_id: SegmentId,
) -> Result<(), ScopeLockError> {
prepare_worker_with_deny(
guard,
worker_name,
pid,
socket,
scope_allow,
scope_deny,
segment_id,
)?;
guard.save()?;
Ok(())
}
/// Admit and stage a registration without publishing it. Owned installation
/// constructs its cleanup authority after admission and before committing.
pub(super) fn prepare_worker_with_deny(
guard: &mut LockFileGuard,
worker_name: String,
pid: u32,
socket: PathBuf,
scope_allow: Vec<ScopeRule>,
scope_deny: Vec<ScopeRule>,
segment_id: SegmentId,
) -> Result<(), ScopeLockError> {
reclaim_stale(guard);
if guard.data().find(&worker_name).is_some() {
@@ -101,7 +125,6 @@ pub fn register_worker_with_deny(
delegated_from: None,
segment_id: Some(segment_id),
});
guard.save()?;
Ok(())
}
@@ -72,7 +72,8 @@ pub struct Allocation {
pub worker_name: String,
/// Owning process. Checked with `kill(pid, 0)` for stale detection.
pub pid: u32,
/// Worker's Unix socket path.
/// Legacy stored endpoint hint. Empty means the Host owns connectivity.
/// Allocation admission and stale reclamation use identity/PID, never this path.
pub socket: PathBuf,
/// Allow rules granted to this Worker (write + read).
pub scope_allow: Vec<ScopeRule>,
@@ -245,6 +246,14 @@ impl LockFileGuard {
/// Persist with atomic replacement while the separate allocation lock stays
/// held. A crash exposes either the old complete table or the new one.
pub fn save(&mut self) -> io::Result<()> {
let mut published = false;
self.save_with_publication(&mut published)
}
/// Report this save's rename while the caller still holds the table lock.
/// In-memory mutation alone is not evidence that an acquisition took place.
pub(super) fn save_with_publication(&mut self, published: &mut bool) -> io::Result<()> {
*published = false;
let json = serde_json::to_vec_pretty(&self.data).map_err(io::Error::other)?;
let mut temp_os = self.data_path.as_os_str().to_owned();
temp_os.push(".tmp");
@@ -264,6 +273,7 @@ impl LockFileGuard {
));
}
fs::rename(&temp_path, &self.data_path)?;
*published = true;
#[cfg(test)]
if take_save_fault(&self.data_path, SaveFaultPoint::AfterRename) {
return Err(io::Error::other(
@@ -338,6 +348,51 @@ mod tests {
assert_eq!(guard.data().allocations[0].worker_name, "a");
}
#[test]
fn empty_legacy_socket_roundtrips_without_path_resolution_or_cleanup() {
let root = TempDir::new().unwrap();
let path = root.path().join("workers.json");
let retained = root.path().join("retained-data");
fs::write(&retained, "do not touch").unwrap();
let segment = sid();
{
let mut guard = open_empty(&path);
register_worker(
&mut guard,
"in-process".into(),
std::process::id(),
PathBuf::new(),
Vec::new(),
segment,
)
.unwrap();
}
let stored: serde_json::Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap();
assert_eq!(stored["allocations"][0]["socket"], "");
let mut guard = LockFileGuard::open(&path).unwrap();
assert!(guard.data().allocations[0].socket.as_os_str().is_empty());
super::super::reclaim_stale_with(&mut guard, |pid| {
assert_eq!(pid, std::process::id());
true
});
assert_eq!(
guard.data().allocations.len(),
1,
"missing endpoint is not stale ownership"
);
assert!(guard.data().find_by_segment(segment).is_some());
super::super::reclaim_stale_with(&mut guard, |_| false);
drop(guard);
assert!(
LockFileGuard::open(&path)
.unwrap()
.data()
.allocations
.is_empty()
);
assert_eq!(fs::read_to_string(retained).unwrap(), "do not touch");
}
#[test]
fn atomic_save_reopens_complete_table_at_both_rename_boundaries() {
let dir = TempDir::new().unwrap();
-165
View File
@@ -1,165 +0,0 @@
use std::ffi::OsString;
use std::fmt;
use std::io;
use std::path::{Path, PathBuf};
const POD_RUNTIME_COMMAND_ENV: &str = "YOI_POD_RUNTIME_COMMAND";
#[derive(Clone, Debug, PartialEq, Eq)]
pub struct WorkerRuntimeCommand {
pub program: PathBuf,
pub prefix_args: Vec<OsString>,
}
impl WorkerRuntimeCommand {
pub fn new(program: impl Into<PathBuf>, prefix_args: Vec<OsString>) -> Self {
Self {
program: program.into(),
prefix_args,
}
}
pub fn for_current_exe() -> io::Result<Self> {
Ok(Self::for_executable(std::env::current_exe()?))
}
pub fn for_executable(program: impl Into<PathBuf>) -> Self {
Self::new(program, vec![OsString::from("worker")])
}
/// Resolve the Worker runtime command used for subprocess launches.
///
/// The default launch path is always the current `yoi` executable plus
/// the unified `worker` prefix argument. During development, a non-empty
/// `YOI_POD_RUNTIME_COMMAND` value replaces only the executable path;
/// the `worker` prefix is still added here and the env value is not parsed as a
/// shell command.
pub fn resolve() -> io::Result<Self> {
Self::resolve_from_env_value(
std::env::var_os(POD_RUNTIME_COMMAND_ENV),
std::env::current_exe,
)
}
fn resolve_from_env_value<F>(
override_program: Option<OsString>,
current_exe: F,
) -> io::Result<Self>
where
F: FnOnce() -> io::Result<PathBuf>,
{
if let Some(program) = override_program.filter(|program| !program.as_os_str().is_empty()) {
return Ok(Self::for_executable(program));
}
Ok(Self::for_executable(current_exe()?))
}
pub fn program(&self) -> &Path {
&self.program
}
pub fn prefix_args(&self) -> &[OsString] {
&self.prefix_args
}
pub fn argv_with<I, S>(&self, args: I) -> Vec<OsString>
where
I: IntoIterator<Item = S>,
S: Into<OsString>,
{
let mut argv = self.prefix_args.clone();
argv.extend(args.into_iter().map(Into::into));
argv
}
}
impl fmt::Display for WorkerRuntimeCommand {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "{}", self.program.display())?;
for arg in &self.prefix_args {
write!(f, " {}", arg.to_string_lossy())?;
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn yoi_binary_defaults_to_worker_prefix() {
let command = WorkerRuntimeCommand::for_executable("/opt/yoi/bin/yoi");
assert_eq!(command.program(), Path::new("/opt/yoi/bin/yoi"));
assert_eq!(command.prefix_args(), [OsString::from("worker")]);
assert_eq!(
command.argv_with(["--worker", "agent"]),
vec!["worker", "--worker", "agent"]
.into_iter()
.map(OsString::from)
.collect::<Vec<_>>()
);
}
#[test]
fn any_runtime_executable_gets_worker_prefix() {
let command = WorkerRuntimeCommand::for_executable("/opt/yoi/bin/custom-runtime");
assert_eq!(command.program(), Path::new("/opt/yoi/bin/custom-runtime"));
assert_eq!(command.prefix_args(), [OsString::from("worker")]);
assert_eq!(
command.argv_with(["--worker", "agent"]),
vec!["worker", "--worker", "agent"]
.into_iter()
.map(OsString::from)
.collect::<Vec<_>>()
);
}
#[test]
fn resolve_uses_current_exe_when_override_is_unset() {
let command = WorkerRuntimeCommand::resolve_from_env_value(None, || {
Ok(PathBuf::from("/opt/yoi/bin/yoi"))
})
.unwrap();
assert_eq!(
command,
WorkerRuntimeCommand::for_executable("/opt/yoi/bin/yoi")
);
}
#[test]
fn resolve_uses_current_exe_when_override_is_empty() {
let command = WorkerRuntimeCommand::resolve_from_env_value(Some(OsString::new()), || {
Ok(PathBuf::from("/opt/yoi/bin/yoi"))
})
.unwrap();
assert_eq!(
command,
WorkerRuntimeCommand::for_executable("/opt/yoi/bin/yoi")
);
}
#[test]
fn resolve_override_replaces_only_program_and_keeps_worker_prefix() {
let command = WorkerRuntimeCommand::resolve_from_env_value(
Some(OsString::from("/tmp/rebuilt yoi")),
|| panic!("override must not inspect current_exe"),
)
.unwrap();
assert_eq!(command.program(), Path::new("/tmp/rebuilt yoi"));
assert_eq!(command.prefix_args(), [OsString::from("worker")]);
assert_eq!(
command.argv_with(["--worker", "agent"]),
vec!["worker", "--worker", "agent"]
.into_iter()
.map(OsString::from)
.collect::<Vec<_>>()
);
}
}
File diff suppressed because it is too large Load Diff
-181
View File
@@ -1,181 +0,0 @@
//! Socket communication retained for the legacy top-level Worker callback protocol.
//!
//! Direct Internal SubWorker lifecycle is exposed through `worker.control` and the
//! canonical Worker tools, not a second SubWorker-specific tool family.
use std::path::Path;
use std::time::Duration;
use protocol::stream::{JsonLineReader, JsonLineWriter};
use protocol::{Event, Method};
use tokio::net::UnixStream;
/// Timeout applied to each socket-level operation — connect, write,
/// read. Kept short so a stuck child doesn't block the spawner's turn.
const SOCKET_OP_TIMEOUT: Duration = Duration::from_secs(5);
/// Connect with a timeout, drain the server's connect-time snapshot,
/// write one `Method` line, flush, and close.
///
/// The Worker socket protocol sends replayed alerts and an initial
/// `Event::Snapshot` before it starts reading client methods. Send-only
/// callers must consume that prefix; otherwise a large snapshot can block
/// the server's writer before it reaches the method-read branch. Any
/// socket error maps to an `io::Error`; the caller decides whether to
/// surface it to the LLM or treat it as "worker stopped".
pub(crate) async fn connect_and_send(socket: &Path, method: &Method) -> std::io::Result<()> {
let stream = tokio::time::timeout(SOCKET_OP_TIMEOUT, UnixStream::connect(socket))
.await
.map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "connect timed out"))??;
let (r, w) = stream.into_split();
let mut reader = JsonLineReader::new(r);
let mut writer = JsonLineWriter::new(w);
drain_initial_snapshot(&mut reader).await?;
tokio::time::timeout(SOCKET_OP_TIMEOUT, writer.write(method))
.await
.map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "write timed out"))??;
Ok(())
}
async fn drain_initial_snapshot<R>(reader: &mut JsonLineReader<R>) -> std::io::Result<()>
where
R: tokio::io::AsyncBufRead + Unpin,
{
loop {
let event = tokio::time::timeout(SOCKET_OP_TIMEOUT, reader.next::<Event>())
.await
.map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "read timed out"))??;
match event {
Some(Event::Snapshot { .. }) => return Ok(()),
Some(_) => continue,
None => {
return Err(std::io::Error::new(
std::io::ErrorKind::UnexpectedEof,
"worker closed connection before Snapshot event",
));
}
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use protocol::{Alert, AlertLevel, AlertSource, Greeting, WorkerEvent, WorkerStatus};
use tempfile::TempDir;
use tokio::net::UnixListener;
use tokio::task::JoinHandle;
fn snapshot(entries: Vec<serde_json::Value>) -> Event {
Event::Snapshot {
session: protocol::SessionSnapshot {
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
entries: entries
.into_iter()
.enumerate()
.map(|(index, value)| protocol::SessionSnapshotEntry {
use_policy: None,
entry_id: format!("test-{index}"),
timestamp: index as u64,
provenance: protocol::SessionEntryProvenance::LegacyUnknown,
derived_from: Vec::new(),
data: protocol::SessionSnapshotEntryData::RunError {
message: value.to_string(),
failure: None,
},
})
.collect(),
},
greeting: Greeting {
worker_name: "server".into(),
cwd: "/tmp".into(),
provider: "test".into(),
model: "test".into(),
scope_summary: String::new(),
tools: Vec::new(),
context_window: 200_000,
context_tokens: 0,
reasoning: None,
context_usage: None,
},
state: WorkerStatus::Idle.into(),
in_flight: Default::default(),
internal_workers: Vec::new(),
}
}
fn serve_initial_events_then_method(
listener: UnixListener,
events: Vec<Event>,
) -> JoinHandle<Option<Method>> {
tokio::spawn(async move {
let (stream, _) = listener.accept().await.ok()?;
let (r, w) = stream.into_split();
let mut reader = JsonLineReader::new(r);
let mut writer = JsonLineWriter::new(w);
for event in events {
writer.write(&event).await.ok()?;
}
reader.next::<Method>().await.ok().flatten()
})
}
#[tokio::test]
async fn connect_and_send_drains_initial_alert_and_snapshot_before_method() {
let tmp = TempDir::new().unwrap();
let socket = tmp.path().join("worker.sock");
let listener = UnixListener::bind(&socket).unwrap();
let received = serve_initial_events_then_method(
listener,
vec![
Event::Alert(Alert {
id: None,
position: None,
level: AlertLevel::Warn,
source: AlertSource::Worker,
message: "replayed alert".into(),
timestamp_ms: 0,
}),
snapshot(Vec::new()),
],
);
let method = Method::Shutdown {
command: protocol::WorkerCommandEnvelope::new(1),
};
connect_and_send(&socket, &method).await.unwrap();
let method = received.await.unwrap().expect("expected method");
assert!(matches!(method, Method::Shutdown { .. }));
}
#[tokio::test]
async fn connect_and_send_delivers_method_after_large_initial_snapshot() {
let tmp = TempDir::new().unwrap();
let socket = tmp.path().join("worker.sock");
let listener = UnixListener::bind(&socket).unwrap();
let large_payload = "x".repeat(2 * 1024 * 1024);
let received = serve_initial_events_then_method(
listener,
vec![snapshot(vec![
serde_json::json!({ "payload": large_payload }),
])],
);
let expected = Method::WorkerEvent(WorkerEvent::TurnEnded {
worker_name: "child".into(),
});
connect_and_send(&socket, &expected).await.unwrap();
let method = received.await.unwrap().expect("expected method");
match method {
Method::WorkerEvent(WorkerEvent::TurnEnded { worker_name }) => {
assert_eq!(worker_name, "child")
}
other => panic!("expected TurnEnded WorkerEvent, got {other:?}"),
}
}
}
-1
View File
@@ -1,4 +1,3 @@
pub mod comm_tools;
mod published;
pub mod registry;
pub mod tool;
+377 -49
View File
@@ -340,6 +340,84 @@ pub struct SpawnedWorkerRegistryLoad {
pub reclaimed_unreachable: bool,
}
/// Read-only restore preparation using the registry that will be published by the Host.
/// The Host must retain its lifecycle exclusion from preparation through application;
/// the metadata reread detects source drift, but is not an atomic store transaction.
/// Dropping an unapplied plan does not reclaim legacy child state.
pub(crate) struct PreparedSpawnedWorkerRegistryLoad<St> {
pub(crate) registry: Arc<SpawnedWorkerRegistry>,
/// Projected outcome, not evidence that reclamation has already happened.
pub(crate) reclaimed_unreachable: bool,
runtime_dir: Arc<RuntimeDir>,
store: St,
worker_name: String,
parent_scope: Option<SharedScope>,
persisted_children: Vec<WorkerSpawnedChild>,
valid_records: Vec<SpawnedWorkerRecord>,
legacy_projection_exists: bool,
}
impl<St> PreparedSpawnedWorkerRegistryLoad<St>
where
St: WorkerMetadataStore + Clone + Send + Sync + 'static,
{
/// Apply only after all other fallible tool initialization, before socket publication.
/// Effects retain their original order: projection, canonical reclaim history, then
/// allocation/scope. An error may follow committed effects; no rollback is attempted.
pub(crate) async fn apply(&mut self) -> io::Result<SpawnedWorkerRegistryLoad> {
let metadata = self
.store
.read_by_name(&self.worker_name)
.map_err(store_error_to_io)?;
let reclaimed: Vec<_> = self
.persisted_children
.iter()
.map(reclaimed_child_from_metadata)
.collect();
// A prior apply may have committed the canonical reclaim and then failed
// in allocation/scope cleanup. Under the host's same lifecycle lock,
// confirm its exact retained facts rather than appending them again.
let already_reclaimed = !reclaimed.is_empty()
&& metadata.as_ref().is_some_and(|metadata| {
metadata.spawned_children.is_empty()
&& metadata.reclaimed_children.ends_with(&reclaimed)
});
if !already_reclaimed
&& metadata
.as_ref()
.map(|metadata| metadata.spawned_children.as_slice())
.unwrap_or_default()
!= self.persisted_children.as_slice()
{
return Err(io::Error::new(
io::ErrorKind::InvalidData,
"spawned-child metadata changed after registry preparation",
));
}
// Runtime projection is migration input only; the normal Internal registry is never
// materialized into spawned_workers.json. Only existence was captured, not its contents.
if !self.persisted_children.is_empty() || self.legacy_projection_exists {
self.runtime_dir.write_spawned_workers(&[]).await?;
}
if !reclaimed.is_empty() && !already_reclaimed {
self.store
.reclaim_spawned_children(&self.worker_name, reclaimed)
.map_err(store_error_to_io)?;
}
for record in &self.valid_records {
reclaim_record(&self.worker_name, self.parent_scope.as_ref(), record)?;
}
let reclaimed_unreachable = self.reclaimed_unreachable;
self.reclaimed_unreachable = false;
Ok(SpawnedWorkerRegistryLoad {
registry: self.registry.clone(),
reclaimed_unreachable,
})
}
}
impl SpawnedWorkerRegistry {
pub(crate) fn bash_output(&self) -> tools::bash_output::BashOutputView {
self.bash_output.lock().unwrap().clone()
@@ -426,6 +504,23 @@ impl SpawnedWorkerRegistry {
worker_name: String,
parent_scope: Option<SharedScope>,
) -> io::Result<SpawnedWorkerRegistryLoad>
where
St: WorkerMetadataStore + Clone + Send + Sync + 'static,
{
Self::prepare_from_worker_state_with_reclaim(runtime_dir, store, worker_name, parent_scope)?
.apply()
.await
}
/// Capture legacy child facts without starting children or changing metadata,
/// projection, allocation, or scope. Tool initialization must clone this registry,
/// not construct a replacement, and defer reclamation to the owned plan's `apply`.
pub(crate) fn prepare_from_worker_state_with_reclaim<St>(
runtime_dir: Arc<RuntimeDir>,
store: St,
worker_name: String,
parent_scope: Option<SharedScope>,
) -> io::Result<PreparedSpawnedWorkerRegistryLoad<St>>
where
St: WorkerMetadataStore + Clone + Send + Sync + 'static,
{
@@ -442,54 +537,33 @@ impl SpawnedWorkerRegistry {
Ok(record) => {
warn!(
worker = %record.worker_name,
"reclaiming legacy persisted process SubWorker during Internal session restore"
"captured legacy persisted process SubWorker for reclamation during Internal session restore"
);
valid_records.push(record);
}
Err(error) => warn!(
error = %error,
worker = %child.worker_name,
"clearing corrupt legacy persisted process SubWorker record"
"captured corrupt legacy persisted process SubWorker record for clearing"
),
}
}
// Runtime projection is migration input only; the normal Internal registry is never
// materialized into spawned_workers.json.
let legacy_projection_exists = runtime_dir.path().join("spawned_workers.json").exists();
if !persisted_children.is_empty() || legacy_projection_exists {
runtime_dir.write_spawned_workers(&[]).await?;
}
if !persisted_children.is_empty() {
let reclaimed = persisted_children
.iter()
.map(reclaimed_child_from_metadata)
.collect();
store
.reclaim_spawned_children(&worker_name, reclaimed)
.map_err(store_error_to_io)?;
}
for record in &valid_records {
reclaim_record(&worker_name, parent_scope.as_ref(), record)?;
}
Ok(SpawnedWorkerRegistryLoad {
registry: Arc::new(Self {
internal_records: std::sync::Mutex::new(Vec::new()),
service_records: std::sync::Mutex::new(Vec::new()),
#[cfg(test)]
fail_service_stops: std::sync::Mutex::new(HashMap::new()),
internal_names: std::sync::Mutex::new(HashSet::new()),
internal_shutting_down: AtomicBool::new(false),
pending_internal_spawns: AtomicUsize::new(0),
pending_internal_notify: Notify::new(),
internal_spawn_cleanup_failed: AtomicBool::new(false),
parent_scope,
bash_output: Mutex::new(tools::bash_output::BashOutputView::new()),
parent_protocol: Mutex::new(None),
protocol_publish_lock: Arc::new(Mutex::new(())),
}),
let registry = match &parent_scope {
Some(scope) => Self::new_internal(worker_name.clone(), scope.clone()),
None => Self::new(runtime_dir.clone()),
};
Ok(PreparedSpawnedWorkerRegistryLoad {
registry,
reclaimed_unreachable: !persisted_children.is_empty(),
runtime_dir,
store,
worker_name,
parent_scope,
persisted_children,
valid_records,
legacy_projection_exists,
})
}
@@ -1140,6 +1214,10 @@ impl Drop for SpawnedWorkerRegistry {
.filter(|record| !record.scope_reclaimed.load(Ordering::Acquire))
.flat_map(delegated_write_rules)
.collect::<Vec<_>>();
// Empty prepared registries own no delegations, including when abandoned or rejected.
if write_rules.is_empty() {
return;
}
let _ = parent_scope.update(|current| current.with_removed_deny_rules(write_rules));
}
}
@@ -1165,19 +1243,18 @@ fn reclaim_record(
parent_scope: Option<&SharedScope>,
record: &SpawnedWorkerRecord,
) -> io::Result<()> {
if let Ok(path) = worker_allocation::default_allocation_path() {
if let Ok(mut guard) = worker_allocation::LockFileGuard::open(&path) {
match worker_allocation::reclaim_delegated_scope(
&mut guard,
parent_name,
&record.worker_name,
&record.scope_delegated,
) {
Ok(()) | Err(worker_allocation::ScopeLockError::UnknownWorker(_)) => {}
Err(error) => return Err(io::Error::other(error)),
}
}
let path = worker_allocation::default_allocation_path().map_err(io::Error::other)?;
let mut guard = worker_allocation::LockFileGuard::open(&path).map_err(io::Error::other)?;
match worker_allocation::reclaim_delegated_scope(
&mut guard,
parent_name,
&record.worker_name,
&record.scope_delegated,
) {
Ok(()) | Err(worker_allocation::ScopeLockError::UnknownWorker(_)) => {}
Err(error) => return Err(io::Error::other(error)),
}
drop(guard);
if let Some(parent_scope) = parent_scope {
let write_rules = record
.scope_delegated
@@ -1240,10 +1317,18 @@ fn store_error_to_io(error: WorkerStoreError) -> io::Error {
#[cfg(test)]
mod tests {
//! Legacy restore cases connect the registry to canonical aggregate storage,
//! runtime projections, and temporary SharedScope rules. Successful apply uses
//! a corrupt legacy record to exercise durable clearing without consulting the
//! process-global legacy allocation file; valid-record scope reclaim is also
//! covered by the existing external restore test.
use std::path::PathBuf;
use std::time::Duration;
use manifest::{Scope, ScopeConfig};
use session_store::LogEntry;
use session_store::{LogEntry, WorkerAggregateStore, WorkerMetadata, WorkerSpawnedScopeRule};
use tempfile::TempDir;
use super::*;
use crate::internal_worker::{
@@ -1251,6 +1336,249 @@ mod tests {
test_internal_worker_session_with_greeting,
};
struct LegacyRestoreFixture {
runtime_dir: Arc<RuntimeDir>,
store: WorkerAggregateStore,
parent_scope: SharedScope,
scope_root: PathBuf,
metadata_path: PathBuf,
// Keep all storage and scope paths alive until the other fixture fields are dropped.
_root: TempDir,
}
impl LegacyRestoreFixture {
async fn new(permission: &str) -> Self {
let root = TempDir::new().unwrap();
let scope_root = root.path().join("scope");
std::fs::create_dir(&scope_root).unwrap();
let aggregate = root.path().join("aggregate");
let store = WorkerAggregateStore::new(&aggregate, "parent").unwrap();
let runtime_dir = Arc::new(
RuntimeDir::create(&root.path().join("runtime"), "parent")
.await
.unwrap(),
);
let mut metadata =
WorkerMetadata::new("parent", None).with_workspace_id("restore-test-workspace");
metadata.spawned_children.push(WorkerSpawnedChild {
worker_name: "legacy-child".into(),
socket_path: root.path().join("legacy.sock"),
callback_address: runtime_dir.path().join("sock"),
scope_delegated: vec![WorkerSpawnedScopeRule {
target: scope_root.clone(),
permission: permission.into(),
recursive: true,
symlink_policy: Default::default(),
}],
});
store.write(&metadata).unwrap();
let write_rule = ScopeRule {
target: scope_root.clone(),
permission: Permission::Write,
recursive: true,
symlink_policy: Default::default(),
};
let parent_scope = SharedScope::new(
Scope::from_config(&ScopeConfig {
allow: vec![write_rule.clone()],
deny: vec![write_rule],
})
.unwrap(),
);
Self {
runtime_dir,
store,
parent_scope,
scope_root,
metadata_path: aggregate.join("metadata.json"),
_root: root,
}
}
fn prepare(&self) -> PreparedSpawnedWorkerRegistryLoad<WorkerAggregateStore> {
SpawnedWorkerRegistry::prepare_from_worker_state_with_reclaim(
self.runtime_dir.clone(),
self.store.clone(),
"parent".into(),
Some(self.parent_scope.clone()),
)
.unwrap()
}
fn projection_path(&self) -> PathBuf {
self.runtime_dir.path().join("spawned_workers.json")
}
}
#[tokio::test]
async fn registry_preparation_and_abandonment_leave_legacy_state_unchanged() {
for projection_exists in [false, true] {
let fixture = LegacyRestoreFixture::new("write").await;
if projection_exists {
// Projection contents are deliberately not decoded during preparation.
std::fs::write(fixture.projection_path(), b"obsolete projection").unwrap();
}
let metadata_before = std::fs::read(&fixture.metadata_path).unwrap();
let projection_before = std::fs::read(fixture.projection_path()).ok();
let scope_before = fixture.parent_scope.snapshot();
assert!(!scope_before.is_writable(&fixture.scope_root));
let prepared = fixture.prepare();
assert!(prepared.reclaimed_unreachable);
assert!(prepared.registry.list_internal().is_empty());
assert!(prepared.registry.internal_worker_snapshots().is_empty());
assert_eq!(
std::fs::read(&fixture.metadata_path).unwrap(),
metadata_before
);
assert_eq!(
std::fs::read(fixture.projection_path()).ok(),
projection_before
);
assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot()));
assert!(!fixture.runtime_dir.path().join("sock").exists());
drop(prepared);
assert_eq!(
std::fs::read(&fixture.metadata_path).unwrap(),
metadata_before
);
assert_eq!(
std::fs::read(fixture.projection_path()).ok(),
projection_before
);
assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot()));
}
}
#[tokio::test]
async fn prepared_apply_clears_canonical_children_and_projection_on_same_registry() {
let fixture = LegacyRestoreFixture::new("unsupported-legacy-permission").await;
std::fs::write(fixture.projection_path(), b"obsolete projection").unwrap();
let scope_before = fixture.parent_scope.snapshot();
let mut prepared = fixture.prepare();
let shared_registry = prepared.registry.clone();
// Unrelated metadata may advance while tools are initialized.
fixture.store.add_peer("parent", "new-peer").unwrap();
let mut expected = fixture.store.read_by_name("parent").unwrap().unwrap();
expected.reclaimed_children.extend(
expected
.spawned_children
.iter()
.map(reclaimed_child_from_metadata),
);
expected.spawned_children.clear();
let loaded = prepared.apply().await.unwrap();
assert!(loaded.reclaimed_unreachable);
assert!(Arc::ptr_eq(&shared_registry, &loaded.registry));
assert!(shared_registry.list_internal().is_empty());
assert_eq!(
fixture.store.read_by_name("parent").unwrap(),
Some(expected)
);
let projection: serde_json::Value =
serde_json::from_slice(&std::fs::read(fixture.projection_path()).unwrap()).unwrap();
assert_eq!(projection, serde_json::json!([]));
// Corrupt records are audited and cleared, not interpreted as scope grants.
assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot()));
assert!(
!fixture
.parent_scope
.snapshot()
.is_writable(&fixture.scope_root)
);
assert!(!fixture.runtime_dir.path().join("sock").exists());
}
#[tokio::test]
async fn prepared_apply_rejects_child_source_drift_before_any_effects() {
for drift in ["scope", "socket", "removal", "metadata-deletion"] {
let fixture = LegacyRestoreFixture::new("write").await;
std::fs::write(fixture.projection_path(), b"obsolete projection").unwrap();
let scope_before = fixture.parent_scope.snapshot();
let mut prepared = fixture.prepare();
if drift == "metadata-deletion" {
fixture.store.delete_by_name("parent").unwrap();
} else {
fixture
.store
.update_by_name("parent", |metadata| match drift {
"scope" => {
metadata.spawned_children[0].scope_delegated[0].recursive = false
}
"socket" => {
metadata.spawned_children[0].socket_path =
fixture.runtime_dir.path().join("replacement.sock");
}
"removal" => metadata.spawned_children.clear(),
_ => unreachable!(),
})
.unwrap();
}
let metadata_before = std::fs::read(&fixture.metadata_path).ok();
let projection_before = std::fs::read(fixture.projection_path()).unwrap();
let error = prepared
.apply()
.await
.err()
.expect("source drift must fail");
assert_eq!(error.kind(), io::ErrorKind::InvalidData);
assert_eq!(std::fs::read(&fixture.metadata_path).ok(), metadata_before);
assert_eq!(
std::fs::read(fixture.projection_path()).unwrap(),
projection_before
);
assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot()));
assert!(!fixture.runtime_dir.path().join("sock").exists());
}
}
#[tokio::test]
async fn prepared_apply_clears_projection_only_without_reporting_child_reclaim() {
let fixture = LegacyRestoreFixture::new("write").await;
fixture
.store
.set_spawned_children("parent", Vec::new())
.unwrap();
std::fs::write(fixture.projection_path(), b"obsolete projection").unwrap();
let metadata_before = std::fs::read(&fixture.metadata_path).unwrap();
let mut prepared = fixture.prepare();
assert!(!prepared.reclaimed_unreachable);
let loaded = prepared.apply().await.unwrap();
assert!(!loaded.reclaimed_unreachable);
assert_eq!(
std::fs::read(&fixture.metadata_path).unwrap(),
metadata_before
);
let projection: serde_json::Value =
serde_json::from_slice(&std::fs::read(fixture.projection_path()).unwrap()).unwrap();
assert_eq!(projection, serde_json::json!([]));
}
#[tokio::test]
async fn prepared_apply_projection_failure_keeps_canonical_children_and_scope() {
let fixture = LegacyRestoreFixture::new("write").await;
std::fs::create_dir(fixture.projection_path()).unwrap();
let metadata_before = std::fs::read(&fixture.metadata_path).unwrap();
let scope_before = fixture.parent_scope.snapshot();
let mut prepared = fixture.prepare();
assert!(prepared.apply().await.is_err());
assert_eq!(
std::fs::read(&fixture.metadata_path).unwrap(),
metadata_before
);
assert!(fixture.projection_path().is_dir());
assert!(Arc::ptr_eq(&scope_before, &fixture.parent_scope.snapshot()));
}
fn registry() -> Arc<SpawnedWorkerRegistry> {
let scope = Scope::from_config(&ScopeConfig {
allow: vec![ScopeRule {
+2
View File
@@ -1108,6 +1108,8 @@ fn sub_worker_spawn_tool_impl(
#[cfg(test)]
mod tests {
mod environment;
use super::*;
use manifest::{DelegationScope, Permission, Scope, SharedScope};
use std::pin::Pin;
@@ -0,0 +1,629 @@
//! Connection tests for model-visible environments and actual routed file operations.
//! Runs the production SubWorker spawn/broker/Worker/prompt/tool path with temporary
//! LocalWorkdirSession providers; only the external LLM is scripted. Selected by
//! the normal worker lib suite, without network credentials or live Workspace data.
use super::*;
use agen::Item;
use serde_json::{Value, json};
use workdir::{WorkdirSessionCapability, WorkdirSessionRouter};
const TASK: &str = "Read marker.txt from each attached Workdir.";
fn prompt_environments(prompt: &str) -> Vec<Value> {
let section = prompt
.split_once("## Workdir execution environment\n")
.expect("Worker must supply the execution environment independently of task text")
.1;
let json = section.split_once("```json\n").unwrap().1;
serde_json::from_str(json.split_once("\n```").unwrap().0).unwrap()
}
#[derive(Clone, Default)]
struct EnvironmentClient {
requests: Arc<Mutex<Vec<Request>>>,
read_issued: Arc<AtomicBool>,
}
#[async_trait]
impl LlmClient for EnvironmentClient {
fn clone_boxed(&self) -> Box<dyn LlmClient> {
Box::new(self.clone())
}
async fn stream(
&self,
request: Request,
) -> Result<Pin<Box<dyn Stream<Item = Result<LlmEvent, ClientError>> + Send>>, ClientError>
{
let environments = prompt_environments(request.system_prompt.as_deref().unwrap());
self.requests.lock().unwrap().push(request);
let mut events = Vec::new();
if !self.read_issued.swap(true, Ordering::SeqCst) {
// Use only the model-visible aliases and the same relative path in
// every session. No test-side cwd prefix is passed to the Read tool.
for (index, environment) in environments.iter().enumerate() {
let alias = environment["alias"].as_str().unwrap();
events.extend([
LlmEvent::tool_use_start(index, format!("read-{alias}"), "Read"),
LlmEvent::tool_input_delta(
index,
json!({"target_workdir": alias, "file_path": "marker.txt"}).to_string(),
),
LlmEvent::tool_use_stop(index),
]);
}
} else {
events.extend([
LlmEvent::text_block_start(0),
LlmEvent::text_delta(0, "read complete"),
LlmEvent::text_block_stop(0, None),
]);
}
events.push(LlmEvent::Status(StatusEvent {
status: ResponseStatus::Completed,
}));
Ok(Box::pin(futures::stream::iter(events.into_iter().map(Ok))))
}
}
struct EnvironmentFixture {
runtime: TempDir,
root: PathBuf,
manifest: WorkerManifest,
profiles: AvailableProfiles,
registry: Arc<SpawnedWorkerRegistry>,
sessions: Arc<WorkdirSessionRouter>,
}
impl EnvironmentFixture {
fn new(aliases: &[(&str, &str)]) -> Self {
let runtime = TempDir::new().unwrap();
let root = runtime.path().join("profile-root");
// Profile discovery stays at this root, deliberately outside every
// attached Workdir. Session cwd must never redirect it into work/nested.
let profiles = write_project_profile_registry(&root, None, &[]);
let mut manifest = parent_manifest(&root, None);
manifest.feature.sub_worker.enabled = true;
let registry = SpawnedWorkerRegistry::new_internal(
"parent".into(),
SharedScope::new(Scope::from_config(&manifest.scope).unwrap()),
);
let sessions = Arc::new(WorkdirSessionRouter::new());
for (alias, cwd) in aliases {
let checkout = runtime.path().join(alias);
for path in ["", "work", "work/nested", "work/nested/deep", "other"] {
std::fs::create_dir_all(checkout.join(path)).unwrap();
std::fs::write(checkout.join(path).join("marker.txt"), marker(alias, path))
.unwrap();
}
let scope = SharedScope::new(Scope::writable(&checkout).unwrap());
let session = Arc::new(workdir::LocalWorkdirSession::materialized_bound(
workdir::Workdir::new(format!("fixture-{alias}")),
checkout.clone(),
checkout.join(cwd),
scope,
workdir::WorkdirSessionCapabilities::ALL,
));
sessions
.attach_at_cwd(
workdir::WorkdirAttachmentAlias::new(*alias).unwrap(),
session,
workdir::WorkdirPath::new(cwd).unwrap(),
)
.unwrap();
}
Self {
runtime,
root,
manifest,
profiles,
registry,
sessions,
}
}
async fn spawn(
&self,
spawner: &str,
registry: Arc<SpawnedWorkerRegistry>,
broker: workdir::WorkdirToolBrokerRouter,
input: Value,
) -> (
crate::spawn::registry::InternalSpawnedWorkerRecord,
EnvironmentClient,
) {
assert_eq!(input["task"], TASK);
let client = EnvironmentClient::default();
let mut manifest = self.manifest.clone();
manifest.worker.name = spawner.into();
let tool = SubWorkerSpawnTool::new(
spawner.into(),
crate::worker::WorkerWorkspaceContext::local_filesystem(None),
ParentNotificationTarget::Durable(Arc::new(|_| {})),
self.runtime.path().to_path_buf(),
self.root.clone(),
Some(broker),
registry.clone(),
manifest,
PromptCatalogSource::builtins_only(),
self.profiles.clone(),
)
.with_internal_client(Box::new(client.clone()));
tool.execute(
&input.to_string(),
agen::tool::ToolExecutionContext::direct(),
)
.await
.expect("spawn through production delegation and Worker construction");
let record = registry
.get_internal(input["name"].as_str().unwrap())
.unwrap();
assert_eq!(
tokio::time::timeout(Duration::from_secs(5), record.session.wait_until_idle())
.await
.expect("scripted read turn must terminate"),
InternalWorkerSessionStatus::Idle,
);
(record, client)
}
}
fn marker(alias: &str, cwd: &str) -> String {
format!(
"{alias}:{}:session-marker",
if cwd.is_empty() { "." } else { cwd }
)
}
fn assert_environment_and_reads(
router: &WorkdirSessionRouter,
client: &EnvironmentClient,
expected: &[(&str, &str, bool)],
) -> String {
let requests = client.requests.lock().unwrap();
assert!(
requests.len() >= 2,
"Read must execute and return to the model"
);
let prompt = requests[0].system_prompt.as_deref().unwrap();
assert!(
!prompt.contains("## Working boundaries"),
"legacy Scope must not override the connected session environment"
);
let environments = prompt_environments(prompt);
assert_eq!(environments.len(), expected.len());
assert_eq!(router.len(), expected.len());
for (alias, cwd, command) in expected {
let environment = router.resolve(Some(alias)).unwrap().session.environment();
assert_eq!(environment.cwd.as_str(), *cwd);
assert_eq!(
environment
.capabilities
.supports(WorkdirSessionCapability::Command),
*command
);
let projected = environments
.iter()
.find(|env| env["alias"] == *alias)
.unwrap();
assert_eq!(projected["cwd"], if cwd.is_empty() { "." } else { *cwd });
assert_eq!(
projected["scope"],
serde_json::to_value(&environment.scope).unwrap()
);
assert_eq!(
projected["restrictions"],
serde_json::to_value(&environment.restrictions).unwrap()
);
assert_eq!(projected["command"], *command);
for (capability, name) in [
(WorkdirSessionCapability::Read, "read"),
(WorkdirSessionCapability::Write, "write"),
(WorkdirSessionCapability::Edit, "edit"),
(WorkdirSessionCapability::Glob, "glob"),
(WorkdirSessionCapability::Grep, "grep"),
(WorkdirSessionCapability::Command, "command"),
] {
assert_eq!(
projected["capabilities"]
.as_array()
.unwrap()
.iter()
.any(|value| value == name),
environment.capabilities.supports(capability),
);
}
let expected_marker = marker(alias, cwd);
let observed_read = requests
.iter()
.flat_map(|request| &request.items)
.any(|item| {
matches!(item, Item::ToolResult { call_id, content: Some(content), .. }
if call_id == &format!("read-{alias}") && content.contains(&expected_marker))
});
assert!(
observed_read,
"{alias}: relative Read did not return {expected_marker}"
);
}
prompt.to_owned()
}
fn assert_retained_environment(
record: &crate::spawn::registry::InternalSpawnedWorkerRecord,
prompt: &str,
) {
let retained = record.session.entries().iter().any(|entry| {
matches!(entry, session_store::LogEntry::AnnotatedSegmentStart { system_prompt: Some(saved), .. }
if saved == prompt)
});
assert!(
retained,
"the model-visible environment must also be retained in session history"
);
}
#[tokio::test]
async fn spawn_root_and_nonroot_cwd_are_projected_without_task_annotations() {
for (name, cwd, command, permission) in [
("root-child", "", false, "read"),
("cwd-child", "work", true, "write"),
] {
let fixture = EnvironmentFixture::new(&[("checkout", "")]);
let (record, client) = fixture
.spawn(
"parent",
fixture.registry.clone(),
workdir::WorkdirToolBrokerRouter::new(fixture.sessions.clone()),
json!({
"name": name, "task": TASK, "cwd": if cwd.is_empty() { "." } else { cwd },
"scope": [{"target": ".", "permission": permission}],
"command": command,
}),
)
.await;
let router = record.workdir_tool_scope.session_router();
let prompt = assert_environment_and_reads(&router, &client, &[("checkout", cwd, command)]);
assert_retained_environment(&record, &prompt);
assert_eq!(
record.installed_tools.iter().any(|tool| tool == "Bash"),
command
);
fixture.registry.remove_internal(name).await.unwrap();
}
}
#[tokio::test]
async fn spawn_multiple_aliases_and_nested_child_keep_real_session_environment() {
let fixture = EnvironmentFixture::new(&[("alpha", ""), ("beta", "")]);
let (child, child_client) = fixture
.spawn(
"parent",
fixture.registry.clone(),
workdir::WorkdirToolBrokerRouter::new(fixture.sessions.clone()),
json!({
"name": "child", "task": TASK, "cwd": "work", "command": true,
"scope": [
{"target_workdir": "alpha", "target": ".", "permission": "write"},
{"target_workdir": "beta", "target": ".", "permission": "write"},
],
}),
)
.await;
let child_router = child.workdir_tool_scope.session_router();
let child_prompt = assert_environment_and_reads(
&child_router,
&child_client,
&[("alpha", "work", true), ("beta", "work", true)],
);
assert_retained_environment(&child, &child_prompt);
assert!(
child
.installed_tools
.iter()
.any(|tool| tool == "SubWorkerSpawn")
);
// Reuse the actual child's registry and attenuated broker. The nested tool
// receives the scripted LLM at the same boundary as the first spawn; session
// construction, routing, delegation and prompt materialization remain real.
let (nested, nested_client) = fixture
.spawn(
"child",
child.child_registry.clone(),
child.workdir_tool_scope.broker_router(),
json!({
"name": "nested", "task": TASK, "cwd": "work/nested",
"scope": [
{"target_workdir": "alpha", "target": "work/nested", "permission": "read"},
{"target_workdir": "beta", "target": "work/nested", "permission": "read"},
],
}),
)
.await;
let nested_router = nested.workdir_tool_scope.session_router();
let nested_prompt = assert_environment_and_reads(
&nested_router,
&nested_client,
&[
("alpha", "work/nested", false),
("beta", "work/nested", false),
],
);
assert_retained_environment(&nested, &nested_prompt);
assert!(!nested.installed_tools.iter().any(|tool| tool == "Bash"));
for alias in ["alpha", "beta"] {
let session = nested_router.resolve(Some(alias)).unwrap().session;
let environment = session.environment();
let scope = environment
.scope
.expect("nested session must remain scoped");
assert_eq!(scope.len(), 1);
assert_eq!(scope[0].target.as_str(), "work/nested");
assert_eq!(
scope[0].permission,
workdir::WorkdirToolScopePermission::Read
);
assert!(
session
.read(workdir::ReadRequest {
path: workdir::WorkdirPath::new_scoped(
fixture
.runtime
.path()
.join(alias)
.join("marker.txt")
.display()
.to_string(),
)
.unwrap(),
offset: 0,
limit: 10,
max_bytes: 1024,
})
.await
.is_err()
);
}
// Grandchild cwd and reduced command authority must not rewrite its parent.
assert_environment_and_reads(
&child_router,
&child_client,
&[("alpha", "work", true), ("beta", "work", true)],
);
child
.child_registry
.remove_internal("nested")
.await
.unwrap();
fixture.registry.remove_internal("child").await.unwrap();
}
#[tokio::test]
async fn spawn_multiple_aliases_keep_root_scope_independent_of_parent_session_cwds() {
let fixture = EnvironmentFixture::new(&[("alpha", "work"), ("beta", "other")]);
let (record, client) = fixture
.spawn(
"parent",
fixture.registry.clone(),
workdir::WorkdirToolBrokerRouter::new(fixture.sessions.clone()),
json!({
"name": "child", "task": TASK, "cwd": "work/nested",
"scope": [
{"target_workdir": "alpha", "target": "work/nested", "permission": "read"},
{"target_workdir": "beta", "target": "work/nested", "permission": "read"},
],
}),
)
.await;
let prompt = assert_environment_and_reads(
&record.workdir_tool_scope.session_router(),
&client,
&[
("alpha", "work/nested", false),
("beta", "work/nested", false),
],
);
assert_retained_environment(&record, &prompt);
for (alias, cwd) in [("alpha", "work"), ("beta", "other")] {
assert_eq!(
fixture
.sessions
.resolve(Some(alias))
.unwrap()
.session
.environment()
.cwd
.as_str(),
cwd
);
}
fixture.registry.remove_internal("child").await.unwrap();
}
#[tokio::test]
async fn nonroot_startup_then_spawn_and_nested_child_keep_root_relative_scope() {
let fixture = EnvironmentFixture::new(&[("workdir", "")]);
let checkout = fixture.runtime.path().join("workdir");
let client = EnvironmentClient::default();
let mut manifest = fixture.manifest.clone();
manifest.scope.allow = vec![abs_rule(&checkout, Permission::Write)];
let mut worker =
Worker::<Box<dyn LlmClient>, EphemeralSessionStore>::from_internal_manifest_with_context(
manifest,
EphemeralSessionStore::default(),
PromptCatalogSource::builtins_only(),
crate::worker::WorkerWorkspaceContext::local_filesystem(None),
WorkerFilesystemAuthority::local(checkout.clone(), checkout.join("work")),
Some(Box::new(client.clone())),
)
.await
.unwrap();
// Do not bind a test router: exercise workdir_sessions_from_authority via
// the real constructor and the normal controller's broker selection.
let startup = worker.workdir_sessions();
register_worker_tools(
&mut worker,
fixture.runtime.path().to_path_buf(),
fixture.registry.clone(),
None,
None,
)
.await
.unwrap();
tokio::time::timeout(Duration::from_secs(5), worker.run_text(TASK))
.await
.expect("startup read turn must terminate")
.unwrap();
assert_environment_and_reads(&startup, &client, &[("workdir", "work", true)]);
let (child, child_client) = fixture
.spawn(
"parent",
fixture.registry.clone(),
workdir::WorkdirToolBrokerRouter::new(startup.clone()),
json!({
"name": "child", "task": TASK, "cwd": "work/nested",
"scope": [{"target": "work/nested", "permission": "read"}],
}),
)
.await;
let child_router = child.workdir_tool_scope.session_router();
let child_prompt = assert_environment_and_reads(
&child_router,
&child_client,
&[("workdir", "work/nested", false)],
);
assert_retained_environment(&child, &child_prompt);
let (nested, nested_client) = fixture
.spawn(
"child",
child.child_registry.clone(),
child.workdir_tool_scope.broker_router(),
json!({
"name": "nested", "task": TASK, "cwd": "work/nested/deep",
"scope": [{"target": "work/nested/deep", "permission": "read"}],
}),
)
.await;
let nested_router = nested.workdir_tool_scope.session_router();
let nested_prompt = assert_environment_and_reads(
&nested_router,
&nested_client,
&[("workdir", "work/nested/deep", false)],
);
assert_retained_environment(&nested, &nested_prompt);
for (router, target) in [
(&child_router, "work/nested"),
(&nested_router, "work/nested/deep"),
] {
let environment = router.resolve(None).unwrap().session.environment();
let rules = environment.scope.unwrap();
assert_eq!(rules.len(), 1);
assert_eq!(rules[0].target.as_str(), target);
}
assert_eq!(
startup
.resolve(None)
.unwrap()
.session
.environment()
.cwd
.as_str(),
"work"
);
let alias = workdir::WorkdirAttachmentAlias::new("workdir").unwrap();
startup.begin_detach(&alias).await.unwrap();
// Both the parent tool view and descendant root-coordinate broker source
// must share this fence, rather than bypassing it through a cached broker.
for router in [&startup, &child_router, &nested_router] {
let session = router.resolve(None).unwrap().session;
assert!(
session
.read(workdir::ReadRequest {
path: workdir::WorkdirPath::new("marker.txt").unwrap(),
offset: 0,
limit: 10,
max_bytes: 1024,
})
.await
.is_err()
);
}
startup.cancel_detach(&alias);
child
.child_registry
.remove_internal("nested")
.await
.unwrap();
fixture.registry.remove_internal("child").await.unwrap();
}
#[tokio::test]
async fn startup_rejects_cwd_outside_workdir_root_without_running_the_model() {
let fixture = EnvironmentFixture::new(&[("workdir", "")]);
let client = EnvironmentClient::default();
let mut manifest = fixture.manifest.clone();
manifest.scope.allow = vec![abs_rule(fixture.runtime.path(), Permission::Write)];
let error =
Worker::<Box<dyn LlmClient>, EphemeralSessionStore>::from_internal_manifest_with_context(
manifest,
EphemeralSessionStore::default(),
PromptCatalogSource::builtins_only(),
crate::worker::WorkerWorkspaceContext::local_filesystem(None),
WorkerFilesystemAuthority::local(
fixture.runtime.path().join("workdir"),
fixture.root.clone(),
),
Some(Box::new(client.clone())),
)
.await
.err()
.expect("a session cwd must be representable within its Workdir root");
assert!(matches!(error, crate::WorkerError::InvalidCwd { .. }));
assert!(client.requests.lock().unwrap().is_empty());
}
#[tokio::test]
async fn worker_multiple_aliases_project_distinct_session_cwds_not_discovery_root() {
for prompt_override in [None, Some("Feature-owned instruction body")] {
let fixture = EnvironmentFixture::new(&[("alpha", "work"), ("beta", "other")]);
let client = EnvironmentClient::default();
let mut worker = Worker::<Box<dyn LlmClient>, EphemeralSessionStore>::from_internal_manifest_with_context(
fixture.manifest.clone(),
EphemeralSessionStore::default(),
PromptCatalogSource::builtins_only(),
crate::worker::WorkerWorkspaceContext::local_filesystem(None),
WorkerFilesystemAuthority::None,
Some(Box::new(client.clone())),
)
.await
.unwrap();
worker.bind_workdir_sessions(fixture.sessions.clone());
register_worker_tools(
&mut worker,
fixture.runtime.path().to_path_buf(),
fixture.registry.clone(),
None,
Some(workdir::WorkdirToolBrokerRouter::new(
fixture.sessions.clone(),
)),
)
.await
.unwrap();
worker.install_system_prompt_contribution(None, prompt_override.map(str::to_owned));
tokio::time::timeout(Duration::from_secs(5), worker.run_text(TASK))
.await
.expect("scripted worker turn must terminate")
.unwrap();
let prompt = assert_environment_and_reads(
&fixture.sessions,
&client,
&[("alpha", "work", true), ("beta", "other", true)],
);
if let Some(body) = prompt_override {
assert!(prompt.starts_with(body));
}
}
}
+2 -6
View File
@@ -678,8 +678,8 @@ pub(crate) fn test_connection(client: Arc<dyn WorkspaceClient>) -> SubjektivHost
mod tests {
use super::*;
use crate::{
Method, WorkerBootstrap, WorkerBootstrapLayout, WorkerControllerTransport,
WorkerFilesystemAuthority, WorkerManifest, WorkerWorkspaceContext,
Method, WorkerBootstrap, WorkerBootstrapLayout, WorkerFilesystemAuthority, WorkerManifest,
WorkerWorkspaceContext,
};
use agen::llm_client::event::{Event as LlmEvent, ResponseStatus, StatusEvent};
use agen::llm_client::{ClientError, LlmClient, Request};
@@ -944,7 +944,6 @@ enabled = true
WorkerBootstrapLayout::Direct {
runtime_base: temp.path().join("runtime"),
},
WorkerControllerTransport::InProcess,
)
.with_model_client(TaskScriptClient::default())
.with_subjektiv_host(host)
@@ -1137,7 +1136,6 @@ enabled = true
WorkerBootstrapLayout::Direct {
runtime_base: temp.path().join("runtime"),
},
WorkerControllerTransport::InProcess,
)
.with_model_client(ScriptedClient(tx))
.with_subjektiv_host(host.clone())
@@ -1318,7 +1316,6 @@ enabled = true
WorkerBootstrapLayout::Direct {
runtime_base: temp.path().join("runtime"),
},
WorkerControllerTransport::InProcess,
)
.with_model_client(ScriptedClient(tx))
.with_subjektiv_host(host.clone())
@@ -1377,7 +1374,6 @@ enabled = true
WorkerBootstrapLayout::Direct {
runtime_base: temp.path().join("runtime"),
},
WorkerControllerTransport::InProcess,
)
.with_model_client(ScriptedClient(tx));
let bootstrap = if attach {
+1918 -240
View File
File diff suppressed because it is too large Load Diff
+33
View File
@@ -0,0 +1,33 @@
use std::io::{self, Write};
use std::sync::{Arc, Mutex};
/// Per-test warning capture; never installs a global subscriber or writes fixture secrets.
#[derive(Clone, Default)]
pub struct CapturedWarnings(Arc<Mutex<Vec<u8>>>);
impl CapturedWarnings {
pub fn subscriber(&self) -> impl tracing::Subscriber + Send + Sync + 'static {
let output = self.clone();
tracing_subscriber::fmt()
.with_writer(move || output.clone())
.without_time()
.with_ansi(false)
.with_max_level(tracing::Level::WARN)
.finish()
}
pub fn text(&self) -> String {
String::from_utf8(self.0.lock().unwrap().clone()).unwrap()
}
}
impl Write for CapturedWarnings {
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
self.0.lock().unwrap().extend_from_slice(bytes);
Ok(bytes.len())
}
fn flush(&mut self) -> io::Result<()> {
Ok(())
}
}
+153 -22
View File
@@ -26,6 +26,9 @@ use tokio::sync::broadcast;
use worker::{Worker, WorkerController, WorkerError};
#[path = "common/diagnostics.rs"]
mod diagnostics;
type TestStore = CombinedStore<FsStore, FsWorkerStore>;
#[derive(Clone)]
@@ -2876,7 +2879,7 @@ const COMPACTION_HOST_WORKSPACE_ID: &str = "journal-transport-host-workspace";
// Host authorization/selection or live-model semantic continuation quality.
struct CompactionJournalHost {
granted: bool,
unavailable: bool,
failure: Option<&'static str>,
snapshot: Mutex<subjektiv::journal::startup::JournalStartupSnapshot>,
reads: Mutex<Vec<worker::subjektiv::SubjektivHostContext>>,
startup_reads: AtomicUsize,
@@ -2886,7 +2889,7 @@ impl CompactionJournalHost {
fn new(snapshot: subjektiv::journal::startup::JournalStartupSnapshot) -> Self {
Self {
granted: true,
unavailable: false,
failure: None,
snapshot: Mutex::new(snapshot),
reads: Mutex::new(vec![]),
startup_reads: AtomicUsize::new(0),
@@ -2917,10 +2920,18 @@ impl worker::subjektiv::SubjektivHost for CompactionJournalHost {
worker::subjektiv::SubjektivHostError,
> {
self.reads.lock().unwrap().push(context.clone());
if self.unavailable {
return Err(worker::subjektiv::SubjektivHostError::Unavailable(
"injected Journal read failure".into(),
));
if let Some(failure) = self.failure {
use worker::subjektiv::SubjektivHostError;
let secret = "PRIVATE_JOURNAL_BODY /private/host/path proof=PRIVATE_PROOF credential=PRIVATE_CREDENTIAL raw_error=PRIVATE_ERROR";
return Err(match failure {
"unavailable" => SubjektivHostError::Unavailable(secret.into()),
"invalid-argument" => SubjektivHostError::InvalidArgument(secret.into()),
"conflict" => SubjektivHostError::Conflict {
code: secret.into(),
message: secret.into(),
},
_ => panic!("unknown failure fixture"),
});
}
Ok(self.snapshot.lock().unwrap().clone())
}
@@ -3466,26 +3477,37 @@ async fn journal_three_compactions_freeze_one_snapshot_preserve_full_summary_tai
}
#[tokio::test]
async fn journal_absent_disabled_unavailable_and_over_byte_budget_keep_conversation_compaction() {
for case in [
"absent",
"disabled",
"unavailable",
"over-byte-budget",
"no-host",
] {
async fn journal_empty_disabled_and_available_compaction_preserve_inputs_without_failure_warnings()
{
assert_journal_compaction_cases(&["absent", "available", "disabled", "no-host"]).await;
}
#[tokio::test]
async fn journal_snapshot_failures_keep_safe_diagnostics_and_continue_compaction() {
assert_journal_compaction_cases(&["unavailable", "invalid-argument", "conflict"]).await;
}
#[tokio::test]
async fn journal_over_byte_budget_keeps_safe_diagnostic_and_continues_compaction() {
assert_journal_compaction_cases(&["over-byte-budget"]).await;
}
async fn assert_journal_compaction_cases(cases: &[&'static str]) {
for &case in cases {
let mut snapshot = compaction_snapshot(0);
if case == "absent" {
snapshot.journals.clear();
}
if case == "over-byte-budget" {
snapshot.journals[0].current_position =
"x".repeat(subjektiv::journal::startup::STARTUP_MAX_BYTES);
"PRIVATE_JOURNAL_BODY".repeat(subjektiv::journal::startup::STARTUP_MAX_BYTES);
}
let expected_json = (case == "absent").then(|| serde_json::to_string(&snapshot).unwrap());
let expected_json = matches!(case, "absent" | "available")
.then(|| serde_json::to_string(&snapshot).unwrap());
let mut host = CompactionJournalHost::new(snapshot);
host.granted = case != "disabled";
host.unavailable = case == "unavailable";
host.failure =
matches!(case, "unavailable" | "invalid-argument" | "conflict").then_some(case);
let host = Arc::new(host);
let client = MockClient::new(vec![
single_text_events("seed"),
@@ -3517,7 +3539,51 @@ async fn journal_absent_disabled_unavailable_and_over_byte_budget_keep_conversat
true,
)
.await;
fixture.worker.compact(0).await.unwrap();
// Scope the subscriber to every poll, not a thread-local guard across await.
// This observes the real Feature warning while compacting into FsStore.
let logs = diagnostics::CapturedWarnings::default();
use tracing::instrument::WithSubscriber;
fixture
.worker
.compact(0)
.with_subscriber(logs.subscriber())
.await
.unwrap();
let log = logs.text();
let failure = match case {
"unavailable" => Some(("snapshot", "host_unavailable")),
"invalid-argument" => Some(("snapshot", "host_invalid_argument")),
"conflict" => Some(("snapshot", "host_conflict")),
"over-byte-budget" => Some(("byte_budget", "startup_max_bytes_exceeded")),
_ => None,
};
if let Some((stage, reason)) = failure {
assert!(log.contains(&format!("stage=\"{stage}\"")), "{case}: {log}");
assert!(
log.contains(&format!("reason=\"{reason}\"")),
"{case}: {log}"
);
assert_eq!(
log.matches("continuing without Journal").count(),
1,
"{case}"
);
} else {
assert!(
!log.contains("Journal compaction snapshot unavailable"),
"{case}: {log}"
);
}
for secret in [
"PRIVATE_",
"/private/host/path",
"JOURNAL_POSITION_0",
"JOURNAL_BODY_0",
"state-token-0",
"fixed-event-reference-0",
] {
assert!(!log.contains(secret), "{case}: diagnostic leaked {secret}");
}
let input = first_compactor_input(&observed.requests()[1]);
let body = if matches!(case, "disabled" | "no-host") {
assert_eq!(
@@ -3532,8 +3598,8 @@ async fn journal_absent_disabled_unavailable_and_over_byte_budget_keep_conversat
String::new()
} else {
let body = journal_body(&fixture.worker);
let expected = if case == "absent" {
"absent"
let expected = if matches!(case, "absent" | "available") {
case
} else {
"unavailable"
};
@@ -3548,6 +3614,71 @@ async fn journal_absent_disabled_unavailable_and_over_byte_budget_keep_conversat
&fixture.worker.prompts().load_full(),
Some(COMPACTION_HOST_WORKSPACE_ID),
);
let seed = replacement_seed(&fixture.worker);
let metadata = seed[1].metadata.input_metadata.as_ref().unwrap();
assert_eq!(
metadata.usage().recording_material,
session_store::InputEvidenceUse::Exclude
);
assert_eq!(
metadata.usage().memory_evidence,
session_store::InputEvidenceUse::Exclude
);
assert_eq!(
metadata.usage().observation,
session_store::InputObservationUse::Hidden
);
let payload: serde_json::Value = metadata
.feature_payload()
.unwrap()
.decode("journal:compaction", 1)
.unwrap();
assert_eq!(payload["status"], expected);
assert_eq!(
payload["prompt_provenance"]["workspace_id"],
COMPACTION_HOST_WORKSPACE_ID
);
let projection = fixture.worker.prompts().load_full().projection().clone();
assert_eq!(
payload["prompt_provenance"]["source_digest"],
projection.source_digest
);
assert_eq!(
payload["prompt_provenance"]["projection_digest"],
projection.catalog_digest
);
if let Some(json) = expected_json.as_deref() {
use sha2::{Digest, Sha256};
assert_eq!(
payload["snapshot"],
serde_json::from_str::<serde_json::Value>(json).unwrap()
);
assert_eq!(
payload["snapshot_sha256"],
Sha256::digest(json.as_bytes())
.iter()
.map(|byte| format!("{byte:02x}"))
.collect::<String>()
);
} else {
assert!(payload["snapshot"].is_null());
assert!(payload["snapshot_sha256"].is_null());
}
// Feature diagnostics do not become model input or a new metadata contract.
let saved = serde_json::to_string(&seed[1]).unwrap();
for forbidden in [
"PRIVATE_",
"/private/host/path",
"host_unavailable",
"host_conflict",
"host_invalid_argument",
"startup_max_bytes_exceeded",
] {
assert!(
!saved.contains(forbidden),
"{case}: saved input leaked {forbidden}"
);
}
body
};
assert_full_initial_summary(&input, &inherited);
@@ -3559,8 +3690,8 @@ async fn journal_absent_disabled_unavailable_and_over_byte_budget_keep_conversat
.is_some_and(|text| text.contains("PREVIOUS_JOURNAL_DELEGATED_CONTEXT"))),
"fallback retains only the newly frozen reference, not an old snapshot as latest"
);
assert!(!body.contains("JOURNAL_POSITION_0"));
assert!(!body.contains("journal-reference-0"));
assert_eq!(body.contains("JOURNAL_POSITION_0"), case == "available");
assert_eq!(body.contains("journal-reference-0"), case == "available");
assert_eq!(
host.reads.lock().unwrap().len(),
usize::from(!matches!(case, "disabled" | "no-host")),
+404 -343
View File
@@ -1,3 +1,4 @@
use std::path::{Path, PathBuf};
use std::pin::Pin;
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
@@ -150,6 +151,19 @@ impl LlmClient for MockClient {
// Helpers
// ---------------------------------------------------------------------------
#[path = "../../../tests/support/restore_environment.rs"]
mod restore_test_environment;
use restore_test_environment::require_restore_test_environment;
fn owned_relative_runtime_base() -> (tempfile::TempDir, PathBuf) {
require_restore_test_environment();
let temp = tempfile::tempdir_in(".").unwrap();
// The relative reference and the TempDir name the same owned tree, without
// putting the checkout prefix into sockaddr_un or changing process cwd.
let runtime_base = Path::new(temp.path().file_name().unwrap()).join("runtime");
(temp, runtime_base)
}
fn simple_text_events() -> Vec<LlmEvent> {
vec![
LlmEvent::text_block_start(0),
@@ -295,6 +309,7 @@ async fn make_worker_with_manifest_and_workspace_context(
manifest: WorkerManifest,
workspace_context: WorkerWorkspaceContext,
) -> (Worker<MockClient, TestStore>, std::path::PathBuf) {
require_restore_test_environment();
let store_tmp = tempfile::tempdir().unwrap();
let store = CombinedStore::new(
FsStore::new(store_tmp.path()).unwrap(),
@@ -320,18 +335,21 @@ async fn make_worker_with_manifest_and_workspace_context(
}
async fn spawn_controller(worker: Worker<MockClient, TestStore>) -> WorkerHandle {
let tmp = tempfile::tempdir().unwrap();
let runtime_base = tmp.path().to_owned();
std::mem::forget(tmp);
let (handle, _shutdown_rx) = WorkerController::spawn(worker, &runtime_base)
let (tmp, runtime_base) = owned_relative_runtime_base();
let (handle, shutdown_rx) = WorkerController::spawn(worker, &runtime_base)
.await
.unwrap();
// Retain ownership until controller shutdown (or test-runtime teardown).
tokio::spawn(async move {
let _ = shutdown_rx.await;
drop(tmp);
});
handle
}
#[tokio::test]
async fn canonical_worker_handle_uploads_and_submits_attachment() {
let root = tempfile::tempdir().unwrap();
let (root, runtime_base) = owned_relative_runtime_base();
let manifest = WorkerManifest::from_toml(MANIFEST_TOML).unwrap();
let session = session_store::WorkerSessionStore::new(root.path().join("session")).unwrap();
let store = CombinedStore::new(
@@ -353,7 +371,7 @@ async fn canonical_worker_handle_uploads_and_submits_attachment() {
)
.await
.unwrap();
let (handle, shutdown_rx) = WorkerController::spawn(worker, &root.path().join("runtime"))
let (handle, shutdown_rx) = WorkerController::spawn(worker, &runtime_base)
.await
.unwrap();
let mut rx = handle.subscribe();
@@ -762,6 +780,227 @@ async fn controller_startup_failure_closes_bound_workdir_session() {
));
}
mod registry_startup_failure {
//! Connect real controller preparation, registry source fences and session cleanup.
//! The Workdir telemetry boundary substitutes a concurrent metadata change at subscription;
//! the real registry must reject drift before starting its prepared observer.
use super::*;
use session_store::WorkerMetadataStore;
use tokio::sync::broadcast;
use workdir::WorkdirSession;
struct ObservedSession {
local: LocalWorkdirSession,
metadata_store: TestStore,
worker_name: String,
events: broadcast::Sender<workdir::CommandEvent>,
subscriptions: AtomicUsize,
receivers_at_close: Mutex<Vec<usize>>,
}
impl std::fmt::Debug for ObservedSession {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("ObservedSession")
.field("worker_name", &self.worker_name)
.finish_non_exhaustive()
}
}
#[async_trait]
impl WorkdirSession for ObservedSession {
fn workdir(&self) -> &Workdir {
self.local.workdir()
}
fn capabilities(&self) -> WorkdirSessionCapabilities {
self.local.capabilities()
}
async fn stat(
&self,
request: workdir::StatRequest,
) -> Result<workdir::StatResult, WorkdirError> {
self.local.stat(request).await
}
async fn read(
&self,
request: workdir::ReadRequest,
) -> Result<workdir::ReadResult, WorkdirError> {
self.local.read(request).await
}
async fn write(
&self,
request: workdir::WriteRequest,
) -> Result<workdir::WriteResult, WorkdirError> {
self.local.write(request).await
}
async fn edit(
&self,
request: workdir::EditRequest,
) -> Result<workdir::EditResult, WorkdirError> {
self.local.edit(request).await
}
async fn list(
&self,
request: workdir::ListRequest,
) -> Result<workdir::ListResult, WorkdirError> {
self.local.list(request).await
}
async fn glob(
&self,
request: workdir::GlobRequest,
) -> Result<workdir::GlobResult, WorkdirError> {
self.local.glob(request).await
}
async fn grep(
&self,
request: workdir::GrepRequest,
) -> Result<workdir::GrepResult, WorkdirError> {
self.local.grep(request).await
}
async fn start_command(
&self,
request: CommandRequest,
) -> Result<workdir::CommandHandle, WorkdirError> {
self.local.start_command(request).await
}
async fn command_status(
&self,
handle: workdir::CommandHandle,
) -> Result<workdir::CommandStatus, WorkdirError> {
self.local.command_status(handle).await
}
async fn command_output(
&self,
request: CommandOutputRequest,
) -> Result<workdir::CommandOutput, WorkdirError> {
self.local.command_output(request).await
}
async fn cancel_command(&self, handle: workdir::CommandHandle) -> Result<(), WorkdirError> {
self.local.cancel_command(handle).await
}
fn subscribe_command_events(&self) -> Option<broadcast::Receiver<workdir::CommandEvent>> {
// Interleave a canonical change after read-only registry preparation.
let mut metadata = self
.metadata_store
.read_by_name(&self.worker_name)
.unwrap()
.unwrap_or_else(|| session_store::WorkerMetadata::new(&self.worker_name, None));
metadata
.spawned_children
.push(session_store::WorkerSpawnedChild {
worker_name: "legacy-drift".into(),
socket_path: PathBuf::new(),
callback_address: PathBuf::new(),
scope_delegated: vec![],
});
self.metadata_store.write(&metadata).unwrap();
self.subscriptions.fetch_add(1, Ordering::SeqCst);
Some(self.events.subscribe())
}
async fn close(&self) -> Result<(), WorkdirError> {
// Record before any await: aborting without joining must not pass by getting a
// scheduler turn during the provider's asynchronous close. The sender stays alive
// even after close, so channel closure cannot accidentally dispose of the observer.
self.receivers_at_close
.lock()
.unwrap()
.push(self.events.receiver_count());
self.local.close().await
}
}
#[tokio::test(flavor = "current_thread")]
async fn registry_drift_drops_prepared_observer_before_closing_session() {
let root = tempfile::tempdir().unwrap();
let run_dir = root.path().join("runs/1");
let client = MockClient::new(simple_text_events());
let manifest = WorkerManifest::from_toml(MANIFEST_TOML).unwrap();
let store = CombinedStore::new(
FsStore::new(root.path().join("sessions")).unwrap(),
FsWorkerStore::new(root.path().join("workers")).unwrap(),
);
let engine =
Engine::<_, agen::state::Mutable, worker::SessionHistoryMetadata>::new_annotated(
client.clone(),
);
let mut worker = Worker::new(
manifest,
engine,
store.clone(),
WorkerWorkspaceContext::local_filesystem(None),
WorkerFilesystemAuthority::local(root.path().into(), root.path().into()),
manifest::Scope::writable(root.path()).unwrap(),
)
.await
.unwrap();
let (events, unused_receiver) = broadcast::channel(8);
drop(unused_receiver);
let session = Arc::new(ObservedSession {
local: LocalWorkdirSession::materialized_bound(
Workdir::new("registry-startup-failure"),
root.path().into(),
root.path().into(),
worker.scope().clone(),
WorkdirSessionCapabilities::ALL,
),
metadata_store: store,
worker_name: worker.manifest().worker.name.clone(),
events,
subscriptions: AtomicUsize::new(0),
receivers_at_close: Mutex::new(Vec::new()),
});
worker.bind_single_workdir_session(Some(session.clone()));
let sessions = worker.workdir_sessions();
let result = tokio::time::timeout(
std::time::Duration::from_secs(5),
WorkerController::spawn_runtime_managed_run(worker, &run_dir),
)
.await
.expect("registry rejection and cleanup must settle");
let error = match result {
Err(error) => error,
Ok(_) => panic!("registry source drift must reject controller startup"),
};
assert_eq!(error.kind(), std::io::ErrorKind::InvalidData);
assert_eq!(session.subscriptions.load(Ordering::SeqCst), 1);
assert_eq!(*session.receivers_at_close.lock().unwrap(), vec![0]);
assert_eq!(session.events.receiver_count(), 0);
assert!(sessions.aliases().is_empty());
// No controller retains the Worker/Engine after the failed startup.
assert_eq!(Arc::strong_count(&client.call_count), 1);
assert_eq!(client.call_count.load(Ordering::SeqCst), 0);
assert!(!run_dir.join("worker.sock").exists());
assert!(!run_dir.join("status.json").exists());
assert!(!run_dir.join("manifest.toml").exists());
assert!(matches!(
session
.start_command(CommandRequest {
command: "printf unreachable".to_owned(),
timeout_secs: 5,
transport_byte_budget: workdir::MAX_COMMAND_TRANSPORT_BYTES,
cwd: workdir::WorkdirPath::root(),
tool_call_id: None,
})
.await,
Err(ref error) if matches!(error.classification_source(), WorkdirError::Unavailable(_))
));
}
}
async fn wait_for_status(handle: &WorkerHandle, status: WorkerStatus) {
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
loop {
@@ -1561,11 +1800,12 @@ permission = "write"
"#;
let client = MockClient::new(simple_text_events());
let worker = make_worker_with_pwd_and_manifest(client, manifest).await.0;
let tmp = tempfile::tempdir().unwrap();
let result = WorkerController::spawn(worker, tmp.path()).await;
let (_tmp, runtime_base) = owned_relative_runtime_base();
let result = WorkerController::spawn(worker, &runtime_base).await;
assert!(
result.is_ok(),
"feature exposure must not imply delegation authority"
"feature exposure must not imply delegation authority: {:?}",
result.as_ref().err()
);
}
@@ -1738,42 +1978,24 @@ async fn snapshot_includes_user_input_for_in_flight_turn() {
// The Running event is the in-flight visibility fence: the committed
// annotated input must already be available to an immediately attaching
// subscriber rather than racing behind this status transition.
let stream = tokio::net::UnixStream::connect(handle.runtime_dir.socket_path())
.await
.unwrap();
let (reader, _writer) = stream.into_split();
let mut reader = protocol::stream::JsonLineReader::new(reader);
loop {
let event = reader.next::<Event>().await.unwrap().unwrap();
match event {
Event::Snapshot { session, .. } => {
let found = session.entries.iter().any(|entry| match &entry.data {
protocol::SessionSnapshotEntryData::UserInput { segments } => {
protocol::Segment::flatten_to_text(segments) == "hello in-flight"
}
protocol::SessionSnapshotEntryData::Message {
role: protocol::SessionMessageRole::User,
content,
} => content.iter().any(|part| {
matches!(
part,
protocol::SessionContentPart::Text { text }
if text == "hello in-flight"
)
}),
_ => false,
});
assert!(
found,
"snapshot must carry the in-flight UserInput entry: {session:?}"
);
return;
let Event::Snapshot { session, .. } = handle.snapshot_event() else {
panic!("expected snapshot");
};
assert!(
session.entries.iter().any(|entry| match &entry.data {
protocol::SessionSnapshotEntryData::UserInput { segments } => {
protocol::Segment::flatten_to_text(segments) == "hello in-flight"
}
Event::Alert(_) => continue,
other => panic!("expected Snapshot first, got {other:?}"),
}
}
protocol::SessionSnapshotEntryData::Message {
role: protocol::SessionMessageRole::User,
content,
} => content.iter().any(|part| matches!(part,
protocol::SessionContentPart::Text { text } if text == "hello in-flight"
)),
_ => false,
}),
"snapshot must carry the in-flight UserInput entry: {session:?}"
);
}
#[tokio::test]
@@ -1855,24 +2077,10 @@ async fn attach_snapshot_includes_current_status() {
.unwrap();
wait_for_status(&handle, WorkerStatus::Running).await;
let stream = tokio::net::UnixStream::connect(handle.runtime_dir.socket_path())
.await
.unwrap();
let (reader, _writer) = stream.into_split();
let mut reader = protocol::stream::JsonLineReader::new(reader);
// First event after connect is the snapshot — it carries the current status.
loop {
let event = reader.next::<Event>().await.unwrap().unwrap();
match event {
Event::Snapshot { state, .. } => {
assert_eq!(state.catalog_status(), WorkerStatus::Running);
return;
}
Event::Alert(_) => continue,
other => panic!("expected Snapshot, got {other:?}"),
}
}
let Event::Snapshot { state, .. } = handle.snapshot_event() else {
panic!("expected snapshot");
};
assert_eq!(state.catalog_status(), WorkerStatus::Running);
}
#[tokio::test]
@@ -2928,7 +3136,7 @@ async fn status_json_reflects_worker_name() {
}
// ---------------------------------------------------------------------------
// Socket transport tests
// Controller lifecycle tests
// ---------------------------------------------------------------------------
#[tokio::test]
@@ -2966,283 +3174,148 @@ async fn shutdown_closes_method_admission_before_terminal_confirmation() {
shutdown_rx.await.unwrap();
}
/// Uses the real Controller and metadata store; only the model client is replaced.
#[tokio::test]
async fn shutdown_joins_socket_server_with_active_connection() {
use tokio::net::UnixStream;
async fn register_peer_returns_existing_response_and_persists_reciprocal_metadata() {
use session_store::{WorkerMetadata, WorkerMetadataStore};
let worker = make_worker(MockClient::new(simple_text_events())).await;
let runtime_base = tempfile::tempdir().unwrap();
let (handle, shutdown_rx) = WorkerController::spawn(worker, runtime_base.path())
.await
let store = worker.store().clone();
store
.write(&WorkerMetadata::new("test-worker", None))
.unwrap();
let socket_path = handle.runtime_dir.socket_path();
let _connection = UnixStream::connect(&socket_path).await.unwrap();
store
.write(&WorkerMetadata::new("peer-worker", None))
.unwrap();
let handle = spawn_controller(worker).await;
let mut events = handle.subscribe();
handle
.send(Method::Shutdown {
command: worker_command(&handle),
.send(Method::RegisterPeer {
name: "peer-worker".into(),
})
.await
.unwrap();
tokio::time::timeout(std::time::Duration::from_secs(5), shutdown_rx)
assert!(
drain_until(
&mut events,
std::time::Duration::from_secs(2),
|event| matches!(
event,
Event::PeerRegistered { result } if result == &serde_json::json!({
"source": "test-worker", "peer": "peer-worker",
})
)
)
.await
.expect("controller should join its socket tasks")
.expect("controller shutdown signal should remain open");
assert!(!socket_path.exists());
);
assert_eq!(
store.read_by_name("test-worker").unwrap().unwrap().peers[0].worker_name,
"peer-worker"
);
assert_eq!(
store.read_by_name("peer-worker").unwrap().unwrap().peers[0].worker_name,
"test-worker"
);
assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle);
}
#[tokio::test]
async fn socket_run_receives_events() {
use protocol::stream::{JsonLineReader, JsonLineWriter};
use tokio::net::UnixStream;
let client = MockClient::new(simple_text_events());
let worker = make_worker(client).await;
async fn register_peer_missing_target_is_rejected_without_starting_a_worker() {
use session_store::{WorkerMetadata, WorkerMetadataStore};
let worker = make_worker(MockClient::new(simple_text_events())).await;
let store = worker.store().clone();
store
.write(&WorkerMetadata::new("test-worker", None))
.unwrap();
let handle = spawn_controller(worker).await;
// Give the socket server a moment to bind
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
let sock_path = handle.runtime_dir.socket_path();
let stream = UnixStream::connect(&sock_path).await.unwrap();
let (reader, writer) = stream.into_split();
let mut reader = JsonLineReader::new(reader);
let mut writer = JsonLineWriter::new(writer);
// Send run method via socket
writer
.write(&Method::submit_text(
protocol::new_submission_request_id(),
"Hello",
))
let mut events = handle.subscribe();
handle
.send(Method::RegisterPeer {
name: "missing-peer".into(),
})
.await
.unwrap();
// Collect events
let mut saw_turn_start = false;
let mut saw_text_delta = false;
let mut saw_turn_end = false;
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
loop {
tokio::select! {
event = reader.next::<Event>() => {
match event {
Ok(Some(Event::TurnStart { .. })) => saw_turn_start = true,
Ok(Some(Event::TextDelta { .. })) => saw_text_delta = true,
Ok(Some(Event::TurnEnd { .. })) => {
saw_turn_end = true;
break;
}
Ok(None) | Err(_) => break,
_ => {}
}
}
_ = tokio::time::sleep_until(deadline) => break,
}
}
assert!(saw_turn_start, "should see turn_start via socket");
assert!(saw_text_delta, "should see text_delta via socket");
assert!(saw_turn_end, "should see turn_end via socket");
}
#[tokio::test]
async fn socket_worker_event_turn_ended_while_idle_auto_starts_turn() {
use protocol::stream::{JsonLineReader, JsonLineWriter};
use tokio::net::UnixStream;
let client = MockClient::new(simple_text_events());
let worker = make_worker(client).await;
let handle = spawn_controller(worker).await;
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
let sock_path = handle.runtime_dir.socket_path();
let stream = UnixStream::connect(&sock_path).await.unwrap();
let (reader, writer) = stream.into_split();
let mut reader = JsonLineReader::new(reader);
let mut writer = JsonLineWriter::new(writer);
writer
.write(&Method::WorkerEvent(protocol::WorkerEvent::TurnEnded {
worker_name: "child".into(),
}))
assert!(
drain_until(
&mut events,
std::time::Duration::from_secs(2),
|event| matches!(
event,
Event::Error { code: worker::ErrorCode::InvalidRequest, message }
if message.contains("missing-peer") && message.contains("does not exist")
)
)
.await
);
assert!(store.read_by_name("missing-peer").unwrap().is_none());
assert!(
store
.read_by_name("test-worker")
.unwrap()
.unwrap()
.peers
.is_empty()
);
}
#[tokio::test]
async fn legacy_process_discovery_methods_require_host_or_runtime_authority() {
let worker = make_worker(MockClient::new(simple_text_events())).await;
let handle = spawn_controller(worker).await;
let mut events = handle.subscribe();
for method in [
Method::ListWorkers,
Method::RestoreWorker {
name: "old-worker".into(),
},
] {
handle.send(method).await.unwrap();
assert!(
drain_until(
&mut events,
std::time::Duration::from_secs(2),
|event| matches!(
event,
Event::Error { code: worker::ErrorCode::InvalidRequest, message }
if message.contains("Host or Runtime")
)
)
.await
);
}
assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle);
}
#[tokio::test]
async fn all_controller_launch_modes_leave_external_transport_to_the_host() {
let root = tempfile::tempdir().unwrap();
for mode in ["direct", "runtime", "run"] {
let worker = make_worker(MockClient::new(simple_text_events())).await;
let base = root.path().join(mode);
let (handle, shutdown) = match mode {
"direct" => WorkerController::spawn(worker, &base).await,
"runtime" => WorkerController::spawn_runtime_managed(worker, &base).await,
"run" => {
WorkerController::spawn_runtime_managed_run(worker, &base.join("runs/1")).await
}
_ => unreachable!(),
}
.unwrap();
let mut saw_worker_event_echo = false;
let mut saw_turn_start = false;
let mut saw_turn_end = false;
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
// The SystemItem and TurnEnd events arrive through independent
// broadcast lanes (sink fan-out vs `working_event_tx`), so their relative
// order on the wire is non-deterministic. Keep reading until both
// are observed (or the deadline trips), rather than breaking on
// the first TurnEnd.
loop {
if saw_worker_event_echo && saw_turn_end {
break;
}
tokio::select! {
event = reader.next::<Event>() => {
match event {
Ok(Some(Event::SystemItem { ref item, .. }))
if item.get("kind").and_then(|k| k.as_str()) == Some("worker_event")
&& item
.pointer("/presentation/event/worker_name")
.and_then(|v| v.as_str()) == Some("child") =>
{
saw_worker_event_echo = true;
}
Ok(Some(Event::TurnStart { .. })) => saw_turn_start = true,
Ok(Some(Event::TurnEnd { .. })) => {
saw_turn_end = true;
}
Ok(None) | Err(_) => break,
_ => {}
}
}
_ = tokio::time::sleep_until(deadline) => break,
}
}
assert!(
saw_worker_event_echo,
"WorkerEvent::TurnEnded via socket should be echoed as Event::SystemItem(WorkerEvent)"
);
assert!(
saw_turn_start,
"WorkerEvent::TurnEnded via socket should auto-start a turn"
);
assert!(
saw_turn_end,
"auto-triggered turn should reach turn_end via socket"
);
}
async fn socket_error_after_method_line(
handle: &WorkerHandle,
line: &[u8],
) -> (worker::ErrorCode, String) {
use protocol::stream::JsonLineReader;
use tokio::io::AsyncWriteExt;
use tokio::net::UnixStream;
let sock_path = handle.runtime_dir.socket_path();
let stream = UnixStream::connect(&sock_path).await.unwrap();
let (reader, mut writer) = stream.into_split();
let mut reader = JsonLineReader::new(reader);
writer.write_all(line).await.unwrap();
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(1);
loop {
tokio::select! {
event = reader.next::<Event>() => {
match event {
Ok(Some(Event::Error { code, message })) => return (code, message),
Ok(Some(_)) => {}
Ok(None) => panic!("socket closed before invalid-method error"),
Err(e) => panic!("socket read failed before invalid-method error: {e}"),
}
}
_ = tokio::time::sleep_until(deadline) => {
panic!("timed out waiting for invalid-method error")
}
}
}
}
#[tokio::test]
async fn socket_schema_invalid_method_returns_error() {
let client = MockClient::new(simple_text_events());
let worker = make_worker(client).await;
let handle = spawn_controller(worker).await;
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
let (code, message) = socket_error_after_method_line(&handle, b"{\"bad\":\"json\"}\n").await;
assert_eq!(code, worker::ErrorCode::InvalidRequest);
assert!(
message.contains("invalid method"),
"expected invalid-method diagnostic, got: {message}"
);
}
#[tokio::test]
async fn socket_malformed_method_returns_error() {
let client = MockClient::new(simple_text_events());
let worker = make_worker(client).await;
let handle = spawn_controller(worker).await;
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
let (code, message) = socket_error_after_method_line(&handle, b"{not-json}\n").await;
assert_eq!(code, worker::ErrorCode::InvalidRequest);
assert!(
message.contains("invalid method"),
"expected invalid-method diagnostic, got: {message}"
);
}
#[tokio::test]
async fn socket_peer_close_without_method_does_not_broadcast_error() {
use protocol::stream::JsonLineReader;
use tokio::net::UnixStream;
let client = MockClient::new(simple_text_events());
let worker = make_worker(client).await;
let handle = spawn_controller(worker).await;
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
let mut broadcast_rx = handle.subscribe();
let sock_path = handle.runtime_dir.socket_path();
let stream = UnixStream::connect(&sock_path).await.unwrap();
let (reader, writer) = stream.into_split();
let mut reader = JsonLineReader::new(reader);
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(1);
loop {
tokio::select! {
event = reader.next::<Event>() => {
match event {
Ok(Some(Event::Snapshot { .. })) => break,
Ok(Some(_)) => {}
Ok(None) => panic!("socket closed before connect-time snapshot"),
Err(e) => panic!("socket read failed before connect-time snapshot: {e}"),
}
}
_ = tokio::time::sleep_until(deadline) => {
panic!("timed out waiting for connect-time snapshot")
}
}
}
drop(writer);
drop(reader);
let deadline = tokio::time::Instant::now() + std::time::Duration::from_millis(200);
loop {
tokio::select! {
event = broadcast_rx.recv() => {
match event {
Ok(Event::Error { code, message }) => {
panic!("peer close without Method broadcast error {code:?}: {message}")
}
Ok(_) => {}
Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
panic!("broadcast receiver lagged while checking peer close: {n}")
}
Err(tokio::sync::broadcast::error::RecvError::Closed) => break,
}
}
_ = tokio::time::sleep_until(deadline) => break,
for legacy in ["pid", "status.json", "manifest.toml", "sock", "worker.sock"] {
assert!(
!handle.runtime_dir.path().join(legacy).exists(),
"{mode} created {legacy}"
);
}
handle
.send(Method::Shutdown {
command: worker_command(&handle),
})
.await
.unwrap();
tokio::time::timeout(std::time::Duration::from_secs(5), shutdown)
.await
.unwrap()
.unwrap();
}
}
@@ -4006,27 +4079,15 @@ fn item_text_contains(item: &Item, needle: &str) -> bool {
}
async fn snapshot_contains_user_input(handle: &WorkerHandle, needle: &str) -> bool {
let stream = tokio::net::UnixStream::connect(handle.runtime_dir.socket_path())
.await
.unwrap();
let (reader, _writer) = stream.into_split();
let mut reader = protocol::stream::JsonLineReader::new(reader);
loop {
let event = reader.next::<Event>().await.unwrap().unwrap();
match event {
Event::Snapshot { session, .. } => {
return session.entries.into_iter().any(|entry| match entry.data {
protocol::SessionSnapshotEntryData::UserInput { segments } => {
protocol::Segment::flatten_to_text(&segments).contains(needle)
}
_ => false,
});
}
Event::Alert(_) => continue,
other => panic!("expected Snapshot first, got {other:?}"),
let Event::Snapshot { session, .. } = handle.snapshot_event() else {
panic!("expected snapshot");
};
session.entries.into_iter().any(|entry| match entry.data {
protocol::SessionSnapshotEntryData::UserInput { segments } => {
protocol::Segment::flatten_to_text(&segments).contains(needle)
}
}
_ => false,
})
}
#[tokio::test]
@@ -187,7 +187,8 @@ async fn materialise_on_first_turn_populates_worker() {
.expect("system prompt materialised")
.to_string();
assert!(rendered.contains("date="));
assert!(rendered.contains(&pwd.display().to_string()));
assert!(rendered.contains("## Workdir execution environment"));
assert!(!rendered.contains(&pwd.display().to_string()));
assert!(rendered.starts_with("date="));
}
@@ -206,7 +207,8 @@ async fn session_start_state_captures_rendered_prompt() {
LogEntry::AnnotatedSegmentStart { system_prompt, .. } => {
let sp = system_prompt.as_deref().expect("system prompt set");
assert!(sp.starts_with("hello"));
assert!(sp.contains(&pwd.display().to_string()));
assert!(sp.contains("## Workdir execution environment"));
assert!(!sp.contains(&pwd.display().to_string()));
}
other => panic!("expected AnnotatedSegmentStart as first entry, got {other:?}"),
}
+4 -36
View File
@@ -1,15 +1,10 @@
//! Legacy process callback events are diagnostics only after Internal SubWorker migration.
//! Agent-visible Internal SubWorker event summaries.
use std::sync::Arc;
use protocol::{Permission, ScopeRule, WorkerEvent};
use tempfile::TempDir;
use worker::ipc::event::{apply_event_side_effects, render_event};
use worker::runtime::dir::RuntimeDir;
use worker::spawn::registry::SpawnedWorkerRegistry;
use protocol::WorkerEvent;
use worker::ipc::event::render_event;
#[test]
fn render_event_keeps_bounded_legacy_diagnostics() {
fn render_event_identifies_child_and_failure() {
let rendered = render_event(&WorkerEvent::Errored {
worker_name: "legacy-child".into(),
message: "boom".into(),
@@ -17,30 +12,3 @@ fn render_event_keeps_bounded_legacy_diagnostics() {
assert!(rendered.contains("legacy-child"));
assert!(rendered.contains("boom"));
}
#[tokio::test]
async fn legacy_callback_cannot_register_process_subworker_authority() {
let runtime_base = TempDir::new().unwrap();
let runtime_dir = Arc::new(
RuntimeDir::create(runtime_base.path(), "parent")
.await
.unwrap(),
);
let registry = SpawnedWorkerRegistry::new(runtime_dir.clone());
let scope_root = TempDir::new().unwrap();
let event = WorkerEvent::ScopeSubDelegated {
parent_worker: "legacy-parent".into(),
sub_worker: "legacy-child".into(),
sub_socket: "/tmp/legacy-child.sock".into(),
scope: vec![ScopeRule {
target: scope_root.path().to_path_buf(),
permission: Permission::Write,
recursive: true,
symlink_policy: Default::default(),
}],
};
apply_event_side_effects(&event, &registry, "parent", &None).await;
assert!(!runtime_dir.path().join("spawned_workers.json").exists());
}