Compare commits
25
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
8b6e3070dd | ||
|
|
b860c67ea2 | ||
|
|
533477c2e7 | ||
|
|
5aa2dc579a | ||
|
|
280b3d2a25 | ||
|
|
ec74f4cdc4 | ||
|
|
5af9dc8e4a | ||
|
|
ca6d4b4b72 | ||
|
|
cccecbe7ca | ||
|
|
7a64c38493 | ||
|
|
b1361d86e9 | ||
|
|
6181426abd | ||
|
|
cd331b93c6 | ||
|
|
60eabea19a | ||
|
|
7394ec6784 | ||
|
|
28b7cc414b | ||
|
|
86d5cf9403 | ||
|
|
6f0a23e5cc | ||
|
|
2f4a46bbdb | ||
|
|
4872cf2aac | ||
|
|
734d4b44ae | ||
|
|
6810f51f46 | ||
|
|
a1748b552c | ||
|
|
62b2d27b4a | ||
|
|
62e2b44305 |
Generated
+4
-8
@@ -6497,6 +6497,7 @@ name = "workdir"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"axum",
|
||||
"fs-operation",
|
||||
"libc",
|
||||
"manifest",
|
||||
@@ -6520,7 +6521,6 @@ dependencies = [
|
||||
"axum",
|
||||
"base64 0.22.1",
|
||||
"chrono",
|
||||
"clap",
|
||||
"client",
|
||||
"config-source",
|
||||
"dotenv",
|
||||
@@ -6558,6 +6558,7 @@ dependencies = [
|
||||
"toml",
|
||||
"tools",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
"tungstenite 0.28.0",
|
||||
"uuid",
|
||||
"wip-client",
|
||||
@@ -6597,6 +6598,7 @@ dependencies = [
|
||||
"tokio",
|
||||
"tokio-tungstenite 0.29.0",
|
||||
"toml",
|
||||
"tools",
|
||||
"tower",
|
||||
"tracing",
|
||||
"tracing-subscriber",
|
||||
@@ -6684,6 +6686,7 @@ dependencies = [
|
||||
"futures",
|
||||
"manifest",
|
||||
"project-record",
|
||||
"protocol",
|
||||
"reqwest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
@@ -6699,7 +6702,6 @@ dependencies = [
|
||||
"tui",
|
||||
"uuid",
|
||||
"workdir",
|
||||
"worker",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -6707,21 +6709,15 @@ name = "yoi-e2e"
|
||||
version = "0.0.0"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"config-source",
|
||||
"libc",
|
||||
"manifest",
|
||||
"protocol",
|
||||
"reqwest",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"server-api",
|
||||
"session-store",
|
||||
"sha2 0.11.0",
|
||||
"tempfile",
|
||||
"tungstenite 0.29.0",
|
||||
"webauthn-authenticator-rs",
|
||||
"webauthn-rs-proto",
|
||||
"worker-runtime",
|
||||
"yoi-workspace-server",
|
||||
]
|
||||
|
||||
|
||||
@@ -100,21 +100,6 @@ pub fn worker_allocation_path() -> Option<PathBuf> {
|
||||
worker_allocation_path_from_runtime_dir(runtime_dir())
|
||||
}
|
||||
|
||||
/// `<runtime_dir>/<worker_name>/` — per-Worker runtime directory.
|
||||
pub fn worker_runtime_dir(worker_name: &str) -> Option<PathBuf> {
|
||||
worker_runtime_dir_from_runtime_dir(runtime_dir(), worker_name)
|
||||
}
|
||||
|
||||
/// `<runtime_dir>/<worker_name>/sock` — Worker's Unix socket path.
|
||||
///
|
||||
/// The `worker` crate's `RuntimeDir::socket_path()` creates the socket inside the
|
||||
/// Worker process. This function **predicts** the path for external callers that
|
||||
/// know the Worker name, such as the TUI attach flow. Both should return the same
|
||||
/// path.
|
||||
pub fn worker_socket_path(worker_name: &str) -> Option<PathBuf> {
|
||||
worker_socket_path_from_runtime_dir(runtime_dir(), worker_name)
|
||||
}
|
||||
|
||||
// ---- internals --------------------------------------------------------------
|
||||
|
||||
fn resolve_config_dir_from_parts(
|
||||
@@ -204,20 +189,6 @@ fn worker_allocation_path_from_runtime_dir(runtime_dir: Option<PathBuf>) -> Opti
|
||||
Some(runtime_dir?.join("workers.json"))
|
||||
}
|
||||
|
||||
fn worker_runtime_dir_from_runtime_dir(
|
||||
runtime_dir: Option<PathBuf>,
|
||||
worker_name: &str,
|
||||
) -> Option<PathBuf> {
|
||||
Some(runtime_dir?.join(worker_name))
|
||||
}
|
||||
|
||||
fn worker_socket_path_from_runtime_dir(
|
||||
runtime_dir: Option<PathBuf>,
|
||||
worker_name: &str,
|
||||
) -> Option<PathBuf> {
|
||||
Some(worker_runtime_dir_from_runtime_dir(runtime_dir, worker_name)?.join("sock"))
|
||||
}
|
||||
|
||||
/// Treats empty environment values as unset. `std::env::var` distinguishes
|
||||
/// `Ok("")` from `Err(NotPresent)`, but path resolution treats both as unset
|
||||
/// for intuitive behavior.
|
||||
@@ -455,16 +426,8 @@ mod tests {
|
||||
PathBuf::from("/sand/sessions")
|
||||
);
|
||||
assert_eq!(
|
||||
worker_allocation_path_from_runtime_dir(runtime_dir.clone()).unwrap(),
|
||||
worker_allocation_path_from_runtime_dir(runtime_dir).unwrap(),
|
||||
PathBuf::from("/sand/run/workers.json")
|
||||
);
|
||||
assert_eq!(
|
||||
worker_runtime_dir_from_runtime_dir(runtime_dir.clone(), "foo").unwrap(),
|
||||
PathBuf::from("/sand/run/foo")
|
||||
);
|
||||
assert_eq!(
|
||||
worker_socket_path_from_runtime_dir(runtime_dir, "foo").unwrap(),
|
||||
PathBuf::from("/sand/run/foo/sock")
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1747,23 +1747,7 @@ fn ticket_event(
|
||||
) -> Result<(), MergeRequestError> {
|
||||
let ticket = &mr.ticket_ids[0];
|
||||
let n:i64=t.query_row("SELECT COALESCE(MAX(event_index),-1)+1 FROM typed_ticket_events WHERE workspace_id=?1 AND ticket_id=?2",params![mr.workspace_id,ticket],|r|r.get(0))?;
|
||||
t.execute(
|
||||
"INSERT INTO typed_ticket_events(
|
||||
workspace_id,ticket_id,event_index,kind,author,at,from_state,to_state,heading,body,event_ref
|
||||
) VALUES(?1,?2,?3,'comment',?4,?5,NULL,NULL,'Merge Request integrated',?6,?7)",
|
||||
params![
|
||||
mr.workspace_id,
|
||||
ticket,
|
||||
n,
|
||||
format!("worker:{}:{}", e.merged_by.runtime_id, e.merged_by.worker_id),
|
||||
e.created_at.to_rfc3339(),
|
||||
format!(
|
||||
"Merge Request `{}` integrated approved source ref `{}`; Ticket remains in progress until guarded Ticket completion.",
|
||||
mr.merge_request_id, e.approved_source_ref
|
||||
),
|
||||
format!("{ticket}:{n}"),
|
||||
],
|
||||
)?;
|
||||
t.execute("INSERT INTO typed_ticket_events(workspace_id,ticket_id,event_index,kind,author,at,from_state,to_state,heading,body)VALUES(?1,?2,?3,'comment',?4,?5,NULL,NULL,'Merge Request integrated',?6)",params![mr.workspace_id,ticket,n,format!("worker:{}:{}",e.merged_by.runtime_id,e.merged_by.worker_id),e.created_at.to_rfc3339(),format!("Merge Request `{}` integrated approved source ref `{}`; Ticket remains in progress until guarded Ticket completion.",mr.merge_request_id,e.approved_source_ref)])?;
|
||||
for (k, v) in [
|
||||
("implementation_assignment_id", a),
|
||||
("merge_request_id", &mr.merge_request_id),
|
||||
|
||||
@@ -53,8 +53,8 @@ fn fixture() -> (tempfile::TempDir, MergeRequestStore) {
|
||||
let d = tempfile::tempdir().unwrap();
|
||||
let p = d.path().join("db");
|
||||
let c = Connection::open(&p).unwrap();
|
||||
c.execute_batch("CREATE TABLE workspaces(workspace_id TEXT PRIMARY KEY);CREATE TABLE repositories(workspace_id TEXT,repository_id TEXT,PRIMARY KEY(workspace_id,repository_id));CREATE TABLE ticket_current_worker_assignments(workspace_id TEXT,ticket_id TEXT,assignment_id TEXT,runtime_id TEXT,worker_id TEXT,updated_at TEXT,PRIMARY KEY(workspace_id,ticket_id));CREATE TABLE typed_tickets(workspace_id TEXT,ticket_id TEXT,workflow_state TEXT,workflow_state_explicit INTEGER,updated_at TEXT,title TEXT NOT NULL,body TEXT NOT NULL,PRIMARY KEY(workspace_id,ticket_id));CREATE TABLE typed_ticket_targets(workspace_id TEXT,ticket_id TEXT,ordinal INTEGER,repository_key TEXT,ref_selector TEXT,access TEXT);CREATE TABLE typed_ticket_events(workspace_id TEXT,ticket_id TEXT,event_index INTEGER,kind TEXT,author TEXT,at TEXT,from_state TEXT,to_state TEXT,heading TEXT,body TEXT,event_ref TEXT NOT NULL DEFAULT '',PRIMARY KEY(workspace_id,ticket_id,event_index),UNIQUE(workspace_id,ticket_id,event_ref));CREATE TABLE typed_ticket_event_attributes(workspace_id TEXT,ticket_id TEXT,event_index INTEGER,key TEXT,value TEXT,PRIMARY KEY(workspace_id,ticket_id,event_index,key));INSERT INTO workspaces VALUES('W');INSERT INTO repositories VALUES('W','R');INSERT INTO repositories VALUES('W','R2');INSERT INTO ticket_current_worker_assignments VALUES('W','T','A','runtime','coder','t');INSERT INTO typed_tickets VALUES('W','T','inprogress',1,'t','Title','Requirements');CREATE VIEW ticket_active_worker_assignments AS SELECT current.* FROM ticket_current_worker_assignments current JOIN typed_tickets ticket ON ticket.workspace_id=current.workspace_id AND ticket.ticket_id=current.ticket_id WHERE ticket.workflow_state NOT IN ('done','closed');").unwrap();
|
||||
c.execute("INSERT INTO typed_ticket_events(workspace_id,ticket_id,event_index,kind,body,event_ref) VALUES('W','T',-1,'comment','Requirements','T:-1')",[]).unwrap();
|
||||
c.execute_batch("CREATE TABLE workspaces(workspace_id TEXT PRIMARY KEY);CREATE TABLE repositories(workspace_id TEXT,repository_id TEXT,PRIMARY KEY(workspace_id,repository_id));CREATE TABLE ticket_current_worker_assignments(workspace_id TEXT,ticket_id TEXT,assignment_id TEXT,runtime_id TEXT,worker_id TEXT,updated_at TEXT,PRIMARY KEY(workspace_id,ticket_id));CREATE TABLE typed_tickets(workspace_id TEXT,ticket_id TEXT,workflow_state TEXT,workflow_state_explicit INTEGER,updated_at TEXT,title TEXT NOT NULL,body TEXT NOT NULL,PRIMARY KEY(workspace_id,ticket_id));CREATE TABLE typed_ticket_targets(workspace_id TEXT,ticket_id TEXT,ordinal INTEGER,repository_key TEXT,ref_selector TEXT,access TEXT);CREATE TABLE typed_ticket_events(workspace_id TEXT,ticket_id TEXT,event_index INTEGER,kind TEXT,author TEXT,at TEXT,from_state TEXT,to_state TEXT,heading TEXT,body TEXT,PRIMARY KEY(workspace_id,ticket_id,event_index));CREATE TABLE typed_ticket_event_attributes(workspace_id TEXT,ticket_id TEXT,event_index INTEGER,key TEXT,value TEXT,PRIMARY KEY(workspace_id,ticket_id,event_index,key));INSERT INTO workspaces VALUES('W');INSERT INTO repositories VALUES('W','R');INSERT INTO repositories VALUES('W','R2');INSERT INTO ticket_current_worker_assignments VALUES('W','T','A','runtime','coder','t');INSERT INTO typed_tickets VALUES('W','T','inprogress',1,'t','Title','Requirements');CREATE VIEW ticket_active_worker_assignments AS SELECT current.* FROM ticket_current_worker_assignments current JOIN typed_tickets ticket ON ticket.workspace_id=current.workspace_id AND ticket.ticket_id=current.ticket_id WHERE ticket.workflow_state NOT IN ('done','closed');").unwrap();
|
||||
c.execute("INSERT INTO typed_ticket_events(workspace_id,ticket_id,event_index,kind,body) VALUES('W','T',-1,'comment','Requirements')",[]).unwrap();
|
||||
drop(c);
|
||||
let a = Assignments(Arc::new(Mutex::new(CurrentAssignment {
|
||||
assignment_id: "A".into(),
|
||||
@@ -457,18 +457,6 @@ fn integration_preserves_selectors_ticket_state_and_assignment_and_replays_its_m
|
||||
.unwrap();
|
||||
assert_eq!(replayed, merged);
|
||||
let connection = Connection::open(d.path().join("db")).unwrap();
|
||||
let ticket_refs = connection
|
||||
.prepare("SELECT event_ref FROM typed_ticket_events WHERE event_index >= 0")
|
||||
.unwrap()
|
||||
.query_map([], |row| row.get::<_, String>(0))
|
||||
.unwrap()
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
ticket_refs,
|
||||
vec!["T:0"],
|
||||
"replay must not append another post"
|
||||
);
|
||||
let (state, active): (String, bool) = connection.query_row(
|
||||
"SELECT workflow_state, EXISTS(SELECT 1 FROM ticket_active_worker_assignments WHERE workspace_id='W' AND ticket_id='T') FROM typed_tickets WHERE workspace_id='W' AND ticket_id='T'",
|
||||
[],
|
||||
@@ -489,87 +477,6 @@ fn integration_preserves_selectors_ticket_state_and_assignment_and_replays_its_m
|
||||
assert!(!json.contains(banned), "{banned} in {json}")
|
||||
}
|
||||
}
|
||||
#[test]
|
||||
fn integration_post_failure_rolls_back_and_same_operation_recovers_with_legacy_empty_ref() {
|
||||
let (dir, store) = fixture();
|
||||
open(&store);
|
||||
let review = approve(&store, "source", "approved");
|
||||
request(&store, "source", "pending");
|
||||
let completion = CompleteMergeRequest {
|
||||
merge_request_id: "MR".into(),
|
||||
ticket_id: "T".into(),
|
||||
operation_id: "recover-integration".into(),
|
||||
approval_event_id: review.event_id,
|
||||
current_subject_ref: "source".into(),
|
||||
target_ref_before: "before".into(),
|
||||
target_ref_after: "after".into(),
|
||||
strategy: MergeStrategy::FastForward,
|
||||
resolution: ConflictResolution::None,
|
||||
auth: auth(),
|
||||
now: at(5),
|
||||
};
|
||||
let connection = Connection::open(dir.path().join("db")).unwrap();
|
||||
// Retain a comment written before integration populated event_ref.
|
||||
connection.execute_batch(
|
||||
"INSERT INTO typed_ticket_events(workspace_id,ticket_id,event_index,kind,body,event_ref)
|
||||
VALUES('W','T',0,'comment','Legacy integration','');
|
||||
CREATE TRIGGER reject_integration_post BEFORE INSERT ON typed_ticket_events
|
||||
WHEN NEW.heading='Merge Request integrated'
|
||||
BEGIN SELECT RAISE(ABORT,'injected integration post failure'); END;",
|
||||
).unwrap();
|
||||
let before = store.get_by_id("W", "MR").unwrap();
|
||||
let error = store.complete(completion.clone()).unwrap_err();
|
||||
assert!(
|
||||
error
|
||||
.to_string()
|
||||
.contains("injected integration post failure")
|
||||
);
|
||||
let failed = store.get_by_id("W", "MR").unwrap();
|
||||
assert_eq!(failed.state, MergeRequestState::Open);
|
||||
assert_eq!(failed.thread, before.thread);
|
||||
assert!(store.authorize_review_submission("MR", "pending").is_ok());
|
||||
assert_eq!(
|
||||
connection
|
||||
.query_row(
|
||||
"SELECT COUNT(*) FROM typed_ticket_event_attributes",
|
||||
[],
|
||||
|row| row.get::<_, i64>(0),
|
||||
)
|
||||
.unwrap(),
|
||||
0
|
||||
);
|
||||
|
||||
connection
|
||||
.execute_batch("DROP TRIGGER reject_integration_post")
|
||||
.unwrap();
|
||||
let merged = store.complete(completion.clone()).unwrap();
|
||||
assert_eq!(store.complete(completion).unwrap(), merged);
|
||||
assert_eq!(
|
||||
store.get_by_id("W", "MR").unwrap().state,
|
||||
MergeRequestState::Merged
|
||||
);
|
||||
assert!(store.authorize_review_submission("MR", "pending").is_err());
|
||||
let posts = connection
|
||||
.prepare("SELECT event_ref,body FROM typed_ticket_events WHERE event_index >= 0 ORDER BY event_index")
|
||||
.unwrap()
|
||||
.query_map([], |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)))
|
||||
.unwrap()
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
posts.len(),
|
||||
2,
|
||||
"retry and replay must append exactly one post"
|
||||
);
|
||||
assert_eq!(posts[0], (String::new(), "Legacy integration".into()));
|
||||
assert_eq!(posts[1].0, "T:1");
|
||||
assert!(posts[1].1.contains("`MR`"));
|
||||
assert_eq!(connection.query_row(
|
||||
"SELECT value FROM typed_ticket_event_attributes WHERE event_index=1 AND key='operation_id'",
|
||||
[], |row| row.get::<_, String>(0),
|
||||
).unwrap(), "recover-integration");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn source_move_cancels_submission_and_old_approval_is_reusable_when_source_returns() {
|
||||
let (_d, s) = fixture();
|
||||
@@ -1048,7 +955,7 @@ fn ticket_rescope_requires_fresh_review_before_merge_and_remains_recoverable() {
|
||||
let connection = Connection::open(dir.path().join("db")).unwrap();
|
||||
connection
|
||||
.execute_batch(
|
||||
"UPDATE typed_tickets SET title='Rescoped' WHERE workspace_id='W' AND ticket_id='T'; INSERT INTO typed_ticket_events VALUES('W','T',1,'item_edit','user','t2',NULL,NULL,NULL,NULL,'T:1')",
|
||||
"UPDATE typed_tickets SET title='Rescoped' WHERE workspace_id='W' AND ticket_id='T'; INSERT INTO typed_ticket_events VALUES('W','T',1,'item_edit','user','t2',NULL,NULL,NULL,NULL)",
|
||||
)
|
||||
.unwrap();
|
||||
connection
|
||||
@@ -1175,7 +1082,7 @@ fn ticket_rescope_after_integration_accepts_fresh_requirement_attestation() {
|
||||
let connection = Connection::open(dir.path().join("db")).unwrap();
|
||||
connection
|
||||
.execute_batch(
|
||||
"UPDATE typed_tickets SET title='Rescoped' WHERE workspace_id='W' AND ticket_id='T'; INSERT INTO typed_ticket_events VALUES('W','T',1,'item_edit','user','t2',NULL,NULL,NULL,NULL,'T:1')",
|
||||
"UPDATE typed_tickets SET title='Rescoped' WHERE workspace_id='W' AND ticket_id='T'; INSERT INTO typed_ticket_events VALUES('W','T',1,'item_edit','user','t2',NULL,NULL,NULL,NULL)",
|
||||
)
|
||||
.unwrap();
|
||||
connection
|
||||
@@ -1400,24 +1307,6 @@ fn partial_and_full_integration_retain_ticket_state_and_active_assignment() {
|
||||
assert_eq!(second_request.merged_result().unwrap(), &second_merge);
|
||||
assert_eq!(first_request.integration_approval().unwrap(), &first);
|
||||
assert_eq!(second_request.integration_approval().unwrap(), &second);
|
||||
let posts = connection
|
||||
.prepare(
|
||||
"SELECT event.event_ref, attribute.value
|
||||
FROM typed_ticket_events event
|
||||
JOIN typed_ticket_event_attributes attribute USING(workspace_id,ticket_id,event_index)
|
||||
WHERE attribute.key='merge_request_id' ORDER BY event.event_index",
|
||||
)
|
||||
.unwrap()
|
||||
.query_map([], |row| {
|
||||
Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?))
|
||||
})
|
||||
.unwrap()
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
posts,
|
||||
vec![("T:0".into(), "MR".into()), ("T:1".into(), "MR-2".into())]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1435,7 +1324,7 @@ fn historical_ticket_completion_payload_remains_readable_and_unchanged_on_migrat
|
||||
}"#;
|
||||
connection.execute_batch(
|
||||
"UPDATE typed_tickets SET workflow_state='done' WHERE workspace_id='W' AND ticket_id='T';
|
||||
INSERT INTO typed_ticket_events VALUES('W','T',0,'state_changed','worker:runtime:coder','2026-07-26T12:00:07Z','inprogress','done','Ticket requirements completed','historical body','T:0');"
|
||||
INSERT INTO typed_ticket_events VALUES('W','T',0,'state_changed','worker:runtime:coder','2026-07-26T12:00:07Z','inprogress','done','Ticket requirements completed','historical body');"
|
||||
).unwrap();
|
||||
connection
|
||||
.execute(
|
||||
|
||||
@@ -52,7 +52,7 @@ pub enum AuthenticatedInputSource {
|
||||
/// Assigned whenever a serialized tracked method crosses an untrusted
|
||||
/// protocol boundary. Receivers must handle it exactly like public input.
|
||||
UntrustedWire,
|
||||
/// Stamped only by the operator-owned standalone in-process transport.
|
||||
/// Stamped only by the operator-owned Standalone Host after local admission.
|
||||
/// This is not a Backend Account or authority accepted from wire fields.
|
||||
LocalOperator,
|
||||
Account {
|
||||
@@ -399,15 +399,12 @@ pub enum Method {
|
||||
|
||||
/// Typed lifecycle events sent from a child Worker to its parent.
|
||||
///
|
||||
/// Delivered as `Method::WorkerEvent` over the parent's Unix socket. The
|
||||
/// parent Controller always applies variant-specific side effects
|
||||
/// (registry / worker-allocation updates). Agent-visible variants are also
|
||||
/// queued into the notification buffer; control-plane-only variants are
|
||||
/// not injected into the parent's LLM context.
|
||||
/// Delivered in-process to the parent Controller's protocol intake. Agent-visible variants
|
||||
/// are queued into the notification buffer. Reports are advisory: receivers must check the
|
||||
/// actual child session before treating its work as complete.
|
||||
///
|
||||
/// Transport is fire-and-forget; receivers must tolerate out-of-order
|
||||
/// delivery (e.g. `TurnEnded` arriving after `ShutDown` for the same
|
||||
/// child Worker).
|
||||
/// Retained legacy control-only variants can be decoded from saved history, but do not
|
||||
/// register live children, delegate scope, or establish a socket callback.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))]
|
||||
@@ -429,16 +426,8 @@ pub enum WorkerEvent {
|
||||
/// Child has stopped (controller loop is exiting).
|
||||
ShutDown { worker_name: String },
|
||||
|
||||
/// Child SubWorker sub-delegated scope to a grandchild SubWorker via `SubWorkerSpawn`.
|
||||
///
|
||||
/// Control-plane only: receivers apply registry side effects and
|
||||
/// propagate upward, but do not expose this as an agent notification.
|
||||
///
|
||||
/// The parent uses this to add the grandchild to its own
|
||||
/// `spawned_workers.json` so it can manage the grandchild directly
|
||||
/// even if the intermediate child dies. The parent then re-fires
|
||||
/// this event upward (if it has a parent of its own) to maintain
|
||||
/// the chain to root.
|
||||
/// Legacy process-child scope report retained for saved Session decoding only.
|
||||
/// Live Controllers ignore it; only the Host-owned Internal registry grants child scope.
|
||||
ScopeSubDelegated {
|
||||
/// Sub-delegating Worker (= the sender itself).
|
||||
parent_worker: String,
|
||||
|
||||
+16
-113
@@ -125,6 +125,8 @@ pub struct WorkdirOperationApiError {
|
||||
pub message: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub diagnostics: Option<Vec<Diagnostic>>,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub audit_failures: Vec<workdir::http::WorkdirAuditFailure>,
|
||||
#[serde(skip, default = "default_repository_error_status")]
|
||||
status: u16,
|
||||
/// Server-local metadata retained by generated error response extensions, never the wire body.
|
||||
@@ -156,6 +158,7 @@ impl WorkdirOperationApiError {
|
||||
code: None,
|
||||
message: error.message,
|
||||
diagnostics: Some(error.diagnostics),
|
||||
audit_failures: Vec::new(),
|
||||
status: error.status,
|
||||
workdir_log: None,
|
||||
denial_reason: None,
|
||||
@@ -170,6 +173,7 @@ impl WorkdirOperationApiError {
|
||||
code: Some(error.code),
|
||||
message: error.message,
|
||||
diagnostics: None,
|
||||
audit_failures: Vec::new(),
|
||||
status,
|
||||
workdir_log: None,
|
||||
denial_reason: error.denial_reason,
|
||||
@@ -225,10 +229,6 @@ impl_openapi_schema!(
|
||||
PasskeyLoginOptionsResponse,
|
||||
PasskeyLoginCompleteRequest,
|
||||
PasskeyAdditionOptionsRequest,
|
||||
PasskeySummary,
|
||||
PasskeyListResponse,
|
||||
PasskeyAdditionResultRequest,
|
||||
PasskeyAdditionResultResponse,
|
||||
PasskeyRegistrationOptionsRequest,
|
||||
PasskeyRegistrationOptionsResponse,
|
||||
PasskeyRegistrationCompleteRequest,
|
||||
@@ -652,33 +652,6 @@ pub trait ServerApi {
|
||||
#[body] request: PasskeyRegistrationCompleteRequest,
|
||||
) -> Result<server_api_responses::AuthPasskeyRegistrationComplete, RepositoryApiError>;
|
||||
|
||||
#[get(
|
||||
"/api/auth/passkeys",
|
||||
status = 200,
|
||||
error_status = 400,
|
||||
additional_error_statuses = [403, 500],
|
||||
bearer_auth = true,
|
||||
browser_auth = true
|
||||
)]
|
||||
async fn auth_passkey_list(
|
||||
&self,
|
||||
#[extension] context: ServerRequestContext,
|
||||
) -> Result<PasskeyListResponse, RepositoryApiError>;
|
||||
|
||||
#[post(
|
||||
"/api/auth/passkeys/addition/result",
|
||||
status = 200,
|
||||
error_status = 400,
|
||||
additional_error_statuses = [403, 500],
|
||||
bearer_auth = true,
|
||||
browser_auth = true
|
||||
)]
|
||||
async fn auth_passkey_addition_result(
|
||||
&self,
|
||||
#[extension] context: ServerRequestContext,
|
||||
#[body] request: PasskeyAdditionResultRequest,
|
||||
) -> Result<PasskeyAdditionResultResponse, RepositoryApiError>;
|
||||
|
||||
#[post(
|
||||
"/api/auth/passkeys/addition/options",
|
||||
status = 200,
|
||||
@@ -4146,8 +4119,6 @@ impl RequestActor {
|
||||
pub struct WhoamiResponse {
|
||||
pub actor: Option<RequestActor>,
|
||||
pub session: BrowserSessionState,
|
||||
/// Non-secret comparison evidence, never an authentication credential.
|
||||
pub authentication_reference: Option<String>,
|
||||
}
|
||||
|
||||
/// Browser navigation evidence, not an authorization grant or a generic 401 classification.
|
||||
@@ -4175,8 +4146,6 @@ pub struct AuthBootstrapUserRequest {
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct AuthUserResponse {
|
||||
pub user: AuthenticatedUser,
|
||||
/// Comparison evidence for the exact newly issued session, not a credential.
|
||||
pub authentication_reference: String,
|
||||
}
|
||||
|
||||
/// Starts new-account registration only; an existing handle is never interpreted as key addition.
|
||||
@@ -4193,64 +4162,16 @@ pub struct PasskeyRegistrationOptionsRequest {
|
||||
pub browser_origin: Option<String>,
|
||||
}
|
||||
|
||||
/// Adds to the current actor. Expected identity/binding fields only fence stale UI;
|
||||
/// they never select an owner or authorize an addition.
|
||||
/// Adds a credential to the current authenticated actor; target identity is never public input.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct PasskeyAdditionOptionsRequest {
|
||||
pub expected_user_id: String,
|
||||
pub expected_account_id: String,
|
||||
pub expected_authentication_reference: String,
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
#[cfg_attr(feature = "typescript", ts(optional = nullable))]
|
||||
pub browser_origin: Option<String>,
|
||||
}
|
||||
|
||||
/// Safe account metadata. Neither the WebAuthn credential ID nor key material is exposed.
|
||||
/// Normal-login verification is observed verifier success, not inferred discoverability.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct PasskeySummary {
|
||||
pub key_reference: String,
|
||||
pub created_at: String,
|
||||
pub last_used_at: Option<String>,
|
||||
pub normal_login_verified_at: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct PasskeyListResponse {
|
||||
pub user: AuthenticatedUser,
|
||||
pub keys: Vec<PasskeySummary>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct PasskeyAdditionResultRequest {
|
||||
pub challenge_id: String,
|
||||
pub expected_user_id: String,
|
||||
pub expected_account_id: String,
|
||||
/// Original options authentication reference; the current same-owner session may differ.
|
||||
pub expected_authentication_reference: String,
|
||||
}
|
||||
|
||||
/// Unknown is not proof that registration failed: completion may still be in flight.
|
||||
/// Saved proves the credential commit, not session issuance or current session health.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(tag = "status", rename_all = "snake_case", deny_unknown_fields)]
|
||||
pub enum PasskeyAdditionResultResponse {
|
||||
Unknown {},
|
||||
Saved {
|
||||
user: AuthenticatedUser,
|
||||
key: PasskeySummary,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
#[serde(deny_unknown_fields)]
|
||||
@@ -7593,6 +7514,9 @@ pub struct CurrentWorkerWorkdirAttachmentResponse {
|
||||
pub alias: String,
|
||||
pub working_directory_id: String,
|
||||
pub capabilities: workdir::WorkdirSessionCapabilities,
|
||||
/// Present for attachment responses, absent after detach.
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub environment: Option<workdir::WorkdirSessionEnvironment>,
|
||||
pub attached: bool,
|
||||
}
|
||||
|
||||
@@ -7605,7 +7529,9 @@ pub struct CurrentWorkerWorkdirOperationRequest {
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||
#[serde(transparent)]
|
||||
pub struct CurrentWorkerWorkdirOperationResponse(pub workdir::http::WorkdirSessionOperationResult);
|
||||
pub struct CurrentWorkerWorkdirOperationResponse(
|
||||
pub workdir::http::WorkdirSessionOperationResponse,
|
||||
);
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)]
|
||||
#[cfg_attr(feature = "typescript", derive(ts_rs::TS))]
|
||||
@@ -11725,10 +11651,6 @@ pub fn auth_api_typescript() -> String {
|
||||
AuthBootstrapUserRequest::decl(&config),
|
||||
AuthUserResponse::decl(&config),
|
||||
PasskeyAdditionOptionsRequest::decl(&config),
|
||||
PasskeySummary::decl(&config),
|
||||
PasskeyListResponse::decl(&config),
|
||||
PasskeyAdditionResultRequest::decl(&config),
|
||||
PasskeyAdditionResultResponse::decl(&config),
|
||||
PasskeyRegistrationOptionsRequest::decl(&config),
|
||||
PasskeyRegistrationOptionsResponse::decl(&config),
|
||||
PasskeyRegistrationCompleteRequest::decl(&config),
|
||||
@@ -13367,6 +13289,7 @@ mod tests {
|
||||
alias: "checkout".to_string(),
|
||||
working_directory_id: "workdir".to_string(),
|
||||
capabilities: workdir::WorkdirSessionCapabilities::EMPTY,
|
||||
environment: None,
|
||||
attached: false,
|
||||
};
|
||||
let value = serde_json::to_value(&response).unwrap();
|
||||
@@ -14156,8 +14079,7 @@ mod tests {
|
||||
"display_name": "Hare",
|
||||
"auth_method": "browser_session"
|
||||
},
|
||||
"session": { "kind": "authenticated", "expires_at": "2026-10-20T00:00:00Z" },
|
||||
"authentication_reference": "synthetic-session-reference"
|
||||
"session": { "kind": "authenticated", "expires_at": "2026-10-20T00:00:00Z" }
|
||||
});
|
||||
let decoded = serde_json::from_value::<WhoamiResponse>(whoami.clone())
|
||||
.expect("server whoami fixture should match shared DTO");
|
||||
@@ -14179,8 +14101,7 @@ mod tests {
|
||||
"account_id": "account-1",
|
||||
"handle": "hare",
|
||||
"display_name": "Hare"
|
||||
},
|
||||
"authentication_reference": "synthetic-session-reference"
|
||||
}
|
||||
});
|
||||
let decoded = serde_json::from_value::<AuthUserResponse>(auth_user.clone())
|
||||
.expect("server auth-user fixture should match shared DTO");
|
||||
@@ -14353,7 +14274,7 @@ mod tests {
|
||||
serde_json::json!({"kind":"expired"}),
|
||||
serde_json::json!({"kind":"authenticated", "expires_at":null}),
|
||||
] {
|
||||
let value = serde_json::json!({"actor":null, "session":session, "authentication_reference": null});
|
||||
let value = serde_json::json!({"actor":null, "session":session});
|
||||
let response: WhoamiResponse = serde_json::from_value(value.clone()).unwrap();
|
||||
assert_eq!(serde_json::to_value(response).unwrap(), value);
|
||||
}
|
||||
@@ -14891,28 +14812,10 @@ mod openapi_artifact_tests {
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
serde_json::from_value::<PasskeyAdditionOptionsRequest>(serde_json::json!({})).is_err()
|
||||
serde_json::from_value::<PasskeyAdditionOptionsRequest>(serde_json::json!({})).is_ok()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn passkey_addition_requires_expected_owner_and_binding() {
|
||||
let expected = serde_json::json!({
|
||||
"expected_user_id": "user-a", "expected_account_id": "account-a",
|
||||
"expected_authentication_reference": "reference"
|
||||
});
|
||||
assert!(serde_json::from_value::<PasskeyAdditionOptionsRequest>(expected.clone()).is_ok());
|
||||
for field in [
|
||||
"expected_user_id",
|
||||
"expected_account_id",
|
||||
"expected_authentication_reference",
|
||||
] {
|
||||
let mut missing = expected.clone();
|
||||
missing.as_object_mut().unwrap().remove(field);
|
||||
assert!(serde_json::from_value::<PasskeyAdditionOptionsRequest>(missing).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn auth_workspace_catalog_and_typed_header_contract_is_strict() {
|
||||
let document = canonical_openapi_document().expect("canonical OpenAPI contract must build");
|
||||
|
||||
@@ -23,7 +23,7 @@ serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
session-store.workspace = true
|
||||
thiserror.workspace = true
|
||||
tokio = { workspace = true, features = ["rt", "sync", "time"] }
|
||||
tokio = { workspace = true, features = ["rt", "sync", "time", "net", "io-util", "io-std", "signal"] }
|
||||
uuid = { workspace = true, features = ["v7"] }
|
||||
worker.workspace = true
|
||||
|
||||
|
||||
@@ -4,7 +4,8 @@ A Host that starts existing Workers in the same process. Assembles interactive W
|
||||
|
||||
## Reference index
|
||||
|
||||
- [`host`] — bootstrap and confirmed shutdown.
|
||||
- [`host`] — bootstrap and confirmed shutdown, with or without a TUI.
|
||||
- [`unix`] — optional operator transport; disconnect does not release execution ownership.
|
||||
- [`launch`] — Profile, cwd, and Subject validation before launch.
|
||||
- [`store`] — Worker records and leases.
|
||||
- [`jobs`] — Job execution and independent result and cleanup lifetimes.
|
||||
|
||||
@@ -24,7 +24,6 @@ use session_store::{
|
||||
};
|
||||
use thiserror::Error;
|
||||
use worker::bootstrap::{WorkerBootstrap, WorkerBootstrapError, WorkerBootstrapLayout};
|
||||
use worker::controller::WorkerControllerTransport;
|
||||
use worker::ipc::protocol_session::{
|
||||
WorkerProtocolSessionStreams, dispatch_worker_protocol_method, live_log_entry_event,
|
||||
subscribe_worker_protocol_session,
|
||||
@@ -222,7 +221,6 @@ impl StandaloneHost {
|
||||
workspace_context,
|
||||
filesystem_authority,
|
||||
WorkerBootstrapLayout::Direct { runtime_base },
|
||||
WorkerControllerTransport::InProcess,
|
||||
);
|
||||
if let Some(subject) = &subject {
|
||||
bootstrap = bootstrap.with_subjektiv_host(subject.host.clone());
|
||||
@@ -430,7 +428,6 @@ impl StandaloneHost {
|
||||
workspace_context,
|
||||
filesystem_authority,
|
||||
WorkerBootstrapLayout::Direct { runtime_base },
|
||||
WorkerControllerTransport::InProcess,
|
||||
);
|
||||
if let Some(subject) = &subject {
|
||||
bootstrap = bootstrap.with_subjektiv_host(subject.host.clone());
|
||||
@@ -520,10 +517,7 @@ impl StandaloneHost {
|
||||
/// Working events, committed session entries, alert snapshots, and the
|
||||
/// initial history snapshot are merged behind the client boundary.
|
||||
pub fn connect(&self) -> Client<InProcessSocket> {
|
||||
let streams = subscribe_worker_protocol_session(&self.handle);
|
||||
let (socket, peer) = InProcessSocket::pair();
|
||||
tokio::spawn(run_protocol_session(self.handle.clone(), streams, peer));
|
||||
Client::new(socket)
|
||||
Client::new(connect_protocol(self.handle.clone()))
|
||||
}
|
||||
|
||||
/// Logical Subject disconnection, not Worker shutdown or completion of its represented work.
|
||||
@@ -565,6 +559,35 @@ impl StandaloneHost {
|
||||
}
|
||||
}
|
||||
|
||||
/// Run without a TUI until an admitted protocol Shutdown, controller exit, or process signal.
|
||||
/// Client EOF only disconnects that client. All exits confirm the same production shutdown
|
||||
/// and lease release as the in-process UI; an unconfirmed shutdown retains the lease.
|
||||
pub async fn serve_unix(
|
||||
self,
|
||||
endpoint: crate::unix::UnixEndpoint,
|
||||
) -> Result<(), crate::HeadlessError> {
|
||||
let mut adapter = crate::unix::UnixAdapter::start(endpoint, self.handle.clone());
|
||||
let signal = async {
|
||||
let mut terminate =
|
||||
tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())?;
|
||||
tokio::select! {
|
||||
result = tokio::signal::ctrl_c() => result,
|
||||
_ = terminate.recv() => Ok(()),
|
||||
}
|
||||
};
|
||||
let result = tokio::select! {
|
||||
_ = self.handle.protocol_closed() => Ok(()),
|
||||
result = signal => result,
|
||||
result = adapter.ended() => result,
|
||||
};
|
||||
// Reclaim the public entry and every connection before releasing execution ownership.
|
||||
let transport = adapter.shutdown().await;
|
||||
self.shutdown().await?;
|
||||
result?;
|
||||
transport?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn with_shutdown_timeout(mut self, shutdown_timeout: Duration) -> Self {
|
||||
self.shutdown_timeout = shutdown_timeout;
|
||||
self
|
||||
@@ -647,9 +670,8 @@ impl Drop for StandaloneHost {
|
||||
}
|
||||
}
|
||||
|
||||
// The in-process channel is owned by the local operator's Host, not an
|
||||
// arbitrary socket or Backend account. decode_method discards wire provenance;
|
||||
// only this boundary can stamp accepted local human input.
|
||||
// Both in-process views and credential-checked Unix clients are admitted by this operator's
|
||||
// Host. decode_method discards wire provenance; only this boundary stamps local human input.
|
||||
fn authorize_local_input(method: protocol::Method) -> protocol::Method {
|
||||
use protocol::{AuthenticatedInputSource, Method};
|
||||
match method {
|
||||
@@ -692,6 +714,13 @@ fn authorize_local_input(method: protocol::Method) -> protocol::Method {
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn connect_protocol(handle: worker::WorkerHandle) -> InProcessSocket {
|
||||
let streams = subscribe_worker_protocol_session(&handle);
|
||||
let (socket, peer) = InProcessSocket::pair();
|
||||
tokio::spawn(run_protocol_session(handle, streams, peer));
|
||||
socket
|
||||
}
|
||||
|
||||
async fn run_protocol_session(
|
||||
handle: worker::WorkerHandle,
|
||||
streams: WorkerProtocolSessionStreams,
|
||||
|
||||
@@ -7,6 +7,44 @@ pub mod launch;
|
||||
pub mod store;
|
||||
pub mod subjektiv;
|
||||
mod subjektiv_sessions;
|
||||
pub mod unix;
|
||||
|
||||
pub use unix::{UnixEndpoint, relay_unix_client, run_unix_client};
|
||||
|
||||
/// Restore reads the saved cwd and Profile, not a newly resolved launch recipe.
|
||||
pub enum HeadlessLaunch {
|
||||
Fresh(ResolvedStandaloneLaunch),
|
||||
Restore {
|
||||
state_dir: std::path::PathBuf,
|
||||
worker_id: WorkerId,
|
||||
},
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum HeadlessError {
|
||||
#[error(transparent)]
|
||||
Transport(#[from] std::io::Error),
|
||||
#[error(transparent)]
|
||||
Startup(#[from] StandaloneStartupError),
|
||||
#[error(transparent)]
|
||||
Shutdown(#[from] StandaloneShutdownError),
|
||||
}
|
||||
|
||||
/// Reserve the optional public transport before creating or restoring execution state.
|
||||
pub async fn run_headless(
|
||||
launch: HeadlessLaunch,
|
||||
endpoint: std::path::PathBuf,
|
||||
) -> Result<(), HeadlessError> {
|
||||
let endpoint = UnixEndpoint::bind(endpoint)?;
|
||||
let host = match launch {
|
||||
HeadlessLaunch::Fresh(launch) => StandaloneHost::start(launch).await?,
|
||||
HeadlessLaunch::Restore {
|
||||
state_dir,
|
||||
worker_id,
|
||||
} => StandaloneHost::restore(state_dir, worker_id).await?,
|
||||
};
|
||||
host.serve_unix(endpoint).await
|
||||
}
|
||||
|
||||
pub use host::{
|
||||
StandaloneHost, StandaloneShutdownError, StandaloneStartupError,
|
||||
|
||||
@@ -127,7 +127,6 @@ async fn local_host_compaction_freezes_pending_journal_through_request_and_durab
|
||||
worker::WorkerWorkspaceContext::no_workspace(),
|
||||
worker::WorkerFilesystemAuthority::None,
|
||||
worker::WorkerBootstrapLayout::Direct { runtime_base: fixture._temp.path().join("runtime") },
|
||||
worker::WorkerControllerTransport::InProcess,
|
||||
).with_model_client(model.clone()).with_subjektiv_host(fixture.host().clone()).prepare_restored("journal-test").await.unwrap();
|
||||
assert_eq!(prepared.worker().session_id().to_string(), fixture.context.session_id);
|
||||
let source_segment = prepared.worker().segment_id();
|
||||
|
||||
@@ -0,0 +1,296 @@
|
||||
//! Optional operator transport owned by the Standalone Host, never by a Worker.
|
||||
//!
|
||||
//! The endpoint's existing parent must be private to the current Unix user. Binding never
|
||||
//! removes an existing entry, including stale sockets. Recovery is an explicit operator action.
|
||||
//! Peers must have that same uid; admitted methods still cross the normal local protocol
|
||||
//! authorization boundary. Disconnect only drops a view, not the Host or its execution lease.
|
||||
|
||||
use std::io;
|
||||
use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use client::transport::Socket;
|
||||
use protocol::stream::{JsonLineWriter, decode_event, decode_method};
|
||||
use tokio::io::{AsyncBufReadExt, BufReader};
|
||||
use tokio::net::{UnixListener, UnixStream};
|
||||
use tokio::sync::oneshot;
|
||||
use tokio::task::{JoinHandle, JoinSet};
|
||||
|
||||
/// A bound endpoint reservation. Drop removes only the exact socket created by this bind.
|
||||
/// It can be reserved before starting a Host, so bind failures do not allocate Workers.
|
||||
pub struct UnixEndpoint {
|
||||
listener: UnixListener,
|
||||
path: PathBuf,
|
||||
device: u64,
|
||||
inode: u64,
|
||||
}
|
||||
|
||||
impl UnixEndpoint {
|
||||
pub fn bind(path: impl AsRef<Path>) -> io::Result<Self> {
|
||||
let path = path.as_ref();
|
||||
let name = path.file_name().ok_or_else(|| {
|
||||
io::Error::new(io::ErrorKind::InvalidInput, "endpoint requires a filename")
|
||||
})?;
|
||||
let parent = std::fs::canonicalize(
|
||||
path.parent()
|
||||
.filter(|p| !p.as_os_str().is_empty())
|
||||
.unwrap_or(Path::new(".")),
|
||||
)?;
|
||||
let metadata = std::fs::metadata(&parent)?;
|
||||
// SAFETY: geteuid has no preconditions and does not mutate process identity.
|
||||
let uid = unsafe { libc::geteuid() };
|
||||
if !metadata.is_dir() || metadata.uid() != uid || metadata.mode() & 0o077 != 0 {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::PermissionDenied,
|
||||
"endpoint parent must be a current-user-owned private directory (mode 0700)",
|
||||
));
|
||||
}
|
||||
let path = parent.join(name);
|
||||
let listener = UnixListener::bind(&path)?;
|
||||
let metadata = std::fs::symlink_metadata(&path)?;
|
||||
let endpoint = Self {
|
||||
listener,
|
||||
path,
|
||||
device: metadata.dev(),
|
||||
inode: metadata.ino(),
|
||||
};
|
||||
std::fs::set_permissions(&endpoint.path, std::fs::Permissions::from_mode(0o600))?;
|
||||
Ok(endpoint)
|
||||
}
|
||||
|
||||
pub fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for UnixEndpoint {
|
||||
fn drop(&mut self) {
|
||||
if let Ok(current) = std::fs::symlink_metadata(&self.path)
|
||||
&& current.file_type().is_socket()
|
||||
&& current.dev() == self.device
|
||||
&& current.ino() == self.inode
|
||||
{
|
||||
let _ = std::fs::remove_file(&self.path);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) struct UnixAdapter {
|
||||
task: Option<JoinHandle<io::Result<()>>>,
|
||||
stop: Option<oneshot::Sender<()>>,
|
||||
}
|
||||
|
||||
impl UnixAdapter {
|
||||
pub(crate) fn start(endpoint: UnixEndpoint, handle: worker::WorkerHandle) -> Self {
|
||||
let (stop, mut stopped) = oneshot::channel();
|
||||
let task = tokio::spawn(async move {
|
||||
let mut connections = JoinSet::new();
|
||||
let result = loop {
|
||||
tokio::select! {
|
||||
_ = &mut stopped => break Ok(()),
|
||||
accepted = endpoint.listener.accept(), if connections.len() < 64 => {
|
||||
match accepted {
|
||||
Ok((stream, _)) => {
|
||||
// The filesystem gate protects connection establishment; credentials
|
||||
// also fence open connections to the Host's operator identity.
|
||||
let allowed = stream.peer_cred().is_ok_and(|cred| {
|
||||
// SAFETY: geteuid has no preconditions.
|
||||
cred.uid() == unsafe { libc::geteuid() }
|
||||
});
|
||||
if allowed {
|
||||
let socket = crate::host::connect_protocol(handle.clone());
|
||||
connections.spawn(bridge(stream, socket));
|
||||
}
|
||||
}
|
||||
Err(error) => break Err(error),
|
||||
}
|
||||
}
|
||||
_ = connections.join_next(), if !connections.is_empty() => {}
|
||||
}
|
||||
};
|
||||
connections.shutdown().await;
|
||||
// Keep the bound endpoint alive until every connection has been reclaimed.
|
||||
drop(endpoint);
|
||||
result
|
||||
});
|
||||
Self {
|
||||
task: Some(task),
|
||||
stop: Some(stop),
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) async fn ended(&mut self) -> io::Result<()> {
|
||||
let result = self.task.as_mut().expect("adapter task").await;
|
||||
self.task.take();
|
||||
result.map_err(io::Error::other)?
|
||||
}
|
||||
|
||||
pub(crate) async fn shutdown(mut self) -> io::Result<()> {
|
||||
if let Some(stop) = self.stop.take() {
|
||||
let _ = stop.send(());
|
||||
}
|
||||
if self.task.is_some() {
|
||||
self.ended().await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for UnixAdapter {
|
||||
fn drop(&mut self) {
|
||||
if let Some(stop) = self.stop.take() {
|
||||
let _ = stop.send(());
|
||||
}
|
||||
// Sending stop lets the task join its connection set and drop its endpoint.
|
||||
// A Host dropped without confirmed shutdown keeps the production execution lease.
|
||||
}
|
||||
}
|
||||
|
||||
async fn bridge(
|
||||
stream: UnixStream,
|
||||
mut socket: client::transport::in_process::Socket,
|
||||
) -> io::Result<()> {
|
||||
let (reader, writer) = stream.into_split();
|
||||
let mut reader = BufReader::new(reader).lines();
|
||||
let mut writer = JsonLineWriter::new(writer);
|
||||
loop {
|
||||
tokio::select! {
|
||||
line = reader.next_line() => {
|
||||
let Some(message) = line? else { return Ok(()); };
|
||||
if message.trim().is_empty() { continue; }
|
||||
socket.send(message).await.map_err(io::Error::other)?;
|
||||
}
|
||||
event = socket.next() => {
|
||||
let Some(message) = event.map_err(io::Error::other)? else { return Ok(()); };
|
||||
let event = decode_event(&message).map_err(io::Error::other)?;
|
||||
writer.write_event(&event).await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Connect a JSONL operator client. EOF disconnects without issuing Shutdown.
|
||||
/// Input is decoded as Worker Methods and output contains Worker Events only.
|
||||
pub async fn run_unix_client(endpoint: PathBuf) -> io::Result<()> {
|
||||
relay_unix_client(endpoint, tokio::io::stdin(), tokio::io::stdout()).await
|
||||
}
|
||||
|
||||
/// The CLI's stream relay, also usable by embedders that own their input/output streams.
|
||||
pub async fn relay_unix_client<R, W>(endpoint: PathBuf, input: R, output: W) -> io::Result<()>
|
||||
where
|
||||
R: tokio::io::AsyncRead + Unpin,
|
||||
W: tokio::io::AsyncWrite + Unpin,
|
||||
{
|
||||
let stream = UnixStream::connect(endpoint).await?;
|
||||
let (reader, writer) = stream.into_split();
|
||||
let mut input = BufReader::new(input).lines();
|
||||
let mut to_host = JsonLineWriter::new(writer);
|
||||
let mut from_host = BufReader::new(reader).lines();
|
||||
let mut output = JsonLineWriter::new(output);
|
||||
loop {
|
||||
tokio::select! {
|
||||
line = input.next_line() => {
|
||||
let Some(line) = line? else { return Ok(()); };
|
||||
if line.trim().is_empty() { continue; }
|
||||
let method = decode_method(&line).map_err(io::Error::other)?;
|
||||
to_host.write(&method).await?;
|
||||
}
|
||||
line = from_host.next_line() => {
|
||||
let Some(line) = line? else { return Ok(()); };
|
||||
let event = decode_event(&line).map_err(io::Error::other)?;
|
||||
output.write_event(&event).await?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn endpoint_requires_private_parent_and_is_owner_only() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
let path = directory.path().join("host.sock");
|
||||
std::fs::set_permissions(directory.path(), std::fs::Permissions::from_mode(0o755)).unwrap();
|
||||
let error = UnixEndpoint::bind(&path)
|
||||
.err()
|
||||
.expect("public parent rejected");
|
||||
assert_eq!(error.kind(), io::ErrorKind::PermissionDenied);
|
||||
assert!(!path.exists());
|
||||
std::fs::set_permissions(directory.path(), std::fs::Permissions::from_mode(0o700)).unwrap();
|
||||
let endpoint = UnixEndpoint::bind(&path).unwrap();
|
||||
assert_eq!(
|
||||
std::fs::metadata(&path).unwrap().permissions().mode() & 0o777,
|
||||
0o600
|
||||
);
|
||||
let client = UnixStream::connect(&path).await.unwrap();
|
||||
let (server, _) = endpoint.listener.accept().await.unwrap();
|
||||
assert_eq!(
|
||||
client.peer_cred().unwrap().uid(),
|
||||
server.peer_cred().unwrap().uid()
|
||||
);
|
||||
drop(endpoint);
|
||||
assert!(!path.exists());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bind_failure_precedes_worker_state_and_startup_failure_releases_only_its_endpoint() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
std::fs::set_permissions(directory.path(), std::fs::Permissions::from_mode(0o700)).unwrap();
|
||||
let path = directory.path().join("host.sock");
|
||||
let owner = UnixEndpoint::bind(&path).unwrap();
|
||||
let state_dir = directory.path().join("must-not-be-opened");
|
||||
let error = crate::run_headless(
|
||||
crate::HeadlessLaunch::Restore {
|
||||
state_dir: state_dir.clone(),
|
||||
worker_id: protocol::WorkerId::now_v7(),
|
||||
},
|
||||
path.clone(),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(error, crate::HeadlessError::Transport(_)));
|
||||
assert!(!state_dir.exists());
|
||||
assert!(path.exists());
|
||||
drop(owner);
|
||||
let error = crate::run_headless(
|
||||
crate::HeadlessLaunch::Restore {
|
||||
state_dir,
|
||||
worker_id: protocol::WorkerId::now_v7(),
|
||||
},
|
||||
path.clone(),
|
||||
)
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(error, crate::HeadlessError::Startup(_)));
|
||||
assert!(
|
||||
!path.exists(),
|
||||
"a failed Host startup must release its own bind"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn bind_failure_and_old_endpoint_drop_preserve_another_owner() {
|
||||
let directory = tempfile::tempdir().unwrap();
|
||||
std::fs::set_permissions(directory.path(), std::fs::Permissions::from_mode(0o700)).unwrap();
|
||||
let path = directory.path().join("host.sock");
|
||||
let first = UnixEndpoint::bind(&path).unwrap();
|
||||
let first_inode = std::fs::metadata(&path).unwrap().ino();
|
||||
assert_eq!(
|
||||
UnixEndpoint::bind(&path).err().unwrap().kind(),
|
||||
io::ErrorKind::AddrInUse
|
||||
);
|
||||
assert_eq!(std::fs::metadata(&path).unwrap().ino(), first_inode);
|
||||
std::fs::rename(&path, directory.path().join("retained.sock")).unwrap();
|
||||
let second = UnixEndpoint::bind(&path).unwrap();
|
||||
let second_inode = std::fs::metadata(&path).unwrap().ino();
|
||||
assert_ne!(first_inode, second_inode);
|
||||
drop(first);
|
||||
assert_eq!(std::fs::metadata(&path).unwrap().ino(), second_inode);
|
||||
let _client = UnixStream::connect(&path).await.unwrap();
|
||||
let _connection = second.listener.accept().await.unwrap();
|
||||
drop(second);
|
||||
assert!(!path.exists());
|
||||
}
|
||||
}
|
||||
@@ -608,6 +608,235 @@ async fn standalone_metadata_fails_closed_on_incomplete_or_newer_records() -> Te
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn headless_unix_client_uses_production_history_lease_and_local_authority() -> TestResult {
|
||||
use protocol::stream::{JsonLineReader, encode_method};
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
let temp = tempfile::tempdir()?;
|
||||
// Only the transport endpoint needs a short private pathname. Persistence remains the
|
||||
// same production store exercised by the in-process Host tests above.
|
||||
std::fs::set_permissions(temp.path(), std::fs::Permissions::from_mode(0o700))?;
|
||||
let endpoint_path = temp.path().join("operator.sock");
|
||||
let endpoint = standalone::UnixEndpoint::bind(&endpoint_path)?;
|
||||
let state_dir = temp.path().join("state");
|
||||
let launch = StandaloneLaunchConfig::new(
|
||||
temp.path(),
|
||||
&state_dir,
|
||||
manifest::ProfileSelector::Default,
|
||||
"headless-operator",
|
||||
)
|
||||
.resolve()?;
|
||||
let model = ScriptedClient::new(vec![vec![
|
||||
LlmEvent::text_block_start(0),
|
||||
LlmEvent::text_delta(0, "headless production response"),
|
||||
LlmEvent::text_block_stop(0, Some(StopReason::EndTurn)),
|
||||
]]);
|
||||
let inspection = model.clone();
|
||||
let host = StandaloneHost::start_with_model_client(launch, model).await?;
|
||||
let worker_id = host.worker_id();
|
||||
let session_id = host.record().active_session_id;
|
||||
let profile = host.record().manifest.clone();
|
||||
let serve = tokio::spawn(host.serve_unix(endpoint));
|
||||
|
||||
// Exercise the real external client's stdin/stdout relay over the actual Unix adapter.
|
||||
let (mut input, relay_input) = tokio::io::duplex(64 * 1024);
|
||||
let (relay_output, output) = tokio::io::duplex(64 * 1024);
|
||||
let relay = tokio::spawn(standalone::relay_unix_client(
|
||||
endpoint_path.clone(),
|
||||
relay_input,
|
||||
relay_output,
|
||||
));
|
||||
let mut events = JsonLineReader::new(output);
|
||||
let snapshot = tokio::time::timeout(Duration::from_secs(10), events.next::<Event>())
|
||||
.await??
|
||||
.expect("snapshot");
|
||||
assert!(matches!(snapshot, Event::Snapshot { .. }));
|
||||
let request = Method::SubmitTracked {
|
||||
submission_request_id: protocol::new_submission_request_id(),
|
||||
input: vec![protocol::Segment::text("headless production input")],
|
||||
source: protocol::AuthenticatedInputSource::Backend {
|
||||
operation_id: "forged-backend-source".into(),
|
||||
},
|
||||
};
|
||||
let wire = encode_method(&request)?;
|
||||
println!("headless input: {wire}");
|
||||
input.write_all(wire.as_bytes()).await?;
|
||||
input.write_all(b"\n").await?;
|
||||
let mut saw_user = false;
|
||||
let mut saw_response = false;
|
||||
let mut saw_state = false;
|
||||
tokio::time::timeout(Duration::from_secs(20), async {
|
||||
loop {
|
||||
let event = events.next::<Event>().await?.expect("live event");
|
||||
match &event {
|
||||
Event::UserMessage { segments, .. } => {
|
||||
saw_user |= format!("{segments:?}").contains("headless production input");
|
||||
}
|
||||
Event::TextDelta { text } => {
|
||||
saw_response |= text == "headless production response";
|
||||
}
|
||||
Event::WorkerState { .. } => saw_state = true,
|
||||
Event::RunEnd { .. } => break,
|
||||
_ => {}
|
||||
}
|
||||
println!(
|
||||
"headless output: {}",
|
||||
protocol::stream::encode_event(&event)?
|
||||
);
|
||||
}
|
||||
Ok::<_, Box<dyn std::error::Error>>(())
|
||||
})
|
||||
.await??;
|
||||
assert!(saw_user && saw_response && saw_state);
|
||||
assert_eq!(inspection.requests().len(), 1);
|
||||
assert!(matches!(
|
||||
StandaloneHost::restore_with_model_client(
|
||||
state_dir.clone(),
|
||||
worker_id,
|
||||
ScriptedClient::new(vec![])
|
||||
)
|
||||
.await,
|
||||
Err(StandaloneStartupError::WorkerActive)
|
||||
));
|
||||
|
||||
// EOF is a disconnect, not a shutdown; another view can reconnect to the same history.
|
||||
drop(input);
|
||||
tokio::time::timeout(Duration::from_secs(10), relay).await???;
|
||||
assert!(!serve.is_finished());
|
||||
let mut malformed = tokio::net::UnixStream::connect(&endpoint_path).await?;
|
||||
malformed.write_all(b"not-json\n").await?;
|
||||
let mut discarded = Vec::new();
|
||||
use tokio::io::AsyncReadExt;
|
||||
tokio::time::timeout(
|
||||
Duration::from_secs(10),
|
||||
malformed.read_to_end(&mut discarded),
|
||||
)
|
||||
.await??;
|
||||
assert!(
|
||||
!serve.is_finished(),
|
||||
"invalid client input must not stop the Host"
|
||||
);
|
||||
let mut socket_client =
|
||||
Client::new(client::transport::unix_socket::Socket::connect(&endpoint_path).await?);
|
||||
let snapshot = tokio::time::timeout(Duration::from_secs(10), async {
|
||||
loop {
|
||||
let event = socket_client.next_event().await?.expect("reconnect event");
|
||||
if matches!(event, Event::Snapshot { .. }) {
|
||||
break Ok::<_, Box<dyn std::error::Error>>(event);
|
||||
}
|
||||
}
|
||||
})
|
||||
.await??;
|
||||
let snapshot_text = format!("{snapshot:?}");
|
||||
assert!(
|
||||
snapshot_text.contains("headless production input"),
|
||||
"{snapshot_text}"
|
||||
);
|
||||
assert!(
|
||||
snapshot_text.contains("headless production response"),
|
||||
"{snapshot_text}"
|
||||
);
|
||||
socket_client
|
||||
.send(&Method::Shutdown {
|
||||
command: protocol::WorkerCommandEnvelope::new(0),
|
||||
})
|
||||
.await?;
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
loop {
|
||||
if let Some(Event::CommandAcknowledged { acknowledgement }) =
|
||||
socket_client.next_event().await?
|
||||
{
|
||||
assert_eq!(
|
||||
acknowledgement.disposition,
|
||||
protocol::WorkerCommandDisposition::StaleCommandId
|
||||
);
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok::<_, Box<dyn std::error::Error>>(())
|
||||
})
|
||||
.await??;
|
||||
assert!(
|
||||
!serve.is_finished(),
|
||||
"stale Shutdown must not bypass Controller admission"
|
||||
);
|
||||
socket_client
|
||||
.send(&Method::Shutdown {
|
||||
command: protocol::WorkerCommandEnvelope::new(100),
|
||||
})
|
||||
.await?;
|
||||
tokio::time::timeout(Duration::from_secs(15), serve).await???;
|
||||
assert!(!endpoint_path.exists());
|
||||
let store = StandaloneWorkerStore::open(&state_dir)?;
|
||||
let saved = store.load(worker_id)?;
|
||||
assert_eq!(saved.status, StandaloneWorkerStatus::Stopped);
|
||||
use session_store::Store;
|
||||
let history = session_store::FsStore::new(store.sessions_dir(worker_id))?.read_all(
|
||||
saved.active_session_id,
|
||||
saved.active_segment_id.expect("active segment"),
|
||||
)?;
|
||||
let origins = history
|
||||
.iter()
|
||||
.filter_map(|entry| match entry {
|
||||
session_store::LogEntry::AnnotatedUserInput { history, .. } => Some(history),
|
||||
_ => None,
|
||||
})
|
||||
.flatten()
|
||||
.map(|entry| &entry.metadata.origin)
|
||||
.collect::<Vec<_>>();
|
||||
assert!(origins.contains(&&session_store::LoggedSessionHistoryOrigin::LocalHumanInput));
|
||||
assert!(!format!("{origins:?}").contains("forged-backend-source"));
|
||||
|
||||
// This is the same restoration path called by the TUI, not a headless catalog/import.
|
||||
let restored = StandaloneHost::restore_with_model_client(
|
||||
state_dir.clone(),
|
||||
worker_id,
|
||||
ScriptedClient::new(vec![]),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(restored.worker_id(), worker_id);
|
||||
assert_eq!(restored.record().active_session_id, session_id);
|
||||
assert_eq!(
|
||||
serde_json::to_value(&restored.record().manifest)?,
|
||||
serde_json::to_value(&profile)?
|
||||
);
|
||||
assert_eq!(
|
||||
restored.record().cwd.canonical_path,
|
||||
temp.path().canonicalize()?
|
||||
);
|
||||
let mut in_process = restored.connect();
|
||||
let snapshot = tokio::time::timeout(Duration::from_secs(10), async {
|
||||
loop {
|
||||
let event = in_process
|
||||
.next_event()
|
||||
.await?
|
||||
.expect("in-process restored event");
|
||||
if matches!(event, Event::Snapshot { .. }) {
|
||||
break Ok::<_, Box<dyn std::error::Error>>(event);
|
||||
}
|
||||
}
|
||||
})
|
||||
.await??;
|
||||
assert!(format!("{snapshot:?}").contains("headless production response"));
|
||||
assert!(
|
||||
!endpoint_path.exists(),
|
||||
"in-process restore must not recreate the adapter"
|
||||
);
|
||||
let runtime = state_dir
|
||||
.join(worker_id.to_string())
|
||||
.join("runtime")
|
||||
.join(format!("standalone-{worker_id}"));
|
||||
assert!(!runtime.join("sock").exists());
|
||||
assert!(!runtime.join("pid").exists());
|
||||
restored.shutdown().await?;
|
||||
println!(
|
||||
"verdict: input/state/response/disconnect/explicit shutdown, shared production restore and concurrent lease exclusion passed for {worker_id}"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn wait_for_run_end(client: &mut Client<InProcessSocket>) -> TestResult {
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
loop {
|
||||
|
||||
+25
-76
@@ -19,6 +19,8 @@ use workdir::{
|
||||
WorkdirSessionRouter,
|
||||
};
|
||||
|
||||
mod cleanup;
|
||||
|
||||
const DEFAULT_TIMEOUT_SECS: u64 = 120;
|
||||
const MAX_TIMEOUT_SECS: u64 = 600;
|
||||
const INLINE_BYTE_BUDGET: usize = 12 * 1024;
|
||||
@@ -28,7 +30,9 @@ struct BashParams {
|
||||
/// Worker-local alias of the Workdir attachment to use.
|
||||
#[serde(default)]
|
||||
target_workdir: Option<String>,
|
||||
/// Workdir-relative directory in which to execute the command.
|
||||
/// Directory in which to execute the command.
|
||||
// Keep provider path-coordinate documentation out of the model-facing schema.
|
||||
#[schemars(with = "String")]
|
||||
cwd: WorkdirPath,
|
||||
command: String,
|
||||
#[serde(default)]
|
||||
@@ -72,80 +76,19 @@ impl Drop for CommandGuard {
|
||||
let Some(handle) = self.handle.take() else {
|
||||
return;
|
||||
};
|
||||
let workdir = self.session.clone();
|
||||
let state = Arc::clone(&self.state);
|
||||
let execution_id = self.execution_id.clone();
|
||||
let execution_nonce = self.execution_nonce;
|
||||
let mut retained = self.retained.take();
|
||||
let cursor = self.cursor;
|
||||
// A dropped provider future is not terminal confirmation. Keep the live
|
||||
// execution registered until cleanup has both requested cancellation and
|
||||
// observed terminal command output, so cancellation/session teardown
|
||||
// cannot race with an apparently empty registry.
|
||||
tokio::spawn(async move {
|
||||
let result = tokio::time::timeout(std::time::Duration::from_secs(10), async {
|
||||
let _ = workdir.cancel_command(handle.clone()).await;
|
||||
let mut cursor = cursor;
|
||||
loop {
|
||||
let page = command_page(
|
||||
&workdir,
|
||||
CommandOutputRequest {
|
||||
handle: handle.clone(),
|
||||
cursor,
|
||||
limit: workdir::MAX_COMMAND_READ_BYTES,
|
||||
wait: true,
|
||||
},
|
||||
)
|
||||
.await?;
|
||||
if let Some(writer) = &mut retained {
|
||||
let mut next = cursor;
|
||||
for chunk in &page.chunks {
|
||||
if chunk.sequence != next {
|
||||
return Err(workdir::WorkdirError::Unavailable(
|
||||
"command cleanup sequence gap".into(),
|
||||
));
|
||||
}
|
||||
writer
|
||||
.append_chunk(chunk)
|
||||
.map_err(workdir::WorkdirError::Unavailable)?;
|
||||
next = next.checked_add(chunk.bytes.len() as u64).ok_or_else(|| {
|
||||
workdir::WorkdirError::Unavailable(
|
||||
"command cleanup cursor overflow".into(),
|
||||
)
|
||||
})?;
|
||||
}
|
||||
if page.next_cursor != next {
|
||||
return Err(workdir::WorkdirError::Unavailable(
|
||||
"command cleanup cursor gap".into(),
|
||||
));
|
||||
}
|
||||
}
|
||||
cursor = page.next_cursor;
|
||||
if let Some(terminal) = page.terminal {
|
||||
if let Some(writer) = &mut retained {
|
||||
writer
|
||||
.finish_transport(&terminal)
|
||||
.map_err(workdir::WorkdirError::Unavailable)?;
|
||||
}
|
||||
return Ok::<(), workdir::WorkdirError>(());
|
||||
}
|
||||
}
|
||||
})
|
||||
.await;
|
||||
// Bounded cleanup can end with an unknown provider outcome. The writer then
|
||||
// terminalizes partial evidence on drop; retaining a dead collector in `active`
|
||||
// would leak registrations and falsely suggest it could still collect output.
|
||||
let _ = result;
|
||||
let mut state = state.lock().unwrap();
|
||||
if state
|
||||
.active
|
||||
.get(&execution_id)
|
||||
.is_some_and(|active| active.execution_nonce == execution_nonce)
|
||||
{
|
||||
state.active.remove(&execution_id);
|
||||
state.cancellation_requested.remove(&execution_id);
|
||||
}
|
||||
});
|
||||
let cleanup = cleanup::Cleanup {
|
||||
session: self.session.clone(),
|
||||
state: Arc::clone(&self.state),
|
||||
execution_id: self.execution_id.clone(),
|
||||
execution_nonce: self.execution_nonce,
|
||||
handle,
|
||||
cursor: self.cursor,
|
||||
retained: self
|
||||
.retained
|
||||
.take()
|
||||
.expect("started command owns output writer"),
|
||||
};
|
||||
tokio::spawn(cleanup.run());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -367,7 +310,7 @@ impl BashTool {
|
||||
guard.handle = None;
|
||||
|
||||
let timed_out = terminal.status == workdir::CommandStatus::TimedOut;
|
||||
let summary =
|
||||
let mut summary =
|
||||
if cancellation_requested || terminal.status == workdir::CommandStatus::Cancelled {
|
||||
format!("$ {cmd_summary} (cancelled)")
|
||||
} else if timed_out {
|
||||
@@ -379,6 +322,12 @@ impl BashTool {
|
||||
None => format!("$ {cmd_summary} (terminated)"),
|
||||
}
|
||||
};
|
||||
if matches!(
|
||||
terminal.output_collection,
|
||||
workdir::CommandOutputCollection::Incomplete { .. }
|
||||
) {
|
||||
summary.push_str(" (output incomplete)");
|
||||
}
|
||||
Ok(BashResult {
|
||||
summary,
|
||||
stdout_preview: String::from_utf8_lossy(&stdout_preview).into_owned(),
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
//! Finite Drop cleanup. A cancel acknowledgement is not termination evidence; only a
|
||||
//! validated terminal fence confirms the process result. No provider text is logged.
|
||||
|
||||
use super::*;
|
||||
use crate::bash_output::{
|
||||
CleanupCancel, CleanupDiagnostics, CleanupDrain, CleanupFailure, CleanupProviderFailure,
|
||||
CleanupRegistration,
|
||||
};
|
||||
use std::time::Duration;
|
||||
|
||||
const CLEANUP_TIMEOUT: Duration = Duration::from_secs(10);
|
||||
// Reserve time for independent terminal observation even if cancel never responds.
|
||||
const CANCEL_TIMEOUT: Duration = Duration::from_secs(2);
|
||||
|
||||
pub(super) struct Cleanup {
|
||||
pub session: WorkdirSessionHandle,
|
||||
pub state: Arc<Mutex<BashExecutionState>>,
|
||||
pub execution_id: String,
|
||||
pub execution_nonce: u64,
|
||||
pub handle: CommandHandle,
|
||||
pub cursor: u64,
|
||||
pub retained: BashOutputWriter,
|
||||
}
|
||||
|
||||
impl Cleanup {
|
||||
pub async fn run(mut self) {
|
||||
let deadline = tokio::time::Instant::now() + CLEANUP_TIMEOUT;
|
||||
let cancel = match tokio::time::timeout(
|
||||
CANCEL_TIMEOUT,
|
||||
self.session.cancel_command(self.handle.clone()),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(Ok(())) => CleanupCancel::Acknowledged,
|
||||
Ok(Err(error)) => CleanupCancel::Failed {
|
||||
reason: CleanupProviderFailure::from(&error),
|
||||
},
|
||||
Err(_) => CleanupCancel::DeadlineExceeded,
|
||||
};
|
||||
let drain = match tokio::time::timeout_at(deadline, self.drain()).await {
|
||||
Ok(Ok(())) => CleanupDrain::TerminalConfirmed,
|
||||
Ok(Err(reason)) => CleanupDrain::Failed { reason },
|
||||
Err(_) => CleanupDrain::DeadlineExceeded,
|
||||
};
|
||||
let deadline_expired = matches!(drain, CleanupDrain::DeadlineExceeded);
|
||||
// Finalize evidence before exposing an empty registry. Keep the lock through
|
||||
// publication so an older cleanup can never remove a newer generation.
|
||||
let mut state = self.state.lock().unwrap();
|
||||
let registration = if state
|
||||
.active
|
||||
.get(&self.execution_id)
|
||||
.is_some_and(|active| active.execution_nonce == self.execution_nonce)
|
||||
{
|
||||
state.active.remove(&self.execution_id);
|
||||
state.cancellation_requested.remove(&self.execution_id);
|
||||
CleanupRegistration::Released
|
||||
} else {
|
||||
CleanupRegistration::Superseded
|
||||
};
|
||||
let diagnostics = CleanupDiagnostics {
|
||||
execution_nonce: self.execution_nonce,
|
||||
cancel,
|
||||
drain,
|
||||
deadline_expired,
|
||||
registration,
|
||||
};
|
||||
tracing::info!(
|
||||
output_id = %self.retained.id(),
|
||||
cleanup = ?diagnostics,
|
||||
"Bash Drop cleanup finished; cancellation and terminal observation are independent"
|
||||
);
|
||||
self.retained.finish_cleanup(diagnostics);
|
||||
drop(self.retained);
|
||||
}
|
||||
|
||||
async fn drain(&mut self) -> Result<(), CleanupFailure> {
|
||||
loop {
|
||||
let page = command_page(
|
||||
&self.session,
|
||||
CommandOutputRequest {
|
||||
handle: self.handle.clone(),
|
||||
cursor: self.cursor,
|
||||
limit: workdir::MAX_COMMAND_READ_BYTES,
|
||||
wait: true,
|
||||
},
|
||||
)
|
||||
.await
|
||||
.map_err(|error| CleanupFailure::Provider {
|
||||
reason: CleanupProviderFailure::from(&error),
|
||||
})?;
|
||||
let mut next = self.cursor;
|
||||
for chunk in &page.chunks {
|
||||
if chunk.sequence != next {
|
||||
return Err(CleanupFailure::SequenceGap);
|
||||
}
|
||||
next = next
|
||||
.checked_add(chunk.bytes.len() as u64)
|
||||
.filter(|next| *next <= crate::bash_output::COMMAND_BYTES as u64)
|
||||
.ok_or(CleanupFailure::DeliveryBudgetExceeded)?;
|
||||
self.retained
|
||||
.append_chunk(chunk)
|
||||
.map_err(|_| CleanupFailure::RetainedChunkInvalid)?;
|
||||
}
|
||||
if page.next_cursor != next {
|
||||
return Err(CleanupFailure::CursorGap);
|
||||
}
|
||||
self.cursor = next;
|
||||
if let Some(terminal) = page.terminal {
|
||||
self.retained
|
||||
.finish_transport(&terminal)
|
||||
.map_err(|_| CleanupFailure::TerminalMetadataInvalid)?;
|
||||
return Ok(());
|
||||
}
|
||||
// An immediately-ready provider must not starve the cleanup deadline.
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
@@ -0,0 +1,397 @@
|
||||
//! Component tests enter through the production CommandGuard Drop, cleanup, page
|
||||
//! validation and retained store. Only the Workdir provider is replaced; paused time
|
||||
//! and a cancel barrier make faults/deadlines deterministic without OS timing.
|
||||
|
||||
use super::*;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use workdir::*;
|
||||
|
||||
const PRIVATE: &str = "SECRET command /private/host/path output password=poison";
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
enum Fault {
|
||||
Cancel,
|
||||
CancelHang,
|
||||
Drain,
|
||||
TerminalTimeout,
|
||||
Sequence,
|
||||
Cursor,
|
||||
RetainedChunk,
|
||||
TerminalMetadata,
|
||||
CollectionIncomplete,
|
||||
Complete,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct Provider {
|
||||
local: LocalWorkdirSession,
|
||||
fault: Fault,
|
||||
cancelling: tokio::sync::Notify,
|
||||
allow_drain: AtomicBool,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl WorkdirSession for Provider {
|
||||
fn workdir(&self) -> &Workdir {
|
||||
self.local.workdir()
|
||||
}
|
||||
fn capabilities(&self) -> WorkdirSessionCapabilities {
|
||||
WorkdirSessionCapabilities::ALL
|
||||
}
|
||||
async fn stat(&self, _: StatRequest) -> Result<StatResult, WorkdirError> {
|
||||
Err(WorkdirError::OperationFailed)
|
||||
}
|
||||
async fn read(&self, _: ReadRequest) -> Result<ReadResult, WorkdirError> {
|
||||
Err(WorkdirError::OperationFailed)
|
||||
}
|
||||
async fn write(&self, _: WriteRequest) -> Result<WriteResult, WorkdirError> {
|
||||
Err(WorkdirError::OperationFailed)
|
||||
}
|
||||
async fn edit(&self, _: EditRequest) -> Result<EditResult, WorkdirError> {
|
||||
Err(WorkdirError::OperationFailed)
|
||||
}
|
||||
async fn list(&self, _: ListRequest) -> Result<ListResult, WorkdirError> {
|
||||
Err(WorkdirError::OperationFailed)
|
||||
}
|
||||
async fn glob(&self, _: GlobRequest) -> Result<GlobResult, WorkdirError> {
|
||||
Err(WorkdirError::OperationFailed)
|
||||
}
|
||||
async fn grep(&self, _: GrepRequest) -> Result<GrepResult, WorkdirError> {
|
||||
Err(WorkdirError::OperationFailed)
|
||||
}
|
||||
async fn start_command(&self, _: CommandRequest) -> Result<CommandHandle, WorkdirError> {
|
||||
Err(WorkdirError::OperationFailed)
|
||||
}
|
||||
async fn command_status(&self, _: CommandHandle) -> Result<CommandStatus, WorkdirError> {
|
||||
Err(WorkdirError::OperationFailed)
|
||||
}
|
||||
|
||||
async fn command_output(
|
||||
&self,
|
||||
request: CommandOutputRequest,
|
||||
) -> Result<CommandOutput, WorkdirError> {
|
||||
assert_eq!(request.cursor, 3);
|
||||
while !self.allow_drain.load(Ordering::Acquire) {
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
match self.fault {
|
||||
Fault::Drain => {
|
||||
return Err(WorkdirError::Transport(PRIVATE.into())
|
||||
.with_failure_reason(WorkdirFailureReason::HttpTransfer));
|
||||
}
|
||||
Fault::TerminalTimeout => return std::future::pending().await,
|
||||
_ => {}
|
||||
}
|
||||
let mut chunk = CommandByteChunk {
|
||||
sequence: 3,
|
||||
offset: 3,
|
||||
stream: CommandStream::Stdout,
|
||||
bytes: b"tail".to_vec(),
|
||||
};
|
||||
if matches!(self.fault, Fault::Sequence) {
|
||||
chunk.sequence = 4;
|
||||
}
|
||||
if matches!(self.fault, Fault::RetainedChunk) {
|
||||
chunk.offset = 42;
|
||||
}
|
||||
let mut terminal = terminal();
|
||||
if matches!(self.fault, Fault::TerminalMetadata) {
|
||||
terminal.stdout_delivered_bytes = 6;
|
||||
}
|
||||
if matches!(self.fault, Fault::CollectionIncomplete) {
|
||||
terminal.output_collection = CommandOutputCollection::Incomplete {
|
||||
failure: CommandFailure {
|
||||
stage: CommandFailureStage::StdoutDrain,
|
||||
reason: CommandIoErrorKind::Other,
|
||||
os_code: None,
|
||||
},
|
||||
};
|
||||
}
|
||||
Ok(CommandOutput {
|
||||
chunks: vec![chunk],
|
||||
next_cursor: if matches!(self.fault, Fault::Cursor) {
|
||||
8
|
||||
} else {
|
||||
7
|
||||
},
|
||||
terminal: Some(terminal),
|
||||
})
|
||||
}
|
||||
|
||||
async fn cancel_command(&self, _: CommandHandle) -> Result<(), WorkdirError> {
|
||||
self.cancelling.notify_one();
|
||||
match self.fault {
|
||||
Fault::Cancel => Err(WorkdirError::Unavailable(PRIVATE.into())),
|
||||
Fault::CancelHang => std::future::pending().await,
|
||||
_ => Ok(()),
|
||||
}
|
||||
}
|
||||
async fn close(&self) -> Result<(), WorkdirError> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn terminal() -> CommandTerminal {
|
||||
CommandTerminal {
|
||||
status: CommandStatus::Completed,
|
||||
exit_code: Some(0),
|
||||
process_failure: None,
|
||||
output_collection: CommandOutputCollection::Complete,
|
||||
stdout_bytes: 7,
|
||||
stderr_bytes: 0,
|
||||
stdout_delivered_bytes: 7,
|
||||
stderr_delivered_bytes: 0,
|
||||
truncation_reason: None,
|
||||
}
|
||||
}
|
||||
|
||||
struct Fixture {
|
||||
_root: tempfile::TempDir,
|
||||
provider: Arc<Provider>,
|
||||
state: Arc<Mutex<BashExecutionState>>,
|
||||
view: BashOutputView,
|
||||
id: String,
|
||||
}
|
||||
|
||||
impl Fixture {
|
||||
fn start(fault: Fault) -> (Self, CommandGuard) {
|
||||
let base =
|
||||
std::path::Path::new(env!("CARGO_MANIFEST_DIR")).join("../../target/cleanup-fixtures");
|
||||
std::fs::create_dir_all(&base).unwrap();
|
||||
let root = tempfile::tempdir_in(base).unwrap();
|
||||
let provider = Arc::new(Provider {
|
||||
local: LocalWorkdirSession::new(
|
||||
manifest::Scope::writable(root.path()).unwrap(),
|
||||
root.path().into(),
|
||||
),
|
||||
fault,
|
||||
cancelling: Default::default(),
|
||||
allow_drain: AtomicBool::new(false),
|
||||
});
|
||||
let view = BashOutputView::new();
|
||||
let mut retained = view.begin().unwrap();
|
||||
retained.append(CommandStream::Stdout, b"pre");
|
||||
let id = retained.id();
|
||||
let state = Arc::new(Mutex::new(BashExecutionState::default()));
|
||||
let handle = CommandHandle(PRIVATE.into());
|
||||
state.lock().unwrap().active.insert(
|
||||
PRIVATE.into(),
|
||||
ActiveCommand {
|
||||
call_id: PRIVATE.into(),
|
||||
execution_nonce: 1,
|
||||
session: provider.clone(),
|
||||
handle: handle.clone(),
|
||||
},
|
||||
);
|
||||
state
|
||||
.lock()
|
||||
.unwrap()
|
||||
.cancellation_requested
|
||||
.insert(PRIVATE.into());
|
||||
let guard = CommandGuard {
|
||||
session: provider.clone(),
|
||||
state: state.clone(),
|
||||
execution_id: PRIVATE.into(),
|
||||
execution_nonce: 1,
|
||||
handle: Some(handle),
|
||||
cursor: 3,
|
||||
retained: Some(retained),
|
||||
};
|
||||
(
|
||||
Self {
|
||||
_root: root,
|
||||
provider,
|
||||
state,
|
||||
view,
|
||||
id,
|
||||
},
|
||||
guard,
|
||||
)
|
||||
}
|
||||
|
||||
async fn finish(&self) -> BashOutputMetadata {
|
||||
self.provider.allow_drain.store(true, Ordering::Release);
|
||||
// The provider's read retry backoff and cleanup deadlines are virtual time.
|
||||
for _ in 0..4 {
|
||||
tokio::task::yield_now().await;
|
||||
tokio::time::advance(Duration::from_secs(3)).await;
|
||||
}
|
||||
for _ in 0..1000 {
|
||||
if let Some(metadata) = self.view.metadata(&self.id) {
|
||||
return metadata;
|
||||
}
|
||||
tokio::task::yield_now().await;
|
||||
}
|
||||
panic!("finite cleanup did not publish partial evidence");
|
||||
}
|
||||
|
||||
fn assert_evidence(&self, m: &BashOutputMetadata, terminal: bool, bytes: &[u8]) {
|
||||
assert_eq!(
|
||||
self.view
|
||||
.read(&self.id, CommandStream::Stdout, 0, 64)
|
||||
.unwrap(),
|
||||
bytes
|
||||
);
|
||||
assert_eq!(m.stdout_saved, bytes.len());
|
||||
assert_eq!(m.stdout_delivered, bytes.len() as u64);
|
||||
assert_eq!(m.stdout_seen, bytes.len() as u64);
|
||||
assert_eq!(
|
||||
m.terminal.status,
|
||||
terminal.then_some(CommandStatus::Completed)
|
||||
);
|
||||
assert_eq!(m.terminal.exit_code, terminal.then_some(0));
|
||||
if !terminal {
|
||||
assert!(m.terminal.output_collection.is_none());
|
||||
assert!(m.terminal.process_failure.is_none());
|
||||
assert!(!m.terminal.transport_complete);
|
||||
}
|
||||
let diagnostics = m.cleanup.as_ref().unwrap();
|
||||
assert_eq!(diagnostics.execution_nonce, 1);
|
||||
let json = serde_json::to_string(diagnostics).unwrap();
|
||||
let debug = format!("{diagnostics:?}");
|
||||
for forbidden in [
|
||||
PRIVATE, "SECRET", "/private", "password", "poison", "tail", "pre",
|
||||
] {
|
||||
assert!(!json.contains(forbidden), "diagnostics leaked data: {json}");
|
||||
assert!(
|
||||
!debug.contains(forbidden),
|
||||
"tracing payload leaked data: {debug}"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn cancel_failure_does_not_preclude_confirmed_terminal() {
|
||||
let (f, guard) = Fixture::start(Fault::Cancel);
|
||||
drop(guard);
|
||||
f.provider.cancelling.notified().await;
|
||||
assert!(f.state.lock().unwrap().active.contains_key(PRIVATE));
|
||||
let m = f.finish().await;
|
||||
f.assert_evidence(&m, true, b"pretail");
|
||||
assert!(m.terminal.transport_complete);
|
||||
assert!(matches!(
|
||||
m.cleanup.as_ref().unwrap().cancel,
|
||||
CleanupCancel::Failed { .. }
|
||||
));
|
||||
assert!(matches!(
|
||||
m.cleanup.as_ref().unwrap().drain,
|
||||
CleanupDrain::TerminalConfirmed
|
||||
));
|
||||
assert!(!m.cleanup.as_ref().unwrap().deadline_expired);
|
||||
assert!(f.state.lock().unwrap().active.is_empty());
|
||||
assert!(f.state.lock().unwrap().cancellation_requested.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn acknowledged_cancel_drain_faults_keep_distinct_reasons_and_partial_bytes() {
|
||||
for (fault, reason, bytes) in [
|
||||
(Fault::Drain, "provider", b"pre".as_slice()),
|
||||
(Fault::Sequence, "sequence_gap", b"pre"),
|
||||
(Fault::Cursor, "cursor_gap", b"pretail"),
|
||||
(Fault::RetainedChunk, "retained_chunk_invalid", b"pre"),
|
||||
(
|
||||
Fault::TerminalMetadata,
|
||||
"terminal_metadata_invalid",
|
||||
b"pretail",
|
||||
),
|
||||
] {
|
||||
let (f, guard) = Fixture::start(fault);
|
||||
drop(guard);
|
||||
f.provider.cancelling.notified().await;
|
||||
let m = f.finish().await;
|
||||
f.assert_evidence(&m, false, bytes);
|
||||
let diagnostic = serde_json::to_value(m.cleanup.as_ref().unwrap()).unwrap();
|
||||
assert_eq!(diagnostic["cancel"]["outcome"], "acknowledged");
|
||||
assert_eq!(diagnostic["drain"]["outcome"], "failed");
|
||||
assert_eq!(diagnostic["drain"]["reason"]["kind"], reason);
|
||||
assert_eq!(diagnostic["registration"], "released");
|
||||
assert_eq!(diagnostic["deadline_expired"], false);
|
||||
assert!(f.state.lock().unwrap().active.is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn terminal_deadline_never_claims_stop_or_leaves_active_collector() {
|
||||
let (f, guard) = Fixture::start(Fault::TerminalTimeout);
|
||||
drop(guard);
|
||||
f.provider.cancelling.notified().await;
|
||||
let m = f.finish().await;
|
||||
f.assert_evidence(&m, false, b"pre");
|
||||
assert!(matches!(
|
||||
m.cleanup.as_ref().unwrap().cancel,
|
||||
CleanupCancel::Acknowledged
|
||||
));
|
||||
assert!(matches!(
|
||||
m.cleanup.as_ref().unwrap().drain,
|
||||
CleanupDrain::DeadlineExceeded
|
||||
));
|
||||
assert!(m.cleanup.as_ref().unwrap().deadline_expired);
|
||||
assert!(f.state.lock().unwrap().active.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn cancel_deadline_still_attempts_independent_terminal_observation() {
|
||||
let (f, guard) = Fixture::start(Fault::CancelHang);
|
||||
drop(guard);
|
||||
f.provider.cancelling.notified().await;
|
||||
let m = f.finish().await;
|
||||
f.assert_evidence(&m, true, b"pretail");
|
||||
assert!(matches!(
|
||||
m.cleanup.as_ref().unwrap().cancel,
|
||||
CleanupCancel::DeadlineExceeded
|
||||
));
|
||||
assert!(matches!(
|
||||
m.cleanup.as_ref().unwrap().drain,
|
||||
CleanupDrain::TerminalConfirmed
|
||||
));
|
||||
assert!(!m.cleanup.as_ref().unwrap().deadline_expired);
|
||||
assert!(f.state.lock().unwrap().active.is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn old_cleanup_preserves_new_execution_and_cancellation_registration() {
|
||||
let (f, guard) = Fixture::start(Fault::Complete);
|
||||
drop(guard);
|
||||
f.provider.cancelling.notified().await;
|
||||
f.state
|
||||
.lock()
|
||||
.unwrap()
|
||||
.active
|
||||
.get_mut(PRIVATE)
|
||||
.unwrap()
|
||||
.execution_nonce = 2;
|
||||
let m = f.finish().await;
|
||||
f.assert_evidence(&m, true, b"pretail");
|
||||
assert!(matches!(
|
||||
m.cleanup.as_ref().unwrap().registration,
|
||||
CleanupRegistration::Superseded
|
||||
));
|
||||
assert_eq!(f.state.lock().unwrap().active[PRIVATE].execution_nonce, 2);
|
||||
assert!(
|
||||
f.state
|
||||
.lock()
|
||||
.unwrap()
|
||||
.cancellation_requested
|
||||
.contains(PRIVATE)
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test(start_paused = true)]
|
||||
async fn terminal_confirmation_is_independent_of_output_collection_completeness() {
|
||||
let (f, guard) = Fixture::start(Fault::CollectionIncomplete);
|
||||
drop(guard);
|
||||
f.provider.cancelling.notified().await;
|
||||
let m = f.finish().await;
|
||||
f.assert_evidence(&m, true, b"pretail");
|
||||
assert!(!m.terminal.transport_complete);
|
||||
assert!(matches!(
|
||||
m.terminal.output_collection,
|
||||
Some(CommandOutputCollection::Incomplete { .. })
|
||||
));
|
||||
assert!(matches!(
|
||||
m.cleanup.as_ref().unwrap().drain,
|
||||
CleanupDrain::TerminalConfirmed
|
||||
));
|
||||
assert!(f.state.lock().unwrap().active.is_empty());
|
||||
}
|
||||
+130
-10
@@ -19,7 +19,7 @@ use std::time::{Duration, Instant};
|
||||
use serde::Serialize;
|
||||
use workdir::{CommandStatus, CommandStream};
|
||||
|
||||
const COMMAND_BYTES: usize = 4 * 1024 * 1024;
|
||||
pub(crate) const COMMAND_BYTES: usize = 4 * 1024 * 1024;
|
||||
const STORE_BYTES: usize = 32 * 1024 * 1024;
|
||||
const STORE_COUNT: usize = 64;
|
||||
const ORDER_COUNT: usize = 4096;
|
||||
@@ -55,6 +55,7 @@ struct Record {
|
||||
transport_counts: Option<(u64, u64)>,
|
||||
transport_truncation: Option<workdir::CommandOutputTruncation>,
|
||||
terminal: Option<OutputTerminal>,
|
||||
cleanup: Option<CleanupDiagnostics>,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
@@ -68,8 +69,10 @@ pub struct OutputRange {
|
||||
pub struct OutputTerminal {
|
||||
pub status: Option<CommandStatus>,
|
||||
pub exit_code: Option<i32>,
|
||||
/// False on a forcibly dropped collector or failed transport: received bytes are retained,
|
||||
/// but they are not a claim about the unobserved process tail.
|
||||
pub process_failure: Option<workdir::CommandFailure>,
|
||||
pub output_collection: Option<workdir::CommandOutputCollection>,
|
||||
/// True only for confirmed EOF collection (quota is reported separately). A terminal
|
||||
/// fence or exit zero alone does not confirm the unobserved process tail.
|
||||
pub transport_complete: bool,
|
||||
}
|
||||
|
||||
@@ -93,10 +96,72 @@ pub struct BashOutputMetadata {
|
||||
pub transport_truncation: Option<workdir::CommandOutputTruncation>,
|
||||
pub storage_truncated: bool,
|
||||
pub terminal: OutputTerminal,
|
||||
pub cleanup: Option<CleanupDiagnostics>,
|
||||
pub text_encoding: &'static str,
|
||||
pub ordering: &'static str,
|
||||
}
|
||||
|
||||
/// Drop cleanup observations, not assertions that cancellation stopped a process.
|
||||
/// Correlate with the enclosing output UUID and this host-assigned execution generation;
|
||||
/// caller identifiers, provider handles, commands and error text are deliberately excluded.
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct CleanupDiagnostics {
|
||||
pub execution_nonce: u64,
|
||||
pub cancel: CleanupCancel,
|
||||
pub drain: CleanupDrain,
|
||||
pub deadline_expired: bool,
|
||||
pub registration: CleanupRegistration,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
#[serde(tag = "outcome", rename_all = "snake_case")]
|
||||
pub enum CleanupCancel {
|
||||
Acknowledged,
|
||||
Failed { reason: CleanupProviderFailure },
|
||||
DeadlineExceeded,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
#[serde(tag = "outcome", rename_all = "snake_case")]
|
||||
pub enum CleanupDrain {
|
||||
TerminalConfirmed,
|
||||
Failed { reason: CleanupFailure },
|
||||
DeadlineExceeded,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case")]
|
||||
pub enum CleanupFailure {
|
||||
Provider { reason: CleanupProviderFailure },
|
||||
SequenceGap,
|
||||
CursorGap,
|
||||
DeliveryBudgetExceeded,
|
||||
RetainedChunkInvalid,
|
||||
TerminalMetadataInvalid,
|
||||
}
|
||||
|
||||
#[derive(Clone, Debug, Serialize)]
|
||||
pub struct CleanupProviderFailure {
|
||||
pub failure_reason: Option<workdir::WorkdirFailureReason>,
|
||||
pub denial_reason: Option<workdir::WorkdirDenialReason>,
|
||||
}
|
||||
|
||||
impl From<&workdir::WorkdirError> for CleanupProviderFailure {
|
||||
fn from(error: &workdir::WorkdirError) -> Self {
|
||||
Self {
|
||||
failure_reason: error.failure_reason(),
|
||||
denial_reason: error.denial_reason(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, Serialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum CleanupRegistration {
|
||||
Released,
|
||||
Superseded,
|
||||
}
|
||||
|
||||
/// A bounded reservation acquired before command start. Drop terminalizes partial evidence.
|
||||
pub struct BashOutputWriter {
|
||||
record: Arc<Mutex<Record>>,
|
||||
@@ -212,6 +277,7 @@ impl BashOutputView {
|
||||
transport_counts: None,
|
||||
transport_truncation: None,
|
||||
terminal: None,
|
||||
cleanup: None,
|
||||
}));
|
||||
store.records.push_back(record.clone());
|
||||
store.reserved += COMMAND_BYTES;
|
||||
@@ -311,6 +377,7 @@ impl Record {
|
||||
transport_truncation: self.transport_truncation.clone(),
|
||||
storage_truncated: self.truncated || self.transport_truncation.is_some(),
|
||||
terminal: self.terminal.clone().expect("finalized record"),
|
||||
cleanup: self.cleanup.clone(),
|
||||
text_encoding: "utf8_lossy",
|
||||
ordering: "provider_read_observation",
|
||||
}
|
||||
@@ -340,6 +407,14 @@ async fn reap(weak: Weak<Mutex<Store>>, mut shutdown: tokio::sync::watch::Receiv
|
||||
}
|
||||
|
||||
impl BashOutputWriter {
|
||||
pub(crate) fn id(&self) -> String {
|
||||
self.record.lock().unwrap().id.clone()
|
||||
}
|
||||
|
||||
pub(crate) fn finish_cleanup(&mut self, diagnostics: CleanupDiagnostics) {
|
||||
self.record.lock().unwrap().cleanup = Some(diagnostics);
|
||||
}
|
||||
|
||||
pub fn append_chunk(&mut self, chunk: &workdir::CommandByteChunk) -> Result<(), String> {
|
||||
let expected = {
|
||||
let r = self.record.lock().unwrap();
|
||||
@@ -405,12 +480,9 @@ impl BashOutputWriter {
|
||||
terminal: &workdir::CommandTerminal,
|
||||
) -> Result<BashOutputMetadata, String> {
|
||||
let mut r = self.record.lock().unwrap();
|
||||
if terminal.stdout_delivered_bytes != r.stdout_seen
|
||||
if !terminal.metadata_valid()
|
||||
|| terminal.stdout_delivered_bytes != r.stdout_seen
|
||||
|| terminal.stderr_delivered_bytes != r.stderr_seen
|
||||
|| terminal.stdout_bytes < r.stdout_seen
|
||||
|| terminal.stderr_bytes < r.stderr_seen
|
||||
|| ((terminal.stdout_bytes != r.stdout_seen || terminal.stderr_bytes != r.stderr_seen)
|
||||
!= terminal.truncation_reason.is_some())
|
||||
{
|
||||
return Err("command terminal counts do not fence delivered bytes".into());
|
||||
}
|
||||
@@ -419,7 +491,12 @@ impl BashOutputWriter {
|
||||
r.terminal = Some(OutputTerminal {
|
||||
status: Some(terminal.status),
|
||||
exit_code: terminal.exit_code,
|
||||
transport_complete: true,
|
||||
transport_complete: matches!(
|
||||
terminal.output_collection,
|
||||
workdir::CommandOutputCollection::Complete
|
||||
),
|
||||
process_failure: terminal.process_failure,
|
||||
output_collection: Some(terminal.output_collection),
|
||||
});
|
||||
Ok(r.metadata())
|
||||
}
|
||||
@@ -439,6 +516,8 @@ impl Drop for BashOutputWriter {
|
||||
status: None,
|
||||
exit_code: None,
|
||||
transport_complete: false,
|
||||
process_failure: None,
|
||||
output_collection: None,
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -456,6 +535,8 @@ mod tests {
|
||||
status: Some(CommandStatus::Completed),
|
||||
exit_code: Some(0),
|
||||
transport_complete: true,
|
||||
process_failure: None,
|
||||
output_collection: Some(workdir::CommandOutputCollection::Complete),
|
||||
})
|
||||
.id
|
||||
}
|
||||
@@ -514,6 +595,8 @@ mod tests {
|
||||
status: Some(CommandStatus::Completed),
|
||||
exit_code: Some(0),
|
||||
transport_complete: true,
|
||||
process_failure: None,
|
||||
output_collection: Some(workdir::CommandOutputCollection::Complete),
|
||||
});
|
||||
assert!(root.metadata(&metadata.id).is_some());
|
||||
let _replacement = sibling.begin().unwrap();
|
||||
@@ -582,7 +665,7 @@ mod tests {
|
||||
let root = BashOutputView::new();
|
||||
let mut writer = root.begin().unwrap();
|
||||
writer.append(CommandStream::Stdout, b"abc");
|
||||
let mut terminal = serde_json::json!({"status":"completed","exit_code":0,"stdout_bytes":9,"stderr_bytes":0,"stdout_delivered_bytes":4,"stderr_delivered_bytes":0,"truncation_reason":"byte_budget"});
|
||||
let mut terminal = serde_json::json!({"status":"completed","exit_code":0,"process_failure":null,"output_collection":{"state":"complete"},"stdout_bytes":9,"stderr_bytes":0,"stdout_delivered_bytes":4,"stderr_delivered_bytes":0,"truncation_reason":"byte_budget"});
|
||||
assert!(
|
||||
writer
|
||||
.finish_transport(&serde_json::from_value(terminal.clone()).unwrap())
|
||||
@@ -602,4 +685,41 @@ mod tests {
|
||||
Some(workdir::CommandOutputTruncation::ByteBudget)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn incomplete_collection_with_quota_retains_both_causes_without_claiming_complete_output() {
|
||||
let root = BashOutputView::new();
|
||||
let mut writer = root.begin().unwrap();
|
||||
writer.append(CommandStream::Stdout, b"abc");
|
||||
let terminal: workdir::CommandTerminal = serde_json::from_value(serde_json::json!({
|
||||
"status":"completed", "exit_code":0, "process_failure":null,
|
||||
"output_collection":{"state":"incomplete","failure":{"stage":"stdout_drain","reason":"other","os_code":null}},
|
||||
"stdout_bytes":9, "stderr_bytes":0, "stdout_delivered_bytes":3, "stderr_delivered_bytes":0,
|
||||
"truncation_reason":"byte_budget"
|
||||
})).unwrap();
|
||||
let metadata = writer.finish_transport(&terminal).unwrap();
|
||||
assert!(!metadata.terminal.transport_complete);
|
||||
assert_eq!(metadata.terminal.exit_code, Some(0));
|
||||
assert_eq!(
|
||||
metadata.terminal.output_collection,
|
||||
Some(terminal.output_collection)
|
||||
);
|
||||
assert_eq!(
|
||||
metadata.transport_truncation,
|
||||
Some(workdir::CommandOutputTruncation::ByteBudget)
|
||||
);
|
||||
assert_eq!(metadata.stdout_seen, 9);
|
||||
assert_eq!(metadata.stdout_saved, 3);
|
||||
assert!(metadata.storage_truncated);
|
||||
assert!(metadata.retention_limit.is_none());
|
||||
assert_eq!(
|
||||
root.read(&metadata.id, CommandStream::Stdout, 0, 100)
|
||||
.unwrap(),
|
||||
b"abc"
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::to_value(root.metadata(&metadata.id).unwrap()).unwrap(),
|
||||
serde_json::to_value(metadata).unwrap()
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,7 +21,7 @@ pub(crate) struct EditParams {
|
||||
/// Worker-local alias of the Workdir attachment to use.
|
||||
#[serde(default)]
|
||||
pub target_workdir: Option<String>,
|
||||
/// Logical path relative to the bound Workdir root.
|
||||
/// Existing file in which to replace text.
|
||||
pub file_path: String,
|
||||
#[serde(flatten)]
|
||||
pub edit: fs_operation::text::EditArgs,
|
||||
|
||||
@@ -167,6 +167,7 @@ pub(crate) fn checked_error(error: workdir::WorkdirError) -> ToolError {
|
||||
| WorkdirError::IsDirectory { .. }
|
||||
| WorkdirError::InvalidGlob { .. }
|
||||
| WorkdirError::InvalidRegex { .. } => "checkout_invalid",
|
||||
WorkdirError::NotDispatched => "checkout_not_dispatched",
|
||||
WorkdirError::Unavailable { .. } | WorkdirError::SessionClosed => "checkout_unavailable",
|
||||
// IO/transport failures may arrive after effects have begun. Do not infer
|
||||
// no-effects from their text or strip explicit OutcomeUnknown evidence.
|
||||
|
||||
@@ -17,7 +17,7 @@ pub(crate) struct GlobParams {
|
||||
pub target_workdir: Option<String>,
|
||||
/// Glob pattern, for example `**/*.rs` or `src/**/test_*.py`.
|
||||
pub pattern: String,
|
||||
/// Logical Workdir-relative directory. Defaults to the Workdir root.
|
||||
/// Directory to search. Defaults to `.`.
|
||||
#[serde(default)]
|
||||
pub path: Option<String>,
|
||||
}
|
||||
@@ -106,7 +106,7 @@ pub(crate) fn routed_glob_tool(router: Arc<WorkdirSessionRouter>) -> ToolDefinit
|
||||
Arc::new(move || {
|
||||
let schema = schemars::schema_for!(GlobParams);
|
||||
let meta = ToolMeta::new("Glob")
|
||||
.description("Find files matching a glob pattern inside the selected Workdir. Results are sorted and capped at 1000 entries. Paths are Workdir-relative.")
|
||||
.description("Find files matching a glob pattern inside the selected Workdir. Results are sorted and capped at 1000 entries.")
|
||||
.input_schema(serde_json::to_value(schema).expect("Glob schema serialization"));
|
||||
let tool: Arc<dyn Tool> = Arc::new(GlobTool {
|
||||
router: router.clone(),
|
||||
|
||||
@@ -27,7 +27,7 @@ pub(crate) struct GrepParams {
|
||||
#[serde(default)]
|
||||
target_workdir: Option<String>,
|
||||
pattern: String,
|
||||
/// Workdir-relative path, or an absolute path covered by readable scope. Defaults to the Workdir root.
|
||||
/// File or directory to search. Defaults to `.`.
|
||||
#[serde(default)]
|
||||
pub path: Option<String>,
|
||||
#[serde(default)]
|
||||
@@ -160,7 +160,7 @@ pub(crate) fn routed_grep_tool(router: Arc<WorkdirSessionRouter>) -> ToolDefinit
|
||||
Arc::new(move || {
|
||||
let schema = schemars::schema_for!(GrepParams);
|
||||
let meta = ToolMeta::new("Grep")
|
||||
.description("Search a file or directory in the selected Workdir with a regex. Content results group lines by file; `>` marks matching lines and unmarked lines are context. Directory traversal executes inside the WorkdirSession provider. Results are bounded and Workdir-relative.")
|
||||
.description("Search a file or directory in the selected Workdir with a regex. Content results group lines by file; `>` marks matching lines and unmarked lines are context. Directory traversal executes inside the WorkdirSession provider. Results are bounded.")
|
||||
.input_schema(serde_json::to_value(schema).expect("Grep schema serialization"));
|
||||
let tool: Arc<dyn Tool> = Arc::new(GrepTool {
|
||||
router: router.clone(),
|
||||
|
||||
@@ -13,7 +13,7 @@ use workdir::{WorkdirPath, WorkdirSessionHandle, WorkdirSessionRouter};
|
||||
const DESCRIPTION: &str = "Read a text file from a Workdir attachment selected by its Worker-local alias. \
|
||||
Supports offset/limit for large files. Returns line-numbered output (1-based). \
|
||||
Directories cannot be read. The file must be read before Write or Edit can \
|
||||
modify it. Paths are Workdir-relative unless an absolute path is explicitly readable. \
|
||||
modify it. \
|
||||
When exactly one Workdir is attached, target_workdir may be omitted.";
|
||||
|
||||
const DEFAULT_LIMIT: usize = 2000;
|
||||
@@ -24,7 +24,7 @@ pub(crate) struct ReadParams {
|
||||
/// Worker-local alias of the Workdir attachment to use.
|
||||
#[serde(default)]
|
||||
pub target_workdir: Option<String>,
|
||||
/// Workdir-relative path, or an absolute path covered by readable scope.
|
||||
/// Text file to read; directories are not supported.
|
||||
pub file_path: String,
|
||||
#[serde(flatten)]
|
||||
pub read: fs_operation::text::LineReadArgs,
|
||||
|
||||
@@ -20,7 +20,7 @@ pub const MAX_IMAGE_BYTES: usize = 10 * 1024 * 1024;
|
||||
const IMAGE_READ_CHUNK_BYTES: usize = 1024 * 1024;
|
||||
|
||||
const DESCRIPTION: &str = "Attach an image from the selected Workdir attachment to the next model request. \
|
||||
The path must be logical and Workdir-relative. Supported formats: PNG, JPEG, GIF, and WebP. \
|
||||
Supported formats: PNG, JPEG, GIF, and WebP. \
|
||||
When exactly one Workdir is attached, target_workdir may be omitted.";
|
||||
|
||||
#[derive(Debug, Deserialize, schemars::JsonSchema)]
|
||||
@@ -28,7 +28,7 @@ struct ViewImageParams {
|
||||
/// Worker-local alias of the Workdir attachment to use.
|
||||
#[serde(default)]
|
||||
target_workdir: Option<String>,
|
||||
/// Logical path relative to the bound Workdir root.
|
||||
/// Image file to attach to the next model request.
|
||||
path: String,
|
||||
}
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ pub(crate) struct WriteParams {
|
||||
/// Worker-local alias of the Workdir attachment to use.
|
||||
#[serde(default)]
|
||||
pub target_workdir: Option<String>,
|
||||
/// Logical path relative to the bound Workdir root.
|
||||
/// File to create or overwrite.
|
||||
pub file_path: String,
|
||||
#[serde(flatten)]
|
||||
pub write: fs_operation::text::WriteArgs,
|
||||
|
||||
@@ -3,6 +3,11 @@
|
||||
//! `ToolServerHandle::register_tool` / `flush_pending` are `pub(crate)` in
|
||||
//! agen, so from here we exercise the factories directly — the same
|
||||
//! code path that `flush_pending()` runs at production time.
|
||||
//!
|
||||
//! Scoped routing cases connect real LocalWorkdirSession, WorkdirToolBroker::scope,
|
||||
//! attachment router and Tool factories without replacing their permission or path checks.
|
||||
//! They use RAII temporary roots and existing image fixtures; command cases require the
|
||||
//! same local shell as the Bash suite and run in the normal `tools --test integration` target.
|
||||
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
@@ -80,6 +85,101 @@ fn setup_routed() -> (TempDir, TempDir, Arc<WorkdirSessionRouter>, Registry) {
|
||||
(left, right, router, Registry::new(definitions))
|
||||
}
|
||||
|
||||
const READ_SAME_FILE: &str = "IFS= read -r line < same.txt && printf '%s\\n' \"$line\"";
|
||||
|
||||
struct ScopedRegistry {
|
||||
roots: Vec<TempDir>,
|
||||
directories: Vec<std::path::PathBuf>,
|
||||
brokers: Vec<WorkdirToolBroker>,
|
||||
leases: Vec<workdir::WorkdirScopeLease>,
|
||||
router: Arc<WorkdirSessionRouter>,
|
||||
reg: Registry,
|
||||
}
|
||||
|
||||
impl ScopedRegistry {
|
||||
// Cwd and scope targets here are delegation declarations in provider coordinates.
|
||||
async fn new(attachments: &[(&str, &str, bool)]) -> Self {
|
||||
let router = Arc::new(WorkdirSessionRouter::new());
|
||||
let mut roots = Vec::new();
|
||||
let mut directories = Vec::new();
|
||||
let mut brokers = Vec::new();
|
||||
let mut leases = Vec::new();
|
||||
for &(alias, cwd, command) in attachments {
|
||||
let root = TempDir::new().unwrap();
|
||||
let directory = root.path().join(cwd);
|
||||
std::fs::create_dir_all(directory.join("subdir")).unwrap();
|
||||
std::fs::write(directory.join("same.txt"), format!("{alias} original\n")).unwrap();
|
||||
std::fs::write(
|
||||
directory.join("subdir/same.txt"),
|
||||
format!("{alias} nested needle\n"),
|
||||
)
|
||||
.unwrap();
|
||||
std::fs::write(
|
||||
directory.join("image.png"),
|
||||
include_bytes!("fixtures/view-image/valid.png"),
|
||||
)
|
||||
.unwrap();
|
||||
if cwd != "." {
|
||||
// Decoys make accidental root-based resolution observable.
|
||||
std::fs::write(root.path().join("same.txt"), "root decoy\n").unwrap();
|
||||
std::fs::write(root.path().join("image.png"), "not an image").unwrap();
|
||||
}
|
||||
let source: WorkdirSessionHandle = Arc::new(LocalWorkdirSession::new(
|
||||
Scope::writable(root.path()).unwrap(),
|
||||
root.path().to_path_buf(),
|
||||
));
|
||||
let broker = WorkdirToolBroker::new(source);
|
||||
let lease = broker
|
||||
.scope(WorkdirToolScope {
|
||||
rules: vec![WorkdirToolScopeRule {
|
||||
target: WorkdirPath::new(cwd).unwrap(),
|
||||
permission: WorkdirToolScopePermission::Write,
|
||||
recursive: true,
|
||||
symlink_policy: Default::default(),
|
||||
}],
|
||||
cwd: WorkdirPath::new(cwd).unwrap(),
|
||||
command,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
router
|
||||
.attach(
|
||||
WorkdirAttachmentAlias::new(alias).unwrap(),
|
||||
lease.tool_session(),
|
||||
)
|
||||
.unwrap();
|
||||
roots.push(root);
|
||||
directories.push(directory);
|
||||
brokers.push(broker);
|
||||
leases.push(lease);
|
||||
}
|
||||
let mut definitions = routed_builtin_tools(
|
||||
router.clone(),
|
||||
Tracker::new(),
|
||||
tools::bash_output::BashOutputView::new(),
|
||||
);
|
||||
definitions.push(routed_view_image_tool(router.clone()));
|
||||
Self {
|
||||
roots,
|
||||
directories,
|
||||
brokers,
|
||||
leases,
|
||||
router,
|
||||
reg: Registry::new(definitions),
|
||||
}
|
||||
}
|
||||
|
||||
async fn close(self) {
|
||||
self.router.close_all().await.unwrap();
|
||||
for lease in &self.leases {
|
||||
lease.close().await.unwrap();
|
||||
}
|
||||
for broker in &self.brokers {
|
||||
broker.close().await.unwrap();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn valid_gif_with_len(target_len: usize) -> Vec<u8> {
|
||||
let fixture = include_bytes!("fixtures/view-image/valid.gif");
|
||||
assert_eq!(
|
||||
@@ -156,6 +256,71 @@ fn routed_tool_schemas_share_optional_target_workdir() {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn routed_tool_help_does_not_override_session_path_coordinates() {
|
||||
fn assert_no_root_coordinates(tool: &str, description: &str) {
|
||||
let normalized = description
|
||||
.split_whitespace()
|
||||
.collect::<Vec<_>>()
|
||||
.join(" ")
|
||||
.to_ascii_lowercase();
|
||||
for obsolete in ["workdir-relative", "workdir root", "root-relative"] {
|
||||
assert!(
|
||||
!normalized.contains(obsolete),
|
||||
"{tool} help overrides session path coordinates: {description}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn check_schema_descriptions(tool: &str, schema: &serde_json::Value) {
|
||||
match schema {
|
||||
serde_json::Value::Object(object) => {
|
||||
if let Some(description) = object.get("description").and_then(|v| v.as_str()) {
|
||||
assert_no_root_coordinates(tool, description);
|
||||
}
|
||||
for value in object.values() {
|
||||
check_schema_descriptions(tool, value);
|
||||
}
|
||||
}
|
||||
serde_json::Value::Array(array) => {
|
||||
for value in array {
|
||||
check_schema_descriptions(tool, value);
|
||||
}
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
|
||||
let router = Arc::new(WorkdirSessionRouter::new());
|
||||
let mut definitions = routed_builtin_tools(
|
||||
router.clone(),
|
||||
Tracker::new(),
|
||||
tools::bash_output::BashOutputView::new(),
|
||||
);
|
||||
definitions.push(routed_view_image_tool(router));
|
||||
for definition in definitions {
|
||||
let (meta, _) = definition();
|
||||
assert_no_root_coordinates(&meta.name, &meta.description);
|
||||
// Referenced type definitions are model-visible too, not just properties.
|
||||
check_schema_descriptions(&meta.name, &meta.input_schema);
|
||||
|
||||
let path_parameter = match meta.name.as_str() {
|
||||
"Read" | "Write" | "Edit" => "file_path",
|
||||
"Glob" | "Grep" | "ViewImage" => "path",
|
||||
"Bash" => "cwd",
|
||||
other => panic!("missing path parameter contract for {other}"),
|
||||
};
|
||||
let help = meta.input_schema["properties"][path_parameter]["description"]
|
||||
.as_str()
|
||||
.unwrap_or_default();
|
||||
assert!(
|
||||
!help.trim().is_empty(),
|
||||
"{}.{path_parameter} must retain argument-specific help",
|
||||
meta.name
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn routed_tools_require_alias_only_when_attachment_set_is_ambiguous() {
|
||||
let (_left, _right, router, reg) = setup_routed();
|
||||
@@ -284,6 +449,427 @@ async fn target_alias_routes_each_tool_and_isolates_read_before_edit() {
|
||||
router.close_all().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_cwd_file_search_and_image_tools_share_reusable_paths() {
|
||||
for cwd in [".", "work"] {
|
||||
let fixture = ScopedRegistry::new(&[("checkout", cwd, false)]).await;
|
||||
let read = fixture.reg.get("Read");
|
||||
let original = call(&read, json!({ "file_path": "same.txt" })).await;
|
||||
assert!(original.content.unwrap().contains("checkout original"));
|
||||
call(
|
||||
&fixture.reg.get("Write"),
|
||||
json!({ "file_path": "same.txt", "content": "written marker\n" }),
|
||||
)
|
||||
.await;
|
||||
call(
|
||||
&fixture.reg.get("Edit"),
|
||||
json!({ "file_path": "same.txt", "old_string": "written", "new_string": "edited" }),
|
||||
)
|
||||
.await;
|
||||
let edited = call(&read, json!({ "file_path": "same.txt" })).await;
|
||||
assert!(edited.content.unwrap().contains("edited marker"));
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(fixture.directories[0].join("same.txt")).unwrap(),
|
||||
"edited marker\n"
|
||||
);
|
||||
|
||||
// Omitted search paths start at cwd, not at the underlying provider root.
|
||||
for (name, input) in [
|
||||
("Glob", json!({ "pattern": "same.txt" })),
|
||||
(
|
||||
"Grep",
|
||||
json!({ "pattern": "edited marker", "output_mode": "files_with_matches" }),
|
||||
),
|
||||
] {
|
||||
let result = call(&fixture.reg.get(name), input).await;
|
||||
assert_eq!(
|
||||
result.content.unwrap().trim(),
|
||||
"same.txt",
|
||||
"{name}, cwd={cwd}"
|
||||
);
|
||||
}
|
||||
|
||||
// A search rooted below cwd still returns cwd-relative reusable paths.
|
||||
for (name, input) in [
|
||||
("Glob", json!({ "pattern": "*.txt", "path": "subdir" })),
|
||||
(
|
||||
"Grep",
|
||||
json!({ "pattern": "needle", "path": "subdir", "output_mode": "files_with_matches" }),
|
||||
),
|
||||
] {
|
||||
let result = call(&fixture.reg.get(name), input).await;
|
||||
let body = result.content.unwrap();
|
||||
let paths: Vec<_> = body.lines().collect();
|
||||
assert_eq!(paths, ["subdir/same.txt"], "{name}, cwd={cwd}");
|
||||
let path = paths[0];
|
||||
let found = call(&read, json!({ "file_path": path })).await;
|
||||
assert!(found.content.unwrap().contains("checkout nested needle"));
|
||||
call(
|
||||
&fixture.reg.get("Edit"),
|
||||
json!({ "file_path": path, "old_string": "needle", "new_string": format!("needle-{name}") }),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(fixture.directories[0].join("subdir/same.txt")).unwrap(),
|
||||
"checkout nested needle-Grep-Glob\n"
|
||||
);
|
||||
let image = call(
|
||||
&fixture.reg.get("ViewImage"),
|
||||
json!({ "path": "image.png" }),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(image.attachments.len(), 1);
|
||||
let agen::tool::Attachment::Image(image) = &image.attachments[0];
|
||||
assert_eq!(image.mime_type(), "image/png");
|
||||
assert_eq!(
|
||||
image.data(),
|
||||
include_bytes!("fixtures/view-image/valid.png")
|
||||
);
|
||||
if cwd != "." {
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(fixture.roots[0].path().join("same.txt")).unwrap(),
|
||||
"root decoy\n"
|
||||
);
|
||||
}
|
||||
fixture.close().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_bash_relative_cwd_and_cd_do_not_change_later_tool_cwd() {
|
||||
for cwd in [".", "work"] {
|
||||
let fixture = ScopedRegistry::new(&[("checkout", cwd, true)]).await;
|
||||
for (command_cwd, command, expected) in [
|
||||
(".", READ_SAME_FILE, "checkout original\n"),
|
||||
("subdir", READ_SAME_FILE, "checkout nested needle\n"),
|
||||
("./subdir//.", READ_SAME_FILE, "checkout nested needle\n"),
|
||||
(
|
||||
".",
|
||||
"cd subdir && IFS= read -r line < same.txt && printf '%s\\n' \"$line\"",
|
||||
"checkout nested needle\n",
|
||||
),
|
||||
(".", READ_SAME_FILE, "checkout original\n"),
|
||||
] {
|
||||
let output = call(
|
||||
&fixture.reg.get("Bash"),
|
||||
json!({ "cwd": command_cwd, "command": command, "timeout": 5 }),
|
||||
)
|
||||
.await;
|
||||
let body: serde_json::Value = serde_json::from_str(&output.content.unwrap()).unwrap();
|
||||
assert_eq!(body["output"]["terminal"]["exit_code"], 0, "{body}");
|
||||
assert_eq!(body["stdout_preview"], expected, "cwd={cwd}, {command}");
|
||||
let subsequent =
|
||||
call(&fixture.reg.get("Read"), json!({ "file_path": "same.txt" })).await;
|
||||
assert!(subsequent.content.unwrap().contains("checkout original"));
|
||||
}
|
||||
fixture.close().await;
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_aliases_with_different_cwds_keep_paths_and_observations_separate() {
|
||||
let fixture =
|
||||
ScopedRegistry::new(&[("left", "work", true), ("right", "other/deep", true)]).await;
|
||||
let read = fixture.reg.get("Read");
|
||||
let edit = fixture.reg.get("Edit");
|
||||
call(
|
||||
&read,
|
||||
json!({ "target_workdir": "left", "file_path": "same.txt" }),
|
||||
)
|
||||
.await;
|
||||
let unread = call_err(
|
||||
&edit,
|
||||
json!({ "target_workdir": "right", "file_path": "same.txt", "old_string": "original", "new_string": "wrong" }),
|
||||
)
|
||||
.await;
|
||||
assert!(unread.to_string().contains("has not been read"), "{unread}");
|
||||
call(
|
||||
&read,
|
||||
json!({ "target_workdir": "right", "file_path": "same.txt" }),
|
||||
)
|
||||
.await;
|
||||
call(
|
||||
&fixture.reg.get("Write"),
|
||||
json!({ "target_workdir": "right", "file_path": "same.txt", "content": "right written\n" }),
|
||||
)
|
||||
.await;
|
||||
call(
|
||||
&edit,
|
||||
json!({ "target_workdir": "right", "file_path": "same.txt", "old_string": "written", "new_string": "edited" }),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(fixture.directories[0].join("same.txt")).unwrap(),
|
||||
"left original\n"
|
||||
);
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(fixture.directories[1].join("same.txt")).unwrap(),
|
||||
"right edited\n"
|
||||
);
|
||||
|
||||
// Different valid bytes make cross-alias image reads observable too.
|
||||
let png: &[u8] = include_bytes!("fixtures/view-image/valid.png");
|
||||
let gif: &[u8] = include_bytes!("fixtures/view-image/valid.gif");
|
||||
std::fs::write(fixture.directories[1].join("image.png"), gif).unwrap();
|
||||
for (alias, expected, image_bytes, mime) in [
|
||||
("left", "left original", png, "image/png"),
|
||||
("right", "right edited", gif, "image/gif"),
|
||||
] {
|
||||
let output = call(
|
||||
&read,
|
||||
json!({ "target_workdir": alias, "file_path": "same.txt" }),
|
||||
)
|
||||
.await;
|
||||
assert!(output.content.unwrap().contains(expected));
|
||||
for (name, input) in [
|
||||
(
|
||||
"Glob",
|
||||
json!({ "target_workdir": alias, "pattern": "*.txt", "path": "subdir" }),
|
||||
),
|
||||
(
|
||||
"Grep",
|
||||
json!({ "target_workdir": alias, "pattern": alias, "path": "subdir", "output_mode": "files_with_matches" }),
|
||||
),
|
||||
] {
|
||||
let output = call(&fixture.reg.get(name), input).await;
|
||||
let path = output.content.unwrap();
|
||||
let path = path.trim();
|
||||
assert_eq!(path, "subdir/same.txt", "{alias}: {name}");
|
||||
let found = call(&read, json!({ "target_workdir": alias, "file_path": path })).await;
|
||||
assert!(
|
||||
found
|
||||
.content
|
||||
.unwrap()
|
||||
.contains(&format!("{alias} nested needle"))
|
||||
);
|
||||
}
|
||||
let image = call(
|
||||
&fixture.reg.get("ViewImage"),
|
||||
json!({ "target_workdir": alias, "path": "image.png" }),
|
||||
)
|
||||
.await;
|
||||
let agen::tool::Attachment::Image(image) = &image.attachments[0];
|
||||
assert_eq!(image.mime_type(), mime);
|
||||
assert_eq!(image.data(), image_bytes);
|
||||
let output = call(
|
||||
&fixture.reg.get("Bash"),
|
||||
json!({ "target_workdir": alias, "cwd": ".", "command": READ_SAME_FILE, "timeout": 5 }),
|
||||
)
|
||||
.await;
|
||||
let body: serde_json::Value = serde_json::from_str(&output.content.unwrap()).unwrap();
|
||||
assert_eq!(body["output"]["terminal"]["exit_code"], 0, "{body}");
|
||||
assert_eq!(body["stdout_preview"], format!("{expected}\n"));
|
||||
}
|
||||
fixture.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_command_false_rejects_bash_without_fallback_to_another_alias() {
|
||||
let fixture =
|
||||
ScopedRegistry::new(&[("enabled", "work", true), ("disabled", "other", false)]).await;
|
||||
for cwd in [".", "subdir"] {
|
||||
let error = call_err(
|
||||
&fixture.reg.get("Bash"),
|
||||
json!({ "target_workdir": "disabled", "cwd": cwd, "command": "printf forbidden > blocked.txt", "timeout": 5 }),
|
||||
)
|
||||
.await;
|
||||
assert!(
|
||||
error.to_string().contains("does not support Command"),
|
||||
"{error}"
|
||||
);
|
||||
}
|
||||
// Write permission remains usable, but does not imply command permission.
|
||||
call(
|
||||
&fixture.reg.get("Write"),
|
||||
json!({ "target_workdir": "disabled", "file_path": "created.txt", "content": "allowed write\n" }),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(fixture.directories[1].join("created.txt")).unwrap(),
|
||||
"allowed write\n"
|
||||
);
|
||||
for (root, directory) in fixture.roots.iter().zip(&fixture.directories) {
|
||||
for path in [
|
||||
root.path().to_path_buf(),
|
||||
directory.clone(),
|
||||
directory.join("subdir"),
|
||||
] {
|
||||
assert!(!path.join("blocked.txt").exists());
|
||||
}
|
||||
}
|
||||
fixture.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_normalized_paths_share_read_before_mutation_history() {
|
||||
let fixture = ScopedRegistry::new(&[("checkout", "work", false)]).await;
|
||||
let read = call(
|
||||
&fixture.reg.get("Read"),
|
||||
json!({ "file_path": "./subdir//./same.txt" }),
|
||||
)
|
||||
.await;
|
||||
assert!(read.content.unwrap().contains("checkout nested needle"));
|
||||
call(
|
||||
&fixture.reg.get("Write"),
|
||||
json!({ "file_path": "subdir/same.txt", "content": "normalized write\n" }),
|
||||
)
|
||||
.await;
|
||||
call(
|
||||
&fixture.reg.get("Edit"),
|
||||
json!({ "file_path": "./subdir/same.txt", "old_string": "write", "new_string": "edit" }),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(fixture.directories[0].join("subdir/same.txt")).unwrap(),
|
||||
"normalized edit\n"
|
||||
);
|
||||
for (name, input) in [
|
||||
("Glob", json!({ "pattern": "*.txt", "path": "./subdir//." })),
|
||||
(
|
||||
"Grep",
|
||||
json!({ "pattern": "normalized edit", "path": "./subdir//.", "output_mode": "files_with_matches" }),
|
||||
),
|
||||
] {
|
||||
let result = call(&fixture.reg.get(name), input).await;
|
||||
assert_eq!(result.content.unwrap().trim(), "subdir/same.txt", "{name}");
|
||||
}
|
||||
let image = call(
|
||||
&fixture.reg.get("ViewImage"),
|
||||
json!({ "path": "././image.png" }),
|
||||
)
|
||||
.await;
|
||||
let agen::tool::Attachment::Image(image) = &image.attachments[0];
|
||||
assert_eq!(
|
||||
image.data(),
|
||||
include_bytes!("fixtures/view-image/valid.png")
|
||||
);
|
||||
fixture.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn scoped_tools_reject_parent_traversal_and_paths_outside_the_grant() {
|
||||
let fixture = ScopedRegistry::new(&[("checkout", "work", true)]).await;
|
||||
let outside = fixture.roots[0].path().join("outside");
|
||||
std::fs::create_dir(&outside).unwrap();
|
||||
std::fs::write(outside.join("secret.txt"), "private needle\n").unwrap();
|
||||
for invalid in [
|
||||
"../outside/secret.txt",
|
||||
"subdir/../../outside/secret.txt",
|
||||
"subdir\\same.txt",
|
||||
] {
|
||||
for (name, input) in [
|
||||
("Read", json!({ "file_path": invalid })),
|
||||
(
|
||||
"Write",
|
||||
json!({ "file_path": invalid, "content": "forbidden" }),
|
||||
),
|
||||
(
|
||||
"Edit",
|
||||
json!({ "file_path": invalid, "old_string": "needle", "new_string": "forbidden" }),
|
||||
),
|
||||
("Glob", json!({ "path": invalid, "pattern": "*" })),
|
||||
("Grep", json!({ "path": invalid, "pattern": "needle" })),
|
||||
("ViewImage", json!({ "path": invalid })),
|
||||
(
|
||||
"Bash",
|
||||
json!({ "cwd": invalid, "command": "printf forbidden > blocked.txt", "timeout": 5 }),
|
||||
),
|
||||
] {
|
||||
let error = call_err(&fixture.reg.get(name), input).await;
|
||||
assert!(
|
||||
error.to_string().contains("path"),
|
||||
"{name}: {invalid}: {error}"
|
||||
);
|
||||
}
|
||||
}
|
||||
// These paths are inside the provider's grant, but outside the child's grant.
|
||||
// Absolute readable paths must not bypass the delegated scope after cwd rebasing.
|
||||
for (name, input) in [
|
||||
("Read", json!({ "file_path": outside.join("secret.txt") })),
|
||||
("Grep", json!({ "path": outside, "pattern": "needle" })),
|
||||
(
|
||||
"Bash",
|
||||
json!({ "cwd": outside, "command": "printf forbidden > blocked.txt", "timeout": 5 }),
|
||||
),
|
||||
] {
|
||||
let error = call_err(&fixture.reg.get(name), input).await;
|
||||
let message = error.to_string();
|
||||
assert!(
|
||||
matches!(error, ToolError::InvalidArgument(_)),
|
||||
"{name}: {message}"
|
||||
);
|
||||
assert!(
|
||||
message.contains("scope") || message.contains("path"),
|
||||
"{name}: {message}"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(outside.join("secret.txt")).unwrap(),
|
||||
"private needle\n"
|
||||
);
|
||||
assert!(!outside.join("blocked.txt").exists());
|
||||
assert!(!fixture.directories[0].join("blocked.txt").exists());
|
||||
assert!(!fixture.roots[0].path().join("blocked.txt").exists());
|
||||
fixture.close().await;
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[tokio::test]
|
||||
async fn scoped_cwd_tools_reject_resolved_symlink_escape_after_a_read() {
|
||||
use std::os::unix::fs::symlink;
|
||||
|
||||
let fixture = ScopedRegistry::new(&[("checkout", "work", true)]).await;
|
||||
let outside = fixture.roots[0].path().join("outside");
|
||||
std::fs::create_dir(&outside).unwrap();
|
||||
// Equal bytes avoid disguising an authority failure as a stale-hash rejection.
|
||||
std::fs::write(outside.join("secret.txt"), "checkout original\n").unwrap();
|
||||
let png = include_bytes!("fixtures/view-image/valid.png");
|
||||
std::fs::write(outside.join("image.png"), png).unwrap();
|
||||
call(&fixture.reg.get("Read"), json!({ "file_path": "same.txt" })).await;
|
||||
std::fs::remove_file(fixture.directories[0].join("same.txt")).unwrap();
|
||||
symlink(
|
||||
"../outside/secret.txt",
|
||||
fixture.directories[0].join("same.txt"),
|
||||
)
|
||||
.unwrap();
|
||||
symlink("../outside", fixture.directories[0].join("escape")).unwrap();
|
||||
|
||||
for (name, input) in [
|
||||
("Read", json!({ "file_path": "same.txt" })),
|
||||
(
|
||||
"Write",
|
||||
json!({ "file_path": "escape/created.txt", "content": "forbidden" }),
|
||||
),
|
||||
(
|
||||
"Edit",
|
||||
json!({ "file_path": "same.txt", "old_string": "original", "new_string": "forbidden" }),
|
||||
),
|
||||
("Glob", json!({ "path": "escape", "pattern": "*" })),
|
||||
("Grep", json!({ "path": "escape", "pattern": "original" })),
|
||||
("ViewImage", json!({ "path": "escape/image.png" })),
|
||||
(
|
||||
"Bash",
|
||||
json!({ "cwd": "escape", "command": "printf forbidden > blocked.txt", "timeout": 5 }),
|
||||
),
|
||||
] {
|
||||
let error = call_err(&fixture.reg.get(name), input).await;
|
||||
let message = error.to_string();
|
||||
assert!(
|
||||
message.contains("scope") || message.contains("symlink"),
|
||||
"{name}: {message}"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(outside.join("secret.txt")).unwrap(),
|
||||
"checkout original\n"
|
||||
);
|
||||
assert_eq!(std::fs::read(outside.join("image.png")).unwrap(), png);
|
||||
assert!(!outside.join("created.txt").exists());
|
||||
assert!(!outside.join("blocked.txt").exists());
|
||||
fixture.close().await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn selected_attachment_capability_is_checked_without_fallback() {
|
||||
let left = TempDir::new().unwrap();
|
||||
@@ -416,6 +1002,7 @@ struct ExternalValidatingSession {
|
||||
truncate_first_binary_response: bool,
|
||||
command_response_losses: std::sync::atomic::AtomicUsize,
|
||||
lose_terminal_only: bool,
|
||||
collection_override: Option<workdir::CommandOutputCollection>,
|
||||
mutate_after_first_binary_response: Option<std::path::PathBuf>,
|
||||
}
|
||||
|
||||
@@ -534,7 +1121,11 @@ impl workdir::WorkdirSession for ExternalValidatingSession {
|
||||
&self,
|
||||
request: workdir::CommandOutputRequest,
|
||||
) -> Result<workdir::CommandOutput, workdir::WorkdirError> {
|
||||
let output = self.inner.command_output(request).await?;
|
||||
let mut output = self.inner.command_output(request).await?;
|
||||
if let (Some(collection), Some(terminal)) = (self.collection_override, &mut output.terminal)
|
||||
{
|
||||
terminal.output_collection = collection;
|
||||
}
|
||||
if (!self.lose_terminal_only || output.terminal.is_some())
|
||||
&& self
|
||||
.command_response_losses
|
||||
@@ -575,6 +1166,7 @@ async fn external_view_image_uses_a_protocol_bounded_request() {
|
||||
truncate_first_binary_response: false,
|
||||
command_response_losses: std::sync::atomic::AtomicUsize::new(0),
|
||||
lose_terminal_only: false,
|
||||
collection_override: None,
|
||||
mutate_after_first_binary_response: None,
|
||||
});
|
||||
let png = include_bytes!("fixtures/view-image/valid.png");
|
||||
@@ -600,6 +1192,7 @@ async fn external_view_image_rejects_truncated_and_cross_version_chunks() {
|
||||
truncate_first_binary_response: true,
|
||||
command_response_losses: std::sync::atomic::AtomicUsize::new(0),
|
||||
lose_terminal_only: false,
|
||||
collection_override: None,
|
||||
mutate_after_first_binary_response: None,
|
||||
});
|
||||
let (_meta, tool) = view_image_tool(truncated)();
|
||||
@@ -618,6 +1211,7 @@ async fn external_view_image_rejects_truncated_and_cross_version_chunks() {
|
||||
truncate_first_binary_response: false,
|
||||
command_response_losses: std::sync::atomic::AtomicUsize::new(0),
|
||||
lose_terminal_only: false,
|
||||
collection_override: None,
|
||||
mutate_after_first_binary_response: Some(image_path),
|
||||
});
|
||||
let (_meta, tool) = view_image_tool(changing)();
|
||||
@@ -1104,7 +1698,20 @@ async fn bash_cancellation_returns_bounded_progress_as_terminal_output() {
|
||||
let execution = tokio::spawn(async move {
|
||||
executing.execute(r#"{"command":"printf 'before'; printf 'err-before' >&2; while :; do :; done","cwd":"."}"#, execution_context).await
|
||||
});
|
||||
await_command_output(&mut events).await;
|
||||
// Cancellation must follow both writes, not merely the first stdout chunk.
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), async {
|
||||
let mut ready = [false; 2];
|
||||
while !ready.iter().all(|seen| *seen) {
|
||||
if let workdir::CommandEvent::Output { stream, .. } = events.recv().await.unwrap() {
|
||||
ready[match stream {
|
||||
workdir::CommandStream::Stdout => 0,
|
||||
workdir::CommandStream::Stderr => 1,
|
||||
}] = true;
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("both streams written before cancellation");
|
||||
bash.cancel_execution(&context).await.unwrap();
|
||||
let ToolError::Cancelled(output) =
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), execution)
|
||||
@@ -1281,6 +1888,7 @@ async fn lost_page_and_terminal_responses_replay_without_restarting_or_double_ap
|
||||
mutate_after_first_binary_response: None,
|
||||
command_response_losses: std::sync::atomic::AtomicUsize::new(1),
|
||||
lose_terminal_only,
|
||||
collection_override: None,
|
||||
});
|
||||
let view = tools::bash_output::BashOutputView::new();
|
||||
let definitions = core_builtin_tools(session.clone(), Tracker::new(), view.clone());
|
||||
@@ -1315,3 +1923,80 @@ async fn lost_page_and_terminal_responses_replay_without_restarting_or_double_ap
|
||||
session.close().await.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
// Provider metadata is the fixture here; actual read/drain injection and HTTP
|
||||
// forwarding are tested by workdir's production-supervisor suite.
|
||||
#[tokio::test]
|
||||
async fn exit_zero_with_incomplete_output_keeps_partial_bytes_and_safe_failure_in_retained_evidence()
|
||||
{
|
||||
use workdir::{
|
||||
CommandFailure, CommandFailureStage as Stage, CommandIoErrorKind, CommandOutputCollection,
|
||||
};
|
||||
for stage in [
|
||||
Stage::StdoutRead,
|
||||
Stage::StderrRead,
|
||||
Stage::StdoutDrain,
|
||||
Stage::StderrDrain,
|
||||
Stage::OutputDelivery,
|
||||
] {
|
||||
let root = TempDir::new().unwrap();
|
||||
let failure = CommandFailure {
|
||||
stage,
|
||||
reason: CommandIoErrorKind::PermissionDenied,
|
||||
os_code: None,
|
||||
};
|
||||
let collection = CommandOutputCollection::Incomplete { failure };
|
||||
let session: WorkdirSessionHandle = Arc::new(ExternalValidatingSession {
|
||||
inner: LocalWorkdirSession::new(
|
||||
Scope::writable(root.path()).unwrap(),
|
||||
root.path().into(),
|
||||
),
|
||||
truncate_first_binary_response: false,
|
||||
mutate_after_first_binary_response: None,
|
||||
command_response_losses: std::sync::atomic::AtomicUsize::new(0),
|
||||
lose_terminal_only: false,
|
||||
collection_override: Some(collection),
|
||||
});
|
||||
let view = tools::bash_output::BashOutputView::new();
|
||||
let (_, bash) = tools::bash_tool(session.clone(), view.clone())();
|
||||
let out = call(
|
||||
&bash,
|
||||
json!({"cwd":".","command":"printf partial; printf err >&2"}),
|
||||
)
|
||||
.await;
|
||||
assert!(out.summary.ends_with("(output incomplete)"));
|
||||
let body: serde_json::Value = serde_json::from_str(&out.content.unwrap()).unwrap();
|
||||
let metadata = &body["output"];
|
||||
assert_eq!(body["stdout_preview"], "partial");
|
||||
assert_eq!(body["stderr_preview"], "err");
|
||||
assert_eq!(body["preview_omitted"], false);
|
||||
assert_eq!(metadata["terminal"]["exit_code"], 0);
|
||||
assert_eq!(metadata["terminal"]["status"], "completed");
|
||||
assert_eq!(metadata["terminal"]["transport_complete"], false);
|
||||
assert_eq!(
|
||||
metadata["terminal"]["output_collection"],
|
||||
serde_json::to_value(collection).unwrap()
|
||||
);
|
||||
assert_eq!(metadata["storage_truncated"], false);
|
||||
assert_eq!(metadata["transport_truncation"], serde_json::Value::Null);
|
||||
assert_eq!(metadata["stdout_seen"], 7);
|
||||
assert_eq!(metadata["stdout_delivered"], 7);
|
||||
let id = metadata["id"].as_str().unwrap();
|
||||
assert_eq!(
|
||||
serde_json::to_value(view.metadata(id).unwrap()).unwrap(),
|
||||
*metadata
|
||||
);
|
||||
assert_eq!(
|
||||
view.read(id, workdir::CommandStream::Stdout, 0, 100)
|
||||
.unwrap(),
|
||||
b"partial"
|
||||
);
|
||||
assert_eq!(
|
||||
view.read(id, workdir::CommandStream::Stderr, 0, 100)
|
||||
.unwrap(),
|
||||
b"err"
|
||||
);
|
||||
assert!(!metadata.to_string().contains(root.path().to_str().unwrap()));
|
||||
session.close().await.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,5 +23,6 @@ thiserror.workspace = true
|
||||
tokio = { workspace = true, features = ["io-util", "process", "rt", "sync", "time"] }
|
||||
|
||||
[dev-dependencies]
|
||||
axum.workspace = true
|
||||
serde_json.workspace = true
|
||||
tempfile.workspace = true
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
//! hashes alone do not accept same-content inode replacements. Callers own Read line numbering
|
||||
//! and per-attachment read history.
|
||||
//!
|
||||
//! Create paths are checkout-root-relative and must be strict descendants of the observed target
|
||||
//! Create paths use the target's session coordinates and must be strict descendants of the observed target
|
||||
//! directory. Existing destinations are never overwritten; results return the updated observation
|
||||
//! of the retained target directory, not the created file. Read results also return the validator
|
||||
//! of the state actually read, never a replacement from a later independent stat.
|
||||
@@ -18,7 +18,8 @@
|
||||
//!
|
||||
//! Search delegates to shared List/Glob/Grep engines. Scope rules form unions within layers and
|
||||
//! intersections across layers, provider authority, and output roots. Wrappers transform coordinates
|
||||
//! once; returned paths and Grep content are relative to the final session's logical root.
|
||||
//! once; returned paths and Grep content are relative to the composed output root (the caller's
|
||||
//! cwd by default).
|
||||
//! Paths are not reverse-parsed from rendered filenames; non-UTF-8 names are not lossily aliased.
|
||||
//! Unsupported providers reject search rather than fall back to broader trait searches. Search is
|
||||
//! live traversal, not an atomic snapshot of all descendants or exclusion of external editors.
|
||||
@@ -556,15 +557,13 @@ mod tests {
|
||||
));
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn checkout_hardlink_save_isolates_alias_and_rejects_symlinks_special_files() {
|
||||
async fn checkout_hardlink_save_isolates_alias_and_rejects_symlinks() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
std::fs::write(root.path().join("a"), "original").unwrap();
|
||||
std::fs::hard_link(root.path().join("a"), root.path().join("alias")).unwrap();
|
||||
std::os::unix::fs::symlink("a", root.path().join("link")).unwrap();
|
||||
let _socket = std::os::unix::net::UnixListener::bind(root.path().join("socket")).unwrap();
|
||||
let session = session(root.path());
|
||||
assert!(session.checkout_observe(path("link")).await.is_err());
|
||||
assert!(session.checkout_observe(path("socket")).await.is_err());
|
||||
let read = checked_read(&session, session.checkout_observe(path("a")).await.unwrap()).await;
|
||||
session
|
||||
.checkout_execute(CheckoutRequest {
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
//! Model-facing session coordinates, separate from provider paths and authority.
|
||||
|
||||
use crate::{WorkdirPath, WorkdirSessionCapabilities, WorkdirToolScopeRule};
|
||||
use schemars::JsonSchema;
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
/// A snapshot of the environment actually used by a session's operations.
|
||||
///
|
||||
/// Cwd is a logical Workdir-root coordinate. Scope targets use that root or
|
||||
/// explicit absolute provider paths. Scope is an additional allow ceiling, not
|
||||
/// a replacement for provider checks. Providers
|
||||
/// may retain further restrictions (including deny rules); this projection
|
||||
/// never authorizes an operation.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct WorkdirSessionEnvironment {
|
||||
#[serde(deserialize_with = "deserialize_cwd")]
|
||||
pub cwd: WorkdirPath,
|
||||
pub scope: Option<Vec<WorkdirToolScopeRule>>,
|
||||
pub capabilities: WorkdirSessionCapabilities,
|
||||
pub restrictions: Vec<String>,
|
||||
}
|
||||
|
||||
fn deserialize_cwd<'de, D: serde::Deserializer<'de>>(
|
||||
deserializer: D,
|
||||
) -> Result<WorkdirPath, D::Error> {
|
||||
let path = String::deserialize(deserializer)?;
|
||||
WorkdirPath::new(path).map_err(serde::de::Error::custom)
|
||||
}
|
||||
|
||||
impl WorkdirSessionEnvironment {
|
||||
pub fn root(capabilities: WorkdirSessionCapabilities) -> Self {
|
||||
Self {
|
||||
cwd: WorkdirPath::root(),
|
||||
scope: None,
|
||||
capabilities,
|
||||
restrictions: Vec::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn wire_cwd_is_normalized_and_cannot_claim_an_absolute_or_parent_base() {
|
||||
let mut wire = serde_json::to_value(WorkdirSessionEnvironment::root(
|
||||
WorkdirSessionCapabilities::READ_ONLY,
|
||||
))
|
||||
.unwrap();
|
||||
for invalid in [
|
||||
"/absolute",
|
||||
"../parent",
|
||||
"nested/../parent",
|
||||
"nested\\other",
|
||||
] {
|
||||
wire["cwd"] = invalid.into();
|
||||
assert!(
|
||||
serde_json::from_value::<WorkdirSessionEnvironment>(wire.clone()).is_err(),
|
||||
"{invalid}"
|
||||
);
|
||||
}
|
||||
wire["cwd"] = "./nested//child".into();
|
||||
let decoded: WorkdirSessionEnvironment = serde_json::from_value(wire).unwrap();
|
||||
assert_eq!(decoded.cwd.as_str(), "nested/child");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
//! Real Local/scoped/read-only/router sessions across loopback HTTP. Only the
|
||||
//! HTTP authentication boundary is a fixture; coordinate and scope checks are
|
||||
//! production implementations. Runs in the normal http-client library suite.
|
||||
#![cfg(all(feature = "http-client", target_os = "linux"))]
|
||||
|
||||
use crate::http::*;
|
||||
use crate::*;
|
||||
use axum::{
|
||||
Json, Router,
|
||||
extract::State,
|
||||
routing::{delete, post},
|
||||
};
|
||||
use std::sync::Arc;
|
||||
|
||||
struct Server(tokio::task::JoinHandle<()>);
|
||||
impl Drop for Server {
|
||||
fn drop(&mut self) {
|
||||
self.0.abort();
|
||||
}
|
||||
}
|
||||
|
||||
async fn open(State(session): State<WorkdirSessionHandle>) -> Json<OpenWorkdirSessionResponse> {
|
||||
Json(OpenWorkdirSessionResponse {
|
||||
session_id: WorkdirSessionId::new("environment-session").unwrap(),
|
||||
workdir_id: session.workdir().id().clone(),
|
||||
capabilities: session.capabilities(),
|
||||
environment: session.environment(),
|
||||
})
|
||||
}
|
||||
async fn operation(
|
||||
State(session): State<WorkdirSessionHandle>,
|
||||
Json(request): Json<WorkdirSessionOperationRequest>,
|
||||
) -> Result<
|
||||
Json<WorkdirSessionOperationResult>,
|
||||
(axum::http::StatusCode, Json<WorkdirTransportError>),
|
||||
> {
|
||||
dispatch_workdir_session_operation(session.as_ref(), request.operation)
|
||||
.await
|
||||
.map(Json)
|
||||
.map_err(|error| {
|
||||
(
|
||||
axum::http::StatusCode::FORBIDDEN,
|
||||
Json(WorkdirTransportError::from_workdir_error(&error)),
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_environment_matches_scoped_readonly_operations_without_rebasing_twice() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
std::fs::create_dir(root.path().join("nested")).unwrap();
|
||||
std::fs::write(root.path().join("nested/file.txt"), "nested marker").unwrap();
|
||||
std::fs::write(root.path().join("file.txt"), "root decoy").unwrap();
|
||||
let broker = WorkdirToolBroker::new(Arc::new(LocalWorkdirSession::new(
|
||||
manifest::Scope::writable(root.path()).unwrap(),
|
||||
root.path().to_path_buf(),
|
||||
)));
|
||||
let lease = broker
|
||||
.scope(WorkdirToolScope {
|
||||
cwd: WorkdirPath::new("nested").unwrap(),
|
||||
rules: vec![WorkdirToolScopeRule {
|
||||
target: WorkdirPath::new("nested").unwrap(),
|
||||
permission: WorkdirToolScopePermission::Write,
|
||||
recursive: true,
|
||||
symlink_policy: Default::default(),
|
||||
}],
|
||||
command: true,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let session: WorkdirSessionHandle = Arc::new(ReadOnlyWorkdirSession::new(lease.tool_session()));
|
||||
let expected = session.environment();
|
||||
let app = Router::new()
|
||||
.route("/v1/working-directories/{id}/sessions", post(open))
|
||||
.route("/v1/workdir-sessions/{id}/operations", post(operation))
|
||||
.route(
|
||||
"/v1/workdir-sessions/{id}",
|
||||
delete(|| async { axum::http::StatusCode::NO_CONTENT }),
|
||||
)
|
||||
.with_state(session.clone());
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let _server = Server(tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
}));
|
||||
let remote: WorkdirSessionHandle = Arc::new(
|
||||
RemoteWorkdirSession::open(
|
||||
reqwest::Client::builder().no_proxy().build().unwrap(),
|
||||
format!("http://{address}").parse().unwrap(),
|
||||
"fixture",
|
||||
session.workdir().id().clone(),
|
||||
OpenWorkdirSessionRequest::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap(),
|
||||
);
|
||||
let router = WorkdirSessionRouter::new();
|
||||
router
|
||||
.attach(WorkdirAttachmentAlias::new("remote").unwrap(), remote)
|
||||
.unwrap();
|
||||
let selected = router.resolve(Some("remote")).unwrap().session;
|
||||
assert_eq!(selected.environment(), expected);
|
||||
assert_eq!(expected.cwd, WorkdirPath::new("nested").unwrap());
|
||||
assert!(
|
||||
!expected
|
||||
.capabilities
|
||||
.supports(WorkdirSessionCapability::Command)
|
||||
);
|
||||
assert!(
|
||||
expected
|
||||
.scope
|
||||
.unwrap()
|
||||
.iter()
|
||||
.all(|rule| rule.permission == WorkdirToolScopePermission::Read)
|
||||
);
|
||||
let found = selected
|
||||
.glob(GlobRequest {
|
||||
path: WorkdirPath::root(),
|
||||
pattern: "*.txt".into(),
|
||||
limit: 10,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(found.paths, vec![WorkdirPath::new("file.txt").unwrap()]);
|
||||
let path = found.paths[0].clone();
|
||||
let read = selected
|
||||
.read(ReadRequest {
|
||||
path: path.clone(),
|
||||
offset: 0,
|
||||
limit: 10,
|
||||
max_bytes: 100,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(read.path, path);
|
||||
assert_eq!(read.bytes, b"nested marker");
|
||||
assert!(
|
||||
selected
|
||||
.write(WriteRequest {
|
||||
path,
|
||||
content: b"denied".to_vec(),
|
||||
expected_hash: Some(read.content_hash)
|
||||
})
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
selected
|
||||
.read(ReadRequest {
|
||||
path: WorkdirPath::new_scoped(root.path().join("file.txt").to_str().unwrap())
|
||||
.unwrap(),
|
||||
offset: 0,
|
||||
limit: 10,
|
||||
max_bytes: 100
|
||||
})
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(
|
||||
selected
|
||||
.start_command(CommandRequest {
|
||||
command: "touch escaped".into(),
|
||||
cwd: WorkdirPath::root(),
|
||||
timeout_secs: 5,
|
||||
transport_byte_budget: 100,
|
||||
tool_call_id: None
|
||||
})
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
assert!(!root.path().join("nested/escaped").exists());
|
||||
router.close_all().await.unwrap();
|
||||
lease.close().await.unwrap();
|
||||
broker.close().await.unwrap();
|
||||
}
|
||||
@@ -924,6 +924,8 @@ mod tests {
|
||||
chunks: vec![],
|
||||
next_cursor: 0,
|
||||
terminal: Some(crate::CommandTerminal {
|
||||
process_failure: None,
|
||||
output_collection: crate::CommandOutputCollection::Complete,
|
||||
status: crate::CommandStatus::Completed,
|
||||
exit_code: Some(0),
|
||||
stdout_bytes: 0,
|
||||
@@ -1082,6 +1084,8 @@ mod tests {
|
||||
}],
|
||||
next_cursor: 3,
|
||||
terminal: Some(crate::CommandTerminal {
|
||||
process_failure: None,
|
||||
output_collection: crate::CommandOutputCollection::Complete,
|
||||
status: crate::CommandStatus::Completed,
|
||||
exit_code: Some(0),
|
||||
stdout_bytes: 100 * 1024 * 1024,
|
||||
|
||||
+113
-9
@@ -54,6 +54,7 @@ pub struct OpenWorkdirSessionResponse {
|
||||
pub session_id: WorkdirSessionId,
|
||||
pub workdir_id: WorkdirId,
|
||||
pub capabilities: WorkdirSessionCapabilities,
|
||||
pub environment: crate::WorkdirSessionEnvironment,
|
||||
}
|
||||
|
||||
/// One provider-side Workdir operation.
|
||||
@@ -119,6 +120,51 @@ pub enum WorkdirSessionOperationResult {
|
||||
CommandCancel,
|
||||
}
|
||||
|
||||
/// Audit persistence is independent of the provider result and never authorizes a resend.
|
||||
/// Only server-generated correlation and closed classifications cross this boundary.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct WorkdirAuditFailure {
|
||||
pub operation_id: String,
|
||||
pub stage: WorkdirAuditStage,
|
||||
pub operation_outcome: WorkdirAuditOperationOutcome,
|
||||
pub reason: WorkdirAuditFailureReason,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum WorkdirAuditStage {
|
||||
BeforeDispatch,
|
||||
DispatchRejected,
|
||||
ResponseReceived,
|
||||
ResponseUnknown,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)]
|
||||
pub enum WorkdirAuditOperationOutcome {
|
||||
NotDispatched,
|
||||
Succeeded,
|
||||
Failed { code: WorkdirTransportErrorCode },
|
||||
Unknown,
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum WorkdirAuditFailureReason {
|
||||
AuditStoreUnavailable,
|
||||
}
|
||||
|
||||
/// A successful operation can still have failed audit persistence. Consumers must
|
||||
/// surface that degradation separately, without discarding the result or resending.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
pub struct WorkdirSessionOperationResponse {
|
||||
#[serde(flatten)]
|
||||
pub result: WorkdirSessionOperationResult,
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub audit_failures: Vec<WorkdirAuditFailure>,
|
||||
}
|
||||
|
||||
/// Dispatch one typed operation against any provider-neutral Workdir session.
|
||||
///
|
||||
/// Runtime HTTP handlers, outbound External providers, and Backend brokers can
|
||||
@@ -208,6 +254,7 @@ pub enum WorkdirTransportErrorCode {
|
||||
NotFound,
|
||||
Conflict,
|
||||
OutcomeUnknown,
|
||||
NotDispatched,
|
||||
Unsupported,
|
||||
InvalidRequest,
|
||||
Denied,
|
||||
@@ -231,6 +278,7 @@ impl WorkdirTransportErrorCode {
|
||||
Self::NotFound => "not_found",
|
||||
Self::Conflict => "conflict",
|
||||
Self::OutcomeUnknown => "outcome_unknown",
|
||||
Self::NotDispatched => "not_dispatched",
|
||||
Self::Unsupported => "unsupported",
|
||||
Self::InvalidRequest => "invalid_request",
|
||||
Self::Denied => "denied",
|
||||
@@ -261,7 +309,7 @@ impl WorkdirTransportErrorCode {
|
||||
| Self::SymlinkTargetIsDirectory
|
||||
| Self::IsDirectory
|
||||
| Self::SymlinkDirectoryNotTraversed => 400,
|
||||
Self::Unavailable => 503,
|
||||
Self::Unavailable | Self::NotDispatched => 503,
|
||||
Self::Io | Self::Internal | Self::OutcomeUnknown => 500,
|
||||
Self::Transport => 502,
|
||||
}
|
||||
@@ -298,6 +346,9 @@ impl WorkdirTransportError {
|
||||
Code::OutcomeUnknown,
|
||||
"Workdir operation may have effects; inspect before retrying",
|
||||
),
|
||||
WorkdirError::NotDispatched => {
|
||||
(Code::NotDispatched, "Workdir operation was not dispatched")
|
||||
}
|
||||
WorkdirError::NotFound(_) => (Code::NotFound, "Workdir path was not found"),
|
||||
WorkdirError::Conflict(_) => (
|
||||
Code::Conflict,
|
||||
@@ -375,12 +426,14 @@ impl WorkdirTransportError {
|
||||
use WorkdirTransportErrorCode as Code;
|
||||
let error = match self.code {
|
||||
Code::NotFound => WorkdirError::NotFound("<remote>".into()),
|
||||
Code::Conflict => WorkdirError::Conflict(self.message),
|
||||
Code::Conflict => WorkdirError::Conflict(
|
||||
"The target file's content or existence changed since it was last observed; read the file again before retrying".into(),
|
||||
),
|
||||
Code::OutcomeUnknown => WorkdirError::OutcomeUnknown(
|
||||
"Workdir operation may have effects; inspect before retrying".into(),
|
||||
),
|
||||
Code::Unsupported => WorkdirError::UnsupportedOperation(self.message),
|
||||
Code::InvalidRequest => WorkdirError::InvalidArgument(self.message),
|
||||
Code::Unsupported => WorkdirError::UnsupportedOperation("Workdir operation is not supported".into()),
|
||||
Code::InvalidRequest => WorkdirError::InvalidArgument("Workdir operation request is invalid".into()),
|
||||
Code::Denied => WorkdirError::Denied(crate::WorkdirDenial {
|
||||
reason: self.denial_reason,
|
||||
// Provider-authored text is not trusted diagnostic data.
|
||||
@@ -409,12 +462,13 @@ impl WorkdirTransportError {
|
||||
target: "<remote-target>".into(),
|
||||
},
|
||||
Code::UnknownCommand => WorkdirError::UnknownCommand("<remote>".to_string()),
|
||||
Code::Unavailable => WorkdirError::Unavailable(self.message),
|
||||
Code::Unavailable => WorkdirError::Unavailable("Workdir session is unavailable".into()),
|
||||
Code::NotDispatched => WorkdirError::NotDispatched,
|
||||
Code::Io => WorkdirError::Io {
|
||||
path: "<remote>".into(),
|
||||
source: std::io::Error::other(self.message),
|
||||
source: std::io::Error::other("Workdir I/O operation failed"),
|
||||
},
|
||||
Code::Transport => WorkdirError::Transport(self.message),
|
||||
Code::Transport => WorkdirError::Transport("Workdir transport failed".into()),
|
||||
Code::Internal => WorkdirError::OperationFailed,
|
||||
};
|
||||
let error = if let Some(reason) = self.failure_reason {
|
||||
@@ -500,6 +554,7 @@ mod client {
|
||||
workdir: Workdir,
|
||||
session_id: WorkdirSessionId,
|
||||
capabilities: WorkdirSessionCapabilities,
|
||||
environment: crate::WorkdirSessionEnvironment,
|
||||
closed: AtomicBool,
|
||||
}
|
||||
|
||||
@@ -549,7 +604,8 @@ mod client {
|
||||
if opened.workdir_id.as_str() != workdir_id.as_str() {
|
||||
return Err(WorkdirError::Unavailable(
|
||||
"Runtime opened a session for a different Workdir".to_string(),
|
||||
));
|
||||
)
|
||||
.with_failure_reason(crate::WorkdirFailureReason::SessionIdentityMismatch));
|
||||
}
|
||||
Ok(Self {
|
||||
client,
|
||||
@@ -558,6 +614,7 @@ mod client {
|
||||
workdir: Workdir::new(opened.workdir_id.as_str()),
|
||||
session_id: opened.session_id,
|
||||
capabilities: opened.capabilities,
|
||||
environment: opened.environment,
|
||||
closed: AtomicBool::new(false),
|
||||
})
|
||||
}
|
||||
@@ -608,6 +665,7 @@ mod client {
|
||||
WorkdirError::Unavailable(format!(
|
||||
"Runtime returned a mismatched Workdir operation result; expected {expected}"
|
||||
))
|
||||
.with_failure_reason(crate::WorkdirFailureReason::OperationResultMismatch)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -621,6 +679,12 @@ mod client {
|
||||
self.capabilities
|
||||
}
|
||||
|
||||
fn environment(&self) -> crate::WorkdirSessionEnvironment {
|
||||
let mut environment = self.environment.clone();
|
||||
environment.capabilities = self.capabilities();
|
||||
environment
|
||||
}
|
||||
|
||||
async fn authorize_scope_path(
|
||||
&self,
|
||||
request: WorkdirScopeAuthorizationRequest,
|
||||
@@ -770,7 +834,8 @@ mod client {
|
||||
}
|
||||
Ok(_) if mutation => Err(WorkdirError::OutcomeUnknown(
|
||||
"mismatched response after checkout mutation".into(),
|
||||
)),
|
||||
)
|
||||
.with_failure_reason(crate::WorkdirFailureReason::OperationResultMismatch)),
|
||||
Ok(_) => Err(Self::mismatch("checkout_execute")),
|
||||
Err(error)
|
||||
if mutation
|
||||
@@ -1153,6 +1218,9 @@ mod client {
|
||||
workdir,
|
||||
session_id: WorkdirSessionId::new("s").unwrap(),
|
||||
capabilities: WorkdirSessionCapabilities::EMPTY,
|
||||
environment: crate::WorkdirSessionEnvironment::root(
|
||||
WorkdirSessionCapabilities::EMPTY,
|
||||
),
|
||||
closed: AtomicBool::new(false),
|
||||
};
|
||||
let stat = StatRequest {
|
||||
@@ -1210,6 +1278,37 @@ pub use client::{
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn operation_response_keeps_success_and_independent_audit_failure() {
|
||||
let original =
|
||||
WorkdirSessionOperationResult::CommandStart(crate::CommandHandle("command-a".into()));
|
||||
let response = WorkdirSessionOperationResponse {
|
||||
result: original.clone(),
|
||||
audit_failures: vec![WorkdirAuditFailure {
|
||||
operation_id: "op-00000000-0000-7000-8000-000000000001".into(),
|
||||
stage: WorkdirAuditStage::ResponseReceived,
|
||||
operation_outcome: WorkdirAuditOperationOutcome::Succeeded,
|
||||
reason: WorkdirAuditFailureReason::AuditStoreUnavailable,
|
||||
}],
|
||||
};
|
||||
let json = serde_json::to_value(&response).unwrap();
|
||||
assert_eq!(json["operation"], "command_start");
|
||||
assert_eq!(json["result"], "command-a");
|
||||
assert_eq!(
|
||||
json["audit_failures"][0]["operation_outcome"]["kind"],
|
||||
"succeeded"
|
||||
);
|
||||
assert_eq!(
|
||||
serde_json::from_value::<WorkdirSessionOperationResponse>(json).unwrap(),
|
||||
response
|
||||
);
|
||||
let normal = serde_json::to_value(&original).unwrap();
|
||||
let decoded: WorkdirSessionOperationResponse =
|
||||
serde_json::from_value(normal.clone()).unwrap();
|
||||
assert!(decoded.audit_failures.is_empty());
|
||||
assert_eq!(serde_json::to_value(decoded).unwrap(), normal);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn transport_error_round_trip_keeps_public_classification() {
|
||||
for (code, expected_status, expected_error) in [
|
||||
@@ -1229,6 +1328,11 @@ mod tests {
|
||||
409,
|
||||
"The target file's content or existence changed since it was last observed",
|
||||
),
|
||||
(
|
||||
WorkdirTransportErrorCode::NotDispatched,
|
||||
503,
|
||||
"not dispatched",
|
||||
),
|
||||
(WorkdirTransportErrorCode::Unsupported, 400, "unsupported"),
|
||||
(WorkdirTransportErrorCode::Denied, 403, "denied"),
|
||||
(
|
||||
|
||||
@@ -5,6 +5,8 @@ pub use checkout::{
|
||||
CheckoutObservation, CheckoutOperation, CheckoutOutput, CheckoutRequest, CheckoutResult,
|
||||
CheckoutSearchOperation, CheckoutSearchRequest, CheckoutSearchResult,
|
||||
};
|
||||
mod environment;
|
||||
pub use environment::WorkdirSessionEnvironment;
|
||||
pub mod external;
|
||||
pub mod http;
|
||||
mod local;
|
||||
@@ -16,6 +18,8 @@ pub mod workspace;
|
||||
#[cfg(test)]
|
||||
mod checkout_search_tests;
|
||||
#[cfg(test)]
|
||||
mod environment_tests;
|
||||
#[cfg(test)]
|
||||
mod external_local_tests;
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
@@ -185,6 +189,9 @@ pub type WriteOutcome = WriteResult;
|
||||
/// Worker-bound operation session for a persistent [`Workdir`].
|
||||
///
|
||||
/// Filesystem search and commands run on the host owning materialization.
|
||||
/// Relative operation paths and reusable search results use the session's cwd;
|
||||
/// root-coordinate providers are adapted once at attachment, not by each Tool.
|
||||
/// Scope declarations retain their own root-coordinate authority contract.
|
||||
/// Session closure is terminal but does not delete the Workdir or files.
|
||||
/// Unsupported checkout/search implementations explicitly reject requests, with no ambient
|
||||
/// filesystem fallback. Providers own identity validators; callers own read hash history
|
||||
@@ -194,6 +201,13 @@ pub trait WorkdirSession: std::fmt::Debug + Send + Sync {
|
||||
fn workdir(&self) -> &Workdir;
|
||||
fn capabilities(&self) -> WorkdirSessionCapabilities;
|
||||
|
||||
/// Describe the coordinates used by this session, not a caller's Profile
|
||||
/// or filesystem authority. Root-coordinate providers use the default;
|
||||
/// coordinate and authority wrappers must project their effective view.
|
||||
fn environment(&self) -> WorkdirSessionEnvironment {
|
||||
WorkdirSessionEnvironment::root(self.capabilities())
|
||||
}
|
||||
|
||||
/// Validate an attenuated filesystem rule at the provider boundary without
|
||||
/// exposing the resolved host path. Providers that cannot resolve symbolic
|
||||
/// links must reject resolved-policy checks rather than downgrade them.
|
||||
@@ -408,12 +422,36 @@ impl From<&str> for WorkdirDenial {
|
||||
pub enum WorkdirFailureReason {
|
||||
SessionClosed,
|
||||
ProviderUnavailable,
|
||||
ProviderGrantExpired,
|
||||
ProviderAdmissionClosed,
|
||||
ProviderDispatchRejected,
|
||||
ProviderResponseLost,
|
||||
ProviderOperationTimeout,
|
||||
CommandTerminalMissing,
|
||||
AuditStoreUnavailable,
|
||||
HttpTimeout,
|
||||
HttpConnect,
|
||||
HttpTransfer,
|
||||
HttpResponseInvalid,
|
||||
HttpResponseTooLarge,
|
||||
OperationResultMismatch,
|
||||
SessionIdentityMismatch,
|
||||
SessionOpenTaskFailed,
|
||||
SessionOpenFailed,
|
||||
AuthorizationMissing,
|
||||
AuthorizationStale,
|
||||
AuthorizationUnavailable,
|
||||
SigningIdentityUnavailable,
|
||||
AuthorizationSignFailed,
|
||||
RuntimeStopped,
|
||||
RuntimeStateUnavailable,
|
||||
RuntimeBackendUnavailable,
|
||||
RuntimeOwnerMismatch,
|
||||
RuntimeInvalidRequest,
|
||||
WorkingDirectoryNotFound,
|
||||
WorkingDirectoryNotActive,
|
||||
WorkingDirectoryRecordInvalid,
|
||||
WorkingDirectoryAccessUnavailable,
|
||||
}
|
||||
|
||||
impl WorkdirFailureReason {
|
||||
@@ -421,12 +459,107 @@ impl WorkdirFailureReason {
|
||||
match self {
|
||||
Self::SessionClosed => "session_closed",
|
||||
Self::ProviderUnavailable => "provider_unavailable",
|
||||
Self::ProviderGrantExpired => "provider_grant_expired",
|
||||
Self::ProviderAdmissionClosed => "provider_admission_closed",
|
||||
Self::ProviderDispatchRejected => "provider_dispatch_rejected",
|
||||
Self::ProviderResponseLost => "provider_response_lost",
|
||||
Self::ProviderOperationTimeout => "provider_operation_timeout",
|
||||
Self::CommandTerminalMissing => "command_terminal_missing",
|
||||
Self::AuditStoreUnavailable => "audit_store_unavailable",
|
||||
Self::HttpTimeout => "http_timeout",
|
||||
Self::HttpConnect => "http_connect",
|
||||
Self::HttpTransfer => "http_transfer",
|
||||
Self::HttpResponseInvalid => "http_response_invalid",
|
||||
Self::HttpResponseTooLarge => "http_response_too_large",
|
||||
Self::OperationResultMismatch => "operation_result_mismatch",
|
||||
Self::SessionIdentityMismatch => "session_identity_mismatch",
|
||||
Self::SessionOpenTaskFailed => "session_open_task_failed",
|
||||
Self::SessionOpenFailed => "session_open_failed",
|
||||
Self::AuthorizationMissing => "authorization_missing",
|
||||
Self::AuthorizationStale => "authorization_stale",
|
||||
Self::AuthorizationUnavailable => "authorization_unavailable",
|
||||
Self::SigningIdentityUnavailable => "signing_identity_unavailable",
|
||||
Self::AuthorizationSignFailed => "authorization_sign_failed",
|
||||
Self::RuntimeStopped => "runtime_stopped",
|
||||
Self::RuntimeStateUnavailable => "runtime_state_unavailable",
|
||||
Self::RuntimeBackendUnavailable => "runtime_backend_unavailable",
|
||||
Self::RuntimeOwnerMismatch => "runtime_owner_mismatch",
|
||||
Self::RuntimeInvalidRequest => "runtime_invalid_request",
|
||||
Self::WorkingDirectoryNotFound => "working_directory_not_found",
|
||||
Self::WorkingDirectoryNotActive => "working_directory_not_active",
|
||||
Self::WorkingDirectoryRecordInvalid => "working_directory_record_invalid",
|
||||
Self::WorkingDirectoryAccessUnavailable => "working_directory_access_unavailable",
|
||||
}
|
||||
}
|
||||
|
||||
/// The observed failing boundary, not a dispatch or effects assertion.
|
||||
pub const fn stage(self) -> WorkdirFailureStage {
|
||||
use WorkdirFailureStage as Stage;
|
||||
match self {
|
||||
Self::SessionClosed => Stage::Session,
|
||||
Self::ProviderUnavailable => Stage::Provider,
|
||||
Self::ProviderGrantExpired | Self::ProviderAdmissionClosed => Stage::ProviderAdmission,
|
||||
Self::ProviderDispatchRejected => Stage::ProviderDispatch,
|
||||
Self::ProviderResponseLost | Self::ProviderOperationTimeout => Stage::ProviderResponse,
|
||||
Self::AuditStoreUnavailable => Stage::Audit,
|
||||
Self::CommandTerminalMissing => Stage::Command,
|
||||
Self::HttpTimeout | Self::HttpConnect | Self::HttpTransfer => Stage::Http,
|
||||
Self::HttpResponseInvalid
|
||||
| Self::HttpResponseTooLarge
|
||||
| Self::OperationResultMismatch => Stage::ResponseValidation,
|
||||
Self::SessionIdentityMismatch
|
||||
| Self::SessionOpenTaskFailed
|
||||
| Self::SessionOpenFailed
|
||||
| Self::WorkingDirectoryNotFound
|
||||
| Self::WorkingDirectoryNotActive
|
||||
| Self::WorkingDirectoryRecordInvalid
|
||||
| Self::WorkingDirectoryAccessUnavailable => Stage::SessionOpen,
|
||||
Self::AuthorizationMissing
|
||||
| Self::AuthorizationStale
|
||||
| Self::AuthorizationUnavailable
|
||||
| Self::SigningIdentityUnavailable
|
||||
| Self::AuthorizationSignFailed => Stage::AuthorizationPreparation,
|
||||
Self::RuntimeStopped
|
||||
| Self::RuntimeStateUnavailable
|
||||
| Self::RuntimeBackendUnavailable
|
||||
| Self::RuntimeOwnerMismatch
|
||||
| Self::RuntimeInvalidRequest => Stage::Runtime,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Internal operational boundary derived only from a bounded reason.
|
||||
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
|
||||
pub enum WorkdirFailureStage {
|
||||
Session,
|
||||
Provider,
|
||||
ProviderAdmission,
|
||||
ProviderDispatch,
|
||||
ProviderResponse,
|
||||
Audit,
|
||||
Command,
|
||||
Http,
|
||||
ResponseValidation,
|
||||
SessionOpen,
|
||||
AuthorizationPreparation,
|
||||
Runtime,
|
||||
}
|
||||
|
||||
impl WorkdirFailureStage {
|
||||
pub const fn as_str(self) -> &'static str {
|
||||
match self {
|
||||
Self::Session => "session",
|
||||
Self::Provider => "provider",
|
||||
Self::ProviderAdmission => "provider_admission",
|
||||
Self::ProviderDispatch => "provider_dispatch",
|
||||
Self::ProviderResponse => "provider_response",
|
||||
Self::Audit => "audit",
|
||||
Self::Command => "command",
|
||||
Self::Http => "http",
|
||||
Self::ResponseValidation => "response_validation",
|
||||
Self::SessionOpen => "session_open",
|
||||
Self::AuthorizationPreparation => "authorization_preparation",
|
||||
Self::Runtime => "runtime",
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -466,6 +599,10 @@ pub enum WorkdirError {
|
||||
#[error("Workdir session is unavailable: {0}")]
|
||||
Unavailable(String),
|
||||
|
||||
/// The broker has proof that this operation never reached the provider.
|
||||
#[error("Workdir operation was not dispatched")]
|
||||
NotDispatched,
|
||||
|
||||
#[error("Workdir operation failed")]
|
||||
OperationFailed,
|
||||
|
||||
|
||||
@@ -1340,6 +1340,52 @@ impl WorkdirSession for LocalWorkdirSession {
|
||||
self.inner.capabilities
|
||||
}
|
||||
|
||||
fn environment(&self) -> crate::WorkdirSessionEnvironment {
|
||||
let scope = self.inner.scope.snapshot();
|
||||
let project = |rule: manifest::ScopeRule| -> Option<crate::WorkdirToolScopeRule> {
|
||||
let target = if self.inner.root.starts_with(&rule.target) && rule.recursive {
|
||||
WorkdirPath::root()
|
||||
} else {
|
||||
let path = rule
|
||||
.target
|
||||
.strip_prefix(&self.inner.root)
|
||||
.unwrap_or(&rule.target);
|
||||
WorkdirPath::new_scoped(path.to_str()?).ok()?
|
||||
};
|
||||
Some(crate::WorkdirToolScopeRule {
|
||||
target,
|
||||
permission: match rule.permission {
|
||||
Permission::Read => WorkdirToolScopePermission::Read,
|
||||
Permission::Write => WorkdirToolScopePermission::Write,
|
||||
},
|
||||
recursive: rule.recursive,
|
||||
symlink_policy: rule.symlink_policy,
|
||||
})
|
||||
};
|
||||
crate::WorkdirSessionEnvironment {
|
||||
cwd: WorkdirPath::root(),
|
||||
scope: Some(
|
||||
scope
|
||||
.allow_rules()
|
||||
.into_iter()
|
||||
.filter_map(project)
|
||||
.collect(),
|
||||
),
|
||||
capabilities: self.capabilities(),
|
||||
restrictions: scope
|
||||
.deny_rules()
|
||||
.into_iter()
|
||||
.map(|rule| {
|
||||
project(rule)
|
||||
.map(|rule| format!("Deny: {rule:?}"))
|
||||
.unwrap_or_else(|| {
|
||||
"Unrepresentable provider deny rule remains enforced".into()
|
||||
})
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn authorize_scope_path(
|
||||
&self,
|
||||
request: WorkdirScopeAuthorizationRequest,
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
//! A replay page survives a lost HTTP response until its next cursor is sent.
|
||||
|
||||
use super::*;
|
||||
use crate::{CommandFailure, CommandFailureStage, CommandIoErrorKind, CommandOutputCollection};
|
||||
use tokio::io::{AsyncRead, AsyncReadExt};
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
@@ -146,6 +147,33 @@ pub(super) fn start(
|
||||
telemetry: CommandTelemetry,
|
||||
environment: BTreeMap<String, String>,
|
||||
resources: Vec<Arc<dyn WorkdirSessionResource>>,
|
||||
) -> LocalCommand {
|
||||
let execution_id = command_id.clone();
|
||||
let execution_telemetry = telemetry.clone();
|
||||
start_with(
|
||||
command_id,
|
||||
telemetry,
|
||||
resources,
|
||||
move |cancel, sender| async move {
|
||||
execute(
|
||||
cwd,
|
||||
request,
|
||||
&execution_id,
|
||||
&execution_telemetry,
|
||||
environment,
|
||||
cancel,
|
||||
&sender,
|
||||
)
|
||||
.await
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
fn start_with<F: std::future::Future<Output = crate::CommandTerminal> + Send + 'static>(
|
||||
command_id: String,
|
||||
telemetry: CommandTelemetry,
|
||||
resources: Vec<Arc<dyn WorkdirSessionResource>>,
|
||||
execution: impl FnOnce(watch::Receiver<bool>, mpsc::Sender<Message>) -> F + Send + 'static,
|
||||
) -> LocalCommand {
|
||||
let (sender, receiver) = mpsc::channel(QUEUE_CHUNKS);
|
||||
let (cancel, cancel_rx) = watch::channel(false);
|
||||
@@ -160,16 +188,7 @@ pub(super) fn start(
|
||||
}));
|
||||
let task = tokio::spawn(async move {
|
||||
let _resources = resources;
|
||||
let terminal = execute(
|
||||
cwd,
|
||||
request,
|
||||
&command_id,
|
||||
&telemetry,
|
||||
environment,
|
||||
cancel_rx,
|
||||
&sender,
|
||||
)
|
||||
.await;
|
||||
let terminal = execution(cancel_rx, sender.clone()).await;
|
||||
telemetry.terminal(&command_id, terminal.status, terminal.exit_code);
|
||||
// Closing the byte queue fences this already-published terminal, without
|
||||
// requiring spare queue capacity or a receiver to release task resources.
|
||||
@@ -192,10 +211,18 @@ struct ByteAccounting {
|
||||
}
|
||||
|
||||
impl ByteAccounting {
|
||||
fn terminal(&self, status: CommandStatus, exit_code: Option<i32>) -> crate::CommandTerminal {
|
||||
fn terminal(
|
||||
&self,
|
||||
status: CommandStatus,
|
||||
exit_code: Option<i32>,
|
||||
process_failure: Option<CommandFailure>,
|
||||
output_collection: CommandOutputCollection,
|
||||
) -> crate::CommandTerminal {
|
||||
crate::CommandTerminal {
|
||||
status,
|
||||
exit_code,
|
||||
process_failure,
|
||||
output_collection,
|
||||
stdout_bytes: self.stdout,
|
||||
stderr_bytes: self.stderr,
|
||||
stdout_delivered_bytes: self.stdout_delivered,
|
||||
@@ -215,7 +242,7 @@ async fn pump<R: AsyncRead + Unpin>(
|
||||
command_id: &str,
|
||||
mut remaining: u64,
|
||||
accounting: &mut ByteAccounting,
|
||||
) -> std::io::Result<()> {
|
||||
) -> Result<(), CommandFailure> {
|
||||
let mut stdout_done = false;
|
||||
let mut stderr_done = false;
|
||||
let mut stdout_decoder = CommandOutputDecoder::default();
|
||||
@@ -228,7 +255,11 @@ async fn pump<R: AsyncRead + Unpin>(
|
||||
// Before quota, reserve first to retain bounded pipe backpressure.
|
||||
let permit = if remaining > 0 {
|
||||
let Ok(permit) = sender.reserve().await else {
|
||||
return Ok(());
|
||||
return Err(CommandFailure {
|
||||
stage: CommandFailureStage::OutputDelivery,
|
||||
reason: CommandIoErrorKind::Unknown,
|
||||
os_code: None,
|
||||
});
|
||||
};
|
||||
Some(permit)
|
||||
} else {
|
||||
@@ -242,7 +273,15 @@ async fn pump<R: AsyncRead + Unpin>(
|
||||
CommandStream::Stdout => &stdout_buffer,
|
||||
CommandStream::Stderr => &stderr_buffer,
|
||||
};
|
||||
let count = result?;
|
||||
let count = result.map_err(|error| {
|
||||
CommandFailure::io(
|
||||
match stream {
|
||||
CommandStream::Stdout => CommandFailureStage::StdoutRead,
|
||||
CommandStream::Stderr => CommandFailureStage::StderrRead,
|
||||
},
|
||||
&error,
|
||||
)
|
||||
})?;
|
||||
let decoder = match stream {
|
||||
CommandStream::Stdout => &mut stdout_decoder,
|
||||
CommandStream::Stderr => &mut stderr_decoder,
|
||||
@@ -288,7 +327,7 @@ async fn execute(
|
||||
command_id: &str,
|
||||
telemetry: &CommandTelemetry,
|
||||
environment: BTreeMap<String, String>,
|
||||
mut cancel: watch::Receiver<bool>,
|
||||
cancel: watch::Receiver<bool>,
|
||||
sender: &mpsc::Sender<Message>,
|
||||
) -> crate::CommandTerminal {
|
||||
telemetry.started(command_id, request.tool_call_id);
|
||||
@@ -304,74 +343,159 @@ async fn execute(
|
||||
.kill_on_drop(true);
|
||||
#[cfg(unix)]
|
||||
command.as_std_mut().process_group(0);
|
||||
let mut accounting = ByteAccounting::default();
|
||||
let Ok(mut child) = command.spawn() else {
|
||||
return accounting.terminal(CommandStatus::Failed, None);
|
||||
let mut child = match command.spawn() {
|
||||
Ok(child) => child,
|
||||
Err(error) => {
|
||||
return ByteAccounting::default().terminal(
|
||||
CommandStatus::Failed,
|
||||
None,
|
||||
Some(CommandFailure::io(CommandFailureStage::Spawn, &error)),
|
||||
CommandOutputCollection::NotStarted,
|
||||
);
|
||||
}
|
||||
};
|
||||
let mut process_group = CommandProcessGroupGuard::new(child.id());
|
||||
let stdout = child.stdout.take().expect("piped stdout");
|
||||
let stderr = child.stderr.take().expect("piped stderr");
|
||||
let (status, exit_code) = {
|
||||
supervise(
|
||||
child,
|
||||
stdout,
|
||||
stderr,
|
||||
request.timeout_secs,
|
||||
request.transport_byte_budget,
|
||||
command_id,
|
||||
telemetry,
|
||||
cancel,
|
||||
sender,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
// Substitute only syscalls and readers: injected faults run the production supervisor.
|
||||
trait CommandProcess: Send {
|
||||
fn id(&self) -> Option<u32>;
|
||||
fn start_kill(&mut self);
|
||||
fn wait(
|
||||
&mut self,
|
||||
) -> impl std::future::Future<Output = std::io::Result<std::process::ExitStatus>> + Send;
|
||||
}
|
||||
|
||||
impl CommandProcess for tokio::process::Child {
|
||||
fn id(&self) -> Option<u32> {
|
||||
self.id()
|
||||
}
|
||||
fn start_kill(&mut self) {
|
||||
let _ = self.start_kill();
|
||||
}
|
||||
async fn wait(&mut self) -> std::io::Result<std::process::ExitStatus> {
|
||||
self.wait().await
|
||||
}
|
||||
}
|
||||
|
||||
fn process_result(
|
||||
result: std::io::Result<std::process::ExitStatus>,
|
||||
) -> (CommandStatus, Option<i32>, Option<CommandFailure>) {
|
||||
match result {
|
||||
Ok(exit) => (
|
||||
if exit.success() {
|
||||
CommandStatus::Completed
|
||||
} else {
|
||||
CommandStatus::Failed
|
||||
},
|
||||
exit.code(),
|
||||
None,
|
||||
),
|
||||
Err(error) => (
|
||||
CommandStatus::Failed,
|
||||
None,
|
||||
Some(CommandFailure::io(CommandFailureStage::Wait, &error)),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn supervise(
|
||||
mut child: impl CommandProcess,
|
||||
stdout: impl AsyncRead + Unpin,
|
||||
stderr: impl AsyncRead + Unpin,
|
||||
timeout_secs: u64,
|
||||
byte_budget: u64,
|
||||
command_id: &str,
|
||||
telemetry: &CommandTelemetry,
|
||||
mut cancel: watch::Receiver<bool>,
|
||||
sender: &mpsc::Sender<Message>,
|
||||
) -> crate::CommandTerminal {
|
||||
let mut process_group = CommandProcessGroupGuard::new(child.id());
|
||||
let mut accounting = ByteAccounting::default();
|
||||
let (status, exit_code, process_failure, output_collection) = {
|
||||
let output = pump(
|
||||
stdout,
|
||||
stderr,
|
||||
sender,
|
||||
telemetry,
|
||||
command_id,
|
||||
request.transport_byte_budget,
|
||||
byte_budget,
|
||||
&mut accounting,
|
||||
);
|
||||
tokio::pin!(output);
|
||||
let mut drained = false;
|
||||
let timeout = tokio::time::sleep(Duration::from_secs(request.timeout_secs.max(1)));
|
||||
let mut collected = None;
|
||||
let timeout = tokio::time::sleep(Duration::from_secs(timeout_secs.max(1)));
|
||||
tokio::pin!(timeout);
|
||||
let terminal = loop {
|
||||
tokio::select! {
|
||||
result = &mut output, if !drained => {
|
||||
drained = true;
|
||||
if result.is_err() || sender.is_closed() {
|
||||
result = &mut output, if collected.is_none() => {
|
||||
let failed = result.is_err();
|
||||
collected = Some(result);
|
||||
if failed {
|
||||
process_group.terminate();
|
||||
let _ = child.start_kill();
|
||||
let _ = child.wait().await;
|
||||
break (CommandStatus::Failed, None);
|
||||
child.start_kill();
|
||||
break process_result(child.wait().await);
|
||||
}
|
||||
}
|
||||
exit = child.wait() => {
|
||||
process_group.terminate();
|
||||
break match exit {
|
||||
Ok(exit) => (if exit.success() { CommandStatus::Completed } else { CommandStatus::Failed }, exit.code()),
|
||||
Err(_) => (CommandStatus::Failed, None),
|
||||
};
|
||||
break process_result(exit);
|
||||
}
|
||||
_ = &mut timeout => {
|
||||
process_group.terminate();
|
||||
let _ = child.start_kill();
|
||||
let code = child.wait().await.ok().and_then(|exit| exit.code());
|
||||
break (CommandStatus::TimedOut, code);
|
||||
child.start_kill();
|
||||
let (_, code, failure) = process_result(child.wait().await);
|
||||
break (CommandStatus::TimedOut, code, failure);
|
||||
}
|
||||
_ = sender.closed() => {
|
||||
process_group.terminate();
|
||||
let _ = child.start_kill();
|
||||
let _ = child.wait().await;
|
||||
break (CommandStatus::Cancelled, None);
|
||||
child.start_kill();
|
||||
let (_, code, failure) = process_result(child.wait().await);
|
||||
break (CommandStatus::Cancelled, code, failure);
|
||||
}
|
||||
changed = cancel.changed() => {
|
||||
if changed.is_err() || *cancel.borrow() {
|
||||
process_group.terminate();
|
||||
let _ = child.start_kill();
|
||||
let code = child.wait().await.ok().and_then(|exit| exit.code());
|
||||
break (CommandStatus::Cancelled, code);
|
||||
child.start_kill();
|
||||
let (_, code, failure) = process_result(child.wait().await);
|
||||
break (CommandStatus::Cancelled, code, failure);
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
// Even after kill, preserve the bounded pipe tail before publishing terminal.
|
||||
if !drained {
|
||||
let _ = output.await;
|
||||
}
|
||||
terminal
|
||||
// Exit is not pipe EOF. Retain read failures first observed during drain.
|
||||
let result = match collected {
|
||||
Some(result) => result,
|
||||
None => output.await.map_err(|mut failure| {
|
||||
failure.stage = match failure.stage {
|
||||
CommandFailureStage::StdoutRead => CommandFailureStage::StdoutDrain,
|
||||
CommandFailureStage::StderrRead => CommandFailureStage::StderrDrain,
|
||||
stage => stage,
|
||||
};
|
||||
failure
|
||||
}),
|
||||
};
|
||||
let collection = match result {
|
||||
Ok(()) => CommandOutputCollection::Complete,
|
||||
Err(failure) => CommandOutputCollection::Incomplete { failure },
|
||||
};
|
||||
(terminal.0, terminal.1, terminal.2, collection)
|
||||
};
|
||||
accounting.terminal(status, exit_code)
|
||||
accounting.terminal(status, exit_code, process_failure, output_collection)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -520,3 +520,349 @@ async fn slow_receiver_backpressures_before_quota_but_full_queue_does_not_block_
|
||||
assert_eq!(queued_bytes, QUEUE_CHUNKS * COMMAND_EVENT_CHUNK_BYTES);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct Gate(Arc<std::sync::Mutex<(bool, Option<std::task::Waker>)>>);
|
||||
|
||||
impl Gate {
|
||||
fn open(&self) {
|
||||
let mut state = self.0.lock().unwrap();
|
||||
state.0 = true;
|
||||
if let Some(waker) = state.1.take() {
|
||||
waker.wake();
|
||||
}
|
||||
}
|
||||
|
||||
fn poll(&self, cx: &mut std::task::Context<'_>) -> std::task::Poll<()> {
|
||||
let mut state = self.0.lock().unwrap();
|
||||
if state.0 {
|
||||
std::task::Poll::Ready(())
|
||||
} else {
|
||||
state.1 = Some(cx.waker().clone());
|
||||
std::task::Poll::Pending
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct FaultProcess {
|
||||
ready: Gate,
|
||||
exited: Gate,
|
||||
wait_error: Option<std::io::ErrorKind>,
|
||||
}
|
||||
|
||||
impl CommandProcess for FaultProcess {
|
||||
fn id(&self) -> Option<u32> {
|
||||
None
|
||||
}
|
||||
fn start_kill(&mut self) {
|
||||
self.ready.open();
|
||||
}
|
||||
async fn wait(&mut self) -> std::io::Result<std::process::ExitStatus> {
|
||||
std::future::poll_fn(|cx| self.ready.poll(cx)).await;
|
||||
self.exited.open();
|
||||
match self.wait_error {
|
||||
Some(kind) => Err(std::io::Error::new(
|
||||
kind,
|
||||
"/private/path TOKEN=secret command-body",
|
||||
)),
|
||||
None => Ok(std::process::ExitStatus::default()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
struct FaultReader {
|
||||
prefix: &'static [u8],
|
||||
prefix_read: Gate,
|
||||
fail_ready: Gate,
|
||||
}
|
||||
|
||||
impl AsyncRead for FaultReader {
|
||||
fn poll_read(
|
||||
mut self: std::pin::Pin<&mut Self>,
|
||||
cx: &mut std::task::Context<'_>,
|
||||
buffer: &mut tokio::io::ReadBuf<'_>,
|
||||
) -> std::task::Poll<std::io::Result<()>> {
|
||||
if !self.prefix.is_empty() {
|
||||
let count = self.prefix.len().min(buffer.remaining());
|
||||
buffer.put_slice(&self.prefix[..count]);
|
||||
self.prefix = &self.prefix[count..];
|
||||
self.prefix_read.open();
|
||||
return std::task::Poll::Ready(Ok(()));
|
||||
}
|
||||
std::task::ready!(self.fail_ready.poll(cx));
|
||||
std::task::Poll::Ready(Err(std::io::Error::new(
|
||||
std::io::ErrorKind::PermissionDenied,
|
||||
"/private/path TOKEN=secret command-body",
|
||||
)))
|
||||
}
|
||||
}
|
||||
|
||||
// Faults replace only OS wait/read boundaries, not collection or finalization.
|
||||
#[tokio::test]
|
||||
async fn read_and_post_exit_drain_faults_keep_prefix_exit_and_safe_original_reason() {
|
||||
for stream in [CommandStream::Stdout, CommandStream::Stderr] {
|
||||
for after_exit in [false, true] {
|
||||
let ready = Gate::default();
|
||||
let exited = Gate::default();
|
||||
let fail_ready = if after_exit {
|
||||
exited.clone()
|
||||
} else {
|
||||
let gate = Gate::default();
|
||||
gate.open();
|
||||
gate
|
||||
};
|
||||
let reader = FaultReader {
|
||||
prefix: b"partial",
|
||||
prefix_read: if after_exit {
|
||||
ready.clone()
|
||||
} else {
|
||||
Gate::default()
|
||||
},
|
||||
fail_ready,
|
||||
};
|
||||
let process = FaultProcess {
|
||||
ready,
|
||||
exited,
|
||||
wait_error: None,
|
||||
};
|
||||
let (sender, mut receiver) = mpsc::channel(QUEUE_CHUNKS);
|
||||
let (_cancel, cancel_rx) = watch::channel(false);
|
||||
let telemetry = CommandTelemetry::new();
|
||||
let terminal = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
match stream {
|
||||
CommandStream::Stdout => {
|
||||
supervise(
|
||||
process,
|
||||
reader,
|
||||
&b""[..],
|
||||
10,
|
||||
100,
|
||||
"fault",
|
||||
&telemetry,
|
||||
cancel_rx,
|
||||
&sender,
|
||||
)
|
||||
.await
|
||||
}
|
||||
CommandStream::Stderr => {
|
||||
supervise(
|
||||
process,
|
||||
&b""[..],
|
||||
reader,
|
||||
10,
|
||||
100,
|
||||
"fault",
|
||||
&telemetry,
|
||||
cancel_rx,
|
||||
&sender,
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let Message::Bytes(chunk) = receiver.recv().await.unwrap();
|
||||
assert_eq!(chunk.bytes, b"partial");
|
||||
assert_eq!(chunk.stream, stream);
|
||||
assert_eq!(chunk.sequence, 0);
|
||||
assert_eq!(chunk.offset, 0);
|
||||
assert!(receiver.try_recv().is_err());
|
||||
assert_eq!(terminal.status, CommandStatus::Completed);
|
||||
assert_eq!(terminal.exit_code, Some(0));
|
||||
assert_eq!(terminal.process_failure, None);
|
||||
assert_eq!(terminal.stdout_bytes + terminal.stderr_bytes, 7);
|
||||
assert_eq!(
|
||||
terminal.stdout_delivered_bytes + terminal.stderr_delivered_bytes,
|
||||
7
|
||||
);
|
||||
assert_eq!(terminal.truncation_reason, None);
|
||||
assert_eq!(
|
||||
terminal.output_collection,
|
||||
CommandOutputCollection::Incomplete {
|
||||
failure: CommandFailure {
|
||||
stage: match (stream, after_exit) {
|
||||
(CommandStream::Stdout, false) => CommandFailureStage::StdoutRead,
|
||||
(CommandStream::Stdout, true) => CommandFailureStage::StdoutDrain,
|
||||
(CommandStream::Stderr, false) => CommandFailureStage::StderrRead,
|
||||
(CommandStream::Stderr, true) => CommandFailureStage::StderrDrain,
|
||||
},
|
||||
reason: CommandIoErrorKind::PermissionDenied,
|
||||
os_code: None
|
||||
},
|
||||
}
|
||||
);
|
||||
let wire = serde_json::to_string(&terminal).unwrap();
|
||||
for secret in ["/private", "TOKEN", "secret", "command-body"] {
|
||||
assert!(!wire.contains(secret));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn wait_fault_keeps_collected_bytes_and_does_not_invent_exit_information() {
|
||||
let ready = Gate::default();
|
||||
ready.open();
|
||||
let process = FaultProcess {
|
||||
ready,
|
||||
exited: Gate::default(),
|
||||
wait_error: Some(std::io::ErrorKind::InvalidData),
|
||||
};
|
||||
let (sender, mut receiver) = mpsc::channel(QUEUE_CHUNKS);
|
||||
let (_cancel, cancel_rx) = watch::channel(false);
|
||||
let terminal = supervise(
|
||||
process,
|
||||
&b"out"[..],
|
||||
&b"err"[..],
|
||||
10,
|
||||
100,
|
||||
"wait-fault",
|
||||
&CommandTelemetry::new(),
|
||||
cancel_rx,
|
||||
&sender,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(terminal.status, CommandStatus::Failed);
|
||||
assert_eq!(terminal.exit_code, None);
|
||||
assert_eq!(
|
||||
terminal.process_failure,
|
||||
Some(CommandFailure {
|
||||
stage: CommandFailureStage::Wait,
|
||||
reason: CommandIoErrorKind::InvalidData,
|
||||
os_code: None
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
terminal.output_collection,
|
||||
CommandOutputCollection::Complete
|
||||
);
|
||||
assert_eq!(terminal.stdout_bytes, 3);
|
||||
assert_eq!(terminal.stderr_bytes, 3);
|
||||
let mut chunks = Vec::new();
|
||||
while let Ok(Message::Bytes(chunk)) = receiver.try_recv() {
|
||||
chunks.push(chunk);
|
||||
}
|
||||
assert_eq!(stream_bytes(&chunks, CommandStream::Stdout), b"out");
|
||||
assert_eq!(stream_bytes(&chunks, CommandStream::Stderr), b"err");
|
||||
assert!(!serde_json::to_string(&terminal).unwrap().contains("secret"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn spawn_fault_retains_safe_reason_without_claiming_output_collection() {
|
||||
let root = TempDir::new().unwrap();
|
||||
let (sender, _receiver) = mpsc::channel(QUEUE_CHUNKS);
|
||||
let (_cancel, cancel_rx) = watch::channel(false);
|
||||
let terminal = execute(
|
||||
root.path().join("missing-secret-path"),
|
||||
Harness::request("secret-command", 10, 100),
|
||||
"spawn-fault",
|
||||
&CommandTelemetry::new(),
|
||||
BTreeMap::new(),
|
||||
cancel_rx,
|
||||
&sender,
|
||||
)
|
||||
.await;
|
||||
assert_eq!(
|
||||
terminal.process_failure,
|
||||
Some(CommandFailure {
|
||||
stage: CommandFailureStage::Spawn,
|
||||
reason: CommandIoErrorKind::NotFound,
|
||||
os_code: Some(libc::ENOENT)
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
terminal.output_collection,
|
||||
CommandOutputCollection::NotStarted
|
||||
);
|
||||
assert_eq!(terminal.exit_code, None);
|
||||
assert_eq!(terminal.status, CommandStatus::Failed);
|
||||
assert_eq!(terminal.stdout_bytes + terminal.stderr_bytes, 0);
|
||||
assert!(!serde_json::to_string(&terminal).unwrap().contains("secret"));
|
||||
}
|
||||
|
||||
#[cfg(feature = "http-client")]
|
||||
mod http;
|
||||
|
||||
#[tokio::test]
|
||||
async fn drain_failure_and_byte_budget_keep_independent_loss_diagnostics() {
|
||||
for stream in [CommandStream::Stdout, CommandStream::Stderr] {
|
||||
let ready = Gate::default();
|
||||
let exited = Gate::default();
|
||||
let process = FaultProcess {
|
||||
ready: ready.clone(),
|
||||
exited: exited.clone(),
|
||||
wait_error: None,
|
||||
};
|
||||
let reader = FaultReader {
|
||||
prefix: b"partial",
|
||||
prefix_read: ready,
|
||||
fail_ready: exited,
|
||||
};
|
||||
let (sender, mut receiver) = mpsc::channel(QUEUE_CHUNKS);
|
||||
let (_cancel, cancel_rx) = watch::channel(false);
|
||||
let telemetry = CommandTelemetry::new();
|
||||
let terminal = tokio::time::timeout(Duration::from_secs(5), async {
|
||||
match stream {
|
||||
CommandStream::Stdout => {
|
||||
supervise(
|
||||
process,
|
||||
reader,
|
||||
&b""[..],
|
||||
10,
|
||||
3,
|
||||
"quota-fault",
|
||||
&telemetry,
|
||||
cancel_rx,
|
||||
&sender,
|
||||
)
|
||||
.await
|
||||
}
|
||||
CommandStream::Stderr => {
|
||||
supervise(
|
||||
process,
|
||||
&b""[..],
|
||||
reader,
|
||||
10,
|
||||
3,
|
||||
"quota-fault",
|
||||
&telemetry,
|
||||
cancel_rx,
|
||||
&sender,
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let Message::Bytes(chunk) = receiver.recv().await.unwrap();
|
||||
assert_eq!(chunk.stream, stream);
|
||||
assert_eq!(chunk.bytes, b"par");
|
||||
assert!(receiver.try_recv().is_err());
|
||||
assert_eq!(terminal.exit_code, Some(0));
|
||||
assert_eq!(terminal.stdout_bytes + terminal.stderr_bytes, 7);
|
||||
assert_eq!(
|
||||
terminal.stdout_delivered_bytes + terminal.stderr_delivered_bytes,
|
||||
3
|
||||
);
|
||||
assert_eq!(
|
||||
terminal.truncation_reason,
|
||||
Some(crate::CommandOutputTruncation::ByteBudget)
|
||||
);
|
||||
assert_eq!(
|
||||
terminal.output_collection,
|
||||
CommandOutputCollection::Incomplete {
|
||||
failure: CommandFailure {
|
||||
stage: match stream {
|
||||
CommandStream::Stdout => CommandFailureStage::StdoutDrain,
|
||||
CommandStream::Stderr => CommandFailureStage::StderrDrain,
|
||||
},
|
||||
reason: CommandIoErrorKind::PermissionDenied,
|
||||
os_code: None,
|
||||
},
|
||||
}
|
||||
);
|
||||
assert!(terminal.metadata_valid());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,241 @@
|
||||
//! Loopback integration of the real supervisor, byte queue/replay, local session,
|
||||
//! HTTP dispatcher/client. OS wait/read are
|
||||
//! deterministic faults; session open and start bind that injected process fixture.
|
||||
//! Authorization is not asserted here (Runtime's signed HTTP suite owns it).
|
||||
|
||||
use super::*;
|
||||
use crate::http::*;
|
||||
use axum::{Json, Router, extract::State, routing::post};
|
||||
|
||||
struct Server(tokio::task::JoinHandle<()>);
|
||||
impl Drop for Server {
|
||||
fn drop(&mut self) {
|
||||
self.0.abort();
|
||||
}
|
||||
}
|
||||
|
||||
async fn operation(
|
||||
State(session): State<Arc<LocalWorkdirSession>>,
|
||||
Json(request): Json<WorkdirSessionOperationRequest>,
|
||||
) -> Json<WorkdirSessionOperationResult> {
|
||||
if matches!(request.operation, WorkdirSessionOperation::CommandStart(_)) {
|
||||
return Json(WorkdirSessionOperationResult::CommandStart(CommandHandle(
|
||||
"injected".into(),
|
||||
)));
|
||||
}
|
||||
Json(
|
||||
dispatch_workdir_session_operation(session.as_ref(), request.operation)
|
||||
.await
|
||||
.unwrap(),
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn pipe_and_wait_faults_cross_real_http_with_prefix_and_terminal_replay() {
|
||||
tokio::time::timeout(Duration::from_secs(20), async {
|
||||
for (stream, after_exit, wait_fault) in [
|
||||
(CommandStream::Stdout, false, false),
|
||||
(CommandStream::Stderr, false, false),
|
||||
(CommandStream::Stdout, true, false),
|
||||
(CommandStream::Stderr, true, false),
|
||||
(CommandStream::Stdout, false, true),
|
||||
] {
|
||||
let root = TempDir::new().unwrap();
|
||||
let session = Arc::new(LocalWorkdirSession::new(
|
||||
Scope::writable(root.path()).unwrap(),
|
||||
root.path().into(),
|
||||
));
|
||||
let ready = Gate::default();
|
||||
let exited = Gate::default();
|
||||
if wait_fault {
|
||||
ready.open();
|
||||
}
|
||||
let process = FaultProcess {
|
||||
ready: ready.clone(),
|
||||
exited: exited.clone(),
|
||||
wait_error: wait_fault.then_some(std::io::ErrorKind::InvalidData),
|
||||
};
|
||||
let fail_ready = if after_exit {
|
||||
exited
|
||||
} else {
|
||||
let gate = Gate::default();
|
||||
gate.open();
|
||||
gate
|
||||
};
|
||||
let reader = FaultReader {
|
||||
prefix: b"partial",
|
||||
prefix_read: if after_exit { ready } else { Gate::default() },
|
||||
fail_ready,
|
||||
};
|
||||
let command = start_with(
|
||||
"injected".into(),
|
||||
CommandTelemetry::new(),
|
||||
vec![],
|
||||
move |cancel, sender| async move {
|
||||
let telemetry = CommandTelemetry::new();
|
||||
if wait_fault {
|
||||
supervise(
|
||||
process,
|
||||
&b"partial"[..],
|
||||
&b""[..],
|
||||
10,
|
||||
100,
|
||||
"injected",
|
||||
&telemetry,
|
||||
cancel,
|
||||
&sender,
|
||||
)
|
||||
.await
|
||||
} else {
|
||||
match stream {
|
||||
CommandStream::Stdout => {
|
||||
supervise(
|
||||
process,
|
||||
reader,
|
||||
&b""[..],
|
||||
10,
|
||||
100,
|
||||
"injected",
|
||||
&telemetry,
|
||||
cancel,
|
||||
&sender,
|
||||
)
|
||||
.await
|
||||
}
|
||||
CommandStream::Stderr => {
|
||||
supervise(
|
||||
process,
|
||||
&b""[..],
|
||||
reader,
|
||||
10,
|
||||
100,
|
||||
"injected",
|
||||
&telemetry,
|
||||
cancel,
|
||||
&sender,
|
||||
)
|
||||
.await
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
);
|
||||
session
|
||||
.inner
|
||||
.commands
|
||||
.lock()
|
||||
.await
|
||||
.insert("injected".into(), command);
|
||||
let app = Router::new()
|
||||
.route(
|
||||
"/v1/working-directories/{id}/sessions",
|
||||
post(|| async {
|
||||
Json(OpenWorkdirSessionResponse {
|
||||
session_id: WorkdirSessionId::new("fault-session").unwrap(),
|
||||
workdir_id: crate::Workdir::new("fault-workdir").id().clone(),
|
||||
capabilities: crate::WorkdirSessionCapabilities::ALL,
|
||||
environment: crate::WorkdirSessionEnvironment::root(
|
||||
crate::WorkdirSessionCapabilities::ALL,
|
||||
),
|
||||
})
|
||||
}),
|
||||
)
|
||||
.route("/v1/workdir-sessions/{id}/operations", post(operation))
|
||||
.with_state(session.clone());
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let base = format!("http://{}", listener.local_addr().unwrap());
|
||||
let _server = Server(tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
}));
|
||||
let remote: crate::WorkdirSessionHandle = Arc::new(
|
||||
RemoteWorkdirSession::open(
|
||||
reqwest::Client::builder().no_proxy().build().unwrap(),
|
||||
base.parse().unwrap(),
|
||||
"fixture-token",
|
||||
crate::Workdir::new("fault-workdir").id().clone(),
|
||||
OpenWorkdirSessionRequest::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap(),
|
||||
);
|
||||
let handle = CommandHandle("injected".into());
|
||||
let mut request = CommandOutputRequest {
|
||||
handle,
|
||||
cursor: 0,
|
||||
limit: 1,
|
||||
wait: true,
|
||||
};
|
||||
let mut bytes = Vec::new();
|
||||
let terminal = loop {
|
||||
let page = remote.command_output(request.clone()).await.unwrap();
|
||||
assert_eq!(remote.command_output(request.clone()).await.unwrap(), page);
|
||||
for chunk in &page.chunks {
|
||||
assert_eq!(chunk.sequence, request.cursor);
|
||||
assert_eq!(chunk.offset, request.cursor);
|
||||
assert_eq!(chunk.stream, stream);
|
||||
bytes.extend_from_slice(&chunk.bytes);
|
||||
request.cursor += chunk.bytes.len() as u64;
|
||||
}
|
||||
assert_eq!(page.next_cursor, request.cursor);
|
||||
if let Some(terminal) = page.terminal {
|
||||
break terminal;
|
||||
}
|
||||
};
|
||||
assert_eq!(bytes, b"partial");
|
||||
assert_eq!(terminal.truncation_reason, None);
|
||||
assert_eq!(terminal.stdout_bytes + terminal.stderr_bytes, 7);
|
||||
assert_eq!(
|
||||
terminal.stdout_delivered_bytes + terminal.stderr_delivered_bytes,
|
||||
7
|
||||
);
|
||||
if wait_fault {
|
||||
assert_eq!(terminal.status, CommandStatus::Failed);
|
||||
assert_eq!(terminal.exit_code, None);
|
||||
assert_eq!(
|
||||
terminal.process_failure,
|
||||
Some(CommandFailure {
|
||||
stage: CommandFailureStage::Wait,
|
||||
reason: CommandIoErrorKind::InvalidData,
|
||||
os_code: None
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
terminal.output_collection,
|
||||
CommandOutputCollection::Complete
|
||||
);
|
||||
} else {
|
||||
assert_eq!(terminal.status, CommandStatus::Completed);
|
||||
assert_eq!(terminal.exit_code, Some(0));
|
||||
let stage = match (stream, after_exit) {
|
||||
(CommandStream::Stdout, false) => CommandFailureStage::StdoutRead,
|
||||
(CommandStream::Stderr, false) => CommandFailureStage::StderrRead,
|
||||
(CommandStream::Stdout, true) => CommandFailureStage::StdoutDrain,
|
||||
(CommandStream::Stderr, true) => CommandFailureStage::StderrDrain,
|
||||
};
|
||||
assert_eq!(
|
||||
terminal.output_collection,
|
||||
CommandOutputCollection::Incomplete {
|
||||
failure: CommandFailure {
|
||||
stage,
|
||||
reason: CommandIoErrorKind::PermissionDenied,
|
||||
os_code: None
|
||||
}
|
||||
}
|
||||
);
|
||||
}
|
||||
let wire = serde_json::to_string(&terminal).unwrap();
|
||||
for secret in [
|
||||
"/private",
|
||||
"TOKEN",
|
||||
"secret",
|
||||
"command-body",
|
||||
"fixture-token",
|
||||
] {
|
||||
assert!(!wire.contains(secret));
|
||||
}
|
||||
session.close().await.unwrap();
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("bounded loopback fault projection");
|
||||
}
|
||||
+257
-10
@@ -132,12 +132,151 @@ pub enum CommandOutputTruncation {
|
||||
ByteBudget,
|
||||
}
|
||||
|
||||
/// Raw totals include discarded pipe bytes. Delivered totals count only prefixes
|
||||
/// admitted to the transport queue, not page reads or retries by a consumer.
|
||||
/// Closed diagnostic vocabulary: never serialize OS messages, paths, or errno text.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum CommandIoErrorKind {
|
||||
NotFound,
|
||||
PermissionDenied,
|
||||
ConnectionRefused,
|
||||
ConnectionReset,
|
||||
ConnectionAborted,
|
||||
NotConnected,
|
||||
BrokenPipe,
|
||||
AlreadyExists,
|
||||
WouldBlock,
|
||||
InvalidInput,
|
||||
InvalidData,
|
||||
TimedOut,
|
||||
WriteZero,
|
||||
Interrupted,
|
||||
UnexpectedEof,
|
||||
OutOfMemory,
|
||||
HostUnreachable,
|
||||
NetworkUnreachable,
|
||||
AddrInUse,
|
||||
AddrNotAvailable,
|
||||
NetworkDown,
|
||||
NotADirectory,
|
||||
IsADirectory,
|
||||
DirectoryNotEmpty,
|
||||
ReadOnlyFilesystem,
|
||||
StaleNetworkFileHandle,
|
||||
StorageFull,
|
||||
NotSeekable,
|
||||
QuotaExceeded,
|
||||
FileTooLarge,
|
||||
ResourceBusy,
|
||||
ExecutableFileBusy,
|
||||
Deadlock,
|
||||
CrossesDevices,
|
||||
TooManyLinks,
|
||||
InvalidFilename,
|
||||
ArgumentListTooLong,
|
||||
Unsupported,
|
||||
Other,
|
||||
Unknown,
|
||||
}
|
||||
|
||||
impl From<std::io::ErrorKind> for CommandIoErrorKind {
|
||||
fn from(kind: std::io::ErrorKind) -> Self {
|
||||
use std::io::ErrorKind as Io;
|
||||
match kind {
|
||||
Io::NotFound => Self::NotFound,
|
||||
Io::PermissionDenied => Self::PermissionDenied,
|
||||
Io::ConnectionRefused => Self::ConnectionRefused,
|
||||
Io::ConnectionReset => Self::ConnectionReset,
|
||||
Io::ConnectionAborted => Self::ConnectionAborted,
|
||||
Io::NotConnected => Self::NotConnected,
|
||||
Io::BrokenPipe => Self::BrokenPipe,
|
||||
Io::AlreadyExists => Self::AlreadyExists,
|
||||
Io::WouldBlock => Self::WouldBlock,
|
||||
Io::InvalidInput => Self::InvalidInput,
|
||||
Io::InvalidData => Self::InvalidData,
|
||||
Io::TimedOut => Self::TimedOut,
|
||||
Io::WriteZero => Self::WriteZero,
|
||||
Io::Interrupted => Self::Interrupted,
|
||||
Io::UnexpectedEof => Self::UnexpectedEof,
|
||||
Io::OutOfMemory => Self::OutOfMemory,
|
||||
Io::HostUnreachable => Self::HostUnreachable,
|
||||
Io::NetworkUnreachable => Self::NetworkUnreachable,
|
||||
Io::AddrInUse => Self::AddrInUse,
|
||||
Io::AddrNotAvailable => Self::AddrNotAvailable,
|
||||
Io::NetworkDown => Self::NetworkDown,
|
||||
Io::NotADirectory => Self::NotADirectory,
|
||||
Io::IsADirectory => Self::IsADirectory,
|
||||
Io::DirectoryNotEmpty => Self::DirectoryNotEmpty,
|
||||
Io::ReadOnlyFilesystem => Self::ReadOnlyFilesystem,
|
||||
Io::StaleNetworkFileHandle => Self::StaleNetworkFileHandle,
|
||||
Io::StorageFull => Self::StorageFull,
|
||||
Io::NotSeekable => Self::NotSeekable,
|
||||
Io::QuotaExceeded => Self::QuotaExceeded,
|
||||
Io::FileTooLarge => Self::FileTooLarge,
|
||||
Io::ResourceBusy => Self::ResourceBusy,
|
||||
Io::ExecutableFileBusy => Self::ExecutableFileBusy,
|
||||
Io::Deadlock => Self::Deadlock,
|
||||
Io::CrossesDevices => Self::CrossesDevices,
|
||||
Io::TooManyLinks => Self::TooManyLinks,
|
||||
Io::InvalidFilename => Self::InvalidFilename,
|
||||
Io::ArgumentListTooLong => Self::ArgumentListTooLong,
|
||||
Io::Unsupported => Self::Unsupported,
|
||||
Io::Other => Self::Other,
|
||||
_ => Self::Unknown,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum CommandFailureStage {
|
||||
Spawn,
|
||||
Wait,
|
||||
StdoutRead,
|
||||
StderrRead,
|
||||
StdoutDrain,
|
||||
StderrDrain,
|
||||
OutputDelivery,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct CommandFailure {
|
||||
pub stage: CommandFailureStage,
|
||||
pub reason: CommandIoErrorKind,
|
||||
/// Observed numeric OS code, if supplied by the source. Never inferred from text.
|
||||
pub os_code: Option<i32>,
|
||||
}
|
||||
|
||||
impl CommandFailure {
|
||||
pub(crate) fn io(stage: CommandFailureStage, error: &std::io::Error) -> Self {
|
||||
Self {
|
||||
stage,
|
||||
reason: error.kind().into(),
|
||||
os_code: error.raw_os_error(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// EOF collection is independent of exit status and the explicit byte budget.
|
||||
/// Incomplete counts are lower bounds on bytes actually produced by the process.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(tag = "state", rename_all = "snake_case", deny_unknown_fields)]
|
||||
pub enum CommandOutputCollection {
|
||||
NotStarted,
|
||||
Complete,
|
||||
Incomplete { failure: CommandFailure },
|
||||
}
|
||||
|
||||
/// Raw totals include discarded *observed* pipe bytes, not an unknown failed tail.
|
||||
/// Delivered totals count only prefixes admitted to the transport queue, not page
|
||||
/// reads or retries. Neither terminal arrival nor exit zero implies EOF collection.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct CommandTerminal {
|
||||
pub status: CommandStatus,
|
||||
pub exit_code: Option<i32>,
|
||||
pub process_failure: Option<CommandFailure>,
|
||||
pub output_collection: CommandOutputCollection,
|
||||
pub stdout_bytes: u64,
|
||||
pub stderr_bytes: u64,
|
||||
pub stdout_delivered_bytes: u64,
|
||||
@@ -145,6 +284,54 @@ pub struct CommandTerminal {
|
||||
pub truncation_reason: Option<CommandOutputTruncation>,
|
||||
}
|
||||
|
||||
impl CommandTerminal {
|
||||
/// Validate provider-authored metadata before publishing it as collected evidence.
|
||||
pub fn metadata_valid(&self) -> bool {
|
||||
let truncated = self.stdout_bytes > self.stdout_delivered_bytes
|
||||
|| self.stderr_bytes > self.stderr_delivered_bytes;
|
||||
if self.status == CommandStatus::Running
|
||||
|| self.stdout_delivered_bytes > self.stdout_bytes
|
||||
|| self.stderr_delivered_bytes > self.stderr_bytes
|
||||
|| self.truncation_reason != truncated.then_some(CommandOutputTruncation::ByteBudget)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
match self.process_failure {
|
||||
Some(CommandFailure {
|
||||
stage: CommandFailureStage::Spawn,
|
||||
..
|
||||
}) => {
|
||||
self.status == CommandStatus::Failed
|
||||
&& self.exit_code.is_none()
|
||||
&& self.output_collection == CommandOutputCollection::NotStarted
|
||||
&& self.stdout_bytes == 0
|
||||
&& self.stderr_bytes == 0
|
||||
}
|
||||
Some(CommandFailure {
|
||||
stage: CommandFailureStage::Wait,
|
||||
..
|
||||
}) => {
|
||||
self.status != CommandStatus::Completed
|
||||
&& self.exit_code.is_none()
|
||||
&& self.collection_started()
|
||||
}
|
||||
Some(_) => false,
|
||||
None => self.collection_started(),
|
||||
}
|
||||
}
|
||||
|
||||
fn collection_started(&self) -> bool {
|
||||
match self.output_collection {
|
||||
CommandOutputCollection::NotStarted => false,
|
||||
CommandOutputCollection::Complete => true,
|
||||
CommandOutputCollection::Incomplete { failure } => !matches!(
|
||||
failure.stage,
|
||||
CommandFailureStage::Spawn | CommandFailureStage::Wait
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Sending next_cursor acknowledges this page. Only the current page may be
|
||||
/// retried; acknowledged bytes expire. Terminal fences all captured bytes,
|
||||
/// including on timeout/cancel. Consumers must pull while the process runs.
|
||||
@@ -186,19 +373,13 @@ impl CommandOutput {
|
||||
self.next_cursor <= MAX_COMMAND_TRANSPORT_BYTES
|
||||
&& end.is_none_or(|end| end == self.next_cursor)
|
||||
&& self.terminal.as_ref().is_none_or(|terminal| {
|
||||
let truncated = terminal.stdout_bytes > terminal.stdout_delivered_bytes
|
||||
|| terminal.stderr_bytes > terminal.stderr_delivered_bytes;
|
||||
stream_ends[0].is_none_or(|end| end == terminal.stdout_delivered_bytes)
|
||||
terminal.metadata_valid()
|
||||
&& stream_ends[0].is_none_or(|end| end == terminal.stdout_delivered_bytes)
|
||||
&& stream_ends[1].is_none_or(|end| end == terminal.stderr_delivered_bytes)
|
||||
&& terminal.status != CommandStatus::Running
|
||||
&& terminal.stdout_delivered_bytes <= terminal.stdout_bytes
|
||||
&& terminal.stderr_delivered_bytes <= terminal.stderr_bytes
|
||||
&& terminal
|
||||
.stdout_delivered_bytes
|
||||
.checked_add(terminal.stderr_delivered_bytes)
|
||||
== Some(self.next_cursor)
|
||||
&& terminal.truncation_reason
|
||||
== truncated.then_some(CommandOutputTruncation::ByteBudget)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -262,4 +443,70 @@ mod tests {
|
||||
.contains(&serde_json::json!("transport_byte_budget"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn failure_wire_is_closed_and_does_not_infer_os_codes_from_custom_messages() {
|
||||
let error = std::io::Error::new(
|
||||
std::io::ErrorKind::Other,
|
||||
"/host/private TOKEN=secret EACCES command",
|
||||
);
|
||||
let failure = CommandFailure::io(CommandFailureStage::Wait, &error);
|
||||
assert_eq!(
|
||||
serde_json::to_value(failure).unwrap(),
|
||||
serde_json::json!({"stage":"wait","reason":"other","os_code":null})
|
||||
);
|
||||
let os_error = std::io::Error::from_raw_os_error(libc::EIO);
|
||||
let actual = CommandFailure::io(CommandFailureStage::StdoutDrain, &os_error);
|
||||
assert_eq!(actual.os_code, Some(libc::EIO));
|
||||
for invalid in [
|
||||
serde_json::json!({"stage":"wait","reason":"/host/private TOKEN=secret"}),
|
||||
serde_json::json!({"stage":"/host/private","reason":"other"}),
|
||||
serde_json::json!({"stage":"wait","reason":"other","message":"secret"}),
|
||||
] {
|
||||
assert!(serde_json::from_value::<CommandFailure>(invalid).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_terminal_rejects_inconsistent_failure_domains_and_unstarted_byte_claims() {
|
||||
let wire = serde_json::json!({"status":"completed","exit_code":0,"process_failure":null,"output_collection":{"state":"incomplete","failure":{"stage":"stdout_drain","reason":"other","os_code":null}},"stdout_bytes":3,"stderr_bytes":0,"stdout_delivered_bytes":3,"stderr_delivered_bytes":0,"truncation_reason":null});
|
||||
let terminal: CommandTerminal = serde_json::from_value(wire.clone()).unwrap();
|
||||
assert!(terminal.metadata_valid());
|
||||
for (field, value) in [
|
||||
(
|
||||
"output_collection",
|
||||
serde_json::json!({"state":"not_started"}),
|
||||
),
|
||||
(
|
||||
"output_collection",
|
||||
serde_json::json!({"state":"incomplete","failure":{"stage":"wait","reason":"other"}}),
|
||||
),
|
||||
(
|
||||
"process_failure",
|
||||
serde_json::json!({"stage":"stdout_read","reason":"other"}),
|
||||
),
|
||||
(
|
||||
"process_failure",
|
||||
serde_json::json!({"stage":"spawn","reason":"not_found"}),
|
||||
),
|
||||
("truncation_reason", serde_json::json!("byte_budget")),
|
||||
] {
|
||||
let mut invalid = wire.clone();
|
||||
invalid[field] = value;
|
||||
assert!(
|
||||
!serde_json::from_value::<CommandTerminal>(invalid)
|
||||
.unwrap()
|
||||
.metadata_valid()
|
||||
);
|
||||
}
|
||||
let mut unclassified = wire.clone();
|
||||
unclassified
|
||||
.as_object_mut()
|
||||
.unwrap()
|
||||
.remove("output_collection");
|
||||
assert!(serde_json::from_value::<CommandTerminal>(unclassified).is_err());
|
||||
let mut raw = wire;
|
||||
raw["message"] = serde_json::json!("/private TOKEN=secret");
|
||||
assert!(serde_json::from_value::<CommandTerminal>(raw).is_err());
|
||||
}
|
||||
}
|
||||
|
||||
+236
-31
@@ -225,16 +225,27 @@ impl Drop for ActiveOperation<'_> {
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RoutedWorkdirSession {
|
||||
state: Arc<RoutedSessionState>,
|
||||
view: Option<WorkdirSessionHandle>,
|
||||
}
|
||||
|
||||
impl RoutedWorkdirSession {
|
||||
fn inner(&self) -> &WorkdirSessionHandle {
|
||||
self.view.as_ref().unwrap_or(&self.state.session)
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl WorkdirSession for RoutedWorkdirSession {
|
||||
fn workdir(&self) -> &Workdir {
|
||||
self.state.session.workdir()
|
||||
self.inner().workdir()
|
||||
}
|
||||
|
||||
fn capabilities(&self) -> WorkdirSessionCapabilities {
|
||||
self.state.session.capabilities()
|
||||
self.inner().capabilities()
|
||||
}
|
||||
|
||||
fn environment(&self) -> crate::WorkdirSessionEnvironment {
|
||||
self.inner().environment()
|
||||
}
|
||||
|
||||
async fn authorize_scope_path(
|
||||
@@ -242,7 +253,7 @@ impl WorkdirSession for RoutedWorkdirSession {
|
||||
request: WorkdirScopeAuthorizationRequest,
|
||||
) -> Result<(), WorkdirError> {
|
||||
let _active = self.state.enter()?;
|
||||
self.state.session.authorize_scope_path(request).await
|
||||
self.inner().authorize_scope_path(request).await
|
||||
}
|
||||
|
||||
async fn scope_rules_overlap(
|
||||
@@ -250,7 +261,7 @@ impl WorkdirSession for RoutedWorkdirSession {
|
||||
request: WorkdirScopeOverlapRequest,
|
||||
) -> Result<bool, WorkdirError> {
|
||||
let _active = self.state.enter()?;
|
||||
self.state.session.scope_rules_overlap(request).await
|
||||
self.inner().scope_rules_overlap(request).await
|
||||
}
|
||||
|
||||
async fn checkout_search(
|
||||
@@ -258,7 +269,7 @@ impl WorkdirSession for RoutedWorkdirSession {
|
||||
request: crate::CheckoutSearchRequest,
|
||||
) -> Result<crate::CheckoutSearchResult, WorkdirError> {
|
||||
let _active = self.state.enter()?;
|
||||
self.state.session.checkout_search(request).await
|
||||
self.inner().checkout_search(request).await
|
||||
}
|
||||
|
||||
async fn checkout_observe(
|
||||
@@ -266,59 +277,59 @@ impl WorkdirSession for RoutedWorkdirSession {
|
||||
path: crate::WorkdirPath,
|
||||
) -> Result<crate::CheckoutObservation, WorkdirError> {
|
||||
let _active = self.state.enter()?;
|
||||
self.state.session.checkout_observe(path).await
|
||||
self.inner().checkout_observe(path).await
|
||||
}
|
||||
async fn checkout_execute(
|
||||
&self,
|
||||
request: crate::CheckoutRequest,
|
||||
) -> Result<crate::CheckoutResult, WorkdirError> {
|
||||
let _active = self.state.enter()?;
|
||||
self.state.session.checkout_execute(request).await
|
||||
self.inner().checkout_execute(request).await
|
||||
}
|
||||
|
||||
async fn stat(&self, request: StatRequest) -> Result<StatResult, WorkdirError> {
|
||||
let _active = self.state.enter()?;
|
||||
self.state.session.stat(request).await
|
||||
self.inner().stat(request).await
|
||||
}
|
||||
|
||||
async fn read(&self, request: ReadRequest) -> Result<ReadResult, WorkdirError> {
|
||||
let _active = self.state.enter()?;
|
||||
self.state.session.read(request).await
|
||||
self.inner().read(request).await
|
||||
}
|
||||
|
||||
async fn read_bytes(&self, request: ReadBytesRequest) -> Result<ReadBytesResult, WorkdirError> {
|
||||
let _active = self.state.enter()?;
|
||||
self.state.session.read_bytes(request).await
|
||||
self.inner().read_bytes(request).await
|
||||
}
|
||||
|
||||
async fn write(&self, request: WriteRequest) -> Result<WriteResult, WorkdirError> {
|
||||
let _active = self.state.enter()?;
|
||||
self.state.session.write(request).await
|
||||
self.inner().write(request).await
|
||||
}
|
||||
|
||||
async fn edit(&self, request: EditRequest) -> Result<EditResult, WorkdirError> {
|
||||
let _active = self.state.enter()?;
|
||||
self.state.session.edit(request).await
|
||||
self.inner().edit(request).await
|
||||
}
|
||||
|
||||
async fn list(&self, request: ListRequest) -> Result<ListResult, WorkdirError> {
|
||||
let _active = self.state.enter()?;
|
||||
self.state.session.list(request).await
|
||||
self.inner().list(request).await
|
||||
}
|
||||
|
||||
async fn glob(&self, request: GlobRequest) -> Result<GlobResult, WorkdirError> {
|
||||
let _active = self.state.enter()?;
|
||||
self.state.session.glob(request).await
|
||||
self.inner().glob(request).await
|
||||
}
|
||||
|
||||
async fn grep(&self, request: GrepRequest) -> Result<GrepResult, WorkdirError> {
|
||||
let _active = self.state.enter()?;
|
||||
self.state.session.grep(request).await
|
||||
self.inner().grep(request).await
|
||||
}
|
||||
|
||||
async fn start_command(&self, request: CommandRequest) -> Result<CommandHandle, WorkdirError> {
|
||||
let _active = self.state.enter()?;
|
||||
let handle = self.state.session.start_command(request).await?;
|
||||
let handle = self.inner().start_command(request).await?;
|
||||
self.state
|
||||
.active_commands
|
||||
.lock()
|
||||
@@ -329,7 +340,7 @@ impl WorkdirSession for RoutedWorkdirSession {
|
||||
|
||||
async fn command_status(&self, handle: CommandHandle) -> Result<CommandStatus, WorkdirError> {
|
||||
let _active = self.state.enter()?;
|
||||
let status = self.state.session.command_status(handle.clone()).await?;
|
||||
let status = self.inner().command_status(handle.clone()).await?;
|
||||
self.state.mark_terminal(&handle, status);
|
||||
Ok(status)
|
||||
}
|
||||
@@ -340,7 +351,7 @@ impl WorkdirSession for RoutedWorkdirSession {
|
||||
) -> Result<CommandOutput, WorkdirError> {
|
||||
let _active = self.state.enter()?;
|
||||
let handle = request.handle.clone();
|
||||
let output = self.state.session.command_output(request).await?;
|
||||
let output = self.inner().command_output(request).await?;
|
||||
if let Some(terminal) = &output.terminal {
|
||||
self.state.mark_terminal(&handle, terminal.status);
|
||||
}
|
||||
@@ -349,7 +360,7 @@ impl WorkdirSession for RoutedWorkdirSession {
|
||||
|
||||
async fn cancel_command(&self, handle: CommandHandle) -> Result<(), WorkdirError> {
|
||||
let _active = self.state.enter()?;
|
||||
self.state.session.cancel_command(handle.clone()).await?;
|
||||
self.inner().cancel_command(handle.clone()).await?;
|
||||
self.state
|
||||
.active_commands
|
||||
.lock()
|
||||
@@ -359,11 +370,11 @@ impl WorkdirSession for RoutedWorkdirSession {
|
||||
}
|
||||
|
||||
fn subscribe_command_events(&self) -> Option<broadcast::Receiver<CommandEvent>> {
|
||||
self.state.session.subscribe_command_events()
|
||||
self.inner().subscribe_command_events()
|
||||
}
|
||||
|
||||
fn command_snapshot(&self) -> Vec<CommandSnapshot> {
|
||||
let snapshots = self.state.session.command_snapshot();
|
||||
let snapshots = self.inner().command_snapshot();
|
||||
self.state.reconcile_terminal_commands(&snapshots);
|
||||
snapshots
|
||||
}
|
||||
@@ -374,7 +385,7 @@ impl WorkdirSession for RoutedWorkdirSession {
|
||||
"Workdir attachment has active operations or commands".to_string(),
|
||||
));
|
||||
}
|
||||
self.state.session.close().await
|
||||
self.inner().close().await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -387,6 +398,7 @@ impl WorkdirSession for RoutedWorkdirSession {
|
||||
pub struct WorkdirSessionRouter {
|
||||
sessions: RwLock<BTreeMap<WorkdirAttachmentAlias, Arc<RoutedSessionState>>>,
|
||||
next_generation: AtomicU64,
|
||||
coordinate_brokers: RwLock<BTreeMap<WorkdirAttachmentAlias, (u64, crate::WorkdirToolBroker)>>,
|
||||
}
|
||||
|
||||
impl WorkdirSessionRouter {
|
||||
@@ -473,9 +485,9 @@ impl WorkdirSessionRouter {
|
||||
}
|
||||
};
|
||||
Ok(ResolvedWorkdirSession {
|
||||
session: self.routed(&selected.0, selected.1.clone()),
|
||||
alias: selected.0,
|
||||
generation: selected.1.generation,
|
||||
session: Arc::new(RoutedWorkdirSession { state: selected.1 }),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -483,13 +495,22 @@ impl WorkdirSessionRouter {
|
||||
&self,
|
||||
alias: WorkdirAttachmentAlias,
|
||||
session: WorkdirSessionHandle,
|
||||
) -> Result<WorkdirSessionHandle, WorkdirError> {
|
||||
self.attach_with_cwd(alias, session, None)
|
||||
}
|
||||
|
||||
fn attach_with_cwd(
|
||||
&self,
|
||||
alias: WorkdirAttachmentAlias,
|
||||
session: WorkdirSessionHandle,
|
||||
cwd: Option<crate::WorkdirPath>,
|
||||
) -> Result<WorkdirSessionHandle, WorkdirError> {
|
||||
let mut sessions = self.sessions.write().map_err(|_| {
|
||||
WorkdirError::Unavailable("Workdir attachment router is poisoned".to_string())
|
||||
WorkdirError::Unavailable("Workdir attachment router is poisoned".into())
|
||||
})?;
|
||||
if sessions.contains_key(&alias) {
|
||||
return Err(WorkdirError::Conflict(format!(
|
||||
"Workdir attachment alias `{alias}` already exists"
|
||||
"Workdir attachment alias \u{0060}{alias}\u{0060} already exists"
|
||||
)));
|
||||
}
|
||||
let state = Arc::new(RoutedSessionState {
|
||||
@@ -499,8 +520,61 @@ impl WorkdirSessionRouter {
|
||||
active_operations: AtomicUsize::new(0),
|
||||
active_commands: Mutex::new(HashSet::new()),
|
||||
});
|
||||
let view = if let Some(cwd) = cwd {
|
||||
let source = Arc::new(RoutedWorkdirSession {
|
||||
state: state.clone(),
|
||||
view: None,
|
||||
});
|
||||
let broker = crate::WorkdirToolBroker::at_cwd(source, cwd)?;
|
||||
let view = broker.tool_session();
|
||||
self.coordinate_brokers
|
||||
.write()
|
||||
.map_err(|_| {
|
||||
WorkdirError::Unavailable("Workdir coordinate broker map is poisoned".into())
|
||||
})?
|
||||
.insert(alias.clone(), (state.generation, broker));
|
||||
Some(view)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
sessions.insert(alias, state.clone());
|
||||
Ok(Arc::new(RoutedWorkdirSession { state }))
|
||||
Ok(Arc::new(RoutedWorkdirSession { state, view }))
|
||||
}
|
||||
|
||||
fn routed(
|
||||
&self,
|
||||
alias: &WorkdirAttachmentAlias,
|
||||
state: Arc<RoutedSessionState>,
|
||||
) -> Arc<RoutedWorkdirSession> {
|
||||
let view = self
|
||||
.coordinate_broker(alias, state.generation)
|
||||
.map(|broker| broker.tool_session());
|
||||
Arc::new(RoutedWorkdirSession { state, view })
|
||||
}
|
||||
|
||||
pub(crate) fn coordinate_broker(
|
||||
&self,
|
||||
alias: &WorkdirAttachmentAlias,
|
||||
generation: u64,
|
||||
) -> Option<crate::WorkdirToolBroker> {
|
||||
self.coordinate_brokers
|
||||
.read()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.get(alias)
|
||||
.filter(|(current, _)| *current == generation)
|
||||
.map(|(_, broker)| broker.clone())
|
||||
}
|
||||
|
||||
/// Bind tool coordinates once while retaining root-coordinate delegation
|
||||
/// and provider authorization. Both views share the route's detach fence.
|
||||
pub fn attach_at_cwd(
|
||||
&self,
|
||||
alias: WorkdirAttachmentAlias,
|
||||
source: WorkdirSessionHandle,
|
||||
cwd: crate::WorkdirPath,
|
||||
) -> Result<WorkdirSessionHandle, WorkdirError> {
|
||||
let cwd = crate::WorkdirPath::new(cwd.as_str())?;
|
||||
self.attach_with_cwd(alias, source, Some(cwd))
|
||||
}
|
||||
|
||||
pub fn session(&self, alias: &WorkdirAttachmentAlias) -> Option<WorkdirSessionHandle> {
|
||||
@@ -509,7 +583,7 @@ impl WorkdirSessionRouter {
|
||||
.ok()?
|
||||
.get(alias)
|
||||
.cloned()
|
||||
.map(|state| Arc::new(RoutedWorkdirSession { state }) as WorkdirSessionHandle)
|
||||
.map(|state| self.routed(alias, state) as WorkdirSessionHandle)
|
||||
}
|
||||
|
||||
/// Compatibility projection used only while filesystem tool schemas have no
|
||||
@@ -521,10 +595,8 @@ impl WorkdirSessionRouter {
|
||||
return None;
|
||||
}
|
||||
sessions
|
||||
.values()
|
||||
.next()
|
||||
.cloned()
|
||||
.map(|state| Arc::new(RoutedWorkdirSession { state }) as WorkdirSessionHandle)
|
||||
.first_key_value()
|
||||
.map(|(alias, state)| self.routed(alias, state.clone()) as WorkdirSessionHandle)
|
||||
}
|
||||
|
||||
pub fn can_detach(&self, alias: &WorkdirAttachmentAlias) -> Result<(), WorkdirError> {
|
||||
@@ -620,6 +692,10 @@ impl WorkdirSessionRouter {
|
||||
.is_some_and(|current| Arc::ptr_eq(current, &state))
|
||||
{
|
||||
sessions.remove(alias);
|
||||
self.coordinate_brokers
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.remove(alias);
|
||||
}
|
||||
Ok(state.session.clone())
|
||||
}
|
||||
@@ -661,6 +737,10 @@ impl WorkdirSessionRouter {
|
||||
.is_some_and(|current| Arc::ptr_eq(current, &state))
|
||||
{
|
||||
sessions.remove(&alias);
|
||||
self.coordinate_brokers
|
||||
.write()
|
||||
.unwrap_or_else(std::sync::PoisonError::into_inner)
|
||||
.remove(&alias);
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
@@ -684,6 +764,131 @@ mod tests {
|
||||
))
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn cwd_views_share_root_delegation_leases_and_attachment_fences() {
|
||||
use crate::{
|
||||
WorkdirPath as P, WorkdirToolBrokerRouter, WorkdirToolScope,
|
||||
WorkdirToolScopePermission as Permission, WorkdirToolScopeRule,
|
||||
};
|
||||
let root = TempDir::new().unwrap();
|
||||
for directory in ["work", "other", "other/inside"] {
|
||||
std::fs::create_dir_all(root.path().join(directory)).unwrap();
|
||||
std::fs::write(root.path().join(directory).join("marker"), directory).unwrap();
|
||||
}
|
||||
let router = Arc::new(WorkdirSessionRouter::new());
|
||||
let alias = WorkdirAttachmentAlias::new("source").unwrap();
|
||||
let parent = router
|
||||
.attach_at_cwd(alias.clone(), session("wd", &root), P::new("work").unwrap())
|
||||
.unwrap();
|
||||
assert_eq!(parent.environment().cwd, P::new("work").unwrap());
|
||||
let request = |cwd: &str| WorkdirToolScope {
|
||||
rules: vec![WorkdirToolScopeRule {
|
||||
target: P::new(cwd).unwrap(),
|
||||
permission: Permission::Write,
|
||||
recursive: true,
|
||||
symlink_policy: Default::default(),
|
||||
}],
|
||||
cwd: P::new(cwd).unwrap(),
|
||||
command: false,
|
||||
};
|
||||
let brokers = WorkdirToolBrokerRouter::new(router.clone());
|
||||
// Declarations stay root-relative, not work/other.
|
||||
let child = brokers
|
||||
.scope(vec![(alias.clone(), request("other"))])
|
||||
.await
|
||||
.unwrap();
|
||||
let grandchild = child
|
||||
.broker_router()
|
||||
.scope(vec![(alias.clone(), request("other/inside"))])
|
||||
.await
|
||||
.unwrap();
|
||||
for (session, cwd) in [
|
||||
(parent.clone(), "work"),
|
||||
(
|
||||
child.session_router().resolve(None).unwrap().session,
|
||||
"other",
|
||||
),
|
||||
(
|
||||
grandchild.session_router().resolve(None).unwrap().session,
|
||||
"other/inside",
|
||||
),
|
||||
] {
|
||||
assert_eq!(session.environment().cwd, P::new(cwd).unwrap());
|
||||
let result = session
|
||||
.read(ReadRequest {
|
||||
path: P::new("marker").unwrap(),
|
||||
offset: 0,
|
||||
limit: 10,
|
||||
max_bytes: 100,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(result.bytes, cwd.as_bytes());
|
||||
assert_eq!(result.path, P::new("marker").unwrap());
|
||||
assert_eq!(
|
||||
session
|
||||
.stat(StatRequest {
|
||||
path: P::new("marker").unwrap()
|
||||
})
|
||||
.await
|
||||
.unwrap()
|
||||
.path,
|
||||
P::new("marker").unwrap()
|
||||
);
|
||||
}
|
||||
// The model's tool view and delegation view share write-lease exclusion.
|
||||
let absolute = P::new_scoped(root.path().join("other/marker").to_str().unwrap()).unwrap();
|
||||
let denied = parent
|
||||
.write(WriteRequest {
|
||||
path: absolute,
|
||||
content: b"forbidden".to_vec(),
|
||||
expected_hash: None,
|
||||
})
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert_eq!(
|
||||
denied.denial_reason(),
|
||||
Some(crate::WorkdirDenialReason::ChildWriteLeaseConflict)
|
||||
);
|
||||
let old_child = child.session_router().resolve(None).unwrap().session;
|
||||
router.begin_detach(&alias).await.unwrap();
|
||||
assert!(
|
||||
old_child
|
||||
.read(ReadRequest {
|
||||
path: P::new("marker").unwrap(),
|
||||
offset: 0,
|
||||
limit: 10,
|
||||
max_bytes: 100
|
||||
})
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
router.cancel_detach(&alias);
|
||||
grandchild.close().await.unwrap();
|
||||
child.close().await.unwrap();
|
||||
router.detach(&alias).await.unwrap();
|
||||
router
|
||||
.attach_at_cwd(
|
||||
alias,
|
||||
session("replacement", &root),
|
||||
P::new("other").unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
router.resolve(None).unwrap().session.environment().cwd,
|
||||
P::new("other").unwrap()
|
||||
);
|
||||
assert!(
|
||||
old_child
|
||||
.stat(StatRequest {
|
||||
path: P::new("marker").unwrap()
|
||||
})
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
router.close_all().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn checkout_routes_use_active_operation_detach_fence_and_old_handles_stay_closed() {
|
||||
let root = TempDir::new().unwrap();
|
||||
|
||||
@@ -108,17 +108,34 @@ impl WorkdirToolBroker {
|
||||
Self::with_capabilities(source, capabilities)
|
||||
}
|
||||
|
||||
/// Set the tool coordinate base without changing provider authority.
|
||||
/// The input is relative to the source session's coordinates; delegation
|
||||
/// declarations retain their existing coordinate contract.
|
||||
pub fn at_cwd(source: WorkdirSessionHandle, cwd: FsPath) -> Result<Self, WorkdirError> {
|
||||
let cwd = FsPath::new(cwd.as_str())?;
|
||||
let capabilities = source.capabilities();
|
||||
Ok(Self::with_capabilities_and_cwd(source, capabilities, cwd))
|
||||
}
|
||||
|
||||
/// Own a session behind an additional Backend-authored capability ceiling.
|
||||
/// The ceiling can only attenuate provider capabilities; it can never add one.
|
||||
pub fn with_capabilities(
|
||||
source: WorkdirSessionHandle,
|
||||
capabilities: WorkdirSessionCapabilities,
|
||||
) -> Self {
|
||||
Self::with_capabilities_and_cwd(source, capabilities, FsPath::root())
|
||||
}
|
||||
|
||||
fn with_capabilities_and_cwd(
|
||||
source: WorkdirSessionHandle,
|
||||
capabilities: WorkdirSessionCapabilities,
|
||||
cwd: FsPath,
|
||||
) -> Self {
|
||||
let capabilities = source.capabilities().intersection(capabilities);
|
||||
let (command_events, _) = broadcast::channel(64);
|
||||
let authority = Arc::new(ScopedWorkdirSession {
|
||||
source,
|
||||
cwd: FsPath::new("").expect("empty Workdir path is valid"),
|
||||
cwd,
|
||||
scope: None,
|
||||
capabilities,
|
||||
validity: SessionValidity::root(),
|
||||
@@ -416,6 +433,9 @@ impl WorkdirToolBrokerRouter {
|
||||
.sessions
|
||||
.resolve(Some(alias.as_str()))
|
||||
.map_err(|error| WorkdirError::InvalidArgument(error.to_string()))?;
|
||||
if let Some(broker) = self.sessions.coordinate_broker(alias, selected.generation) {
|
||||
return Ok(broker);
|
||||
}
|
||||
let mut brokers = self.brokers.lock().map_err(|_| {
|
||||
WorkdirError::Unavailable("Workdir tool broker router is poisoned".to_string())
|
||||
})?;
|
||||
@@ -684,7 +704,7 @@ impl ScopedWorkdirSession {
|
||||
}
|
||||
|
||||
fn resolve_path(&self, path: &FsPath) -> Result<FsPath, WorkdirError> {
|
||||
if self.cwd.as_str().is_empty() {
|
||||
if self.cwd.as_str().is_empty() || Path::new(path.as_str()).is_absolute() {
|
||||
return Ok(path.clone());
|
||||
}
|
||||
let joined = Path::new(self.cwd.as_str()).join(path.as_str());
|
||||
@@ -1085,6 +1105,43 @@ impl WorkdirSession for ScopedWorkdirSession {
|
||||
self.capabilities
|
||||
}
|
||||
|
||||
fn environment(&self) -> crate::WorkdirSessionEnvironment {
|
||||
let mut environment = self.source.environment();
|
||||
let source_cwd = environment.cwd.clone();
|
||||
environment.cwd = FsPath::new(
|
||||
Path::new(source_cwd.as_str())
|
||||
.join(self.cwd.as_str())
|
||||
.to_str()
|
||||
.expect("validated UTF-8 paths"),
|
||||
)
|
||||
.expect("validated session cwd");
|
||||
if let Some(scope) = &self.scope {
|
||||
if let Some(inherited) = environment.scope.take() {
|
||||
environment
|
||||
.restrictions
|
||||
.push(format!("Provider allow ceiling: {inherited:?}"));
|
||||
}
|
||||
environment.scope = Some(
|
||||
scope
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(|mut rule| {
|
||||
rule.target = FsPath::new(
|
||||
Path::new(source_cwd.as_str())
|
||||
.join(rule.target.as_str())
|
||||
.to_str()
|
||||
.expect("validated UTF-8 paths"),
|
||||
)
|
||||
.expect("validated scope target");
|
||||
rule
|
||||
})
|
||||
.collect(),
|
||||
);
|
||||
}
|
||||
environment.capabilities = self.capabilities();
|
||||
environment
|
||||
}
|
||||
|
||||
async fn authorize_scope_path(
|
||||
&self,
|
||||
mut request: WorkdirScopeAuthorizationRequest,
|
||||
@@ -1273,8 +1330,11 @@ impl WorkdirSession for ScopedWorkdirSession {
|
||||
.resolve_operation_path(&request.path, WorkdirToolScopePermission::Read)
|
||||
.await?;
|
||||
self.ensure_read(&path, WorkdirSessionCapability::Read)?;
|
||||
let caller_path = request.path.clone();
|
||||
request.path = path;
|
||||
self.source.stat(request).await
|
||||
let mut result = self.source.stat(request).await?;
|
||||
result.path = caller_path;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
async fn read(&self, mut request: ReadRequest) -> Result<ReadResult, WorkdirError> {
|
||||
@@ -1282,8 +1342,11 @@ impl WorkdirSession for ScopedWorkdirSession {
|
||||
.resolve_operation_path(&request.path, WorkdirToolScopePermission::Read)
|
||||
.await?;
|
||||
self.ensure_read(&path, WorkdirSessionCapability::Read)?;
|
||||
let caller_path = request.path.clone();
|
||||
request.path = path;
|
||||
self.source.read(request).await
|
||||
let mut result = self.source.read(request).await?;
|
||||
result.path = caller_path;
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
async fn read_bytes(
|
||||
@@ -1422,9 +1485,10 @@ impl WorkdirSession for ScopedWorkdirSession {
|
||||
)));
|
||||
}
|
||||
let tool_call_id = request.tool_call_id.clone();
|
||||
if self.scope.is_some() {
|
||||
request.cwd = self.resolve_path(&request.cwd)?;
|
||||
}
|
||||
request.cwd = self
|
||||
.resolve_operation_path(&request.cwd, WorkdirToolScopePermission::Read)
|
||||
.await?;
|
||||
self.ensure_path(&request.cwd, WorkdirToolScopePermission::Read)?;
|
||||
if let Some(tool_call_id) = &tool_call_id {
|
||||
self.starting_tool_calls
|
||||
.lock()
|
||||
@@ -1601,7 +1665,20 @@ impl WorkdirSession for ReadOnlyWorkdirSession {
|
||||
}
|
||||
|
||||
fn capabilities(&self) -> WorkdirSessionCapabilities {
|
||||
WorkdirSessionCapabilities::READ_ONLY
|
||||
self.inner
|
||||
.capabilities()
|
||||
.intersection(WorkdirSessionCapabilities::READ_ONLY)
|
||||
}
|
||||
|
||||
fn environment(&self) -> crate::WorkdirSessionEnvironment {
|
||||
let mut environment = self.inner.environment();
|
||||
environment.capabilities = self.capabilities();
|
||||
if let Some(scope) = &mut environment.scope {
|
||||
for rule in scope {
|
||||
rule.permission = WorkdirToolScopePermission::Read;
|
||||
}
|
||||
}
|
||||
environment
|
||||
}
|
||||
|
||||
async fn authorize_scope_path(
|
||||
|
||||
@@ -83,3 +83,34 @@ fn provider_cannot_inject_free_text_failure_reason() {
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn provider_authored_operational_messages_never_enter_error_display_or_debug() {
|
||||
for code in [
|
||||
Code::Conflict,
|
||||
Code::OutcomeUnknown,
|
||||
Code::Unsupported,
|
||||
Code::InvalidRequest,
|
||||
Code::Unavailable,
|
||||
Code::Io,
|
||||
Code::Transport,
|
||||
Code::Internal,
|
||||
] {
|
||||
let wire = serde_json::json!({
|
||||
"code": code, "message":"/host/private command-secret body-secret proof-secret key-secret Authorization: Bearer bearer-secret",
|
||||
"failure_reason": "session_open_task_failed"
|
||||
});
|
||||
let decoded: WorkdirTransportError = serde_json::from_value(wire).unwrap();
|
||||
let error = decoded.into_workdir_error();
|
||||
for text in [
|
||||
format!("{error}"),
|
||||
format!("{error:?}"),
|
||||
serde_json::to_string(&WorkdirTransportError::from_workdir_error(&error)).unwrap(),
|
||||
] {
|
||||
assert!(!text.contains("secret"), "{text}");
|
||||
assert!(!text.contains("/host/private"), "{text}");
|
||||
}
|
||||
assert_eq!(error.failure_reason(), Some(Reason::SessionOpenTaskFailed));
|
||||
assert_eq!(WorkdirTransportError::from_workdir_error(&error).code, code);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,6 +14,18 @@ use workdir::{
|
||||
},
|
||||
};
|
||||
|
||||
fn opened(workdir_id: &str) -> String {
|
||||
serde_json::to_string(&workdir::http::OpenWorkdirSessionResponse {
|
||||
session_id: workdir::http::WorkdirSessionId::new("s").unwrap(),
|
||||
workdir_id: Workdir::new(workdir_id).id().clone(),
|
||||
capabilities: workdir::WorkdirSessionCapabilities::ALL,
|
||||
environment: workdir::WorkdirSessionEnvironment::root(
|
||||
workdir::WorkdirSessionCapabilities::ALL,
|
||||
),
|
||||
})
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
fn reply(listener: &TcpListener, status: &str, body: &str) {
|
||||
listener.set_nonblocking(true).unwrap();
|
||||
let deadline = Instant::now() + Duration::from_secs(5);
|
||||
@@ -73,11 +85,7 @@ async fn remote_http_denials_forward_only_typed_reasons_and_old_missing_reason()
|
||||
error["denial_reason"] = serde_json::to_value(reason).unwrap();
|
||||
}
|
||||
let server = thread::spawn(move || {
|
||||
reply(
|
||||
&listener,
|
||||
"200 OK",
|
||||
r#"{"session_id":"s","workdir_id":"w","capabilities":{"bits":63}}"#,
|
||||
);
|
||||
reply(&listener, "200 OK", &opened("w"));
|
||||
reply(&listener, "403 Forbidden", &error.to_string());
|
||||
});
|
||||
let remote = RemoteWorkdirSession::open(
|
||||
@@ -163,11 +171,7 @@ async fn remote_checkout_diagnostics_preserve_mutation_outcome_and_read_failures
|
||||
let address = listener.local_addr().unwrap();
|
||||
let error = serde_json::json!({"code":code, "message":"/secret/host provider-secret", "denial_reason":reason});
|
||||
let server = thread::spawn(move || {
|
||||
reply(
|
||||
&listener,
|
||||
"200 OK",
|
||||
r#"{"session_id":"s","workdir_id":"w","capabilities":{"bits":63}}"#,
|
||||
);
|
||||
reply(&listener, "200 OK", &opened("w"));
|
||||
reply(
|
||||
&listener,
|
||||
&format!("{} Fixture failure", code.http_status()),
|
||||
@@ -216,3 +220,106 @@ async fn remote_checkout_diagnostics_preserve_mutation_outcome_and_read_failures
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn mismatched_http_results_keep_cause_and_mutation_uncertainty() {
|
||||
use workdir::{
|
||||
CheckoutOperation, CheckoutRequest, CommandHandle, CommandOutputRequest,
|
||||
WorkdirFailureReason as Failure,
|
||||
};
|
||||
for case in ["variant", "bytes", "mutation", "identity"] {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = thread::spawn(move || {
|
||||
reply(
|
||||
&listener,
|
||||
"200 OK",
|
||||
&opened(if case == "identity" {
|
||||
"wrong-secret"
|
||||
} else {
|
||||
"w"
|
||||
}),
|
||||
);
|
||||
if case != "identity" {
|
||||
let result = if case == "bytes" {
|
||||
serde_json::json!({"operation":"command_output","result":{
|
||||
"chunks": [], "next_cursor": 123, "terminal": null
|
||||
}})
|
||||
} else {
|
||||
serde_json::to_value(
|
||||
workdir::http::WorkdirSessionOperationResult::CommandCancel,
|
||||
)
|
||||
.unwrap()
|
||||
};
|
||||
reply(&listener, "200 OK", &result.to_string());
|
||||
}
|
||||
});
|
||||
let opened = RemoteWorkdirSession::open(
|
||||
reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.timeout(Duration::from_secs(5))
|
||||
.build()
|
||||
.unwrap(),
|
||||
format!("http://{address}").parse().unwrap(),
|
||||
"request-secret",
|
||||
Workdir::new("w").id().clone(),
|
||||
OpenWorkdirSessionRequest::default(),
|
||||
)
|
||||
.await;
|
||||
let error = if case == "identity" {
|
||||
opened.unwrap_err()
|
||||
} else {
|
||||
let remote = opened.unwrap();
|
||||
match case {
|
||||
"variant" => remote
|
||||
.stat(StatRequest {
|
||||
path: WorkdirPath::new("file-secret").unwrap(),
|
||||
})
|
||||
.await
|
||||
.unwrap_err(),
|
||||
"bytes" => remote
|
||||
.command_output(CommandOutputRequest {
|
||||
handle: CommandHandle("command-secret".into()),
|
||||
cursor: 0,
|
||||
limit: 16,
|
||||
wait: false,
|
||||
})
|
||||
.await
|
||||
.unwrap_err(),
|
||||
"mutation" => remote
|
||||
.checkout_execute(CheckoutRequest {
|
||||
target: WorkdirPath::new("file-secret").unwrap(),
|
||||
validator: vec![1],
|
||||
operation: CheckoutOperation::Write {
|
||||
content: b"body-secret".to_vec(),
|
||||
expected_hash: [0; 32],
|
||||
},
|
||||
})
|
||||
.await
|
||||
.unwrap_err(),
|
||||
_ => unreachable!(),
|
||||
}
|
||||
};
|
||||
server.join().unwrap();
|
||||
let expected = if case == "identity" {
|
||||
Failure::SessionIdentityMismatch
|
||||
} else {
|
||||
Failure::OperationResultMismatch
|
||||
};
|
||||
assert_eq!(error.failure_reason(), Some(expected), "{case}: {error:?}");
|
||||
let transport = WorkdirTransportError::from_workdir_error(&error);
|
||||
assert_eq!(
|
||||
transport.code,
|
||||
if case == "mutation" {
|
||||
Code::OutcomeUnknown
|
||||
} else {
|
||||
Code::Unavailable
|
||||
}
|
||||
);
|
||||
let wire = serde_json::to_string(&transport).unwrap();
|
||||
assert!(!wire.contains("secret"), "{wire}");
|
||||
assert!(!format!("{error:?}").contains("secret"));
|
||||
let forwarded = transport.into_workdir_error();
|
||||
assert_eq!(forwarded.failure_reason(), Some(expected));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,8 +19,6 @@ required-features = ["ws-server", "fs-store"]
|
||||
[features]
|
||||
default = ["ws-server", "fs-store"]
|
||||
fs-store = ["dep:fs4"]
|
||||
# Offline fixture tooling only; never enabled by the product defaults.
|
||||
test-support = ["fs-store", "ws-server"]
|
||||
http-server = ["dep:axum", "dep:tower", "dep:reqwest"]
|
||||
ws-server = ["http-server", "axum/ws", "dep:futures", "tokio/sync"]
|
||||
|
||||
@@ -58,6 +56,7 @@ server-api = { path = "../server-api" }
|
||||
workdir.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tools.workspace = true
|
||||
workdir = { workspace = true, features = ["http-client"] }
|
||||
futures.workspace = true
|
||||
agen.workspace = true
|
||||
|
||||
@@ -8,34 +8,3 @@ Manages Worker creation, stopping, and Restore, execution connections within the
|
||||
- [`execution`] — execution backends and operation-specific results.
|
||||
- [`working_directory`] — repository materialization, session resources, and safe deletion.
|
||||
- [`workspace_issuer`] — Workspace signature verification and replay prevention.
|
||||
|
||||
## Offline stopped-Worker fixtures
|
||||
|
||||
The nondefault `test-support` feature exposes `test_support::StoppedWorkerTemplate`.
|
||||
It is intentionally limited to current-schema, stopped, attachment-free artificial
|
||||
conversations with one Segment, text history, and no pending execution/activation.
|
||||
It does not add a product endpoint or change create/restore authorization.
|
||||
|
||||
1. With the source Runtime offline, call `StoppedWorkerTemplate::extract(worker_dir)`.
|
||||
The source must have its existing Runtime owner lock and identity. Extraction is
|
||||
read-only; the returned serde template excludes source auth, saved Manifest,
|
||||
request/config references, run directories, and Runtime operation history.
|
||||
2. Initialize **new** Runtime identity and Workspace trust in a fresh destination
|
||||
using normal setup. Preserve the template's Runtime and Workspace IDs.
|
||||
3. Call `template.install(&options, request, &resolved_manifest)` before starting
|
||||
the Runtime. Supply a fresh typed `CreateWorkerRequest` (including config/profile
|
||||
references and trusted memory settings) and `manifest::WorkerManifest` resolved
|
||||
for that environment. Its Worker name must equal `template.worker_name()`.
|
||||
4. Start normally and use the ordinary explicit restore operation. Installation
|
||||
itself creates no execution and leaves the Worker stopped with a fresh guard.
|
||||
|
||||
The template preserves the original Session manifest/JSONL bytes, including stable
|
||||
entry IDs, timestamps, and settled submission receipts. It rejects resumable queues
|
||||
and Feature invocation history. A saved system prompt is required for restore;
|
||||
checkpoint receipts must match the Worker's typed receipt schema. Preserved records
|
||||
must use canonical writer field shapes: unknown/duplicate fields (including nested
|
||||
fields), discarded optional fields, and opaque Feature history payloads are rejected,
|
||||
not stripped or rewritten. Portability/secret-marker checks do not replace
|
||||
review of the supplied artificial conversation or its independently verified hashes.
|
||||
The installer refuses any destination content other than initialized `auth.toml`;
|
||||
interrupted publication is fail-closed, not an overwrite/recovery facility.
|
||||
|
||||
@@ -39,10 +39,6 @@ use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU8, AtomicU64, Ordering};
|
||||
|
||||
#[cfg(feature = "test-support")]
|
||||
#[path = "test_support.rs"]
|
||||
pub mod test_support;
|
||||
|
||||
const SCHEMA_VERSION: u32 = 12;
|
||||
// Frozen source schemas: do not move their meaning when adding a migration.
|
||||
const SOURCE_SCHEMA_VERSION: u32 = 11;
|
||||
|
||||
@@ -869,6 +869,7 @@ async fn open_workdir_session(
|
||||
session_id: session_id.clone(),
|
||||
workdir_id: session.workdir().id().clone(),
|
||||
capabilities: session.capabilities(),
|
||||
environment: session.environment(),
|
||||
};
|
||||
state
|
||||
.workdir_sessions
|
||||
@@ -2187,7 +2188,25 @@ impl RuntimeHttpWorkdirError {
|
||||
}
|
||||
|
||||
fn runtime(error: RuntimeError) -> Self {
|
||||
match error {
|
||||
use workdir::WorkdirFailureReason as Reason;
|
||||
let reason = match &error {
|
||||
RuntimeError::WorkingDirectory(diagnostic) => match diagnostic.code.as_str() {
|
||||
"working_directory_not_found" => Reason::WorkingDirectoryNotFound,
|
||||
"working_directory_not_active" => Reason::WorkingDirectoryNotActive,
|
||||
"working_directory_record_invalid" => Reason::WorkingDirectoryRecordInvalid,
|
||||
"working_directory_repository_access_unavailable" => {
|
||||
Reason::WorkingDirectoryAccessUnavailable
|
||||
}
|
||||
_ => Reason::SessionOpenFailed,
|
||||
},
|
||||
RuntimeError::RuntimeStopped => Reason::RuntimeStopped,
|
||||
RuntimeError::StatePoisoned => Reason::RuntimeStateUnavailable,
|
||||
RuntimeError::ExecutionBackendUnavailable { .. } => Reason::RuntimeBackendUnavailable,
|
||||
RuntimeError::WorkspaceOwnerMismatch { .. } => Reason::RuntimeOwnerMismatch,
|
||||
RuntimeError::InvalidRequest(_) => Reason::RuntimeInvalidRequest,
|
||||
_ => Reason::SessionOpenFailed,
|
||||
};
|
||||
let mut response = match error {
|
||||
RuntimeError::WorkingDirectory(diagnostic)
|
||||
if diagnostic.code == "working_directory_not_found" =>
|
||||
{
|
||||
@@ -2200,7 +2219,9 @@ impl RuntimeHttpWorkdirError {
|
||||
WorkdirTransportErrorCode::Unavailable,
|
||||
"Workdir session is unavailable",
|
||||
),
|
||||
}
|
||||
};
|
||||
response.payload.failure_reason = Some(reason);
|
||||
response
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2222,6 +2243,7 @@ impl IntoResponse for RuntimeHttpWorkdirError {
|
||||
status = self.status.as_u16(),
|
||||
code = self.payload.code.as_str(),
|
||||
failure_reason = reason.as_str(),
|
||||
failure_stage = reason.stage().as_str(),
|
||||
"Workdir provider operation failed"
|
||||
);
|
||||
}
|
||||
@@ -2434,6 +2456,7 @@ pub enum RuntimeHttpServerError {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
mod command_transport_tests;
|
||||
mod runtime_workdir_failure_tests;
|
||||
#[cfg(feature = "fs-store")]
|
||||
mod session_material_tests;
|
||||
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
//! pre-seeded session is a fixture; no Runtime Worker/materializer is claimed here.
|
||||
|
||||
use super::*;
|
||||
use std::collections::BTreeMap;
|
||||
use std::time::Duration;
|
||||
use workdir::http::{
|
||||
RemoteWorkdirSession, WorkdirHttpAuthorization, WorkdirHttpAuthorizationFuture,
|
||||
@@ -70,14 +71,23 @@ impl Harness {
|
||||
}
|
||||
|
||||
async fn new() -> Self {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let session: WorkdirSessionHandle = Arc::new(LocalWorkdirSession::materialized_bound(
|
||||
Workdir::new("wd-transport"),
|
||||
root.path().to_path_buf(),
|
||||
root.path().to_path_buf(),
|
||||
SharedScope::new(Scope::writable(root.path()).unwrap()),
|
||||
WorkdirSessionCapabilities::ALL,
|
||||
));
|
||||
Self::with_environment(tempfile::tempdir().unwrap(), BTreeMap::new()).await
|
||||
}
|
||||
|
||||
async fn with_environment(
|
||||
root: tempfile::TempDir,
|
||||
command_environment: BTreeMap<String, String>,
|
||||
) -> Self {
|
||||
let session: WorkdirSessionHandle =
|
||||
Arc::new(LocalWorkdirSession::materialized_bound_with_environment(
|
||||
Workdir::new("wd-transport"),
|
||||
root.path().to_path_buf(),
|
||||
root.path().to_path_buf(),
|
||||
SharedScope::new(Scope::writable(root.path()).unwrap()),
|
||||
WorkdirSessionCapabilities::ALL,
|
||||
command_environment,
|
||||
Vec::new(),
|
||||
));
|
||||
let identity = RuntimeIdentityMaterial::generate("transport-key").unwrap();
|
||||
let trust = workspace_trust(&identity);
|
||||
let state = RuntimeHttpState {
|
||||
@@ -101,6 +111,9 @@ impl Harness {
|
||||
session_id: WorkdirSessionId::new("session-transport").unwrap(),
|
||||
workdir_id: Workdir::new("wd-transport").id().clone(),
|
||||
capabilities: WorkdirSessionCapabilities::ALL,
|
||||
environment: workdir::WorkdirSessionEnvironment::root(
|
||||
WorkdirSessionCapabilities::ALL,
|
||||
),
|
||||
})
|
||||
}),
|
||||
)
|
||||
@@ -222,6 +235,234 @@ async fn real_http_waiting_command_survives_long_poll_expiry() {
|
||||
.expect("bounded long-poll continuation over HTTP");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn real_http_spawn_failure_preserves_safe_cause_counts_and_terminal_replay() {
|
||||
tokio::time::timeout(Duration::from_secs(10), async {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let missing_path = root.path().join("private-spawn-path-must-not-exist");
|
||||
assert!(!missing_path.exists());
|
||||
let private_path = missing_path.to_str().unwrap().to_owned();
|
||||
let environment_secret = "spawn-environment-secret-must-not-leak";
|
||||
let command_secret = "spawn-command-secret-must-not-leak";
|
||||
let tool_call_secret = "spawn-tool-call-secret-must-not-leak";
|
||||
// This is an OS spawn failure in the production local provider, not a
|
||||
// fabricated terminal: PATH contains no bash. The test process environment
|
||||
// is untouched, so parallel command tests retain their normal PATH.
|
||||
let harness = Harness::with_environment(
|
||||
root,
|
||||
BTreeMap::from([
|
||||
("PATH".into(), private_path.clone()),
|
||||
("PRIVATE_AUTH_TOKEN".into(), environment_secret.into()),
|
||||
]),
|
||||
)
|
||||
.await;
|
||||
let command = format!("printf '{command_secret}'; printf ran > must-not-run");
|
||||
let handle = harness
|
||||
.remote
|
||||
.start_command(CommandRequest {
|
||||
transport_byte_budget: 1024,
|
||||
command: command.clone(),
|
||||
timeout_secs: 5,
|
||||
cwd: WorkdirPath::root(),
|
||||
tool_call_id: Some(tool_call_secret.into()),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let request = CommandOutputRequest {
|
||||
handle,
|
||||
cursor: 0,
|
||||
limit: 16,
|
||||
wait: true,
|
||||
};
|
||||
let page = harness.remote.command_output(request.clone()).await.unwrap();
|
||||
assert!(page.chunks.is_empty());
|
||||
assert_eq!(page.next_cursor, 0);
|
||||
let terminal = page.terminal.as_ref().expect("spawn failure is terminal");
|
||||
assert_eq!(terminal.status, workdir::CommandStatus::Failed);
|
||||
assert_eq!(terminal.exit_code, None);
|
||||
assert_eq!(
|
||||
terminal.process_failure,
|
||||
Some(workdir::CommandFailure {
|
||||
stage: workdir::CommandFailureStage::Spawn,
|
||||
reason: workdir::CommandIoErrorKind::NotFound,
|
||||
os_code: Some(libc::ENOENT),
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
terminal.output_collection,
|
||||
workdir::CommandOutputCollection::NotStarted
|
||||
);
|
||||
assert_eq!(terminal.stdout_bytes, 0);
|
||||
assert_eq!(terminal.stderr_bytes, 0);
|
||||
assert_eq!(terminal.stdout_delivered_bytes, 0);
|
||||
assert_eq!(terminal.stderr_delivered_bytes, 0);
|
||||
assert_eq!(terminal.truncation_reason, None);
|
||||
|
||||
// A zero-byte terminal page remains replayable at its unchanged cursor;
|
||||
// neither a retry nor a smaller nonwaiting read changes its accounting.
|
||||
assert_eq!(
|
||||
harness.remote.command_output(request.clone()).await.unwrap(),
|
||||
page
|
||||
);
|
||||
let replay_request = CommandOutputRequest {
|
||||
wait: false,
|
||||
limit: 1,
|
||||
..request
|
||||
};
|
||||
assert_eq!(
|
||||
harness
|
||||
.remote
|
||||
.command_output(replay_request.clone())
|
||||
.await
|
||||
.unwrap(),
|
||||
page
|
||||
);
|
||||
let response = harness
|
||||
.raw_operation(
|
||||
serde_json::to_value(WorkdirSessionOperation::CommandOutput(
|
||||
replay_request.clone(),
|
||||
))
|
||||
.unwrap(),
|
||||
)
|
||||
.await;
|
||||
assert_eq!(response.status(), reqwest::StatusCode::OK);
|
||||
let body = response.text().await.unwrap();
|
||||
for secret in [
|
||||
private_path.as_str(),
|
||||
harness._root.path().to_str().unwrap(),
|
||||
"private-spawn-path-must-not-exist",
|
||||
"PRIVATE_AUTH_TOKEN",
|
||||
environment_secret,
|
||||
command_secret,
|
||||
command.as_str(),
|
||||
tool_call_secret,
|
||||
] {
|
||||
assert!(
|
||||
!body.contains(secret),
|
||||
"private input leaked into HTTP terminal"
|
||||
);
|
||||
}
|
||||
let wire: serde_json::Value = serde_json::from_str(&body).unwrap();
|
||||
assert_eq!(
|
||||
wire,
|
||||
serde_json::json!({
|
||||
"operation": "command_output",
|
||||
"result": {
|
||||
"chunks": [],
|
||||
"next_cursor": 0,
|
||||
"terminal": {
|
||||
"status": "failed",
|
||||
"exit_code": null,
|
||||
"process_failure": {"stage": "spawn", "reason": "not_found", "os_code": libc::ENOENT},
|
||||
"output_collection": {"state": "not_started"},
|
||||
"stdout_bytes": 0,
|
||||
"stderr_bytes": 0,
|
||||
"stdout_delivered_bytes": 0,
|
||||
"stderr_delivered_bytes": 0,
|
||||
"truncation_reason": null
|
||||
}
|
||||
}
|
||||
})
|
||||
);
|
||||
assert_eq!(
|
||||
harness.remote.command_output(replay_request).await.unwrap(),
|
||||
page
|
||||
);
|
||||
// Exercise the same real spawn failure through Bash, not a terminal
|
||||
// fixture. Opening another remote client reuses the pre-seeded provider
|
||||
// session and still passes through the signed HTTP middleware.
|
||||
let bash_remote: WorkdirSessionHandle = Arc::new(
|
||||
RemoteWorkdirSession::open_with_authorization(
|
||||
harness.client.clone(),
|
||||
harness.operation_url.join("/").unwrap(),
|
||||
harness.authorization.clone(),
|
||||
Workdir::new("wd-transport").id().clone(),
|
||||
OpenWorkdirSessionRequest::default(),
|
||||
)
|
||||
.await
|
||||
.unwrap(),
|
||||
);
|
||||
let view = tools::bash_output::BashOutputView::new();
|
||||
let (_, bash) = tools::bash_tool(bash_remote, view.clone())();
|
||||
let input = serde_json::json!({
|
||||
"command": command,
|
||||
"cwd": ".",
|
||||
"timeout": 5
|
||||
})
|
||||
.to_string();
|
||||
let output = bash
|
||||
.execute(
|
||||
&input,
|
||||
agen::tool::ToolExecutionContext::new(tool_call_secret, "http-spawn-attempt", 0),
|
||||
)
|
||||
.await
|
||||
.expect("Bash returns a structured failed-command result");
|
||||
let content = output.content.as_deref().expect("Bash ToolOutput body");
|
||||
let tool_body: serde_json::Value = serde_json::from_str(content).unwrap();
|
||||
assert_eq!(tool_body["stdout_preview"], "");
|
||||
assert_eq!(tool_body["stderr_preview"], "");
|
||||
assert_eq!(tool_body["preview_omitted"], false);
|
||||
assert_eq!(
|
||||
tool_body["output"]["terminal"],
|
||||
serde_json::json!({
|
||||
"status": "failed",
|
||||
"exit_code": null,
|
||||
"process_failure": {"stage": "spawn", "reason": "not_found", "os_code": libc::ENOENT},
|
||||
"output_collection": {"state": "not_started"},
|
||||
"transport_complete": false
|
||||
})
|
||||
);
|
||||
let output_id = tool_body["output"]["id"].as_str().unwrap();
|
||||
let retained = view.metadata(output_id).expect("retained spawn failure");
|
||||
assert_eq!(retained.terminal.status, Some(workdir::CommandStatus::Failed));
|
||||
assert_eq!(retained.terminal.exit_code, None);
|
||||
assert_eq!(retained.terminal.process_failure, terminal.process_failure);
|
||||
assert_eq!(
|
||||
retained.terminal.output_collection,
|
||||
Some(workdir::CommandOutputCollection::NotStarted)
|
||||
);
|
||||
assert!(!retained.terminal.transport_complete);
|
||||
assert_eq!(retained.stdout_saved, 0);
|
||||
assert_eq!(retained.stderr_saved, 0);
|
||||
assert_eq!(retained.stdout_seen, 0);
|
||||
assert_eq!(retained.stderr_seen, 0);
|
||||
assert_eq!(retained.stdout_delivered, 0);
|
||||
assert_eq!(retained.stderr_delivered, 0);
|
||||
assert!(!retained.storage_truncated);
|
||||
assert!(retained.retention_limit.is_none());
|
||||
assert_eq!(retained.transport_truncation, None);
|
||||
let retained_wire = serde_json::to_value(&retained).unwrap();
|
||||
assert_eq!(retained_wire, tool_body["output"]);
|
||||
for stream in [CommandStream::Stdout, CommandStream::Stderr] {
|
||||
assert_eq!(view.read(output_id, stream, 0, 1), Some(Vec::new()));
|
||||
}
|
||||
for secret in [
|
||||
private_path.as_str(),
|
||||
harness._root.path().to_str().unwrap(),
|
||||
"private-spawn-path-must-not-exist",
|
||||
"PRIVATE_AUTH_TOKEN",
|
||||
environment_secret,
|
||||
command_secret,
|
||||
command.as_str(),
|
||||
tool_call_secret,
|
||||
] {
|
||||
assert!(
|
||||
!content.contains(secret),
|
||||
"private input leaked into Bash ToolOutput body"
|
||||
);
|
||||
}
|
||||
assert!(!harness._root.path().join("must-not-run").exists());
|
||||
harness.remote.close().await.unwrap();
|
||||
assert!(harness.session.command_snapshot().is_empty());
|
||||
assert_eq!(
|
||||
serde_json::to_value(view.metadata(output_id).unwrap()).unwrap(),
|
||||
retained_wire
|
||||
);
|
||||
})
|
||||
.await
|
||||
.expect("bounded real spawn failure over signed HTTP");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn real_http_closed_session_retains_failure_reason() {
|
||||
let harness = Harness::new().await;
|
||||
@@ -291,6 +532,11 @@ async fn real_http_command_pages_preserve_binary_streams_replay_and_nonzero_exit
|
||||
assert_eq!(stderr, b"stderr\xe6\x97\xa5");
|
||||
assert_eq!(terminal.status, workdir::CommandStatus::Failed);
|
||||
assert_eq!(terminal.exit_code, Some(9));
|
||||
assert_eq!(terminal.process_failure, None);
|
||||
assert_eq!(
|
||||
terminal.output_collection,
|
||||
workdir::CommandOutputCollection::Complete
|
||||
);
|
||||
assert!(harness.remote.command_output(CommandOutputRequest { handle, cursor: 0, limit: 1, wait: false }).await.is_err());
|
||||
harness.remote.close().await.unwrap();
|
||||
assert!(harness.session.command_snapshot().is_empty());
|
||||
@@ -435,6 +681,11 @@ async fn real_http_command_budget_bounds_unique_payload_and_reports_larger_seen_
|
||||
);
|
||||
assert_eq!(terminal.status, workdir::CommandStatus::Failed);
|
||||
assert_eq!(terminal.exit_code, Some(9));
|
||||
assert_eq!(terminal.process_failure, None);
|
||||
assert_eq!(
|
||||
terminal.output_collection,
|
||||
workdir::CommandOutputCollection::Complete
|
||||
);
|
||||
assert!(
|
||||
harness
|
||||
.remote
|
||||
|
||||
@@ -0,0 +1,528 @@
|
||||
//! Session-open integration through the production signed router, Runtime owner/binding
|
||||
//! checks, HTTP DTO and tracing boundary. Only the execution backend is replaced
|
||||
//! for deterministic lookup/open faults and inert owner-binding Worker fixtures;
|
||||
//! no handler, middleware or grant is replaced. Runs in the normal http-server lib
|
||||
//! suite with isolated loopback sockets.
|
||||
|
||||
use super::*;
|
||||
use crate::working_directory::WorkingDirectoryDiagnostic;
|
||||
use std::io::{self, Write};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::time::Duration;
|
||||
|
||||
const OPEN_PATH: &str = "/v1/working-directories/private-workdir-marker/sessions";
|
||||
const PRIVATE_DIAGNOSTIC: &str = concat!(
|
||||
"raw-runtime-error-marker /private/host-path-marker ",
|
||||
"command=private-command-marker contents=private-file-marker ",
|
||||
"proof=private-proof-marker Authorization: Bearer private-header-marker ",
|
||||
"working_directory_not_found authorization_stale runtime_stopped",
|
||||
);
|
||||
const PRIVATE_MARKERS: &[&str] = &[
|
||||
"raw-runtime-error-marker",
|
||||
"/private/host-path-marker",
|
||||
"private-command-marker",
|
||||
"private-file-marker",
|
||||
"private-proof-marker",
|
||||
"private-header-marker",
|
||||
"private-workdir-marker",
|
||||
"private-owner-marker",
|
||||
];
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct LogCapture(Arc<Mutex<Vec<u8>>>);
|
||||
|
||||
impl Write for LogCapture {
|
||||
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
||||
self.0.lock().unwrap().extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
impl LogCapture {
|
||||
fn subscriber(&self) -> impl tracing::Subscriber + Send + Sync + 'static {
|
||||
let writer = self.clone();
|
||||
tracing_subscriber::fmt()
|
||||
.with_ansi(false)
|
||||
.without_time()
|
||||
.with_max_level(tracing::Level::WARN)
|
||||
.with_writer(move || writer.clone())
|
||||
.finish()
|
||||
}
|
||||
|
||||
fn take(&self) -> String {
|
||||
String::from_utf8(std::mem::take(&mut *self.0.lock().unwrap())).unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_private_absent(text: &str) {
|
||||
for marker in PRIVATE_MARKERS {
|
||||
assert!(
|
||||
!text.contains(marker),
|
||||
"private input escaped error boundary"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_failure_log(log: &str, status: StatusCode, code: &str, reason: &str, stage: &str) {
|
||||
let events: Vec<_> = log
|
||||
.lines()
|
||||
.filter(|line| line.contains("workdir_provider_failure"))
|
||||
.collect();
|
||||
assert_eq!(events.len(), 1, "one diagnostic per failed request: {log}");
|
||||
let event = events[0];
|
||||
for field in [
|
||||
"WARN".to_string(),
|
||||
"worker_runtime::workdir".to_string(),
|
||||
format!("status={}", status.as_u16()),
|
||||
format!("code=\"{code}\""),
|
||||
format!("failure_reason=\"{reason}\""),
|
||||
format!("failure_stage=\"{stage}\""),
|
||||
] {
|
||||
assert!(event.contains(&field), "missing {field}: {event}");
|
||||
}
|
||||
assert_private_absent(log);
|
||||
}
|
||||
|
||||
fn assert_failure_body(body: &str, code: &str, message: &str, reason: &str) {
|
||||
assert_private_absent(body);
|
||||
// Exact safe transport vocabulary; raw diagnostics and stage are not wire fields.
|
||||
assert_eq!(
|
||||
serde_json::from_str::<serde_json::Value>(body).unwrap(),
|
||||
serde_json::json!({"code": code, "message": message, "failure_reason": reason}),
|
||||
);
|
||||
}
|
||||
|
||||
struct Harness {
|
||||
server: tokio::task::JoinHandle<()>,
|
||||
client: reqwest::Client,
|
||||
base: String,
|
||||
identity: RuntimeIdentityMaterial,
|
||||
trust: WorkspaceIssuerTrustRecord,
|
||||
}
|
||||
|
||||
impl Drop for Harness {
|
||||
fn drop(&mut self) {
|
||||
self.server.abort();
|
||||
}
|
||||
}
|
||||
|
||||
impl Harness {
|
||||
async fn new(runtime: Runtime) -> Self {
|
||||
let identity = RuntimeIdentityMaterial::generate("runtime-failure-test-key").unwrap();
|
||||
let trust = workspace_trust(&identity);
|
||||
let app = runtime_http_router_with_workspace_auth(
|
||||
runtime,
|
||||
None,
|
||||
workspace_test_auth(vec![trust.clone()]),
|
||||
);
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let base = format!("http://{}", listener.local_addr().unwrap());
|
||||
let server = tokio::spawn(async move {
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
});
|
||||
Self {
|
||||
server,
|
||||
client: reqwest::Client::builder()
|
||||
.no_proxy()
|
||||
.timeout(Duration::from_secs(5))
|
||||
.build()
|
||||
.unwrap(),
|
||||
base,
|
||||
identity,
|
||||
trust,
|
||||
}
|
||||
}
|
||||
|
||||
async fn send(
|
||||
&self,
|
||||
claims: &WorkspaceCapabilityClaims,
|
||||
body: Vec<u8>,
|
||||
workspace: &str,
|
||||
) -> (StatusCode, String, String) {
|
||||
let token = issue_workspace_capability_token(&self.identity.signing_key().unwrap(), claims)
|
||||
.unwrap();
|
||||
let response = self
|
||||
.client
|
||||
.post(format!("{}{OPEN_PATH}", self.base))
|
||||
.bearer_auth(&token)
|
||||
.header(header::CONTENT_TYPE, "application/json")
|
||||
.header(RUNTIME_WORKSPACE_SCOPE_HEADER, workspace)
|
||||
.header("x-private-test-header", "private-header-marker")
|
||||
.body(body)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
let status = StatusCode::from_u16(response.status().as_u16()).unwrap();
|
||||
let headers = format!("{:?}", response.headers());
|
||||
let body = response.text().await.unwrap();
|
||||
assert_private_absent(&headers);
|
||||
for output in [&body, &headers] {
|
||||
assert!(!output.contains(&token), "signed bearer token leaked");
|
||||
assert!(
|
||||
!output.contains(&self.identity.private_key),
|
||||
"signing key leaked"
|
||||
);
|
||||
}
|
||||
(status, body, token)
|
||||
}
|
||||
|
||||
async fn open(&self) -> (StatusCode, String, String) {
|
||||
let body = serde_json::to_vec(&OpenWorkdirSessionRequest::default()).unwrap();
|
||||
let claims = signed_claims(&self.trust, "POST", OPEN_PATH, &body);
|
||||
self.send(&claims, body, "workspace-a").await
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone, Copy)]
|
||||
enum FailureAt {
|
||||
Lookup,
|
||||
Open,
|
||||
}
|
||||
|
||||
struct FailingBackend {
|
||||
at: FailureAt,
|
||||
code: &'static str,
|
||||
lookups: AtomicUsize,
|
||||
opens: AtomicUsize,
|
||||
}
|
||||
|
||||
impl FailingBackend {
|
||||
fn runtime(at: FailureAt, code: &'static str) -> (Runtime, Arc<Self>) {
|
||||
let backend = Arc::new(Self {
|
||||
at,
|
||||
code,
|
||||
lookups: AtomicUsize::new(0),
|
||||
opens: AtomicUsize::new(0),
|
||||
});
|
||||
let runtime =
|
||||
Runtime::with_execution_backend(RuntimeOptions::default(), backend.clone()).unwrap();
|
||||
(runtime, backend)
|
||||
}
|
||||
|
||||
fn failure(&self) -> WorkingDirectoryDiagnostic {
|
||||
WorkingDirectoryDiagnostic::rejected(self.code, PRIVATE_DIAGNOSTIC)
|
||||
}
|
||||
}
|
||||
|
||||
impl WorkerExecutionBackend for FailingBackend {
|
||||
fn backend_id(&self) -> &str {
|
||||
"runtime-failure-test"
|
||||
}
|
||||
|
||||
fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult {
|
||||
AcceptingBackend.spawn_worker(request)
|
||||
}
|
||||
|
||||
fn dispatch_input(&self, worker: &WorkerRef, input: WorkerInput) -> WorkerExecutionResult {
|
||||
AcceptingBackend.dispatch_input(worker, input)
|
||||
}
|
||||
|
||||
fn stop_worker(&self, worker: &WorkerRef) -> WorkerExecutionResult {
|
||||
AcceptingBackend.stop_worker(worker)
|
||||
}
|
||||
|
||||
fn working_directory(
|
||||
&self,
|
||||
id: &str,
|
||||
) -> Result<WorkingDirectoryStatus, WorkingDirectoryDiagnostic> {
|
||||
self.lookups.fetch_add(1, Ordering::SeqCst);
|
||||
match self.at {
|
||||
FailureAt::Lookup => Err(self.failure()),
|
||||
FailureAt::Open => Ok(WorkingDirectoryStatus {
|
||||
summary: crate::catalog::WorkingDirectorySummary {
|
||||
working_directory_id: id.into(),
|
||||
display_name: None,
|
||||
repository_id: "repository-test".into(),
|
||||
creation_selector: None,
|
||||
creation_ref: None,
|
||||
creation_tree: None,
|
||||
current_selector: None,
|
||||
current_ref: None,
|
||||
current_tree: None,
|
||||
observed_at_epoch_seconds: None,
|
||||
materializer_kind: crate::catalog::MaterializerKind::RuntimeGitClone,
|
||||
cleanup_target: None,
|
||||
status: crate::catalog::WorkingDirectoryStatusKind::Active,
|
||||
cleanliness: None,
|
||||
occupied_by: None,
|
||||
},
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn open_workdir_session(
|
||||
&self,
|
||||
_: &str,
|
||||
) -> Result<WorkdirSessionHandle, WorkingDirectoryDiagnostic> {
|
||||
self.opens.fetch_add(1, Ordering::SeqCst);
|
||||
Err(self.failure())
|
||||
}
|
||||
}
|
||||
|
||||
// Current-thread execution keeps the server tasks inside the thread-local tracing
|
||||
// capture across awaits, without replacing the process-global subscriber.
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn signed_http_session_open_preserves_runtime_availability_and_owner_failures() {
|
||||
let capture = LogCapture::default();
|
||||
let _guard = tracing::subscriber::set_default(capture.subscriber());
|
||||
tracing::callsite::rebuild_interest_cache();
|
||||
tokio::time::timeout(Duration::from_secs(15), async {
|
||||
let stopped = Runtime::new_memory();
|
||||
stopped.stop_runtime().unwrap();
|
||||
let (owned, backend) = FailingBackend::runtime(FailureAt::Lookup, "must_not_reach_backend");
|
||||
let mut request = task_request("bind owner");
|
||||
request.config_bundle = None;
|
||||
request.memory_settings.as_mut().unwrap().workspace_id = "workspace-a".into();
|
||||
owned
|
||||
.create_worker_scoped(
|
||||
&RuntimeWorkspaceScope::new("workspace-a", "private-owner-marker"),
|
||||
request,
|
||||
)
|
||||
.unwrap();
|
||||
for (runtime, status, code, message, reason) in [
|
||||
(
|
||||
stopped,
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"unavailable",
|
||||
"Workdir session is unavailable",
|
||||
"runtime_stopped",
|
||||
),
|
||||
(
|
||||
Runtime::new_memory(),
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"unavailable",
|
||||
"Workdir session is unavailable",
|
||||
"runtime_backend_unavailable",
|
||||
),
|
||||
(
|
||||
owned,
|
||||
StatusCode::NOT_FOUND,
|
||||
"not_found",
|
||||
"Workdir session was not found",
|
||||
"runtime_owner_mismatch",
|
||||
),
|
||||
] {
|
||||
let harness = Harness::new(runtime).await;
|
||||
capture.take();
|
||||
let (actual, body, token) = harness.open().await;
|
||||
assert_eq!(actual, status);
|
||||
assert_failure_body(&body, code, message, reason);
|
||||
let log = capture.take();
|
||||
assert_failure_log(&log, status, code, reason, "runtime");
|
||||
assert!(!log.contains(&token));
|
||||
assert!(!log.contains(&harness.identity.private_key));
|
||||
}
|
||||
assert_eq!(backend.lookups.load(Ordering::SeqCst), 0);
|
||||
assert_eq!(backend.opens.load(Ordering::SeqCst), 0);
|
||||
})
|
||||
.await
|
||||
.expect("bounded Runtime failure HTTP requests");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn signed_http_session_open_allowlists_diagnostic_codes_without_guessing_unknown_causes() {
|
||||
let capture = LogCapture::default();
|
||||
let _guard = tracing::subscriber::set_default(capture.subscriber());
|
||||
tracing::callsite::rebuild_interest_cache();
|
||||
tokio::time::timeout(Duration::from_secs(20), async {
|
||||
for (diagnostic, reason) in [
|
||||
("working_directory_not_found", "working_directory_not_found"),
|
||||
(
|
||||
"working_directory_not_active",
|
||||
"working_directory_not_active",
|
||||
),
|
||||
(
|
||||
"working_directory_record_invalid",
|
||||
"working_directory_record_invalid",
|
||||
),
|
||||
(
|
||||
"working_directory_repository_access_unavailable",
|
||||
"working_directory_access_unavailable",
|
||||
),
|
||||
// Plausible provider text/code is not evidence of a typed cause.
|
||||
("authorization_stale", "session_open_failed"),
|
||||
(
|
||||
"working_directory_not_found_private-proof-marker",
|
||||
"session_open_failed",
|
||||
),
|
||||
] {
|
||||
for at in [FailureAt::Lookup, FailureAt::Open] {
|
||||
let (runtime, backend) = FailingBackend::runtime(at, diagnostic);
|
||||
let harness = Harness::new(runtime).await;
|
||||
capture.take();
|
||||
let (status, body, token) = harness.open().await;
|
||||
let (expected, code, message) = if diagnostic == "working_directory_not_found" {
|
||||
(
|
||||
StatusCode::NOT_FOUND,
|
||||
"not_found",
|
||||
"Workdir session was not found",
|
||||
)
|
||||
} else {
|
||||
(
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"unavailable",
|
||||
"Workdir session is unavailable",
|
||||
)
|
||||
};
|
||||
assert_eq!(status, expected, "{diagnostic}");
|
||||
assert_failure_body(&body, code, message, reason);
|
||||
let log = capture.take();
|
||||
assert_failure_log(&log, status, code, reason, "session_open");
|
||||
assert!(!log.contains(&token));
|
||||
assert!(!log.contains(&harness.identity.private_key));
|
||||
assert_eq!(backend.lookups.load(Ordering::SeqCst), 1, "no hidden retry");
|
||||
assert_eq!(
|
||||
backend.opens.load(Ordering::SeqCst),
|
||||
usize::from(matches!(at, FailureAt::Open)),
|
||||
);
|
||||
}
|
||||
}
|
||||
})
|
||||
.await
|
||||
.expect("bounded diagnostic fault matrix");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn signed_http_session_open_rejects_bad_capability_scope_and_owner_binding_before_backend() {
|
||||
let capture = LogCapture::default();
|
||||
let _guard = tracing::subscriber::set_default(capture.subscriber());
|
||||
tracing::callsite::rebuild_interest_cache();
|
||||
tokio::time::timeout(Duration::from_secs(15), async {
|
||||
let (runtime, backend) =
|
||||
FailingBackend::runtime(FailureAt::Open, "working_directory_record_invalid");
|
||||
let mut request = task_request("foreign workspace binding");
|
||||
request.config_bundle = None;
|
||||
request.memory_settings.as_mut().unwrap().workspace_id = "other-workspace".into();
|
||||
let foreign_worker = runtime
|
||||
.create_worker_scoped(
|
||||
&RuntimeWorkspaceScope::new("other-workspace", "https://backend.test"),
|
||||
request,
|
||||
)
|
||||
.unwrap()
|
||||
.worker_ref;
|
||||
let harness = Harness::new(runtime).await;
|
||||
for case in ["permission", "body", "workspace", "owner_binding"] {
|
||||
let body = if case == "owner_binding" {
|
||||
serde_json::to_vec(&OpenWorkdirSessionRequest {
|
||||
owner_worker_id: Some(foreign_worker.worker_id.to_string()),
|
||||
})
|
||||
.unwrap()
|
||||
} else {
|
||||
b"{}".to_vec()
|
||||
};
|
||||
let mut claims = signed_claims(&harness.trust, "POST", OPEN_PATH, &body);
|
||||
if case == "permission" {
|
||||
claims.operation = "workers:read".into();
|
||||
}
|
||||
if case == "workspace" {
|
||||
claims.issuer_workspace_id = "other-workspace".into();
|
||||
}
|
||||
let sent = if case == "body" {
|
||||
b"{ }".to_vec()
|
||||
} else {
|
||||
body
|
||||
};
|
||||
capture.take();
|
||||
let (status, body, token) = harness.send(&claims, sent, "workspace-a").await;
|
||||
assert_eq!(
|
||||
status,
|
||||
if case == "owner_binding" {
|
||||
StatusCode::NOT_FOUND
|
||||
} else {
|
||||
StatusCode::UNAUTHORIZED
|
||||
},
|
||||
"{case}",
|
||||
);
|
||||
assert_private_absent(&body);
|
||||
let log = capture.take();
|
||||
assert_private_absent(&log);
|
||||
assert!(!log.contains(&token));
|
||||
assert!(!log.contains(&harness.identity.private_key));
|
||||
if case == "owner_binding" {
|
||||
assert_failure_body(
|
||||
&body,
|
||||
"not_found",
|
||||
"Workdir session was not found",
|
||||
"working_directory_not_found",
|
||||
);
|
||||
assert_failure_log(
|
||||
&log,
|
||||
status,
|
||||
"not_found",
|
||||
"working_directory_not_found",
|
||||
"session_open",
|
||||
);
|
||||
} else {
|
||||
let wire: serde_json::Value = serde_json::from_str(&body).unwrap();
|
||||
assert_eq!(wire["error"]["code"], "unauthorized");
|
||||
assert!(
|
||||
!log.contains("workdir_provider_failure"),
|
||||
"auth rejection is not a provider fault",
|
||||
);
|
||||
}
|
||||
}
|
||||
assert_eq!(backend.lookups.load(Ordering::SeqCst), 0);
|
||||
assert_eq!(backend.opens.load(Ordering::SeqCst), 0);
|
||||
})
|
||||
.await
|
||||
.expect("bounded authorization rejection requests");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "current_thread")]
|
||||
async fn runtime_error_response_keeps_raw_details_private_and_unclassified_causes_generic() {
|
||||
let capture = LogCapture::default();
|
||||
let _guard = tracing::subscriber::set_default(capture.subscriber());
|
||||
tracing::callsite::rebuild_interest_cache();
|
||||
for (error, status, code, message, reason, stage) in [
|
||||
(
|
||||
RuntimeError::ExecutionBackendUnavailable {
|
||||
message: PRIVATE_DIAGNOSTIC.into(),
|
||||
},
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"unavailable",
|
||||
"Workdir session is unavailable",
|
||||
"runtime_backend_unavailable",
|
||||
"runtime",
|
||||
),
|
||||
(
|
||||
RuntimeError::StatePoisoned,
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"unavailable",
|
||||
"Workdir session is unavailable",
|
||||
"runtime_state_unavailable",
|
||||
"runtime",
|
||||
),
|
||||
(
|
||||
RuntimeError::InvalidRequest(PRIVATE_DIAGNOSTIC.into()),
|
||||
StatusCode::BAD_REQUEST,
|
||||
"invalid_request",
|
||||
"Workdir operation request is invalid",
|
||||
"runtime_invalid_request",
|
||||
"runtime",
|
||||
),
|
||||
(
|
||||
RuntimeError::StoreCorrupt {
|
||||
operation: "session_open",
|
||||
path: PathBuf::from("/private/host-path-marker"),
|
||||
message: PRIVATE_DIAGNOSTIC.into(),
|
||||
},
|
||||
StatusCode::SERVICE_UNAVAILABLE,
|
||||
"unavailable",
|
||||
"Workdir session is unavailable",
|
||||
"session_open_failed",
|
||||
"session_open",
|
||||
),
|
||||
] {
|
||||
capture.take();
|
||||
let response = RuntimeHttpWorkdirError::runtime(error).into_response();
|
||||
assert_eq!(response.status(), status);
|
||||
let bytes = to_bytes(response.into_body(), 16384).await.unwrap();
|
||||
let body = std::str::from_utf8(&bytes).unwrap();
|
||||
assert_failure_body(body, code, message, reason);
|
||||
assert_failure_log(&capture.take(), status, code, reason, stage);
|
||||
}
|
||||
}
|
||||
@@ -32,8 +32,6 @@ pub mod workspace_issuer;
|
||||
#[cfg(feature = "http-server")]
|
||||
pub mod workspace_request;
|
||||
|
||||
#[cfg(feature = "test-support")]
|
||||
pub use fs_store::test_support;
|
||||
#[cfg(feature = "fs-store")]
|
||||
pub use fs_store::{FsRuntimeStore, FsRuntimeStoreOptions};
|
||||
pub use management::RuntimeOptions;
|
||||
|
||||
@@ -682,8 +682,10 @@ fn runtime_v5_rejects_broken_or_future_source_without_rewriting_evidence() {
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn current_runtime_session_migration_still_rejects_live_worker_sockets() {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let root = temp.path().join("runtime");
|
||||
// Relative socket names keep this fixture independent of the checkout/TMPDIR
|
||||
// length. The TempDir still owns the same tree; no process cwd is changed.
|
||||
let temp = tempfile::tempdir_in(".").unwrap();
|
||||
let root = Path::new(temp.path().file_name().unwrap()).join("runtime");
|
||||
let (worker, _, _) = runtime_fixture(&root, 12);
|
||||
let before = tree_bytes(&root);
|
||||
let runs = root.join("workers").join(worker.to_string()).join("runs");
|
||||
|
||||
@@ -55,6 +55,10 @@ impl WorkdirSession for RetainedWorkdirSession {
|
||||
self.inner.capabilities()
|
||||
}
|
||||
|
||||
fn environment(&self) -> workdir::WorkdirSessionEnvironment {
|
||||
self.inner.environment()
|
||||
}
|
||||
|
||||
async fn authorize_scope_path(
|
||||
&self,
|
||||
request: WorkdirScopeAuthorizationRequest,
|
||||
|
||||
@@ -1,859 +0,0 @@
|
||||
//! Offline support for the approved, attachment-free stopped Worker fixture.
|
||||
//!
|
||||
//! Enabled only by `test-support`. This is not an import API: extraction accepts a
|
||||
//! current-schema, settled, single-Segment aggregate; installation accepts only a
|
||||
//! fresh Runtime directory containing its newly initialized `auth.toml`. No
|
||||
//! Runtime or execution backend is constructed. Original Session bytes are never
|
||||
//! rewritten. The caller must separately review/attest the artificial conversation;
|
||||
//! structural checks and secret-marker checks cannot prove arbitrary prose safe.
|
||||
|
||||
use super::*;
|
||||
use crate::catalog::ProfileSourceArchiveSource;
|
||||
use crate::workspace_issuer::{
|
||||
WorkspaceIssuerTrustRecord, WorkspaceIssuerTrustState, validate_workspace_issuer_trust_records,
|
||||
};
|
||||
use session_store::{
|
||||
SegmentId, SessionId, WorkerActiveSegmentRef, WorkerMetadata, WorkerSessionStore,
|
||||
};
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "test_support_tests.rs"]
|
||||
mod tests;
|
||||
|
||||
const TEMPLATE_VERSION: u32 = 1;
|
||||
const MAX_SESSION_BYTES: usize = 8 * 1024 * 1024;
|
||||
|
||||
/// Portable fixture, intentionally without Debug (Session material is private).
|
||||
/// Fields are private and every deserialized template is validated on install.
|
||||
/// Serialize this value to retain the fixture; do not serialize its source aggregate.
|
||||
#[derive(Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct StoppedWorkerTemplate {
|
||||
template_version: u32,
|
||||
runtime_schema_version: u32,
|
||||
runtime_id: String,
|
||||
workspace_id: String,
|
||||
worker_id: WorkerId,
|
||||
created_at_ms: Option<u64>,
|
||||
session_id: SessionId,
|
||||
segment_id: SegmentId,
|
||||
session_manifest: Vec<u8>,
|
||||
segment_jsonl: Vec<u8>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct SessionManifest {
|
||||
schema_version: u32,
|
||||
session_id: SessionId,
|
||||
}
|
||||
|
||||
// Read-only projection: never retain source key material in a template.
|
||||
#[derive(Deserialize)]
|
||||
struct AuthIdentity {
|
||||
identity_id: String,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct SourceAuth {
|
||||
identity: AuthIdentity,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct InitializedAuth {
|
||||
identity: crate::auth::RuntimeIdentityMaterial,
|
||||
workspace_issuers: Vec<WorkspaceIssuerTrustRecord>,
|
||||
#[serde(default)]
|
||||
archived_legacy_issuers: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct GuardState {
|
||||
active_request_id: Option<String>,
|
||||
owners: BTreeMap<String, GuardOwner>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct GuardOwner {
|
||||
receipt: Option<serde::de::IgnoredAny>,
|
||||
}
|
||||
|
||||
fn reject(message: &'static str) -> RuntimeError {
|
||||
RuntimeError::InvalidRequest(format!("stopped Worker fixture: {message}"))
|
||||
}
|
||||
|
||||
fn io<T>(result: std::io::Result<T>) -> Result<T, RuntimeError> {
|
||||
result.map_err(|_| reject("filesystem operation failed"))
|
||||
}
|
||||
|
||||
fn decode<T: serde::de::DeserializeOwned>(bytes: &[u8]) -> Result<T, RuntimeError> {
|
||||
serde_json::from_slice(bytes).map_err(|_| reject("invalid typed document"))
|
||||
}
|
||||
|
||||
fn no_symlink_components(path: &Path) -> Result<(), RuntimeError> {
|
||||
let mut current = PathBuf::new();
|
||||
for part in path.components() {
|
||||
if matches!(part, std::path::Component::ParentDir) {
|
||||
return Err(reject("parent traversal is unsupported"));
|
||||
}
|
||||
current.push(part);
|
||||
let metadata = io(fs::symlink_metadata(¤t))?;
|
||||
if metadata.file_type().is_symlink() {
|
||||
return Err(reject("symlinks are unsupported"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn check_tree(path: &Path, budget: &mut usize) -> Result<(), RuntimeError> {
|
||||
if *budget == 0 {
|
||||
return Err(reject("file count exceeds fixture limit"));
|
||||
}
|
||||
*budget -= 1;
|
||||
let metadata = io(fs::symlink_metadata(path))?;
|
||||
if metadata.is_dir() {
|
||||
for entry in io(fs::read_dir(path))? {
|
||||
check_tree(&io(entry)?.path(), budget)?;
|
||||
}
|
||||
} else if !metadata.is_file() {
|
||||
return Err(reject("symlinks and special files are unsupported"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn regular_file(path: &Path) -> Result<(), RuntimeError> {
|
||||
no_symlink_components(path)?;
|
||||
if !io(fs::symlink_metadata(path))?.is_file() {
|
||||
return Err(reject("expected a regular file"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn bytes(path: &Path) -> Result<Vec<u8>, RuntimeError> {
|
||||
regular_file(path)?;
|
||||
let mut options = OpenOptions::new();
|
||||
options.read(true);
|
||||
#[cfg(target_os = "linux")]
|
||||
{
|
||||
use std::os::unix::fs::OpenOptionsExt;
|
||||
options.custom_flags(libc::O_NOATIME | libc::O_NOFOLLOW);
|
||||
}
|
||||
let file = io(options.open(path))?;
|
||||
let mut result = Vec::new();
|
||||
io(file
|
||||
.take(MAX_SESSION_BYTES as u64 + 1)
|
||||
.read_to_end(&mut result))?;
|
||||
if result.len() > MAX_SESSION_BYTES {
|
||||
return Err(reject("file exceeds fixture limit"));
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
|
||||
fn text(path: &Path) -> Result<String, RuntimeError> {
|
||||
String::from_utf8(bytes(path)?).map_err(|_| reject("invalid UTF-8 document"))
|
||||
}
|
||||
|
||||
fn portable_id(id: &str) -> bool {
|
||||
!id.is_empty()
|
||||
&& id.len() <= 256
|
||||
&& id
|
||||
.bytes()
|
||||
.all(|c| c.is_ascii_alphanumeric() || b"-_".contains(&c))
|
||||
}
|
||||
|
||||
impl StoppedWorkerTemplate {
|
||||
pub fn runtime_id(&self) -> &str {
|
||||
&self.runtime_id
|
||||
}
|
||||
pub fn workspace_id(&self) -> &str {
|
||||
&self.workspace_id
|
||||
}
|
||||
pub fn worker_id(&self) -> WorkerId {
|
||||
self.worker_id
|
||||
}
|
||||
pub fn worker_name(&self) -> String {
|
||||
runtime_worker_name(self.worker_id)
|
||||
}
|
||||
pub fn session_id(&self) -> SessionId {
|
||||
self.session_id
|
||||
}
|
||||
pub fn segment_id(&self) -> SegmentId {
|
||||
self.segment_id
|
||||
}
|
||||
|
||||
/// Extract without migration, rewriting metadata, or copying credentials,
|
||||
/// resolved manifests, execution-operation receipts, or run directories.
|
||||
/// The source Runtime must be offline; its existing owner lock is held for
|
||||
/// the entire read. The path must be `<runtime>/workers/<WorkerId>`.
|
||||
pub fn extract(worker_dir: &Path) -> Result<Self, RuntimeError> {
|
||||
no_symlink_components(worker_dir)?;
|
||||
let workers = worker_dir
|
||||
.parent()
|
||||
.ok_or_else(|| reject("invalid aggregate path"))?;
|
||||
if workers.file_name().and_then(|s| s.to_str()) != Some(WORKERS_DIR) {
|
||||
return Err(reject("invalid aggregate path"));
|
||||
}
|
||||
let root = workers
|
||||
.parent()
|
||||
.ok_or_else(|| reject("invalid Runtime path"))?;
|
||||
// Require the existing lock, rather than creating a file in the source.
|
||||
let lock_path = root.with_file_name(format!(
|
||||
".{}.runtime-owner.lock",
|
||||
root.file_name()
|
||||
.and_then(|s| s.to_str())
|
||||
.ok_or_else(|| reject("invalid Runtime path"))?
|
||||
));
|
||||
regular_file(&lock_path)?;
|
||||
let lock = io(OpenOptions::new().read(true).open(lock_path))?;
|
||||
if !io(FileExt::try_lock_shared(&lock))? {
|
||||
return Err(reject("source Runtime is not offline"));
|
||||
}
|
||||
let _lock = RuntimeStoreOwnerLock { file: lock };
|
||||
check_tree(worker_dir, &mut 256)?;
|
||||
let snapshot: RuntimeSnapshot = decode(&bytes(&root.join(RUNTIME_FILE))?)?;
|
||||
snapshot
|
||||
.validate(&root.join(RUNTIME_FILE))
|
||||
.map_err(|_| reject("unsupported Runtime schema"))?;
|
||||
if !snapshot.subject_create_recovery_fences.is_empty()
|
||||
|| !snapshot.subject_releases.is_empty()
|
||||
{
|
||||
return Err(reject("source has lifecycle operations"));
|
||||
}
|
||||
let record: WorkerAggregateRecord = decode(&bytes(&worker_dir.join(WORKER_FILE))?)?;
|
||||
let record = record
|
||||
.validate(&worker_dir.join(WORKER_FILE))
|
||||
.map_err(|_| reject("invalid Worker schema or identity"))?;
|
||||
let guard: GuardState = decode(
|
||||
&serde_json::to_vec(&record.restore_guard)
|
||||
.map_err(|_| reject("invalid restore guard"))?,
|
||||
)?;
|
||||
if record.status != WorkerStatus::Stopped
|
||||
|| guard.active_request_id.is_some()
|
||||
|| guard.owners.values().any(|owner| owner.receipt.is_none())
|
||||
{
|
||||
return Err(reject("Worker is not settled and stopped"));
|
||||
}
|
||||
let PersistedWorkerExecutionState::Available(execution) = &record.execution_state else {
|
||||
return Err(reject("pending or unavailable execution metadata"));
|
||||
};
|
||||
// A stopped aggregate may retain the empty historical binding marker.
|
||||
// It is deliberately not copied to the installed Worker.
|
||||
if execution.restore_intent != WorkerRestoreIntent::Explicit {
|
||||
return Err(reject("Worker requires explicit restore"));
|
||||
}
|
||||
if !record.workdir_attachments.is_empty() || !record.logical_workdir_attachments.is_empty()
|
||||
{
|
||||
return Err(reject("attachments are unsupported"));
|
||||
}
|
||||
check_request(&execution.request, false)?;
|
||||
let workspace_id = record
|
||||
.workspace_id
|
||||
.ok_or_else(|| reject("missing Workspace identity"))?;
|
||||
let metadata: WorkerMetadata = decode(&bytes(&worker_dir.join(WORKER_METADATA_FILE))?)?;
|
||||
crate::worker_backend::validate_fixture_restore_metadata(&metadata, &execution.request)
|
||||
.map_err(|_| reject("invalid source restore Manifest or settings"))?;
|
||||
if metadata.worker_name != runtime_worker_name(record.worker_id)
|
||||
|| metadata.workspace_id.as_deref() != Some(&workspace_id)
|
||||
|| !metadata.spawned_children.is_empty()
|
||||
|| !metadata.reclaimed_children.is_empty()
|
||||
|| !metadata.peers.is_empty()
|
||||
|| metadata.subjektiv_session_attribution.is_some()
|
||||
{
|
||||
return Err(reject("unsupported or inconsistent Worker metadata"));
|
||||
}
|
||||
let active = metadata
|
||||
.active
|
||||
.ok_or_else(|| reject("missing active Session"))?;
|
||||
let segment_id = active
|
||||
.segment_id
|
||||
.ok_or_else(|| reject("unmaterialized Session"))?;
|
||||
let source_auth: SourceAuth = toml::from_str(&text(&root.join("auth.toml"))?)
|
||||
.map_err(|_| reject("missing Runtime identity"))?;
|
||||
let session = worker_dir.join("session");
|
||||
let fixture = Self {
|
||||
template_version: TEMPLATE_VERSION,
|
||||
runtime_schema_version: SCHEMA_VERSION,
|
||||
runtime_id: source_auth.identity.identity_id,
|
||||
workspace_id,
|
||||
worker_id: record.worker_id,
|
||||
created_at_ms: record.created_at_ms,
|
||||
session_id: active.session_id,
|
||||
segment_id,
|
||||
session_manifest: bytes(&session.join("session.json"))?,
|
||||
segment_jsonl: bytes(&session.join("segments").join(format!("{segment_id}.jsonl")))?,
|
||||
};
|
||||
fixture.validate_shape()?;
|
||||
fixture.validate_session(&session)?;
|
||||
Ok(fixture)
|
||||
}
|
||||
|
||||
/// Install after normal identity/trust initialization, before Runtime start.
|
||||
/// `request` and `manifest` supply NEW backend/config/profile references and
|
||||
/// freshly resolved policy, not a launch. The saved source Manifest is never
|
||||
/// reused. `manifest.worker.name` must equal `worker_name()`.
|
||||
/// Only `auth.toml` may already exist. The owner lock fences live Runtimes;
|
||||
/// staged files are synced and published without overwriting any Worker.
|
||||
/// An interrupted publication fails closed as an incomplete Runtime layout.
|
||||
pub fn install(
|
||||
&self,
|
||||
options: &FsRuntimeStoreOptions,
|
||||
request: CreateWorkerRequest,
|
||||
manifest: &manifest::WorkerManifest,
|
||||
) -> Result<(), RuntimeError> {
|
||||
self.validate_shape()?;
|
||||
check_request(&request, true)?;
|
||||
if manifest.worker.name != self.worker_name() {
|
||||
return Err(reject("resolved Manifest Worker name mismatch"));
|
||||
}
|
||||
let mut metadata = WorkerMetadata::new(
|
||||
self.worker_name(),
|
||||
Some(WorkerActiveSegmentRef {
|
||||
session_id: self.session_id,
|
||||
segment_id: Some(self.segment_id),
|
||||
}),
|
||||
)
|
||||
.with_workspace_id(&self.workspace_id);
|
||||
metadata.resolved_manifest_snapshot = Some(
|
||||
manifest::write_persisted_worker_manifest_snapshot(manifest)
|
||||
.map_err(|_| reject("invalid resolved Manifest"))?,
|
||||
);
|
||||
crate::worker_backend::validate_fixture_restore_metadata(&metadata, &request)
|
||||
.map_err(|_| reject("resolved Manifest does not match fresh request settings"))?;
|
||||
if options.runtime_id != self.runtime_id
|
||||
|| request.worker_id != self.worker_id
|
||||
|| request
|
||||
.workspace_api
|
||||
.as_ref()
|
||||
.map(|api| api.workspace_id.as_str())
|
||||
!= Some(&self.workspace_id)
|
||||
{
|
||||
return Err(reject("installation identity mismatch"));
|
||||
}
|
||||
no_symlink_components(&options.root)?;
|
||||
check_tree(&options.root, &mut 256)?;
|
||||
// Validate the lock location before the ordinary owner-lock opener.
|
||||
let lock_path = options.root.with_file_name(format!(
|
||||
".{}.runtime-owner.lock",
|
||||
options
|
||||
.root
|
||||
.file_name()
|
||||
.and_then(|s| s.to_str())
|
||||
.ok_or_else(|| reject("invalid Runtime path"))?
|
||||
));
|
||||
if fs::symlink_metadata(&lock_path).is_ok() {
|
||||
regular_file(&lock_path)?;
|
||||
}
|
||||
let lock = acquire_runtime_store_owner_lock(&options.root)?;
|
||||
validate_fresh_runtime_store_layout(&options.root)
|
||||
.map_err(|_| reject("destination is not fresh; refusing overwrite"))?;
|
||||
let auth: InitializedAuth = toml::from_str(&text(&options.root.join("auth.toml"))?)
|
||||
.map_err(|_| reject("initialize Runtime identity and trust before install"))?;
|
||||
if auth.identity.identity_id != self.runtime_id
|
||||
|| !auth.archived_legacy_issuers.is_empty()
|
||||
|| auth.identity.signing_key().is_err()
|
||||
{
|
||||
return Err(reject("invalid destination Runtime identity"));
|
||||
}
|
||||
validate_workspace_issuer_trust_records(&auth.workspace_issuers)
|
||||
.map_err(|_| reject("invalid destination Workspace trust"))?;
|
||||
let api = request
|
||||
.workspace_api
|
||||
.as_ref()
|
||||
.expect("validated Workspace binding");
|
||||
if !auth.workspace_issuers.iter().any(|trust| {
|
||||
trust.workspace_id == self.workspace_id
|
||||
&& trust.backend_url == api.base_url
|
||||
&& trust.state == WorkspaceIssuerTrustState::Active
|
||||
}) {
|
||||
return Err(reject("request does not match initialized Workspace trust"));
|
||||
}
|
||||
let stage = options
|
||||
.root
|
||||
.join(format!(".fixture-{}", uuid::Uuid::now_v7()));
|
||||
io(fs::create_dir(&stage))?;
|
||||
let store = FsRuntimeStore {
|
||||
root: stage.clone(),
|
||||
_owner_lock: Some(lock),
|
||||
worker_write_fault: Arc::new(AtomicU8::new(AtomicWriteFault::None as u8)),
|
||||
};
|
||||
let result = (|| {
|
||||
let worker = PersistedWorkerRecord {
|
||||
restore_guard: crate::runtime::RestoreGuard::default(),
|
||||
worker_ref: WorkerRef::new(self.worker_id),
|
||||
worker_id: self.worker_id,
|
||||
profile: request.profile.clone(),
|
||||
display_name: request.display_name.clone(),
|
||||
profile_source: request.profile_source.reference(),
|
||||
config_bundle: request.config_bundle.clone(),
|
||||
created_at_ms: self.created_at_ms,
|
||||
status: WorkerStatus::Stopped,
|
||||
last_finished_submission_request_id: None,
|
||||
workspace_id: Some(self.workspace_id.clone()),
|
||||
workdir_attachments: Vec::new(),
|
||||
logical_workdir_attachments: Vec::new(),
|
||||
execution_state: PersistedWorkerExecutionState::Available(
|
||||
PersistedWorkerExecution {
|
||||
request,
|
||||
binding: None,
|
||||
restore_intent: WorkerRestoreIntent::Explicit,
|
||||
},
|
||||
),
|
||||
};
|
||||
WorkerAggregateRecord::from_persisted(&worker)
|
||||
.validate(&store.worker_dir(&self.worker_id).join(WORKER_FILE))
|
||||
.map_err(|_| reject("invalid installation request"))?;
|
||||
store.write_worker_record(&worker)?;
|
||||
let worker_dir = store.worker_dir(&self.worker_id);
|
||||
atomic_write_json(
|
||||
&worker_dir.join(WORKER_METADATA_FILE),
|
||||
&metadata,
|
||||
"write fixture metadata",
|
||||
)?;
|
||||
let session = worker_dir.join("session");
|
||||
io(fs::create_dir_all(session.join("segments")))?;
|
||||
write_new(&session.join("session.json"), &self.session_manifest)?;
|
||||
write_new(
|
||||
&session
|
||||
.join("segments")
|
||||
.join(format!("{}.jsonl", self.segment_id)),
|
||||
&self.segment_jsonl,
|
||||
)?;
|
||||
self.validate_session(&session)?;
|
||||
store.write_runtime_snapshot(&PersistedRuntimeState {
|
||||
subject_create_recovery_fences: BTreeMap::new(),
|
||||
subject_releases: Vec::new(),
|
||||
// Match normal store initialization: this is catalog availability,
|
||||
// not a process or Worker execution start.
|
||||
display_name: options.display_name.clone(),
|
||||
status: RuntimeStatus::Running,
|
||||
next_diagnostic_id: 1,
|
||||
workers: BTreeMap::new(),
|
||||
workspace_owners: BTreeMap::new(),
|
||||
diagnostics: Vec::new(),
|
||||
})?;
|
||||
let loaded = store.load_runtime_state()?;
|
||||
if loaded.workers.len() != 1 || !loaded.diagnostics.is_empty() {
|
||||
return Err(reject("staged aggregate failed ordinary store roundtrip"));
|
||||
}
|
||||
sync_runtime_tree(&stage)?;
|
||||
// No destination existed when checked under the owner lock. Hard-link
|
||||
// the snapshot last with create-new semantics; never replace a file.
|
||||
// Reserve the destination catalog with create-new semantics. Even
|
||||
// an empty preexisting workers directory must never be replaced.
|
||||
let destination_workers = options.root.join(WORKERS_DIR);
|
||||
io(fs::create_dir(&destination_workers))?;
|
||||
io(fs::rename(
|
||||
store.worker_dir(&self.worker_id),
|
||||
destination_workers.join(self.worker_id.to_string()),
|
||||
))?;
|
||||
sync_directory(&destination_workers, "publish stopped Worker aggregate")?;
|
||||
io(fs::hard_link(
|
||||
stage.join(RUNTIME_FILE),
|
||||
options.root.join(RUNTIME_FILE),
|
||||
))?;
|
||||
sync_directory(&options.root, "publish stopped fixture")?;
|
||||
Ok(())
|
||||
})();
|
||||
// Only remove our private staging directory, never existing destination data.
|
||||
let cleanup = fs::remove_dir_all(&stage);
|
||||
result?;
|
||||
io(cleanup)?;
|
||||
sync_directory(&options.root, "finish stopped fixture")
|
||||
}
|
||||
|
||||
fn validate_shape(&self) -> Result<(), RuntimeError> {
|
||||
if self.template_version != TEMPLATE_VERSION
|
||||
|| self.runtime_schema_version != SCHEMA_VERSION
|
||||
{
|
||||
return Err(reject("unsupported template schema"));
|
||||
}
|
||||
if !portable_id(&self.runtime_id)
|
||||
|| !portable_id(&self.workspace_id)
|
||||
|| self.session_id.get_version_num() != 7
|
||||
|| self.segment_id.get_version_num() != 7
|
||||
|| self.session_manifest.len() > MAX_SESSION_BYTES
|
||||
|| self.segment_jsonl.len() > MAX_SESSION_BYTES
|
||||
{
|
||||
return Err(reject("invalid portable identity or size"));
|
||||
}
|
||||
let manifest: SessionManifest = decode(&self.session_manifest)?;
|
||||
if manifest.session_id != self.session_id
|
||||
|| manifest.schema_version == 0
|
||||
|| !self.segment_jsonl.ends_with(b"\n")
|
||||
{
|
||||
return Err(reject("Session identity or committed boundary mismatch"));
|
||||
}
|
||||
// Read-only JSON traversal, never mutation/reconstruction of Session bytes.
|
||||
// Reject known secret/host markers even in otherwise valid typed records.
|
||||
for line in self
|
||||
.segment_jsonl
|
||||
.split(|b| *b == b'\n')
|
||||
.filter(|line| !line.is_empty())
|
||||
{
|
||||
let _: PortableJson = decode(line)?;
|
||||
validate_preserved_record(line)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_history_entry(
|
||||
&self,
|
||||
entry: &session_store::LoggedHistoryEntry,
|
||||
ids: &mut std::collections::BTreeSet<String>,
|
||||
) -> Result<(), RuntimeError> {
|
||||
if !portable_id(&entry.metadata.entry_id.0)
|
||||
|| !ids.insert(entry.metadata.entry_id.0.clone())
|
||||
{
|
||||
return Err(reject("duplicate or invalid entry identity"));
|
||||
}
|
||||
if entry
|
||||
.metadata
|
||||
.input_metadata
|
||||
.as_ref()
|
||||
.is_some_and(|metadata| {
|
||||
metadata.feature_payload().is_some() || metadata.public_item().is_some()
|
||||
})
|
||||
{
|
||||
// These are opaque owner-specific payloads, not part of the narrow
|
||||
// artificial conversation contract. Never preserve unchecked JSON.
|
||||
return Err(reject("Feature history payloads are unsupported"));
|
||||
}
|
||||
if !matches!(
|
||||
entry.item,
|
||||
session_store::LoggedItem::Message { .. }
|
||||
| session_store::LoggedItem::AssistantResponseBoundary { .. }
|
||||
) {
|
||||
return Err(reject("only text conversation records are supported"));
|
||||
}
|
||||
if let session_store::LoggedSessionHistoryOrigin::ModelOutput { worker } =
|
||||
&entry.metadata.origin
|
||||
{
|
||||
// Session producer IDs identify the original execution, not the
|
||||
// Workspace catalog Worker. Preserve them; do not rebind/rewrite them.
|
||||
if !portable_id(&worker.worker_id)
|
||||
|| worker
|
||||
.workspace_id
|
||||
.as_ref()
|
||||
.is_some_and(|id| id != &self.workspace_id)
|
||||
|| worker
|
||||
.runtime_id
|
||||
.as_ref()
|
||||
.is_some_and(|id| id != &self.runtime_id)
|
||||
{
|
||||
return Err(reject("Session origin identity mismatch"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn validate_session(&self, root: &Path) -> Result<(), RuntimeError> {
|
||||
for entry in io(fs::read_dir(root))? {
|
||||
let entry = io(entry)?;
|
||||
match entry.file_name().to_str() {
|
||||
Some("session.json" | "segments") => {}
|
||||
Some("artifacts") => check_empty_artifacts(&entry.path())?,
|
||||
_ => return Err(reject("unsupported Session files")),
|
||||
}
|
||||
}
|
||||
let entries = io(fs::read_dir(root.join("segments")))?.collect::<Result<Vec<_>, _>>();
|
||||
let entries = io(entries)?;
|
||||
if entries.len() != 1
|
||||
|| entries[0].file_name() != format!("{}.jsonl", self.segment_id).as_str()
|
||||
{
|
||||
return Err(reject("only the fixed single-Segment fixture is supported"));
|
||||
}
|
||||
let store = WorkerSessionStore::open_read_only(root)
|
||||
.map_err(|_| reject("unsupported Session schema"))?;
|
||||
let logs = store
|
||||
.read_all_read_only_bounded(self.session_id, self.segment_id, MAX_SESSION_BYTES as u64)
|
||||
.map_err(|_| reject("invalid canonical Session records"))?
|
||||
.0;
|
||||
if !matches!(logs.first(), Some(session_store::LogEntry::AnnotatedSegmentStart {
|
||||
session_id, system_prompt: Some(_), forked_from: None, compacted_from: None, ..
|
||||
}) if *session_id == self.session_id)
|
||||
{
|
||||
return Err(reject(
|
||||
"missing system prompt, invalid Segment seed, or unsupported lineage",
|
||||
));
|
||||
}
|
||||
let mut ids = std::collections::BTreeSet::new();
|
||||
for (index, log) in logs.iter().enumerate() {
|
||||
use session_store::LogEntry;
|
||||
match log {
|
||||
LogEntry::AnnotatedSegmentStart { history, .. }
|
||||
if index == 0 && history.is_empty() => {}
|
||||
LogEntry::AnnotatedUserInput {
|
||||
history,
|
||||
extensions,
|
||||
..
|
||||
} => {
|
||||
for extension in extensions {
|
||||
validate_settled_activation(extension)?;
|
||||
}
|
||||
for entry in history {
|
||||
self.validate_history_entry(entry, &mut ids)?;
|
||||
}
|
||||
}
|
||||
LogEntry::AnnotatedAssistantItem { entry, .. } => {
|
||||
self.validate_history_entry(entry, &mut ids)?
|
||||
}
|
||||
LogEntry::RunCompleted {
|
||||
entry_id: Some(id), ..
|
||||
} => {
|
||||
if !portable_id(&id.0) || !ids.insert(id.0.clone()) {
|
||||
return Err(reject("duplicate or invalid entry identity"));
|
||||
}
|
||||
}
|
||||
LogEntry::Invoke { .. } | LogEntry::LlmUsage { .. } | LogEntry::TurnEnd { .. } => {}
|
||||
_ => {
|
||||
return Err(reject(
|
||||
"unsupported conversation record or pending operation",
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
let state = session_store::collect_state(&logs);
|
||||
if state.last_run_interrupted || state.active_run_turn_count.is_some() {
|
||||
return Err(reject("Session has pending execution"));
|
||||
}
|
||||
for segments in &state.user_segments {
|
||||
if segments
|
||||
.iter()
|
||||
.any(|segment| !matches!(segment, protocol::Segment::Text { .. }))
|
||||
{
|
||||
return Err(reject("Session attachments are unsupported"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
// Compare *fields*, not re-encoded bytes or scalar values (e.g. f32 JSON
|
||||
// normalization). Serde's permissive/untagged/flattened compatibility readers
|
||||
// can drop unknown data without surfacing it to an ignored-field callback. A
|
||||
// retained field must survive the owner's typed projection at every depth.
|
||||
// This intentionally accepts only canonical writer shapes: even an explicit
|
||||
// empty optional field omitted by that writer is rejected, not repaired.
|
||||
// The projection is validation-only; installation always writes original bytes.
|
||||
fn validate_preserved_record(bytes: &[u8]) -> Result<(), RuntimeError> {
|
||||
let original: serde_json::Value = decode(bytes)?;
|
||||
let record: session_store::LogEntry = decode(bytes)?;
|
||||
let projection =
|
||||
serde_json::to_value(&record).map_err(|_| reject("invalid typed Session record"))?;
|
||||
require_preserved_fields(&original, &projection)
|
||||
}
|
||||
|
||||
fn require_preserved_fields(
|
||||
original: &serde_json::Value,
|
||||
projection: &serde_json::Value,
|
||||
) -> Result<(), RuntimeError> {
|
||||
use serde_json::Value;
|
||||
match (original, projection) {
|
||||
(Value::Object(original), Value::Object(projection)) => {
|
||||
for (key, value) in original {
|
||||
let projected = projection
|
||||
.get(key)
|
||||
.ok_or_else(|| reject("unknown or noncanonical Session field"))?;
|
||||
require_preserved_fields(value, projected)?;
|
||||
}
|
||||
}
|
||||
(Value::Array(original), Value::Array(projection))
|
||||
if original.len() == projection.len() =>
|
||||
{
|
||||
for (value, projected) in original.iter().zip(projection) {
|
||||
require_preserved_fields(value, projected)?;
|
||||
}
|
||||
}
|
||||
(Value::Null, Value::Null)
|
||||
| (Value::Bool(_), Value::Bool(_))
|
||||
| (Value::Number(_), Value::Number(_))
|
||||
| (Value::String(_), Value::String(_)) => {}
|
||||
_ => return Err(reject("noncanonical Session field shape")),
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Exact receipt wire fields owned by worker::PendingActivationState. Keep these
|
||||
// fixture-only mirrors strict; do not change the production compatibility reader.
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct SubmissionReceipt {
|
||||
submission_request_id: String,
|
||||
source_namespace: String,
|
||||
submission_id: String,
|
||||
payload_digest: String,
|
||||
disposition: protocol::SubmissionDisposition,
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct NotificationReceipt {
|
||||
notification_request_id: String,
|
||||
source_namespace: String,
|
||||
payload_digest: String,
|
||||
}
|
||||
|
||||
// The approved conversation has committed submission checkpoints in its original
|
||||
// Session records. Keep their receipt bytes, but reject any resumable queue or
|
||||
// Feature invocation history. This projection confers no execution authority.
|
||||
fn validate_settled_activation(
|
||||
extension: &session_store::SessionExtension,
|
||||
) -> Result<(), RuntimeError> {
|
||||
#[derive(Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct Checkpoint {
|
||||
#[serde(rename = "next_activation_sequence")]
|
||||
_next_activation_sequence: u64,
|
||||
activating: Option<serde::de::IgnoredAny>,
|
||||
activating_notification: Vec<serde::de::IgnoredAny>,
|
||||
pending: Vec<serde::de::IgnoredAny>,
|
||||
pending_notifications: Vec<serde::de::IgnoredAny>,
|
||||
#[serde(rename = "receipts")]
|
||||
_receipts: Vec<SubmissionReceipt>,
|
||||
#[serde(rename = "notification_receipts")]
|
||||
_notification_receipts: Vec<NotificationReceipt>,
|
||||
invocation_receipts: Vec<serde::de::IgnoredAny>,
|
||||
invocation_recovery_blocked: bool,
|
||||
}
|
||||
if extension.domain != "worker.pending_activations.v1" {
|
||||
return Err(reject("unsupported Session extension"));
|
||||
}
|
||||
let checkpoint: Checkpoint = serde_json::from_value(extension.payload.clone())
|
||||
.map_err(|_| reject("invalid Session activation checkpoint"))?;
|
||||
if checkpoint.activating.is_some()
|
||||
|| !checkpoint.activating_notification.is_empty()
|
||||
|| !checkpoint.pending.is_empty()
|
||||
|| !checkpoint.pending_notifications.is_empty()
|
||||
|| !checkpoint.invocation_receipts.is_empty()
|
||||
|| checkpoint.invocation_recovery_blocked
|
||||
{
|
||||
return Err(reject(
|
||||
"Session has pending activation or invocation history",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn check_request(request: &CreateWorkerRequest, install: bool) -> Result<(), RuntimeError> {
|
||||
if !request.workdir_attachment_requests.is_empty()
|
||||
|| !request.workdir_attachments.is_empty()
|
||||
|| request.backend_job.is_some()
|
||||
|| request.subjektiv_attached
|
||||
|| request.worker_observation_enabled
|
||||
|| !request.worker_observation_grants.is_empty()
|
||||
|| (install && request.initial_input.is_some())
|
||||
{
|
||||
return Err(reject(
|
||||
"unsupported capabilities, attachments, or initial input",
|
||||
));
|
||||
}
|
||||
if install
|
||||
&& (request.create_fingerprint.trim().is_empty()
|
||||
|| request.config_bundle.is_none()
|
||||
|| !matches!(
|
||||
request.profile_source,
|
||||
ProfileSourceArchiveSource::WorkspaceConfig { .. }
|
||||
))
|
||||
{
|
||||
return Err(reject(
|
||||
"fresh config/profile references and create fingerprint are required",
|
||||
));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn check_empty_artifacts(root: &Path) -> Result<(), RuntimeError> {
|
||||
for entry in io(fs::read_dir(root))? {
|
||||
let entry = io(entry)?;
|
||||
let metadata = io(entry.metadata())?;
|
||||
if metadata.is_dir() {
|
||||
check_empty_artifacts(&entry.path())?;
|
||||
} else if entry.file_name() != ".aggregate.lock" || metadata.len() != 0 {
|
||||
return Err(reject("Session artifacts are unsupported"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn write_new(path: &Path, bytes: &[u8]) -> Result<(), RuntimeError> {
|
||||
let mut file = io(OpenOptions::new().write(true).create_new(true).open(path))?;
|
||||
io(file.write_all(bytes))?;
|
||||
io(file.sync_all())
|
||||
}
|
||||
|
||||
// A streaming validation visitor, not a generic document available to callers.
|
||||
struct PortableJson;
|
||||
impl<'de> Deserialize<'de> for PortableJson {
|
||||
fn deserialize<D: serde::Deserializer<'de>>(deserializer: D) -> Result<Self, D::Error> {
|
||||
struct Visitor;
|
||||
impl<'de> serde::de::Visitor<'de> for Visitor {
|
||||
type Value = PortableJson;
|
||||
fn expecting(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.write_str("portable fixture JSON")
|
||||
}
|
||||
fn visit_bool<E: serde::de::Error>(self, _: bool) -> Result<PortableJson, E> {
|
||||
Ok(PortableJson)
|
||||
}
|
||||
fn visit_i64<E: serde::de::Error>(self, _: i64) -> Result<PortableJson, E> {
|
||||
Ok(PortableJson)
|
||||
}
|
||||
fn visit_u64<E: serde::de::Error>(self, _: u64) -> Result<PortableJson, E> {
|
||||
Ok(PortableJson)
|
||||
}
|
||||
fn visit_f64<E: serde::de::Error>(self, _: f64) -> Result<PortableJson, E> {
|
||||
Ok(PortableJson)
|
||||
}
|
||||
fn visit_unit<E: serde::de::Error>(self) -> Result<PortableJson, E> {
|
||||
Ok(PortableJson)
|
||||
}
|
||||
fn visit_str<E: serde::de::Error>(self, value: &str) -> Result<PortableJson, E> {
|
||||
let lower = value.to_ascii_lowercase();
|
||||
if [
|
||||
"://",
|
||||
"private key",
|
||||
"bearer ",
|
||||
"sk-",
|
||||
"api_key",
|
||||
"access_token",
|
||||
"refresh_token",
|
||||
"private_key",
|
||||
"authorization",
|
||||
]
|
||||
.iter()
|
||||
.any(|marker| lower.contains(marker))
|
||||
|| value
|
||||
.split_whitespace()
|
||||
.any(|word| word.starts_with('/') || word.contains(":\\"))
|
||||
{
|
||||
return Err(E::custom("nonportable or sensitive fixture material"));
|
||||
}
|
||||
Ok(PortableJson)
|
||||
}
|
||||
fn visit_seq<A: serde::de::SeqAccess<'de>>(
|
||||
self,
|
||||
mut seq: A,
|
||||
) -> Result<PortableJson, A::Error> {
|
||||
while seq.next_element::<PortableJson>()?.is_some() {}
|
||||
Ok(PortableJson)
|
||||
}
|
||||
fn visit_map<A: serde::de::MapAccess<'de>>(
|
||||
self,
|
||||
mut map: A,
|
||||
) -> Result<PortableJson, A::Error> {
|
||||
let mut keys = std::collections::BTreeSet::new();
|
||||
while let Some(key) = map.next_key::<String>()? {
|
||||
// A duplicate key can hide discarded data even inside an
|
||||
// otherwise opaque extension payload. Never retain it.
|
||||
if !keys.insert(key.clone()) {
|
||||
return Err(serde::de::Error::custom("duplicate fixture JSON field"));
|
||||
}
|
||||
serde::de::Visitor::visit_str::<A::Error>(Visitor, &key)?;
|
||||
map.next_value::<PortableJson>()?;
|
||||
}
|
||||
Ok(PortableJson)
|
||||
}
|
||||
}
|
||||
deserializer.deserialize_any(Visitor)
|
||||
}
|
||||
}
|
||||
@@ -1,885 +0,0 @@
|
||||
use super::*;
|
||||
use crate::auth::RuntimeIdentityMaterial;
|
||||
use crate::catalog::{ProfileSelector, WorkspaceApiRef};
|
||||
use crate::profile_archive::ProfileSourceGraphSummary;
|
||||
use session_store::{LogEntry, Store};
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
const RUNTIME: &str = "fixture-runtime";
|
||||
const WORKSPACE: &str = "fixture-workspace";
|
||||
const NEW_URL: &str = "http://127.0.0.1:34567";
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(untagged)]
|
||||
enum ReceiptFixture<T> {
|
||||
Valid(T),
|
||||
Malformed(u8),
|
||||
}
|
||||
|
||||
#[derive(Default, Serialize)]
|
||||
struct CheckpointFixture {
|
||||
next_activation_sequence: u64,
|
||||
activating: Option<u8>,
|
||||
activating_notification: Vec<u8>,
|
||||
pending: Vec<u8>,
|
||||
pending_notifications: Vec<u8>,
|
||||
receipts: Vec<ReceiptFixture<SubmissionReceipt>>,
|
||||
notification_receipts: Vec<ReceiptFixture<NotificationReceipt>>,
|
||||
invocation_receipts: Vec<u8>,
|
||||
invocation_recovery_blocked: bool,
|
||||
}
|
||||
|
||||
fn settled_checkpoint() -> CheckpointFixture {
|
||||
CheckpointFixture {
|
||||
next_activation_sequence: 2,
|
||||
receipts: vec![ReceiptFixture::Valid(SubmissionReceipt {
|
||||
submission_request_id: "completed-submission-request".into(),
|
||||
source_namespace: "fixture-source".into(),
|
||||
submission_id: "completed-submission".into(),
|
||||
payload_digest: "fixture-payload-digest".into(),
|
||||
disposition: protocol::SubmissionDisposition::Started,
|
||||
})],
|
||||
notification_receipts: vec![ReceiptFixture::Valid(NotificationReceipt {
|
||||
notification_request_id: "completed-notification-request".into(),
|
||||
source_namespace: "fixture-source".into(),
|
||||
payload_digest: "fixture-notification-digest".into(),
|
||||
})],
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
fn checkpoint_extension(checkpoint: &CheckpointFixture) -> session_store::SessionExtension {
|
||||
session_store::SessionExtension::new(
|
||||
"worker.pending_activations.v1",
|
||||
serde_json::to_value(checkpoint).unwrap(),
|
||||
)
|
||||
}
|
||||
|
||||
fn worker_id() -> WorkerId {
|
||||
"01a1657d-b762-7791-87ae-a49f28cadbce".parse().unwrap()
|
||||
}
|
||||
|
||||
fn resolved() -> manifest::WorkerManifest {
|
||||
let mut config = manifest::WorkerManifestConfig::resolution_defaults();
|
||||
config.worker.name = Some(runtime_worker_name(worker_id()));
|
||||
manifest::WorkerManifest::try_from(config).unwrap()
|
||||
}
|
||||
|
||||
fn request() -> CreateWorkerRequest {
|
||||
CreateWorkerRequest {
|
||||
worker_id: worker_id(),
|
||||
create_fingerprint: "new-create-fingerprint".into(),
|
||||
profile: ProfileSelector::Builtin("default".into()),
|
||||
display_name: Some("fixture".into()),
|
||||
profile_source: ProfileSourceArchiveSource::WorkspaceConfig {
|
||||
archive: ProfileSourceArchiveRef {
|
||||
id: "new-profile".into(),
|
||||
digest: "new-profile-digest".into(),
|
||||
size_bytes: 1,
|
||||
source_graph: ProfileSourceGraphSummary {
|
||||
source_count: 1,
|
||||
total_source_bytes: 1,
|
||||
entrypoints: BTreeMap::from([("builtin:default".into(), "default.dcl".into())]),
|
||||
import_count: 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
config_bundle: Some(ConfigBundleRef {
|
||||
id: "new-config".into(),
|
||||
digest: "new-config-digest".into(),
|
||||
}),
|
||||
workspace_api: Some(WorkspaceApiRef {
|
||||
workspace_id: WORKSPACE.into(),
|
||||
base_url: NEW_URL.into(),
|
||||
}),
|
||||
initial_input: None,
|
||||
workdir_attachment_requests: Vec::new(),
|
||||
workdir_attachments: Vec::new(),
|
||||
worker_observation_enabled: false,
|
||||
worker_observation_grants: Vec::new(),
|
||||
memory_settings: Some(manifest::WorkspaceMemorySettingsSnapshot {
|
||||
workspace_id: WORKSPACE.into(),
|
||||
language: "en".into(),
|
||||
}),
|
||||
subjektiv_attached: false,
|
||||
backend_job: None,
|
||||
}
|
||||
}
|
||||
|
||||
fn initialize(root: &Path, runtime_id: &str, workspace_id: &str, url: &str) {
|
||||
#[derive(Serialize)]
|
||||
struct Auth {
|
||||
identity: RuntimeIdentityMaterial,
|
||||
workspace_issuers: Vec<WorkspaceIssuerTrustRecord>,
|
||||
}
|
||||
let issuer = RuntimeIdentityMaterial::generate("fixture-issuer").unwrap();
|
||||
let digest = Sha256::digest(crate::auth::decode_public_key(&issuer.public_key).unwrap());
|
||||
let fingerprint = format!(
|
||||
"sha256:{}",
|
||||
digest
|
||||
.iter()
|
||||
.map(|byte| format!("{byte:02x}"))
|
||||
.collect::<String>()
|
||||
);
|
||||
let trust = WorkspaceIssuerTrustRecord::from_bundle(
|
||||
server_api::WorkspacePublicIdentityBundle {
|
||||
workspace_id: workspace_id.into(),
|
||||
backend_url: url.into(),
|
||||
key_id: "fixture-issuer".into(),
|
||||
algorithm: "ed25519".into(),
|
||||
public_key: issuer.public_key,
|
||||
public_key_fingerprint: fingerprint,
|
||||
},
|
||||
"fixture-trust".into(),
|
||||
100,
|
||||
)
|
||||
.unwrap();
|
||||
fs::create_dir_all(root).unwrap();
|
||||
fs::write(
|
||||
root.join("auth.toml"),
|
||||
toml::to_string(&Auth {
|
||||
identity: RuntimeIdentityMaterial::generate(runtime_id).unwrap(),
|
||||
workspace_issuers: vec![trust],
|
||||
})
|
||||
.unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
fn record(request: CreateWorkerRequest) -> PersistedWorkerRecord {
|
||||
PersistedWorkerRecord {
|
||||
restore_guard: crate::runtime::RestoreGuard::default(),
|
||||
worker_ref: WorkerRef::new(request.worker_id),
|
||||
worker_id: request.worker_id,
|
||||
profile: request.profile.clone(),
|
||||
display_name: request.display_name.clone(),
|
||||
profile_source: request.profile_source.reference(),
|
||||
config_bundle: request.config_bundle.clone(),
|
||||
created_at_ms: Some(123456),
|
||||
status: WorkerStatus::Stopped,
|
||||
last_finished_submission_request_id: Some("old-submission-operation".into()),
|
||||
workspace_id: Some(WORKSPACE.into()),
|
||||
workdir_attachments: vec![],
|
||||
logical_workdir_attachments: vec![],
|
||||
execution_state: PersistedWorkerExecutionState::Available(PersistedWorkerExecution {
|
||||
request,
|
||||
binding: Some(PersistedWorkerExecutionBinding {}),
|
||||
restore_intent: WorkerRestoreIntent::Explicit,
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
fn source() -> (tempfile::TempDir, PathBuf, StoppedWorkerTemplate) {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let root = temp.path().join("runtime");
|
||||
initialize(&root, RUNTIME, WORKSPACE, "http://127.0.0.1:23456");
|
||||
drop(
|
||||
crate::Runtime::with_fs_store(FsRuntimeStoreOptions::new(&root).with_runtime_id(RUNTIME))
|
||||
.unwrap(),
|
||||
);
|
||||
let opened = FsRuntimeStore::open_or_create(root.clone(), RUNTIME).unwrap();
|
||||
let mut old = request();
|
||||
old.workspace_api.as_mut().unwrap().base_url = "http://127.0.0.1:23456".into();
|
||||
old.config_bundle.as_mut().unwrap().id = "old-config".into();
|
||||
opened.store.write_worker_record(&record(old)).unwrap();
|
||||
let worker = opened.store.worker_dir(&worker_id());
|
||||
let session_id = "01a1657d-b800-7777-8888-111111111111".parse().unwrap();
|
||||
let segment_id = "01a1657d-b818-77c1-bd73-176172a4bd82".parse().unwrap();
|
||||
let mut metadata = WorkerMetadata::new(
|
||||
runtime_worker_name(worker_id()),
|
||||
Some(WorkerActiveSegmentRef {
|
||||
session_id,
|
||||
segment_id: Some(segment_id),
|
||||
}),
|
||||
)
|
||||
.with_workspace_id(WORKSPACE)
|
||||
.with_workspace_root(PathBuf::from("/old/host/path"));
|
||||
metadata.resolved_manifest_snapshot =
|
||||
Some(manifest::write_persisted_worker_manifest_snapshot(&resolved()).unwrap());
|
||||
atomic_write_json(
|
||||
&worker.join(WORKER_METADATA_FILE),
|
||||
&metadata,
|
||||
"test metadata",
|
||||
)
|
||||
.unwrap();
|
||||
let sessions = WorkerSessionStore::new(worker.join("session")).unwrap();
|
||||
let history = session_store::LoggedHistoryEntry {
|
||||
item: session_store::LoggedItem::Message {
|
||||
role: session_store::LoggedRole::Assistant,
|
||||
content: vec![session_store::LoggedContentPart::Text {
|
||||
text: "artificial fixture reply".into(),
|
||||
}],
|
||||
},
|
||||
metadata: session_store::LoggedSessionHistoryMetadata {
|
||||
entry_id: session_store::LoggedSessionHistoryEntryId("fixed-entry-id".into()),
|
||||
origin: session_store::LoggedSessionHistoryOrigin::ModelOutput {
|
||||
worker: session_store::LoggedWorkerSubject {
|
||||
workspace_id: Some(WORKSPACE.into()),
|
||||
runtime_id: Some(RUNTIME.into()),
|
||||
worker_id: worker_id().to_string(),
|
||||
},
|
||||
},
|
||||
derivation: None,
|
||||
input_metadata: None,
|
||||
},
|
||||
};
|
||||
sessions
|
||||
.create_segment(
|
||||
session_id,
|
||||
segment_id,
|
||||
&[
|
||||
LogEntry::AnnotatedSegmentStart {
|
||||
ts: 123,
|
||||
session_id,
|
||||
system_prompt: Some("Harmless artificial fixture instructions.".into()),
|
||||
config: Default::default(),
|
||||
history: vec![],
|
||||
forked_from: None,
|
||||
compacted_from: None,
|
||||
},
|
||||
LogEntry::Invoke {
|
||||
ts: 124,
|
||||
trigger: protocol::InvokeKind::UserSend,
|
||||
},
|
||||
LogEntry::AnnotatedUserInput {
|
||||
ts: 124,
|
||||
segments: vec![protocol::Segment::text(
|
||||
"Harmless artificial fixture input.",
|
||||
)],
|
||||
extensions: vec![checkpoint_extension(&settled_checkpoint())],
|
||||
history: vec![session_store::LoggedHistoryEntry {
|
||||
item: session_store::LoggedItem::Message {
|
||||
role: session_store::LoggedRole::User,
|
||||
content: vec![session_store::LoggedContentPart::Text {
|
||||
text: "Harmless artificial fixture input.".into(),
|
||||
}],
|
||||
},
|
||||
metadata: session_store::LoggedSessionHistoryMetadata {
|
||||
entry_id: session_store::LoggedSessionHistoryEntryId(
|
||||
"fixed-user-id".into(),
|
||||
),
|
||||
origin: session_store::LoggedSessionHistoryOrigin::BackendInstruction {
|
||||
operation_id: None,
|
||||
},
|
||||
derivation: None,
|
||||
input_metadata: None,
|
||||
},
|
||||
}],
|
||||
},
|
||||
LogEntry::AnnotatedAssistantItem {
|
||||
ts: 125,
|
||||
entry: history,
|
||||
},
|
||||
LogEntry::LlmUsage {
|
||||
ts: 125,
|
||||
history_len: 2,
|
||||
input_total_tokens: 1,
|
||||
cache_read_tokens: 0,
|
||||
cache_write_tokens: 0,
|
||||
output_tokens: 1,
|
||||
},
|
||||
LogEntry::TurnEnd {
|
||||
ts: 126,
|
||||
turn_count: 1,
|
||||
},
|
||||
LogEntry::RunCompleted {
|
||||
entry_id: Some(session_store::LoggedSessionHistoryEntryId(
|
||||
"fixed-terminal-id".into(),
|
||||
)),
|
||||
ts: 126,
|
||||
interrupted: false,
|
||||
result: agen::EngineResult::Finished,
|
||||
active_run_turn_count: None,
|
||||
},
|
||||
],
|
||||
)
|
||||
.unwrap();
|
||||
fs::create_dir_all(worker.join("runs/old-run")).unwrap();
|
||||
fs::write(worker.join("runs/old-run/private.log"), "not exported").unwrap();
|
||||
drop(opened);
|
||||
let template = StoppedWorkerTemplate::extract(&worker).unwrap();
|
||||
(temp, worker, template)
|
||||
}
|
||||
|
||||
fn destination(template: &StoppedWorkerTemplate) -> (tempfile::TempDir, FsRuntimeStoreOptions) {
|
||||
let temp = tempfile::tempdir().unwrap();
|
||||
let root = temp.path().join("runtime");
|
||||
initialize(
|
||||
&root,
|
||||
template.runtime_id(),
|
||||
template.workspace_id(),
|
||||
NEW_URL,
|
||||
);
|
||||
(
|
||||
temp,
|
||||
FsRuntimeStoreOptions::new(root).with_runtime_id(template.runtime_id()),
|
||||
)
|
||||
}
|
||||
|
||||
fn assert_fresh(options: &FsRuntimeStoreOptions) {
|
||||
assert_eq!(fs::read_dir(&options.root).unwrap().count(), 1);
|
||||
assert!(options.root.join("auth.toml").is_file());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extraction_install_roundtrip_preserves_bytes_and_is_inactive() {
|
||||
let (_source, worker, template) = source();
|
||||
let original_worker = fs::read(worker.join(WORKER_FILE)).unwrap();
|
||||
let original_metadata = fs::read(worker.join(WORKER_METADATA_FILE)).unwrap();
|
||||
let exported = serde_json::to_vec(&template).unwrap();
|
||||
let exported_text = String::from_utf8(exported.clone()).unwrap();
|
||||
for forbidden in [
|
||||
"old-config",
|
||||
"old-submission-operation",
|
||||
"restore_guard",
|
||||
"private_key",
|
||||
"http://",
|
||||
"/old/host",
|
||||
"private.log",
|
||||
"resolved_manifest_snapshot",
|
||||
] {
|
||||
assert!(!exported_text.contains(forbidden));
|
||||
}
|
||||
let template: StoppedWorkerTemplate = serde_json::from_slice(&exported).unwrap();
|
||||
let (_target, options) = destination(&template);
|
||||
let auth_before = fs::read(options.root.join("auth.toml")).unwrap();
|
||||
template.install(&options, request(), &resolved()).unwrap();
|
||||
assert_eq!(
|
||||
fs::read(options.root.join("auth.toml")).unwrap(),
|
||||
auth_before
|
||||
);
|
||||
assert_eq!(fs::read(worker.join(WORKER_FILE)).unwrap(), original_worker);
|
||||
assert_eq!(
|
||||
fs::read(worker.join(WORKER_METADATA_FILE)).unwrap(),
|
||||
original_metadata
|
||||
);
|
||||
let installed = options
|
||||
.root
|
||||
.join(WORKERS_DIR)
|
||||
.join(template.worker_id().to_string());
|
||||
assert_eq!(
|
||||
fs::read(installed.join("session/session.json")).unwrap(),
|
||||
template.session_manifest
|
||||
);
|
||||
assert_eq!(
|
||||
fs::read(installed.join(format!("session/segments/{}.jsonl", template.segment_id())))
|
||||
.unwrap(),
|
||||
template.segment_jsonl
|
||||
);
|
||||
assert!(!installed.join("runs").exists());
|
||||
let metadata: WorkerMetadata =
|
||||
decode(&fs::read(installed.join(WORKER_METADATA_FILE)).unwrap()).unwrap();
|
||||
assert_eq!(metadata.active.unwrap().session_id, template.session_id());
|
||||
assert!(metadata.workspace_root.is_none() && metadata.resolved_manifest_snapshot.is_some());
|
||||
|
||||
let opened = FsRuntimeStore::open_or_create(options.root.clone(), RUNTIME).unwrap();
|
||||
let state = opened.state.unwrap();
|
||||
assert!(state.diagnostics.is_empty());
|
||||
let saved = state.workers.get(&worker_id()).unwrap();
|
||||
assert_eq!(saved.status, WorkerStatus::Stopped);
|
||||
assert!(saved.last_finished_submission_request_id.is_none());
|
||||
let PersistedWorkerExecutionState::Available(execution) = &saved.execution_state else {
|
||||
panic!("not available")
|
||||
};
|
||||
assert_eq!(execution.restore_intent, WorkerRestoreIntent::Explicit);
|
||||
assert!(execution.binding.is_none());
|
||||
assert_eq!(execution.request, request());
|
||||
// Fresh restore guard, no operation owner/receipt history.
|
||||
assert_ne!(
|
||||
serde_json::to_vec(&saved.restore_guard).unwrap(),
|
||||
serde_json::to_vec(
|
||||
&decode::<WorkerAggregateRecord>(&original_worker)
|
||||
.unwrap()
|
||||
.restore_guard
|
||||
)
|
||||
.unwrap()
|
||||
);
|
||||
let guard: GuardState = decode(&serde_json::to_vec(&saved.restore_guard).unwrap()).unwrap();
|
||||
assert!(guard.active_request_id.is_none() && guard.owners.is_empty());
|
||||
drop(opened.store);
|
||||
|
||||
struct NoExecution;
|
||||
impl crate::execution::WorkerExecutionBackend for NoExecution {
|
||||
fn backend_id(&self) -> &str {
|
||||
"fixture-no-execution"
|
||||
}
|
||||
fn spawn_worker(
|
||||
&self,
|
||||
_: crate::execution::WorkerExecutionSpawnRequest,
|
||||
) -> crate::execution::WorkerExecutionSpawnResult {
|
||||
panic!("fixture executed")
|
||||
}
|
||||
fn dispatch_input(
|
||||
&self,
|
||||
_: &WorkerRef,
|
||||
_: crate::interaction::WorkerInput,
|
||||
) -> crate::execution::WorkerExecutionResult {
|
||||
panic!("fixture submitted")
|
||||
}
|
||||
fn restore_worker(
|
||||
&self,
|
||||
_: crate::execution::WorkerExecutionRestoreRequest,
|
||||
) -> crate::execution::WorkerExecutionSpawnResult {
|
||||
panic!("fixture auto-restored")
|
||||
}
|
||||
}
|
||||
let runtime =
|
||||
crate::Runtime::with_fs_store_and_execution_backend(options.clone(), Arc::new(NoExecution))
|
||||
.unwrap();
|
||||
assert_eq!(runtime.list_workers().unwrap().len(), 1);
|
||||
assert_eq!(runtime.summary().unwrap().active_worker_count, 0);
|
||||
assert_eq!(
|
||||
runtime
|
||||
.worker_detail(&WorkerRef::new(worker_id()))
|
||||
.unwrap()
|
||||
.status,
|
||||
WorkerStatus::Stopped
|
||||
);
|
||||
drop(runtime);
|
||||
let runtime = crate::Runtime::with_fs_store(options).unwrap();
|
||||
assert_eq!(
|
||||
runtime.list_workers().unwrap()[0].status,
|
||||
WorkerStatus::Stopped
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn install_rejects_schema_identity_input_and_overwrite() {
|
||||
let (_source, _, template) = source();
|
||||
for change in [
|
||||
"template",
|
||||
"runtime-schema",
|
||||
"session-schema",
|
||||
"session-id",
|
||||
"segment-id",
|
||||
"runtime-id",
|
||||
"workspace-id",
|
||||
"worker-id",
|
||||
"origin-id",
|
||||
"duplicate-id",
|
||||
"truncated",
|
||||
] {
|
||||
let (_target, options) = destination(&template);
|
||||
let mut invalid = template.clone();
|
||||
match change {
|
||||
"template" => invalid.template_version += 1,
|
||||
"runtime-schema" => invalid.runtime_schema_version += 1,
|
||||
"session-schema" => {
|
||||
invalid.session_manifest = format!(
|
||||
"{{\"schema_version\":999,\"session_id\":\"{}\"}}",
|
||||
invalid.session_id
|
||||
)
|
||||
.into_bytes()
|
||||
}
|
||||
"session-id" => invalid.session_id = uuid::Uuid::now_v7(),
|
||||
"segment-id" => invalid.segment_id = uuid::Uuid::nil(), // no external attestation: valid new Segment ID is otherwise portable
|
||||
"runtime-id" => invalid.runtime_id = "other-runtime".into(),
|
||||
"workspace-id" => invalid.workspace_id = "other-workspace".into(),
|
||||
"worker-id" => invalid.worker_id = WorkerId::now_v7(),
|
||||
"origin-id" => {
|
||||
invalid.segment_jsonl = String::from_utf8(invalid.segment_jsonl)
|
||||
.unwrap()
|
||||
.replace(RUNTIME, "other-runtime")
|
||||
.into_bytes()
|
||||
}
|
||||
"duplicate-id" => {
|
||||
let last = invalid
|
||||
.segment_jsonl
|
||||
.split_inclusive(|b| *b == b'\n')
|
||||
.last()
|
||||
.unwrap()
|
||||
.to_vec();
|
||||
invalid.segment_jsonl.extend(last);
|
||||
}
|
||||
"truncated" => {
|
||||
invalid.segment_jsonl.pop();
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
assert!(
|
||||
invalid.install(&options, request(), &resolved()).is_err(),
|
||||
"accepted {change}"
|
||||
);
|
||||
assert_fresh(&options);
|
||||
}
|
||||
let (_target, options) = destination(&template);
|
||||
let mut launch = request();
|
||||
launch.initial_input = Some(crate::interaction::WorkerInput::user("do not execute"));
|
||||
assert!(template.install(&options, launch, &resolved()).is_err());
|
||||
assert_fresh(&options);
|
||||
template.install(&options, request(), &resolved()).unwrap();
|
||||
let worker_path = options
|
||||
.root
|
||||
.join(WORKERS_DIR)
|
||||
.join(worker_id().to_string())
|
||||
.join(WORKER_FILE);
|
||||
let before = fs::read(&worker_path).unwrap();
|
||||
assert!(template.install(&options, request(), &resolved()).is_err());
|
||||
assert_eq!(fs::read(worker_path).unwrap(), before);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_uninitialized_untrusted_live_and_partial_destination() {
|
||||
let (_source, _, template) = source();
|
||||
let (_target, options) = destination(&template);
|
||||
let held = acquire_runtime_store_owner_lock(&options.root).unwrap();
|
||||
assert!(template.install(&options, request(), &resolved()).is_err());
|
||||
assert_fresh(&options);
|
||||
drop(held);
|
||||
initialize(&options.root, "wrong-runtime", WORKSPACE, NEW_URL);
|
||||
assert!(template.install(&options, request(), &resolved()).is_err());
|
||||
initialize(&options.root, RUNTIME, WORKSPACE, "http://127.0.0.1:23456");
|
||||
assert!(template.install(&options, request(), &resolved()).is_err());
|
||||
fs::remove_file(options.root.join("auth.toml")).unwrap();
|
||||
assert!(template.install(&options, request(), &resolved()).is_err());
|
||||
initialize(&options.root, RUNTIME, WORKSPACE, NEW_URL);
|
||||
fs::create_dir(options.root.join(WORKERS_DIR)).unwrap();
|
||||
assert!(template.install(&options, request(), &resolved()).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extraction_rejects_nonstopped_pending_schema_and_metadata_mismatch() {
|
||||
for change in [
|
||||
"running",
|
||||
"pending",
|
||||
"guard-pending",
|
||||
"attachment",
|
||||
"schema",
|
||||
"worker-id",
|
||||
"metadata-workspace",
|
||||
"metadata-session",
|
||||
"artifact",
|
||||
"secret",
|
||||
"url",
|
||||
"path",
|
||||
] {
|
||||
let (_source, worker, _) = source();
|
||||
let mut aggregate: WorkerAggregateRecord =
|
||||
decode(&fs::read(worker.join(WORKER_FILE)).unwrap()).unwrap();
|
||||
match change {
|
||||
"running" => aggregate.status = WorkerStatus::Running,
|
||||
"pending" => {
|
||||
aggregate.execution_state = WorkerExecutionStateRecord::ReconciliationRequired(
|
||||
PersistedWorkerLifecycleOperation::Restore(PersistedWorkerRestoreOperation {
|
||||
operation_id: WorkerLifecycleOperationId::new(),
|
||||
mode: PersistedWorkerRestoreMode::Explicit,
|
||||
request: request(),
|
||||
binding: None,
|
||||
last_settled_status: WorkerStatus::Stopped,
|
||||
}),
|
||||
)
|
||||
}
|
||||
"guard-pending" => aggregate.restore_guard = serde_json::from_str(r#"{"observation_token":"old-token","active_request_id":"pending-restore","owners":{}}"#).unwrap(),
|
||||
"attachment" => aggregate.logical_workdir_attachments.push(LogicalWorkdirAttachment {
|
||||
alias: workdir::WorkdirAttachmentAlias::new("source").unwrap(),
|
||||
working_directory_id: "old-workdir".into(),
|
||||
capabilities: workdir::WorkdirSessionCapabilities::READ_ONLY,
|
||||
}),
|
||||
"schema" => aggregate.schema_version += 1,
|
||||
"worker-id" => aggregate.worker_id = WorkerId::now_v7(),
|
||||
"metadata-workspace" | "metadata-session" => {
|
||||
let mut metadata: WorkerMetadata =
|
||||
decode(&fs::read(worker.join(WORKER_METADATA_FILE)).unwrap()).unwrap();
|
||||
if change == "metadata-workspace" {
|
||||
metadata.workspace_id = Some("wrong-workspace".into());
|
||||
} else {
|
||||
metadata.active.as_mut().unwrap().session_id = uuid::Uuid::now_v7();
|
||||
}
|
||||
atomic_write_json(
|
||||
&worker.join(WORKER_METADATA_FILE),
|
||||
&metadata,
|
||||
"test mismatch",
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
"artifact" => {
|
||||
fs::create_dir_all(worker.join("session/artifacts/paste")).unwrap();
|
||||
fs::write(
|
||||
worker.join("session/artifacts/paste/payload"),
|
||||
"unsupported",
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
"secret" | "url" | "path" => {
|
||||
let segment = fs::read_dir(worker.join("session/segments"))
|
||||
.unwrap()
|
||||
.next()
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.path();
|
||||
let replacement = match change {
|
||||
"secret" => "Bearer secret",
|
||||
"url" => "https://old-host.invalid",
|
||||
_ => "/old/host/path",
|
||||
};
|
||||
let text = fs::read_to_string(&segment)
|
||||
.unwrap()
|
||||
.replace("artificial fixture reply", replacement);
|
||||
fs::write(segment, text).unwrap();
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
atomic_write_json(&worker.join(WORKER_FILE), &aggregate, "test aggregate").unwrap();
|
||||
assert!(
|
||||
StoppedWorkerTemplate::extract(&worker).is_err(),
|
||||
"accepted {change}"
|
||||
);
|
||||
}
|
||||
let (_source, worker, _) = source();
|
||||
let held =
|
||||
acquire_runtime_store_owner_lock(worker.parent().unwrap().parent().unwrap()).unwrap();
|
||||
assert!(StoppedWorkerTemplate::extract(&worker).is_err());
|
||||
drop(held);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fresh_manifest_and_memory_settings_are_required() {
|
||||
let (_source, _, template) = source();
|
||||
let (_target, options) = destination(&template);
|
||||
let mut manifest = resolved();
|
||||
manifest.worker.name = "wrong-worker".into();
|
||||
assert!(template.install(&options, request(), &manifest).is_err());
|
||||
let mut binding = request();
|
||||
binding.memory_settings = None;
|
||||
assert!(template.install(&options, binding, &resolved()).is_err());
|
||||
let mut binding = request();
|
||||
binding.memory_settings.as_mut().unwrap().workspace_id = "other-workspace".into();
|
||||
assert!(template.install(&options, binding, &resolved()).is_err());
|
||||
assert_fresh(&options);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn settled_checkpoints_preserve_receipts_but_refuse_pending_work() {
|
||||
let mut checkpoint = settled_checkpoint();
|
||||
assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_ok());
|
||||
checkpoint.pending.push(1);
|
||||
assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_err());
|
||||
checkpoint.pending.clear();
|
||||
checkpoint.activating = Some(1);
|
||||
assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_err());
|
||||
checkpoint.activating = None;
|
||||
checkpoint.invocation_receipts.push(1);
|
||||
assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_err());
|
||||
checkpoint.invocation_receipts.clear();
|
||||
checkpoint.invocation_recovery_blocked = true;
|
||||
assert!(validate_settled_activation(&checkpoint_extension(&checkpoint)).is_err());
|
||||
}
|
||||
|
||||
// Deliberately corrupt only test-owned synthetic source files, never the lab.
|
||||
fn assert_rejected_session_bytes(worker: &Path, template: &StoppedWorkerTemplate, bytes: Vec<u8>) {
|
||||
let path = worker.join(format!("session/segments/{}.jsonl", template.segment_id()));
|
||||
fs::write(path, &bytes).unwrap();
|
||||
let extraction_error = StoppedWorkerTemplate::extract(worker)
|
||||
.err()
|
||||
.expect("invalid source accepted");
|
||||
assert!(
|
||||
!extraction_error
|
||||
.to_string()
|
||||
.contains("synthetic-discarded-material")
|
||||
);
|
||||
let (_target, options) = destination(template);
|
||||
let mut invalid = template.clone();
|
||||
invalid.segment_jsonl = bytes;
|
||||
let installation_error = invalid
|
||||
.install(&options, request(), &resolved())
|
||||
.unwrap_err();
|
||||
assert!(
|
||||
!installation_error
|
||||
.to_string()
|
||||
.contains("synthetic-discarded-material")
|
||||
);
|
||||
assert_fresh(&options);
|
||||
}
|
||||
|
||||
fn synthetic_logs(template: &StoppedWorkerTemplate) -> Vec<LogEntry> {
|
||||
template
|
||||
.segment_jsonl
|
||||
.split(|byte| *byte == b'\n')
|
||||
.filter(|line| !line.is_empty())
|
||||
.map(|line| serde_json::from_slice(line).unwrap())
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn encode_synthetic_logs(logs: &[LogEntry]) -> Vec<u8> {
|
||||
let mut bytes = Vec::new();
|
||||
for log in logs {
|
||||
serde_json::to_writer(&mut bytes, log).unwrap();
|
||||
bytes.push(b'\n');
|
||||
}
|
||||
bytes
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn missing_system_prompt_matches_worker_restore_rejection() {
|
||||
let (_source, worker_dir, template) = source();
|
||||
let mut logs = synthetic_logs(&template);
|
||||
assert!(session_store::collect_state(&logs).system_prompt.is_some());
|
||||
let LogEntry::AnnotatedSegmentStart { system_prompt, .. } = &mut logs[0] else {
|
||||
panic!("missing seed")
|
||||
};
|
||||
*system_prompt = None;
|
||||
assert_rejected_session_bytes(&worker_dir, &template, encode_synthetic_logs(&logs));
|
||||
// The owning Worker constructor rejects this before model-client creation,
|
||||
// allocation, or execution. Do not start a controller or submit any input.
|
||||
let store = session_store::CombinedStore::new(
|
||||
WorkerSessionStore::open_read_only(worker_dir.join("session")).unwrap(),
|
||||
session_store::WorkerAggregateStore::open_read_only(&worker_dir, template.worker_name())
|
||||
.unwrap(),
|
||||
);
|
||||
let restored = worker::Worker::restore_from_manifest(
|
||||
template.session_id(),
|
||||
template.segment_id(),
|
||||
resolved(),
|
||||
store,
|
||||
worker::PromptCatalogSource::builtins_only(),
|
||||
)
|
||||
.await;
|
||||
assert!(matches!(
|
||||
restored,
|
||||
Err(worker::WorkerError::SegmentSystemPromptMissing { .. })
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_submission_and_notification_receipts_fail_extraction_and_install() {
|
||||
let (_source, worker, template) = source();
|
||||
for notification in [false, true] {
|
||||
let mut checkpoint = settled_checkpoint();
|
||||
if notification {
|
||||
checkpoint.notification_receipts =
|
||||
vec![ReceiptFixture::Malformed(1), ReceiptFixture::Malformed(2)];
|
||||
} else {
|
||||
checkpoint.receipts = vec![ReceiptFixture::Malformed(1), ReceiptFixture::Malformed(2)];
|
||||
}
|
||||
let mut logs = synthetic_logs(&template);
|
||||
for log in &mut logs {
|
||||
if let LogEntry::AnnotatedUserInput { extensions, .. } = log {
|
||||
*extensions = vec![checkpoint_extension(&checkpoint)];
|
||||
}
|
||||
}
|
||||
assert_rejected_session_bytes(&worker, &template, encode_synthetic_logs(&logs));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_fields_at_every_retained_record_depth_are_rejected_without_secret_diagnostics() {
|
||||
let (_source, worker, template) = source();
|
||||
let original = String::from_utf8(template.segment_jsonl.clone()).unwrap();
|
||||
// These values are harmless synthetic markers, not lab/private material.
|
||||
// The compatibility decoder normally discards many of these fields.
|
||||
let locations = [
|
||||
r#""kind":"annotated_segment_start""#,
|
||||
r#""kind":"invoke""#,
|
||||
r#""kind":"annotated_user_input""#,
|
||||
r#""kind":"annotated_assistant_item""#,
|
||||
r#""kind":"llm_usage""#,
|
||||
r#""kind":"turn_end""#,
|
||||
r#""kind":"run_completed""#,
|
||||
r#""max_tokens":null"#,
|
||||
r#""kind":"message","role":"assistant""#,
|
||||
r#""kind":"text","text":"artificial fixture reply""#,
|
||||
r#""kind":"text","content":"Harmless artificial fixture input.""#,
|
||||
r#""entry_id":"fixed-entry-id""#,
|
||||
r#""kind":"model_output""#,
|
||||
r#""worker_id":"01a1657d-b762-7791-87ae-a49f28cadbce""#,
|
||||
r#""domain":"worker.pending_activations.v1""#,
|
||||
r#""next_activation_sequence":2"#,
|
||||
r#""submission_request_id":"completed-submission-request""#,
|
||||
r#""notification_request_id":"completed-notification-request""#,
|
||||
];
|
||||
for location in locations {
|
||||
assert!(original.contains(location));
|
||||
for extra in [
|
||||
r#", "reasoning":{"text":"synthetic-discarded-material"}"#,
|
||||
r#", "encrypted_content":"synthetic-discarded-material""#,
|
||||
] {
|
||||
let altered = original.replacen(location, &format!("{location}{extra}"), 1);
|
||||
assert_rejected_session_bytes(&worker, &template, altered.into_bytes());
|
||||
}
|
||||
}
|
||||
let duplicate = original.replacen(
|
||||
r#""role":"assistant""#,
|
||||
r#""role":"assistant","role":"assistant""#,
|
||||
1,
|
||||
);
|
||||
assert_rejected_session_bytes(&worker, &template, duplicate.into_bytes());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn opaque_feature_payloads_cannot_bypass_strict_session_validation() {
|
||||
let (_source, worker, template) = source();
|
||||
let mut logs = synthetic_logs(&template);
|
||||
for log in &mut logs {
|
||||
if let LogEntry::AnnotatedUserInput { history, .. } = log {
|
||||
if let session_store::LoggedItem::Message { role, .. } = &mut history[0].item {
|
||||
*role = session_store::LoggedRole::System;
|
||||
}
|
||||
history[0].metadata.input_metadata = Some(
|
||||
session_store::HistoryInputMetadata::new(
|
||||
session_store::HistoryInputUsePolicy {
|
||||
recording_material: session_store::InputEvidenceUse::Exclude,
|
||||
memory_evidence: session_store::InputEvidenceUse::Exclude,
|
||||
observation: session_store::InputObservationUse::Hidden,
|
||||
compaction: session_store::InputCompactionUse::Retain,
|
||||
},
|
||||
Some(
|
||||
session_store::FeatureInputPayload::encode(
|
||||
"fixture.test",
|
||||
1,
|
||||
&BTreeMap::from([("reasoning", "harmless synthetic payload")]),
|
||||
)
|
||||
.unwrap(),
|
||||
),
|
||||
None,
|
||||
)
|
||||
.unwrap(),
|
||||
);
|
||||
}
|
||||
}
|
||||
assert_rejected_session_bytes(&worker, &template, encode_synthetic_logs(&logs));
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
#[test]
|
||||
fn refuses_source_and_destination_symlinks() {
|
||||
let (_source, worker, template) = source();
|
||||
let (_target, options) = destination(&template);
|
||||
std::os::unix::fs::symlink(worker.join(WORKER_FILE), options.root.join("linked")).unwrap();
|
||||
assert!(template.install(&options, request(), &resolved()).is_err());
|
||||
fs::remove_file(options.root.join("linked")).unwrap();
|
||||
std::os::unix::fs::symlink(worker.join(WORKER_FILE), worker.join("runs/link")).unwrap();
|
||||
assert!(StoppedWorkerTemplate::extract(&worker).is_err());
|
||||
}
|
||||
|
||||
/// Opt-in only: source path supplied explicitly; assertions never display content.
|
||||
#[test]
|
||||
#[ignore = "requires the approved artificial lab aggregate via YOI_STOPPED_FIXTURE_SOURCE"]
|
||||
fn approved_lab_extract_install() {
|
||||
let path = PathBuf::from(
|
||||
std::env::var_os("YOI_STOPPED_FIXTURE_SOURCE").expect("explicit source required"),
|
||||
);
|
||||
let template = StoppedWorkerTemplate::extract(&path).unwrap();
|
||||
let (_target, options) = destination(&template);
|
||||
let mut bind = request();
|
||||
bind.worker_id = template.worker_id();
|
||||
bind.workspace_api.as_mut().unwrap().workspace_id = template.workspace_id().into();
|
||||
bind.memory_settings.as_mut().unwrap().workspace_id = template.workspace_id().into();
|
||||
let mut manifest = resolved();
|
||||
manifest.worker.name = template.worker_name();
|
||||
template.install(&options, bind, &manifest).unwrap();
|
||||
let installed = options
|
||||
.root
|
||||
.join(WORKERS_DIR)
|
||||
.join(template.worker_id().to_string());
|
||||
for relative in [
|
||||
"session/session.json".to_string(),
|
||||
format!("session/segments/{}.jsonl", template.segment_id()),
|
||||
] {
|
||||
assert!(
|
||||
fs::read(path.join(&relative)).unwrap() == fs::read(installed.join(relative)).unwrap()
|
||||
);
|
||||
}
|
||||
let runtime = crate::Runtime::with_fs_store(options).unwrap();
|
||||
assert_eq!(runtime.list_workers().unwrap().len(), 1);
|
||||
assert_eq!(runtime.summary().unwrap().active_worker_count, 0);
|
||||
}
|
||||
@@ -146,9 +146,9 @@ use worker::feature::builtin::{
|
||||
use worker::ipc::protocol_session::{WorkerProtocolSessionStreams, live_log_entry_event};
|
||||
use worker::{
|
||||
PreparedWorker, PromptCatalogSource, SegmentLogSink, SubjektivSessionAttributionLifecycle,
|
||||
Worker, WorkerBootstrap, WorkerBootstrapError, WorkerBootstrapLayout,
|
||||
WorkerControllerTransport, WorkerError, WorkerFilesystemAuthority, WorkerHandle,
|
||||
WorkerSharedState, WorkerWorkspaceContext, WorkspaceClient, WorkspaceId,
|
||||
Worker, WorkerBootstrap, WorkerBootstrapError, WorkerBootstrapLayout, WorkerError,
|
||||
WorkerFilesystemAuthority, WorkerHandle, WorkerSharedState, WorkerWorkspaceContext,
|
||||
WorkspaceClient, WorkspaceId,
|
||||
};
|
||||
|
||||
#[cfg(feature = "ws-server")]
|
||||
@@ -508,7 +508,6 @@ pub struct ProfileRuntimeWorkerFactory {
|
||||
worker_mutation_identity: Option<RuntimeIdentityMaterial>,
|
||||
workspace_request_clients: Arc<HashMap<String, RuntimeWorkspaceRequestClient>>,
|
||||
embedded_worker_mutation_dispatcher: Option<Arc<dyn EmbeddedWorkerMutationDispatcher>>,
|
||||
controller_transport: WorkerControllerTransport,
|
||||
failed_restore_sessions: Arc<
|
||||
Mutex<
|
||||
HashMap<
|
||||
@@ -536,7 +535,6 @@ impl ProfileRuntimeWorkerFactory {
|
||||
worker_mutation_identity: None,
|
||||
workspace_request_clients: Arc::new(HashMap::new()),
|
||||
embedded_worker_mutation_dispatcher: None,
|
||||
controller_transport: WorkerControllerTransport::InProcess,
|
||||
failed_restore_sessions: Arc::new(Mutex::new(HashMap::new())),
|
||||
}
|
||||
}
|
||||
@@ -585,14 +583,6 @@ impl ProfileRuntimeWorkerFactory {
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_controller_transport(
|
||||
mut self,
|
||||
controller_transport: WorkerControllerTransport,
|
||||
) -> Self {
|
||||
self.controller_transport = controller_transport;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn with_runtime_store_dir(mut self, runtime_store_dir: impl Into<PathBuf>) -> Self {
|
||||
self.worker_aggregate_root = Some(runtime_store_dir.into().join("workers"));
|
||||
self
|
||||
@@ -1137,8 +1127,17 @@ fn runtime_local_workdir_router(
|
||||
),
|
||||
None => source,
|
||||
};
|
||||
let cwd = binding
|
||||
.cwd()
|
||||
.strip_prefix(binding.root())
|
||||
.map_err(|_| format!("Workdir attachment `{alias}` cwd is outside its logical root"))?;
|
||||
let cwd = workdir::WorkdirPath::new(
|
||||
cwd.to_str()
|
||||
.ok_or_else(|| format!("Workdir attachment `{alias}` cwd is not UTF-8"))?,
|
||||
)
|
||||
.map_err(|error| error.to_string())?;
|
||||
router
|
||||
.attach(alias.clone(), source)
|
||||
.attach_at_cwd(alias.clone(), source, cwd)
|
||||
.map_err(|error| format!("bind Workdir attachment `{alias}`: {error}"))?;
|
||||
}
|
||||
Ok(router)
|
||||
@@ -1389,15 +1388,6 @@ fn bind_workspace_memory_settings(
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Read-only validation shared with the offline, feature-gated fixture installer.
|
||||
#[cfg(feature = "test-support")]
|
||||
pub(crate) fn validate_fixture_restore_metadata(
|
||||
metadata: &session_store::WorkerMetadata,
|
||||
request: &CreateWorkerRequest,
|
||||
) -> Result<(), String> {
|
||||
ProfileRuntimeWorkerFactory::manifest_for_restore(metadata, request).map(|_| ())
|
||||
}
|
||||
|
||||
fn validate_worker_memory_settings(
|
||||
manifest: &manifest::WorkerManifest,
|
||||
request: &CreateWorkerRequest,
|
||||
@@ -1607,13 +1597,13 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory {
|
||||
.resolve_profile(selector, &worker_root, &worker_name)
|
||||
.map_err(|err| format!("failed to resolve profile source archive: {err}"))?;
|
||||
if has_local_filesystem {
|
||||
worker::entrypoint::resolve_runtime_profile_manifest_from_manifest(
|
||||
worker::profile_manifest::resolve_runtime_profile_manifest_from_manifest(
|
||||
manifest,
|
||||
&worker_root,
|
||||
&worker_name,
|
||||
)?
|
||||
} else {
|
||||
worker::entrypoint::resolve_runtime_profile_manifest_from_manifest_without_filesystem(
|
||||
worker::profile_manifest::resolve_runtime_profile_manifest_from_manifest_without_filesystem(
|
||||
manifest,
|
||||
&worker_root,
|
||||
&worker_name,
|
||||
@@ -1660,7 +1650,6 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory {
|
||||
WorkerBootstrapLayout::RuntimeManagedRun {
|
||||
run_dir: run_dir.clone(),
|
||||
},
|
||||
self.controller_transport,
|
||||
);
|
||||
let bootstrap = match &self.spawn_model_client_factory {
|
||||
Some(factory) => bootstrap.with_model_client(factory()),
|
||||
@@ -2013,7 +2002,6 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory {
|
||||
WorkerBootstrapLayout::RuntimeManagedRun {
|
||||
run_dir: run_dir.clone(),
|
||||
},
|
||||
self.controller_transport,
|
||||
)
|
||||
.start()
|
||||
.await
|
||||
@@ -6524,8 +6512,7 @@ mod tests {
|
||||
let backend = WorkerRuntimeExecutionBackend::new(
|
||||
ProfileRuntimeWorkerFactory::new(root.path().join("missing-profiles"))
|
||||
.with_runtime_store_dir(&store)
|
||||
.with_runtime_id("runtime-subjektiv-test")
|
||||
.with_controller_transport(WorkerControllerTransport::InProcess),
|
||||
.with_runtime_id("runtime-subjektiv-test"),
|
||||
)
|
||||
.unwrap();
|
||||
backend.preflight_restore(&request).unwrap();
|
||||
@@ -9141,17 +9128,6 @@ mod tests {
|
||||
assert!(error.contains("replacement Worker is required"), "{error}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn profile_runtime_factory_defaults_to_in_process_controller_transport() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let factory = ProfileRuntimeWorkerFactory::new(root.path());
|
||||
|
||||
assert!(matches!(
|
||||
factory.controller_transport,
|
||||
WorkerControllerTransport::InProcess
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
#[serial_test::serial(worker_allocation)]
|
||||
async fn in_process_restore_does_not_bind_unix_socket_under_overlong_store_path() {
|
||||
@@ -9214,7 +9190,6 @@ mod tests {
|
||||
|
||||
let controller = ProfileRuntimeWorkerFactory::new(root.path())
|
||||
.with_runtime_store_dir(&runtime_store_dir)
|
||||
.with_controller_transport(WorkerControllerTransport::InProcess)
|
||||
.restore_controller(WorkerExecutionRestoreRequest {
|
||||
operation_id: crate::execution::WorkerLifecycleOperationId::new(),
|
||||
worker_ref: worker_ref.clone(),
|
||||
@@ -9923,12 +9898,25 @@ mod tests {
|
||||
})
|
||||
.await;
|
||||
session.close().await.unwrap();
|
||||
Ok(result.unwrap_err().to_string())
|
||||
Ok(result.unwrap_err())
|
||||
})
|
||||
.unwrap();
|
||||
// The String-returning opener cannot safely classify its raw cause. The
|
||||
// deferred boundary must preserve its typed stage without exposing that text.
|
||||
assert_eq!(
|
||||
error.failure_reason(),
|
||||
Some(workdir::WorkdirFailureReason::SessionOpenFailed),
|
||||
"{damage}: {error:?}"
|
||||
);
|
||||
assert!(
|
||||
error.contains("working_directory_cleanup_"),
|
||||
"{damage}: {error}"
|
||||
matches!(error.classification_source(), workdir::WorkdirError::Unavailable(message)
|
||||
if message == "Workdir session could not be opened"),
|
||||
"{damage}: {error:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
error.denial_reason(),
|
||||
None,
|
||||
"{damage}: no guessed denial cause"
|
||||
);
|
||||
assert_eq!(
|
||||
observed.lock().unwrap().len(),
|
||||
@@ -9941,6 +9929,13 @@ mod tests {
|
||||
"damaged checkout must be retained"
|
||||
);
|
||||
assert!(backend.stop_worker(&request.worker_ref).is_accepted());
|
||||
println!(
|
||||
"damage={damage}; reason={:?}; classification={:?}; cwd_bind_count={}; retained_files={}; retained_data_unchanged=true; stopped=true",
|
||||
error.failure_reason(),
|
||||
error.classification_source(),
|
||||
observed.lock().unwrap().len(),
|
||||
before.len(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10564,6 +10559,60 @@ mod tests {
|
||||
assert!(!binding.working_directory_root().exists());
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test]
|
||||
async fn local_attachment_router_keeps_each_binding_cwd_and_capability_ceiling() {
|
||||
let runtime_base = tempfile::tempdir().unwrap();
|
||||
let repo = create_clean_repo();
|
||||
let materializer = RuntimeGitMaterializer::new(runtime_base.path());
|
||||
let root_binding = materializer
|
||||
.create(&working_directory_request(repo.path()))
|
||||
.unwrap();
|
||||
std::fs::create_dir(root_binding.root().join("nested")).unwrap();
|
||||
std::fs::write(root_binding.root().join("marker"), "root").unwrap();
|
||||
std::fs::write(root_binding.root().join("nested/marker"), "nested").unwrap();
|
||||
let mut nested_binding = root_binding.clone();
|
||||
nested_binding.cwd = root_binding.root().join("nested");
|
||||
let root_alias = WorkdirAttachmentAlias::new("root").unwrap();
|
||||
let nested_alias = WorkdirAttachmentAlias::new("nested").unwrap();
|
||||
let attachments = BTreeMap::from([
|
||||
(root_alias.clone(), root_binding),
|
||||
(nested_alias.clone(), nested_binding),
|
||||
]);
|
||||
let ceilings = BTreeMap::from([
|
||||
(root_alias, WorkdirSessionCapabilities::ALL),
|
||||
(nested_alias, WorkdirSessionCapabilities::READ_ONLY),
|
||||
]);
|
||||
let router = runtime_local_workdir_router(&attachments, &ceilings, None).unwrap();
|
||||
assert!(router.resolve(None).is_err());
|
||||
for (alias, cwd, content, command) in [
|
||||
("root", "", "root", true),
|
||||
("nested", "nested", "nested", false),
|
||||
] {
|
||||
let selected = router.resolve(Some(alias)).unwrap().session;
|
||||
let environment = selected.environment();
|
||||
assert_eq!(environment.cwd.as_str(), cwd);
|
||||
assert_eq!(
|
||||
environment
|
||||
.capabilities
|
||||
.supports(workdir::WorkdirSessionCapability::Command),
|
||||
command
|
||||
);
|
||||
let read = selected
|
||||
.read(workdir::ReadRequest {
|
||||
path: workdir::WorkdirPath::new("marker").unwrap(),
|
||||
offset: 0,
|
||||
limit: 10,
|
||||
max_bytes: 100,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(read.bytes, content.as_bytes());
|
||||
assert_eq!(read.path.as_str(), "marker");
|
||||
}
|
||||
router.close_all().await.unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn spawn_failure_with_existing_working_directory_preserves_workdir() {
|
||||
let runtime_base = tempfile::tempdir().unwrap();
|
||||
|
||||
@@ -66,13 +66,32 @@ impl DeferredWorkdirSession {
|
||||
let open = self.open.clone();
|
||||
let session = tokio::task::spawn_blocking(move || open())
|
||||
.await
|
||||
.map_err(|_| WorkdirError::Unavailable("Workdir session opener failed".into()))?
|
||||
.map_err(WorkdirError::Unavailable)?;
|
||||
.map_err(|_| {
|
||||
WorkdirError::Unavailable("Workdir session opener failed".into())
|
||||
.with_failure_reason(WorkdirFailureReason::SessionOpenTaskFailed)
|
||||
})?
|
||||
.map_err(|_| {
|
||||
// The opener's string contract cannot prove a more specific cause.
|
||||
WorkdirError::Unavailable("Workdir session could not be opened".into())
|
||||
.with_failure_reason(WorkdirFailureReason::SessionOpenFailed)
|
||||
})?;
|
||||
if session.workdir().id() != self.workdir.id() {
|
||||
session.close().await?;
|
||||
return Err(WorkdirError::Unavailable(
|
||||
"Workdir session identity mismatch".into(),
|
||||
));
|
||||
if let Err(error) = session.close().await {
|
||||
// Retain a failed-close owner for explicit close; never use it.
|
||||
state.closed = true;
|
||||
state.session = Some(session);
|
||||
tracing::warn!(
|
||||
target: "worker_runtime::workdir",
|
||||
event = "workdir_session_identity_cleanup_failed",
|
||||
failure_reason = error.failure_reason().map(WorkdirFailureReason::as_str),
|
||||
denial_reason = error.denial_reason().map(WorkdirDenialReason::as_str),
|
||||
"Mismatched Workdir session cleanup failed"
|
||||
);
|
||||
}
|
||||
return Err(
|
||||
WorkdirError::Unavailable("Workdir session identity mismatch".into())
|
||||
.with_failure_reason(WorkdirFailureReason::SessionIdentityMismatch),
|
||||
);
|
||||
}
|
||||
if let Some(mut source) = session.subscribe_command_events() {
|
||||
let events = self.events.clone();
|
||||
@@ -105,6 +124,18 @@ macro_rules! impl_deferred_session {
|
||||
self.capabilities
|
||||
}
|
||||
|
||||
fn environment(&self) -> WorkdirSessionEnvironment {
|
||||
let mut environment = self.state.try_lock().ok().and_then(|state| {
|
||||
state.session.as_ref().map(|session| session.environment())
|
||||
}).unwrap_or_else(|| {
|
||||
let mut environment = WorkdirSessionEnvironment::root(self.capabilities);
|
||||
environment.restrictions.push("Deferred provider: scope is checked on connection and every operation".into());
|
||||
environment
|
||||
});
|
||||
environment.capabilities = self.capabilities;
|
||||
environment
|
||||
}
|
||||
|
||||
$(
|
||||
async fn $method(&self, $request: $request_type) -> Result<$result, WorkdirError> {
|
||||
self.session().await?.$method($request).await
|
||||
@@ -224,10 +255,16 @@ mod tests {
|
||||
},
|
||||
);
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 0);
|
||||
let failed = session.read(read_request()).await.unwrap_err();
|
||||
assert!(matches!(
|
||||
session.read(read_request()).await,
|
||||
Err(WorkdirError::Unavailable(_))
|
||||
failed.classification_source(),
|
||||
WorkdirError::Unavailable(_)
|
||||
));
|
||||
assert_eq!(
|
||||
failed.failure_reason(),
|
||||
Some(WorkdirFailureReason::SessionOpenFailed)
|
||||
);
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 1);
|
||||
assert_eq!(
|
||||
session.read(read_request()).await.unwrap().bytes,
|
||||
b"restored"
|
||||
@@ -253,4 +290,126 @@ mod tests {
|
||||
);
|
||||
session.close().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn opener_task_and_open_errors_keep_distinct_safe_causes() {
|
||||
for panic in [false, true] {
|
||||
let attempts = Arc::new(AtomicUsize::new(0));
|
||||
let observed = attempts.clone();
|
||||
let session = DeferredWorkdirSession::handle(
|
||||
"saved",
|
||||
WorkdirSessionCapabilities::READ_ONLY,
|
||||
move || {
|
||||
observed.fetch_add(1, Ordering::SeqCst);
|
||||
if panic {
|
||||
panic!(
|
||||
"/host/private bearer-secret command-secret proof-secret key-secret"
|
||||
);
|
||||
}
|
||||
Err("/host/private bearer-secret command-secret proof-secret key-secret".into())
|
||||
},
|
||||
);
|
||||
let error = session.read(read_request()).await.unwrap_err();
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 1);
|
||||
let reason = if panic {
|
||||
WorkdirFailureReason::SessionOpenTaskFailed
|
||||
} else {
|
||||
WorkdirFailureReason::SessionOpenFailed
|
||||
};
|
||||
assert_eq!(error.failure_reason(), Some(reason));
|
||||
assert_eq!(reason.stage().as_str(), "session_open");
|
||||
let transport = workdir::http::WorkdirTransportError::from_workdir_error(&error);
|
||||
assert_eq!(transport.code.http_status(), 503);
|
||||
assert!(
|
||||
!serde_json::to_string(&transport)
|
||||
.unwrap()
|
||||
.contains("secret")
|
||||
);
|
||||
assert!(!format!("{error:?}").contains("secret"));
|
||||
session.close().await.unwrap();
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct MismatchedSession {
|
||||
workdir: Workdir,
|
||||
closes: Arc<AtomicUsize>,
|
||||
fail_close: bool,
|
||||
}
|
||||
|
||||
macro_rules! mismatched_session {
|
||||
($( $method:ident($request:ident: $request_type:ty) -> $result:ty; )*) => {
|
||||
#[async_trait]
|
||||
impl WorkdirSession for MismatchedSession {
|
||||
fn workdir(&self) -> &Workdir { &self.workdir }
|
||||
fn capabilities(&self) -> WorkdirSessionCapabilities { WorkdirSessionCapabilities::READ_ONLY }
|
||||
$(async fn $method(&self, _: $request_type) -> Result<$result, WorkdirError> {
|
||||
panic!("mismatched session must never be used");
|
||||
})*
|
||||
async fn close(&self) -> Result<(), WorkdirError> {
|
||||
let attempt = self.closes.fetch_add(1, Ordering::SeqCst);
|
||||
if self.fail_close && attempt == 0 {
|
||||
Err(WorkdirError::Transport("/host/private key-secret".into())
|
||||
.with_failure_reason(WorkdirFailureReason::HttpTransfer))
|
||||
} else { Ok(()) }
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
mismatched_session! {
|
||||
stat(request: StatRequest) -> StatResult;
|
||||
read(request: ReadRequest) -> ReadResult;
|
||||
write(request: WriteRequest) -> WriteResult;
|
||||
edit(request: EditRequest) -> EditResult;
|
||||
list(request: ListRequest) -> ListResult;
|
||||
glob(request: GlobRequest) -> GlobResult;
|
||||
grep(request: GrepRequest) -> GrepResult;
|
||||
start_command(request: CommandRequest) -> CommandHandle;
|
||||
command_status(handle: CommandHandle) -> CommandStatus;
|
||||
command_output(request: CommandOutputRequest) -> CommandOutput;
|
||||
cancel_command(handle: CommandHandle) -> ();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn identity_mismatch_preserves_primary_cause_and_failed_close_owner() {
|
||||
for fail_close in [false, true] {
|
||||
let closes = Arc::new(AtomicUsize::new(0));
|
||||
let raw: WorkdirSessionHandle = Arc::new(MismatchedSession {
|
||||
workdir: Workdir::new("wrong-secret"),
|
||||
closes: closes.clone(),
|
||||
fail_close,
|
||||
});
|
||||
let attempts = Arc::new(AtomicUsize::new(0));
|
||||
let observed = attempts.clone();
|
||||
let session = DeferredWorkdirSession::handle(
|
||||
"saved",
|
||||
WorkdirSessionCapabilities::READ_ONLY,
|
||||
move || {
|
||||
observed.fetch_add(1, Ordering::SeqCst);
|
||||
Ok(raw.clone())
|
||||
},
|
||||
);
|
||||
let error = session.read(read_request()).await.unwrap_err();
|
||||
assert_eq!(
|
||||
error.failure_reason(),
|
||||
Some(WorkdirFailureReason::SessionIdentityMismatch)
|
||||
);
|
||||
assert!(!format!("{error:?}").contains("secret"));
|
||||
assert_eq!(closes.load(Ordering::SeqCst), 1);
|
||||
if fail_close {
|
||||
assert!(matches!(
|
||||
session.read(read_request()).await,
|
||||
Err(WorkdirError::SessionClosed)
|
||||
));
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
session.close().await.unwrap();
|
||||
assert_eq!(
|
||||
closes.load(Ordering::SeqCst),
|
||||
if fail_close { 2 } else { 1 }
|
||||
);
|
||||
assert_eq!(attempts.load(Ordering::SeqCst), 1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3677,7 +3677,11 @@ mod tests {
|
||||
#[test]
|
||||
fn bound_ssh_workdir_uses_attachment_scoped_agent_and_cleans_it_up() {
|
||||
let repo = create_clean_repo();
|
||||
let runtime_root = tempfile::tempdir().unwrap();
|
||||
// ssh-agent and the broker receive relative socket paths in this owned
|
||||
// fixture tree, so the test does not depend on checkout/TMPDIR length.
|
||||
// Neither this process nor its peers change cwd.
|
||||
let runtime_root = tempfile::tempdir_in(".").unwrap();
|
||||
let runtime_path = Path::new(runtime_root.path().file_name().unwrap());
|
||||
let key_root = tempfile::tempdir().unwrap();
|
||||
let key_path = key_root.path().join("id_ed25519");
|
||||
let fallback_key_path = key_root.path().join("id_ed25519_fallback");
|
||||
@@ -3689,7 +3693,7 @@ mod tests {
|
||||
.unwrap();
|
||||
assert!(status.success());
|
||||
}
|
||||
let materializer = RuntimeGitMaterializer::new(runtime_root.path());
|
||||
let materializer = RuntimeGitMaterializer::new(runtime_path);
|
||||
let mut request = request(repo.path());
|
||||
request.materialization = Some(crate::catalog::RepositoryMaterializationContext {
|
||||
workspace_id: "workspace-1".to_string(),
|
||||
@@ -3731,7 +3735,7 @@ mod tests {
|
||||
kind: server_api::RepositorySourceKind::Ssh,
|
||||
uri: "ssh://git@example.test/repo.git".to_string(),
|
||||
};
|
||||
let command_access = RepositoryCommandAccess::prepare(runtime_root.path(), &ssh_operation)
|
||||
let command_access = RepositoryCommandAccess::prepare(runtime_path, &ssh_operation)
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
assert!(!command_access.root.join("identity").exists());
|
||||
@@ -3873,18 +3877,17 @@ mod tests {
|
||||
.unwrap();
|
||||
// Restore preflight must validate without consuming this one-shot SSH
|
||||
// delivery or creating agent/session resources needed by actual bind.
|
||||
let record_before =
|
||||
fs::read(runtime_root.path().join(&id).join(MATERIALIZATION_RECORD)).unwrap();
|
||||
let record_before = fs::read(runtime_path.join(&id).join(MATERIALIZATION_RECORD)).unwrap();
|
||||
materializer
|
||||
.preflight_bind_working_directory(&id, None)
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
fs::read(runtime_root.path().join(&id).join(MATERIALIZATION_RECORD)).unwrap(),
|
||||
fs::read(runtime_path.join(&id).join(MATERIALIZATION_RECORD)).unwrap(),
|
||||
record_before
|
||||
);
|
||||
assert_eq!(materializer.list_working_directories().unwrap().len(), 1);
|
||||
assert_eq!(
|
||||
fs::read_dir(runtime_root.path().join(".repository-agents"))
|
||||
fs::read_dir(runtime_path.join(".repository-agents"))
|
||||
.map(|entries| entries.count())
|
||||
.unwrap_or_default(),
|
||||
0
|
||||
@@ -3898,7 +3901,7 @@ mod tests {
|
||||
assert!(ssh_policy.contains("__repository-ssh"));
|
||||
assert!(!ssh_policy.contains("SSH_AUTH_SOCK"));
|
||||
assert_eq!(
|
||||
fs::read_dir(runtime_root.path().join(".repository-agents"))
|
||||
fs::read_dir(runtime_path.join(".repository-agents"))
|
||||
.map(|entries| entries.count())
|
||||
.unwrap_or_default(),
|
||||
1
|
||||
@@ -3916,7 +3919,7 @@ mod tests {
|
||||
"consumed delivery expiry must not revoke a live Workdir session"
|
||||
);
|
||||
assert_eq!(
|
||||
fs::read_dir(runtime_root.path().join(".repository-agents"))
|
||||
fs::read_dir(runtime_path.join(".repository-agents"))
|
||||
.map(|entries| entries.count())
|
||||
.unwrap_or_default(),
|
||||
1,
|
||||
@@ -3925,7 +3928,7 @@ mod tests {
|
||||
drop(binding);
|
||||
assert!(!ssh_command.exists());
|
||||
assert_eq!(
|
||||
fs::read_dir(runtime_root.path().join(".repository-agents"))
|
||||
fs::read_dir(runtime_path.join(".repository-agents"))
|
||||
.map(|entries| entries.count())
|
||||
.unwrap_or_default(),
|
||||
0
|
||||
@@ -3971,7 +3974,7 @@ mod tests {
|
||||
],
|
||||
&read_only_command_policy,
|
||||
));
|
||||
let broker_socket = fs::read_dir(runtime_root.path().join(REPOSITORY_ACCESS_DIR))
|
||||
let broker_socket = fs::read_dir(runtime_path.join(REPOSITORY_ACCESS_DIR))
|
||||
.unwrap()
|
||||
.filter_map(Result::ok)
|
||||
.map(|entry| entry.path().join("b.sock"))
|
||||
@@ -4104,7 +4107,7 @@ mod tests {
|
||||
.code,
|
||||
"working_directory_repository_access_expired"
|
||||
);
|
||||
let restored = RuntimeGitMaterializer::new(runtime_root.path());
|
||||
let restored = RuntimeGitMaterializer::new(runtime_path);
|
||||
assert_eq!(
|
||||
restored.bind_working_directory(&id, None).unwrap_err().code,
|
||||
"working_directory_remote_repository_access_required"
|
||||
|
||||
@@ -11,7 +11,6 @@ default = []
|
||||
|
||||
[dependencies]
|
||||
async-trait = { workspace = true }
|
||||
clap = { version = "4.6.0", features = ["derive"] }
|
||||
agen = { workspace = true, features = ["codex"] }
|
||||
session-store = { workspace = true }
|
||||
secrets = { workspace = true }
|
||||
@@ -58,6 +57,7 @@ wip-protocol = { workspace = true }
|
||||
wip-text-view = { workspace = true }
|
||||
|
||||
[dev-dependencies]
|
||||
tracing-subscriber.workspace = true
|
||||
axum = { workspace = true }
|
||||
workdir = { workspace = true, features = ["http-client"] }
|
||||
dotenv = "0.15.0"
|
||||
|
||||
@@ -206,26 +206,36 @@ impl AuthorizedWorkdirSession {
|
||||
.map(CheckoutSearchResult::Grep),
|
||||
};
|
||||
};
|
||||
scoped
|
||||
.authorize_scope_path(WorkdirScopeAuthorizationRequest {
|
||||
rules: vec![WorkdirToolScopeRule {
|
||||
target: WorkdirPath::root(),
|
||||
..rule.clone()
|
||||
}],
|
||||
path: relative_path(&root, operation.path())?,
|
||||
permission: WorkdirToolScopePermission::Read,
|
||||
})
|
||||
.await?;
|
||||
// Lease cwd rebases paths. Use the same provider scoped-traversal contract
|
||||
// with attachment output_root instead, retaining typed paths, cursors and
|
||||
// Grep's rendered filenames without parsing or post-filtering output.
|
||||
let mut request = CheckoutSearchRequest::new(operation);
|
||||
request.scope_layers.push(vec![rule]);
|
||||
self.session.checkout_search(request).await
|
||||
self.search_with_scope(scoped, root, rule, CheckoutSearchRequest::new(operation))
|
||||
.await
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn search_with_scope(
|
||||
&self,
|
||||
scoped: WorkdirSessionHandle,
|
||||
root: WorkdirPath,
|
||||
rule: WorkdirToolScopeRule,
|
||||
mut request: CheckoutSearchRequest,
|
||||
) -> Result<CheckoutSearchResult, WorkdirError> {
|
||||
scoped
|
||||
.authorize_scope_path(WorkdirScopeAuthorizationRequest {
|
||||
rules: vec![WorkdirToolScopeRule {
|
||||
target: WorkdirPath::root(),
|
||||
..rule.clone()
|
||||
}],
|
||||
path: relative_path(&root, request.operation.path())?,
|
||||
permission: WorkdirToolScopePermission::Read,
|
||||
})
|
||||
.await?;
|
||||
// Preserve the caller's coordinate root, cursors and all existing
|
||||
// attenuation layers. The provider applies the Host ceiling during
|
||||
// traversal, not by filtering already-rendered filenames afterward.
|
||||
request.scope_layers.push(vec![rule]);
|
||||
self.session.checkout_search(request).await
|
||||
}
|
||||
|
||||
fn authorize_command(&self) -> Result<(), WorkdirError> {
|
||||
// cwd is not a sandbox: arbitrary shell can access the entire attachment.
|
||||
self.authorize(COMMAND, &WorkdirPath::root(), true)
|
||||
@@ -261,6 +271,12 @@ impl WorkdirSession for AuthorizedWorkdirSession {
|
||||
self.session.capabilities()
|
||||
}
|
||||
|
||||
fn environment(&self) -> workdir::WorkdirSessionEnvironment {
|
||||
let mut environment = self.session.environment();
|
||||
environment.capabilities = environment.capabilities.intersection(self.capabilities());
|
||||
environment
|
||||
}
|
||||
|
||||
async fn authorize_scope_path(
|
||||
&self,
|
||||
mut request: WorkdirScopeAuthorizationRequest,
|
||||
@@ -296,8 +312,14 @@ impl WorkdirSession for AuthorizedWorkdirSession {
|
||||
&self,
|
||||
request: CheckoutSearchRequest,
|
||||
) -> Result<CheckoutSearchResult, WorkdirError> {
|
||||
self.require_legacy(READ)?;
|
||||
self.session.checkout_search(request).await
|
||||
let path = request.operation.path().clone();
|
||||
self.with_authorized(READ, &path, true, |scoped, root, rule| async move {
|
||||
match rule {
|
||||
Some(rule) => self.search_with_scope(scoped, root, rule, request).await,
|
||||
None => self.session.checkout_search(request).await,
|
||||
}
|
||||
})
|
||||
.await
|
||||
}
|
||||
|
||||
async fn checkout_observe(
|
||||
@@ -490,6 +512,8 @@ mod tests {
|
||||
//! Component tests connect this adapter to the real LocalWorkdirSession.
|
||||
//! Only the Host authorization transport/policy is replaced; these are not E2E tests.
|
||||
|
||||
mod environment;
|
||||
|
||||
use super::*;
|
||||
use crate::{WorkspaceClientError, WorkspaceRequest, WorkspaceResponse};
|
||||
use server_api::authorization::ScopedPermission;
|
||||
@@ -842,18 +866,6 @@ mod tests {
|
||||
.await,
|
||||
Err(WorkdirError::Denied(_))
|
||||
));
|
||||
assert!(matches!(
|
||||
session
|
||||
.checkout_search(CheckoutSearchRequest::new(CheckoutSearchOperation::List(
|
||||
ListRequest {
|
||||
path: WorkdirPath::root(),
|
||||
after: None,
|
||||
limit: 10,
|
||||
}
|
||||
)))
|
||||
.await,
|
||||
Err(WorkdirError::Denied(_))
|
||||
));
|
||||
let rule = WorkdirToolScopeRule {
|
||||
target: WorkdirPath::root(),
|
||||
permission: WorkdirToolScopePermission::Read,
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
//! Normal lib tests connecting live Host-grant checks to real provider traversal
|
||||
//! and router cwd projection. Only the Host transport is replaced by the parent
|
||||
//! module's grant fixture; no commands, network or ambient Workspace state are used.
|
||||
|
||||
use super::*;
|
||||
|
||||
#[tokio::test]
|
||||
async fn authorization_wrapper_preserves_inner_environment_and_capability_ceiling() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
std::fs::create_dir(root.path().join("src")).unwrap();
|
||||
let provider = Arc::new(LocalWorkdirSession::materialized(
|
||||
root.path().to_path_buf(),
|
||||
root.path().to_path_buf(),
|
||||
manifest::SharedScope::new(manifest::Scope::writable(root.path()).unwrap()),
|
||||
WorkdirSessionCapabilities::READ_ONLY,
|
||||
));
|
||||
let broker = WorkdirToolBroker::at_cwd(provider, WorkdirPath::new("src").unwrap()).unwrap();
|
||||
let inner = broker.tool_session();
|
||||
let expected = inner.environment();
|
||||
let client = AuthorizationClient::new(vec![permission(READ, "src", true)]);
|
||||
let wrapped = authorized_workdir_session(inner, client.clone(), "attachment-id".into());
|
||||
assert_eq!(wrapped.environment(), expected);
|
||||
assert_eq!(wrapped.environment().cwd.as_str(), "src");
|
||||
assert_eq!(
|
||||
wrapped.environment().capabilities,
|
||||
WorkdirSessionCapabilities::READ_ONLY
|
||||
);
|
||||
assert!(
|
||||
client.requests.lock().unwrap().is_empty(),
|
||||
"environment is metadata, not a cached grant decision"
|
||||
);
|
||||
wrapped.close().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn nonroot_routed_search_retains_host_grants_output_coordinates_and_cursors() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
std::fs::create_dir(root.path().join("src")).unwrap();
|
||||
std::fs::create_dir(root.path().join("private")).unwrap();
|
||||
for name in ["a.txt", "b.txt"] {
|
||||
std::fs::write(root.path().join("src").join(name), "needle public marker").unwrap();
|
||||
}
|
||||
std::fs::write(
|
||||
root.path().join("private/secret.txt"),
|
||||
"needle private marker",
|
||||
)
|
||||
.unwrap();
|
||||
let client = AuthorizationClient::new(vec![permission(READ, "src", true)]);
|
||||
let authorized = session(&root, client.clone());
|
||||
let router = workdir::WorkdirSessionRouter::new();
|
||||
let alias = workdir::WorkdirAttachmentAlias::new("checkout").unwrap();
|
||||
router
|
||||
.attach_at_cwd(alias.clone(), authorized, WorkdirPath::new("src").unwrap())
|
||||
.unwrap();
|
||||
let routed = router.resolve(Some("checkout")).unwrap().session;
|
||||
assert_eq!(routed.environment().cwd.as_str(), "src");
|
||||
let paths = routed
|
||||
.glob(GlobRequest {
|
||||
path: WorkdirPath::root(),
|
||||
pattern: "**/*.txt".into(),
|
||||
limit: 100,
|
||||
})
|
||||
.await
|
||||
.unwrap()
|
||||
.paths;
|
||||
assert_eq!(
|
||||
paths,
|
||||
vec![
|
||||
WorkdirPath::new("a.txt").unwrap(),
|
||||
WorkdirPath::new("b.txt").unwrap()
|
||||
]
|
||||
);
|
||||
// Search output is reusable as a path without adding src/ at the caller.
|
||||
assert_eq!(
|
||||
routed.read(read(paths[0].as_str())).await.unwrap().bytes,
|
||||
b"needle public marker"
|
||||
);
|
||||
for output_mode in [
|
||||
workdir::GrepOutputMode::Content,
|
||||
workdir::GrepOutputMode::FilesWithMatches,
|
||||
workdir::GrepOutputMode::Count,
|
||||
] {
|
||||
let result = routed
|
||||
.grep(GrepRequest {
|
||||
path: WorkdirPath::root(),
|
||||
pattern: "needle".into(),
|
||||
glob: None,
|
||||
file_type: None,
|
||||
case_insensitive: false,
|
||||
before_context: 0,
|
||||
after_context: 0,
|
||||
multiline: false,
|
||||
output_mode,
|
||||
limit: 100,
|
||||
offset: 0,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
let mut grep_paths = result.paths;
|
||||
grep_paths.sort_by(|left, right| left.as_str().cmp(right.as_str()));
|
||||
assert_eq!(grep_paths, paths);
|
||||
assert!(result.output.contains("a.txt"));
|
||||
assert!(!result.output.contains("src/"));
|
||||
assert!(!result.output.contains("secret"));
|
||||
}
|
||||
let first = routed
|
||||
.list(ListRequest {
|
||||
path: WorkdirPath::root(),
|
||||
after: None,
|
||||
limit: 1,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(first.entries[0].path.as_str(), "a.txt");
|
||||
assert_eq!(first.next_after.as_ref().unwrap().path.as_str(), "a.txt");
|
||||
let second = routed
|
||||
.list(ListRequest {
|
||||
path: WorkdirPath::root(),
|
||||
after: first.next_after,
|
||||
limit: 1,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(second.entries[0].path.as_str(), "b.txt");
|
||||
assert!(second.next_after.is_none());
|
||||
|
||||
// An existing caller ceiling must survive the extra Host layer. An empty
|
||||
// layer is invalid and cannot silently turn into the broader Host grant.
|
||||
let mut denied = CheckoutSearchRequest::new(CheckoutSearchOperation::Glob(GlobRequest {
|
||||
path: WorkdirPath::root(),
|
||||
pattern: "**/*.txt".into(),
|
||||
limit: 100,
|
||||
}));
|
||||
denied.scope_layers.push(Vec::new());
|
||||
assert!(routed.checkout_search(denied).await.is_err());
|
||||
assert!(client.requests.lock().unwrap().iter().all(|request| {
|
||||
matches!(&request.resource, PermissionResource::Workdir { path, .. } if path == "src" || path == "src/a.txt")
|
||||
}));
|
||||
client.revoked.store(true, Ordering::SeqCst);
|
||||
assert!(
|
||||
routed
|
||||
.glob(GlobRequest {
|
||||
path: WorkdirPath::root(),
|
||||
pattern: "**/*.txt".into(),
|
||||
limit: 100,
|
||||
})
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
router.detach(&alias).await.unwrap();
|
||||
}
|
||||
|
||||
#[cfg(target_os = "linux")]
|
||||
#[tokio::test]
|
||||
async fn scoped_checkout_search_preserves_caller_layer_and_rejects_resolved_escape() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
std::fs::create_dir_all(root.path().join("src/nested")).unwrap();
|
||||
std::fs::create_dir(root.path().join("private")).unwrap();
|
||||
std::fs::write(root.path().join("src/public.txt"), "needle public").unwrap();
|
||||
std::fs::write(root.path().join("src/nested/hidden.txt"), "needle hidden").unwrap();
|
||||
std::fs::write(root.path().join("private/secret.txt"), "needle secret").unwrap();
|
||||
std::os::unix::fs::symlink("../private", root.path().join("src/escape")).unwrap();
|
||||
std::os::unix::fs::symlink("../private/secret.txt", root.path().join("src/alias.txt")).unwrap();
|
||||
let client = AuthorizationClient::new(vec![permission(READ, "src", true)]);
|
||||
let authorized = session(&root, client);
|
||||
let request = CheckoutSearchRequest {
|
||||
operation: CheckoutSearchOperation::Glob(GlobRequest {
|
||||
path: WorkdirPath::new("src").unwrap(),
|
||||
pattern: "**/*.txt".into(),
|
||||
limit: 100,
|
||||
}),
|
||||
output_root: WorkdirPath::new("src").unwrap(),
|
||||
scope_layers: vec![vec![WorkdirToolScopeRule {
|
||||
target: WorkdirPath::new("src").unwrap(),
|
||||
recursive: false,
|
||||
permission: WorkdirToolScopePermission::Read,
|
||||
symlink_policy: manifest::SymlinkPolicy::Resolved,
|
||||
}]],
|
||||
};
|
||||
let result = authorized.checkout_search(request).await.unwrap();
|
||||
let CheckoutSearchResult::Glob(result) = result else {
|
||||
panic!("expected Glob result")
|
||||
};
|
||||
assert_eq!(result.paths, vec![WorkdirPath::new("public.txt").unwrap()]);
|
||||
assert!(
|
||||
authorized
|
||||
.checkout_search(CheckoutSearchRequest::new(CheckoutSearchOperation::Glob(
|
||||
GlobRequest {
|
||||
path: WorkdirPath::new("private").unwrap(),
|
||||
pattern: "**/*.txt".into(),
|
||||
limit: 100,
|
||||
}
|
||||
)))
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
authorized.close().await.unwrap();
|
||||
}
|
||||
@@ -6,9 +6,7 @@ use thiserror::Error;
|
||||
use workdir::WorkdirSessionHandle;
|
||||
|
||||
use crate::PromptCatalogSource;
|
||||
use crate::controller::{
|
||||
ShutdownReceiver, WorkerController, WorkerControllerTransport, WorkerHandle,
|
||||
};
|
||||
use crate::controller::{ShutdownReceiver, WorkerController, WorkerHandle};
|
||||
use crate::worker::{Worker, WorkerError, WorkerFilesystemAuthority, WorkerWorkspaceContext};
|
||||
use manifest::WorkerManifest;
|
||||
|
||||
@@ -30,7 +28,6 @@ pub struct WorkerBootstrap<St> {
|
||||
workspace_context: WorkerWorkspaceContext,
|
||||
filesystem_authority: WorkerFilesystemAuthority,
|
||||
layout: WorkerBootstrapLayout,
|
||||
transport: WorkerControllerTransport,
|
||||
model_client: Option<Box<dyn LlmClient>>,
|
||||
workdir_session: Option<WorkdirSessionHandle>,
|
||||
subjektiv_host: Option<std::sync::Arc<dyn crate::subjektiv::SubjektivHost>>,
|
||||
@@ -41,7 +38,6 @@ pub struct WorkerBootstrap<St> {
|
||||
pub struct PreparedWorker<C: LlmClient, St: Store> {
|
||||
worker: Worker<C, St>,
|
||||
layout: WorkerBootstrapLayout,
|
||||
transport: WorkerControllerTransport,
|
||||
}
|
||||
|
||||
/// Live controller returned only after Worker construction and feature
|
||||
@@ -68,7 +64,6 @@ impl<St> WorkerBootstrap<St>
|
||||
where
|
||||
St: Store + WorkerMetadataStore + Clone + Send + Sync + 'static,
|
||||
{
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn new(
|
||||
manifest: WorkerManifest,
|
||||
store: St,
|
||||
@@ -76,7 +71,6 @@ where
|
||||
workspace_context: WorkerWorkspaceContext,
|
||||
filesystem_authority: WorkerFilesystemAuthority,
|
||||
layout: WorkerBootstrapLayout,
|
||||
transport: WorkerControllerTransport,
|
||||
) -> Self {
|
||||
Self {
|
||||
manifest,
|
||||
@@ -85,7 +79,6 @@ where
|
||||
workspace_context,
|
||||
filesystem_authority,
|
||||
layout,
|
||||
transport,
|
||||
model_client: None,
|
||||
workdir_session: None,
|
||||
subjektiv_host: None,
|
||||
@@ -165,7 +158,7 @@ where
|
||||
)
|
||||
.map_err(WorkerBootstrapError::Worker)?;
|
||||
}
|
||||
Ok(PreparedWorker::new(worker, self.layout, self.transport))
|
||||
Ok(PreparedWorker::new(worker, self.layout))
|
||||
}
|
||||
|
||||
pub async fn prepare_restored(
|
||||
@@ -206,7 +199,7 @@ where
|
||||
)
|
||||
.map_err(WorkerBootstrapError::Worker)?;
|
||||
}
|
||||
Ok(PreparedWorker::new(worker, self.layout, self.transport))
|
||||
Ok(PreparedWorker::new(worker, self.layout))
|
||||
}
|
||||
|
||||
pub async fn start(self) -> Result<BootstrappedWorker, WorkerBootstrapError> {
|
||||
@@ -221,16 +214,8 @@ where
|
||||
{
|
||||
/// Wrap a restored Worker in the same pre-exposure lifecycle used by fresh
|
||||
/// bootstraps.
|
||||
pub fn new(
|
||||
worker: Worker<C, St>,
|
||||
layout: WorkerBootstrapLayout,
|
||||
transport: WorkerControllerTransport,
|
||||
) -> Self {
|
||||
Self {
|
||||
worker,
|
||||
layout,
|
||||
transport,
|
||||
}
|
||||
pub fn new(worker: Worker<C, St>, layout: WorkerBootstrapLayout) -> Self {
|
||||
Self { worker, layout }
|
||||
}
|
||||
|
||||
pub fn worker(&self) -> &Worker<C, St> {
|
||||
@@ -242,7 +227,7 @@ where
|
||||
}
|
||||
|
||||
pub async fn start(self) -> Result<BootstrappedWorker, WorkerBootstrapError> {
|
||||
start_worker_controller(self.worker, self.layout, self.transport).await
|
||||
start_worker_controller(self.worker, self.layout).await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -252,7 +237,6 @@ where
|
||||
pub async fn start_worker_controller<C, St>(
|
||||
worker: Worker<C, St>,
|
||||
layout: WorkerBootstrapLayout,
|
||||
transport: WorkerControllerTransport,
|
||||
) -> Result<BootstrappedWorker, WorkerBootstrapError>
|
||||
where
|
||||
C: LlmClient + Clone + 'static,
|
||||
@@ -261,13 +245,10 @@ where
|
||||
let cleanup_sessions = worker.workdir_sessions();
|
||||
let controller = match layout {
|
||||
WorkerBootstrapLayout::Direct { runtime_base } => {
|
||||
WorkerController::spawn_with_transport_owned(worker, &runtime_base, transport).await
|
||||
WorkerController::spawn_owned(worker, &runtime_base).await
|
||||
}
|
||||
WorkerBootstrapLayout::RuntimeManagedRun { run_dir } => {
|
||||
WorkerController::spawn_runtime_managed_run_with_transport_owned(
|
||||
worker, &run_dir, transport,
|
||||
)
|
||||
.await
|
||||
WorkerController::spawn_runtime_managed_run_owned(worker, &run_dir).await
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -121,6 +121,7 @@ async fn open_session(
|
||||
session_id: WorkdirSessionId::new(SESSION).unwrap(),
|
||||
workdir_id: state.session.workdir().id().clone(),
|
||||
capabilities: state.session.capabilities(),
|
||||
environment: state.session.environment(),
|
||||
})
|
||||
.into_response()
|
||||
}
|
||||
@@ -251,6 +252,10 @@ impl Drop for HttpFixture {
|
||||
|
||||
impl HttpFixture {
|
||||
async fn new(capabilities: WorkdirSessionCapabilities) -> Self {
|
||||
Self::at_cwd(capabilities, "").await
|
||||
}
|
||||
|
||||
async fn at_cwd(capabilities: WorkdirSessionCapabilities, cwd: &str) -> Self {
|
||||
let dir = TempDir::new().unwrap();
|
||||
std::fs::create_dir_all(dir.path().join("src/deep")).unwrap();
|
||||
std::fs::write(
|
||||
@@ -264,6 +269,13 @@ impl HttpFixture {
|
||||
manifest::SharedScope::new(manifest::Scope::writable(dir.path()).unwrap()),
|
||||
capabilities,
|
||||
));
|
||||
let session: WorkdirSessionHandle = if cwd.is_empty() {
|
||||
session
|
||||
} else {
|
||||
workdir::WorkdirToolBroker::at_cwd(session, WorkdirPath::new(cwd).unwrap())
|
||||
.unwrap()
|
||||
.tool_session()
|
||||
};
|
||||
let state = Arc::new(ProviderState {
|
||||
session,
|
||||
requests: Mutex::new(Vec::new()),
|
||||
@@ -349,6 +361,31 @@ impl HttpFixture {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn remote_open_keeps_provider_environment_and_nonroot_operation_coordinates() {
|
||||
let fixture = HttpFixture::at_cwd(WorkdirSessionCapabilities::READ_ONLY, "src/deep").await;
|
||||
let remote = fixture.open(TOKEN).await.unwrap();
|
||||
assert_eq!(remote.environment(), fixture.state.session.environment());
|
||||
assert_eq!(remote.environment().cwd.as_str(), "src/deep");
|
||||
assert_eq!(
|
||||
remote.environment().capabilities,
|
||||
WorkdirSessionCapabilities::READ_ONLY
|
||||
);
|
||||
let result = remote
|
||||
.read(workdir::ReadRequest {
|
||||
path: WorkdirPath::new("a.txt").unwrap(),
|
||||
offset: 0,
|
||||
limit: 10,
|
||||
max_bytes: 4096,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(result.bytes, b"first\nneedle needle\nlast\n");
|
||||
fixture.public_output(&serde_json::to_string(&remote.environment()).unwrap());
|
||||
remote.close().await.unwrap();
|
||||
fixture.assert_endpoints();
|
||||
}
|
||||
|
||||
fn native_runtime(remote: Arc<RemoteWorkdirSession>) -> (WipRuntime, Arc<WorkdirSessionRouter>) {
|
||||
let router = Arc::new(WorkdirSessionRouter::new());
|
||||
router
|
||||
|
||||
@@ -78,6 +78,12 @@ impl WorkdirSession for HookedSession {
|
||||
self.local.capabilities()
|
||||
}
|
||||
|
||||
fn environment(&self) -> workdir::WorkdirSessionEnvironment {
|
||||
let mut environment = self.local.environment();
|
||||
environment.capabilities = environment.capabilities.intersection(self.capabilities());
|
||||
environment
|
||||
}
|
||||
|
||||
async fn authorize_scope_path(
|
||||
&self,
|
||||
request: WorkdirScopeAuthorizationRequest,
|
||||
@@ -201,11 +207,10 @@ async fn checkout_wip_file_operations_share_core_and_keep_prior_read_boundary()
|
||||
let dir = TempDir::new().unwrap();
|
||||
std::fs::write(dir.path().join("text.txt"), "旧 旧\n").unwrap();
|
||||
let router = Arc::new(WorkdirSessionRouter::new());
|
||||
let hooked = HookedSession::new(&dir);
|
||||
assert_eq!(hooked.environment(), hooked.local.environment());
|
||||
router
|
||||
.attach(
|
||||
WorkdirAttachmentAlias::new("main").unwrap(),
|
||||
HookedSession::new(&dir),
|
||||
)
|
||||
.attach(WorkdirAttachmentAlias::new("main").unwrap(), hooked)
|
||||
.unwrap();
|
||||
let provider = provider(router);
|
||||
let route = format!("{}/text.txt", checkout_root("main"));
|
||||
|
||||
+59
-679
File diff suppressed because it is too large
Load Diff
@@ -65,7 +65,6 @@ async fn prepared(
|
||||
WorkerBootstrapLayout::Direct {
|
||||
runtime_base: temp.path().join("runtime"),
|
||||
},
|
||||
WorkerControllerTransport::InProcess,
|
||||
)
|
||||
.with_model_client(client)
|
||||
.prepare()
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -4,6 +4,7 @@
|
||||
//! same descriptor-approved registry path used by feature modules. They are not
|
||||
//! an external plugin-loading surface.
|
||||
|
||||
mod backend_diagnostic;
|
||||
pub mod backend_job_result;
|
||||
pub(crate) mod bash;
|
||||
|
||||
|
||||
@@ -0,0 +1,195 @@
|
||||
//! Internal adapter diagnostics are independent of public failure classifications.
|
||||
//!
|
||||
//! Only closed vocabularies, numeric status and opaque correlation tokens cross this boundary.
|
||||
//! In particular, a transport error's Display/Debug and server-controlled text are never recorded.
|
||||
use sha2::{Digest, Sha256};
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum Adapter {
|
||||
Drive,
|
||||
WorkspaceConfig,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum Stage {
|
||||
Request,
|
||||
Response,
|
||||
Completion,
|
||||
Reconcile,
|
||||
Attach,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub(super) enum Reason {
|
||||
Deadline,
|
||||
TaskJoin,
|
||||
TransportUnavailable,
|
||||
InvalidPath,
|
||||
Communication,
|
||||
Decode,
|
||||
RequestSize,
|
||||
ResponseSize,
|
||||
ResponseLength,
|
||||
Identity,
|
||||
InvalidMetadata,
|
||||
BackendRejected,
|
||||
AccessCeiling,
|
||||
AttachConversion,
|
||||
BackendUnknown,
|
||||
UnrecognizedResponse,
|
||||
NotCommittedSnapshot,
|
||||
IncompleteReceipt,
|
||||
Reconciled,
|
||||
}
|
||||
|
||||
/// One logical call, including its original-request reconciliation. Request identities are hashed
|
||||
/// rather than logged verbatim: callers and responses need not contain safe log text.
|
||||
#[derive(Clone)]
|
||||
pub(super) struct Diagnostic {
|
||||
adapter: Adapter,
|
||||
operation: &'static str,
|
||||
mutation: bool,
|
||||
correlation: String,
|
||||
dispatch: tracing::Dispatch,
|
||||
stage_override: Option<Stage>,
|
||||
}
|
||||
|
||||
impl Diagnostic {
|
||||
pub(super) fn new(
|
||||
adapter: Adapter,
|
||||
operation: &'static str,
|
||||
mutation: bool,
|
||||
request_id: Option<&str>,
|
||||
) -> Self {
|
||||
Self {
|
||||
adapter,
|
||||
operation,
|
||||
mutation,
|
||||
correlation: request_id
|
||||
.map(|id| {
|
||||
let hash: String = Sha256::digest(id.as_bytes())
|
||||
.iter()
|
||||
.map(|b| format!("{b:02x}"))
|
||||
.collect();
|
||||
format!("request:{hash}")
|
||||
})
|
||||
.unwrap_or_else(|| uuid::Uuid::now_v7().to_string()),
|
||||
dispatch: tracing::dispatcher::get_default(Clone::clone),
|
||||
stage_override: None,
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn at_stage(&self, stage: Stage) -> Self {
|
||||
let mut diagnostic = self.clone();
|
||||
diagnostic.stage_override = Some(stage);
|
||||
diagnostic
|
||||
}
|
||||
|
||||
pub(super) fn record(&self, stage: Stage, reason: Reason, status: Option<u16>) {
|
||||
let stage = self.stage_override.unwrap_or(stage);
|
||||
tracing::dispatcher::with_default(&self.dispatch, || {
|
||||
tracing::warn!(
|
||||
target: "worker::backend_diagnostic",
|
||||
adapter = ?self.adapter,
|
||||
operation = self.operation,
|
||||
mutation = self.mutation,
|
||||
correlation = %self.correlation,
|
||||
stage = ?stage,
|
||||
reason = ?reason,
|
||||
status,
|
||||
"Workspace adapter diagnostic"
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
pub(super) fn transport(&self, error: &crate::worker::WorkspaceClientError) {
|
||||
use crate::worker::WorkspaceClientError;
|
||||
self.record(
|
||||
Stage::Request,
|
||||
match error {
|
||||
WorkspaceClientError::Unavailable(_) => Reason::TransportUnavailable,
|
||||
WorkspaceClientError::InvalidPath(_) => Reason::InvalidPath,
|
||||
WorkspaceClientError::Request(_) => Reason::Communication,
|
||||
},
|
||||
None,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) mod tests {
|
||||
use super::{Adapter, Diagnostic, Reason, Stage};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tracing::{Event, Metadata, Subscriber, field::Visit, span};
|
||||
|
||||
#[test]
|
||||
fn unsafe_request_identity_is_correlated_without_logging_its_value() {
|
||||
let capture = Capture::default();
|
||||
tracing::dispatcher::with_default(&capture.dispatch(), || {
|
||||
for id in [
|
||||
"secret /host/path\ncredential",
|
||||
"secret /host/path\ncredential",
|
||||
"different",
|
||||
] {
|
||||
Diagnostic::new(Adapter::Drive, "mutate", true, Some(id)).record(
|
||||
Stage::Request,
|
||||
Reason::Communication,
|
||||
None,
|
||||
);
|
||||
}
|
||||
});
|
||||
let text = capture.text();
|
||||
for secret in ["secret", "/host/path", "credential", "different"] {
|
||||
assert!(!text.contains(secret));
|
||||
}
|
||||
let correlations: Vec<_> = text
|
||||
.lines()
|
||||
.map(|line| {
|
||||
line.split("correlation=")
|
||||
.nth(1)
|
||||
.unwrap()
|
||||
.split(';')
|
||||
.next()
|
||||
.unwrap()
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(correlations.len(), 3);
|
||||
assert_eq!(correlations[0], correlations[1]);
|
||||
assert_ne!(correlations[0], correlations[2]);
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
pub(crate) struct Capture(pub(crate) Arc<Mutex<Vec<String>>>);
|
||||
impl Capture {
|
||||
pub(crate) fn dispatch(&self) -> tracing::Dispatch {
|
||||
tracing::Dispatch::new(self.clone())
|
||||
}
|
||||
pub(crate) fn text(&self) -> String {
|
||||
self.0.lock().unwrap().join("\n")
|
||||
}
|
||||
}
|
||||
struct Fields(String);
|
||||
impl Visit for Fields {
|
||||
fn record_debug(&mut self, field: &tracing::field::Field, value: &dyn std::fmt::Debug) {
|
||||
use std::fmt::Write;
|
||||
write!(&mut self.0, "{}={value:?};", field.name()).unwrap();
|
||||
}
|
||||
}
|
||||
impl Subscriber for Capture {
|
||||
fn enabled(&self, metadata: &Metadata<'_>) -> bool {
|
||||
metadata.target() == "worker::backend_diagnostic"
|
||||
}
|
||||
fn new_span(&self, _: &span::Attributes<'_>) -> span::Id {
|
||||
span::Id::from_u64(1)
|
||||
}
|
||||
fn record(&self, _: &span::Id, _: &span::Record<'_>) {}
|
||||
fn record_follows_from(&self, _: &span::Id, _: &span::Id) {}
|
||||
fn event(&self, event: &Event<'_>) {
|
||||
let mut fields = Fields(String::new());
|
||||
event.record(&mut fields);
|
||||
self.0.lock().unwrap().push(fields.0);
|
||||
}
|
||||
fn enter(&self, _: &span::Id) {}
|
||||
fn exit(&self, _: &span::Id) {}
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,8 @@
|
||||
//! cases retain an unknown outcome with the original request ID. Never blindly resend with a new ID.
|
||||
use std::{sync::Arc, time::Duration};
|
||||
|
||||
use super::super::backend_diagnostic::{Adapter, Diagnostic, Reason, Stage};
|
||||
|
||||
use crate::worker::{
|
||||
WorkspaceBinaryRequest, WorkspaceClient, WorkspaceRequest, WorkspaceRequestMethod,
|
||||
};
|
||||
@@ -23,6 +25,7 @@ const RESPONSE_LIMIT: usize = 2 * 1024 * 1024;
|
||||
#[derive(Clone)]
|
||||
pub struct DriveBackend {
|
||||
client: Arc<dyn WorkspaceClient>,
|
||||
deadline: Duration,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, thiserror::Error)]
|
||||
@@ -49,7 +52,10 @@ pub enum DriveError {
|
||||
|
||||
impl DriveBackend {
|
||||
pub fn new(client: Arc<dyn WorkspaceClient>) -> Self {
|
||||
Self { client }
|
||||
Self {
|
||||
client,
|
||||
deadline: DEADLINE,
|
||||
}
|
||||
}
|
||||
pub fn workspace(&self) -> Result<&str, DriveError> {
|
||||
self.client
|
||||
@@ -61,15 +67,28 @@ impl DriveBackend {
|
||||
Ok(format!("/api/w/{}/drive", encode(self.workspace()?)))
|
||||
}
|
||||
pub(super) fn validate_entry(&self, entry: &DriveEntry) -> Result<(), DriveError> {
|
||||
self.validate_ref(&entry.entry)?;
|
||||
self.validate_entry_at(
|
||||
entry,
|
||||
&Diagnostic::new(Adapter::Drive, "validate_entry", false, None),
|
||||
)
|
||||
}
|
||||
fn validate_entry_at(
|
||||
&self,
|
||||
entry: &DriveEntry,
|
||||
diagnostic: &Diagnostic,
|
||||
) -> Result<(), DriveError> {
|
||||
self.validate_ref(&entry.entry)
|
||||
.inspect_err(|_| diagnostic.record(Stage::Completion, Reason::Identity, None))?;
|
||||
if let Some(parent) = &entry.parent {
|
||||
self.validate_ref(parent)?;
|
||||
self.validate_ref(parent)
|
||||
.inspect_err(|_| diagnostic.record(Stage::Completion, Reason::Identity, None))?;
|
||||
}
|
||||
if (entry.last_mutation_id.len() > 128
|
||||
|| entry.last_mutation_id.chars().any(char::is_control)
|
||||
|| (entry.parent.is_some() && entry.last_mutation_id.is_empty()))
|
||||
|| entry.size.is_some_and(|n| n > DRIVE_FILE_MAX_BYTES)
|
||||
{
|
||||
diagnostic.record(Stage::Completion, Reason::InvalidMetadata, None);
|
||||
return Err(DriveError::Unavailable);
|
||||
}
|
||||
Ok(())
|
||||
@@ -90,11 +109,13 @@ impl DriveBackend {
|
||||
request: WorkspaceRequest,
|
||||
mutation: bool,
|
||||
request_id: &str,
|
||||
diagnostic: &Diagnostic,
|
||||
) -> Result<T, DriveError> {
|
||||
let client = self.client.clone();
|
||||
let deadline = self.deadline;
|
||||
let response = tokio::time::timeout(
|
||||
DEADLINE,
|
||||
tokio::task::spawn_blocking(move || client.execute_with_timeout(request, DEADLINE)),
|
||||
deadline,
|
||||
tokio::task::spawn_blocking(move || client.execute_with_timeout(request, deadline)),
|
||||
)
|
||||
.await;
|
||||
let unknown = || {
|
||||
@@ -105,10 +126,20 @@ impl DriveBackend {
|
||||
}
|
||||
};
|
||||
let response = response
|
||||
.map_err(|_| unknown())?
|
||||
.map_err(|_| unknown())?
|
||||
.map_err(|_| unknown())?;
|
||||
.map_err(|_| {
|
||||
diagnostic.record(Stage::Request, Reason::Deadline, None);
|
||||
unknown()
|
||||
})?
|
||||
.map_err(|_| {
|
||||
diagnostic.record(Stage::Request, Reason::TaskJoin, None);
|
||||
unknown()
|
||||
})?
|
||||
.map_err(|error| {
|
||||
diagnostic.transport(&error);
|
||||
unknown()
|
||||
})?;
|
||||
if response.body.len() > RESPONSE_LIMIT {
|
||||
diagnostic.record(Stage::Response, Reason::ResponseSize, None);
|
||||
return Err(if mutation {
|
||||
unknown()
|
||||
} else {
|
||||
@@ -121,42 +152,57 @@ impl DriveBackend {
|
||||
response.body.as_bytes(),
|
||||
mutation,
|
||||
request_id,
|
||||
diagnostic,
|
||||
));
|
||||
}
|
||||
serde_json::from_str(&response.body).map_err(|_| unknown())
|
||||
serde_json::from_str(&response.body).map_err(|_| {
|
||||
diagnostic.record(Stage::Response, Reason::Decode, None);
|
||||
unknown()
|
||||
})
|
||||
}
|
||||
async fn get<T: DeserializeOwned + Send + 'static, Q: Serialize>(
|
||||
&self,
|
||||
suffix: &str,
|
||||
suffix: &'static str,
|
||||
query: &Q,
|
||||
diagnostic: &Diagnostic,
|
||||
) -> Result<T, DriveError> {
|
||||
let query = query_string(query)?;
|
||||
self.json(
|
||||
WorkspaceRequest::get(format!("{}{suffix}{query}", self.base()?)),
|
||||
false,
|
||||
"",
|
||||
diagnostic,
|
||||
)
|
||||
.await
|
||||
}
|
||||
pub async fn capabilities(&self) -> Result<DriveCapabilities, DriveError> {
|
||||
self.get("/capabilities", &serde_json::json!({})).await
|
||||
self.get(
|
||||
"/capabilities",
|
||||
&serde_json::json!({}),
|
||||
&Diagnostic::new(Adapter::Drive, "capabilities", false, None),
|
||||
)
|
||||
.await
|
||||
}
|
||||
pub async fn root(&self) -> Result<DriveEntry, DriveError> {
|
||||
let diagnostic = Diagnostic::new(Adapter::Drive, "root", false, None);
|
||||
let entry: DriveEntry = self
|
||||
.json(
|
||||
WorkspaceRequest::get(format!("{}/root", self.base()?)),
|
||||
false,
|
||||
"",
|
||||
&diagnostic,
|
||||
)
|
||||
.await?;
|
||||
self.validate_entry(&entry)?;
|
||||
self.validate_entry_at(&entry, &diagnostic)?;
|
||||
if entry.kind != DriveEntryKind::Folder || entry.parent.is_some() {
|
||||
diagnostic.record(Stage::Completion, Reason::InvalidMetadata, None);
|
||||
return Err(DriveError::Unavailable);
|
||||
}
|
||||
Ok(entry)
|
||||
}
|
||||
pub async fn metadata(&self, entry: &DriveEntryRef) -> Result<DriveEntry, DriveError> {
|
||||
self.validate_ref(entry)?;
|
||||
let diagnostic = Diagnostic::new(Adapter::Drive, "metadata", false, None);
|
||||
let result: DriveEntry = self
|
||||
.get(
|
||||
"/metadata",
|
||||
@@ -164,10 +210,12 @@ impl DriveBackend {
|
||||
entry_workspace_id: entry.workspace_id.clone(),
|
||||
id: entry.node_id.clone(),
|
||||
},
|
||||
&diagnostic,
|
||||
)
|
||||
.await?;
|
||||
self.validate_entry(&result)?;
|
||||
self.validate_entry_at(&result, &diagnostic)?;
|
||||
if result.entry != *entry {
|
||||
diagnostic.record(Stage::Completion, Reason::Identity, None);
|
||||
return Err(DriveError::Unavailable);
|
||||
}
|
||||
Ok(result)
|
||||
@@ -179,6 +227,7 @@ impl DriveBackend {
|
||||
after: Option<String>,
|
||||
) -> Result<DriveListResponse, DriveError> {
|
||||
self.validate_ref(parent)?;
|
||||
let diagnostic = Diagnostic::new(Adapter::Drive, "list", false, None);
|
||||
let result: DriveListResponse = self
|
||||
.get(
|
||||
"/list",
|
||||
@@ -188,6 +237,7 @@ impl DriveBackend {
|
||||
limit,
|
||||
after,
|
||||
},
|
||||
&diagnostic,
|
||||
)
|
||||
.await?;
|
||||
self.validate_page(
|
||||
@@ -195,33 +245,43 @@ impl DriveBackend {
|
||||
limit
|
||||
.unwrap_or(DRIVE_PAGE_MAX_LIMIT)
|
||||
.min(DRIVE_PAGE_MAX_LIMIT),
|
||||
&diagnostic,
|
||||
)?;
|
||||
if result
|
||||
.entries
|
||||
.iter()
|
||||
.any(|entry| entry.parent.as_ref() != Some(parent))
|
||||
{
|
||||
diagnostic.record(Stage::Completion, Reason::Identity, None);
|
||||
return Err(DriveError::Unavailable);
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
pub async fn search(&self, query: DriveSearchQuery) -> Result<DriveListResponse, DriveError> {
|
||||
let result: DriveListResponse = self.get("/search", &query).await?;
|
||||
let diagnostic = Diagnostic::new(Adapter::Drive, "search", false, None);
|
||||
let result: DriveListResponse = self.get("/search", &query, &diagnostic).await?;
|
||||
self.validate_page(
|
||||
&result,
|
||||
query
|
||||
.limit
|
||||
.unwrap_or(DRIVE_SEARCH_MAX_LIMIT)
|
||||
.min(DRIVE_SEARCH_MAX_LIMIT),
|
||||
&diagnostic,
|
||||
)?;
|
||||
Ok(result)
|
||||
}
|
||||
fn validate_page(&self, page: &DriveListResponse, limit: u32) -> Result<(), DriveError> {
|
||||
fn validate_page(
|
||||
&self,
|
||||
page: &DriveListResponse,
|
||||
limit: u32,
|
||||
diagnostic: &Diagnostic,
|
||||
) -> Result<(), DriveError> {
|
||||
if page.entries.len() > limit as usize {
|
||||
diagnostic.record(Stage::Completion, Reason::ResponseSize, None);
|
||||
return Err(DriveError::Limit);
|
||||
}
|
||||
for entry in &page.entries {
|
||||
self.validate_entry(entry)?;
|
||||
self.validate_entry_at(entry, diagnostic)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -231,6 +291,7 @@ impl DriveBackend {
|
||||
max_bytes: u32,
|
||||
) -> Result<DriveReadTextResponse, DriveError> {
|
||||
self.validate_ref(entry)?;
|
||||
let diagnostic = Diagnostic::new(Adapter::Drive, "read_text", false, None);
|
||||
let result: DriveReadTextResponse = self
|
||||
.get(
|
||||
"/read-text",
|
||||
@@ -239,10 +300,16 @@ impl DriveBackend {
|
||||
id: entry.node_id.clone(),
|
||||
max_bytes,
|
||||
},
|
||||
&diagnostic,
|
||||
)
|
||||
.await?;
|
||||
self.validate_entry(&result.entry)?;
|
||||
if result.entry.entry != *entry || result.text.len() > max_bytes as usize {
|
||||
self.validate_entry_at(&result.entry, &diagnostic)?;
|
||||
if result.entry.entry != *entry {
|
||||
diagnostic.record(Stage::Completion, Reason::Identity, None);
|
||||
return Err(DriveError::Unavailable);
|
||||
}
|
||||
if result.text.len() > max_bytes as usize {
|
||||
diagnostic.record(Stage::Completion, Reason::ResponseSize, None);
|
||||
return Err(DriveError::Unavailable);
|
||||
}
|
||||
Ok(result)
|
||||
@@ -255,6 +322,7 @@ impl DriveBackend {
|
||||
if total as usize > limit {
|
||||
return Err(DriveError::Limit);
|
||||
}
|
||||
let diagnostic = Diagnostic::new(Adapter::Drive, "read_chunk", false, None);
|
||||
let mut bytes = Vec::with_capacity(total as usize);
|
||||
// Even an empty file gets a read bound to the observed committed mutation and current authorization.
|
||||
loop {
|
||||
@@ -274,21 +342,42 @@ impl DriveBackend {
|
||||
max_response_bytes: DRIVE_CHUNK_MAX_BYTES as usize,
|
||||
};
|
||||
let client = self.client.clone();
|
||||
let deadline = self.deadline;
|
||||
let response = tokio::time::timeout(
|
||||
DEADLINE,
|
||||
deadline,
|
||||
tokio::task::spawn_blocking(move || {
|
||||
client.execute_binary_with_timeout(request, DEADLINE)
|
||||
client.execute_binary_with_timeout(request, deadline)
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| DriveError::Unavailable)?
|
||||
.map_err(|_| DriveError::Unavailable)?
|
||||
.map_err(|_| DriveError::Unavailable)?;
|
||||
.map_err(|_| {
|
||||
diagnostic.record(Stage::Request, Reason::Deadline, None);
|
||||
DriveError::Unavailable
|
||||
})?
|
||||
.map_err(|_| {
|
||||
diagnostic.record(Stage::Request, Reason::TaskJoin, None);
|
||||
DriveError::Unavailable
|
||||
})?
|
||||
.map_err(|error| {
|
||||
diagnostic.transport(&error);
|
||||
DriveError::Unavailable
|
||||
})?;
|
||||
if response.body.len() > DRIVE_CHUNK_MAX_BYTES as usize {
|
||||
diagnostic.record(Stage::Response, Reason::ResponseSize, None);
|
||||
return Err(DriveError::Unavailable);
|
||||
}
|
||||
if !(200..300).contains(&response.status) {
|
||||
return Err(classify(response.status, &response.body, false, ""));
|
||||
return Err(classify(
|
||||
response.status,
|
||||
&response.body,
|
||||
false,
|
||||
"",
|
||||
&diagnostic,
|
||||
));
|
||||
}
|
||||
let expected = total.saturating_sub(offset).min(length) as usize;
|
||||
if response.body.len() != expected {
|
||||
diagnostic.record(Stage::Response, Reason::ResponseLength, None);
|
||||
return Err(DriveError::Unavailable);
|
||||
}
|
||||
bytes.extend_from_slice(&response.body);
|
||||
@@ -301,22 +390,40 @@ impl DriveBackend {
|
||||
&self,
|
||||
id: &str,
|
||||
result: Result<DriveMutationResponse, DriveError>,
|
||||
diagnostic: &Diagnostic,
|
||||
) -> Result<DriveMutationResponse, DriveError> {
|
||||
// A parseable success can still be an invalid completion. Classify it
|
||||
// before reconciliation so it also gets one original-request inquiry.
|
||||
let result = result.and_then(|response| {
|
||||
self.validate_completion(id, &response)
|
||||
self.validate_completion(id, &response, diagnostic)
|
||||
.map_err(|_| DriveError::OutcomeUnknown(id.into()))?;
|
||||
Ok(response)
|
||||
});
|
||||
match result {
|
||||
Err(DriveError::OutcomeUnknown(_)) => match self.status(id).await {
|
||||
Err(DriveError::OutcomeUnknown(_)) => match self
|
||||
.status_with_diagnostic(id, &diagnostic.at_stage(Stage::Reconcile))
|
||||
.await
|
||||
{
|
||||
Ok(DriveRequestStatusResponse {
|
||||
state: DriveRequestState::Committed,
|
||||
response: Some(response),
|
||||
..
|
||||
}) => Ok(response), // status validates both IDs and entry metadata
|
||||
_ => Err(DriveError::OutcomeUnknown(id.into())),
|
||||
}) => {
|
||||
diagnostic.record(Stage::Reconcile, Reason::Reconciled, None);
|
||||
Ok(response) // status validates both IDs and entry metadata
|
||||
}
|
||||
Ok(status) => {
|
||||
diagnostic.record(
|
||||
Stage::Reconcile,
|
||||
match status.state {
|
||||
DriveRequestState::Uncommitted => Reason::NotCommittedSnapshot,
|
||||
DriveRequestState::Committed => Reason::IncompleteReceipt,
|
||||
},
|
||||
None,
|
||||
);
|
||||
Err(DriveError::OutcomeUnknown(id.into()))
|
||||
}
|
||||
Err(_) => Err(DriveError::OutcomeUnknown(id.into())),
|
||||
},
|
||||
other => other,
|
||||
}
|
||||
@@ -325,12 +432,14 @@ impl DriveBackend {
|
||||
&self,
|
||||
id: &str,
|
||||
response: &DriveMutationResponse,
|
||||
diagnostic: &Diagnostic,
|
||||
) -> Result<(), DriveError> {
|
||||
if response.request_id != id {
|
||||
diagnostic.record(Stage::Completion, Reason::Identity, None);
|
||||
return Err(DriveError::Unavailable);
|
||||
}
|
||||
if let Some(entry) = &response.entry {
|
||||
self.validate_entry(entry)?;
|
||||
self.validate_entry_at(entry, diagnostic)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -338,9 +447,11 @@ impl DriveBackend {
|
||||
&self,
|
||||
request: DriveMutationRequest,
|
||||
) -> Result<DriveMutationResponse, DriveError> {
|
||||
let diagnostic = Diagnostic::new(Adapter::Drive, "mutate", true, Some(&request.request_id));
|
||||
let body = serde_json::to_string(&request)
|
||||
.map_err(|_| DriveError::Invalid("invalid mutation".into()))?;
|
||||
if body.len() > 512 * 1024 {
|
||||
diagnostic.record(Stage::Request, Reason::RequestSize, None);
|
||||
return Err(DriveError::Limit);
|
||||
}
|
||||
let result = self
|
||||
@@ -352,16 +463,20 @@ impl DriveBackend {
|
||||
),
|
||||
true,
|
||||
&request.request_id,
|
||||
&diagnostic,
|
||||
)
|
||||
.await;
|
||||
self.reconcile(&request.request_id, result).await
|
||||
self.reconcile(&request.request_id, result, &diagnostic)
|
||||
.await
|
||||
}
|
||||
pub async fn upload(
|
||||
&self,
|
||||
query: DriveUploadQuery,
|
||||
bytes: Vec<u8>,
|
||||
) -> Result<DriveMutationResponse, DriveError> {
|
||||
let diagnostic = Diagnostic::new(Adapter::Drive, "upload", true, Some(&query.request_id));
|
||||
if bytes.len() > DRIVE_FILE_MAX_BYTES as usize {
|
||||
diagnostic.record(Stage::Request, Reason::RequestSize, None);
|
||||
return Err(DriveError::Limit);
|
||||
}
|
||||
let path = format!("{}/upload{}", self.base()?, query_string(&query)?);
|
||||
@@ -372,29 +487,56 @@ impl DriveBackend {
|
||||
max_response_bytes: RESPONSE_LIMIT,
|
||||
};
|
||||
let client = self.client.clone();
|
||||
let deadline = self.deadline;
|
||||
let result = tokio::time::timeout(
|
||||
DEADLINE,
|
||||
deadline,
|
||||
tokio::task::spawn_blocking(move || {
|
||||
client.execute_binary_with_timeout(request, DEADLINE)
|
||||
client.execute_binary_with_timeout(request, deadline)
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
let result = match result {
|
||||
Ok(Ok(Ok(response))) if response.body.len() > RESPONSE_LIMIT => {
|
||||
diagnostic.record(Stage::Response, Reason::ResponseSize, None);
|
||||
Err(DriveError::OutcomeUnknown(query.request_id.clone()))
|
||||
}
|
||||
Ok(Ok(Ok(response))) if (200..300).contains(&response.status) => {
|
||||
serde_json::from_slice(&response.body)
|
||||
.map_err(|_| DriveError::OutcomeUnknown(query.request_id.clone()))
|
||||
serde_json::from_slice(&response.body).map_err(|_| {
|
||||
diagnostic.record(Stage::Response, Reason::Decode, None);
|
||||
DriveError::OutcomeUnknown(query.request_id.clone())
|
||||
})
|
||||
}
|
||||
Ok(Ok(Ok(response))) => Err(classify(
|
||||
response.status,
|
||||
&response.body,
|
||||
true,
|
||||
&query.request_id,
|
||||
&diagnostic,
|
||||
)),
|
||||
_ => Err(DriveError::OutcomeUnknown(query.request_id.clone())),
|
||||
Err(_) => {
|
||||
diagnostic.record(Stage::Request, Reason::Deadline, None);
|
||||
Err(DriveError::OutcomeUnknown(query.request_id.clone()))
|
||||
}
|
||||
Ok(Err(_)) => {
|
||||
diagnostic.record(Stage::Request, Reason::TaskJoin, None);
|
||||
Err(DriveError::OutcomeUnknown(query.request_id.clone()))
|
||||
}
|
||||
Ok(Ok(Err(error))) => {
|
||||
diagnostic.transport(&error);
|
||||
Err(DriveError::OutcomeUnknown(query.request_id.clone()))
|
||||
}
|
||||
};
|
||||
self.reconcile(&query.request_id, result).await
|
||||
self.reconcile(&query.request_id, result, &diagnostic).await
|
||||
}
|
||||
pub async fn status(&self, request_id: &str) -> Result<DriveRequestStatusResponse, DriveError> {
|
||||
let diagnostic = Diagnostic::new(Adapter::Drive, "request_status", false, Some(request_id));
|
||||
self.status_with_diagnostic(request_id, &diagnostic).await
|
||||
}
|
||||
async fn status_with_diagnostic(
|
||||
&self,
|
||||
request_id: &str,
|
||||
diagnostic: &Diagnostic,
|
||||
) -> Result<DriveRequestStatusResponse, DriveError> {
|
||||
if request_id.is_empty() || request_id.len() > 256 {
|
||||
return Err(DriveError::Invalid("invalid request ID".into()));
|
||||
}
|
||||
@@ -403,13 +545,15 @@ impl DriveBackend {
|
||||
WorkspaceRequest::get(format!("{}/requests/{}", self.base()?, encode(request_id))),
|
||||
false,
|
||||
"",
|
||||
diagnostic,
|
||||
)
|
||||
.await?;
|
||||
if response.request_id != request_id {
|
||||
diagnostic.record(Stage::Reconcile, Reason::Identity, None);
|
||||
return Err(DriveError::Unavailable);
|
||||
}
|
||||
if let Some(result) = &response.response {
|
||||
self.validate_completion(request_id, result)?;
|
||||
self.validate_completion(request_id, result, diagnostic)?;
|
||||
}
|
||||
Ok(response)
|
||||
}
|
||||
@@ -463,11 +607,21 @@ fn query_string<T: Serialize>(value: &T) -> Result<String, DriveError> {
|
||||
format!("?{}", pairs.join("&"))
|
||||
})
|
||||
}
|
||||
fn classify(status: u16, body: &[u8], mutation: bool, id: &str) -> DriveError {
|
||||
fn classify(
|
||||
status: u16,
|
||||
body: &[u8],
|
||||
mutation: bool,
|
||||
id: &str,
|
||||
diagnostic: &Diagnostic,
|
||||
) -> DriveError {
|
||||
if let Ok(error) = serde_json::from_slice::<DriveApiError>(body) {
|
||||
if error.classification == DriveFailureClassification::Unknown {
|
||||
if error.classification == DriveFailureClassification::Unknown
|
||||
|| error.code == DriveApiErrorCode::OutcomeUnknown
|
||||
{
|
||||
diagnostic.record(Stage::Response, Reason::BackendUnknown, Some(status));
|
||||
return DriveError::OutcomeUnknown(id.into());
|
||||
}
|
||||
diagnostic.record(Stage::Response, Reason::BackendRejected, Some(status));
|
||||
return match error.code {
|
||||
DriveApiErrorCode::Denied => DriveError::Denied,
|
||||
DriveApiErrorCode::NotFound => DriveError::NotFound,
|
||||
@@ -478,6 +632,7 @@ fn classify(status: u16, body: &[u8], mutation: bool, id: &str) -> DriveError {
|
||||
DriveApiErrorCode::OutcomeUnknown => DriveError::OutcomeUnknown(id.into()),
|
||||
};
|
||||
}
|
||||
diagnostic.record(Stage::Response, Reason::UnrecognizedResponse, Some(status));
|
||||
if matches!(status, 401 | 403) {
|
||||
DriveError::Denied
|
||||
} else if mutation {
|
||||
@@ -486,3 +641,7 @@ fn classify(status: u16, body: &[u8], mutation: bool, id: &str) -> DriveError {
|
||||
DriveError::Unavailable
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "backend_tests.rs"]
|
||||
mod tests;
|
||||
|
||||
@@ -0,0 +1,644 @@
|
||||
//! Production Drive adapter fault injection. Only the Workspace transport boundary is substituted;
|
||||
//! retained fixture effects and original-request inquiries are inspected independently of replies.
|
||||
//! The HTTP case additionally connects the adapter to reqwest and an isolated loopback router.
|
||||
use super::*;
|
||||
use crate::feature::builtin::backend_diagnostic::{Reason, tests::Capture};
|
||||
use crate::worker::{WorkspaceBinaryResponse, WorkspaceClientError, WorkspaceResponse};
|
||||
use std::sync::{Condvar, Mutex};
|
||||
use tracing::instrument::WithSubscriber;
|
||||
|
||||
const SECRET: &str = "secret-token /private/drive credential=unsafe";
|
||||
const ID: &str = "host-request-1";
|
||||
|
||||
fn entry() -> DriveEntry {
|
||||
DriveEntry {
|
||||
entry: DriveEntryRef {
|
||||
workspace_id: "workspace".into(),
|
||||
node_id: "2".into(),
|
||||
},
|
||||
parent: Some(DriveEntryRef {
|
||||
workspace_id: "workspace".into(),
|
||||
node_id: "1".into(),
|
||||
}),
|
||||
name: SECRET.into(),
|
||||
kind: DriveEntryKind::File,
|
||||
last_mutation_id: "old-request".into(),
|
||||
size: Some(5),
|
||||
content_type: Some("text/plain".into()),
|
||||
updated_by: SECRET.into(),
|
||||
updated_at: SECRET.into(),
|
||||
latest_url: SECRET.into(),
|
||||
}
|
||||
}
|
||||
fn mutation() -> DriveMutationRequest {
|
||||
DriveMutationRequest {
|
||||
request_id: ID.into(),
|
||||
mutation: DriveMutation::UpdateText {
|
||||
id: entry().entry,
|
||||
expected_mutation_id: "old-request".into(),
|
||||
text: SECRET.into(),
|
||||
content_type: "text/plain".into(),
|
||||
},
|
||||
}
|
||||
}
|
||||
fn json(value: impl Serialize) -> WorkspaceResponse {
|
||||
WorkspaceResponse {
|
||||
status: 200,
|
||||
body: serde_json::to_string(&value).unwrap(),
|
||||
}
|
||||
}
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
enum Fault {
|
||||
Communication,
|
||||
Unavailable,
|
||||
InvalidPath,
|
||||
Decode,
|
||||
Size,
|
||||
Length,
|
||||
Identity,
|
||||
Metadata,
|
||||
Rejected,
|
||||
Unknown,
|
||||
Unrecognized,
|
||||
Incomplete,
|
||||
Panic,
|
||||
Deadline,
|
||||
None,
|
||||
}
|
||||
#[derive(Debug, Default)]
|
||||
struct State {
|
||||
mutations: usize,
|
||||
saved: Option<String>,
|
||||
requests: Vec<WorkspaceRequest>,
|
||||
binary: Vec<WorkspaceBinaryRequest>,
|
||||
released: bool,
|
||||
started: bool,
|
||||
finished: bool,
|
||||
}
|
||||
#[derive(Debug)]
|
||||
struct Client {
|
||||
fault: Fault,
|
||||
receipt: Fault,
|
||||
state: Mutex<State>,
|
||||
changed: Condvar,
|
||||
started: tokio::sync::Notify,
|
||||
}
|
||||
impl Client {
|
||||
fn new(fault: Fault, receipt: Fault) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
fault,
|
||||
receipt,
|
||||
state: Mutex::new(State::default()),
|
||||
changed: Condvar::new(),
|
||||
started: tokio::sync::Notify::new(),
|
||||
})
|
||||
}
|
||||
fn fail(
|
||||
&self,
|
||||
fault: Fault,
|
||||
success: WorkspaceResponse,
|
||||
) -> Result<WorkspaceResponse, WorkspaceClientError> {
|
||||
match fault {
|
||||
Fault::Communication => Err(WorkspaceClientError::Request(SECRET.into())),
|
||||
Fault::Unavailable => Err(WorkspaceClientError::Unavailable(SECRET.into())),
|
||||
Fault::InvalidPath => Err(WorkspaceClientError::InvalidPath(SECRET.into())),
|
||||
Fault::Panic => panic!("injected adapter task panic"),
|
||||
Fault::Decode => Ok(WorkspaceResponse {
|
||||
status: 200,
|
||||
body: SECRET.into(),
|
||||
}),
|
||||
Fault::Size => Ok(WorkspaceResponse {
|
||||
status: 200,
|
||||
body: SECRET.repeat(RESPONSE_LIMIT / SECRET.len() + 1),
|
||||
}),
|
||||
Fault::Rejected | Fault::Unknown => Ok(WorkspaceResponse {
|
||||
status: 409,
|
||||
body: serde_json::to_string(&DriveApiError {
|
||||
code: DriveApiErrorCode::Conflict,
|
||||
message: SECRET.into(),
|
||||
classification: if matches!(fault, Fault::Unknown) {
|
||||
DriveFailureClassification::Unknown
|
||||
} else {
|
||||
DriveFailureClassification::NotCommitted
|
||||
},
|
||||
})
|
||||
.unwrap(),
|
||||
}),
|
||||
Fault::Unrecognized => Ok(WorkspaceResponse {
|
||||
status: 503,
|
||||
body: SECRET.into(),
|
||||
}),
|
||||
_ => Ok(success),
|
||||
}
|
||||
}
|
||||
fn release(&self) {
|
||||
self.state.lock().unwrap().released = true;
|
||||
self.changed.notify_all();
|
||||
}
|
||||
fn wait(&self, finished: bool) {
|
||||
let (state, timeout) = self
|
||||
.changed
|
||||
.wait_timeout_while(self.state.lock().unwrap(), Duration::from_secs(5), |s| {
|
||||
if finished { !s.finished } else { !s.started }
|
||||
})
|
||||
.unwrap();
|
||||
assert!(!timeout.timed_out());
|
||||
assert!(if finished {
|
||||
state.finished
|
||||
} else {
|
||||
state.started
|
||||
});
|
||||
}
|
||||
}
|
||||
struct Release(Arc<Client>);
|
||||
impl Drop for Release {
|
||||
fn drop(&mut self) {
|
||||
self.0.release();
|
||||
}
|
||||
}
|
||||
impl WorkspaceClient for Client {
|
||||
fn workspace_id(&self) -> Option<&str> {
|
||||
Some("workspace")
|
||||
}
|
||||
fn kind(&self) -> &str {
|
||||
"drive-diagnostic-fixture"
|
||||
}
|
||||
fn is_available(&self) -> bool {
|
||||
true
|
||||
}
|
||||
fn execute(
|
||||
&self,
|
||||
request: WorkspaceRequest,
|
||||
) -> Result<WorkspaceResponse, WorkspaceClientError> {
|
||||
let is_status = request.path.contains("/requests/");
|
||||
let fault = if is_status { self.receipt } else { self.fault };
|
||||
let mutation = request.method == WorkspaceRequestMethod::Post;
|
||||
{
|
||||
let mut state = self.state.lock().unwrap();
|
||||
if mutation
|
||||
&& !matches!(
|
||||
fault,
|
||||
Fault::Rejected | Fault::InvalidPath | Fault::Unavailable
|
||||
)
|
||||
{
|
||||
let input: DriveMutationRequest =
|
||||
serde_json::from_str(request.body.as_ref().unwrap()).unwrap();
|
||||
assert_eq!(input, super::tests::mutation());
|
||||
state.mutations += 1;
|
||||
if let DriveMutation::UpdateText { text, .. } = input.mutation {
|
||||
state.saved = Some(text);
|
||||
}
|
||||
}
|
||||
state.requests.push(request);
|
||||
state.started = true;
|
||||
self.started.notify_one();
|
||||
self.changed.notify_all();
|
||||
if matches!(fault, Fault::Deadline) {
|
||||
let (_state, timeout) = self
|
||||
.changed
|
||||
.wait_timeout_while(state, Duration::from_secs(5), |s| !s.released)
|
||||
.unwrap();
|
||||
assert!(
|
||||
!timeout.timed_out(),
|
||||
"transport must be explicitly released"
|
||||
);
|
||||
}
|
||||
}
|
||||
let mut metadata = entry();
|
||||
if matches!(fault, Fault::Identity) {
|
||||
metadata.entry.node_id = "3".into();
|
||||
}
|
||||
if matches!(fault, Fault::Metadata) {
|
||||
metadata.size = Some(DRIVE_FILE_MAX_BYTES + 1);
|
||||
}
|
||||
let completion = DriveMutationResponse {
|
||||
request_id: if matches!(fault, Fault::Identity) {
|
||||
"other-request"
|
||||
} else {
|
||||
ID
|
||||
}
|
||||
.into(),
|
||||
entry: Some(metadata.clone()),
|
||||
};
|
||||
let value = if is_status {
|
||||
json(DriveRequestStatusResponse {
|
||||
request_id: if matches!(fault, Fault::Identity) {
|
||||
"other-request"
|
||||
} else {
|
||||
ID
|
||||
}
|
||||
.into(),
|
||||
state: if matches!(
|
||||
fault,
|
||||
Fault::None | Fault::Identity | Fault::Metadata | Fault::Incomplete
|
||||
) {
|
||||
DriveRequestState::Committed
|
||||
} else {
|
||||
DriveRequestState::Uncommitted
|
||||
},
|
||||
response: if matches!(fault, Fault::None | Fault::Identity | Fault::Metadata) {
|
||||
Some(completion)
|
||||
} else {
|
||||
None
|
||||
},
|
||||
})
|
||||
} else if mutation {
|
||||
json(completion)
|
||||
} else {
|
||||
json(metadata)
|
||||
};
|
||||
if !is_status {
|
||||
self.state.lock().unwrap().finished = true;
|
||||
}
|
||||
self.changed.notify_all();
|
||||
self.fail(fault, value)
|
||||
}
|
||||
fn execute_binary(
|
||||
&self,
|
||||
request: WorkspaceBinaryRequest,
|
||||
) -> Result<WorkspaceBinaryResponse, WorkspaceClientError> {
|
||||
let upload = request.method == WorkspaceRequestMethod::Put;
|
||||
{
|
||||
let mut state = self.state.lock().unwrap();
|
||||
if upload {
|
||||
state.mutations += 1;
|
||||
state.saved = Some(SECRET.into());
|
||||
}
|
||||
state.binary.push(request);
|
||||
}
|
||||
let success = if upload {
|
||||
json(DriveMutationResponse {
|
||||
request_id: ID.into(),
|
||||
entry: Some(entry()),
|
||||
})
|
||||
} else {
|
||||
WorkspaceResponse {
|
||||
status: 200,
|
||||
body: if matches!(self.fault, Fault::Length) {
|
||||
"x"
|
||||
} else {
|
||||
"12345"
|
||||
}
|
||||
.into(),
|
||||
}
|
||||
};
|
||||
self.fail(self.fault, success)
|
||||
.map(|r| WorkspaceBinaryResponse {
|
||||
status: r.status,
|
||||
body: r.body.into_bytes(),
|
||||
})
|
||||
}
|
||||
}
|
||||
fn safe(capture: &Capture, reason: Reason, public: &str) {
|
||||
let text = capture.text();
|
||||
assert!(text.contains(&format!("reason={reason:?}")), "{text}");
|
||||
assert!(text.contains("adapter=Drive"), "{text}");
|
||||
assert!(text.contains("correlation="), "{text}");
|
||||
for secret in [
|
||||
SECRET,
|
||||
"secret-token",
|
||||
"/private/drive",
|
||||
"credential=unsafe",
|
||||
] {
|
||||
assert!(!text.contains(secret), "internal diagnostic leaked content");
|
||||
assert!(!public.contains(secret), "public error leaked content");
|
||||
}
|
||||
}
|
||||
fn reason(fault: Fault) -> Reason {
|
||||
match fault {
|
||||
Fault::Communication => Reason::Communication,
|
||||
Fault::Unavailable => Reason::TransportUnavailable,
|
||||
Fault::InvalidPath => Reason::InvalidPath,
|
||||
Fault::Decode => Reason::Decode,
|
||||
Fault::Size => Reason::ResponseSize,
|
||||
Fault::Length => Reason::ResponseLength,
|
||||
Fault::Identity => Reason::Identity,
|
||||
Fault::Metadata => Reason::InvalidMetadata,
|
||||
Fault::Rejected => Reason::BackendRejected,
|
||||
Fault::Unknown => Reason::BackendUnknown,
|
||||
Fault::Unrecognized => Reason::UnrecognizedResponse,
|
||||
Fault::Panic => Reason::TaskJoin,
|
||||
Fault::Deadline => Reason::Deadline,
|
||||
Fault::Incomplete => Reason::IncompleteReceipt,
|
||||
Fault::None => panic!("no fault"),
|
||||
}
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn production_read_failures_retain_public_classification_and_safe_cause() {
|
||||
for fault in [
|
||||
Fault::Communication,
|
||||
Fault::Unavailable,
|
||||
Fault::InvalidPath,
|
||||
Fault::Decode,
|
||||
Fault::Size,
|
||||
Fault::Identity,
|
||||
Fault::Metadata,
|
||||
Fault::Rejected,
|
||||
Fault::Unknown,
|
||||
Fault::Unrecognized,
|
||||
Fault::Panic,
|
||||
] {
|
||||
let client = Client::new(fault, Fault::None);
|
||||
let backend = DriveBackend::new(client.clone());
|
||||
let capture = Capture::default();
|
||||
let error = backend
|
||||
.metadata(&entry().entry)
|
||||
.with_subscriber(capture.dispatch())
|
||||
.await
|
||||
.unwrap_err();
|
||||
match fault {
|
||||
Fault::Size => assert!(matches!(error, DriveError::Limit)),
|
||||
Fault::Rejected => assert!(matches!(error, DriveError::Conflict)),
|
||||
Fault::Unknown => assert!(matches!(error, DriveError::OutcomeUnknown(_))),
|
||||
_ => assert!(matches!(error, DriveError::Unavailable)),
|
||||
}
|
||||
safe(&capture, reason(fault), &error.to_string());
|
||||
let state = client.state.lock().unwrap();
|
||||
assert_eq!(state.requests.len(), 1);
|
||||
assert_eq!(state.mutations, 0);
|
||||
}
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn production_mutation_loss_reconciles_only_original_identity_and_never_resends() {
|
||||
for fault in [
|
||||
Fault::Communication,
|
||||
Fault::Decode,
|
||||
Fault::Size,
|
||||
Fault::Identity,
|
||||
Fault::Metadata,
|
||||
Fault::Unknown,
|
||||
Fault::Unrecognized,
|
||||
Fault::Panic,
|
||||
] {
|
||||
for receipt in [
|
||||
Fault::None,
|
||||
Fault::Identity,
|
||||
Fault::Metadata,
|
||||
Fault::Decode,
|
||||
Fault::Communication,
|
||||
Fault::Unrecognized,
|
||||
Fault::Length,
|
||||
Fault::Incomplete,
|
||||
] {
|
||||
let client = Client::new(fault, receipt);
|
||||
let backend = DriveBackend::new(client.clone());
|
||||
let capture = Capture::default();
|
||||
let result = backend
|
||||
.mutate(mutation())
|
||||
.with_subscriber(capture.dispatch())
|
||||
.await;
|
||||
if matches!(receipt, Fault::None) {
|
||||
assert_eq!(result.unwrap().request_id, ID);
|
||||
} else {
|
||||
assert!(matches!(result, Err(DriveError::OutcomeUnknown(ref id)) if id == ID));
|
||||
}
|
||||
safe(&capture, reason(fault), "");
|
||||
let text = capture.text();
|
||||
if matches!(receipt, Fault::Length | Fault::Incomplete) {
|
||||
safe(
|
||||
&capture,
|
||||
if matches!(receipt, Fault::Length) {
|
||||
Reason::NotCommittedSnapshot
|
||||
} else {
|
||||
Reason::IncompleteReceipt
|
||||
},
|
||||
"",
|
||||
);
|
||||
}
|
||||
assert!(text.contains("stage=Reconcile"), "{text}");
|
||||
let correlations: std::collections::BTreeSet<_> = text
|
||||
.lines()
|
||||
.filter_map(|l| l.split("correlation=").nth(1))
|
||||
.filter_map(|l| l.split(';').next())
|
||||
.collect();
|
||||
assert_eq!(
|
||||
correlations.len(),
|
||||
1,
|
||||
"original loss and inquiry must correlate"
|
||||
);
|
||||
assert!(
|
||||
!text.contains(ID),
|
||||
"request identity must be opaque in logs"
|
||||
);
|
||||
let state = client.state.lock().unwrap();
|
||||
assert_eq!(state.mutations, 1);
|
||||
assert_eq!(state.saved.as_deref(), Some(SECRET));
|
||||
assert_eq!(state.requests.len(), 2);
|
||||
assert_eq!(state.requests[1].method, WorkspaceRequestMethod::Get);
|
||||
assert!(state.requests[1].path.ends_with(&format!("/requests/{ID}")));
|
||||
}
|
||||
}
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn production_rejection_has_no_effect_or_status_inquiry() {
|
||||
let client = Client::new(Fault::Rejected, Fault::None);
|
||||
let backend = DriveBackend::new(client.clone());
|
||||
let capture = Capture::default();
|
||||
let error = backend
|
||||
.mutate(mutation())
|
||||
.with_subscriber(capture.dispatch())
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(error, DriveError::Conflict));
|
||||
safe(&capture, Reason::BackendRejected, &error.to_string());
|
||||
let state = client.state.lock().unwrap();
|
||||
assert_eq!(state.mutations, 0);
|
||||
assert_eq!(state.requests.len(), 1);
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn production_binary_length_transport_and_upload_loss_are_diagnosed() {
|
||||
for fault in [
|
||||
Fault::Length,
|
||||
Fault::Communication,
|
||||
Fault::Unavailable,
|
||||
Fault::Panic,
|
||||
] {
|
||||
let client = Client::new(fault, Fault::None);
|
||||
let backend = DriveBackend::new(client.clone());
|
||||
let capture = Capture::default();
|
||||
let error = backend
|
||||
.bytes(&entry(), 5)
|
||||
.with_subscriber(capture.dispatch())
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(error, DriveError::Unavailable));
|
||||
safe(&capture, reason(fault), &error.to_string());
|
||||
assert_eq!(client.state.lock().unwrap().binary.len(), 1);
|
||||
}
|
||||
for fault in [Fault::Decode, Fault::Communication, Fault::Size] {
|
||||
let client = Client::new(fault, Fault::Unrecognized);
|
||||
let backend = DriveBackend::new(client.clone());
|
||||
let capture = Capture::default();
|
||||
let query = DriveUploadQuery {
|
||||
operation: DriveUploadOperation::Update,
|
||||
request_id: ID.into(),
|
||||
entry_workspace_id: "workspace".into(),
|
||||
parent_id: None,
|
||||
name: None,
|
||||
id: Some("2".into()),
|
||||
expected_mutation_id: Some("old-request".into()),
|
||||
content_type: "text/plain".into(),
|
||||
size: 5,
|
||||
sha256: "hash".into(),
|
||||
};
|
||||
let error = backend
|
||||
.upload(query, b"12345".to_vec())
|
||||
.with_subscriber(capture.dispatch())
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(error, DriveError::OutcomeUnknown(ref id) if id == ID));
|
||||
safe(&capture, reason(fault), &error.to_string());
|
||||
let state = client.state.lock().unwrap();
|
||||
assert_eq!(state.mutations, 1);
|
||||
assert_eq!(state.binary.len(), 1);
|
||||
assert_eq!(state.requests.len(), 1);
|
||||
assert!(state.requests[0].path.ends_with(&format!("/requests/{ID}")));
|
||||
}
|
||||
}
|
||||
#[tokio::test]
|
||||
async fn production_deadline_does_not_cancel_effect_or_infer_uncommitted_from_snapshot() {
|
||||
for mutation_call in [false, true] {
|
||||
let client = Client::new(Fault::Deadline, Fault::Unrecognized);
|
||||
let release = Release(client.clone());
|
||||
let mut backend = DriveBackend::new(client.clone());
|
||||
backend.deadline = Duration::from_secs(1);
|
||||
let capture = Capture::default();
|
||||
let request = async {
|
||||
if mutation_call {
|
||||
backend.mutate(mutation()).await.map(|_| ())
|
||||
} else {
|
||||
backend.metadata(&entry().entry).await.map(|_| ())
|
||||
}
|
||||
}
|
||||
.with_subscriber(capture.dispatch());
|
||||
tokio::pin!(request);
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = client.started.notified() => {},
|
||||
_ = &mut request => panic!("request must start before timeout"),
|
||||
}
|
||||
{
|
||||
let state = client.state.lock().unwrap();
|
||||
assert_eq!(state.mutations, usize::from(mutation_call));
|
||||
assert!(!state.released && !state.finished);
|
||||
}
|
||||
let result = request.await;
|
||||
let error = result.unwrap_err();
|
||||
assert!(if mutation_call {
|
||||
matches!(error, DriveError::OutcomeUnknown(ref id) if id == ID)
|
||||
} else {
|
||||
matches!(error, DriveError::Unavailable)
|
||||
});
|
||||
safe(&capture, Reason::Deadline, &error.to_string());
|
||||
drop(release);
|
||||
let waiting = client.clone();
|
||||
tokio::task::spawn_blocking(move || waiting.wait(true))
|
||||
.await
|
||||
.unwrap();
|
||||
let state = client.state.lock().unwrap();
|
||||
assert_eq!(state.mutations, usize::from(mutation_call));
|
||||
assert_eq!(state.saved.is_some(), mutation_call);
|
||||
assert_eq!(state.requests.len(), if mutation_call { 2 } else { 1 });
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn http_reply_loss_keeps_original_receipt_and_retained_effect_without_reexecution() {
|
||||
use axum::{
|
||||
Router,
|
||||
extract::State as HttpState,
|
||||
routing::{get, post},
|
||||
};
|
||||
#[derive(Default)]
|
||||
struct Stored {
|
||||
updates: usize,
|
||||
content: String,
|
||||
ids: Vec<String>,
|
||||
}
|
||||
struct Server(tokio::task::JoinHandle<()>);
|
||||
impl Drop for Server {
|
||||
fn drop(&mut self) {
|
||||
self.0.abort();
|
||||
}
|
||||
}
|
||||
let stored = Arc::new(Mutex::new(Stored::default()));
|
||||
let router = Router::new()
|
||||
.route("/api/w/workspace/drive/metadata", get(|| async { SECRET }))
|
||||
.route(
|
||||
"/api/w/workspace/drive/mutate",
|
||||
post(
|
||||
|HttpState(state): HttpState<Arc<Mutex<Stored>>>,
|
||||
body: axum::body::Bytes| async move {
|
||||
let request: DriveMutationRequest = serde_json::from_slice(&body).unwrap();
|
||||
let mut state = state.lock().unwrap();
|
||||
state.ids.push(request.request_id);
|
||||
if let DriveMutation::UpdateText {
|
||||
text,
|
||||
expected_mutation_id,
|
||||
..
|
||||
} = request.mutation
|
||||
{
|
||||
assert_eq!(expected_mutation_id, "old-request");
|
||||
state.content = text;
|
||||
state.updates += 1;
|
||||
} else {
|
||||
panic!("unexpected mutation");
|
||||
}
|
||||
// The effect is retained; only the success response is malformed.
|
||||
SECRET
|
||||
},
|
||||
),
|
||||
)
|
||||
.route(
|
||||
"/api/w/workspace/drive/requests/{id}",
|
||||
get(
|
||||
|HttpState(state): HttpState<Arc<Mutex<Stored>>>,
|
||||
axum::extract::Path(id): axum::extract::Path<String>| async move {
|
||||
let state = state.lock().unwrap();
|
||||
assert_eq!(state.ids, [id.clone()]);
|
||||
axum::Json(DriveRequestStatusResponse {
|
||||
request_id: id.clone(),
|
||||
state: DriveRequestState::Committed,
|
||||
response: Some(DriveMutationResponse {
|
||||
request_id: id,
|
||||
entry: Some(entry()),
|
||||
}),
|
||||
})
|
||||
},
|
||||
),
|
||||
)
|
||||
.with_state(stored.clone());
|
||||
let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let address = listener.local_addr().unwrap();
|
||||
let server = Server(tokio::spawn(async move {
|
||||
axum::serve(listener, router).await.unwrap();
|
||||
}));
|
||||
let client = Arc::new(crate::worker::TestWorkspaceHttpClient::new(
|
||||
"workspace",
|
||||
format!("http://{address}"),
|
||||
));
|
||||
let backend = DriveBackend::new(client);
|
||||
let capture = Capture::default();
|
||||
let error = backend
|
||||
.metadata(&entry().entry)
|
||||
.with_subscriber(capture.dispatch())
|
||||
.await
|
||||
.unwrap_err();
|
||||
assert!(matches!(error, DriveError::Unavailable));
|
||||
safe(&capture, Reason::Decode, &error.to_string());
|
||||
let capture = Capture::default();
|
||||
let response = backend
|
||||
.mutate(mutation())
|
||||
.with_subscriber(capture.dispatch())
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(response.request_id, ID);
|
||||
safe(&capture, Reason::Decode, "");
|
||||
assert!(capture.text().contains("reason=Reconciled"));
|
||||
let state = stored.lock().unwrap();
|
||||
assert_eq!(state.updates, 1);
|
||||
assert_eq!(state.content, SECRET);
|
||||
assert_eq!(state.ids, [ID]);
|
||||
drop(state);
|
||||
drop(server);
|
||||
}
|
||||
@@ -59,6 +59,53 @@ impl JournalReadStatus {
|
||||
}
|
||||
}
|
||||
|
||||
// Only bounded classifications cross into diagnostics. Host messages/codes and
|
||||
// io::Error payloads can contain Journal text, credentials, proofs or host paths.
|
||||
// These diagnostics belong to the Feature, not persisted/model-visible input.
|
||||
enum JournalCompactionFailure {
|
||||
Connection(std::io::ErrorKind),
|
||||
SnapshotConflict,
|
||||
SnapshotInvalidArgument,
|
||||
SnapshotUnavailable,
|
||||
Serialization,
|
||||
ByteBudget,
|
||||
}
|
||||
|
||||
impl JournalCompactionFailure {
|
||||
fn snapshot(error: SubjektivHostError) -> Self {
|
||||
match error {
|
||||
SubjektivHostError::Conflict { .. } => Self::SnapshotConflict,
|
||||
SubjektivHostError::InvalidArgument(_) => Self::SnapshotInvalidArgument,
|
||||
SubjektivHostError::Unavailable(_) => Self::SnapshotUnavailable,
|
||||
}
|
||||
}
|
||||
|
||||
fn stage(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Connection(_) => "connection",
|
||||
Self::SnapshotConflict | Self::SnapshotInvalidArgument | Self::SnapshotUnavailable => {
|
||||
"snapshot"
|
||||
}
|
||||
Self::Serialization => "serialization",
|
||||
Self::ByteBudget => "byte_budget",
|
||||
}
|
||||
}
|
||||
|
||||
fn reason(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Connection(std::io::ErrorKind::PermissionDenied) => "permission_denied",
|
||||
Self::Connection(std::io::ErrorKind::InvalidInput) => "invalid_input",
|
||||
Self::Connection(std::io::ErrorKind::NotConnected) => "not_connected",
|
||||
Self::Connection(_) => "io_error",
|
||||
Self::SnapshotConflict => "host_conflict",
|
||||
Self::SnapshotInvalidArgument => "host_invalid_argument",
|
||||
Self::SnapshotUnavailable => "host_unavailable",
|
||||
Self::Serialization => "json_serialization_failed",
|
||||
Self::ByteBudget => "startup_max_bytes_exceeded",
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct JournalCompactionInput {
|
||||
@@ -203,33 +250,46 @@ pub(crate) fn freeze_compaction_journal(
|
||||
.and_then(|connection| connection.host.workspace_id())
|
||||
});
|
||||
let read = match connection {
|
||||
Ok(Some(connection)) if connection.host.journal_compaction_granted() => {
|
||||
Some(connection.journal_compaction_context().map_err(|_| ()))
|
||||
}
|
||||
Ok(Some(connection)) if connection.host.journal_compaction_granted() => Some(
|
||||
connection
|
||||
.journal_compaction_context()
|
||||
.map_err(JournalCompactionFailure::snapshot),
|
||||
),
|
||||
Ok(_) => None,
|
||||
Err(_) => Some(Err(())),
|
||||
Err(error) => Some(Err(JournalCompactionFailure::Connection(error.kind()))),
|
||||
};
|
||||
let snapshot = read.as_ref().and_then(|read| read.as_ref().ok()).cloned();
|
||||
let (status, json) = match read {
|
||||
None => (Disabled, String::new()),
|
||||
let prepared = match read {
|
||||
None => Ok((Disabled, String::new())),
|
||||
Some(Ok(snapshot)) => {
|
||||
let status = if snapshot.journals.is_empty() {
|
||||
Absent
|
||||
} else {
|
||||
Available
|
||||
};
|
||||
match serde_json::to_string(&snapshot) {
|
||||
Ok(json) if json.len() <= subjektiv::journal::startup::STARTUP_MAX_BYTES => {
|
||||
(status, json)
|
||||
}
|
||||
_ => (Unavailable, String::new()),
|
||||
}
|
||||
// The current snapshot has only derived, JSON-safe fields (strings,
|
||||
// integers, vectors, enums and a UUID). There is no known serialization
|
||||
// failure fixture; retain this guard if the domain type changes.
|
||||
serde_json::to_string(&snapshot)
|
||||
.map_err(|_| JournalCompactionFailure::Serialization)
|
||||
.and_then(|json| {
|
||||
if json.len() <= subjektiv::journal::startup::STARTUP_MAX_BYTES {
|
||||
Ok((status, json))
|
||||
} else {
|
||||
Err(JournalCompactionFailure::ByteBudget)
|
||||
}
|
||||
})
|
||||
}
|
||||
Some(Err(())) => (Unavailable, String::new()),
|
||||
Some(Err(failure)) => Err(failure),
|
||||
};
|
||||
if status == Unavailable {
|
||||
tracing::warn!("Journal compaction snapshot unavailable; continuing without Journal");
|
||||
}
|
||||
let (status, json) = prepared.unwrap_or_else(|failure| {
|
||||
tracing::warn!(
|
||||
stage = failure.stage(),
|
||||
reason = failure.reason(),
|
||||
"Journal compaction snapshot unavailable; continuing without Journal"
|
||||
);
|
||||
(Unavailable, String::new())
|
||||
});
|
||||
use sha2::{Digest, Sha256};
|
||||
let snapshot_sha256: Option<String> = (!json.is_empty()).then(|| {
|
||||
Sha256::digest(json.as_bytes())
|
||||
@@ -726,12 +786,72 @@ fn tool_schema(name: &str) -> Value {
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "../../../tests/common/diagnostics.rs"]
|
||||
mod diagnostic_test_support;
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::Mutex;
|
||||
use subjektiv::journal::JournalReceipt;
|
||||
|
||||
#[test]
|
||||
fn compaction_connection_failure_keeps_safe_reason_without_exposing_error_or_changing_input() {
|
||||
let prompts = crate::prompt::catalog::PromptCatalog::builtins_only().unwrap();
|
||||
for (kind, reason) in [
|
||||
(std::io::ErrorKind::PermissionDenied, "permission_denied"),
|
||||
(std::io::ErrorKind::InvalidInput, "invalid_input"),
|
||||
(std::io::ErrorKind::NotConnected, "not_connected"),
|
||||
(std::io::ErrorKind::Other, "io_error"),
|
||||
] {
|
||||
let logs = diagnostic_test_support::CapturedWarnings::default();
|
||||
let input = tracing::subscriber::with_default(logs.subscriber(), || {
|
||||
freeze_compaction_journal(
|
||||
Err(std::io::Error::new(
|
||||
kind,
|
||||
"PRIVATE_JOURNAL_BODY /private/host/path proof=PRIVATE_PROOF credential=PRIVATE_CREDENTIAL raw_error=PRIVATE_ERROR",
|
||||
)),
|
||||
&prompts,
|
||||
)
|
||||
.unwrap()
|
||||
});
|
||||
let log = logs.text();
|
||||
assert!(log.contains("stage=\"connection\""), "{log}");
|
||||
assert!(log.contains(&format!("reason=\"{reason}\"")), "{log}");
|
||||
assert_eq!(log.matches("continuing without Journal").count(), 1);
|
||||
assert!(!log.contains("PRIVATE_"));
|
||||
assert!(!log.contains("/private/host/path"));
|
||||
assert!(input.body().contains("Journal read status: unavailable."));
|
||||
let persisted = serde_json::to_string(&input).unwrap();
|
||||
assert!(!persisted.contains("PRIVATE_"));
|
||||
assert!(!persisted.contains("/private/host/path"));
|
||||
assert!(!persisted.contains(reason));
|
||||
let payload: JournalCompactionInput = input
|
||||
.metadata()
|
||||
.feature_payload()
|
||||
.unwrap()
|
||||
.decode("journal:compaction", 1)
|
||||
.unwrap();
|
||||
assert!(payload.snapshot.is_none());
|
||||
assert!(payload.snapshot_sha256.is_none());
|
||||
assert_eq!(
|
||||
payload.prompt_provenance.source_digest,
|
||||
prompts.projection().source_digest
|
||||
);
|
||||
assert_eq!(
|
||||
payload.prompt_provenance.projection_digest,
|
||||
prompts.projection().catalog_digest
|
||||
);
|
||||
}
|
||||
let logs = diagnostic_test_support::CapturedWarnings::default();
|
||||
let input = tracing::subscriber::with_default(logs.subscriber(), || {
|
||||
freeze_compaction_journal(Ok(None), &prompts).unwrap()
|
||||
});
|
||||
assert!(input.body().contains("Journal read status: disabled."));
|
||||
assert!(logs.text().is_empty());
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Host {
|
||||
requests: Mutex<Vec<JournalJobRequest>>,
|
||||
|
||||
@@ -226,6 +226,7 @@ pub struct WorkspaceAttachedWorkdirSession {
|
||||
target_workdir: String,
|
||||
workdir: Workdir,
|
||||
capabilities: WorkdirSessionCapabilities,
|
||||
environment: workdir::WorkdirSessionEnvironment,
|
||||
}
|
||||
|
||||
impl WorkspaceAttachedWorkdirSession {
|
||||
@@ -249,17 +250,38 @@ impl WorkspaceAttachedWorkdirSession {
|
||||
)
|
||||
}
|
||||
|
||||
/// Bind a known root-coordinate attachment. Opened provider sessions must
|
||||
/// instead pass their metadata to [`Self::handle_for_workdir_with_environment`].
|
||||
pub fn handle_for_workdir_with_capabilities(
|
||||
client: Arc<dyn WorkspaceClient>,
|
||||
target_workdir: impl Into<String>,
|
||||
workdir_id: impl Into<String>,
|
||||
capabilities: WorkdirSessionCapabilities,
|
||||
) -> WorkdirSessionHandle {
|
||||
Self::handle_for_workdir_with_environment(
|
||||
client,
|
||||
target_workdir,
|
||||
workdir_id,
|
||||
capabilities,
|
||||
workdir::WorkdirSessionEnvironment::root(capabilities),
|
||||
)
|
||||
}
|
||||
|
||||
/// Bind provider-authored coordinates without allowing metadata to expand
|
||||
/// the Backend attachment capability ceiling.
|
||||
pub fn handle_for_workdir_with_environment(
|
||||
client: Arc<dyn WorkspaceClient>,
|
||||
target_workdir: impl Into<String>,
|
||||
workdir_id: impl Into<String>,
|
||||
capabilities: WorkdirSessionCapabilities,
|
||||
environment: workdir::WorkdirSessionEnvironment,
|
||||
) -> WorkdirSessionHandle {
|
||||
Arc::new(Self {
|
||||
client,
|
||||
target_workdir: target_workdir.into(),
|
||||
workdir: Workdir::new(workdir_id),
|
||||
capabilities,
|
||||
capabilities: capabilities.intersection(environment.capabilities),
|
||||
environment,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -291,6 +313,11 @@ impl WorkspaceAttachedWorkdirSession {
|
||||
.execute(request)
|
||||
.map_err(workspace_workdir_error)?;
|
||||
if !response.is_success() {
|
||||
if let Ok(error) =
|
||||
serde_json::from_str::<server_api::WorkdirOperationApiError>(&response.body)
|
||||
{
|
||||
notify_workdir_audit_failures(&error.audit_failures);
|
||||
}
|
||||
return Err(
|
||||
serde_json::from_str::<workdir::http::WorkdirTransportError>(&response.body)
|
||||
.map(workdir::http::WorkdirTransportError::into_workdir_error)
|
||||
@@ -302,11 +329,14 @@ impl WorkspaceAttachedWorkdirSession {
|
||||
}),
|
||||
);
|
||||
}
|
||||
serde_json::from_str(&response.body).map_err(|error| {
|
||||
WorkdirError::Transport(format!(
|
||||
"failed to decode Workspace Workdir operation result: {error}"
|
||||
))
|
||||
})
|
||||
let response: workdir::http::WorkdirSessionOperationResponse =
|
||||
serde_json::from_str(&response.body).map_err(|_| {
|
||||
WorkdirError::Transport(
|
||||
"failed to decode Workspace Workdir operation result".into(),
|
||||
)
|
||||
})?;
|
||||
notify_workdir_audit_failures(&response.audit_failures);
|
||||
Ok(response.result)
|
||||
}
|
||||
|
||||
fn mismatch(expected: &str) -> WorkdirError {
|
||||
@@ -316,6 +346,27 @@ impl WorkspaceAttachedWorkdirSession {
|
||||
}
|
||||
}
|
||||
|
||||
// Notification is separate from the operation return: callers retain successful command
|
||||
// handles, checkout observations and original failures, and must not resend for audit loss.
|
||||
fn notify_workdir_audit_failures(failures: &[workdir::http::WorkdirAuditFailure]) {
|
||||
use sha2::Digest;
|
||||
for failure in failures {
|
||||
// Response strings are untrusted. Never copy them into the receiving process log.
|
||||
let operation_id_hash = sha2::Sha256::digest(failure.operation_id.as_bytes())
|
||||
.iter()
|
||||
.map(|byte| format!("{byte:02x}"))
|
||||
.collect::<String>();
|
||||
tracing::error!(
|
||||
event = "external_workdir_audit_failed",
|
||||
operation_id_hash,
|
||||
audit_stage = ?failure.stage,
|
||||
audit_reason = ?failure.reason,
|
||||
operation_outcome = ?failure.operation_outcome,
|
||||
"External Workdir audit persistence failed; operation result is independent; do not resend"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
fn workspace_workdir_error(error: WorkspaceClientError) -> WorkdirError {
|
||||
match error {
|
||||
WorkspaceClientError::Unavailable(message) => WorkdirError::Unavailable(message),
|
||||
@@ -333,6 +384,12 @@ impl WorkdirSession for WorkspaceAttachedWorkdirSession {
|
||||
self.capabilities
|
||||
}
|
||||
|
||||
fn environment(&self) -> workdir::WorkdirSessionEnvironment {
|
||||
let mut environment = self.environment.clone();
|
||||
environment.capabilities = environment.capabilities.intersection(self.capabilities());
|
||||
environment
|
||||
}
|
||||
|
||||
async fn authorize_scope_path(
|
||||
&self,
|
||||
request: WorkdirScopeAuthorizationRequest,
|
||||
@@ -655,11 +712,12 @@ impl WorkspaceHttpWorkdirBackend {
|
||||
let response = self.attach_response(alias, working_directory_id)?;
|
||||
if let Err(error) = self.session_router.attach(
|
||||
alias_key.clone(),
|
||||
WorkspaceAttachedWorkdirSession::handle_for_workdir_with_capabilities(
|
||||
WorkspaceAttachedWorkdirSession::handle_for_workdir_with_environment(
|
||||
self.client.clone(),
|
||||
alias,
|
||||
working_directory_id,
|
||||
response.capabilities,
|
||||
response.attachment.capabilities,
|
||||
response.environment.clone(),
|
||||
),
|
||||
) {
|
||||
let _ = self.detach(alias);
|
||||
@@ -677,9 +735,9 @@ impl WorkspaceHttpWorkdirBackend {
|
||||
&self,
|
||||
alias: &str,
|
||||
working_directory_id: &str,
|
||||
) -> Result<WorkdirAttachmentResponse, ToolError> {
|
||||
) -> Result<WorkdirSessionAttachmentResponse, ToolError> {
|
||||
let workspace_id = encode_path_segment(self.workspace_id()?);
|
||||
self.execute_json::<WorkdirAttachmentResponse>(WorkspaceRequest::json(
|
||||
self.execute_json::<WorkdirSessionAttachmentResponse>(WorkspaceRequest::json(
|
||||
WorkspaceRequestMethod::Post,
|
||||
format!("/api/w/{workspace_id}/workers/self/workdir-attachments"),
|
||||
serde_json::to_string(&WorkdirAttachRequest {
|
||||
@@ -1163,6 +1221,13 @@ struct WorkdirAttachmentResponse {
|
||||
attached: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, PartialEq, Eq)]
|
||||
struct WorkdirSessionAttachmentResponse {
|
||||
#[serde(flatten)]
|
||||
attachment: WorkdirAttachmentResponse,
|
||||
environment: workdir::WorkdirSessionEnvironment,
|
||||
}
|
||||
|
||||
#[derive(Debug, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
struct WorkdirDeleteInput {
|
||||
@@ -1473,6 +1538,7 @@ mod tests {
|
||||
"alias": "checkout",
|
||||
"working_directory_id": "wd-created",
|
||||
"capabilities": {"bits": 63},
|
||||
"environment": workdir::WorkdirSessionEnvironment::root(WorkdirSessionCapabilities::ALL),
|
||||
"attached": true
|
||||
})),
|
||||
response(json!({
|
||||
@@ -1653,6 +1719,67 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn attached_session_keeps_operation_results_when_audit_persistence_failed() {
|
||||
let failure = json!({
|
||||
"operation_id":"op-safe-id", "stage":"response_received",
|
||||
"reason":"audit_store_unavailable", "operation_outcome":{"kind":"succeeded"}
|
||||
});
|
||||
let mut responses = vec![response(json!({
|
||||
"operation":"command_start", "result":"retained-command",
|
||||
"audit_failures":[failure]
|
||||
}))];
|
||||
for (code, status, outcome) in [
|
||||
("denied", 403, json!({"kind":"failed","code":"denied"})),
|
||||
("outcome_unknown", 500, json!({"kind":"unknown"})),
|
||||
("not_dispatched", 503, json!({"kind":"not_dispatched"})),
|
||||
] {
|
||||
let mut failure = failure.clone();
|
||||
failure["operation_outcome"] = outcome;
|
||||
responses.push(WorkspaceResponse { status, body: json!({
|
||||
"code":code, "message":"safe provider classification", "audit_failures":[failure]
|
||||
}).to_string() });
|
||||
}
|
||||
let client = Arc::new(RecordingWorkspaceClient::new(responses));
|
||||
let session = WorkspaceAttachedWorkdirSession::handle(client.clone(), "checkout");
|
||||
let handle = session
|
||||
.start_command(CommandRequest {
|
||||
command: "true".into(),
|
||||
timeout_secs: 1,
|
||||
transport_byte_budget: workdir::MAX_COMMAND_TRANSPORT_BYTES,
|
||||
cwd: workdir::WorkdirPath::root(),
|
||||
tool_call_id: None,
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
assert_eq!(handle.0, "retained-command");
|
||||
let request = workdir::CheckoutRequest {
|
||||
target: workdir::WorkdirPath::new("file.txt").unwrap(),
|
||||
validator: vec![1],
|
||||
operation: workdir::CheckoutOperation::Write {
|
||||
content: b"new".to_vec(),
|
||||
expected_hash: [0; 32],
|
||||
},
|
||||
};
|
||||
assert!(matches!(
|
||||
session.checkout_execute(request.clone()).await,
|
||||
Err(WorkdirError::Denied(_))
|
||||
));
|
||||
assert!(matches!(
|
||||
session.checkout_execute(request.clone()).await,
|
||||
Err(WorkdirError::OutcomeUnknown(_))
|
||||
));
|
||||
assert!(matches!(
|
||||
session.checkout_execute(request).await,
|
||||
Err(WorkdirError::NotDispatched)
|
||||
));
|
||||
assert_eq!(
|
||||
client.requests().len(),
|
||||
4,
|
||||
"audit loss must not trigger a resend"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn attached_session_does_not_send_worker_local_bash_spill_path() {
|
||||
let client = Arc::new(RecordingWorkspaceClient::new(vec![response(json!({
|
||||
@@ -1914,6 +2041,7 @@ mod tests {
|
||||
"alias": "checkout",
|
||||
"working_directory_id": "wd-attached",
|
||||
"capabilities": {"bits": 63},
|
||||
"environment": workdir::WorkdirSessionEnvironment::root(WorkdirSessionCapabilities::ALL),
|
||||
"attached": true
|
||||
})),
|
||||
response(json!({
|
||||
@@ -1973,16 +2101,29 @@ mod tests {
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn attach_propagates_provider_read_only_capabilities_to_the_alias_session() {
|
||||
#[tokio::test]
|
||||
async fn attach_projects_provider_environment_without_expanding_capability_ceiling() {
|
||||
let dir = tempfile::tempdir().unwrap();
|
||||
std::fs::create_dir(dir.path().join("src")).unwrap();
|
||||
let provider = Arc::new(workdir::LocalWorkdirSession::materialized(
|
||||
dir.path().to_path_buf(),
|
||||
dir.path().to_path_buf(),
|
||||
manifest::SharedScope::new(manifest::Scope::writable(dir.path()).unwrap()),
|
||||
WorkdirSessionCapabilities::ALL,
|
||||
));
|
||||
let broker =
|
||||
workdir::WorkdirToolBroker::at_cwd(provider, workdir::WorkdirPath::new("src").unwrap())
|
||||
.unwrap();
|
||||
let mut environment = broker.tool_session().environment();
|
||||
let client = Arc::new(RecordingWorkspaceClient::new(vec![response(json!({
|
||||
"workspace_id": "workspace/test",
|
||||
"alias": "external",
|
||||
"working_directory_id": "external-workdir",
|
||||
"capabilities": {"bits": 25},
|
||||
"environment": environment,
|
||||
"attached": true
|
||||
}))]));
|
||||
let backend = WorkspaceHttpWorkdirBackend::new(client);
|
||||
let backend = WorkspaceHttpWorkdirBackend::new(client.clone());
|
||||
backend
|
||||
.attach(WorkdirAttachInput {
|
||||
alias: "external".to_string(),
|
||||
@@ -1994,10 +2135,33 @@ mod tests {
|
||||
.session_router
|
||||
.session(&workdir::WorkdirAttachmentAlias::new("external").unwrap())
|
||||
.unwrap();
|
||||
environment.capabilities = WorkdirSessionCapabilities::READ_ONLY;
|
||||
assert_eq!(session.environment(), environment);
|
||||
assert_eq!(
|
||||
session.capabilities(),
|
||||
WorkdirSessionCapabilities::READ_ONLY
|
||||
);
|
||||
assert!(matches!(
|
||||
session
|
||||
.checkout_execute(workdir::CheckoutRequest {
|
||||
target: workdir::WorkdirPath::root(),
|
||||
validator: vec![1],
|
||||
operation: workdir::CheckoutOperation::Create {
|
||||
path: workdir::WorkdirPath::new("new.txt").unwrap(),
|
||||
content: b"denied".to_vec(),
|
||||
},
|
||||
})
|
||||
.await,
|
||||
Err(WorkdirError::Unsupported(
|
||||
workdir::WorkdirSessionCapability::Write
|
||||
))
|
||||
));
|
||||
assert_eq!(
|
||||
client.requests().len(),
|
||||
1,
|
||||
"denied mutation must not reach the Backend"
|
||||
);
|
||||
broker.tool_session().close().await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -2007,6 +2171,7 @@ mod tests {
|
||||
"alias": "checkout",
|
||||
"working_directory_id": "wd-attached",
|
||||
"capabilities": {"bits": 63},
|
||||
"environment": workdir::WorkdirSessionEnvironment::root(WorkdirSessionCapabilities::ALL),
|
||||
"attached": true
|
||||
}))]));
|
||||
let reopen_calls = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
@@ -1253,7 +1253,7 @@ pub(crate) mod tests {
|
||||
state.generation += 1;
|
||||
let raw = json!({"alias":body["alias"], "working_directory_id":body["working_directory_id"], "connection_id":format!("connection-{}", state.generation), "capabilities":{"bits":63}});
|
||||
state.attachments.push(raw);
|
||||
json!({"workspace_id":"test-workspace", "alias":body["alias"], "working_directory_id":body["working_directory_id"], "capabilities":{"bits":63}, "attached":true})
|
||||
json!({"workspace_id":"test-workspace", "alias":body["alias"], "working_directory_id":body["working_directory_id"], "capabilities":{"bits":63}, "environment":workdir::WorkdirSessionEnvironment::root(workdir::WorkdirSessionCapabilities::ALL), "attached":true})
|
||||
}
|
||||
(WorkspaceRequestMethod::Delete, p)
|
||||
if p.starts_with("workers/self/workdir-attachments/") =>
|
||||
|
||||
@@ -7,6 +7,7 @@
|
||||
//! undecodable mutation responses do not prove non-commit; dispatched requests are never resent.
|
||||
//! This adapter does not reimplement Backend source-persistence transactions or activation.
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use async_trait::async_trait;
|
||||
use serde::{Serialize, de::DeserializeOwned};
|
||||
@@ -20,6 +21,7 @@ use super::{
|
||||
ATTACHMENT_ALIAS, CONTENT_ROOT, ConfigAccess, ConfigAttachError, ConfigAttachment, DEADLINE,
|
||||
WorkspaceConfigAttachmentBackend,
|
||||
};
|
||||
use crate::feature::builtin::backend_diagnostic::{Adapter, Diagnostic, Reason, Stage};
|
||||
use crate::wip::WipOperationError;
|
||||
use crate::worker::{
|
||||
WorkspaceClient, WorkspaceClientError, WorkspaceRequest, WorkspaceRequestMethod,
|
||||
@@ -34,11 +36,21 @@ const MAX_RESPONSE_BYTES: usize = 2 * 1024 * 1024;
|
||||
#[derive(Clone)]
|
||||
pub struct WorkspaceConfigBackend {
|
||||
client: Arc<dyn WorkspaceClient>,
|
||||
deadline: Duration,
|
||||
}
|
||||
|
||||
impl WorkspaceConfigBackend {
|
||||
pub fn new(client: Arc<dyn WorkspaceClient>) -> Self {
|
||||
Self { client }
|
||||
Self {
|
||||
client,
|
||||
deadline: DEADLINE,
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
pub(super) fn with_deadline(mut self, deadline: Duration) -> Self {
|
||||
self.deadline = deadline;
|
||||
self
|
||||
}
|
||||
|
||||
fn base(&self) -> Result<String, WipOperationError> {
|
||||
@@ -47,6 +59,7 @@ impl WorkspaceConfigBackend {
|
||||
.workspace_id()
|
||||
.filter(|id| !id.is_empty())
|
||||
.ok_or_else(|| {
|
||||
diagnostic("authority", false).record(Stage::Request, Reason::Identity, None);
|
||||
failure(
|
||||
ProtocolErrorCode::PermissionDenied,
|
||||
"Workspace config authority unavailable",
|
||||
@@ -69,11 +82,13 @@ impl WorkspaceConfigBackend {
|
||||
&self,
|
||||
request: WorkspaceRequest,
|
||||
mutation: bool,
|
||||
diagnostic: &Diagnostic,
|
||||
) -> Result<T, WipOperationError> {
|
||||
let response = self
|
||||
.client
|
||||
.execute_with_timeout(request, DEADLINE)
|
||||
.execute_with_timeout(request, self.deadline)
|
||||
.map_err(|error| {
|
||||
diagnostic.transport(&error);
|
||||
if mutation && !matches!(error, WorkspaceClientError::InvalidPath(_)) {
|
||||
unknown()
|
||||
} else {
|
||||
@@ -84,6 +99,7 @@ impl WorkspaceConfigBackend {
|
||||
}
|
||||
})?;
|
||||
if response.body.len() > MAX_RESPONSE_BYTES {
|
||||
diagnostic.record(Stage::Response, Reason::ResponseSize, Some(response.status));
|
||||
return Err(if mutation {
|
||||
unknown()
|
||||
} else {
|
||||
@@ -96,8 +112,18 @@ impl WorkspaceConfigBackend {
|
||||
if !response.is_success() {
|
||||
if let Ok(error) = serde_json::from_str::<WorkspaceConfigApiError>(&response.body) {
|
||||
if error.classification == WorkspaceConfigFailureClassification::Unknown {
|
||||
diagnostic.record(
|
||||
Stage::Response,
|
||||
Reason::BackendUnknown,
|
||||
Some(response.status),
|
||||
);
|
||||
return Err(unknown());
|
||||
}
|
||||
diagnostic.record(
|
||||
Stage::Response,
|
||||
Reason::BackendRejected,
|
||||
Some(response.status),
|
||||
);
|
||||
// Do not reflect arbitrary transport/provider text into model output.
|
||||
let code = match error.status {
|
||||
401 | 403 | 404 => ProtocolErrorCode::PermissionDenied,
|
||||
@@ -111,6 +137,11 @@ impl WorkspaceConfigBackend {
|
||||
"Workspace config request rejected without commit",
|
||||
));
|
||||
}
|
||||
diagnostic.record(
|
||||
Stage::Response,
|
||||
Reason::UnrecognizedResponse,
|
||||
Some(response.status),
|
||||
);
|
||||
return Err(if mutation {
|
||||
unknown()
|
||||
} else {
|
||||
@@ -125,6 +156,7 @@ impl WorkspaceConfigBackend {
|
||||
});
|
||||
}
|
||||
serde_json::from_str(&response.body).map_err(|_| {
|
||||
diagnostic.record(Stage::Response, Reason::Decode, Some(response.status));
|
||||
if mutation {
|
||||
unknown()
|
||||
} else {
|
||||
@@ -138,10 +170,17 @@ impl WorkspaceConfigBackend {
|
||||
|
||||
/// GET checks the current grant even when the Worker has not attached yet.
|
||||
pub fn current(&self) -> Result<Option<WorkspaceConfigAttachment>, WipOperationError> {
|
||||
self.current_diagnosed(&diagnostic("current", false))
|
||||
}
|
||||
|
||||
fn current_diagnosed(
|
||||
&self,
|
||||
diagnostic: &Diagnostic,
|
||||
) -> Result<Option<WorkspaceConfigAttachment>, WipOperationError> {
|
||||
let result: Option<WorkspaceConfigAttachment> =
|
||||
self.request(WorkspaceRequest::get(self.base()?), false)?;
|
||||
self.request(WorkspaceRequest::get(self.base()?), false, diagnostic)?;
|
||||
if let Some(attachment) = &result {
|
||||
self.validate_attachment(attachment)?;
|
||||
self.validate_attachment(attachment, diagnostic)?;
|
||||
}
|
||||
Ok(result)
|
||||
}
|
||||
@@ -149,6 +188,7 @@ impl WorkspaceConfigBackend {
|
||||
fn validate_attachment(
|
||||
&self,
|
||||
attachment: &WorkspaceConfigAttachment,
|
||||
diagnostic: &Diagnostic,
|
||||
) -> Result<(), WipOperationError> {
|
||||
if Some(attachment.workspace_id.as_str()) != self.client.workspace_id()
|
||||
|| attachment.alias != ATTACHMENT_ALIAS
|
||||
@@ -160,6 +200,7 @@ impl WorkspaceConfigBackend {
|
||||
.bytes()
|
||||
.all(|b| b.is_ascii_alphanumeric() || matches!(b, b'-' | b'_'))
|
||||
{
|
||||
diagnostic.record(Stage::Response, Reason::Identity, None);
|
||||
return Err(failure(
|
||||
ProtocolErrorCode::PermissionDenied,
|
||||
"Invalid Workspace config attachment identity",
|
||||
@@ -173,14 +214,34 @@ impl WorkspaceConfigBackend {
|
||||
suffix: &str,
|
||||
request: &R,
|
||||
mutation: bool,
|
||||
) -> Result<T, WipOperationError> {
|
||||
let operation = match suffix {
|
||||
"/observe" => "observe",
|
||||
"/read" => "read",
|
||||
"/commit" => "commit",
|
||||
"" => "attach",
|
||||
_ => "post",
|
||||
};
|
||||
self.post_diagnosed(suffix, request, mutation, diagnostic(operation, mutation))
|
||||
.await
|
||||
}
|
||||
|
||||
pub(super) async fn post_diagnosed<T: DeserializeOwned + Send + 'static, R: Serialize>(
|
||||
&self,
|
||||
suffix: &str,
|
||||
request: &R,
|
||||
mutation: bool,
|
||||
diagnostic: Diagnostic,
|
||||
) -> Result<T, WipOperationError> {
|
||||
let body = serde_json::to_string(request).map_err(|_| {
|
||||
diagnostic.record(Stage::Request, Reason::Decode, None);
|
||||
failure(
|
||||
ProtocolErrorCode::InvalidArguments,
|
||||
"Invalid Workspace config request",
|
||||
)
|
||||
})?;
|
||||
if body.len() > MAX_RESPONSE_BYTES {
|
||||
diagnostic.record(Stage::Request, Reason::RequestSize, None);
|
||||
return Err(failure(
|
||||
ProtocolErrorCode::ResourceLimitExceeded,
|
||||
"Workspace config request exceeds limit",
|
||||
@@ -192,12 +253,16 @@ impl WorkspaceConfigBackend {
|
||||
body,
|
||||
);
|
||||
let this = self.clone();
|
||||
let request_diagnostic = diagnostic.clone();
|
||||
tokio::time::timeout(
|
||||
DEADLINE,
|
||||
tokio::task::spawn_blocking(move || this.request(request, mutation)),
|
||||
self.deadline,
|
||||
tokio::task::spawn_blocking(move || {
|
||||
this.request(request, mutation, &request_diagnostic)
|
||||
}),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
diagnostic.record(Stage::Completion, Reason::Deadline, None);
|
||||
if mutation {
|
||||
unknown()
|
||||
} else {
|
||||
@@ -208,6 +273,7 @@ impl WorkspaceConfigBackend {
|
||||
}
|
||||
})?
|
||||
.map_err(|_| {
|
||||
diagnostic.record(Stage::Completion, Reason::TaskJoin, None);
|
||||
if mutation {
|
||||
unknown()
|
||||
} else {
|
||||
@@ -223,18 +289,22 @@ impl WorkspaceConfigBackend {
|
||||
&self,
|
||||
) -> Result<Option<WorkspaceConfigAttachment>, WipOperationError> {
|
||||
let this = self.clone();
|
||||
let diagnostic = diagnostic("current", false);
|
||||
let request_diagnostic = diagnostic.clone();
|
||||
tokio::time::timeout(
|
||||
DEADLINE,
|
||||
tokio::task::spawn_blocking(move || this.current()),
|
||||
self.deadline,
|
||||
tokio::task::spawn_blocking(move || this.current_diagnosed(&request_diagnostic)),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
diagnostic.record(Stage::Completion, Reason::Deadline, None);
|
||||
failure(
|
||||
ProtocolErrorCode::ResourceLimitExceeded,
|
||||
"Workspace config request deadline exceeded",
|
||||
)
|
||||
})?
|
||||
.map_err(|_| {
|
||||
diagnostic.record(Stage::Completion, Reason::TaskJoin, None);
|
||||
failure(
|
||||
ProtocolErrorCode::Internal,
|
||||
"Workspace config transport unavailable",
|
||||
@@ -249,10 +319,14 @@ impl WorkspaceConfigAttachmentBackend for WorkspaceConfigBackend {
|
||||
self.client.is_available()
|
||||
}
|
||||
fn validate_access(&self, access: Option<ConfigAccess>) -> Result<(), ConfigAttachError> {
|
||||
let current = self.current().map_err(attach_error)?;
|
||||
let diagnostic = diagnostic("validate_access", false);
|
||||
let current = self
|
||||
.current_diagnosed(&diagnostic)
|
||||
.map_err(|error| attach_error(error, &diagnostic))?;
|
||||
if access == Some(ConfigAccess::ReadWrite)
|
||||
&& current.is_some_and(|a| a.access == WorkspaceConfigAccess::ReadOnly)
|
||||
{
|
||||
diagnostic.record(Stage::Attach, Reason::AccessCeiling, None);
|
||||
return Err(ConfigAttachError::Denied);
|
||||
}
|
||||
Ok(())
|
||||
@@ -271,10 +345,16 @@ impl WorkspaceConfigAttachmentBackend for WorkspaceConfigBackend {
|
||||
ConfigAccess::ReadWrite => WorkspaceConfigAccess::ReadWrite,
|
||||
}),
|
||||
};
|
||||
let attachment: WorkspaceConfigAttachment =
|
||||
self.post("", &request, true).await.map_err(attach_error)?;
|
||||
self.validate_attachment(&attachment)
|
||||
.map_err(|_| ConfigAttachError::OutcomeUnknown)?;
|
||||
let diagnostic = diagnostic("attach", true);
|
||||
let attachment: WorkspaceConfigAttachment = self
|
||||
.post_diagnosed("", &request, true, diagnostic.clone())
|
||||
.await
|
||||
.map_err(|error| attach_error(error, &diagnostic))?;
|
||||
self.validate_attachment(&attachment, &diagnostic)
|
||||
.map_err(|_| {
|
||||
diagnostic.record(Stage::Attach, Reason::Identity, None);
|
||||
ConfigAttachError::OutcomeUnknown
|
||||
})?;
|
||||
Ok(ConfigAttachment {
|
||||
connection_id: attachment.connection_id,
|
||||
access: match attachment.access {
|
||||
@@ -286,7 +366,8 @@ impl WorkspaceConfigAttachmentBackend for WorkspaceConfigBackend {
|
||||
}
|
||||
}
|
||||
|
||||
fn attach_error(error: WipOperationError) -> ConfigAttachError {
|
||||
fn attach_error(error: WipOperationError, diagnostic: &Diagnostic) -> ConfigAttachError {
|
||||
diagnostic.record(Stage::Attach, Reason::AttachConversion, None);
|
||||
match error {
|
||||
WipOperationError::OutcomeUnknown(_) => ConfigAttachError::OutcomeUnknown,
|
||||
WipOperationError::Protocol(error) => match error.code {
|
||||
@@ -298,6 +379,10 @@ fn attach_error(error: WipOperationError) -> ConfigAttachError {
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn diagnostic(operation: &'static str, mutation: bool) -> Diagnostic {
|
||||
Diagnostic::new(Adapter::WorkspaceConfig, operation, mutation, None)
|
||||
}
|
||||
|
||||
pub(super) fn failure(code: ProtocolErrorCode, message: &str) -> WipOperationError {
|
||||
WipOperationError::Protocol(ProtocolError {
|
||||
code,
|
||||
@@ -307,3 +392,7 @@ pub(super) fn failure(code: ProtocolErrorCode, message: &str) -> WipOperationErr
|
||||
pub(super) fn unknown() -> WipOperationError {
|
||||
WipOperationError::OutcomeUnknown("Workspace config outcome unknown; refresh before any subsequent operation; no automatic retry".into())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
#[path = "backend_tests.rs"]
|
||||
mod tests;
|
||||
|
||||
@@ -0,0 +1,580 @@
|
||||
//! Production WorkspaceConfigBackend with only WorkspaceClient substituted. The transport fixture
|
||||
//! applies effects before injected reply loss, retains its state for rereading, and gates blocking
|
||||
//! completion explicitly. These ordinary worker lib tests do not claim Backend transaction coverage.
|
||||
use super::*;
|
||||
use crate::feature::builtin::backend_diagnostic::tests::Capture;
|
||||
use crate::worker::WorkspaceResponse;
|
||||
use std::sync::{Condvar, Mutex};
|
||||
use tracing::instrument::WithSubscriber;
|
||||
|
||||
const UNSAFE: &str = "secret-token /private/config credential=unsafe";
|
||||
|
||||
#[derive(Clone, Copy, Debug)]
|
||||
enum Fault {
|
||||
Unavailable,
|
||||
InvalidPath,
|
||||
Communication,
|
||||
Decode,
|
||||
Size,
|
||||
Rejected(u16),
|
||||
Unknown,
|
||||
Unrecognized,
|
||||
Identity,
|
||||
Panic,
|
||||
Deadline,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct Effects {
|
||||
requests: usize,
|
||||
mutations: usize,
|
||||
saved: Option<String>,
|
||||
completed: bool,
|
||||
released: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct Client {
|
||||
fault: Fault,
|
||||
state: Mutex<Effects>,
|
||||
changed: Condvar,
|
||||
started: tokio::sync::Notify,
|
||||
}
|
||||
impl Client {
|
||||
fn new(fault: Fault) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
fault,
|
||||
state: Mutex::new(Effects::default()),
|
||||
changed: Condvar::new(),
|
||||
started: tokio::sync::Notify::new(),
|
||||
})
|
||||
}
|
||||
fn finish(&self) {
|
||||
self.state.lock().unwrap().completed = true;
|
||||
self.changed.notify_all();
|
||||
}
|
||||
fn release(&self) {
|
||||
self.state.lock().unwrap().released = true;
|
||||
self.changed.notify_all();
|
||||
}
|
||||
fn wait_completed(&self) {
|
||||
let (state, timeout) = self
|
||||
.changed
|
||||
.wait_timeout_while(self.state.lock().unwrap(), Duration::from_secs(5), |s| {
|
||||
!s.completed
|
||||
})
|
||||
.unwrap();
|
||||
assert!(
|
||||
!timeout.timed_out() && state.completed,
|
||||
"blocking request did not finish"
|
||||
);
|
||||
}
|
||||
}
|
||||
struct Release(Arc<Client>);
|
||||
impl Drop for Release {
|
||||
fn drop(&mut self) {
|
||||
self.0.release();
|
||||
}
|
||||
}
|
||||
|
||||
fn attachment() -> WorkspaceConfigAttachment {
|
||||
WorkspaceConfigAttachment {
|
||||
workspace_id: "workspace".into(),
|
||||
connection_id: "connection-1".into(),
|
||||
alias: ATTACHMENT_ALIAS.into(),
|
||||
content_path: CONTENT_ROOT.into(),
|
||||
working_directory_id: "grant".into(),
|
||||
access: WorkspaceConfigAccess::ReadWrite,
|
||||
name: UNSAFE.into(),
|
||||
purpose: UNSAFE.into(),
|
||||
already_attached: false,
|
||||
}
|
||||
}
|
||||
|
||||
impl WorkspaceClient for Client {
|
||||
fn workspace_id(&self) -> Option<&str> {
|
||||
Some("workspace")
|
||||
}
|
||||
fn kind(&self) -> &str {
|
||||
"diagnostic-test-boundary"
|
||||
}
|
||||
fn is_available(&self) -> bool {
|
||||
true
|
||||
}
|
||||
fn execute(
|
||||
&self,
|
||||
request: WorkspaceRequest,
|
||||
) -> Result<WorkspaceResponse, WorkspaceClientError> {
|
||||
{
|
||||
let mut state = self.state.lock().unwrap();
|
||||
state.requests += 1;
|
||||
if request.method == WorkspaceRequestMethod::Post
|
||||
&& !request.path.ends_with("/read")
|
||||
&& !matches!(
|
||||
self.fault,
|
||||
Fault::Unavailable | Fault::InvalidPath | Fault::Rejected(_)
|
||||
)
|
||||
{
|
||||
state.mutations += 1;
|
||||
state.saved = request.body.clone();
|
||||
}
|
||||
self.started.notify_one();
|
||||
if matches!(self.fault, Fault::Deadline) {
|
||||
let (_state, timeout) = self
|
||||
.changed
|
||||
.wait_timeout_while(state, Duration::from_secs(5), |s| !s.released)
|
||||
.unwrap();
|
||||
assert!(!timeout.timed_out(), "test must release blocked transport");
|
||||
}
|
||||
}
|
||||
self.finish();
|
||||
match self.fault {
|
||||
Fault::Unavailable => Err(WorkspaceClientError::Unavailable(UNSAFE.into())),
|
||||
Fault::InvalidPath => Err(WorkspaceClientError::InvalidPath(UNSAFE.into())),
|
||||
Fault::Communication => Err(WorkspaceClientError::Request(UNSAFE.into())),
|
||||
Fault::Panic => panic!("injected blocking task failure"),
|
||||
Fault::Decode => Ok(WorkspaceResponse {
|
||||
status: 200,
|
||||
body: UNSAFE.into(),
|
||||
}),
|
||||
Fault::Size => Ok(WorkspaceResponse {
|
||||
status: 200,
|
||||
body: UNSAFE.repeat(MAX_RESPONSE_BYTES / UNSAFE.len() + 1),
|
||||
}),
|
||||
Fault::Rejected(_) | Fault::Unknown => {
|
||||
let status = match self.fault {
|
||||
Fault::Rejected(status) => status,
|
||||
_ => 503,
|
||||
};
|
||||
Ok(WorkspaceResponse {
|
||||
status,
|
||||
body: serde_json::to_string(&WorkspaceConfigApiError {
|
||||
status,
|
||||
code: UNSAFE.into(),
|
||||
message: UNSAFE.into(),
|
||||
classification: if matches!(self.fault, Fault::Unknown) {
|
||||
WorkspaceConfigFailureClassification::Unknown
|
||||
} else {
|
||||
WorkspaceConfigFailureClassification::NotCommitted
|
||||
},
|
||||
})
|
||||
.unwrap(),
|
||||
})
|
||||
}
|
||||
Fault::Unrecognized => Ok(WorkspaceResponse {
|
||||
status: 503,
|
||||
body: UNSAFE.into(),
|
||||
}),
|
||||
Fault::Identity | Fault::Deadline => {
|
||||
let mut value = attachment();
|
||||
if matches!(self.fault, Fault::Identity) {
|
||||
value.workspace_id = UNSAFE.into();
|
||||
}
|
||||
Ok(WorkspaceResponse {
|
||||
status: 200,
|
||||
body: serde_json::to_string(&value).unwrap(),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn assert_safe(capture: &Capture, reason: Reason, stage: Stage, public: &str) {
|
||||
let text = capture.text();
|
||||
assert!(
|
||||
text.contains(&format!("reason={reason:?}")),
|
||||
"missing reason: {text}"
|
||||
);
|
||||
assert!(
|
||||
text.contains(&format!("stage={stage:?}")),
|
||||
"missing stage: {text}"
|
||||
);
|
||||
assert!(text.contains("adapter=WorkspaceConfig"), "{text}");
|
||||
assert!(text.contains("correlation="), "{text}");
|
||||
for secret in [
|
||||
UNSAFE,
|
||||
"secret-token",
|
||||
"/private/config",
|
||||
"credential=unsafe",
|
||||
] {
|
||||
assert!(
|
||||
!text.contains(secret),
|
||||
"diagnostics leaked unsafe boundary data"
|
||||
);
|
||||
assert!(
|
||||
!public.contains(secret),
|
||||
"public result leaked unsafe boundary data"
|
||||
);
|
||||
}
|
||||
}
|
||||
fn public_error(
|
||||
result: Result<serde_json::Value, WipOperationError>,
|
||||
code: Option<ProtocolErrorCode>,
|
||||
) -> String {
|
||||
match (result, code) {
|
||||
(Err(WipOperationError::OutcomeUnknown(message)), None) => {
|
||||
assert!(message.contains("no automatic retry"));
|
||||
message
|
||||
}
|
||||
(Err(WipOperationError::Protocol(error)), Some(expected)) => {
|
||||
assert_eq!(error.code, expected);
|
||||
error.message
|
||||
}
|
||||
_ => panic!("unexpected public error classification"),
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn production_reply_failures_retain_reasons_without_retrying_mutations() {
|
||||
let cases = [
|
||||
(
|
||||
Fault::Unavailable,
|
||||
Reason::TransportUnavailable,
|
||||
Stage::Request,
|
||||
ProtocolErrorCode::Internal,
|
||||
false,
|
||||
),
|
||||
(
|
||||
Fault::InvalidPath,
|
||||
Reason::InvalidPath,
|
||||
Stage::Request,
|
||||
ProtocolErrorCode::Internal,
|
||||
false,
|
||||
),
|
||||
(
|
||||
Fault::Communication,
|
||||
Reason::Communication,
|
||||
Stage::Request,
|
||||
ProtocolErrorCode::Internal,
|
||||
true,
|
||||
),
|
||||
(
|
||||
Fault::Decode,
|
||||
Reason::Decode,
|
||||
Stage::Response,
|
||||
ProtocolErrorCode::Internal,
|
||||
true,
|
||||
),
|
||||
(
|
||||
Fault::Size,
|
||||
Reason::ResponseSize,
|
||||
Stage::Response,
|
||||
ProtocolErrorCode::ResourceLimitExceeded,
|
||||
true,
|
||||
),
|
||||
(
|
||||
Fault::Unknown,
|
||||
Reason::BackendUnknown,
|
||||
Stage::Response,
|
||||
ProtocolErrorCode::Internal,
|
||||
true,
|
||||
),
|
||||
(
|
||||
Fault::Unrecognized,
|
||||
Reason::UnrecognizedResponse,
|
||||
Stage::Response,
|
||||
ProtocolErrorCode::Internal,
|
||||
true,
|
||||
),
|
||||
(
|
||||
Fault::Panic,
|
||||
Reason::TaskJoin,
|
||||
Stage::Completion,
|
||||
ProtocolErrorCode::Internal,
|
||||
true,
|
||||
),
|
||||
];
|
||||
for (fault, reason, stage, read_code, applies) in cases {
|
||||
for mutation in [false, true] {
|
||||
let client = Client::new(fault);
|
||||
let backend = WorkspaceConfigBackend::new(client.clone());
|
||||
let capture = Capture::default();
|
||||
let result = backend
|
||||
.post::<serde_json::Value, _>(
|
||||
if mutation { "/commit" } else { "/read" },
|
||||
&serde_json::json!({"content": UNSAFE}),
|
||||
mutation,
|
||||
)
|
||||
.with_subscriber(capture.dispatch())
|
||||
.await;
|
||||
let code = if (mutation && !matches!(fault, Fault::InvalidPath))
|
||||
|| matches!(fault, Fault::Unknown)
|
||||
{
|
||||
None
|
||||
} else {
|
||||
Some(read_code)
|
||||
};
|
||||
let public = public_error(result, code);
|
||||
assert_safe(&capture, reason, stage, &public);
|
||||
let state = client.state.lock().unwrap();
|
||||
assert_eq!(state.requests, 1);
|
||||
assert_eq!(state.mutations, usize::from(mutation && applies));
|
||||
assert_eq!(state.saved.is_some(), mutation && applies);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn production_typed_rejections_preserve_public_codes_and_never_apply() {
|
||||
for (status, code) in [
|
||||
(403, ProtocolErrorCode::PermissionDenied),
|
||||
(409, ProtocolErrorCode::ValidatorMismatch),
|
||||
(413, ProtocolErrorCode::ResourceLimitExceeded),
|
||||
(422, ProtocolErrorCode::InvalidArguments),
|
||||
(500, ProtocolErrorCode::Internal),
|
||||
] {
|
||||
for mutation in [false, true] {
|
||||
let client = Client::new(Fault::Rejected(status));
|
||||
let backend = WorkspaceConfigBackend::new(client.clone());
|
||||
let capture = Capture::default();
|
||||
let public = public_error(
|
||||
backend
|
||||
.post::<serde_json::Value, _>(
|
||||
if mutation { "/commit" } else { "/read" },
|
||||
&(),
|
||||
mutation,
|
||||
)
|
||||
.with_subscriber(capture.dispatch())
|
||||
.await,
|
||||
Some(code),
|
||||
);
|
||||
assert_safe(&capture, Reason::BackendRejected, Stage::Response, &public);
|
||||
assert!(capture.text().contains(&format!("status={status}")));
|
||||
let state = client.state.lock().unwrap();
|
||||
assert_eq!(state.requests, 1);
|
||||
assert_eq!(state.mutations, 0);
|
||||
assert!(state.saved.is_none());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn production_deadline_is_unknown_after_effect_and_blocking_completion_is_not_retried() {
|
||||
for mutation in [false, true] {
|
||||
let client = Client::new(Fault::Deadline);
|
||||
let release = Release(client.clone());
|
||||
let backend =
|
||||
WorkspaceConfigBackend::new(client.clone()).with_deadline(Duration::from_secs(1));
|
||||
let capture = Capture::default();
|
||||
let body = serde_json::json!({"content": UNSAFE});
|
||||
let request = backend
|
||||
.post::<serde_json::Value, _>(
|
||||
if mutation { "/commit" } else { "/read" },
|
||||
&body,
|
||||
mutation,
|
||||
)
|
||||
.with_subscriber(capture.dispatch());
|
||||
tokio::pin!(request);
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = client.started.notified() => {},
|
||||
_ = &mut request => panic!("request must start and apply its effect before timeout is observed"),
|
||||
}
|
||||
{
|
||||
let state = client.state.lock().unwrap();
|
||||
assert_eq!(state.mutations, usize::from(mutation));
|
||||
assert_eq!(state.saved.is_some(), mutation);
|
||||
assert!(!state.completed && !state.released);
|
||||
}
|
||||
let result = request.await;
|
||||
let public = public_error(
|
||||
result,
|
||||
if mutation {
|
||||
None
|
||||
} else {
|
||||
Some(ProtocolErrorCode::ResourceLimitExceeded)
|
||||
},
|
||||
);
|
||||
assert_safe(&capture, Reason::Deadline, Stage::Completion, &public);
|
||||
drop(release);
|
||||
let completed = client.clone();
|
||||
tokio::task::spawn_blocking(move || completed.wait_completed())
|
||||
.await
|
||||
.unwrap();
|
||||
let state = client.state.lock().unwrap();
|
||||
assert_eq!(state.requests, 1);
|
||||
assert_eq!(state.mutations, usize::from(mutation));
|
||||
assert_eq!(state.saved.is_some(), mutation);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn production_current_identity_and_task_loss_remain_safe() {
|
||||
for (fault, reason, stage, code) in [
|
||||
(
|
||||
Fault::Identity,
|
||||
Reason::Identity,
|
||||
Stage::Response,
|
||||
ProtocolErrorCode::PermissionDenied,
|
||||
),
|
||||
(
|
||||
Fault::Panic,
|
||||
Reason::TaskJoin,
|
||||
Stage::Completion,
|
||||
ProtocolErrorCode::Internal,
|
||||
),
|
||||
(
|
||||
Fault::Decode,
|
||||
Reason::Decode,
|
||||
Stage::Response,
|
||||
ProtocolErrorCode::Internal,
|
||||
),
|
||||
] {
|
||||
let client = Client::new(fault);
|
||||
let backend = WorkspaceConfigBackend::new(client.clone());
|
||||
let capture = Capture::default();
|
||||
let result = backend
|
||||
.current_async()
|
||||
.with_subscriber(capture.dispatch())
|
||||
.await;
|
||||
let message = match result {
|
||||
Err(WipOperationError::Protocol(error)) => {
|
||||
assert_eq!(error.code, code);
|
||||
error.message
|
||||
}
|
||||
_ => panic!("unexpected current classification"),
|
||||
};
|
||||
assert_safe(&capture, reason, stage, &message);
|
||||
assert_eq!(client.state.lock().unwrap().mutations, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn production_attach_conversion_retains_original_reason_and_effect_count() {
|
||||
for (fault, reason, expected, mutations) in [
|
||||
(
|
||||
Fault::Communication,
|
||||
Reason::Communication,
|
||||
ConfigAttachError::OutcomeUnknown,
|
||||
1,
|
||||
),
|
||||
(
|
||||
Fault::Identity,
|
||||
Reason::Identity,
|
||||
ConfigAttachError::OutcomeUnknown,
|
||||
1,
|
||||
),
|
||||
(
|
||||
Fault::Rejected(403),
|
||||
Reason::BackendRejected,
|
||||
ConfigAttachError::Denied,
|
||||
0,
|
||||
),
|
||||
(
|
||||
Fault::Rejected(422),
|
||||
Reason::BackendRejected,
|
||||
ConfigAttachError::InvalidRequest,
|
||||
0,
|
||||
),
|
||||
(
|
||||
Fault::Rejected(500),
|
||||
Reason::BackendRejected,
|
||||
ConfigAttachError::Unavailable,
|
||||
0,
|
||||
),
|
||||
] {
|
||||
let client = Client::new(fault);
|
||||
let backend = WorkspaceConfigBackend::new(client.clone());
|
||||
let capture = Capture::default();
|
||||
assert_eq!(
|
||||
backend
|
||||
.attach("host-key-not-logged", None)
|
||||
.with_subscriber(capture.dispatch())
|
||||
.await,
|
||||
Err(expected)
|
||||
);
|
||||
assert_safe(&capture, reason, Stage::Attach, expected.message());
|
||||
let text = capture.text();
|
||||
if !matches!(fault, Fault::Identity) {
|
||||
assert!(text.contains("reason=AttachConversion"), "{text}");
|
||||
}
|
||||
assert!(!text.contains("host-key-not-logged"));
|
||||
let correlations: std::collections::BTreeSet<_> = text
|
||||
.lines()
|
||||
.filter_map(|line| line.split("correlation=").nth(1))
|
||||
.filter_map(|tail| tail.split(';').next())
|
||||
.collect();
|
||||
assert_eq!(
|
||||
correlations.len(),
|
||||
1,
|
||||
"request and attach conversion must correlate"
|
||||
);
|
||||
let state = client.state.lock().unwrap();
|
||||
assert_eq!(state.requests, 1);
|
||||
assert_eq!(state.mutations, mutations);
|
||||
assert_eq!(state.saved.is_some(), mutations == 1);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn production_oversized_request_is_rejected_before_dispatch() {
|
||||
let client = Client::new(Fault::Communication);
|
||||
let backend = WorkspaceConfigBackend::new(client.clone());
|
||||
let capture = Capture::default();
|
||||
let public = public_error(
|
||||
backend
|
||||
.post::<serde_json::Value, _>("/commit", &"x".repeat(MAX_RESPONSE_BYTES), true)
|
||||
.with_subscriber(capture.dispatch())
|
||||
.await,
|
||||
Some(ProtocolErrorCode::ResourceLimitExceeded),
|
||||
);
|
||||
assert_safe(&capture, Reason::RequestSize, Stage::Request, &public);
|
||||
let state = client.state.lock().unwrap();
|
||||
assert_eq!(state.requests, 0);
|
||||
assert_eq!(state.mutations, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn production_current_deadline_remains_bounded_and_has_no_mutation() {
|
||||
let client = Client::new(Fault::Deadline);
|
||||
let release = Release(client.clone());
|
||||
let backend = WorkspaceConfigBackend::new(client.clone()).with_deadline(Duration::from_secs(1));
|
||||
let capture = Capture::default();
|
||||
let request = backend.current_async().with_subscriber(capture.dispatch());
|
||||
tokio::pin!(request);
|
||||
tokio::select! {
|
||||
biased;
|
||||
_ = client.started.notified() => {},
|
||||
_ = &mut request => panic!("current request must enter the gate before timeout is observed"),
|
||||
}
|
||||
assert_eq!(client.state.lock().unwrap().mutations, 0);
|
||||
let error = request.await;
|
||||
let message = match error {
|
||||
Err(WipOperationError::Protocol(error)) => {
|
||||
assert_eq!(error.code, ProtocolErrorCode::ResourceLimitExceeded);
|
||||
error.message
|
||||
}
|
||||
_ => panic!("current deadline must remain a public resource-limit error"),
|
||||
};
|
||||
assert_safe(&capture, Reason::Deadline, Stage::Completion, &message);
|
||||
drop(release);
|
||||
let completed = client.clone();
|
||||
tokio::task::spawn_blocking(move || completed.wait_completed())
|
||||
.await
|
||||
.unwrap();
|
||||
let state = client.state.lock().unwrap();
|
||||
assert_eq!(state.requests, 1);
|
||||
assert_eq!(state.mutations, 0);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn production_request_encoding_failure_is_safe_and_never_dispatched() {
|
||||
struct Unencodable;
|
||||
impl Serialize for Unencodable {
|
||||
fn serialize<S: serde::Serializer>(&self, _: S) -> Result<S::Ok, S::Error> {
|
||||
Err(serde::ser::Error::custom(UNSAFE))
|
||||
}
|
||||
}
|
||||
let client = Client::new(Fault::Communication);
|
||||
let backend = WorkspaceConfigBackend::new(client.clone());
|
||||
let capture = Capture::default();
|
||||
let result = backend
|
||||
.post::<serde_json::Value, _>("/commit", &Unencodable, true)
|
||||
.with_subscriber(capture.dispatch())
|
||||
.await;
|
||||
let message = public_error(result, Some(ProtocolErrorCode::InvalidArguments));
|
||||
assert_safe(&capture, Reason::Decode, Stage::Request, &message);
|
||||
let state = client.state.lock().unwrap();
|
||||
assert_eq!(state.requests, 0);
|
||||
assert_eq!(state.mutations, 0);
|
||||
}
|
||||
@@ -36,9 +36,11 @@ use wip_protocol::{
|
||||
};
|
||||
|
||||
use super::backend::{
|
||||
MAX_DEPTH, MAX_NODES, MAX_PATH_BYTES, MAX_TEXT_BYTES, WorkspaceConfigBackend, failure, unknown,
|
||||
MAX_DEPTH, MAX_NODES, MAX_PATH_BYTES, MAX_TEXT_BYTES, WorkspaceConfigBackend, diagnostic,
|
||||
failure, unknown,
|
||||
};
|
||||
use super::{CONTENT_ROOT, WorkspaceConfigFeature};
|
||||
use crate::feature::builtin::backend_diagnostic::{Diagnostic, Reason, Stage};
|
||||
use crate::wip::{
|
||||
WipCallContext, WipOperationError, WipOperationHandler, WipOperationOutput, WipProjection,
|
||||
WipProjectionKind, WipSubtreeProvider, contextual_reference, json_to_wip,
|
||||
@@ -138,6 +140,23 @@ async fn observe(
|
||||
connection: &str,
|
||||
paths: Vec<String>,
|
||||
depth: u32,
|
||||
) -> Result<WorkspaceConfigObserveResponse, WipOperationError> {
|
||||
observe_diagnosed(
|
||||
backend,
|
||||
connection,
|
||||
paths,
|
||||
depth,
|
||||
diagnostic("observe", false),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn observe_diagnosed(
|
||||
backend: &WorkspaceConfigBackend,
|
||||
connection: &str,
|
||||
paths: Vec<String>,
|
||||
depth: u32,
|
||||
diagnostic: Diagnostic,
|
||||
) -> Result<WorkspaceConfigObserveResponse, WipOperationError> {
|
||||
if paths.len() > MAX_NODES || depth > MAX_DEPTH {
|
||||
return Err(invalid());
|
||||
@@ -148,7 +167,7 @@ async fn observe(
|
||||
}
|
||||
}
|
||||
let result: WorkspaceConfigObserveResponse = backend
|
||||
.post(
|
||||
.post_diagnosed(
|
||||
"/observe",
|
||||
&WorkspaceConfigObserveRequest {
|
||||
connection_id: connection.into(),
|
||||
@@ -156,6 +175,7 @@ async fn observe(
|
||||
depth,
|
||||
},
|
||||
false,
|
||||
diagnostic.clone(),
|
||||
)
|
||||
.await?;
|
||||
if result.connection_id != connection
|
||||
@@ -164,24 +184,37 @@ async fn observe(
|
||||
|| result.validator.len() > 4096
|
||||
|| result.digest.is_empty()
|
||||
{
|
||||
diagnostic.record(
|
||||
Stage::Response,
|
||||
if result.connection_id != connection {
|
||||
Reason::Identity
|
||||
} else {
|
||||
Reason::InvalidMetadata
|
||||
},
|
||||
None,
|
||||
);
|
||||
return Err(failure(
|
||||
ProtocolErrorCode::Internal,
|
||||
"Invalid Workspace config metadata",
|
||||
));
|
||||
}
|
||||
if result.entrypoints.is_empty() || result.entrypoints.len() > MAX_NODES {
|
||||
diagnostic.record(Stage::Response, Reason::InvalidMetadata, None);
|
||||
return Err(failure(
|
||||
ProtocolErrorCode::Internal,
|
||||
"Invalid Workspace config entrypoints",
|
||||
));
|
||||
}
|
||||
for path in &result.entrypoints {
|
||||
validate_source(path)?;
|
||||
validate_source(path)
|
||||
.inspect_err(|_| diagnostic.record(Stage::Response, Reason::InvalidMetadata, None))?;
|
||||
}
|
||||
let mut seen = BTreeSet::new();
|
||||
for node in &result.nodes {
|
||||
if !node.path.is_empty() {
|
||||
validate_source(&node.path)?;
|
||||
validate_source(&node.path).inspect_err(|_| {
|
||||
diagnostic.record(Stage::Response, Reason::InvalidMetadata, None)
|
||||
})?;
|
||||
}
|
||||
if !seen.insert(&node.path)
|
||||
|| node.validator.is_empty()
|
||||
@@ -198,6 +231,7 @@ async fn observe(
|
||||
suffix.is_some_and(|s| s.split('/').count() <= depth as usize)
|
||||
})
|
||||
{
|
||||
diagnostic.record(Stage::Response, Reason::InvalidMetadata, None);
|
||||
return Err(failure(
|
||||
ProtocolErrorCode::Internal,
|
||||
"Invalid Workspace config metadata scope",
|
||||
@@ -490,9 +524,10 @@ fn content_type(value: Option<&Value>) -> Result<ConfigContentType, WipOperation
|
||||
|
||||
impl ConfigHandler {
|
||||
async fn read(&self) -> Result<WorkspaceConfigReadResponse, WipOperationError> {
|
||||
let diagnostic = diagnostic("read", false);
|
||||
let response: WorkspaceConfigReadResponse = self
|
||||
.backend
|
||||
.post(
|
||||
.post_diagnosed(
|
||||
"/read",
|
||||
&WorkspaceConfigReadRequest {
|
||||
connection_id: self.attachment.connection_id.clone(),
|
||||
@@ -500,6 +535,7 @@ impl ConfigHandler {
|
||||
validator: self.node.validator.clone(),
|
||||
},
|
||||
false,
|
||||
diagnostic.clone(),
|
||||
)
|
||||
.await?;
|
||||
if response.path != self.node.path
|
||||
@@ -507,12 +543,14 @@ impl ConfigHandler {
|
||||
|| Some(&response.digest) != self.node.digest.as_ref()
|
||||
|| Some(&response.content_type) != self.node.content_type.as_ref()
|
||||
{
|
||||
diagnostic.record(Stage::Response, Reason::Identity, None);
|
||||
return Err(failure(
|
||||
ProtocolErrorCode::Internal,
|
||||
"Invalid Workspace config read response",
|
||||
));
|
||||
}
|
||||
if response.content.len() > MAX_TEXT_BYTES {
|
||||
diagnostic.record(Stage::Response, Reason::ResponseSize, None);
|
||||
return Err(failure(
|
||||
ProtocolErrorCode::ResourceLimitExceeded,
|
||||
"Workspace config source exceeds text limit",
|
||||
@@ -630,12 +668,17 @@ impl WipOperationHandler for ConfigHandler {
|
||||
context: WipCallContext,
|
||||
) -> Result<WipOperationOutput, WipOperationError> {
|
||||
let dispatched = std::sync::atomic::AtomicBool::new(false);
|
||||
let diagnostic = diagnostic(
|
||||
"operation",
|
||||
!matches!(operation, "read" | "read_lines" | "search_text"),
|
||||
);
|
||||
tokio::time::timeout(
|
||||
super::DEADLINE,
|
||||
self.execute(operation, arguments, context, &dispatched),
|
||||
self.execute(operation, arguments, context, &dispatched, &diagnostic),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| {
|
||||
diagnostic.record(Stage::Completion, Reason::Deadline, None);
|
||||
if dispatched.load(std::sync::atomic::Ordering::Acquire) {
|
||||
unknown()
|
||||
} else {
|
||||
@@ -655,6 +698,7 @@ impl ConfigHandler {
|
||||
arguments: &BTreeMap<String, Value>,
|
||||
context: WipCallContext,
|
||||
dispatched: &std::sync::atomic::AtomicBool,
|
||||
operation_diagnostic: &Diagnostic,
|
||||
) -> Result<WipOperationOutput, WipOperationError> {
|
||||
if !self.operations.iter().any(|name| name == operation) {
|
||||
return Err(failure(
|
||||
@@ -798,9 +842,10 @@ impl ConfigHandler {
|
||||
};
|
||||
let changes = self.canonical_changes(changes).await?;
|
||||
dispatched.store(true, std::sync::atomic::Ordering::Release);
|
||||
let diagnostic = operation_diagnostic.clone();
|
||||
let result: WorkspaceConfigCommitResponse = self
|
||||
.backend
|
||||
.post(
|
||||
.post_diagnosed(
|
||||
"/commit",
|
||||
&WorkspaceConfigCommitRequest {
|
||||
connection_id: self.attachment.connection_id.clone(),
|
||||
@@ -812,28 +857,37 @@ impl ConfigHandler {
|
||||
},
|
||||
},
|
||||
true,
|
||||
diagnostic.clone(),
|
||||
)
|
||||
.await?;
|
||||
// Commit response supplies ROOT state. Obtain the exact post-commit PATH
|
||||
// validator, never substitute the root validator for a file validator.
|
||||
let updated = observe(
|
||||
let updated = observe_diagnosed(
|
||||
&self.backend,
|
||||
&self.attachment.connection_id,
|
||||
vec![self.node.path.clone()],
|
||||
0,
|
||||
diagnostic.at_stage(Stage::Reconcile),
|
||||
)
|
||||
.await
|
||||
.map_err(|_| unknown())?;
|
||||
if updated.digest != result.digest || updated.validator != result.validator {
|
||||
diagnostic.record(Stage::Reconcile, Reason::Identity, None);
|
||||
return Err(unknown());
|
||||
}
|
||||
let node = updated
|
||||
.nodes
|
||||
.iter()
|
||||
.find(|n| n.path == self.node.path)
|
||||
.ok_or_else(unknown)?;
|
||||
.ok_or_else(|| {
|
||||
diagnostic.record(Stage::Reconcile, Reason::InvalidMetadata, None);
|
||||
unknown()
|
||||
})?;
|
||||
let value = json_to_wip(&json!({"digest": result.digest,
|
||||
"activation": "Saved to canonical Workspace configuration; existing Worker manifests are not rewritten"})).map_err(|_| unknown())?;
|
||||
"activation": "Saved to canonical Workspace configuration; existing Worker manifests are not rewritten"})).map_err(|_| {
|
||||
diagnostic.record(Stage::Completion, Reason::Decode, None);
|
||||
unknown()
|
||||
})?;
|
||||
Ok(WipOperationOutput::native_with_validator(
|
||||
value,
|
||||
object_validator(
|
||||
|
||||
@@ -1,8 +1,10 @@
|
||||
//! Worker-side E2E across real Feature invocation, WIP Client/Host and typed
|
||||
//! Workspace transport. Canonical persistence/projection tests live in Backend;
|
||||
//! this deterministic router models its published auth/CAS/response contract.
|
||||
//! Worker-side integration across real Feature invocation, WIP Client/Host and the production
|
||||
//! WorkspaceConfigBackend. Only WorkspaceClient transport is substituted by a deterministic router
|
||||
//! retaining applied effects and modeling the published auth/CAS/response contract. These ordinary
|
||||
//! worker lib tests do not replace canonical persistence/projection tests owned by Backend.
|
||||
use super::*;
|
||||
use crate::feature::FeatureRegistryBuilder;
|
||||
use crate::feature::builtin::backend_diagnostic::tests::Capture;
|
||||
use crate::hook::HookRegistryBuilder;
|
||||
use crate::wip::{WipMountRegistry, WipRuntime};
|
||||
use crate::worker::{
|
||||
@@ -16,6 +18,7 @@ use serde_json::{Value as Json, json};
|
||||
use server_api::*;
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tracing::instrument::WithSubscriber;
|
||||
|
||||
#[derive(Debug)]
|
||||
struct State {
|
||||
@@ -26,6 +29,10 @@ struct State {
|
||||
entries: BTreeMap<String, (ConfigContentType, String)>,
|
||||
commits: Vec<WorkspaceConfigCommitRequest>,
|
||||
requests: Vec<WorkspaceRequest>,
|
||||
applied_commits: usize,
|
||||
malformed_read_reply: Option<&'static str>,
|
||||
malformed_observe_reply: Option<&'static str>,
|
||||
lose_postcommit_observe_reply: bool,
|
||||
fail_commit: bool,
|
||||
fail_current: bool,
|
||||
lose_commit_reply: bool,
|
||||
@@ -58,6 +65,10 @@ impl Router {
|
||||
]),
|
||||
commits: Vec::new(),
|
||||
requests: Vec::new(),
|
||||
applied_commits: 0,
|
||||
malformed_read_reply: None,
|
||||
malformed_observe_reply: None,
|
||||
lose_postcommit_observe_reply: false,
|
||||
fail_commit: false,
|
||||
fail_current: false,
|
||||
lose_commit_reply: false,
|
||||
@@ -222,6 +233,11 @@ impl WorkspaceClient for Router {
|
||||
}
|
||||
match suffix {
|
||||
"/observe" => {
|
||||
if state.lose_postcommit_observe_reply && state.applied_commits > 0 {
|
||||
return Err(WorkspaceClientError::Request(
|
||||
"secret-token /private/host/config".into(),
|
||||
));
|
||||
}
|
||||
let request: WorkspaceConfigObserveRequest = serde_json::from_value(input).unwrap();
|
||||
assert!(request.depth <= 8 && request.paths.len() <= 256);
|
||||
let mut kinds =
|
||||
@@ -268,13 +284,23 @@ impl WorkspaceClient for Router {
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
Ok(response(&WorkspaceConfigObserveResponse {
|
||||
let mut result = WorkspaceConfigObserveResponse {
|
||||
connection_id: state.active.as_ref().unwrap().connection_id.clone(),
|
||||
validator: state.validator(""),
|
||||
digest: state.tree_digest(),
|
||||
entrypoints: vec!["main.dcdl".into()],
|
||||
nodes,
|
||||
}))
|
||||
};
|
||||
match state.malformed_observe_reply {
|
||||
Some("connection") => {
|
||||
result.connection_id = "secret-token /private/host/config".into()
|
||||
}
|
||||
Some("entrypoints") => result.entrypoints.clear(),
|
||||
Some("scope") => result.nodes[0].path = "outside/path.dcdl".into(),
|
||||
None => {}
|
||||
Some(_) => unreachable!(),
|
||||
}
|
||||
Ok(response(&result))
|
||||
}
|
||||
"/read" => {
|
||||
let request: WorkspaceConfigReadRequest = serde_json::from_value(input).unwrap();
|
||||
@@ -290,13 +316,20 @@ impl WorkspaceClient for Router {
|
||||
WorkspaceConfigFailureClassification::NotCommitted,
|
||||
));
|
||||
};
|
||||
Ok(response(&WorkspaceConfigReadResponse {
|
||||
let mut result = WorkspaceConfigReadResponse {
|
||||
path: request.path.clone(),
|
||||
content: content.clone(),
|
||||
content_type: *content_type,
|
||||
digest: digest(content),
|
||||
validator: state.validator(&request.path),
|
||||
}))
|
||||
};
|
||||
match state.malformed_read_reply {
|
||||
Some("identity") => result.path = "secret-token /private/host/config".into(),
|
||||
Some("size") => result.content = "x".repeat(256 * 1024 + 1),
|
||||
None => {}
|
||||
Some(_) => unreachable!(),
|
||||
}
|
||||
Ok(response(&result))
|
||||
}
|
||||
"/commit" => {
|
||||
let request: WorkspaceConfigCommitRequest = serde_json::from_value(input).unwrap();
|
||||
@@ -403,6 +436,7 @@ impl WorkspaceClient for Router {
|
||||
));
|
||||
}
|
||||
state.entries = entries;
|
||||
state.applied_commits += 1;
|
||||
if state.lose_commit_reply {
|
||||
state.lose_commit_reply = false;
|
||||
return Err(WorkspaceClientError::Request(
|
||||
@@ -797,14 +831,20 @@ async fn post_commit_observation_mismatch_reports_unknown_without_replaying_writ
|
||||
let path = "/workspace-config/main.dcdl";
|
||||
observe_interface(&runtime, path, true).await;
|
||||
client.state.lock().unwrap().mismatched_commit_reply = Some(field);
|
||||
let capture = Capture::default();
|
||||
let error = call(&runtime, path, "write", json!({"content":"saved"}))
|
||||
.with_subscriber(capture.dispatch())
|
||||
.await
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(error.contains("unknown"), "{field}: {error}");
|
||||
assert!(capture.text().contains("stage=Reconcile"));
|
||||
assert!(capture.text().contains("reason=Identity"));
|
||||
assert!(!capture.text().contains("saved"));
|
||||
let state = client.state.lock().unwrap();
|
||||
assert_eq!(state.commits.len(), 1);
|
||||
assert_eq!(state.entries["main.dcdl"].1, "saved");
|
||||
assert_eq!(state.applied_commits, 1);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1155,3 +1195,110 @@ async fn workspace_config_backend_validators_reject_path_and_connection_transpla
|
||||
assert!(result.content.unwrap().contains("model = updated"));
|
||||
assert_eq!(client.state.lock().unwrap().commits.len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn production_read_validation_retains_reason_without_committing() {
|
||||
for (fault, reason) in [("identity", "Identity"), ("size", "ResponseSize")] {
|
||||
let client = Router::new();
|
||||
let (feature, runtime) = runtime(client.clone());
|
||||
feature.attach("initial", None).await.unwrap();
|
||||
let path = "/workspace-config/main.dcdl";
|
||||
observe_interface(&runtime, path, true).await;
|
||||
client.state.lock().unwrap().malformed_read_reply = Some(fault);
|
||||
let capture = Capture::default();
|
||||
let error = call(&runtime, path, "read", json!({}))
|
||||
.with_subscriber(capture.dispatch())
|
||||
.await
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
let text = capture.text();
|
||||
assert!(text.contains(&format!("reason={reason}")), "{text}");
|
||||
assert!(text.contains("stage=Response"));
|
||||
assert!(!error.contains("unknown"));
|
||||
assert!(!text.contains("secret-token") && !text.contains("/private/host"));
|
||||
assert!(!error.contains("secret-token") && !error.contains("/private/host"));
|
||||
let state = client.state.lock().unwrap();
|
||||
assert_eq!(state.applied_commits, 0);
|
||||
assert_eq!(state.entries["main.dcdl"].1, "model = old");
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn production_observation_validation_retains_reason_without_committing() {
|
||||
for (fault, reason) in [
|
||||
("connection", "Identity"),
|
||||
("entrypoints", "InvalidMetadata"),
|
||||
("scope", "InvalidMetadata"),
|
||||
] {
|
||||
let client = Router::new();
|
||||
let (feature, runtime) = runtime(client.clone());
|
||||
feature.attach("initial", None).await.unwrap();
|
||||
client.state.lock().unwrap().malformed_observe_reply = Some(fault);
|
||||
let capture = Capture::default();
|
||||
let error = runtime
|
||||
.tree("/workspace-config/main.dcdl".into(), 0, true)
|
||||
.with_subscriber(capture.dispatch())
|
||||
.await
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
let text = capture.text();
|
||||
assert!(text.contains(&format!("reason={reason}")), "{text}");
|
||||
assert!(text.contains("stage=Response"));
|
||||
assert!(!text.contains("secret-token") && !text.contains("/private/host"));
|
||||
assert!(!error.contains("secret-token") && !error.contains("/private/host"));
|
||||
assert_eq!(client.state.lock().unwrap().applied_commits, 0);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn production_postcommit_transport_loss_retains_cause_and_unknown_without_retry() {
|
||||
let client = Router::new();
|
||||
let (feature, runtime) = runtime(client.clone());
|
||||
feature.attach("initial", None).await.unwrap();
|
||||
let path = "/workspace-config/main.dcdl";
|
||||
observe_interface(&runtime, path, true).await;
|
||||
client.state.lock().unwrap().lose_postcommit_observe_reply = true;
|
||||
let capture = Capture::default();
|
||||
let error = call(
|
||||
&runtime,
|
||||
path,
|
||||
"write",
|
||||
json!({"content":"saved-before-observation-loss"}),
|
||||
)
|
||||
.with_subscriber(capture.dispatch())
|
||||
.await
|
||||
.unwrap_err()
|
||||
.to_string();
|
||||
assert!(error.contains("unknown"));
|
||||
let text = capture.text();
|
||||
assert!(
|
||||
text.lines().any(
|
||||
|event| event.contains("reason=Communication") && event.contains("stage=Reconcile")
|
||||
),
|
||||
"{text}"
|
||||
);
|
||||
assert!(
|
||||
!text.contains("reason=UnrecognizedResponse"),
|
||||
"must not invent a response-decode cause for transport loss"
|
||||
);
|
||||
assert!(!text.contains("secret-token") && !text.contains("/private/host"));
|
||||
assert!(!text.contains("saved-before-observation-loss"));
|
||||
assert!(!error.contains("secret-token") && !error.contains("/private/host"));
|
||||
let correlations: BTreeSet<_> = text
|
||||
.lines()
|
||||
.filter_map(|line| line.split("correlation=").nth(1))
|
||||
.filter_map(|tail| tail.split(';').next())
|
||||
.collect();
|
||||
assert_eq!(
|
||||
correlations.len(),
|
||||
1,
|
||||
"lost postcommit observation must correlate to original commit"
|
||||
);
|
||||
let state = client.state.lock().unwrap();
|
||||
assert_eq!(state.commits.len(), 1);
|
||||
assert_eq!(state.applied_commits, 1);
|
||||
assert_eq!(
|
||||
state.entries["main.dcdl"].1,
|
||||
"saved-before-observation-loss"
|
||||
);
|
||||
}
|
||||
|
||||
@@ -161,7 +161,14 @@ impl WipOperationHandler for AttachHandler {
|
||||
"attachment_path": format!("/workdir-attachments/{}", encode_identity(&result.connection_id)),
|
||||
"content_path": CONTENT_ROOT,
|
||||
"already_attached": result.already_attached,
|
||||
})).map_err(|_| WipOperationError::OutcomeUnknown("Completed config attachment response unavailable".into()))?;
|
||||
})).map_err(|_| {
|
||||
super::backend::diagnostic("attach_output", true).record(
|
||||
crate::feature::builtin::backend_diagnostic::Stage::Attach,
|
||||
crate::feature::builtin::backend_diagnostic::Reason::Decode,
|
||||
None,
|
||||
);
|
||||
WipOperationError::OutcomeUnknown("Completed config attachment response unavailable".into())
|
||||
})?;
|
||||
Ok(WipOperationOutput::native(value))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,57 +1,6 @@
|
||||
//! `WorkerEvent` send / receive helpers.
|
||||
//!
|
||||
//! This module owns the parent-facing lifecycle-event primitive
|
||||
//! (`WorkerEvent`) that children fire upward on turn-end / error /
|
||||
//! shutdown / scope-sub-delegation. Three responsibilities live here:
|
||||
//!
|
||||
//! - **Send** a `Method::WorkerEvent` to the parent socket, fire-and-forget,
|
||||
//! logging failures without blocking the child.
|
||||
//! - **Render** agent-visible variants into human-readable strings for the
|
||||
//! parent's notification buffer. Control-plane-only variants may still have
|
||||
//! a renderer for diagnostics, but receive-side classification keeps them
|
||||
//! out of LLM history/context.
|
||||
//! - **Apply side effects** on the parent (registry / worker-allocation
|
||||
//! updates) so that the receive path is idempotent and tolerant of
|
||||
//! out-of-order delivery.
|
||||
//!
|
||||
//! Transport is fire-and-forget — the ticket's decision is that
|
||||
//! callbacks are an optimisation and `ListWorkers` + `reclaim_stale` are
|
||||
//! the real fallback. This module is allowed to drop events on the
|
||||
//! floor (with a warn log) rather than retry.
|
||||
//!
|
||||
//! `apply_event_side_effects` takes its dependencies (registry, scope
|
||||
//! lock path, self identity) by reference so the caller owns lifetime
|
||||
//! and locking concerns.
|
||||
//! Agent-visible rendering of typed Internal SubWorker lifecycle events.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
|
||||
use protocol::{Method, WorkerEvent};
|
||||
|
||||
use crate::spawn::comm_tools::connect_and_send;
|
||||
use crate::spawn::registry::SpawnedWorkerRegistry;
|
||||
|
||||
/// Connect to `socket`, send a single `Method::WorkerEvent(event)`, and
|
||||
/// return. Used by children to report up to their parent.
|
||||
///
|
||||
/// This is a synchronous helper — callers that want fire-and-forget
|
||||
/// semantics should wrap the call in `tokio::spawn` themselves.
|
||||
pub async fn send_worker_event(socket: &Path, event: WorkerEvent) -> std::io::Result<()> {
|
||||
connect_and_send(socket, &Method::WorkerEvent(event)).await
|
||||
}
|
||||
|
||||
/// Spawn a fire-and-forget task that sends `event` to `socket`. If
|
||||
/// `socket` is `None`, no send happens (top-level Workers have no parent).
|
||||
/// Any send failure is logged at warn level but otherwise ignored —
|
||||
/// the parent is treated as best-effort.
|
||||
pub fn fire_and_forget(socket: Option<PathBuf>, event: WorkerEvent) {
|
||||
let Some(socket) = socket else { return };
|
||||
tokio::spawn(async move {
|
||||
if let Err(e) = send_worker_event(&socket, event).await {
|
||||
tracing::warn!(error = %e, socket = %socket.display(), "WorkerEvent send failed");
|
||||
}
|
||||
});
|
||||
}
|
||||
use protocol::WorkerEvent;
|
||||
|
||||
/// Render a variant into a one-line human-readable string.
|
||||
///
|
||||
@@ -83,16 +32,3 @@ pub fn render_event(event: &WorkerEvent) -> String {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Legacy process callback events have no SubWorker registry authority.
|
||||
///
|
||||
/// Internal SubWorker lifecycle is applied directly through typed session handles. A callback from
|
||||
/// an externally adopted Worker may still be rendered for diagnostics, but it cannot add/remove
|
||||
/// Internal children or transfer filesystem authority.
|
||||
pub async fn apply_event_side_effects(
|
||||
_event: &WorkerEvent,
|
||||
_registry: &Arc<SpawnedWorkerRegistry>,
|
||||
_self_name: &str,
|
||||
_self_parent_socket: &Option<PathBuf>,
|
||||
) {
|
||||
}
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
pub mod alerter;
|
||||
pub mod event;
|
||||
pub mod protocol_session;
|
||||
pub mod server;
|
||||
|
||||
pub(crate) mod interceptor;
|
||||
pub(crate) mod notify_buffer;
|
||||
|
||||
@@ -1,224 +0,0 @@
|
||||
use std::io;
|
||||
use std::io::ErrorKind;
|
||||
use std::path::PathBuf;
|
||||
|
||||
use protocol::stream::{JsonLineReader, JsonLineWriter};
|
||||
use tokio::net::UnixListener;
|
||||
use tokio::sync::oneshot;
|
||||
use tokio::task::{JoinHandle, JoinSet};
|
||||
|
||||
use crate::controller::WorkerHandle;
|
||||
use crate::ipc::protocol_session::{
|
||||
dispatch_worker_protocol_method, live_log_entry_event, subscribe_worker_protocol_session,
|
||||
};
|
||||
use protocol::{ErrorCode, Event};
|
||||
|
||||
/// Unix socket server for Worker Protocol.
|
||||
///
|
||||
/// Listens on the Worker's runtime directory socket path.
|
||||
/// Each client connection gets bidirectional JSONL:
|
||||
/// - Client writes Method lines → forwarded to WorkerController
|
||||
/// - Worker events → written as Event lines to all connected clients
|
||||
pub struct SocketServer {
|
||||
accept_task: Option<JoinHandle<()>>,
|
||||
shutdown: Option<oneshot::Sender<()>>,
|
||||
path: PathBuf,
|
||||
}
|
||||
|
||||
impl SocketServer {
|
||||
/// Start listening on the WorkerHandle's socket path.
|
||||
pub async fn start(handle: &WorkerHandle) -> Result<Self, io::Error> {
|
||||
let path = handle.runtime_dir.socket_path();
|
||||
|
||||
// Remove stale socket file if it exists
|
||||
let _ = tokio::fs::remove_file(&path).await;
|
||||
|
||||
let listener = UnixListener::bind(&path)?;
|
||||
let handle = handle.clone();
|
||||
let (shutdown_tx, mut shutdown_rx) = oneshot::channel();
|
||||
|
||||
let accept_task = tokio::spawn(async move {
|
||||
let mut connections = JoinSet::new();
|
||||
loop {
|
||||
tokio::select! {
|
||||
_ = &mut shutdown_rx => break,
|
||||
accepted = listener.accept() => match accepted {
|
||||
Ok((stream, _)) => {
|
||||
let handle = handle.clone();
|
||||
connections.spawn(handle_connection(stream, handle));
|
||||
}
|
||||
Err(_) => break,
|
||||
},
|
||||
completed = connections.join_next(), if !connections.is_empty() => {
|
||||
let _ = completed;
|
||||
}
|
||||
}
|
||||
}
|
||||
connections.shutdown().await;
|
||||
});
|
||||
|
||||
Ok(Self {
|
||||
accept_task: Some(accept_task),
|
||||
shutdown: Some(shutdown_tx),
|
||||
path,
|
||||
})
|
||||
}
|
||||
|
||||
/// Stop accepting connections and join the server task. Dropping its
|
||||
/// `JoinSet` cancels every active connection task before this returns.
|
||||
pub async fn shutdown(mut self) {
|
||||
if let Some(shutdown) = self.shutdown.take() {
|
||||
let _ = shutdown.send(());
|
||||
}
|
||||
if let Some(task) = self.accept_task.take() {
|
||||
let _ = task.await;
|
||||
}
|
||||
let _ = tokio::fs::remove_file(&self.path).await;
|
||||
}
|
||||
|
||||
/// The socket file path.
|
||||
pub fn path(&self) -> &std::path::Path {
|
||||
&self.path
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for SocketServer {
|
||||
fn drop(&mut self) {
|
||||
if let Some(shutdown) = self.shutdown.take() {
|
||||
let _ = shutdown.send(());
|
||||
}
|
||||
let _ = self.accept_task.take();
|
||||
let _ = std::fs::remove_file(&self.path);
|
||||
}
|
||||
}
|
||||
|
||||
fn is_peer_disconnect_read_error(error: &io::Error) -> bool {
|
||||
matches!(
|
||||
error.kind(),
|
||||
ErrorKind::ConnectionReset
|
||||
| ErrorKind::ConnectionAborted
|
||||
| ErrorKind::BrokenPipe
|
||||
| ErrorKind::UnexpectedEof
|
||||
)
|
||||
}
|
||||
|
||||
async fn handle_connection(stream: tokio::net::UnixStream, handle: WorkerHandle) {
|
||||
let (reader, writer) = stream.into_split();
|
||||
let mut reader = JsonLineReader::new(reader);
|
||||
let mut writer = JsonLineWriter::new(writer);
|
||||
|
||||
// Hold the in-flight stream lock while taking the session-log mirror
|
||||
// snapshot. `LogEntry::AnnotatedAssistantItem` is mirror-only for live clients,
|
||||
// so a finalized assistant block must be observed either as an already
|
||||
// committed entry or as the still-present in-flight block. This lock
|
||||
// order matches `append_entry` (in-flight clear before sink publish) and
|
||||
// keeps the snapshot/live boundary gap-free.
|
||||
let mut streams = subscribe_worker_protocol_session(&handle);
|
||||
for alert in streams.alert_snapshot {
|
||||
if writer.write_event(&Event::Alert(alert)).await.is_err() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
// Send the typed snapshot up front so late attachers can
|
||||
// reconstruct view state without an extra round trip.
|
||||
if writer.write_event(&streams.snapshot_event).await.is_err() {
|
||||
return;
|
||||
}
|
||||
|
||||
loop {
|
||||
tokio::select! {
|
||||
// Live session-log entries → dispatched as the role-specific
|
||||
// wire events. `SegmentLogSink` only broadcasts committed log
|
||||
// entries with live UI meaning; `UserInput` travels this lane so
|
||||
// the visible user line is ordered with `SegmentStart` rotation.
|
||||
entry = streams.log_entries.recv() => {
|
||||
match entry {
|
||||
Ok(entry) => {
|
||||
if let Some(event) = live_log_entry_event(entry) {
|
||||
if writer.write_event(&event).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => {
|
||||
// Slow client fell behind the broadcast buffer.
|
||||
// Drop the connection so the next reconnect
|
||||
// re-seeds the prefix via subscribe_with_snapshot.
|
||||
break;
|
||||
}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Closed) => break,
|
||||
}
|
||||
}
|
||||
// Broadcast events → this client
|
||||
event = streams.events.recv() => {
|
||||
match event {
|
||||
Ok(event) => {
|
||||
let Ok(event) = handle.alerter.freshen_event(event) else {
|
||||
// The queued Alert was evicted; reconnect must rebuild
|
||||
// the bounded prefix rather than forward a stale anchor.
|
||||
break;
|
||||
};
|
||||
if writer.write_event(&event).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
// Client methods → handle or forward to controller
|
||||
method = reader.next_method() => {
|
||||
match method {
|
||||
Ok(Some(method)) => {
|
||||
if let Some(response) = dispatch_worker_protocol_method(&handle, method).await
|
||||
&& writer.write_event(&response).await.is_err()
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
Ok(None) => break,
|
||||
Err(e) if is_peer_disconnect_read_error(&e) => break,
|
||||
Err(e) => {
|
||||
if writer
|
||||
.write(&Event::Error {
|
||||
code: ErrorCode::InvalidRequest,
|
||||
message: format!("invalid method: {e}"),
|
||||
})
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn peer_disconnect_read_errors_are_connection_close() {
|
||||
for kind in [
|
||||
ErrorKind::ConnectionReset,
|
||||
ErrorKind::ConnectionAborted,
|
||||
ErrorKind::BrokenPipe,
|
||||
ErrorKind::UnexpectedEof,
|
||||
] {
|
||||
let error = io::Error::new(kind, "peer disconnected");
|
||||
assert!(
|
||||
is_peer_disconnect_read_error(&error),
|
||||
"{kind:?} should be treated as a normal peer disconnect"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_data_is_not_peer_disconnect() {
|
||||
let error = io::Error::new(ErrorKind::InvalidData, "malformed method");
|
||||
assert!(!is_peer_disconnect_read_error(&error));
|
||||
}
|
||||
}
|
||||
@@ -13,7 +13,6 @@
|
||||
mod result;
|
||||
pub use result::{JobResultFeature, JobResultFinalizer, JobResultSink, SUBMIT_JOB_RESULT_TOOL};
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -154,7 +153,7 @@ fn prepare_job_from_registry_with_grant(
|
||||
)?;
|
||||
let mut system_prompt = template.render(&SystemPromptContext {
|
||||
now: chrono::Utc::now(),
|
||||
cwd: Cow::Borrowed("(no filesystem)"),
|
||||
workdir_sessions: &workdir::WorkdirSessionRouter::new(),
|
||||
language: &manifest.engine.language,
|
||||
scope: &scope,
|
||||
tool_names,
|
||||
|
||||
@@ -7,8 +7,6 @@ pub mod bootstrap;
|
||||
pub mod checkout;
|
||||
pub mod compact;
|
||||
pub mod controller;
|
||||
pub mod discovery;
|
||||
pub mod entrypoint;
|
||||
pub mod feature;
|
||||
mod file_operation;
|
||||
pub mod fs_view;
|
||||
@@ -19,9 +17,9 @@ pub mod job;
|
||||
mod metadata_lifecycle;
|
||||
pub mod model_client;
|
||||
mod paste_artifact_tool;
|
||||
pub mod profile_manifest;
|
||||
pub mod prompt;
|
||||
pub mod runtime;
|
||||
pub mod runtime_command;
|
||||
pub mod segment_log_sink;
|
||||
mod session_capture;
|
||||
mod session_history;
|
||||
@@ -37,6 +35,7 @@ pub mod wip;
|
||||
|
||||
mod internal_worker;
|
||||
mod interrupt_prep;
|
||||
mod peer;
|
||||
mod permission;
|
||||
mod worker;
|
||||
|
||||
@@ -45,10 +44,9 @@ pub use bootstrap::{
|
||||
WorkerBootstrapLayout, start_worker_controller,
|
||||
};
|
||||
pub use compact::token_counter::{EstimateSource, SplitPoint, TokenEstimate};
|
||||
pub use controller::{ShutdownReceiver, WorkerController, WorkerControllerTransport, WorkerHandle};
|
||||
pub use controller::{ShutdownReceiver, WorkerController, WorkerHandle};
|
||||
pub use hook::{Hook, HookEventKind, HookRegistryBuilder};
|
||||
pub use ipc::alerter::Alerter;
|
||||
pub use ipc::server::SocketServer;
|
||||
pub use manifest::{
|
||||
AuthRef, ModelManifest, SchemeKind, Scope, WorkerManifest, WorkerManifestConfig,
|
||||
WorkerMetaConfig,
|
||||
@@ -71,10 +69,9 @@ pub use session_history::{
|
||||
};
|
||||
pub use shared_state::WorkerSharedState;
|
||||
pub use worker::{
|
||||
LocalWorkingDirectory, ReviewerChildWorkspaceClient, ReviewerContext,
|
||||
SubjektivSessionAttributionLifecycle, Worker, WorkerError, WorkerFilesystemAuthority,
|
||||
WorkerRunResult, WorkerWorkspaceContext, WorkspaceBinaryRequest, WorkspaceBinaryResponse,
|
||||
WorkspaceClient, WorkspaceClientError, WorkspaceId, WorkspaceIdError,
|
||||
LocalWorkingDirectory, SubjektivSessionAttributionLifecycle, Worker, WorkerError,
|
||||
WorkerFilesystemAuthority, WorkerRunResult, WorkerWorkspaceContext, WorkspaceBinaryRequest,
|
||||
WorkspaceBinaryResponse, WorkspaceClient, WorkspaceClientError, WorkspaceId, WorkspaceIdError,
|
||||
WorkspacePromptCatalogResolution, WorkspaceRequest, WorkspaceRequestMethod, WorkspaceResponse,
|
||||
WorkspaceServerOperation, WorkspaceWorkerDiscoveryRequest, apply_worker_manifest,
|
||||
marker_workspace_client, unavailable_workspace_client,
|
||||
|
||||
@@ -0,0 +1,186 @@
|
||||
//! Reciprocal peer metadata registration, independent of discovery and transport.
|
||||
//!
|
||||
//! Registering a peer does not start a Worker or grant filesystem, observation, or
|
||||
//! messaging authority. The two metadata writes are not a transaction: a failed
|
||||
//! reciprocal write triggers best-effort restoration of the source's prior peers.
|
||||
|
||||
use serde::Serialize;
|
||||
use session_store::{WorkerMetadataStore, WorkerStoreError, validate_worker_name};
|
||||
|
||||
#[derive(Debug, Serialize, PartialEq, Eq)]
|
||||
pub(crate) struct PeerRegistrationResult {
|
||||
pub source: String,
|
||||
pub peer: String,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub(crate) enum PeerRegistrationError {
|
||||
#[error("worker state missing for `{worker_name}`")]
|
||||
StateMissing { worker_name: String },
|
||||
#[error("worker `{worker_name}` cannot be registered as a peer of itself")]
|
||||
SelfPeer { worker_name: String },
|
||||
#[error("worker `{worker_name}` does not exist")]
|
||||
MissingWorker { worker_name: String },
|
||||
#[error("worker store error: {0}")]
|
||||
WorkerStore(#[from] WorkerStoreError),
|
||||
}
|
||||
|
||||
pub(crate) fn register_peer(
|
||||
store: &impl WorkerMetadataStore,
|
||||
source: &str,
|
||||
peer: &str,
|
||||
) -> Result<PeerRegistrationResult, PeerRegistrationError> {
|
||||
ensure_existing_peer(store, source, peer)?.ok_or_else(|| PeerRegistrationError::MissingWorker {
|
||||
worker_name: peer.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
fn ensure_existing_peer(
|
||||
store: &impl WorkerMetadataStore,
|
||||
source: &str,
|
||||
peer: &str,
|
||||
) -> Result<Option<PeerRegistrationResult>, PeerRegistrationError> {
|
||||
validate_worker_name(peer)?;
|
||||
if peer == source {
|
||||
return Err(PeerRegistrationError::SelfPeer {
|
||||
worker_name: peer.to_string(),
|
||||
});
|
||||
}
|
||||
let prior_source_peers = store
|
||||
.read_by_name(source)?
|
||||
.ok_or_else(|| PeerRegistrationError::StateMissing {
|
||||
worker_name: source.to_string(),
|
||||
})?
|
||||
.peers;
|
||||
if store.read_by_name(peer)?.is_none() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
store.add_peer(source, peer)?;
|
||||
if let Err(error) = store.add_peer(peer, source) {
|
||||
let _ = store.set_peers(source, prior_source_peers);
|
||||
return Err(PeerRegistrationError::WorkerStore(error));
|
||||
}
|
||||
Ok(Some(PeerRegistrationResult {
|
||||
source: source.to_string(),
|
||||
peer: peer.to_string(),
|
||||
}))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use session_store::{FsWorkerStore, WorkerMetadata};
|
||||
|
||||
#[test]
|
||||
fn register_peer_persists_reciprocal_metadata_without_duplicates() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let store = FsWorkerStore::new(root.path()).unwrap();
|
||||
store.write(&WorkerMetadata::new("source", None)).unwrap();
|
||||
store.write(&WorkerMetadata::new("target", None)).unwrap();
|
||||
for _ in 0..2 {
|
||||
let result = register_peer(&store, "source", "target").unwrap();
|
||||
assert_eq!(
|
||||
serde_json::to_value(result).unwrap(),
|
||||
serde_json::json!({
|
||||
"source": "source", "peer": "target",
|
||||
})
|
||||
);
|
||||
}
|
||||
// Reopen the real store rather than asserting only the write calls.
|
||||
let persisted = FsWorkerStore::new(root.path()).unwrap();
|
||||
let source = persisted.read_by_name("source").unwrap().unwrap();
|
||||
let target = persisted.read_by_name("target").unwrap().unwrap();
|
||||
assert_eq!(source.peers.len(), 1);
|
||||
assert_eq!(source.peers[0].worker_name, "target");
|
||||
assert_eq!(target.peers.len(), 1);
|
||||
assert_eq!(target.peers[0].worker_name, "source");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn invalid_self_or_missing_peer_leaves_metadata_unchanged() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let store = FsWorkerStore::new(root.path()).unwrap();
|
||||
store.write(&WorkerMetadata::new("source", None)).unwrap();
|
||||
assert!(matches!(
|
||||
register_peer(&store, "source", "source"),
|
||||
Err(PeerRegistrationError::SelfPeer { .. })
|
||||
));
|
||||
assert!(matches!(
|
||||
register_peer(&store, "source", "missing"),
|
||||
Err(PeerRegistrationError::MissingWorker { .. })
|
||||
));
|
||||
assert!(matches!(
|
||||
register_peer(&store, "source", "../escape"),
|
||||
Err(PeerRegistrationError::WorkerStore(_))
|
||||
));
|
||||
assert!(matches!(
|
||||
register_peer(&store, "missing-source", "source"),
|
||||
Err(PeerRegistrationError::StateMissing { .. })
|
||||
));
|
||||
assert!(
|
||||
store
|
||||
.read_by_name("source")
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.peers
|
||||
.is_empty()
|
||||
);
|
||||
assert_eq!(store.list_names().unwrap(), vec!["source"]);
|
||||
}
|
||||
|
||||
struct FailTargetPeerStore(FsWorkerStore);
|
||||
|
||||
impl WorkerMetadataStore for FailTargetPeerStore {
|
||||
fn write(&self, metadata: &WorkerMetadata) -> Result<(), WorkerStoreError> {
|
||||
if metadata.worker_name == "target" && !metadata.peers.is_empty() {
|
||||
return Err(WorkerStoreError::Io(std::io::Error::other(
|
||||
"target write failed",
|
||||
)));
|
||||
}
|
||||
self.0.write(metadata)
|
||||
}
|
||||
|
||||
fn read_by_name(&self, name: &str) -> Result<Option<WorkerMetadata>, WorkerStoreError> {
|
||||
self.0.read_by_name(name)
|
||||
}
|
||||
|
||||
fn list_names(&self) -> Result<Vec<String>, WorkerStoreError> {
|
||||
self.0.list_names()
|
||||
}
|
||||
|
||||
fn delete_by_name(&self, name: &str) -> Result<(), WorkerStoreError> {
|
||||
self.0.delete_by_name(name)
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reciprocal_write_failure_restores_exact_prior_source_peers() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let inner = FsWorkerStore::new(root.path()).unwrap();
|
||||
for name in ["source", "target", "existing"] {
|
||||
inner.write(&WorkerMetadata::new(name, None)).unwrap();
|
||||
}
|
||||
inner.add_peer("source", "existing").unwrap();
|
||||
for had_target in [false, true] {
|
||||
if had_target {
|
||||
inner.add_peer("source", "target").unwrap();
|
||||
}
|
||||
let before = inner.read_by_name("source").unwrap().unwrap().peers;
|
||||
let store = FailTargetPeerStore(inner.clone());
|
||||
assert!(matches!(
|
||||
register_peer(&store, "source", "target"),
|
||||
Err(PeerRegistrationError::WorkerStore(_))
|
||||
));
|
||||
assert_eq!(inner.read_by_name("source").unwrap().unwrap().peers, before);
|
||||
assert!(
|
||||
inner
|
||||
.read_by_name("target")
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.peers
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,183 @@
|
||||
//! Runtime manifest preparation from Backend-resolved profiles.
|
||||
//!
|
||||
//! This module has no process entrypoint or local profile discovery authority.
|
||||
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use crate::PromptCatalogSource;
|
||||
use manifest::{Permission, ScopeConfig, ScopeRule, WorkerManifest};
|
||||
|
||||
pub fn resolve_runtime_profile_manifest_from_manifest(
|
||||
mut manifest: WorkerManifest,
|
||||
workspace_root: &Path,
|
||||
worker_name: &str,
|
||||
) -> Result<(WorkerManifest, PromptCatalogSource), String> {
|
||||
if manifest.worker.name.is_empty() {
|
||||
manifest.worker.name = worker_name.to_string();
|
||||
}
|
||||
apply_profile_launch_policy(&mut manifest, workspace_root);
|
||||
// Do not run plugin discovery here: runtime-created Workers receive their
|
||||
// resolved manifest/profile archive from Backend authority, not by scanning
|
||||
// materialized workdir-local plugin stores.
|
||||
Ok((manifest, PromptCatalogSource::builtins_only()))
|
||||
}
|
||||
|
||||
pub fn resolve_runtime_profile_manifest_from_manifest_without_filesystem(
|
||||
mut manifest: WorkerManifest,
|
||||
_workspace_root: &Path,
|
||||
worker_name: &str,
|
||||
) -> Result<(WorkerManifest, PromptCatalogSource), String> {
|
||||
if manifest.worker.name.is_empty() {
|
||||
manifest.worker.name = worker_name.to_string();
|
||||
}
|
||||
manifest.scope = ScopeConfig::default();
|
||||
manifest.delegation_scope = ScopeConfig::default();
|
||||
// Same as the filesystem-capable runtime path: no local discovery.
|
||||
Ok((manifest, PromptCatalogSource::builtins_only()))
|
||||
}
|
||||
|
||||
fn read_rule(target: PathBuf) -> ScopeRule {
|
||||
ScopeRule {
|
||||
target,
|
||||
permission: Permission::Read,
|
||||
recursive: true,
|
||||
symlink_policy: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn write_rule(target: PathBuf) -> ScopeRule {
|
||||
ScopeRule {
|
||||
target,
|
||||
permission: Permission::Write,
|
||||
recursive: true,
|
||||
symlink_policy: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn workspace_scope(
|
||||
workspace_root: &Path,
|
||||
permission: Permission,
|
||||
deny_write: &[PathBuf],
|
||||
) -> ScopeConfig {
|
||||
let allow_rule = ScopeRule {
|
||||
target: workspace_root.to_path_buf(),
|
||||
permission,
|
||||
recursive: true,
|
||||
symlink_policy: Default::default(),
|
||||
};
|
||||
let deny = deny_write
|
||||
.iter()
|
||||
.cloned()
|
||||
.map(write_rule)
|
||||
.collect::<Vec<_>>();
|
||||
ScopeConfig {
|
||||
allow: vec![allow_rule],
|
||||
deny,
|
||||
}
|
||||
}
|
||||
|
||||
fn workspace_worktree_delegation(workspace_root: &Path) -> ScopeConfig {
|
||||
ScopeConfig {
|
||||
allow: vec![
|
||||
read_rule(workspace_root.to_path_buf()),
|
||||
write_rule(workspace_root.join(".worktree")),
|
||||
],
|
||||
deny: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn append_missing_rules(target: &mut Vec<ScopeRule>, defaults: Vec<ScopeRule>) {
|
||||
for rule in defaults {
|
||||
if !target.contains(&rule) {
|
||||
target.push(rule);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn apply_scope_launch_defaults(scope: &mut ScopeConfig, defaults: ScopeConfig) {
|
||||
// Profile resolution has already applied explicit profile/workspace override scope rules.
|
||||
// Launch policy contributes runtime defaults on top rather than replacing those grants.
|
||||
append_missing_rules(&mut scope.allow, defaults.allow);
|
||||
append_missing_rules(&mut scope.deny, defaults.deny);
|
||||
}
|
||||
|
||||
fn apply_profile_launch_policy(manifest: &mut WorkerManifest, workspace_root: &Path) {
|
||||
let worktree_root = workspace_root.join(".worktree");
|
||||
let default_scope = workspace_scope(
|
||||
workspace_root,
|
||||
Permission::Write,
|
||||
std::slice::from_ref(&worktree_root),
|
||||
);
|
||||
apply_scope_launch_defaults(&mut manifest.scope, default_scope);
|
||||
manifest.delegation_scope = workspace_worktree_delegation(workspace_root);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
fn manifest() -> WorkerManifest {
|
||||
let mut config = manifest::WorkerManifestConfig::resolution_defaults();
|
||||
config.worker.name = Some("resolved".into());
|
||||
WorkerManifest::try_from(config).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_launch_preserves_explicit_scope_without_repository_discovery() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
std::fs::create_dir(root.path().join(".yoi")).unwrap();
|
||||
std::fs::write(root.path().join(".yoi/manifest.toml"), "invalid TOML").unwrap();
|
||||
let mut original = manifest();
|
||||
let explicit = read_rule(root.path().join("reference"));
|
||||
original.scope.allow.push(explicit.clone());
|
||||
let (resolved, _) =
|
||||
resolve_runtime_profile_manifest_from_manifest(original, root.path(), "fallback")
|
||||
.unwrap();
|
||||
assert_eq!(resolved.worker.name, "resolved");
|
||||
assert!(resolved.scope.allow.contains(&explicit));
|
||||
assert!(
|
||||
resolved
|
||||
.scope
|
||||
.allow
|
||||
.contains(&write_rule(root.path().to_path_buf()))
|
||||
);
|
||||
assert!(
|
||||
resolved
|
||||
.scope
|
||||
.deny
|
||||
.contains(&write_rule(root.path().join(".worktree")))
|
||||
);
|
||||
assert_eq!(
|
||||
resolved.delegation_scope.allow,
|
||||
vec![
|
||||
read_rule(root.path().to_path_buf()),
|
||||
write_rule(root.path().join(".worktree")),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn runtime_without_filesystem_clears_scope_and_delegation_grants() {
|
||||
let mut original = manifest();
|
||||
original.worker.name.clear();
|
||||
original
|
||||
.scope
|
||||
.allow
|
||||
.push(write_rule(PathBuf::from("/work")));
|
||||
original
|
||||
.delegation_scope
|
||||
.allow
|
||||
.push(write_rule(PathBuf::from("/work")));
|
||||
let (resolved, _) = resolve_runtime_profile_manifest_from_manifest_without_filesystem(
|
||||
original,
|
||||
Path::new("/unused"),
|
||||
"runtime-worker",
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(resolved.worker.name, "runtime-worker");
|
||||
assert!(resolved.scope.allow.is_empty());
|
||||
assert!(resolved.scope.deny.is_empty());
|
||||
assert!(resolved.delegation_scope.allow.is_empty());
|
||||
assert!(resolved.delegation_scope.deny.is_empty());
|
||||
}
|
||||
}
|
||||
@@ -834,10 +834,7 @@ mod tests {
|
||||
assert!(common.contains("target-only movement preserves approval"));
|
||||
assert!(reviewer.contains("complete linked MR/result snapshot"));
|
||||
assert!(reviewer.contains("Submit `ReviewMergeRequest` once"));
|
||||
assert!(reviewer.contains("Bash"));
|
||||
assert!(reviewer.contains("Git"));
|
||||
assert!(!reviewer.contains("without modifying"));
|
||||
assert!(!reviewer.contains("read-only"));
|
||||
assert!(reviewer.contains("without modifying implementation files"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -7,13 +7,12 @@
|
||||
//! eagerly syntax-checks it at Worker construction. The final system
|
||||
//! prompt is materialised exactly once just before the first LLM turn:
|
||||
//! the rendered body is appended with a fixed trailing section carrying
|
||||
//! the Worker's `Scope` summary, (if present) the project's `AGENTS.md`
|
||||
//! contents, resident memory sections, and conditional Worker-orchestration
|
||||
//! guidance, then the whole string is handed to the Engine via
|
||||
//! `set_system_prompt`. Subsequent turns and compactions reuse that
|
||||
//! the Worker's `Scope` summary, actual Workdir session environments, (if
|
||||
//! present) the project's `AGENTS.md` contents, resident memory sections, and
|
||||
//! conditional Worker-orchestration guidance, then the whole string is handed
|
||||
//! to the Engine via `set_system_prompt`. Subsequent turns and compactions reuse that
|
||||
//! materialised string verbatim.
|
||||
|
||||
use std::borrow::Cow;
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -105,7 +104,8 @@ impl std::fmt::Debug for SystemPromptTemplate {
|
||||
/// templates cannot drop them on the floor.
|
||||
pub struct SystemPromptContext<'a> {
|
||||
pub now: DateTime<Utc>,
|
||||
pub cwd: Cow<'a, str>,
|
||||
/// The same session router used by the Worker's filesystem tools.
|
||||
pub workdir_sessions: &'a workdir::WorkdirSessionRouter,
|
||||
/// Language policy exposed to instruction templates as `{{ language }}`.
|
||||
pub language: &'a str,
|
||||
pub scope: &'a Scope,
|
||||
@@ -140,7 +140,14 @@ impl<'a> SystemPromptContext<'a> {
|
||||
"datetime".into(),
|
||||
Value::from(self.now.to_rfc3339_opts(SecondsFormat::Secs, true)),
|
||||
);
|
||||
root.insert("cwd".into(), Value::from(self.cwd.as_ref()));
|
||||
// A singular template cwd is meaningful only with one attachment. Never
|
||||
// substitute the local Profile/Memory discovery directory for a session cwd.
|
||||
let cwd = self
|
||||
.workdir_sessions
|
||||
.only_session()
|
||||
.map(|session| display_workdir_path(&session.environment().cwd).to_owned())
|
||||
.unwrap_or_else(|| "no single Workdir session cwd; see execution environment".into());
|
||||
root.insert("cwd".into(), Value::from(cwd));
|
||||
root.insert("language".into(), Value::from(self.language));
|
||||
root.insert(
|
||||
"tools".into(),
|
||||
@@ -255,6 +262,70 @@ fn exact_prompt_name(reference: &str) -> Option<String> {
|
||||
}
|
||||
}
|
||||
|
||||
fn display_workdir_path(path: &workdir::WorkdirPath) -> &str {
|
||||
if path.as_str().is_empty() {
|
||||
"."
|
||||
} else {
|
||||
path.as_str()
|
||||
}
|
||||
}
|
||||
|
||||
/// Project provider-owned session metadata, rather than reconstructing authority
|
||||
/// from a manifest Scope or the directory used to discover Profiles and Memory.
|
||||
pub(crate) fn workdir_environment_section(
|
||||
router: &workdir::WorkdirSessionRouter,
|
||||
) -> Result<String, SystemPromptError> {
|
||||
use workdir::WorkdirSessionCapability::{Command, Edit, Glob, Grep, Read, Write};
|
||||
|
||||
let mut environments = Vec::new();
|
||||
for alias in router.aliases() {
|
||||
let resolved = router
|
||||
.resolve(Some(alias.as_str()))
|
||||
.map_err(|error| SystemPromptError::Render(error.to_string()))?;
|
||||
let environment = resolved.session.environment();
|
||||
let capabilities: Vec<_> = [
|
||||
(Read, "read"),
|
||||
(Write, "write"),
|
||||
(Edit, "edit"),
|
||||
(Glob, "glob"),
|
||||
(Grep, "grep"),
|
||||
(Command, "command"),
|
||||
]
|
||||
.into_iter()
|
||||
.filter_map(|(capability, name)| {
|
||||
environment
|
||||
.capabilities
|
||||
.supports(capability)
|
||||
.then_some(name)
|
||||
})
|
||||
.collect();
|
||||
environments.push(serde_json::json!({
|
||||
"alias": alias.as_str(),
|
||||
"cwd": display_workdir_path(&environment.cwd),
|
||||
"scope": environment.scope,
|
||||
"restrictions": environment.restrictions,
|
||||
"capabilities": capabilities,
|
||||
"command": environment.capabilities.supports(Command),
|
||||
}));
|
||||
}
|
||||
let json = serde_json::to_string_pretty(&environments)
|
||||
.map_err(|error| SystemPromptError::Render(error.to_string()))?;
|
||||
Ok(format!(
|
||||
"## Workdir execution environment\n\n\
|
||||
Select a Workdir by its alias (`target_workdir`); omission is allowed only with one attachment. \
|
||||
Relative filesystem operation paths are resolved from that selected session's cwd, not its Workdir root. \
|
||||
Paths returned by Glob/Grep use the same cwd-relative coordinates and can be passed unchanged to other filesystem tools. \
|
||||
Bash starts at the session cwd; an explicit relative cwd is resolved from it, and shell cd does not change the session cwd.\n\n\
|
||||
In the metadata below, cwd and relative scope targets are Workdir-root-relative (`.` or an empty path denotes the root); explicit absolute provider scope targets stay absolute. \
|
||||
A null scope means the provider's full available scope, not unrestricted host access; an empty scope grants no paths. \
|
||||
Restrictions retain additional provider or ancestor ceilings, including deny rules. \
|
||||
Capabilities and command availability are session ceilings, not new grants. \
|
||||
Resolved targets remain subject to scope, symlink and command authorization. \
|
||||
SubWorker scope declarations remain Workdir-root-relative; other Scope declarations and Profile/Memory roots retain their own meaning and are not rebased by session cwd.\n\n\
|
||||
```json\n{json}\n```"
|
||||
))
|
||||
}
|
||||
|
||||
/// Build the final system prompt by appending the fixed trailing
|
||||
/// section to `body`. The Rust side owns the layout (blank-line
|
||||
/// separators, trailing-whitespace trim); each section's header + body
|
||||
@@ -276,8 +347,12 @@ fn append_trailing_section(
|
||||
}
|
||||
out.push('\n');
|
||||
|
||||
let boundaries = prompts.working_boundaries_section(&scope.summary())?;
|
||||
out.push_str(boundaries.trim_end_matches(&['\n', ' '][..]));
|
||||
if ctx.workdir_sessions.is_empty() {
|
||||
let boundaries = prompts.working_boundaries_section(&scope.summary())?;
|
||||
out.push_str(boundaries.trim_end_matches(&['\n', ' '][..]));
|
||||
out.push('\n');
|
||||
}
|
||||
out.push_str(&workdir_environment_section(ctx.workdir_sessions)?);
|
||||
out.push('\n');
|
||||
if let Some(agents) = agents_md {
|
||||
out.push('\n');
|
||||
@@ -392,7 +467,7 @@ mod tests {
|
||||
now: DateTime::parse_from_rfc3339("2026-10-08T00:00:00Z")
|
||||
.unwrap()
|
||||
.with_timezone(&Utc),
|
||||
cwd: Cow::Borrowed("."),
|
||||
workdir_sessions: &workdir::WorkdirSessionRouter::new(),
|
||||
language: "en",
|
||||
scope: &scope,
|
||||
tool_names: metas.iter().map(|meta| meta.name.clone()).collect(),
|
||||
|
||||
@@ -1,17 +1,12 @@
|
||||
use std::io;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use manifest::{ScopeRule, paths};
|
||||
use manifest::ScopeRule;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tokio::fs;
|
||||
|
||||
use crate::shared_state::WorkerSharedState;
|
||||
|
||||
/// One spawned-child record mirrored to `spawned_workers.json`.
|
||||
///
|
||||
/// Written by the spawner after registry changes so runtime-local tools
|
||||
/// have a materialised snapshot. Durable restore uses Worker state metadata;
|
||||
/// this file is not the authoritative source.
|
||||
/// Legacy process-child projection used only by persisted scope reclamation.
|
||||
/// Socket fields retain the stored format; they are never connected to.
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct SpawnedWorkerRecord {
|
||||
/// Spawned Worker's identity.
|
||||
@@ -25,56 +20,33 @@ pub struct SpawnedWorkerRecord {
|
||||
pub callback_address: PathBuf,
|
||||
}
|
||||
|
||||
/// Manages the Worker's runtime directory on tmpfs.
|
||||
/// Artifact directory retained by an in-process Controller.
|
||||
///
|
||||
/// ```text
|
||||
/// <runtime_dir>/{worker_name}/
|
||||
/// ├── pid
|
||||
/// ├── status.json
|
||||
/// ├── manifest.toml
|
||||
/// ├── history.json
|
||||
/// └── sock (created by socket listener, not by RuntimeDir)
|
||||
/// ```
|
||||
///
|
||||
/// `<runtime_dir>` is resolved via [`manifest::paths::runtime_dir`].
|
||||
/// Files are written atomically (write tmp → rename).
|
||||
/// The directory is removed on drop.
|
||||
/// Worker liveness and configuration belong to the Host, not filesystem projections.
|
||||
/// Persistent run directories and pre-existing data are never removed on drop.
|
||||
pub struct RuntimeDir {
|
||||
path: PathBuf,
|
||||
write_legacy_snapshots: bool,
|
||||
preserve_on_drop: bool,
|
||||
socket_file_name: &'static str,
|
||||
}
|
||||
|
||||
impl RuntimeDir {
|
||||
/// Create the runtime directory and write the PID file.
|
||||
/// Create an ephemeral directory without PID, status, manifest or socket files.
|
||||
pub async fn create(base: &Path, worker_name: &str) -> Result<Self, io::Error> {
|
||||
let path = base.join(worker_name);
|
||||
fs::create_dir_all(&path).await?;
|
||||
|
||||
let pid = std::process::id().to_string();
|
||||
fs::write(path.join("pid"), pid.as_bytes()).await?;
|
||||
|
||||
Ok(Self {
|
||||
path,
|
||||
write_legacy_snapshots: true,
|
||||
preserve_on_drop: false,
|
||||
socket_file_name: "sock",
|
||||
})
|
||||
Self::create_transient(base, worker_name).await
|
||||
}
|
||||
|
||||
/// Create an ephemeral Runtime-owned artifact directory.
|
||||
///
|
||||
/// Runtime-managed Workers keep their status in the owning Runtime and do
|
||||
/// not materialize legacy pid/status/manifest projections.
|
||||
/// Preserve pre-existing directories rather than claiming their contents for cleanup.
|
||||
pub async fn create_transient(base: &Path, worker_name: &str) -> Result<Self, io::Error> {
|
||||
fs::create_dir_all(base).await?;
|
||||
let path = base.join(worker_name);
|
||||
fs::create_dir_all(&path).await?;
|
||||
let preserve_on_drop = match fs::create_dir(&path).await {
|
||||
Ok(()) => false,
|
||||
Err(error) if error.kind() == io::ErrorKind::AlreadyExists && path.is_dir() => true,
|
||||
Err(error) => return Err(error),
|
||||
};
|
||||
Ok(Self {
|
||||
path,
|
||||
write_legacy_snapshots: false,
|
||||
preserve_on_drop: false,
|
||||
socket_file_name: "sock",
|
||||
preserve_on_drop,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -100,39 +72,11 @@ impl RuntimeDir {
|
||||
std::fs::File::open(parent)?.sync_all()?;
|
||||
Ok(Self {
|
||||
path: path.to_path_buf(),
|
||||
write_legacy_snapshots: false,
|
||||
preserve_on_drop: true,
|
||||
socket_file_name: "worker.sock",
|
||||
})
|
||||
}
|
||||
|
||||
/// Create in the default base directory resolved via
|
||||
/// [`manifest::paths::runtime_dir`].
|
||||
pub async fn create_default(worker_name: &str) -> Result<Self, io::Error> {
|
||||
let base = default_base()?;
|
||||
Self::create(&base, worker_name).await
|
||||
}
|
||||
|
||||
/// Write status.json atomically.
|
||||
pub async fn write_status(&self, state: &WorkerSharedState) -> Result<(), io::Error> {
|
||||
if !self.write_legacy_snapshots {
|
||||
return Ok(());
|
||||
}
|
||||
let content = state.status_json();
|
||||
atomic_write(&self.path.join("status.json"), content.as_bytes()).await
|
||||
}
|
||||
|
||||
/// Write manifest.toml (typically once at startup).
|
||||
pub async fn write_manifest(&self, toml: &str) -> Result<(), io::Error> {
|
||||
if !self.write_legacy_snapshots {
|
||||
return Ok(());
|
||||
}
|
||||
atomic_write(&self.path.join("manifest.toml"), toml.as_bytes()).await
|
||||
}
|
||||
|
||||
/// Write `spawned_workers.json` atomically. The entries are the full
|
||||
/// set of spawned children known to this Worker — callers pass the
|
||||
/// replacement list, no incremental merge.
|
||||
/// Rewrite the legacy projection during persisted process-child reclamation.
|
||||
pub async fn write_spawned_workers(
|
||||
&self,
|
||||
records: &[SpawnedWorkerRecord],
|
||||
@@ -145,26 +89,10 @@ impl RuntimeDir {
|
||||
pub fn path(&self) -> &Path {
|
||||
&self.path
|
||||
}
|
||||
|
||||
/// Path where the Unix socket should be created. External callers
|
||||
/// that only know the worker name (e.g. the TUI's attach flow)
|
||||
/// predict the same path via [`manifest::paths::worker_socket_path`].
|
||||
pub fn socket_path(&self) -> PathBuf {
|
||||
self.path.join(self.socket_file_name)
|
||||
}
|
||||
|
||||
pub async fn close_socket(&self) -> Result<(), io::Error> {
|
||||
match fs::remove_file(self.socket_path()).await {
|
||||
Ok(()) => Ok(()),
|
||||
Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(()),
|
||||
Err(error) => Err(error),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RuntimeDir {
|
||||
fn drop(&mut self) {
|
||||
let _ = std::fs::remove_file(self.socket_path());
|
||||
if !self.preserve_on_drop {
|
||||
let _ = std::fs::remove_dir_all(&self.path);
|
||||
}
|
||||
@@ -179,156 +107,72 @@ async fn atomic_write(target: &Path, content: &[u8]) -> Result<(), io::Error> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Resolve the default base directory for runtime data.
|
||||
///
|
||||
/// Thin wrapper over [`manifest::paths::runtime_dir`] that converts a
|
||||
/// missing-env situation into an `io::Error`.
|
||||
pub fn default_base() -> Result<PathBuf, io::Error> {
|
||||
paths::runtime_dir().ok_or_else(|| {
|
||||
io::Error::new(
|
||||
io::ErrorKind::NotFound,
|
||||
"could not resolve runtime directory (no YOI_HOME / \
|
||||
YOI_RUNTIME_DIR / XDG_RUNTIME_DIR / HOME)",
|
||||
)
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::shared_state::WorkerSharedState;
|
||||
|
||||
fn test_state() -> WorkerSharedState {
|
||||
WorkerSharedState::new(
|
||||
"test-worker".into(),
|
||||
session_store::new_segment_id(),
|
||||
"[engine]\nname = \"test-worker\"".into(),
|
||||
protocol::Greeting {
|
||||
worker_name: "test-worker".into(),
|
||||
cwd: "/tmp".into(),
|
||||
provider: "anthropic".into(),
|
||||
model: "claude".into(),
|
||||
scope_summary: String::new(),
|
||||
tools: Vec::new(),
|
||||
context_window: 200_000,
|
||||
context_tokens: 0,
|
||||
reasoning: None,
|
||||
context_usage: None,
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn creates_directory_and_pid() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let rt = RuntimeDir::create(tmp.path(), "my-worker").await.unwrap();
|
||||
|
||||
assert!(rt.path().join("pid").exists());
|
||||
let pid = std::fs::read_to_string(rt.path().join("pid")).unwrap();
|
||||
assert_eq!(pid, std::process::id().to_string());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn write_status_creates_file() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let rt = RuntimeDir::create(tmp.path(), "my-worker").await.unwrap();
|
||||
let state = test_state();
|
||||
|
||||
rt.write_status(&state).await.unwrap();
|
||||
|
||||
let content = std::fs::read_to_string(rt.path().join("status.json")).unwrap();
|
||||
let parsed: serde_json::Value = serde_json::from_str(&content).unwrap();
|
||||
assert_eq!(parsed["state"], "idle");
|
||||
assert_eq!(parsed["worker_name"], "test-worker");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn write_status_reflects_changes() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let rt = RuntimeDir::create(tmp.path(), "my-worker").await.unwrap();
|
||||
let state = test_state();
|
||||
|
||||
state.transition(protocol::WorkerState::Busy(protocol::WorkerBusyState::Run(
|
||||
protocol::WorkerRunState::Running,
|
||||
)));
|
||||
rt.write_status(&state).await.unwrap();
|
||||
|
||||
let content = std::fs::read_to_string(rt.path().join("status.json")).unwrap();
|
||||
let parsed: serde_json::Value = serde_json::from_str(&content).unwrap();
|
||||
assert_eq!(parsed["state"], "running");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn write_manifest_creates_file() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let rt = RuntimeDir::create(tmp.path(), "my-worker").await.unwrap();
|
||||
|
||||
rt.write_manifest("[engine]\nname = \"test\"")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
let content = std::fs::read_to_string(rt.path().join("manifest.toml")).unwrap();
|
||||
assert_eq!(content, "[engine]\nname = \"test\"");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn transient_directory_does_not_write_liveness_snapshots() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let rt = RuntimeDir::create_transient(tmp.path(), "runtime-worker")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
rt.write_status(&test_state()).await.unwrap();
|
||||
rt.write_manifest("[worker]\nname = \"runtime-worker\"")
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(!rt.path().join("pid").exists());
|
||||
assert!(!rt.path().join("status.json").exists());
|
||||
assert!(!rt.path().join("manifest.toml").exists());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn write_spawned_workers_creates_file() {
|
||||
use manifest::{Permission, ScopeRule};
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let rt = RuntimeDir::create(tmp.path(), "my-worker").await.unwrap();
|
||||
|
||||
let records = vec![SpawnedWorkerRecord {
|
||||
worker_name: "child".into(),
|
||||
socket_path: "/run/yoi/child/sock".into(),
|
||||
scope_delegated: vec![ScopeRule {
|
||||
target: "/tmp/work".into(),
|
||||
permission: Permission::Write,
|
||||
recursive: true,
|
||||
symlink_policy: Default::default(),
|
||||
}],
|
||||
callback_address: "/run/yoi/my-worker/sock".into(),
|
||||
}];
|
||||
rt.write_spawned_workers(&records).await.unwrap();
|
||||
|
||||
let content = std::fs::read_to_string(rt.path().join("spawned_workers.json")).unwrap();
|
||||
let parsed: Vec<SpawnedWorkerRecord> = serde_json::from_str(&content).unwrap();
|
||||
assert_eq!(parsed.len(), 1);
|
||||
assert_eq!(parsed[0].worker_name, "child");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn socket_path() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let rt = RuntimeDir::create(tmp.path(), "my-worker").await.unwrap();
|
||||
assert_eq!(rt.socket_path(), rt.path().join("sock"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn drop_removes_directory() {
|
||||
let tmp = tempfile::tempdir().unwrap();
|
||||
let dir_path;
|
||||
async fn transient_controller_directory_has_no_liveness_projections_and_is_released() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let path;
|
||||
{
|
||||
let rt = RuntimeDir::create(tmp.path(), "my-worker").await.unwrap();
|
||||
dir_path = rt.path().to_owned();
|
||||
assert!(dir_path.exists());
|
||||
let runtime = RuntimeDir::create(root.path(), "worker").await.unwrap();
|
||||
path = runtime.path().to_owned();
|
||||
assert_eq!(std::fs::read_dir(&path).unwrap().count(), 0);
|
||||
}
|
||||
assert!(!dir_path.exists());
|
||||
assert!(!path.exists());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn preexisting_data_is_not_overwritten_or_removed_on_drop() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let path = root.path().join("worker");
|
||||
std::fs::create_dir(&path).unwrap();
|
||||
std::fs::write(path.join("pid"), "retained legacy data").unwrap();
|
||||
{
|
||||
let _runtime = RuntimeDir::create(root.path(), "worker").await.unwrap();
|
||||
}
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(path.join("pid")).unwrap(),
|
||||
"retained legacy data"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn persistent_run_preserves_logs_and_rejects_directory_reuse() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let path = root.path().join("runs/1");
|
||||
{
|
||||
let _runtime = RuntimeDir::create_worker_run(&path).await.unwrap();
|
||||
for name in ["artifacts", "spawned", "worker.out.log", "worker.err.log"] {
|
||||
assert!(path.join(name).exists(), "missing {name}");
|
||||
}
|
||||
}
|
||||
assert!(path.exists());
|
||||
let error = RuntimeDir::create_worker_run(&path).await.err().unwrap();
|
||||
assert_eq!(error.kind(), io::ErrorKind::AlreadyExists);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_reclamation_projection_retains_its_stored_format() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
let runtime = RuntimeDir::create(root.path(), "worker").await.unwrap();
|
||||
let record: SpawnedWorkerRecord = serde_json::from_value(serde_json::json!({
|
||||
"worker_name": "legacy-child", "socket_path": "/old/child/sock",
|
||||
"scope_delegated": [], "callback_address": "/old/parent/sock",
|
||||
}))
|
||||
.unwrap();
|
||||
runtime.write_spawned_workers(&[record]).await.unwrap();
|
||||
let value: serde_json::Value = serde_json::from_slice(
|
||||
&std::fs::read(runtime.path().join("spawned_workers.json")).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
assert_eq!(value[0]["socket_path"], "/old/child/sock");
|
||||
assert_eq!(value[0]["callback_address"], "/old/parent/sock");
|
||||
runtime.write_spawned_workers(&[]).await.unwrap();
|
||||
assert_eq!(
|
||||
std::fs::read_to_string(runtime.path().join("spawned_workers.json")).unwrap(),
|
||||
"[]"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -72,7 +72,8 @@ pub struct Allocation {
|
||||
pub worker_name: String,
|
||||
/// Owning process. Checked with `kill(pid, 0)` for stale detection.
|
||||
pub pid: u32,
|
||||
/// Worker's Unix socket path.
|
||||
/// Legacy stored endpoint hint. Empty means the Host owns connectivity.
|
||||
/// Allocation admission and stale reclamation use identity/PID, never this path.
|
||||
pub socket: PathBuf,
|
||||
/// Allow rules granted to this Worker (write + read).
|
||||
pub scope_allow: Vec<ScopeRule>,
|
||||
@@ -338,6 +339,51 @@ mod tests {
|
||||
assert_eq!(guard.data().allocations[0].worker_name, "a");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_legacy_socket_roundtrips_without_path_resolution_or_cleanup() {
|
||||
let root = TempDir::new().unwrap();
|
||||
let path = root.path().join("workers.json");
|
||||
let retained = root.path().join("retained-data");
|
||||
fs::write(&retained, "do not touch").unwrap();
|
||||
let segment = sid();
|
||||
{
|
||||
let mut guard = open_empty(&path);
|
||||
register_worker(
|
||||
&mut guard,
|
||||
"in-process".into(),
|
||||
std::process::id(),
|
||||
PathBuf::new(),
|
||||
Vec::new(),
|
||||
segment,
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
let stored: serde_json::Value = serde_json::from_slice(&fs::read(&path).unwrap()).unwrap();
|
||||
assert_eq!(stored["allocations"][0]["socket"], "");
|
||||
let mut guard = LockFileGuard::open(&path).unwrap();
|
||||
assert!(guard.data().allocations[0].socket.as_os_str().is_empty());
|
||||
super::super::reclaim_stale_with(&mut guard, |pid| {
|
||||
assert_eq!(pid, std::process::id());
|
||||
true
|
||||
});
|
||||
assert_eq!(
|
||||
guard.data().allocations.len(),
|
||||
1,
|
||||
"missing endpoint is not stale ownership"
|
||||
);
|
||||
assert!(guard.data().find_by_segment(segment).is_some());
|
||||
super::super::reclaim_stale_with(&mut guard, |_| false);
|
||||
drop(guard);
|
||||
assert!(
|
||||
LockFileGuard::open(&path)
|
||||
.unwrap()
|
||||
.data()
|
||||
.allocations
|
||||
.is_empty()
|
||||
);
|
||||
assert_eq!(fs::read_to_string(retained).unwrap(), "do not touch");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn atomic_save_reopens_complete_table_at_both_rename_boundaries() {
|
||||
let dir = TempDir::new().unwrap();
|
||||
|
||||
@@ -1,165 +0,0 @@
|
||||
use std::ffi::OsString;
|
||||
use std::fmt;
|
||||
use std::io;
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
const POD_RUNTIME_COMMAND_ENV: &str = "YOI_POD_RUNTIME_COMMAND";
|
||||
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
pub struct WorkerRuntimeCommand {
|
||||
pub program: PathBuf,
|
||||
pub prefix_args: Vec<OsString>,
|
||||
}
|
||||
|
||||
impl WorkerRuntimeCommand {
|
||||
pub fn new(program: impl Into<PathBuf>, prefix_args: Vec<OsString>) -> Self {
|
||||
Self {
|
||||
program: program.into(),
|
||||
prefix_args,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn for_current_exe() -> io::Result<Self> {
|
||||
Ok(Self::for_executable(std::env::current_exe()?))
|
||||
}
|
||||
|
||||
pub fn for_executable(program: impl Into<PathBuf>) -> Self {
|
||||
Self::new(program, vec![OsString::from("worker")])
|
||||
}
|
||||
|
||||
/// Resolve the Worker runtime command used for subprocess launches.
|
||||
///
|
||||
/// The default launch path is always the current `yoi` executable plus
|
||||
/// the unified `worker` prefix argument. During development, a non-empty
|
||||
/// `YOI_POD_RUNTIME_COMMAND` value replaces only the executable path;
|
||||
/// the `worker` prefix is still added here and the env value is not parsed as a
|
||||
/// shell command.
|
||||
pub fn resolve() -> io::Result<Self> {
|
||||
Self::resolve_from_env_value(
|
||||
std::env::var_os(POD_RUNTIME_COMMAND_ENV),
|
||||
std::env::current_exe,
|
||||
)
|
||||
}
|
||||
|
||||
fn resolve_from_env_value<F>(
|
||||
override_program: Option<OsString>,
|
||||
current_exe: F,
|
||||
) -> io::Result<Self>
|
||||
where
|
||||
F: FnOnce() -> io::Result<PathBuf>,
|
||||
{
|
||||
if let Some(program) = override_program.filter(|program| !program.as_os_str().is_empty()) {
|
||||
return Ok(Self::for_executable(program));
|
||||
}
|
||||
|
||||
Ok(Self::for_executable(current_exe()?))
|
||||
}
|
||||
|
||||
pub fn program(&self) -> &Path {
|
||||
&self.program
|
||||
}
|
||||
|
||||
pub fn prefix_args(&self) -> &[OsString] {
|
||||
&self.prefix_args
|
||||
}
|
||||
|
||||
pub fn argv_with<I, S>(&self, args: I) -> Vec<OsString>
|
||||
where
|
||||
I: IntoIterator<Item = S>,
|
||||
S: Into<OsString>,
|
||||
{
|
||||
let mut argv = self.prefix_args.clone();
|
||||
argv.extend(args.into_iter().map(Into::into));
|
||||
argv
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for WorkerRuntimeCommand {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{}", self.program.display())?;
|
||||
for arg in &self.prefix_args {
|
||||
write!(f, " {}", arg.to_string_lossy())?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn yoi_binary_defaults_to_worker_prefix() {
|
||||
let command = WorkerRuntimeCommand::for_executable("/opt/yoi/bin/yoi");
|
||||
|
||||
assert_eq!(command.program(), Path::new("/opt/yoi/bin/yoi"));
|
||||
assert_eq!(command.prefix_args(), [OsString::from("worker")]);
|
||||
assert_eq!(
|
||||
command.argv_with(["--worker", "agent"]),
|
||||
vec!["worker", "--worker", "agent"]
|
||||
.into_iter()
|
||||
.map(OsString::from)
|
||||
.collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn any_runtime_executable_gets_worker_prefix() {
|
||||
let command = WorkerRuntimeCommand::for_executable("/opt/yoi/bin/custom-runtime");
|
||||
|
||||
assert_eq!(command.program(), Path::new("/opt/yoi/bin/custom-runtime"));
|
||||
assert_eq!(command.prefix_args(), [OsString::from("worker")]);
|
||||
assert_eq!(
|
||||
command.argv_with(["--worker", "agent"]),
|
||||
vec!["worker", "--worker", "agent"]
|
||||
.into_iter()
|
||||
.map(OsString::from)
|
||||
.collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_uses_current_exe_when_override_is_unset() {
|
||||
let command = WorkerRuntimeCommand::resolve_from_env_value(None, || {
|
||||
Ok(PathBuf::from("/opt/yoi/bin/yoi"))
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
command,
|
||||
WorkerRuntimeCommand::for_executable("/opt/yoi/bin/yoi")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_uses_current_exe_when_override_is_empty() {
|
||||
let command = WorkerRuntimeCommand::resolve_from_env_value(Some(OsString::new()), || {
|
||||
Ok(PathBuf::from("/opt/yoi/bin/yoi"))
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
command,
|
||||
WorkerRuntimeCommand::for_executable("/opt/yoi/bin/yoi")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resolve_override_replaces_only_program_and_keeps_worker_prefix() {
|
||||
let command = WorkerRuntimeCommand::resolve_from_env_value(
|
||||
Some(OsString::from("/tmp/rebuilt yoi")),
|
||||
|| panic!("override must not inspect current_exe"),
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(command.program(), Path::new("/tmp/rebuilt yoi"));
|
||||
assert_eq!(command.prefix_args(), [OsString::from("worker")]);
|
||||
assert_eq!(
|
||||
command.argv_with(["--worker", "agent"]),
|
||||
vec!["worker", "--worker", "agent"]
|
||||
.into_iter()
|
||||
.map(OsString::from)
|
||||
.collect::<Vec<_>>()
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,181 +0,0 @@
|
||||
//! Socket communication retained for the legacy top-level Worker callback protocol.
|
||||
//!
|
||||
//! Direct Internal SubWorker lifecycle is exposed through `worker.control` and the
|
||||
//! canonical Worker tools, not a second SubWorker-specific tool family.
|
||||
|
||||
use std::path::Path;
|
||||
use std::time::Duration;
|
||||
|
||||
use protocol::stream::{JsonLineReader, JsonLineWriter};
|
||||
use protocol::{Event, Method};
|
||||
use tokio::net::UnixStream;
|
||||
|
||||
/// Timeout applied to each socket-level operation — connect, write,
|
||||
/// read. Kept short so a stuck child doesn't block the spawner's turn.
|
||||
const SOCKET_OP_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
|
||||
/// Connect with a timeout, drain the server's connect-time snapshot,
|
||||
/// write one `Method` line, flush, and close.
|
||||
///
|
||||
/// The Worker socket protocol sends replayed alerts and an initial
|
||||
/// `Event::Snapshot` before it starts reading client methods. Send-only
|
||||
/// callers must consume that prefix; otherwise a large snapshot can block
|
||||
/// the server's writer before it reaches the method-read branch. Any
|
||||
/// socket error maps to an `io::Error`; the caller decides whether to
|
||||
/// surface it to the LLM or treat it as "worker stopped".
|
||||
pub(crate) async fn connect_and_send(socket: &Path, method: &Method) -> std::io::Result<()> {
|
||||
let stream = tokio::time::timeout(SOCKET_OP_TIMEOUT, UnixStream::connect(socket))
|
||||
.await
|
||||
.map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "connect timed out"))??;
|
||||
let (r, w) = stream.into_split();
|
||||
let mut reader = JsonLineReader::new(r);
|
||||
let mut writer = JsonLineWriter::new(w);
|
||||
|
||||
drain_initial_snapshot(&mut reader).await?;
|
||||
|
||||
tokio::time::timeout(SOCKET_OP_TIMEOUT, writer.write(method))
|
||||
.await
|
||||
.map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "write timed out"))??;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn drain_initial_snapshot<R>(reader: &mut JsonLineReader<R>) -> std::io::Result<()>
|
||||
where
|
||||
R: tokio::io::AsyncBufRead + Unpin,
|
||||
{
|
||||
loop {
|
||||
let event = tokio::time::timeout(SOCKET_OP_TIMEOUT, reader.next::<Event>())
|
||||
.await
|
||||
.map_err(|_| std::io::Error::new(std::io::ErrorKind::TimedOut, "read timed out"))??;
|
||||
match event {
|
||||
Some(Event::Snapshot { .. }) => return Ok(()),
|
||||
Some(_) => continue,
|
||||
None => {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::UnexpectedEof,
|
||||
"worker closed connection before Snapshot event",
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
use protocol::{Alert, AlertLevel, AlertSource, Greeting, WorkerEvent, WorkerStatus};
|
||||
use tempfile::TempDir;
|
||||
use tokio::net::UnixListener;
|
||||
use tokio::task::JoinHandle;
|
||||
|
||||
fn snapshot(entries: Vec<serde_json::Value>) -> Event {
|
||||
Event::Snapshot {
|
||||
session: protocol::SessionSnapshot {
|
||||
pending_submissions: protocol::PendingSubmissionsSnapshot::default(),
|
||||
entries: entries
|
||||
.into_iter()
|
||||
.enumerate()
|
||||
.map(|(index, value)| protocol::SessionSnapshotEntry {
|
||||
use_policy: None,
|
||||
entry_id: format!("test-{index}"),
|
||||
timestamp: index as u64,
|
||||
provenance: protocol::SessionEntryProvenance::LegacyUnknown,
|
||||
derived_from: Vec::new(),
|
||||
data: protocol::SessionSnapshotEntryData::RunError {
|
||||
message: value.to_string(),
|
||||
failure: None,
|
||||
},
|
||||
})
|
||||
.collect(),
|
||||
},
|
||||
greeting: Greeting {
|
||||
worker_name: "server".into(),
|
||||
cwd: "/tmp".into(),
|
||||
provider: "test".into(),
|
||||
model: "test".into(),
|
||||
scope_summary: String::new(),
|
||||
tools: Vec::new(),
|
||||
context_window: 200_000,
|
||||
context_tokens: 0,
|
||||
reasoning: None,
|
||||
context_usage: None,
|
||||
},
|
||||
state: WorkerStatus::Idle.into(),
|
||||
in_flight: Default::default(),
|
||||
internal_workers: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
fn serve_initial_events_then_method(
|
||||
listener: UnixListener,
|
||||
events: Vec<Event>,
|
||||
) -> JoinHandle<Option<Method>> {
|
||||
tokio::spawn(async move {
|
||||
let (stream, _) = listener.accept().await.ok()?;
|
||||
let (r, w) = stream.into_split();
|
||||
let mut reader = JsonLineReader::new(r);
|
||||
let mut writer = JsonLineWriter::new(w);
|
||||
for event in events {
|
||||
writer.write(&event).await.ok()?;
|
||||
}
|
||||
reader.next::<Method>().await.ok().flatten()
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn connect_and_send_drains_initial_alert_and_snapshot_before_method() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let socket = tmp.path().join("worker.sock");
|
||||
let listener = UnixListener::bind(&socket).unwrap();
|
||||
let received = serve_initial_events_then_method(
|
||||
listener,
|
||||
vec![
|
||||
Event::Alert(Alert {
|
||||
id: None,
|
||||
position: None,
|
||||
level: AlertLevel::Warn,
|
||||
source: AlertSource::Worker,
|
||||
message: "replayed alert".into(),
|
||||
timestamp_ms: 0,
|
||||
}),
|
||||
snapshot(Vec::new()),
|
||||
],
|
||||
);
|
||||
|
||||
let method = Method::Shutdown {
|
||||
command: protocol::WorkerCommandEnvelope::new(1),
|
||||
};
|
||||
connect_and_send(&socket, &method).await.unwrap();
|
||||
|
||||
let method = received.await.unwrap().expect("expected method");
|
||||
assert!(matches!(method, Method::Shutdown { .. }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn connect_and_send_delivers_method_after_large_initial_snapshot() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let socket = tmp.path().join("worker.sock");
|
||||
let listener = UnixListener::bind(&socket).unwrap();
|
||||
let large_payload = "x".repeat(2 * 1024 * 1024);
|
||||
let received = serve_initial_events_then_method(
|
||||
listener,
|
||||
vec![snapshot(vec![
|
||||
serde_json::json!({ "payload": large_payload }),
|
||||
])],
|
||||
);
|
||||
let expected = Method::WorkerEvent(WorkerEvent::TurnEnded {
|
||||
worker_name: "child".into(),
|
||||
});
|
||||
|
||||
connect_and_send(&socket, &expected).await.unwrap();
|
||||
|
||||
let method = received.await.unwrap().expect("expected method");
|
||||
match method {
|
||||
Method::WorkerEvent(WorkerEvent::TurnEnded { worker_name }) => {
|
||||
assert_eq!(worker_name, "child")
|
||||
}
|
||||
other => panic!("expected TurnEnded WorkerEvent, got {other:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,3 @@
|
||||
pub mod comm_tools;
|
||||
mod published;
|
||||
pub mod registry;
|
||||
pub mod tool;
|
||||
|
||||
@@ -70,8 +70,7 @@ struct SubWorkerSpawnInput {
|
||||
/// Explicitly grant command execution through the parent-owned Workdir tool broker.
|
||||
#[serde(default)]
|
||||
command: bool,
|
||||
/// Binds an actual builtin Reviewer child to the current Merge Request candidate.
|
||||
/// Bash may be enabled for Git and other commands.
|
||||
/// Binds an actual read-only builtin Reviewer child to the current Merge Request candidate.
|
||||
/// Review capability material is generated by the trusted spawn layer.
|
||||
#[serde(default)]
|
||||
review: Option<ReviewerHandoffInput>,
|
||||
@@ -1108,6 +1107,8 @@ fn sub_worker_spawn_tool_impl(
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
mod environment;
|
||||
|
||||
use super::*;
|
||||
use manifest::{DelegationScope, Permission, Scope, SharedScope};
|
||||
use std::pin::Pin;
|
||||
@@ -1175,17 +1176,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reviewer_handoff_schema_allows_bash_for_git_commands() {
|
||||
let schema = serde_json::to_value(schemars::schema_for!(SubWorkerSpawnInput)).unwrap();
|
||||
let description = schema["properties"]["review"]["description"]
|
||||
.as_str()
|
||||
.expect("review handoff description");
|
||||
assert!(description.contains("Bash"));
|
||||
assert!(description.contains("Git"));
|
||||
assert!(!description.contains("read-only"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reviewer_handoff_requires_explicit_builtin_profile_and_writable_scope() {
|
||||
let valid: SubWorkerSpawnInput = serde_json::from_value(serde_json::json!({
|
||||
|
||||
@@ -0,0 +1,629 @@
|
||||
//! Connection tests for model-visible environments and actual routed file operations.
|
||||
//! Runs the production SubWorker spawn/broker/Worker/prompt/tool path with temporary
|
||||
//! LocalWorkdirSession providers; only the external LLM is scripted. Selected by
|
||||
//! the normal worker lib suite, without network credentials or live Workspace data.
|
||||
|
||||
use super::*;
|
||||
use agen::Item;
|
||||
use serde_json::{Value, json};
|
||||
use workdir::{WorkdirSessionCapability, WorkdirSessionRouter};
|
||||
|
||||
const TASK: &str = "Read marker.txt from each attached Workdir.";
|
||||
|
||||
fn prompt_environments(prompt: &str) -> Vec<Value> {
|
||||
let section = prompt
|
||||
.split_once("## Workdir execution environment\n")
|
||||
.expect("Worker must supply the execution environment independently of task text")
|
||||
.1;
|
||||
let json = section.split_once("```json\n").unwrap().1;
|
||||
serde_json::from_str(json.split_once("\n```").unwrap().0).unwrap()
|
||||
}
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct EnvironmentClient {
|
||||
requests: Arc<Mutex<Vec<Request>>>,
|
||||
read_issued: Arc<AtomicBool>,
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl LlmClient for EnvironmentClient {
|
||||
fn clone_boxed(&self) -> Box<dyn LlmClient> {
|
||||
Box::new(self.clone())
|
||||
}
|
||||
|
||||
async fn stream(
|
||||
&self,
|
||||
request: Request,
|
||||
) -> Result<Pin<Box<dyn Stream<Item = Result<LlmEvent, ClientError>> + Send>>, ClientError>
|
||||
{
|
||||
let environments = prompt_environments(request.system_prompt.as_deref().unwrap());
|
||||
self.requests.lock().unwrap().push(request);
|
||||
let mut events = Vec::new();
|
||||
if !self.read_issued.swap(true, Ordering::SeqCst) {
|
||||
// Use only the model-visible aliases and the same relative path in
|
||||
// every session. No test-side cwd prefix is passed to the Read tool.
|
||||
for (index, environment) in environments.iter().enumerate() {
|
||||
let alias = environment["alias"].as_str().unwrap();
|
||||
events.extend([
|
||||
LlmEvent::tool_use_start(index, format!("read-{alias}"), "Read"),
|
||||
LlmEvent::tool_input_delta(
|
||||
index,
|
||||
json!({"target_workdir": alias, "file_path": "marker.txt"}).to_string(),
|
||||
),
|
||||
LlmEvent::tool_use_stop(index),
|
||||
]);
|
||||
}
|
||||
} else {
|
||||
events.extend([
|
||||
LlmEvent::text_block_start(0),
|
||||
LlmEvent::text_delta(0, "read complete"),
|
||||
LlmEvent::text_block_stop(0, None),
|
||||
]);
|
||||
}
|
||||
events.push(LlmEvent::Status(StatusEvent {
|
||||
status: ResponseStatus::Completed,
|
||||
}));
|
||||
Ok(Box::pin(futures::stream::iter(events.into_iter().map(Ok))))
|
||||
}
|
||||
}
|
||||
|
||||
struct EnvironmentFixture {
|
||||
runtime: TempDir,
|
||||
root: PathBuf,
|
||||
manifest: WorkerManifest,
|
||||
profiles: AvailableProfiles,
|
||||
registry: Arc<SpawnedWorkerRegistry>,
|
||||
sessions: Arc<WorkdirSessionRouter>,
|
||||
}
|
||||
|
||||
impl EnvironmentFixture {
|
||||
fn new(aliases: &[(&str, &str)]) -> Self {
|
||||
let runtime = TempDir::new().unwrap();
|
||||
let root = runtime.path().join("profile-root");
|
||||
// Profile discovery stays at this root, deliberately outside every
|
||||
// attached Workdir. Session cwd must never redirect it into work/nested.
|
||||
let profiles = write_project_profile_registry(&root, None, &[]);
|
||||
let mut manifest = parent_manifest(&root, None);
|
||||
manifest.feature.sub_worker.enabled = true;
|
||||
let registry = SpawnedWorkerRegistry::new_internal(
|
||||
"parent".into(),
|
||||
SharedScope::new(Scope::from_config(&manifest.scope).unwrap()),
|
||||
);
|
||||
let sessions = Arc::new(WorkdirSessionRouter::new());
|
||||
for (alias, cwd) in aliases {
|
||||
let checkout = runtime.path().join(alias);
|
||||
for path in ["", "work", "work/nested", "work/nested/deep", "other"] {
|
||||
std::fs::create_dir_all(checkout.join(path)).unwrap();
|
||||
std::fs::write(checkout.join(path).join("marker.txt"), marker(alias, path))
|
||||
.unwrap();
|
||||
}
|
||||
let scope = SharedScope::new(Scope::writable(&checkout).unwrap());
|
||||
let session = Arc::new(workdir::LocalWorkdirSession::materialized_bound(
|
||||
workdir::Workdir::new(format!("fixture-{alias}")),
|
||||
checkout.clone(),
|
||||
checkout.join(cwd),
|
||||
scope,
|
||||
workdir::WorkdirSessionCapabilities::ALL,
|
||||
));
|
||||
sessions
|
||||
.attach_at_cwd(
|
||||
workdir::WorkdirAttachmentAlias::new(*alias).unwrap(),
|
||||
session,
|
||||
workdir::WorkdirPath::new(cwd).unwrap(),
|
||||
)
|
||||
.unwrap();
|
||||
}
|
||||
Self {
|
||||
runtime,
|
||||
root,
|
||||
manifest,
|
||||
profiles,
|
||||
registry,
|
||||
sessions,
|
||||
}
|
||||
}
|
||||
|
||||
async fn spawn(
|
||||
&self,
|
||||
spawner: &str,
|
||||
registry: Arc<SpawnedWorkerRegistry>,
|
||||
broker: workdir::WorkdirToolBrokerRouter,
|
||||
input: Value,
|
||||
) -> (
|
||||
crate::spawn::registry::InternalSpawnedWorkerRecord,
|
||||
EnvironmentClient,
|
||||
) {
|
||||
assert_eq!(input["task"], TASK);
|
||||
let client = EnvironmentClient::default();
|
||||
let mut manifest = self.manifest.clone();
|
||||
manifest.worker.name = spawner.into();
|
||||
let tool = SubWorkerSpawnTool::new(
|
||||
spawner.into(),
|
||||
crate::worker::WorkerWorkspaceContext::local_filesystem(None),
|
||||
ParentNotificationTarget::Durable(Arc::new(|_| {})),
|
||||
self.runtime.path().to_path_buf(),
|
||||
self.root.clone(),
|
||||
Some(broker),
|
||||
registry.clone(),
|
||||
manifest,
|
||||
PromptCatalogSource::builtins_only(),
|
||||
self.profiles.clone(),
|
||||
)
|
||||
.with_internal_client(Box::new(client.clone()));
|
||||
tool.execute(
|
||||
&input.to_string(),
|
||||
agen::tool::ToolExecutionContext::direct(),
|
||||
)
|
||||
.await
|
||||
.expect("spawn through production delegation and Worker construction");
|
||||
let record = registry
|
||||
.get_internal(input["name"].as_str().unwrap())
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
tokio::time::timeout(Duration::from_secs(5), record.session.wait_until_idle())
|
||||
.await
|
||||
.expect("scripted read turn must terminate"),
|
||||
InternalWorkerSessionStatus::Idle,
|
||||
);
|
||||
(record, client)
|
||||
}
|
||||
}
|
||||
|
||||
fn marker(alias: &str, cwd: &str) -> String {
|
||||
format!(
|
||||
"{alias}:{}:session-marker",
|
||||
if cwd.is_empty() { "." } else { cwd }
|
||||
)
|
||||
}
|
||||
|
||||
fn assert_environment_and_reads(
|
||||
router: &WorkdirSessionRouter,
|
||||
client: &EnvironmentClient,
|
||||
expected: &[(&str, &str, bool)],
|
||||
) -> String {
|
||||
let requests = client.requests.lock().unwrap();
|
||||
assert!(
|
||||
requests.len() >= 2,
|
||||
"Read must execute and return to the model"
|
||||
);
|
||||
let prompt = requests[0].system_prompt.as_deref().unwrap();
|
||||
assert!(
|
||||
!prompt.contains("## Working boundaries"),
|
||||
"legacy Scope must not override the connected session environment"
|
||||
);
|
||||
let environments = prompt_environments(prompt);
|
||||
assert_eq!(environments.len(), expected.len());
|
||||
assert_eq!(router.len(), expected.len());
|
||||
for (alias, cwd, command) in expected {
|
||||
let environment = router.resolve(Some(alias)).unwrap().session.environment();
|
||||
assert_eq!(environment.cwd.as_str(), *cwd);
|
||||
assert_eq!(
|
||||
environment
|
||||
.capabilities
|
||||
.supports(WorkdirSessionCapability::Command),
|
||||
*command
|
||||
);
|
||||
let projected = environments
|
||||
.iter()
|
||||
.find(|env| env["alias"] == *alias)
|
||||
.unwrap();
|
||||
assert_eq!(projected["cwd"], if cwd.is_empty() { "." } else { *cwd });
|
||||
assert_eq!(
|
||||
projected["scope"],
|
||||
serde_json::to_value(&environment.scope).unwrap()
|
||||
);
|
||||
assert_eq!(
|
||||
projected["restrictions"],
|
||||
serde_json::to_value(&environment.restrictions).unwrap()
|
||||
);
|
||||
assert_eq!(projected["command"], *command);
|
||||
for (capability, name) in [
|
||||
(WorkdirSessionCapability::Read, "read"),
|
||||
(WorkdirSessionCapability::Write, "write"),
|
||||
(WorkdirSessionCapability::Edit, "edit"),
|
||||
(WorkdirSessionCapability::Glob, "glob"),
|
||||
(WorkdirSessionCapability::Grep, "grep"),
|
||||
(WorkdirSessionCapability::Command, "command"),
|
||||
] {
|
||||
assert_eq!(
|
||||
projected["capabilities"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|value| value == name),
|
||||
environment.capabilities.supports(capability),
|
||||
);
|
||||
}
|
||||
let expected_marker = marker(alias, cwd);
|
||||
let observed_read = requests
|
||||
.iter()
|
||||
.flat_map(|request| &request.items)
|
||||
.any(|item| {
|
||||
matches!(item, Item::ToolResult { call_id, content: Some(content), .. }
|
||||
if call_id == &format!("read-{alias}") && content.contains(&expected_marker))
|
||||
});
|
||||
assert!(
|
||||
observed_read,
|
||||
"{alias}: relative Read did not return {expected_marker}"
|
||||
);
|
||||
}
|
||||
prompt.to_owned()
|
||||
}
|
||||
|
||||
fn assert_retained_environment(
|
||||
record: &crate::spawn::registry::InternalSpawnedWorkerRecord,
|
||||
prompt: &str,
|
||||
) {
|
||||
let retained = record.session.entries().iter().any(|entry| {
|
||||
matches!(entry, session_store::LogEntry::AnnotatedSegmentStart { system_prompt: Some(saved), .. }
|
||||
if saved == prompt)
|
||||
});
|
||||
assert!(
|
||||
retained,
|
||||
"the model-visible environment must also be retained in session history"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn spawn_root_and_nonroot_cwd_are_projected_without_task_annotations() {
|
||||
for (name, cwd, command, permission) in [
|
||||
("root-child", "", false, "read"),
|
||||
("cwd-child", "work", true, "write"),
|
||||
] {
|
||||
let fixture = EnvironmentFixture::new(&[("checkout", "")]);
|
||||
let (record, client) = fixture
|
||||
.spawn(
|
||||
"parent",
|
||||
fixture.registry.clone(),
|
||||
workdir::WorkdirToolBrokerRouter::new(fixture.sessions.clone()),
|
||||
json!({
|
||||
"name": name, "task": TASK, "cwd": if cwd.is_empty() { "." } else { cwd },
|
||||
"scope": [{"target": ".", "permission": permission}],
|
||||
"command": command,
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
let router = record.workdir_tool_scope.session_router();
|
||||
let prompt = assert_environment_and_reads(&router, &client, &[("checkout", cwd, command)]);
|
||||
assert_retained_environment(&record, &prompt);
|
||||
assert_eq!(
|
||||
record.installed_tools.iter().any(|tool| tool == "Bash"),
|
||||
command
|
||||
);
|
||||
fixture.registry.remove_internal(name).await.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn spawn_multiple_aliases_and_nested_child_keep_real_session_environment() {
|
||||
let fixture = EnvironmentFixture::new(&[("alpha", ""), ("beta", "")]);
|
||||
let (child, child_client) = fixture
|
||||
.spawn(
|
||||
"parent",
|
||||
fixture.registry.clone(),
|
||||
workdir::WorkdirToolBrokerRouter::new(fixture.sessions.clone()),
|
||||
json!({
|
||||
"name": "child", "task": TASK, "cwd": "work", "command": true,
|
||||
"scope": [
|
||||
{"target_workdir": "alpha", "target": ".", "permission": "write"},
|
||||
{"target_workdir": "beta", "target": ".", "permission": "write"},
|
||||
],
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
let child_router = child.workdir_tool_scope.session_router();
|
||||
let child_prompt = assert_environment_and_reads(
|
||||
&child_router,
|
||||
&child_client,
|
||||
&[("alpha", "work", true), ("beta", "work", true)],
|
||||
);
|
||||
assert_retained_environment(&child, &child_prompt);
|
||||
assert!(
|
||||
child
|
||||
.installed_tools
|
||||
.iter()
|
||||
.any(|tool| tool == "SubWorkerSpawn")
|
||||
);
|
||||
|
||||
// Reuse the actual child's registry and attenuated broker. The nested tool
|
||||
// receives the scripted LLM at the same boundary as the first spawn; session
|
||||
// construction, routing, delegation and prompt materialization remain real.
|
||||
let (nested, nested_client) = fixture
|
||||
.spawn(
|
||||
"child",
|
||||
child.child_registry.clone(),
|
||||
child.workdir_tool_scope.broker_router(),
|
||||
json!({
|
||||
"name": "nested", "task": TASK, "cwd": "work/nested",
|
||||
"scope": [
|
||||
{"target_workdir": "alpha", "target": "work/nested", "permission": "read"},
|
||||
{"target_workdir": "beta", "target": "work/nested", "permission": "read"},
|
||||
],
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
let nested_router = nested.workdir_tool_scope.session_router();
|
||||
let nested_prompt = assert_environment_and_reads(
|
||||
&nested_router,
|
||||
&nested_client,
|
||||
&[
|
||||
("alpha", "work/nested", false),
|
||||
("beta", "work/nested", false),
|
||||
],
|
||||
);
|
||||
assert_retained_environment(&nested, &nested_prompt);
|
||||
assert!(!nested.installed_tools.iter().any(|tool| tool == "Bash"));
|
||||
for alias in ["alpha", "beta"] {
|
||||
let session = nested_router.resolve(Some(alias)).unwrap().session;
|
||||
let environment = session.environment();
|
||||
let scope = environment
|
||||
.scope
|
||||
.expect("nested session must remain scoped");
|
||||
assert_eq!(scope.len(), 1);
|
||||
assert_eq!(scope[0].target.as_str(), "work/nested");
|
||||
assert_eq!(
|
||||
scope[0].permission,
|
||||
workdir::WorkdirToolScopePermission::Read
|
||||
);
|
||||
assert!(
|
||||
session
|
||||
.read(workdir::ReadRequest {
|
||||
path: workdir::WorkdirPath::new_scoped(
|
||||
fixture
|
||||
.runtime
|
||||
.path()
|
||||
.join(alias)
|
||||
.join("marker.txt")
|
||||
.display()
|
||||
.to_string(),
|
||||
)
|
||||
.unwrap(),
|
||||
offset: 0,
|
||||
limit: 10,
|
||||
max_bytes: 1024,
|
||||
})
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
// Grandchild cwd and reduced command authority must not rewrite its parent.
|
||||
assert_environment_and_reads(
|
||||
&child_router,
|
||||
&child_client,
|
||||
&[("alpha", "work", true), ("beta", "work", true)],
|
||||
);
|
||||
child
|
||||
.child_registry
|
||||
.remove_internal("nested")
|
||||
.await
|
||||
.unwrap();
|
||||
fixture.registry.remove_internal("child").await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn spawn_multiple_aliases_keep_root_scope_independent_of_parent_session_cwds() {
|
||||
let fixture = EnvironmentFixture::new(&[("alpha", "work"), ("beta", "other")]);
|
||||
let (record, client) = fixture
|
||||
.spawn(
|
||||
"parent",
|
||||
fixture.registry.clone(),
|
||||
workdir::WorkdirToolBrokerRouter::new(fixture.sessions.clone()),
|
||||
json!({
|
||||
"name": "child", "task": TASK, "cwd": "work/nested",
|
||||
"scope": [
|
||||
{"target_workdir": "alpha", "target": "work/nested", "permission": "read"},
|
||||
{"target_workdir": "beta", "target": "work/nested", "permission": "read"},
|
||||
],
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
let prompt = assert_environment_and_reads(
|
||||
&record.workdir_tool_scope.session_router(),
|
||||
&client,
|
||||
&[
|
||||
("alpha", "work/nested", false),
|
||||
("beta", "work/nested", false),
|
||||
],
|
||||
);
|
||||
assert_retained_environment(&record, &prompt);
|
||||
for (alias, cwd) in [("alpha", "work"), ("beta", "other")] {
|
||||
assert_eq!(
|
||||
fixture
|
||||
.sessions
|
||||
.resolve(Some(alias))
|
||||
.unwrap()
|
||||
.session
|
||||
.environment()
|
||||
.cwd
|
||||
.as_str(),
|
||||
cwd
|
||||
);
|
||||
}
|
||||
fixture.registry.remove_internal("child").await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn nonroot_startup_then_spawn_and_nested_child_keep_root_relative_scope() {
|
||||
let fixture = EnvironmentFixture::new(&[("workdir", "")]);
|
||||
let checkout = fixture.runtime.path().join("workdir");
|
||||
let client = EnvironmentClient::default();
|
||||
let mut manifest = fixture.manifest.clone();
|
||||
manifest.scope.allow = vec![abs_rule(&checkout, Permission::Write)];
|
||||
let mut worker =
|
||||
Worker::<Box<dyn LlmClient>, EphemeralSessionStore>::from_internal_manifest_with_context(
|
||||
manifest,
|
||||
EphemeralSessionStore::default(),
|
||||
PromptCatalogSource::builtins_only(),
|
||||
crate::worker::WorkerWorkspaceContext::local_filesystem(None),
|
||||
WorkerFilesystemAuthority::local(checkout.clone(), checkout.join("work")),
|
||||
Some(Box::new(client.clone())),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
// Do not bind a test router: exercise workdir_sessions_from_authority via
|
||||
// the real constructor and the normal controller's broker selection.
|
||||
let startup = worker.workdir_sessions();
|
||||
register_worker_tools(
|
||||
&mut worker,
|
||||
fixture.runtime.path().to_path_buf(),
|
||||
fixture.registry.clone(),
|
||||
None,
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::timeout(Duration::from_secs(5), worker.run_text(TASK))
|
||||
.await
|
||||
.expect("startup read turn must terminate")
|
||||
.unwrap();
|
||||
assert_environment_and_reads(&startup, &client, &[("workdir", "work", true)]);
|
||||
|
||||
let (child, child_client) = fixture
|
||||
.spawn(
|
||||
"parent",
|
||||
fixture.registry.clone(),
|
||||
workdir::WorkdirToolBrokerRouter::new(startup.clone()),
|
||||
json!({
|
||||
"name": "child", "task": TASK, "cwd": "work/nested",
|
||||
"scope": [{"target": "work/nested", "permission": "read"}],
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
let child_router = child.workdir_tool_scope.session_router();
|
||||
let child_prompt = assert_environment_and_reads(
|
||||
&child_router,
|
||||
&child_client,
|
||||
&[("workdir", "work/nested", false)],
|
||||
);
|
||||
assert_retained_environment(&child, &child_prompt);
|
||||
let (nested, nested_client) = fixture
|
||||
.spawn(
|
||||
"child",
|
||||
child.child_registry.clone(),
|
||||
child.workdir_tool_scope.broker_router(),
|
||||
json!({
|
||||
"name": "nested", "task": TASK, "cwd": "work/nested/deep",
|
||||
"scope": [{"target": "work/nested/deep", "permission": "read"}],
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
let nested_router = nested.workdir_tool_scope.session_router();
|
||||
let nested_prompt = assert_environment_and_reads(
|
||||
&nested_router,
|
||||
&nested_client,
|
||||
&[("workdir", "work/nested/deep", false)],
|
||||
);
|
||||
assert_retained_environment(&nested, &nested_prompt);
|
||||
for (router, target) in [
|
||||
(&child_router, "work/nested"),
|
||||
(&nested_router, "work/nested/deep"),
|
||||
] {
|
||||
let environment = router.resolve(None).unwrap().session.environment();
|
||||
let rules = environment.scope.unwrap();
|
||||
assert_eq!(rules.len(), 1);
|
||||
assert_eq!(rules[0].target.as_str(), target);
|
||||
}
|
||||
assert_eq!(
|
||||
startup
|
||||
.resolve(None)
|
||||
.unwrap()
|
||||
.session
|
||||
.environment()
|
||||
.cwd
|
||||
.as_str(),
|
||||
"work"
|
||||
);
|
||||
let alias = workdir::WorkdirAttachmentAlias::new("workdir").unwrap();
|
||||
startup.begin_detach(&alias).await.unwrap();
|
||||
// Both the parent tool view and descendant root-coordinate broker source
|
||||
// must share this fence, rather than bypassing it through a cached broker.
|
||||
for router in [&startup, &child_router, &nested_router] {
|
||||
let session = router.resolve(None).unwrap().session;
|
||||
assert!(
|
||||
session
|
||||
.read(workdir::ReadRequest {
|
||||
path: workdir::WorkdirPath::new("marker.txt").unwrap(),
|
||||
offset: 0,
|
||||
limit: 10,
|
||||
max_bytes: 1024,
|
||||
})
|
||||
.await
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
startup.cancel_detach(&alias);
|
||||
child
|
||||
.child_registry
|
||||
.remove_internal("nested")
|
||||
.await
|
||||
.unwrap();
|
||||
fixture.registry.remove_internal("child").await.unwrap();
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn startup_rejects_cwd_outside_workdir_root_without_running_the_model() {
|
||||
let fixture = EnvironmentFixture::new(&[("workdir", "")]);
|
||||
let client = EnvironmentClient::default();
|
||||
let mut manifest = fixture.manifest.clone();
|
||||
manifest.scope.allow = vec![abs_rule(fixture.runtime.path(), Permission::Write)];
|
||||
let error =
|
||||
Worker::<Box<dyn LlmClient>, EphemeralSessionStore>::from_internal_manifest_with_context(
|
||||
manifest,
|
||||
EphemeralSessionStore::default(),
|
||||
PromptCatalogSource::builtins_only(),
|
||||
crate::worker::WorkerWorkspaceContext::local_filesystem(None),
|
||||
WorkerFilesystemAuthority::local(
|
||||
fixture.runtime.path().join("workdir"),
|
||||
fixture.root.clone(),
|
||||
),
|
||||
Some(Box::new(client.clone())),
|
||||
)
|
||||
.await
|
||||
.err()
|
||||
.expect("a session cwd must be representable within its Workdir root");
|
||||
assert!(matches!(error, crate::WorkerError::InvalidCwd { .. }));
|
||||
assert!(client.requests.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn worker_multiple_aliases_project_distinct_session_cwds_not_discovery_root() {
|
||||
for prompt_override in [None, Some("Feature-owned instruction body")] {
|
||||
let fixture = EnvironmentFixture::new(&[("alpha", "work"), ("beta", "other")]);
|
||||
let client = EnvironmentClient::default();
|
||||
let mut worker = Worker::<Box<dyn LlmClient>, EphemeralSessionStore>::from_internal_manifest_with_context(
|
||||
fixture.manifest.clone(),
|
||||
EphemeralSessionStore::default(),
|
||||
PromptCatalogSource::builtins_only(),
|
||||
crate::worker::WorkerWorkspaceContext::local_filesystem(None),
|
||||
WorkerFilesystemAuthority::None,
|
||||
Some(Box::new(client.clone())),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
worker.bind_workdir_sessions(fixture.sessions.clone());
|
||||
register_worker_tools(
|
||||
&mut worker,
|
||||
fixture.runtime.path().to_path_buf(),
|
||||
fixture.registry.clone(),
|
||||
None,
|
||||
Some(workdir::WorkdirToolBrokerRouter::new(
|
||||
fixture.sessions.clone(),
|
||||
)),
|
||||
)
|
||||
.await
|
||||
.unwrap();
|
||||
worker.install_system_prompt_contribution(None, prompt_override.map(str::to_owned));
|
||||
tokio::time::timeout(Duration::from_secs(5), worker.run_text(TASK))
|
||||
.await
|
||||
.expect("scripted worker turn must terminate")
|
||||
.unwrap();
|
||||
let prompt = assert_environment_and_reads(
|
||||
&fixture.sessions,
|
||||
&client,
|
||||
&[("alpha", "work", true), ("beta", "other", true)],
|
||||
);
|
||||
if let Some(body) = prompt_override {
|
||||
assert!(prompt.starts_with(body));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -678,8 +678,8 @@ pub(crate) fn test_connection(client: Arc<dyn WorkspaceClient>) -> SubjektivHost
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::{
|
||||
Method, WorkerBootstrap, WorkerBootstrapLayout, WorkerControllerTransport,
|
||||
WorkerFilesystemAuthority, WorkerManifest, WorkerWorkspaceContext,
|
||||
Method, WorkerBootstrap, WorkerBootstrapLayout, WorkerFilesystemAuthority, WorkerManifest,
|
||||
WorkerWorkspaceContext,
|
||||
};
|
||||
use agen::llm_client::event::{Event as LlmEvent, ResponseStatus, StatusEvent};
|
||||
use agen::llm_client::{ClientError, LlmClient, Request};
|
||||
@@ -944,7 +944,6 @@ enabled = true
|
||||
WorkerBootstrapLayout::Direct {
|
||||
runtime_base: temp.path().join("runtime"),
|
||||
},
|
||||
WorkerControllerTransport::InProcess,
|
||||
)
|
||||
.with_model_client(TaskScriptClient::default())
|
||||
.with_subjektiv_host(host)
|
||||
@@ -1137,7 +1136,6 @@ enabled = true
|
||||
WorkerBootstrapLayout::Direct {
|
||||
runtime_base: temp.path().join("runtime"),
|
||||
},
|
||||
WorkerControllerTransport::InProcess,
|
||||
)
|
||||
.with_model_client(ScriptedClient(tx))
|
||||
.with_subjektiv_host(host.clone())
|
||||
@@ -1318,7 +1316,6 @@ enabled = true
|
||||
WorkerBootstrapLayout::Direct {
|
||||
runtime_base: temp.path().join("runtime"),
|
||||
},
|
||||
WorkerControllerTransport::InProcess,
|
||||
)
|
||||
.with_model_client(ScriptedClient(tx))
|
||||
.with_subjektiv_host(host.clone())
|
||||
@@ -1377,7 +1374,6 @@ enabled = true
|
||||
WorkerBootstrapLayout::Direct {
|
||||
runtime_base: temp.path().join("runtime"),
|
||||
},
|
||||
WorkerControllerTransport::InProcess,
|
||||
)
|
||||
.with_model_client(ScriptedClient(tx));
|
||||
let bootstrap = if attach {
|
||||
|
||||
+47
-249
@@ -531,7 +531,6 @@ use crate::prompt::agents_md::read_agents_md;
|
||||
use crate::prompt::catalog::{CatalogError, PromptCatalog, WorkspacePromptProjection};
|
||||
use crate::prompt::source::PromptCatalogSource;
|
||||
use crate::prompt::system::{SystemPromptContext, SystemPromptError, SystemPromptTemplate};
|
||||
use crate::runtime::dir;
|
||||
use crate::runtime::worker_allocation::{
|
||||
self, ScopeAllocationGuard, ScopeLockError, SegmentActivationGuard,
|
||||
};
|
||||
@@ -1332,28 +1331,15 @@ impl WorkspaceClient for ReviewerChildWorkspaceClient {
|
||||
mut request: WorkspaceRequest,
|
||||
) -> Result<WorkspaceResponse, WorkspaceClientError> {
|
||||
let workspace_id = self.workspace_id().unwrap_or_default();
|
||||
let ticket_id = &self.context.ticket_id;
|
||||
let ticket_query_path = format!("/api/w/{workspace_id}/tickets/query");
|
||||
let ticket_show_prefix = format!("/api/w/{workspace_id}/tickets/");
|
||||
// ShowTicket uses POST for bounded read parameters. Like Ticket query
|
||||
// and GET reads, it must allow related Tickets through the bound Host's
|
||||
// read authority. Only review submission is restricted to the attested MR.
|
||||
// Match one literal reference segment, not arbitrary POST routes.
|
||||
let is_ticket_show = request
|
||||
.path
|
||||
.strip_prefix(&ticket_show_prefix)
|
||||
.and_then(|path| path.strip_suffix("/show"))
|
||||
.is_some_and(|reference| {
|
||||
!reference.is_empty()
|
||||
&& reference
|
||||
.bytes()
|
||||
.all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_'))
|
||||
});
|
||||
let ticket_show_path = format!("/api/w/{workspace_id}/tickets/{ticket_id}/show");
|
||||
let merge_request_id = &self.context.merge_request_id;
|
||||
let review_path =
|
||||
format!("/api/w/{workspace_id}/merge-requests/{merge_request_id}/reviews");
|
||||
let read_allowed = request.method == WorkspaceRequestMethod::Get
|
||||
|| (request.method == WorkspaceRequestMethod::Post
|
||||
&& (request.path == ticket_query_path || is_ticket_show));
|
||||
&& (request.path == ticket_query_path || request.path == ticket_show_path));
|
||||
if request.method == WorkspaceRequestMethod::Post && request.path == review_path {
|
||||
let body = request.body.take().ok_or_else(|| {
|
||||
WorkspaceClientError::Request("review submission requires a JSON body".to_string())
|
||||
@@ -1881,11 +1867,16 @@ mod reviewer_client_tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reviewer_child_client_rejects_mutation_posts() {
|
||||
fn reviewer_child_client_rejects_other_ticket_and_mutation_posts() {
|
||||
let inner: Arc<dyn WorkspaceClient> = Arc::new(RecordingWorkspaceClient::default());
|
||||
let client = reviewer_client(inner);
|
||||
|
||||
for request in [
|
||||
WorkspaceRequest::json(
|
||||
WorkspaceRequestMethod::Post,
|
||||
"/api/w/ws/tickets/T2/show",
|
||||
"{}".to_string(),
|
||||
),
|
||||
WorkspaceRequest::json(
|
||||
WorkspaceRequestMethod::Post,
|
||||
"/api/w/ws/tickets/T1/comments",
|
||||
@@ -1902,54 +1893,6 @@ mod reviewer_client_tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reviewer_ticket_reads_reject_route_injection_and_mutations_without_transport() {
|
||||
let inner = Arc::new(RecordingWorkspaceClient::default());
|
||||
let client = reviewer_client(inner.clone());
|
||||
for path in [
|
||||
"/api/w/other/tickets/T-1/show",
|
||||
"/api/w/ws/tickets//show",
|
||||
"/api/w/ws/tickets/T-1/comments",
|
||||
"/api/w/ws/tickets/T-1/posts",
|
||||
"/api/w/ws/tickets/T-1/show/",
|
||||
"/api/w/ws/tickets/T-1/show?query=1",
|
||||
"/api/w/ws/tickets/T-1?query=1/show",
|
||||
"/api/w/ws/tickets/T-1#fragment/show",
|
||||
"/api/w/ws/tickets/../show",
|
||||
"/api/w/ws/tickets/T-1/../T-2/show",
|
||||
"/api/w/ws/tickets/T%2D1/show",
|
||||
"/api/w/ws/tickets/T-1%2fcomments/show",
|
||||
"/api/w/ws/merge-requests/MR2/reviews",
|
||||
] {
|
||||
assert!(
|
||||
client
|
||||
.execute(WorkspaceRequest::json(
|
||||
WorkspaceRequestMethod::Post,
|
||||
path,
|
||||
"{}",
|
||||
))
|
||||
.is_err(),
|
||||
"{path}"
|
||||
);
|
||||
}
|
||||
for method in [
|
||||
WorkspaceRequestMethod::Put,
|
||||
WorkspaceRequestMethod::Patch,
|
||||
WorkspaceRequestMethod::Delete,
|
||||
] {
|
||||
assert!(
|
||||
client
|
||||
.execute(WorkspaceRequest::json(
|
||||
method,
|
||||
"/api/w/ws/tickets/T-1/show",
|
||||
"{}"
|
||||
))
|
||||
.is_err()
|
||||
);
|
||||
}
|
||||
assert!(inner.requests.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reviewer_child_client_injects_capability_only_for_attested_review() {
|
||||
let inner = Arc::new(RecordingWorkspaceClient::default());
|
||||
@@ -3512,18 +3455,13 @@ pub struct Worker<C: LlmClient, St: Store> {
|
||||
/// authority before the outer allocation guard reacquires the same lock.
|
||||
failed_activation_authority: Mutex<Option<SegmentActivationGuard>>,
|
||||
/// Scope allocation in the machine-wide lock file. `Some` for
|
||||
/// Workers built via `from_manifest` / `from_manifest_spawned` /
|
||||
/// Workers built via `from_manifest` /
|
||||
/// `restore_from_manifest` (production paths); `None` for the
|
||||
/// low-level `Worker::new` constructor used in tests, which bypasses
|
||||
/// the registry. Kept purely for its `Drop` impl, which releases
|
||||
/// the allocation when the Worker is dropped.
|
||||
#[allow(dead_code)]
|
||||
scope_allocation: Option<ScopeAllocationGuard>,
|
||||
/// Socket path of the spawning Worker. `Some` only for Workers built via
|
||||
/// `from_manifest_spawned`. Consumed by the controller to fire
|
||||
/// `Method::WorkerEvent` reports upward (turn end, error, shutdown,
|
||||
/// scope sub-delegation).
|
||||
callback_socket: Option<PathBuf>,
|
||||
/// Transient launch role for Ticket role sessions. This is process-local
|
||||
/// runtime identity used by controller policy; it is not model-visible and
|
||||
/// is not persisted into Ticket claim/session records.
|
||||
@@ -3762,7 +3700,7 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
|
||||
let prompts = Arc::new(ArcSwap::from(PromptCatalog::builtins_only()?));
|
||||
DelegationScope::from_config(&manifest.delegation_scope).map_err(WorkerError::Scope)?;
|
||||
let scope = SharedScope::new(scope);
|
||||
let workdir_sessions = workdir_sessions_from_authority(&filesystem_authority, &scope);
|
||||
let workdir_sessions = workdir_sessions_from_authority(&filesystem_authority, &scope)?;
|
||||
let mut worker = Self {
|
||||
backend_job: std::sync::OnceLock::new(),
|
||||
manifest,
|
||||
@@ -3807,7 +3745,6 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
|
||||
pending_committed_history: Arc::new(Mutex::new(std::collections::VecDeque::new())),
|
||||
failed_activation_authority: Mutex::new(None),
|
||||
scope_allocation: None,
|
||||
callback_socket: None,
|
||||
runtime_ticket_role: None,
|
||||
prompts,
|
||||
inject_resident_summary: true,
|
||||
@@ -3972,8 +3909,8 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
|
||||
self.runtime_ticket_role.as_deref()
|
||||
}
|
||||
|
||||
/// Set the process-local Ticket role marker. Intended for entrypoint
|
||||
/// launch metadata, not for model-visible prompts or durable claims.
|
||||
/// Set the process-local Ticket role marker supplied by the Host at launch,
|
||||
/// not by model-visible prompts or durable claims.
|
||||
pub fn set_runtime_ticket_role(&mut self, role: Option<String>) {
|
||||
self.runtime_ticket_role = role;
|
||||
}
|
||||
@@ -4844,15 +4781,6 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
|
||||
self.pending_notifies.clone()
|
||||
}
|
||||
|
||||
/// Parent callback socket set by `from_manifest_spawned`.
|
||||
///
|
||||
/// Consumed by the Controller to fire `Method::WorkerEvent` upward on
|
||||
/// lifecycle transitions. `None` for top-level Workers, in which case
|
||||
/// the Controller silently skips the send.
|
||||
pub fn callback_socket(&self) -> Option<&PathBuf> {
|
||||
self.callback_socket.as_ref()
|
||||
}
|
||||
|
||||
pub(crate) fn install_compaction_budget(
|
||||
&mut self,
|
||||
budget: Option<crate::compact::worker::CompactWorkerInterceptor>,
|
||||
@@ -5052,11 +4980,14 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
|
||||
let Some(template) = self.system_prompt_template.as_ref() else {
|
||||
return Ok(());
|
||||
};
|
||||
if let Some(rendered) = self.feature_system_prompt_override.as_ref() {
|
||||
if let Some(body) = self.feature_system_prompt_override.as_ref() {
|
||||
let environment =
|
||||
crate::prompt::system::workdir_environment_section(&self.workdir_sessions)
|
||||
.map_err(|source| WorkerError::SystemPromptRender { source })?;
|
||||
self.engine
|
||||
.as_mut()
|
||||
.expect("worker present")
|
||||
.set_system_prompt(rendered.clone());
|
||||
.set_system_prompt(format!("{body}\n\n{environment}"));
|
||||
return Ok(());
|
||||
}
|
||||
let alerter = self.alerter.clone();
|
||||
@@ -5101,14 +5032,10 @@ impl<C: LlmClient + 'static, St: Store> Worker<C, St> {
|
||||
};
|
||||
let worker_language = worker_language(&self.manifest.engine);
|
||||
let scope_snapshot = self.scope.snapshot();
|
||||
let cwd_for_prompt = self
|
||||
.local_working_directory()
|
||||
.map(|local| local.cwd.display().to_string())
|
||||
.unwrap_or_else(|| "no local working directory".to_string());
|
||||
let prompt_catalog = self.prompts.load_full();
|
||||
let ctx = SystemPromptContext {
|
||||
now: chrono::Utc::now(),
|
||||
cwd: cwd_for_prompt.into(),
|
||||
workdir_sessions: &self.workdir_sessions,
|
||||
language: worker_language,
|
||||
scope: &scope_snapshot,
|
||||
tool_names,
|
||||
@@ -8360,14 +8287,10 @@ where
|
||||
// Register this Worker in the machine-wide worker-allocation
|
||||
// before building anything else, so a spawn that conflicts on
|
||||
// scope fails fast.
|
||||
let socket_path = dir::default_base()
|
||||
.map_err(ScopeLockError::from)?
|
||||
.join(&manifest.worker.name)
|
||||
.join("sock");
|
||||
let scope_allocation = worker_allocation::install_top_level(
|
||||
manifest.worker.name.clone(),
|
||||
std::process::id(),
|
||||
socket_path,
|
||||
PathBuf::new(), // Reserved legacy allocation field; no Worker endpoint exists.
|
||||
common.scope.allow_rules(),
|
||||
segment_id,
|
||||
)?;
|
||||
@@ -8382,7 +8305,7 @@ where
|
||||
let worker_metadata_segment_cas = Some(worker_metadata_segment_cas_for_store(&store));
|
||||
let scope = SharedScope::new(common.scope);
|
||||
let workdir_sessions =
|
||||
workdir_sessions_from_authority(&common.filesystem_authority, &scope);
|
||||
workdir_sessions_from_authority(&common.filesystem_authority, &scope)?;
|
||||
|
||||
let mut worker = Self {
|
||||
backend_job: std::sync::OnceLock::new(),
|
||||
@@ -8428,7 +8351,6 @@ where
|
||||
pending_committed_history: Arc::new(Mutex::new(std::collections::VecDeque::new())),
|
||||
failed_activation_authority: Mutex::new(None),
|
||||
scope_allocation: Some(scope_allocation),
|
||||
callback_socket: None,
|
||||
runtime_ticket_role: None,
|
||||
prompts: common.prompts,
|
||||
inject_resident_summary: true,
|
||||
@@ -8784,7 +8706,7 @@ where
|
||||
engine.set_cache_key(Some(segment_id.to_string()));
|
||||
let scope = SharedScope::new(common.scope);
|
||||
let workdir_sessions =
|
||||
workdir_sessions_from_authority(&common.filesystem_authority, &scope);
|
||||
workdir_sessions_from_authority(&common.filesystem_authority, &scope)?;
|
||||
let mut worker = Self {
|
||||
backend_job: std::sync::OnceLock::new(),
|
||||
manifest,
|
||||
@@ -8829,7 +8751,6 @@ where
|
||||
pending_committed_history: Arc::new(Mutex::new(std::collections::VecDeque::new())),
|
||||
failed_activation_authority: Mutex::new(None),
|
||||
scope_allocation: None,
|
||||
callback_socket: None,
|
||||
runtime_ticket_role: None,
|
||||
prompts: common.prompts,
|
||||
inject_resident_summary: true,
|
||||
@@ -8852,141 +8773,6 @@ where
|
||||
Ok(worker)
|
||||
}
|
||||
|
||||
/// Build a Worker spawned by another Worker (sibling process).
|
||||
///
|
||||
/// Behaves like [`Worker::from_manifest`] but claims the scope
|
||||
/// allocation that the spawner pre-registered via
|
||||
/// [`worker_allocation::delegate_scope`], rather than installing a new
|
||||
/// top-level entry. `callback_socket` carries the spawner's
|
||||
/// Unix-socket path so the spawned Worker can send `Method::Notify`
|
||||
/// back to the spawner.
|
||||
pub async fn from_manifest_spawned(
|
||||
manifest: WorkerManifest,
|
||||
store: St,
|
||||
loader: PromptCatalogSource,
|
||||
callback_socket: PathBuf,
|
||||
) -> Result<Self, WorkerError> {
|
||||
let cwd = current_cwd()?;
|
||||
let authority = WorkerFilesystemAuthority::local(cwd.clone(), cwd.clone());
|
||||
let workspace_context = WorkerWorkspaceContext::local_filesystem(None);
|
||||
Self::from_manifest_spawned_with_context(
|
||||
manifest,
|
||||
store,
|
||||
loader,
|
||||
callback_socket,
|
||||
workspace_context,
|
||||
authority,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
pub async fn from_manifest_spawned_with_context(
|
||||
manifest: WorkerManifest,
|
||||
store: St,
|
||||
loader: PromptCatalogSource,
|
||||
callback_socket: PathBuf,
|
||||
workspace_context: WorkerWorkspaceContext,
|
||||
filesystem_authority: WorkerFilesystemAuthority,
|
||||
) -> Result<Self, WorkerError> {
|
||||
let common = prepare_worker_common_with_context(
|
||||
&manifest,
|
||||
&loader,
|
||||
/* parse_template */ true,
|
||||
workspace_context,
|
||||
filesystem_authority,
|
||||
manifest.scope.clone(),
|
||||
)?;
|
||||
|
||||
// A spawned child starts its own conversation, so it mints a
|
||||
// fresh Session rather than joining the spawner's.
|
||||
let session_id = session_store::new_session_id();
|
||||
let segment_id = session_store::new_segment_id();
|
||||
let scope_allocation = worker_allocation::adopt_allocation(
|
||||
manifest.worker.name.clone(),
|
||||
std::process::id(),
|
||||
segment_id,
|
||||
)?;
|
||||
|
||||
let mut worker =
|
||||
Engine::<Box<dyn LlmClient>, Mutable, SessionHistoryMetadata>::new_annotated(
|
||||
common.client,
|
||||
);
|
||||
apply_worker_manifest(&mut worker, &manifest.engine);
|
||||
worker.set_cache_key(Some(segment_id.to_string()));
|
||||
let worker_metadata_writer = Some(worker_metadata_writer_for_store(&store));
|
||||
let worker_metadata_segment_cas = Some(worker_metadata_segment_cas_for_store(&store));
|
||||
let scope = SharedScope::new(common.scope);
|
||||
let workdir_sessions =
|
||||
workdir_sessions_from_authority(&common.filesystem_authority, &scope);
|
||||
|
||||
let mut worker = Self {
|
||||
backend_job: std::sync::OnceLock::new(),
|
||||
manifest,
|
||||
engine: Some(worker),
|
||||
session: WorkerSession::new(session_id, Vec::new()),
|
||||
last_run_interrupted: false,
|
||||
store,
|
||||
worker_metadata_writer,
|
||||
worker_metadata_segment_cas,
|
||||
segment_state: SegmentState::new(session_id, segment_id, 0),
|
||||
filesystem_authority: common.filesystem_authority,
|
||||
workdir_sessions,
|
||||
workspace_context: common.workspace_context,
|
||||
flow_runtime_state: Arc::new(Mutex::new(None)),
|
||||
flow_feature_enabled: false,
|
||||
scope,
|
||||
hook_builder: HookRegistryBuilder::new(),
|
||||
hook_registry: None,
|
||||
feature_background_tasks: FeatureBackgroundTaskRegistry::default(),
|
||||
feature_invocations: FeatureInvocationRegistry::default(),
|
||||
manifest_lifecycle_features_enabled: true,
|
||||
interceptor_installed: false,
|
||||
compaction_service_budget: None,
|
||||
compact_state: None,
|
||||
usage_tracker: Arc::new(UsageTracker::new()),
|
||||
metrics_tracker: Arc::new(crate::compact::metrics_tracker::MetricsTracker::new()),
|
||||
usage_history: Arc::new(Mutex::new(Vec::new())),
|
||||
tracker: None,
|
||||
task_feature: TaskFeature::new(),
|
||||
worker_observation_provider: None,
|
||||
system_prompt_template: common.system_prompt_template,
|
||||
feature_instructions: common.feature_instructions,
|
||||
alerter: None,
|
||||
working_event_tx: None,
|
||||
internal_worker_registry: None,
|
||||
pending_compaction_cleanup: Mutex::new(None),
|
||||
in_flight: None,
|
||||
ai_activity_counter: Arc::new(AtomicUsize::new(0)),
|
||||
pending_notifies: NotifyBuffer::new(),
|
||||
notification_coalesce_delay: DEFAULT_NOTIFICATION_COALESCE_DELAY,
|
||||
pending_attachments: Arc::new(Mutex::new(Vec::<SystemItem>::new())),
|
||||
pending_committed_history: Arc::new(Mutex::new(std::collections::VecDeque::new())),
|
||||
failed_activation_authority: Mutex::new(None),
|
||||
scope_allocation: Some(scope_allocation),
|
||||
callback_socket: Some(callback_socket),
|
||||
runtime_ticket_role: None,
|
||||
prompts: common.prompts,
|
||||
inject_resident_summary: true,
|
||||
feature_resident_summary_source: None,
|
||||
session_context_source: None,
|
||||
prepared_initial_seed: None,
|
||||
prepared_initial_prompt_completion: None,
|
||||
subjektiv_host: None,
|
||||
feature_installation: crate::feature::FeatureInstallation::Activate,
|
||||
restore_feature_lifecycle_pending: false,
|
||||
feature_system_prompt_override: None,
|
||||
user_segments: Vec::new(),
|
||||
sink: SegmentLogSink::new(),
|
||||
history_persistence_wired: false,
|
||||
log_writer: None,
|
||||
notification_committer: None,
|
||||
};
|
||||
worker.apply_permissions_from_manifest();
|
||||
worker.apply_prune_from_manifest();
|
||||
worker.write_worker_metadata_pending()?;
|
||||
Ok(worker)
|
||||
}
|
||||
|
||||
/// Restore a Worker by resolving its name-keyed metadata to an active
|
||||
/// `(SessionId, SegmentId)` and then using the normal session-log restore
|
||||
/// path. The metadata stores only the active pointer; lineage and origin
|
||||
@@ -9271,14 +9057,10 @@ where
|
||||
// the lookup + install inside a single `LockFileGuard` is what
|
||||
// makes "no two live Workers write to the same session log"
|
||||
// actually structural rather than a hopeful pre-check.
|
||||
let socket_path = dir::default_base()
|
||||
.map_err(ScopeLockError::from)?
|
||||
.join(&manifest.worker.name)
|
||||
.join("sock");
|
||||
let scope_allocation = worker_allocation::install_top_level_with_deny(
|
||||
manifest.worker.name.clone(),
|
||||
std::process::id(),
|
||||
socket_path,
|
||||
PathBuf::new(), // Reserved legacy allocation field; no Worker endpoint exists.
|
||||
common.scope.allow_rules(),
|
||||
common.scope.deny_rules(),
|
||||
segment_id,
|
||||
@@ -9326,7 +9108,7 @@ where
|
||||
let worker_metadata_segment_cas = Some(worker_metadata_segment_cas_for_store(&store));
|
||||
let scope = SharedScope::new(common.scope);
|
||||
let workdir_sessions =
|
||||
workdir_sessions_from_authority(&common.filesystem_authority, &scope);
|
||||
workdir_sessions_from_authority(&common.filesystem_authority, &scope)?;
|
||||
|
||||
let mut worker = Self {
|
||||
backend_job: std::sync::OnceLock::new(),
|
||||
@@ -9376,7 +9158,6 @@ where
|
||||
pending_committed_history: Arc::new(Mutex::new(std::collections::VecDeque::new())),
|
||||
failed_activation_authority: Mutex::new(None),
|
||||
scope_allocation: Some(scope_allocation),
|
||||
callback_socket: None,
|
||||
runtime_ticket_role: None,
|
||||
prompts: common.prompts,
|
||||
inject_resident_summary: true,
|
||||
@@ -10438,9 +10219,25 @@ pub enum WorkerError {
|
||||
fn workdir_sessions_from_authority(
|
||||
authority: &WorkerFilesystemAuthority,
|
||||
scope: &SharedScope,
|
||||
) -> Arc<WorkdirSessionRouter> {
|
||||
) -> Result<Arc<WorkdirSessionRouter>, WorkerError> {
|
||||
let router = Arc::new(WorkdirSessionRouter::new());
|
||||
if let Some(local) = authority.as_local() {
|
||||
let invalid_cwd = |error: String| WorkerError::InvalidCwd {
|
||||
cwd: local.cwd.clone(),
|
||||
source: std::io::Error::new(std::io::ErrorKind::InvalidInput, error),
|
||||
};
|
||||
let relative_cwd = local.cwd.strip_prefix(&local.root).map_err(|error| {
|
||||
invalid_cwd(format!(
|
||||
"session cwd must be within its Workdir root: {error}"
|
||||
))
|
||||
})?;
|
||||
let relative_cwd = relative_cwd
|
||||
.to_str()
|
||||
.ok_or_else(|| invalid_cwd("session cwd must be UTF-8".into()))?;
|
||||
let relative_cwd = workdir::WorkdirPath::new(relative_cwd)
|
||||
.map_err(|error| invalid_cwd(error.to_string()))?;
|
||||
// The provider keeps root-coordinate operations. Only the routed tool
|
||||
// view uses cwd; delegation reuses the same root-coordinate broker.
|
||||
let session = Arc::new(LocalWorkdirSession::materialized(
|
||||
local.root.clone(),
|
||||
local.cwd.clone(),
|
||||
@@ -10448,13 +10245,14 @@ fn workdir_sessions_from_authority(
|
||||
WorkdirSessionCapabilities::ALL,
|
||||
)) as WorkdirSessionHandle;
|
||||
router
|
||||
.attach(
|
||||
.attach_at_cwd(
|
||||
WorkdirAttachmentAlias::new("workdir").expect("static alias is valid"),
|
||||
session,
|
||||
relative_cwd,
|
||||
)
|
||||
.expect("fresh Workdir router accepts its only attachment");
|
||||
.map_err(|error| invalid_cwd(error.to_string()))?;
|
||||
}
|
||||
router
|
||||
Ok(router)
|
||||
}
|
||||
|
||||
/// Bundle of resources that every high-level Worker constructor needs:
|
||||
@@ -10472,6 +10270,8 @@ struct WorkerCommon {
|
||||
feature_instructions: Vec<FeatureInstructionDeclaration>,
|
||||
}
|
||||
|
||||
/// Migration-only safety probe for retained process-child metadata.
|
||||
/// New Internal SubWorkers never publish socket metadata or use this path.
|
||||
async fn restored_child_reachable(child: &WorkerSpawnedChild) -> bool {
|
||||
tokio::time::timeout(
|
||||
RESTORE_RECONCILIATION_REACHABILITY_TIMEOUT,
|
||||
@@ -18018,7 +17818,6 @@ permission = "write"
|
||||
WorkerWorkspaceContext::no_workspace(),
|
||||
WorkerFilesystemAuthority::None,
|
||||
layout,
|
||||
crate::WorkerControllerTransport::InProcess,
|
||||
)
|
||||
.with_model_client(NoopClient)
|
||||
.with_subjektiv_host(host)
|
||||
@@ -18338,7 +18137,6 @@ permission = "write"
|
||||
.join("runtime/runs")
|
||||
.join(uuid::Uuid::now_v7().to_string()),
|
||||
},
|
||||
crate::WorkerControllerTransport::InProcess,
|
||||
)
|
||||
.with_model_client(NoopClient)
|
||||
}
|
||||
|
||||
@@ -0,0 +1,33 @@
|
||||
use std::io::{self, Write};
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
/// Per-test warning capture; never installs a global subscriber or writes fixture secrets.
|
||||
#[derive(Clone, Default)]
|
||||
pub struct CapturedWarnings(Arc<Mutex<Vec<u8>>>);
|
||||
|
||||
impl CapturedWarnings {
|
||||
pub fn subscriber(&self) -> impl tracing::Subscriber + Send + Sync + 'static {
|
||||
let output = self.clone();
|
||||
tracing_subscriber::fmt()
|
||||
.with_writer(move || output.clone())
|
||||
.without_time()
|
||||
.with_ansi(false)
|
||||
.with_max_level(tracing::Level::WARN)
|
||||
.finish()
|
||||
}
|
||||
|
||||
pub fn text(&self) -> String {
|
||||
String::from_utf8(self.0.lock().unwrap().clone()).unwrap()
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for CapturedWarnings {
|
||||
fn write(&mut self, bytes: &[u8]) -> io::Result<usize> {
|
||||
self.0.lock().unwrap().extend_from_slice(bytes);
|
||||
Ok(bytes.len())
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -26,6 +26,9 @@ use tokio::sync::broadcast;
|
||||
|
||||
use worker::{Worker, WorkerController, WorkerError};
|
||||
|
||||
#[path = "common/diagnostics.rs"]
|
||||
mod diagnostics;
|
||||
|
||||
type TestStore = CombinedStore<FsStore, FsWorkerStore>;
|
||||
|
||||
#[derive(Clone)]
|
||||
@@ -2876,7 +2879,7 @@ const COMPACTION_HOST_WORKSPACE_ID: &str = "journal-transport-host-workspace";
|
||||
// Host authorization/selection or live-model semantic continuation quality.
|
||||
struct CompactionJournalHost {
|
||||
granted: bool,
|
||||
unavailable: bool,
|
||||
failure: Option<&'static str>,
|
||||
snapshot: Mutex<subjektiv::journal::startup::JournalStartupSnapshot>,
|
||||
reads: Mutex<Vec<worker::subjektiv::SubjektivHostContext>>,
|
||||
startup_reads: AtomicUsize,
|
||||
@@ -2886,7 +2889,7 @@ impl CompactionJournalHost {
|
||||
fn new(snapshot: subjektiv::journal::startup::JournalStartupSnapshot) -> Self {
|
||||
Self {
|
||||
granted: true,
|
||||
unavailable: false,
|
||||
failure: None,
|
||||
snapshot: Mutex::new(snapshot),
|
||||
reads: Mutex::new(vec![]),
|
||||
startup_reads: AtomicUsize::new(0),
|
||||
@@ -2917,10 +2920,18 @@ impl worker::subjektiv::SubjektivHost for CompactionJournalHost {
|
||||
worker::subjektiv::SubjektivHostError,
|
||||
> {
|
||||
self.reads.lock().unwrap().push(context.clone());
|
||||
if self.unavailable {
|
||||
return Err(worker::subjektiv::SubjektivHostError::Unavailable(
|
||||
"injected Journal read failure".into(),
|
||||
));
|
||||
if let Some(failure) = self.failure {
|
||||
use worker::subjektiv::SubjektivHostError;
|
||||
let secret = "PRIVATE_JOURNAL_BODY /private/host/path proof=PRIVATE_PROOF credential=PRIVATE_CREDENTIAL raw_error=PRIVATE_ERROR";
|
||||
return Err(match failure {
|
||||
"unavailable" => SubjektivHostError::Unavailable(secret.into()),
|
||||
"invalid-argument" => SubjektivHostError::InvalidArgument(secret.into()),
|
||||
"conflict" => SubjektivHostError::Conflict {
|
||||
code: secret.into(),
|
||||
message: secret.into(),
|
||||
},
|
||||
_ => panic!("unknown failure fixture"),
|
||||
});
|
||||
}
|
||||
Ok(self.snapshot.lock().unwrap().clone())
|
||||
}
|
||||
@@ -3466,26 +3477,37 @@ async fn journal_three_compactions_freeze_one_snapshot_preserve_full_summary_tai
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn journal_absent_disabled_unavailable_and_over_byte_budget_keep_conversation_compaction() {
|
||||
for case in [
|
||||
"absent",
|
||||
"disabled",
|
||||
"unavailable",
|
||||
"over-byte-budget",
|
||||
"no-host",
|
||||
] {
|
||||
async fn journal_empty_disabled_and_available_compaction_preserve_inputs_without_failure_warnings()
|
||||
{
|
||||
assert_journal_compaction_cases(&["absent", "available", "disabled", "no-host"]).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn journal_snapshot_failures_keep_safe_diagnostics_and_continue_compaction() {
|
||||
assert_journal_compaction_cases(&["unavailable", "invalid-argument", "conflict"]).await;
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn journal_over_byte_budget_keeps_safe_diagnostic_and_continues_compaction() {
|
||||
assert_journal_compaction_cases(&["over-byte-budget"]).await;
|
||||
}
|
||||
|
||||
async fn assert_journal_compaction_cases(cases: &[&'static str]) {
|
||||
for &case in cases {
|
||||
let mut snapshot = compaction_snapshot(0);
|
||||
if case == "absent" {
|
||||
snapshot.journals.clear();
|
||||
}
|
||||
if case == "over-byte-budget" {
|
||||
snapshot.journals[0].current_position =
|
||||
"x".repeat(subjektiv::journal::startup::STARTUP_MAX_BYTES);
|
||||
"PRIVATE_JOURNAL_BODY".repeat(subjektiv::journal::startup::STARTUP_MAX_BYTES);
|
||||
}
|
||||
let expected_json = (case == "absent").then(|| serde_json::to_string(&snapshot).unwrap());
|
||||
let expected_json = matches!(case, "absent" | "available")
|
||||
.then(|| serde_json::to_string(&snapshot).unwrap());
|
||||
let mut host = CompactionJournalHost::new(snapshot);
|
||||
host.granted = case != "disabled";
|
||||
host.unavailable = case == "unavailable";
|
||||
host.failure =
|
||||
matches!(case, "unavailable" | "invalid-argument" | "conflict").then_some(case);
|
||||
let host = Arc::new(host);
|
||||
let client = MockClient::new(vec![
|
||||
single_text_events("seed"),
|
||||
@@ -3517,7 +3539,51 @@ async fn journal_absent_disabled_unavailable_and_over_byte_budget_keep_conversat
|
||||
true,
|
||||
)
|
||||
.await;
|
||||
fixture.worker.compact(0).await.unwrap();
|
||||
// Scope the subscriber to every poll, not a thread-local guard across await.
|
||||
// This observes the real Feature warning while compacting into FsStore.
|
||||
let logs = diagnostics::CapturedWarnings::default();
|
||||
use tracing::instrument::WithSubscriber;
|
||||
fixture
|
||||
.worker
|
||||
.compact(0)
|
||||
.with_subscriber(logs.subscriber())
|
||||
.await
|
||||
.unwrap();
|
||||
let log = logs.text();
|
||||
let failure = match case {
|
||||
"unavailable" => Some(("snapshot", "host_unavailable")),
|
||||
"invalid-argument" => Some(("snapshot", "host_invalid_argument")),
|
||||
"conflict" => Some(("snapshot", "host_conflict")),
|
||||
"over-byte-budget" => Some(("byte_budget", "startup_max_bytes_exceeded")),
|
||||
_ => None,
|
||||
};
|
||||
if let Some((stage, reason)) = failure {
|
||||
assert!(log.contains(&format!("stage=\"{stage}\"")), "{case}: {log}");
|
||||
assert!(
|
||||
log.contains(&format!("reason=\"{reason}\"")),
|
||||
"{case}: {log}"
|
||||
);
|
||||
assert_eq!(
|
||||
log.matches("continuing without Journal").count(),
|
||||
1,
|
||||
"{case}"
|
||||
);
|
||||
} else {
|
||||
assert!(
|
||||
!log.contains("Journal compaction snapshot unavailable"),
|
||||
"{case}: {log}"
|
||||
);
|
||||
}
|
||||
for secret in [
|
||||
"PRIVATE_",
|
||||
"/private/host/path",
|
||||
"JOURNAL_POSITION_0",
|
||||
"JOURNAL_BODY_0",
|
||||
"state-token-0",
|
||||
"fixed-event-reference-0",
|
||||
] {
|
||||
assert!(!log.contains(secret), "{case}: diagnostic leaked {secret}");
|
||||
}
|
||||
let input = first_compactor_input(&observed.requests()[1]);
|
||||
let body = if matches!(case, "disabled" | "no-host") {
|
||||
assert_eq!(
|
||||
@@ -3532,8 +3598,8 @@ async fn journal_absent_disabled_unavailable_and_over_byte_budget_keep_conversat
|
||||
String::new()
|
||||
} else {
|
||||
let body = journal_body(&fixture.worker);
|
||||
let expected = if case == "absent" {
|
||||
"absent"
|
||||
let expected = if matches!(case, "absent" | "available") {
|
||||
case
|
||||
} else {
|
||||
"unavailable"
|
||||
};
|
||||
@@ -3548,6 +3614,71 @@ async fn journal_absent_disabled_unavailable_and_over_byte_budget_keep_conversat
|
||||
&fixture.worker.prompts().load_full(),
|
||||
Some(COMPACTION_HOST_WORKSPACE_ID),
|
||||
);
|
||||
let seed = replacement_seed(&fixture.worker);
|
||||
let metadata = seed[1].metadata.input_metadata.as_ref().unwrap();
|
||||
assert_eq!(
|
||||
metadata.usage().recording_material,
|
||||
session_store::InputEvidenceUse::Exclude
|
||||
);
|
||||
assert_eq!(
|
||||
metadata.usage().memory_evidence,
|
||||
session_store::InputEvidenceUse::Exclude
|
||||
);
|
||||
assert_eq!(
|
||||
metadata.usage().observation,
|
||||
session_store::InputObservationUse::Hidden
|
||||
);
|
||||
let payload: serde_json::Value = metadata
|
||||
.feature_payload()
|
||||
.unwrap()
|
||||
.decode("journal:compaction", 1)
|
||||
.unwrap();
|
||||
assert_eq!(payload["status"], expected);
|
||||
assert_eq!(
|
||||
payload["prompt_provenance"]["workspace_id"],
|
||||
COMPACTION_HOST_WORKSPACE_ID
|
||||
);
|
||||
let projection = fixture.worker.prompts().load_full().projection().clone();
|
||||
assert_eq!(
|
||||
payload["prompt_provenance"]["source_digest"],
|
||||
projection.source_digest
|
||||
);
|
||||
assert_eq!(
|
||||
payload["prompt_provenance"]["projection_digest"],
|
||||
projection.catalog_digest
|
||||
);
|
||||
if let Some(json) = expected_json.as_deref() {
|
||||
use sha2::{Digest, Sha256};
|
||||
assert_eq!(
|
||||
payload["snapshot"],
|
||||
serde_json::from_str::<serde_json::Value>(json).unwrap()
|
||||
);
|
||||
assert_eq!(
|
||||
payload["snapshot_sha256"],
|
||||
Sha256::digest(json.as_bytes())
|
||||
.iter()
|
||||
.map(|byte| format!("{byte:02x}"))
|
||||
.collect::<String>()
|
||||
);
|
||||
} else {
|
||||
assert!(payload["snapshot"].is_null());
|
||||
assert!(payload["snapshot_sha256"].is_null());
|
||||
}
|
||||
// Feature diagnostics do not become model input or a new metadata contract.
|
||||
let saved = serde_json::to_string(&seed[1]).unwrap();
|
||||
for forbidden in [
|
||||
"PRIVATE_",
|
||||
"/private/host/path",
|
||||
"host_unavailable",
|
||||
"host_conflict",
|
||||
"host_invalid_argument",
|
||||
"startup_max_bytes_exceeded",
|
||||
] {
|
||||
assert!(
|
||||
!saved.contains(forbidden),
|
||||
"{case}: saved input leaked {forbidden}"
|
||||
);
|
||||
}
|
||||
body
|
||||
};
|
||||
assert_full_initial_summary(&input, &inherited);
|
||||
@@ -3559,8 +3690,8 @@ async fn journal_absent_disabled_unavailable_and_over_byte_budget_keep_conversat
|
||||
.is_some_and(|text| text.contains("PREVIOUS_JOURNAL_DELEGATED_CONTEXT"))),
|
||||
"fallback retains only the newly frozen reference, not an old snapshot as latest"
|
||||
);
|
||||
assert!(!body.contains("JOURNAL_POSITION_0"));
|
||||
assert!(!body.contains("journal-reference-0"));
|
||||
assert_eq!(body.contains("JOURNAL_POSITION_0"), case == "available");
|
||||
assert_eq!(body.contains("journal-reference-0"), case == "available");
|
||||
assert_eq!(
|
||||
host.reads.lock().unwrap().len(),
|
||||
usize::from(!matches!(case, "disabled" | "no-host")),
|
||||
|
||||
@@ -1738,42 +1738,24 @@ async fn snapshot_includes_user_input_for_in_flight_turn() {
|
||||
// The Running event is the in-flight visibility fence: the committed
|
||||
// annotated input must already be available to an immediately attaching
|
||||
// subscriber rather than racing behind this status transition.
|
||||
let stream = tokio::net::UnixStream::connect(handle.runtime_dir.socket_path())
|
||||
.await
|
||||
.unwrap();
|
||||
let (reader, _writer) = stream.into_split();
|
||||
let mut reader = protocol::stream::JsonLineReader::new(reader);
|
||||
|
||||
loop {
|
||||
let event = reader.next::<Event>().await.unwrap().unwrap();
|
||||
match event {
|
||||
Event::Snapshot { session, .. } => {
|
||||
let found = session.entries.iter().any(|entry| match &entry.data {
|
||||
protocol::SessionSnapshotEntryData::UserInput { segments } => {
|
||||
protocol::Segment::flatten_to_text(segments) == "hello in-flight"
|
||||
}
|
||||
protocol::SessionSnapshotEntryData::Message {
|
||||
role: protocol::SessionMessageRole::User,
|
||||
content,
|
||||
} => content.iter().any(|part| {
|
||||
matches!(
|
||||
part,
|
||||
protocol::SessionContentPart::Text { text }
|
||||
if text == "hello in-flight"
|
||||
)
|
||||
}),
|
||||
_ => false,
|
||||
});
|
||||
assert!(
|
||||
found,
|
||||
"snapshot must carry the in-flight UserInput entry: {session:?}"
|
||||
);
|
||||
return;
|
||||
let Event::Snapshot { session, .. } = handle.snapshot_event() else {
|
||||
panic!("expected snapshot");
|
||||
};
|
||||
assert!(
|
||||
session.entries.iter().any(|entry| match &entry.data {
|
||||
protocol::SessionSnapshotEntryData::UserInput { segments } => {
|
||||
protocol::Segment::flatten_to_text(segments) == "hello in-flight"
|
||||
}
|
||||
Event::Alert(_) => continue,
|
||||
other => panic!("expected Snapshot first, got {other:?}"),
|
||||
}
|
||||
}
|
||||
protocol::SessionSnapshotEntryData::Message {
|
||||
role: protocol::SessionMessageRole::User,
|
||||
content,
|
||||
} => content.iter().any(|part| matches!(part,
|
||||
protocol::SessionContentPart::Text { text } if text == "hello in-flight"
|
||||
)),
|
||||
_ => false,
|
||||
}),
|
||||
"snapshot must carry the in-flight UserInput entry: {session:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -1855,24 +1837,10 @@ async fn attach_snapshot_includes_current_status() {
|
||||
.unwrap();
|
||||
wait_for_status(&handle, WorkerStatus::Running).await;
|
||||
|
||||
let stream = tokio::net::UnixStream::connect(handle.runtime_dir.socket_path())
|
||||
.await
|
||||
.unwrap();
|
||||
let (reader, _writer) = stream.into_split();
|
||||
let mut reader = protocol::stream::JsonLineReader::new(reader);
|
||||
|
||||
// First event after connect is the snapshot — it carries the current status.
|
||||
loop {
|
||||
let event = reader.next::<Event>().await.unwrap().unwrap();
|
||||
match event {
|
||||
Event::Snapshot { state, .. } => {
|
||||
assert_eq!(state.catalog_status(), WorkerStatus::Running);
|
||||
return;
|
||||
}
|
||||
Event::Alert(_) => continue,
|
||||
other => panic!("expected Snapshot, got {other:?}"),
|
||||
}
|
||||
}
|
||||
let Event::Snapshot { state, .. } = handle.snapshot_event() else {
|
||||
panic!("expected snapshot");
|
||||
};
|
||||
assert_eq!(state.catalog_status(), WorkerStatus::Running);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -2928,7 +2896,7 @@ async fn status_json_reflects_worker_name() {
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Socket transport tests
|
||||
// Controller lifecycle tests
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
#[tokio::test]
|
||||
@@ -2966,283 +2934,148 @@ async fn shutdown_closes_method_admission_before_terminal_confirmation() {
|
||||
shutdown_rx.await.unwrap();
|
||||
}
|
||||
|
||||
/// Uses the real Controller and metadata store; only the model client is replaced.
|
||||
#[tokio::test]
|
||||
async fn shutdown_joins_socket_server_with_active_connection() {
|
||||
use tokio::net::UnixStream;
|
||||
|
||||
async fn register_peer_returns_existing_response_and_persists_reciprocal_metadata() {
|
||||
use session_store::{WorkerMetadata, WorkerMetadataStore};
|
||||
let worker = make_worker(MockClient::new(simple_text_events())).await;
|
||||
let runtime_base = tempfile::tempdir().unwrap();
|
||||
let (handle, shutdown_rx) = WorkerController::spawn(worker, runtime_base.path())
|
||||
.await
|
||||
let store = worker.store().clone();
|
||||
store
|
||||
.write(&WorkerMetadata::new("test-worker", None))
|
||||
.unwrap();
|
||||
let socket_path = handle.runtime_dir.socket_path();
|
||||
let _connection = UnixStream::connect(&socket_path).await.unwrap();
|
||||
|
||||
store
|
||||
.write(&WorkerMetadata::new("peer-worker", None))
|
||||
.unwrap();
|
||||
let handle = spawn_controller(worker).await;
|
||||
let mut events = handle.subscribe();
|
||||
handle
|
||||
.send(Method::Shutdown {
|
||||
command: worker_command(&handle),
|
||||
.send(Method::RegisterPeer {
|
||||
name: "peer-worker".into(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), shutdown_rx)
|
||||
assert!(
|
||||
drain_until(
|
||||
&mut events,
|
||||
std::time::Duration::from_secs(2),
|
||||
|event| matches!(
|
||||
event,
|
||||
Event::PeerRegistered { result } if result == &serde_json::json!({
|
||||
"source": "test-worker", "peer": "peer-worker",
|
||||
})
|
||||
)
|
||||
)
|
||||
.await
|
||||
.expect("controller should join its socket tasks")
|
||||
.expect("controller shutdown signal should remain open");
|
||||
assert!(!socket_path.exists());
|
||||
);
|
||||
assert_eq!(
|
||||
store.read_by_name("test-worker").unwrap().unwrap().peers[0].worker_name,
|
||||
"peer-worker"
|
||||
);
|
||||
assert_eq!(
|
||||
store.read_by_name("peer-worker").unwrap().unwrap().peers[0].worker_name,
|
||||
"test-worker"
|
||||
);
|
||||
assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn socket_run_receives_events() {
|
||||
use protocol::stream::{JsonLineReader, JsonLineWriter};
|
||||
use tokio::net::UnixStream;
|
||||
|
||||
let client = MockClient::new(simple_text_events());
|
||||
let worker = make_worker(client).await;
|
||||
async fn register_peer_missing_target_is_rejected_without_starting_a_worker() {
|
||||
use session_store::{WorkerMetadata, WorkerMetadataStore};
|
||||
let worker = make_worker(MockClient::new(simple_text_events())).await;
|
||||
let store = worker.store().clone();
|
||||
store
|
||||
.write(&WorkerMetadata::new("test-worker", None))
|
||||
.unwrap();
|
||||
let handle = spawn_controller(worker).await;
|
||||
|
||||
// Give the socket server a moment to bind
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
|
||||
let sock_path = handle.runtime_dir.socket_path();
|
||||
let stream = UnixStream::connect(&sock_path).await.unwrap();
|
||||
let (reader, writer) = stream.into_split();
|
||||
let mut reader = JsonLineReader::new(reader);
|
||||
let mut writer = JsonLineWriter::new(writer);
|
||||
|
||||
// Send run method via socket
|
||||
writer
|
||||
.write(&Method::submit_text(
|
||||
protocol::new_submission_request_id(),
|
||||
"Hello",
|
||||
))
|
||||
let mut events = handle.subscribe();
|
||||
handle
|
||||
.send(Method::RegisterPeer {
|
||||
name: "missing-peer".into(),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Collect events
|
||||
let mut saw_turn_start = false;
|
||||
let mut saw_text_delta = false;
|
||||
let mut saw_turn_end = false;
|
||||
|
||||
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
|
||||
loop {
|
||||
tokio::select! {
|
||||
event = reader.next::<Event>() => {
|
||||
match event {
|
||||
Ok(Some(Event::TurnStart { .. })) => saw_turn_start = true,
|
||||
Ok(Some(Event::TextDelta { .. })) => saw_text_delta = true,
|
||||
Ok(Some(Event::TurnEnd { .. })) => {
|
||||
saw_turn_end = true;
|
||||
break;
|
||||
}
|
||||
Ok(None) | Err(_) => break,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
_ = tokio::time::sleep_until(deadline) => break,
|
||||
}
|
||||
}
|
||||
|
||||
assert!(saw_turn_start, "should see turn_start via socket");
|
||||
assert!(saw_text_delta, "should see text_delta via socket");
|
||||
assert!(saw_turn_end, "should see turn_end via socket");
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn socket_worker_event_turn_ended_while_idle_auto_starts_turn() {
|
||||
use protocol::stream::{JsonLineReader, JsonLineWriter};
|
||||
use tokio::net::UnixStream;
|
||||
|
||||
let client = MockClient::new(simple_text_events());
|
||||
let worker = make_worker(client).await;
|
||||
let handle = spawn_controller(worker).await;
|
||||
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
|
||||
let sock_path = handle.runtime_dir.socket_path();
|
||||
let stream = UnixStream::connect(&sock_path).await.unwrap();
|
||||
let (reader, writer) = stream.into_split();
|
||||
let mut reader = JsonLineReader::new(reader);
|
||||
let mut writer = JsonLineWriter::new(writer);
|
||||
|
||||
writer
|
||||
.write(&Method::WorkerEvent(protocol::WorkerEvent::TurnEnded {
|
||||
worker_name: "child".into(),
|
||||
}))
|
||||
assert!(
|
||||
drain_until(
|
||||
&mut events,
|
||||
std::time::Duration::from_secs(2),
|
||||
|event| matches!(
|
||||
event,
|
||||
Event::Error { code: worker::ErrorCode::InvalidRequest, message }
|
||||
if message.contains("missing-peer") && message.contains("does not exist")
|
||||
)
|
||||
)
|
||||
.await
|
||||
);
|
||||
assert!(store.read_by_name("missing-peer").unwrap().is_none());
|
||||
assert!(
|
||||
store
|
||||
.read_by_name("test-worker")
|
||||
.unwrap()
|
||||
.unwrap()
|
||||
.peers
|
||||
.is_empty()
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_process_discovery_methods_require_host_or_runtime_authority() {
|
||||
let worker = make_worker(MockClient::new(simple_text_events())).await;
|
||||
let handle = spawn_controller(worker).await;
|
||||
let mut events = handle.subscribe();
|
||||
for method in [
|
||||
Method::ListWorkers,
|
||||
Method::RestoreWorker {
|
||||
name: "old-worker".into(),
|
||||
},
|
||||
] {
|
||||
handle.send(method).await.unwrap();
|
||||
assert!(
|
||||
drain_until(
|
||||
&mut events,
|
||||
std::time::Duration::from_secs(2),
|
||||
|event| matches!(
|
||||
event,
|
||||
Event::Error { code: worker::ErrorCode::InvalidRequest, message }
|
||||
if message.contains("Host or Runtime")
|
||||
)
|
||||
)
|
||||
.await
|
||||
);
|
||||
}
|
||||
assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn all_controller_launch_modes_leave_external_transport_to_the_host() {
|
||||
let root = tempfile::tempdir().unwrap();
|
||||
for mode in ["direct", "runtime", "run"] {
|
||||
let worker = make_worker(MockClient::new(simple_text_events())).await;
|
||||
let base = root.path().join(mode);
|
||||
let (handle, shutdown) = match mode {
|
||||
"direct" => WorkerController::spawn(worker, &base).await,
|
||||
"runtime" => WorkerController::spawn_runtime_managed(worker, &base).await,
|
||||
"run" => {
|
||||
WorkerController::spawn_runtime_managed_run(worker, &base.join("runs/1")).await
|
||||
}
|
||||
_ => unreachable!(),
|
||||
}
|
||||
.unwrap();
|
||||
|
||||
let mut saw_worker_event_echo = false;
|
||||
let mut saw_turn_start = false;
|
||||
let mut saw_turn_end = false;
|
||||
|
||||
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2);
|
||||
// The SystemItem and TurnEnd events arrive through independent
|
||||
// broadcast lanes (sink fan-out vs `working_event_tx`), so their relative
|
||||
// order on the wire is non-deterministic. Keep reading until both
|
||||
// are observed (or the deadline trips), rather than breaking on
|
||||
// the first TurnEnd.
|
||||
loop {
|
||||
if saw_worker_event_echo && saw_turn_end {
|
||||
break;
|
||||
}
|
||||
tokio::select! {
|
||||
event = reader.next::<Event>() => {
|
||||
match event {
|
||||
Ok(Some(Event::SystemItem { ref item, .. }))
|
||||
if item.get("kind").and_then(|k| k.as_str()) == Some("worker_event")
|
||||
&& item
|
||||
.pointer("/presentation/event/worker_name")
|
||||
.and_then(|v| v.as_str()) == Some("child") =>
|
||||
{
|
||||
saw_worker_event_echo = true;
|
||||
}
|
||||
Ok(Some(Event::TurnStart { .. })) => saw_turn_start = true,
|
||||
Ok(Some(Event::TurnEnd { .. })) => {
|
||||
saw_turn_end = true;
|
||||
}
|
||||
Ok(None) | Err(_) => break,
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
_ = tokio::time::sleep_until(deadline) => break,
|
||||
}
|
||||
}
|
||||
|
||||
assert!(
|
||||
saw_worker_event_echo,
|
||||
"WorkerEvent::TurnEnded via socket should be echoed as Event::SystemItem(WorkerEvent)"
|
||||
);
|
||||
assert!(
|
||||
saw_turn_start,
|
||||
"WorkerEvent::TurnEnded via socket should auto-start a turn"
|
||||
);
|
||||
assert!(
|
||||
saw_turn_end,
|
||||
"auto-triggered turn should reach turn_end via socket"
|
||||
);
|
||||
}
|
||||
|
||||
async fn socket_error_after_method_line(
|
||||
handle: &WorkerHandle,
|
||||
line: &[u8],
|
||||
) -> (worker::ErrorCode, String) {
|
||||
use protocol::stream::JsonLineReader;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use tokio::net::UnixStream;
|
||||
|
||||
let sock_path = handle.runtime_dir.socket_path();
|
||||
let stream = UnixStream::connect(&sock_path).await.unwrap();
|
||||
let (reader, mut writer) = stream.into_split();
|
||||
let mut reader = JsonLineReader::new(reader);
|
||||
|
||||
writer.write_all(line).await.unwrap();
|
||||
|
||||
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(1);
|
||||
loop {
|
||||
tokio::select! {
|
||||
event = reader.next::<Event>() => {
|
||||
match event {
|
||||
Ok(Some(Event::Error { code, message })) => return (code, message),
|
||||
Ok(Some(_)) => {}
|
||||
Ok(None) => panic!("socket closed before invalid-method error"),
|
||||
Err(e) => panic!("socket read failed before invalid-method error: {e}"),
|
||||
}
|
||||
}
|
||||
_ = tokio::time::sleep_until(deadline) => {
|
||||
panic!("timed out waiting for invalid-method error")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn socket_schema_invalid_method_returns_error() {
|
||||
let client = MockClient::new(simple_text_events());
|
||||
let worker = make_worker(client).await;
|
||||
let handle = spawn_controller(worker).await;
|
||||
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
|
||||
let (code, message) = socket_error_after_method_line(&handle, b"{\"bad\":\"json\"}\n").await;
|
||||
|
||||
assert_eq!(code, worker::ErrorCode::InvalidRequest);
|
||||
assert!(
|
||||
message.contains("invalid method"),
|
||||
"expected invalid-method diagnostic, got: {message}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn socket_malformed_method_returns_error() {
|
||||
let client = MockClient::new(simple_text_events());
|
||||
let worker = make_worker(client).await;
|
||||
let handle = spawn_controller(worker).await;
|
||||
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
|
||||
let (code, message) = socket_error_after_method_line(&handle, b"{not-json}\n").await;
|
||||
|
||||
assert_eq!(code, worker::ErrorCode::InvalidRequest);
|
||||
assert!(
|
||||
message.contains("invalid method"),
|
||||
"expected invalid-method diagnostic, got: {message}"
|
||||
);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn socket_peer_close_without_method_does_not_broadcast_error() {
|
||||
use protocol::stream::JsonLineReader;
|
||||
use tokio::net::UnixStream;
|
||||
|
||||
let client = MockClient::new(simple_text_events());
|
||||
let worker = make_worker(client).await;
|
||||
let handle = spawn_controller(worker).await;
|
||||
|
||||
tokio::time::sleep(std::time::Duration::from_millis(50)).await;
|
||||
|
||||
let mut broadcast_rx = handle.subscribe();
|
||||
let sock_path = handle.runtime_dir.socket_path();
|
||||
let stream = UnixStream::connect(&sock_path).await.unwrap();
|
||||
let (reader, writer) = stream.into_split();
|
||||
let mut reader = JsonLineReader::new(reader);
|
||||
|
||||
let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(1);
|
||||
loop {
|
||||
tokio::select! {
|
||||
event = reader.next::<Event>() => {
|
||||
match event {
|
||||
Ok(Some(Event::Snapshot { .. })) => break,
|
||||
Ok(Some(_)) => {}
|
||||
Ok(None) => panic!("socket closed before connect-time snapshot"),
|
||||
Err(e) => panic!("socket read failed before connect-time snapshot: {e}"),
|
||||
}
|
||||
}
|
||||
_ = tokio::time::sleep_until(deadline) => {
|
||||
panic!("timed out waiting for connect-time snapshot")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
drop(writer);
|
||||
drop(reader);
|
||||
|
||||
let deadline = tokio::time::Instant::now() + std::time::Duration::from_millis(200);
|
||||
loop {
|
||||
tokio::select! {
|
||||
event = broadcast_rx.recv() => {
|
||||
match event {
|
||||
Ok(Event::Error { code, message }) => {
|
||||
panic!("peer close without Method broadcast error {code:?}: {message}")
|
||||
}
|
||||
Ok(_) => {}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
|
||||
panic!("broadcast receiver lagged while checking peer close: {n}")
|
||||
}
|
||||
Err(tokio::sync::broadcast::error::RecvError::Closed) => break,
|
||||
}
|
||||
}
|
||||
_ = tokio::time::sleep_until(deadline) => break,
|
||||
for legacy in ["pid", "status.json", "manifest.toml", "sock", "worker.sock"] {
|
||||
assert!(
|
||||
!handle.runtime_dir.path().join(legacy).exists(),
|
||||
"{mode} created {legacy}"
|
||||
);
|
||||
}
|
||||
handle
|
||||
.send(Method::Shutdown {
|
||||
command: worker_command(&handle),
|
||||
})
|
||||
.await
|
||||
.unwrap();
|
||||
tokio::time::timeout(std::time::Duration::from_secs(5), shutdown)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4006,27 +3839,15 @@ fn item_text_contains(item: &Item, needle: &str) -> bool {
|
||||
}
|
||||
|
||||
async fn snapshot_contains_user_input(handle: &WorkerHandle, needle: &str) -> bool {
|
||||
let stream = tokio::net::UnixStream::connect(handle.runtime_dir.socket_path())
|
||||
.await
|
||||
.unwrap();
|
||||
let (reader, _writer) = stream.into_split();
|
||||
let mut reader = protocol::stream::JsonLineReader::new(reader);
|
||||
|
||||
loop {
|
||||
let event = reader.next::<Event>().await.unwrap().unwrap();
|
||||
match event {
|
||||
Event::Snapshot { session, .. } => {
|
||||
return session.entries.into_iter().any(|entry| match entry.data {
|
||||
protocol::SessionSnapshotEntryData::UserInput { segments } => {
|
||||
protocol::Segment::flatten_to_text(&segments).contains(needle)
|
||||
}
|
||||
_ => false,
|
||||
});
|
||||
}
|
||||
Event::Alert(_) => continue,
|
||||
other => panic!("expected Snapshot first, got {other:?}"),
|
||||
let Event::Snapshot { session, .. } = handle.snapshot_event() else {
|
||||
panic!("expected snapshot");
|
||||
};
|
||||
session.entries.into_iter().any(|entry| match entry.data {
|
||||
protocol::SessionSnapshotEntryData::UserInput { segments } => {
|
||||
protocol::Segment::flatten_to_text(&segments).contains(needle)
|
||||
}
|
||||
}
|
||||
_ => false,
|
||||
})
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
|
||||
@@ -187,7 +187,8 @@ async fn materialise_on_first_turn_populates_worker() {
|
||||
.expect("system prompt materialised")
|
||||
.to_string();
|
||||
assert!(rendered.contains("date="));
|
||||
assert!(rendered.contains(&pwd.display().to_string()));
|
||||
assert!(rendered.contains("## Workdir execution environment"));
|
||||
assert!(!rendered.contains(&pwd.display().to_string()));
|
||||
assert!(rendered.starts_with("date="));
|
||||
}
|
||||
|
||||
@@ -206,7 +207,8 @@ async fn session_start_state_captures_rendered_prompt() {
|
||||
LogEntry::AnnotatedSegmentStart { system_prompt, .. } => {
|
||||
let sp = system_prompt.as_deref().expect("system prompt set");
|
||||
assert!(sp.starts_with("hello"));
|
||||
assert!(sp.contains(&pwd.display().to_string()));
|
||||
assert!(sp.contains("## Workdir execution environment"));
|
||||
assert!(!sp.contains(&pwd.display().to_string()));
|
||||
}
|
||||
other => panic!("expected AnnotatedSegmentStart as first entry, got {other:?}"),
|
||||
}
|
||||
|
||||
@@ -1,15 +1,10 @@
|
||||
//! Legacy process callback events are diagnostics only after Internal SubWorker migration.
|
||||
//! Agent-visible Internal SubWorker event summaries.
|
||||
|
||||
use std::sync::Arc;
|
||||
|
||||
use protocol::{Permission, ScopeRule, WorkerEvent};
|
||||
use tempfile::TempDir;
|
||||
use worker::ipc::event::{apply_event_side_effects, render_event};
|
||||
use worker::runtime::dir::RuntimeDir;
|
||||
use worker::spawn::registry::SpawnedWorkerRegistry;
|
||||
use protocol::WorkerEvent;
|
||||
use worker::ipc::event::render_event;
|
||||
|
||||
#[test]
|
||||
fn render_event_keeps_bounded_legacy_diagnostics() {
|
||||
fn render_event_identifies_child_and_failure() {
|
||||
let rendered = render_event(&WorkerEvent::Errored {
|
||||
worker_name: "legacy-child".into(),
|
||||
message: "boom".into(),
|
||||
@@ -17,30 +12,3 @@ fn render_event_keeps_bounded_legacy_diagnostics() {
|
||||
assert!(rendered.contains("legacy-child"));
|
||||
assert!(rendered.contains("boom"));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn legacy_callback_cannot_register_process_subworker_authority() {
|
||||
let runtime_base = TempDir::new().unwrap();
|
||||
let runtime_dir = Arc::new(
|
||||
RuntimeDir::create(runtime_base.path(), "parent")
|
||||
.await
|
||||
.unwrap(),
|
||||
);
|
||||
let registry = SpawnedWorkerRegistry::new(runtime_dir.clone());
|
||||
let scope_root = TempDir::new().unwrap();
|
||||
let event = WorkerEvent::ScopeSubDelegated {
|
||||
parent_worker: "legacy-parent".into(),
|
||||
sub_worker: "legacy-child".into(),
|
||||
sub_socket: "/tmp/legacy-child.sock".into(),
|
||||
scope: vec![ScopeRule {
|
||||
target: scope_root.path().to_path_buf(),
|
||||
permission: Permission::Write,
|
||||
recursive: true,
|
||||
symlink_policy: Default::default(),
|
||||
}],
|
||||
};
|
||||
|
||||
apply_event_side_effects(&event, ®istry, "parent", &None).await;
|
||||
|
||||
assert!(!runtime_dir.path().join("spawned_workers.json").exists());
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user