diff --git a/Cargo.lock b/Cargo.lock index e6038be2..8c6a126e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6650,6 +6650,7 @@ dependencies = [ "serial_test", "session-metrics", "session-store", + "sha2 0.11.0", "tempfile", "thiserror 2.0.18", "ticket", diff --git a/crates/client/src/client.rs b/crates/client/src/client.rs index 517d6a7c..bdd73375 100644 --- a/crates/client/src/client.rs +++ b/crates/client/src/client.rs @@ -112,26 +112,27 @@ mod tests { async fn encodes_methods_and_decodes_events_above_transport() { let mut socket = TestSocket::default(); socket.incoming.push_back( - encode_event(&Event::Status { - status: WorkerStatus::Idle, + encode_event(&Event::WorkerState { + snapshot: WorkerStatus::Idle.into(), }) .expect("encode event"), ); let mut client = Client::new(socket); client - .send(&Method::run_text("hello")) + .send(&Method::submit_text( + protocol::new_submission_request_id(), + "hello", + )) .await .expect("send method"); assert!(matches!( decode_method(&client.socket.sent[0]), - Ok(Method::Run { .. }) + Ok(Method::Submit { .. }) )); assert!(matches!( client.next_event().await, - Ok(Some(Event::Status { - status: WorkerStatus::Idle - })) + Ok(Some(Event::WorkerState { .. })) )); } } diff --git a/crates/client/src/transport/in_process.rs b/crates/client/src/transport/in_process.rs index b1db8e6b..ed50be54 100644 --- a/crates/client/src/transport/in_process.rs +++ b/crates/client/src/transport/in_process.rs @@ -89,17 +89,20 @@ mod tests { let mut client = Client::new(socket); client - .send(&Method::run_text("hello")) + .send(&Method::submit_text( + protocol::new_submission_request_id(), + "hello", + )) .await .expect("send method"); assert!(matches!( peer.next().await.as_deref().map(decode_method), - Some(Ok(Method::Run { .. })) + Some(Ok(Method::Submit { .. })) )); peer.send( - encode_event(&Event::Status { - status: WorkerStatus::Idle, + encode_event(&Event::WorkerState { + snapshot: WorkerStatus::Idle.into(), }) .expect("encode event"), ) @@ -107,9 +110,7 @@ mod tests { .expect("send event"); assert!(matches!( client.next_event().await, - Ok(Some(Event::Status { - status: WorkerStatus::Idle - })) + Ok(Some(Event::WorkerState { .. })) )); } } diff --git a/crates/client/src/transport/unix_socket.rs b/crates/client/src/transport/unix_socket.rs index 0262bff2..8b57bb2c 100644 --- a/crates/client/src/transport/unix_socket.rs +++ b/crates/client/src/transport/unix_socket.rs @@ -113,8 +113,8 @@ mod tests { let listener = UnixListener::bind(&socket_path).unwrap(); let server = tokio::spawn(async move { let (mut stream, _) = listener.accept().await.unwrap(); - let event = encode_event(&Event::Status { - status: WorkerStatus::Idle, + let event = encode_event(&Event::WorkerState { + snapshot: WorkerStatus::Idle.into(), }) .unwrap(); stream.write_all(event.as_bytes()).await.unwrap(); @@ -126,12 +126,7 @@ mod tests { .await .expect("client should receive event while alive") .expect("transport should succeed"); - assert!(matches!( - event, - Some(Event::Status { - status: WorkerStatus::Idle - }) - )); + assert!(matches!(event, Some(Event::WorkerState { .. }))); server.await.unwrap(); } @@ -147,12 +142,18 @@ mod tests { let mut client = Client::new(Socket::connect(&socket_path).await.unwrap()); client - .send(&Method::run_text("hello")) + .send(&Method::submit_text( + protocol::new_submission_request_id(), + "hello", + )) .await .expect("send method"); let received = server.await.unwrap().expect("method message"); - assert!(matches!(decode_method(&received), Ok(Method::Run { .. }))); + assert!(matches!( + decode_method(&received), + Ok(Method::Submit { .. }) + )); } #[tokio::test] diff --git a/crates/client/src/transport/websocket.rs b/crates/client/src/transport/websocket.rs index e8640573..883af12e 100644 --- a/crates/client/src/transport/websocket.rs +++ b/crates/client/src/transport/websocket.rs @@ -114,10 +114,10 @@ mod tests { assert!(matches!( message, Message::Text(ref text) - if matches!(decode_method(text), Ok(Method::Run { .. })) + if matches!(decode_method(text), Ok(Method::Submit { .. })) )); - let event = encode_event(&Event::Status { - status: WorkerStatus::Idle, + let event = encode_event(&Event::WorkerState { + snapshot: WorkerStatus::Idle.into(), }) .unwrap(); socket.send(Message::Text(event.into())).await.unwrap(); @@ -126,14 +126,15 @@ mod tests { let request = format!("ws://{address}").into_client_request().unwrap(); let mut client = Client::new(Socket::connect(request).await.unwrap()); client - .send(&Method::run_text("hello")) + .send(&Method::submit_text( + protocol::new_submission_request_id(), + "hello", + )) .await .expect("send method"); assert!(matches!( client.next_event().await, - Ok(Some(Event::Status { - status: WorkerStatus::Idle - })) + Ok(Some(Event::WorkerState { .. })) )); server.await.unwrap(); } diff --git a/crates/client/src/workspace_product.rs b/crates/client/src/workspace_product.rs index 70b6a6af..99d72538 100644 --- a/crates/client/src/workspace_product.rs +++ b/crates/client/src/workspace_product.rs @@ -12,8 +12,10 @@ use workspace_api::{ BrowserCreateWorkerResponse, BrowserWorkspaceOrchestratorResponse, CreateWorkspaceWorkerRequest, ListResponse, MemoryDocumentResponse, MemoryStagingListResponse, ObjectiveCreateRequest, ObjectiveDetail, ObjectiveEditRequest, ObjectiveLinkTicketRequest, - ObjectiveStateRequest, ObjectiveSummary, TICKET_ORCHESTRATION_PLANS_QUERY_PATH, - TICKET_RELATIONS_QUERY_PATH, WorkerLaunchOptionsResponse, + ObjectiveStateRequest, ObjectiveSummary, PutRuntimeTrustKeyRequest, + RevokeRuntimeTrustKeyRequest, RuntimeTrustKeyRevealResponse, + TICKET_ORCHESTRATION_PLANS_QUERY_PATH, TICKET_RELATIONS_QUERY_PATH, + WorkerLaunchOptionsResponse, WorkspaceRuntimeDetail, WorkspaceRuntimeResource, }; use crate::{BackendApiClient, BackendWorkspaceClientError}; @@ -241,6 +243,53 @@ impl BackendWorkspaceProductClient { ) } + pub fn list_runtimes( + &self, + ) -> Result, BackendWorkspaceClientError> { + self.get_json("/runtimes") + } + + pub fn runtime_detail( + &self, + runtime_id: &str, + ) -> Result { + self.get_json(&format!("/runtimes/{}", encode_path_segment(runtime_id))) + } + + pub fn reveal_runtime_trust_key( + &self, + runtime_id: &str, + ) -> Result { + self.get_json(&format!( + "/runtimes/{}/trust-key", + encode_path_segment(runtime_id) + )) + } + + pub fn put_runtime_trust_key( + &self, + runtime_id: &str, + request: &PutRuntimeTrustKeyRequest, + ) -> Result { + self.send_json( + Method::PUT, + &format!("/runtimes/{}/trust-key", encode_path_segment(runtime_id)), + Some(request), + ) + } + + pub fn revoke_runtime_trust_key( + &self, + runtime_id: &str, + request: &RevokeRuntimeTrustKeyRequest, + ) -> Result { + self.send_json( + Method::DELETE, + &format!("/runtimes/{}/trust-key", encode_path_segment(runtime_id)), + Some(request), + ) + } + pub fn memory_document(&self) -> Result { self.get_json("/memory") } diff --git a/crates/protocol/src/lib.rs b/crates/protocol/src/lib.rs index 9f0ab12c..d54c9232 100644 --- a/crates/protocol/src/lib.rs +++ b/crates/protocol/src/lib.rs @@ -11,6 +11,11 @@ use serde::{Deserialize, Serialize}; pub use identity::{WorkerId, WorkerIdParseError}; +/// Allocate an opaque idempotency key for one client Submit request. +pub fn new_submission_request_id() -> String { + uuid::Uuid::now_v7().to_string() +} + fn default_true() -> bool { true } @@ -27,21 +32,264 @@ fn is_false(value: &bool) -> bool { // Method (Client → Worker via Unix Socket) // --------------------------------------------------------------------------- +/// Trusted Server → Runtime transport header carrying the authenticated +/// browser Account identity for one Worker protocol connection. +/// +/// Runtime accepts this only after its normal HTTP authentication succeeds; +/// serialized [`Method`] payloads cannot set authenticated source identity. +pub const AUTHENTICATED_ACCOUNT_ID_HEADER: &str = "x-yoi-authenticated-account-id"; + +/// Trusted source identity attached by an authenticated transport boundary. +/// +/// Public clients cannot select this value directly. Runtime/Backend adapters +/// stamp it before forwarding an accepted Submit or Notify to a Worker. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum AuthenticatedInputSource { + /// Assigned whenever a serialized tracked method crosses an untrusted + /// protocol boundary. Receivers must handle it exactly like public input. + UntrustedWire, + Account { + account_id: String, + }, + Worker { + runtime_id: String, + worker_id: String, + }, + SubWorker { + session_id: String, + }, + Backend { + operation_id: String, + }, +} + +impl Default for AuthenticatedInputSource { + fn default() -> Self { + Self::UntrustedWire + } +} + +impl AuthenticatedInputSource { + pub fn namespace(&self) -> String { + match self { + Self::UntrustedWire => "untrusted-wire".into(), + Self::Account { account_id } => format!("account:{account_id}"), + Self::Worker { + runtime_id, + worker_id, + } => format!("worker:{runtime_id}:{worker_id}"), + Self::SubWorker { session_id } => format!("sub_worker:{session_id}"), + Self::Backend { operation_id } => format!("backend:{operation_id}"), + } + } +} + +/// Immutable identity and revision fence for one state-changing Worker command. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +pub struct WorkerCommandEnvelope { + /// Caller-owned sequence. A controller accepts command ids in strictly + /// increasing order for one execution generation. + pub command_id: u64, + pub expected_execution_generation: u64, + pub expected_worker_state_revision: u64, +} + +impl WorkerCommandEnvelope { + pub fn for_snapshot(command_id: u64, snapshot: &WorkerStateSnapshot) -> Self { + Self { + command_id, + expected_execution_generation: snapshot.execution_generation, + expected_worker_state_revision: snapshot.revision, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(rename_all = "snake_case")] +pub enum WorkerCommandKind { + Resume, + Cancel, + Pause, + Compact, + Shutdown, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(rename_all = "snake_case")] +pub enum WorkerCommandDisposition { + Accepted, + StaleExecutionGeneration, + StaleWorkerStateRevision, + StaleCommandId, + Conflict, + InvalidState, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +pub struct WorkerCommandAcknowledgement { + pub command_id: u64, + pub command: WorkerCommandKind, + pub disposition: WorkerCommandDisposition, + /// The complete authoritative state observed after command admission. + pub state: WorkerStateSnapshot, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(tag = "kind", content = "state", rename_all = "snake_case")] +pub enum WorkerState { + Idle, + Busy(WorkerBusyState), +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(tag = "kind", content = "state", rename_all = "snake_case")] +pub enum WorkerBusyState { + Run(WorkerRunState), + Maintenance(WorkerMaintenanceState), +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(rename_all = "snake_case")] +pub enum WorkerRunState { + Running, + Pausing, + Paused, + Cancelling, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(rename_all = "snake_case")] +pub enum WorkerMaintenanceState { + Compacting, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +pub struct WorkerStateSnapshot { + pub execution_generation: u64, + pub revision: u64, + /// Highest lifecycle command id observed by this controller generation. + pub last_command_id: u64, + pub state: WorkerState, +} + +impl WorkerStateSnapshot { + pub fn initial(execution_generation: u64) -> Self { + Self { + execution_generation, + revision: 0, + last_command_id: 0, + state: WorkerState::Idle, + } + } + + /// Compatibility projection for Runtime catalog lifecycle. This value is + /// never command-admission authority and cannot produce `Stopped`. + pub fn catalog_status(&self) -> WorkerStatus { + match self.state { + WorkerState::Idle => WorkerStatus::Idle, + WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused)) => WorkerStatus::Paused, + WorkerState::Busy(WorkerBusyState::Run(_)) + | WorkerState::Busy(WorkerBusyState::Maintenance(_)) => WorkerStatus::Running, + } + } +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WorkerStateSnapshotApply { + Applied, + Duplicate, + Stale, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct WorkerStateSnapshotConflict { + pub execution_generation: u64, + pub revision: u64, +} + +impl std::fmt::Display for WorkerStateSnapshotConflict { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + formatter, + "conflicting worker state snapshots at generation {} revision {}", + self.execution_generation, self.revision + ) + } +} + +impl std::error::Error for WorkerStateSnapshotConflict {} + +pub fn apply_worker_state_snapshot( + current: &mut WorkerStateSnapshot, + incoming: &WorkerStateSnapshot, +) -> Result { + use std::cmp::Ordering; + + let ordering = (incoming.execution_generation, incoming.revision) + .cmp(&(current.execution_generation, current.revision)); + match ordering { + Ordering::Greater => { + *current = incoming.clone(); + Ok(WorkerStateSnapshotApply::Applied) + } + Ordering::Less => Ok(WorkerStateSnapshotApply::Stale), + Ordering::Equal if incoming == current => Ok(WorkerStateSnapshotApply::Duplicate), + Ordering::Equal => Err(WorkerStateSnapshotConflict { + execution_generation: incoming.execution_generation, + revision: incoming.revision, + }), + } +} + +impl From for WorkerStateSnapshot { + fn from(status: WorkerStatus) -> Self { + let state = match status { + WorkerStatus::Idle | WorkerStatus::Stopped => WorkerState::Idle, + WorkerStatus::Running => { + WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)) + } + WorkerStatus::Paused => WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused)), + }; + Self { + execution_generation: 1, + revision: 0, + last_command_id: 0, + state, + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[serde(tag = "method", content = "params", rename_all = "snake_case")] pub enum Method { - Run { + /// Durably accept typed input for immediate activation or the session FIFO. + /// + /// `submission_request_id` is generated by the authenticated caller and is + /// used only for idempotent retry. Worker allocates the durable + /// `submission_id` returned by [`Event::SubmissionAccepted`]. + Submit { + submission_request_id: String, input: Vec, }, - /// Runtime-internal Run carrying an opaque correlation id that is committed - /// with the resulting UserInput entry. This variant is not serializable on - /// the public Client → Worker protocol. - #[serde(skip)] + /// Authenticated transport form of Submit. Trusted adapters replace + /// public Submit before forwarding it to the Worker. #[cfg_attr(feature = "typescript", ts(skip))] - RunTracked { + SubmitTracked { + submission_request_id: String, input: Vec, - submission_id: String, + #[serde(skip_deserializing, default)] + source: AuthenticatedInputSource, }, /// Human-readable text injected into the target Worker's LLM context /// as a non-blocking system message. `auto_run` controls whether an @@ -50,26 +298,63 @@ pub enum Method { /// No side effects beyond LLM context; use `WorkerEvent` for typed /// lifecycle reports. Notify { + notification_request_id: String, message: String, #[serde(default = "default_true", skip_serializing_if = "is_true")] auto_run: bool, }, + /// Authenticated transport form of Notify. + #[cfg_attr(feature = "typescript", ts(skip))] + NotifyTracked { + notification_request_id: String, + message: String, + #[serde(default = "default_true", skip_serializing_if = "is_true")] + auto_run: bool, + #[serde(skip_deserializing, default)] + source: AuthenticatedInputSource, + }, /// Typed lifecycle report from a child Worker to its direct parent. WorkerEvent(WorkerEvent), - Resume, - Cancel, + /// Return the authoritative FIFO summary without exposing queued payloads. + ListPendingSubmissions, + /// Remove one queued submission. Running or already activated submissions + /// are immutable and therefore cannot be cancelled here. + CancelPendingSubmission { + submission_id: String, + expected_revision: u64, + }, + /// Remove every queued submission while preserving the active run. + ClearPendingSubmissions { + expected_revision: u64, + }, + /// Activate the next queued submission while the Worker is idle. This is an + /// explicit recovery operation and never resumes a paused run implicitly. + ContinuePending { + expected_revision: u64, + expected_head_id: String, + }, + Resume { + command: WorkerCommandEnvelope, + }, + Cancel { + command: WorkerCommandEnvelope, + }, /// Stop the in-flight turn and transition to `Paused`. /// /// Unlike `Cancel` (which discards and returns to `Idle`), a paused - /// Worker can resume the interrupted work via `Resume`, or start a - /// fresh turn via `Run` (orphan `tool_use` items are closed with a + /// Worker can resume the interrupted work via `Resume`, or accept a + /// fresh `Submit` (orphan `tool_use` items are closed with a /// synthetic tool result before the new user message is appended). - Pause, + Pause { + command: WorkerCommandEnvelope, + }, /// Request an explicit compaction while the Worker is otherwise idle. /// /// This is a typed control method: clients must not send `compact` as a - /// `Method::Run` user message. - Compact, + /// `Method::Submit` user message. + Compact { + command: WorkerCommandEnvelope, + }, /// Ask the Worker to list valid rewind targets from its authoritative session log. ListRewindTargets, /// Truncate the current session back to the selected rewind target and @@ -78,7 +363,9 @@ pub enum Method { target: RewindTargetId, expected_head_entries: usize, }, - Shutdown, + Shutdown { + command: WorkerCommandEnvelope, + }, /// Request a list of completion candidates from the Worker. /// /// Reply is sent on the same socket as `Event::Completions` (not @@ -181,7 +468,7 @@ impl WorkerEvent { /// One typed piece of a user submission. /// -/// `Method::Run` and `Event::UserMessage` carry `Vec`. Dumb +/// `Method::Submit` and `Event::UserMessage` carry `Vec`. Dumb /// clients (CLI piping, scripts) only need to produce a single /// `Segment::Text`; richer clients (TUI / GUI) construct typed atoms /// (paste chips, file refs) and @@ -404,12 +691,13 @@ impl Segment { } impl Method { - /// Convenience: a `Run` carrying a single `Segment::Text`. + /// Convenience: a `Submit` carrying a single `Segment::Text`. /// Used by dumb clients, inter-Worker tools, and tests that only have /// a string to forward. - pub fn run_text(s: impl Into) -> Self { - Self::Run { - input: vec![Segment::text(s)], + pub fn submit_text(submission_request_id: impl Into, text: impl Into) -> Self { + Self::Submit { + submission_request_id: submission_request_id.into(), + input: vec![Segment::text(text)], } } } @@ -503,6 +791,39 @@ pub enum ToolResultDisposition { OutcomeUnknown, } +/// Durable acceptance result for one idempotent Submit request. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(rename_all = "snake_case")] +pub enum SubmissionDisposition { + Started, + Queued, +} + +/// Bounded public projection of one pending submission. Payload segments and +/// provenance remain in the session log and are intentionally not exposed. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +pub struct PendingSubmissionSummary { + pub submission_id: String, + pub accepted_at_ms: u64, + pub segment_count: u32, + pub byte_len: u64, +} + +/// Revisioned session-owned FIFO projection used by snapshots and live events. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +pub struct PendingSubmissionsSnapshot { + pub revision: u64, + #[serde(default)] + pub notification_count: u32, + #[serde(default)] + pub head_id: Option, + #[serde(default)] + pub submissions: Vec, +} + /// Canonical, storage-independent projection of committed session history. /// /// Worker protocols expose this DTO instead of append-log records. New @@ -511,6 +832,8 @@ pub enum ToolResultDisposition { #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] pub struct SessionSnapshot { + #[serde(default)] + pub pending_submissions: PendingSubmissionsSnapshot, pub entries: Vec, } @@ -609,16 +932,27 @@ pub struct SessionToolAttachment { #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[serde(tag = "event", content = "data", rename_all = "snake_case")] pub enum Event { - /// A user input message was accepted, persisted as - /// `LogEntry::AnnotatedUserInput`, and is about to start a new turn. - /// Broadcast to every subscribed client so TUI / GUI instances show - /// the same user line that reconnect snapshots would replay from - /// history; clients must not synthesize a separate pending/fake - /// message for accepted runs. - /// - /// Fires exactly once per committed user input, after - /// `InvokeStart { kind: UserSend }` and before the first - /// `TurnStart`. Rejected runs (e.g. `AlreadyRunning`) do not emit. + /// Durable Submit acceptance. A `Started` receipt follows the atomic + /// UserInput commit; a `Queued` receipt follows the durable FIFO checkpoint. + /// Repeating the same request id and exact payload returns the same receipt + /// without appending or activating twice. + SubmissionAccepted { + submission_request_id: String, + submission_id: String, + disposition: SubmissionDisposition, + }, + /// Correlated rejection before durable acceptance. + SubmissionRejected { + submission_request_id: String, + message: String, + }, + /// Revisioned FIFO replacement following enqueue, activation, cancel, or clear. + PendingSubmissionsChanged { + pending: PendingSubmissionsSnapshot, + }, + /// A user input message persisted as `LogEntry::AnnotatedUserInput` and + /// activated for a turn. Broadcast to every subscribed client so TUI / GUI + /// instances show the same user line that reconnect snapshots replay. UserMessage { segments: Vec, }, @@ -641,7 +975,7 @@ pub enum Event { /// /// Marker event for the start of an Invoke range; the range extends /// implicitly until the next `InvokeStart`. Fires for every accepted - /// `Method::Run` (kind=`UserSend`), `Method::Notify` (kind=`Notify`), + /// `Method::Submit` (kind=`UserSend`), `Method::Notify` (kind=`Notify`), /// `Method::WorkerEvent` re-injection (kind=`WorkerEvent`), and any other /// IDLE-breaking trigger. Mid-run interrupts (e.g. hook output, /// typed system reminder insertion that doesn't break IDLE) do not @@ -798,8 +1132,9 @@ pub enum Event { Snapshot { session: SessionSnapshot, greeting: Greeting, - #[serde(default)] - status: WorkerStatus, + /// Full revisioned live execution state. `Stopped` remains Runtime + /// catalog authority and is deliberately not represented here. + state: WorkerStateSnapshot, /// Unfinished model output that has already streamed in the current /// run but is not yet represented by committed snapshot entries. #[serde(default, skip_serializing_if = "InFlightSnapshot::is_empty")] @@ -836,8 +1171,11 @@ pub enum Event { }, /// Current Worker controller status. Broadcast on every controller-level /// transition and included in `History` snapshots for late attach. - Status { - status: WorkerStatus, + WorkerState { + snapshot: WorkerStateSnapshot, + }, + CommandAcknowledged { + acknowledgement: WorkerCommandAcknowledgement, }, /// Bounded, provider-owned command telemetry for the live Console. This is /// intentionally not a history entry and is reconstructed from @@ -1193,7 +1531,7 @@ pub enum TurnResult { #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[serde(rename_all = "snake_case")] pub enum InvokeKind { - /// `Method::Run` — a user submission. + /// `Method::Submit` — a user submission. UserSend, /// `Method::Notify` — free-text notification injected into history. Notify, @@ -1216,7 +1554,7 @@ pub enum RunResult { Finished, Paused, LimitReached, - /// The accepted Method::Run produced no assistant/tool output before + /// The accepted Method::Submit produced no assistant/tool output before /// user interruption, so the Worker rolled the submit-time turn state back /// to its pre-submit snapshot. Clients should treat the Worker as Idle and /// restore the just-submitted input into the editable composer if desired. @@ -1285,26 +1623,82 @@ mod tests { use super::*; #[test] - fn method_run_json_roundtrip() { - let json = r#"{"method":"run","params":{"input":[{"kind":"text","content":"Hello"}]}}"#; + fn worker_state_snapshot_apply_is_monotonic_and_detects_conflicts() { + let mut current = WorkerStateSnapshot::initial(4); + let mut newer = current.clone(); + newer.revision = 1; + newer.state = WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)); + + assert_eq!( + apply_worker_state_snapshot(&mut current, &newer), + Ok(WorkerStateSnapshotApply::Applied) + ); + assert_eq!( + apply_worker_state_snapshot(&mut current, &newer), + Ok(WorkerStateSnapshotApply::Duplicate) + ); + + let stale_revision = WorkerStateSnapshot::initial(4); + assert_eq!( + apply_worker_state_snapshot(&mut current, &stale_revision), + Ok(WorkerStateSnapshotApply::Stale) + ); + let stale_generation = WorkerStateSnapshot { + execution_generation: 3, + revision: u64::MAX, + ..newer.clone() + }; + assert_eq!( + apply_worker_state_snapshot(&mut current, &stale_generation), + Ok(WorkerStateSnapshotApply::Stale) + ); + + let conflicting = WorkerStateSnapshot { + state: WorkerState::Idle, + ..newer.clone() + }; + assert_eq!( + apply_worker_state_snapshot(&mut current, &conflicting), + Err(WorkerStateSnapshotConflict { + execution_generation: 4, + revision: 1, + }) + ); + assert_eq!(current, newer); + + let next_generation = WorkerStateSnapshot::initial(5); + assert_eq!( + apply_worker_state_snapshot(&mut current, &next_generation), + Ok(WorkerStateSnapshotApply::Applied) + ); + assert_eq!(current, next_generation); + } + + #[test] + fn method_submit_json_roundtrip_and_run_is_rejected() { + let json = r#"{"method":"submit","params":{"submission_request_id":"request-1","input":[{"kind":"text","content":"Hello"}]}}"#; let method: Method = serde_json::from_str(json).unwrap(); match &method { - Method::Run { input } => { + Method::Submit { input, .. } => { assert_eq!(input.len(), 1); match &input[0] { Segment::Text { content } => assert_eq!(content, "Hello"), other => panic!("expected Text, got {other:?}"), } } - other => panic!("expected Run, got {other:?}"), + other => panic!("expected Submit, got {other:?}"), } let serialized = serde_json::to_string(&method).unwrap(); assert_eq!(serialized, json); + assert!( + serde_json::from_str::(r#"{"method":"run","params":{"input":[]}}"#).is_err() + ); } #[test] - fn method_run_paste_segment_roundtrip() { - let method = Method::Run { + fn method_submit_paste_segment_roundtrip() { + let method = Method::Submit { + submission_request_id: "request-1".to_string(), input: vec![ Segment::text("see "), Segment::Paste { @@ -1318,7 +1712,7 @@ mod tests { let json = serde_json::to_string(&method).unwrap(); let decoded: Method = serde_json::from_str(&json).unwrap(); match decoded { - Method::Run { input } => { + Method::Submit { input, .. } => { assert_eq!(input.len(), 2); match &input[1] { Segment::Paste { @@ -1335,7 +1729,7 @@ mod tests { other => panic!("expected Paste, got {other:?}"), } } - other => panic!("expected Run, got {other:?}"), + other => panic!("expected Submit, got {other:?}"), } } @@ -1389,8 +1783,9 @@ mod tests { } #[test] - fn method_run_flow_segment_roundtrip() { - let method = Method::Run { + fn method_submit_flow_segment_roundtrip() { + let method = Method::Submit { + submission_request_id: "request-1".to_string(), input: vec![ Segment::Flow { selector: "builtin:coder-review".to_string(), @@ -1404,7 +1799,7 @@ mod tests { let decoded = serde_json::from_str::(&json).unwrap(); assert!(matches!( decoded, - Method::Run { input } + Method::Submit { input, .. } if matches!( input.as_slice(), [ @@ -1416,15 +1811,26 @@ mod tests { } #[test] - fn runtime_tracked_run_is_not_public_protocol_json() { - let method = Method::RunTracked { + fn authenticated_submit_replaces_wire_source_with_transport_identity() { + let method = Method::SubmitTracked { input: vec![Segment::text("private")], - submission_id: "submission-1".to_string(), + submission_request_id: "request-1".to_string(), + source: AuthenticatedInputSource::Account { + account_id: "account-1".into(), + }, }; - assert!(serde_json::to_string(&method).is_err()); + let json = serde_json::to_string(&method).unwrap(); + let decoded = serde_json::from_str::(&json).unwrap(); + assert!(matches!( + decoded, + Method::SubmitTracked { + source: AuthenticatedInputSource::UntrustedWire, + .. + } + )); assert!( serde_json::from_str::( - r#"{"method":"run_tracked","input":[],"submission_id":"forged"}"#, + r#"{"method":"submit_tracked","input":[],"submission_request_id":"forged"}"#, ) .is_err() ); @@ -1442,42 +1848,53 @@ mod tests { } #[test] - fn method_run_with_unknown_segment_decodes() { - let json = r#"{"method":"run","params":{"input":[{"kind":"text","content":"hi"},{"kind":"future_thing","x":1}]}}"#; + fn method_submit_with_unknown_segment_decodes() { + let json = r#"{"method":"submit","params":{"submission_request_id":"request-1","input":[{"kind":"text","content":"hi"},{"kind":"future_thing","x":1}]}}"#; let method: Method = serde_json::from_str(json).unwrap(); match method { - Method::Run { input } => { + Method::Submit { input, .. } => { assert_eq!(input.len(), 2); assert!(matches!(input[0], Segment::Text { .. })); assert!(matches!(input[1], Segment::Unknown)); } - other => panic!("expected Run, got {other:?}"), + other => panic!("expected Submit, got {other:?}"), } } #[test] - fn method_without_params() { - let json = r#"{"method":"resume"}"#; - let method: Method = serde_json::from_str(json).unwrap(); - assert!(matches!(method, Method::Resume)); + fn lifecycle_method_without_command_fails_closed() { + let error = serde_json::from_str::(r#"{"method":"resume"}"#).unwrap_err(); + assert!(error.to_string().contains("params")); } #[test] - fn method_pause_roundtrip() { - let json = r#"{"method":"pause"}"#; - let method: Method = serde_json::from_str(json).unwrap(); - assert!(matches!(method, Method::Pause)); - let serialized = serde_json::to_string(&method).unwrap(); - assert_eq!(serialized, json); - } - - #[test] - fn method_compact_roundtrip() { - let json = r#"{"method":"compact"}"#; - let method: Method = serde_json::from_str(json).unwrap(); - assert!(matches!(method, Method::Compact)); - let serialized = serde_json::to_string(&method).unwrap(); - assert_eq!(serialized, json); + fn lifecycle_methods_roundtrip_with_fences() { + for method in [ + Method::Pause { + command: WorkerCommandEnvelope { + command_id: 11, + expected_execution_generation: 4, + expected_worker_state_revision: 8, + }, + }, + Method::Compact { + command: WorkerCommandEnvelope { + command_id: 12, + expected_execution_generation: 4, + expected_worker_state_revision: 9, + }, + }, + ] { + let json = serde_json::to_string(&method).unwrap(); + let decoded: Method = serde_json::from_str(&json).unwrap(); + match decoded { + Method::Pause { command } | Method::Compact { command } => { + assert_eq!(command.expected_execution_generation, 4); + assert!(command.command_id >= 11); + } + other => panic!("unexpected lifecycle method: {other:?}"), + } + } } #[test] @@ -1648,11 +2065,11 @@ mod tests { #[test] fn method_notify_json_roundtrip_defaults_to_auto_run() { - let json = r#"{"method":"notify","params":{"message":"turn done"}}"#; + let json = r#"{"method":"notify","params":{"notification_request_id":"notification-1","message":"turn done"}}"#; let method: Method = serde_json::from_str(json).unwrap(); assert!(matches!( method, - Method::Notify { ref message, auto_run: true } if message == "turn done" + Method::Notify { ref message, auto_run: true, .. } if message == "turn done" )); let serialized = serde_json::to_string(&method).unwrap(); assert_eq!(serialized, json); @@ -1660,11 +2077,11 @@ mod tests { #[test] fn method_notify_weak_json_roundtrip_serializes_auto_run_false() { - let json = r#"{"method":"notify","params":{"message":"progress","auto_run":false}}"#; + let json = r#"{"method":"notify","params":{"notification_request_id":"notification-1","message":"progress","auto_run":false}}"#; let method: Method = serde_json::from_str(json).unwrap(); assert!(matches!( method, - Method::Notify { ref message, auto_run: false } if message == "progress" + Method::Notify { ref message, auto_run: false, .. } if message == "progress" )); assert_eq!(serde_json::to_string(&method).unwrap(), json); } @@ -1725,6 +2142,7 @@ mod tests { fn event_snapshot_format() { let event = Event::Snapshot { session: SessionSnapshot { + pending_submissions: PendingSubmissionsSnapshot::default(), entries: vec![SessionSnapshotEntry { entry_id: "entry-1".into(), timestamp: 1, @@ -1745,7 +2163,7 @@ mod tests { context_window: 200_000, context_tokens: 42_000, }, - status: WorkerStatus::Paused, + state: WorkerStatus::Paused.into(), in_flight: InFlightSnapshot::default(), internal_workers: Vec::new(), }; @@ -1762,12 +2180,13 @@ mod tests { assert_eq!(parsed["data"]["greeting"]["tools"][0], "Read"); assert_eq!(parsed["data"]["greeting"]["context_window"], 200_000); assert_eq!(parsed["data"]["greeting"]["context_tokens"], 42_000); - assert_eq!(parsed["data"]["status"], "paused"); + assert_eq!(parsed["data"]["state"]["state"]["kind"], "busy"); + assert_eq!(parsed["data"]["state"]["state"]["state"]["state"], "paused"); } #[test] fn event_snapshot_in_flight_roundtrip_and_default() { - let inbound = r#"{"event":"snapshot","data":{"session":{"entries":[]},"greeting":{"worker_name":"test","cwd":"/tmp","provider":"p","model":"m","scope_summary":"s","tools":[]},"status":"running"}}"#; + let inbound = r#"{"event":"snapshot","data":{"session":{"entries":[]},"greeting":{"worker_name":"test","cwd":"/tmp","provider":"p","model":"m","scope_summary":"s","tools":[]},"state":{"execution_generation":1,"revision":1,"last_command_id":0,"state":{"kind":"busy","state":{"kind":"run","state":"running"}}}}}"#; let decoded: Event = serde_json::from_str(inbound).unwrap(); match decoded { Event::Snapshot { in_flight, .. } => assert!(in_flight.is_empty()), @@ -1776,6 +2195,7 @@ mod tests { let event = Event::Snapshot { session: SessionSnapshot { + pending_submissions: PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, greeting: Greeting { @@ -1788,7 +2208,7 @@ mod tests { context_window: 0, context_tokens: 0, }, - status: WorkerStatus::Running, + state: WorkerStatus::Running.into(), in_flight: InFlightSnapshot { blocks: vec![ InFlightBlock::Text { @@ -1844,6 +2264,7 @@ mod tests { fn event_segment_rotated_roundtrip() { let event = Event::SegmentRotated { session: SessionSnapshot { + pending_submissions: PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, }; @@ -1875,20 +2296,32 @@ mod tests { } #[test] - fn event_status_format() { - let event = Event::Status { - status: WorkerStatus::Running, + fn event_worker_state_format() { + let event = Event::WorkerState { + snapshot: WorkerStateSnapshot { + execution_generation: 7, + revision: 3, + last_command_id: 9, + state: WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)), + }, }; let json = serde_json::to_string(&event).unwrap(); let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); - assert_eq!(parsed["event"], "status"); - assert_eq!(parsed["data"]["status"], "running"); + assert_eq!(parsed["event"], "worker_state"); + assert_eq!(parsed["data"]["snapshot"]["execution_generation"], 7); + assert_eq!(parsed["data"]["snapshot"]["revision"], 3); + assert_eq!(parsed["data"]["snapshot"]["state"]["kind"], "busy"); let decoded: Event = serde_json::from_str(&json).unwrap(); assert!(matches!( decoded, - Event::Status { - status: WorkerStatus::Running + Event::WorkerState { + snapshot: WorkerStateSnapshot { + execution_generation: 7, + revision: 3, + state: WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)), + .. + } } )); } @@ -1929,19 +2362,10 @@ mod tests { } #[test] - fn event_snapshot_without_status_defaults_to_idle() { + fn event_snapshot_without_worker_state_fails_closed() { let json = r#"{"event":"snapshot","data":{"session":{"entries":[]},"greeting":{"worker_name":"test","cwd":"/tmp","provider":"anthropic","model":"claude","scope_summary":"","tools":[]}}}"#; - let decoded: Event = serde_json::from_str(json).unwrap(); - match decoded { - Event::Snapshot { - status, greeting, .. - } => { - assert_eq!(status, WorkerStatus::Idle); - assert_eq!(greeting.context_window, 0); - assert_eq!(greeting.context_tokens, 0); - } - other => panic!("expected Snapshot, got {other:?}"), - } + let error = serde_json::from_str::(json).unwrap_err(); + assert!(error.to_string().contains("state")); } #[test] @@ -2354,7 +2778,12 @@ mod tests { "scope_summary": "scope", "tools": [] }, - "status": "idle" + "state": { + "execution_generation": 1, + "revision": 0, + "last_command_id": 0, + "state": { "kind": "idle" } + } } })) .unwrap(); diff --git a/crates/protocol/src/subscription.rs b/crates/protocol/src/subscription.rs index 468e241c..bf4a8635 100644 --- a/crates/protocol/src/subscription.rs +++ b/crates/protocol/src/subscription.rs @@ -573,6 +573,11 @@ pub struct SubscriptionWorker { pub resource_key: Option, /// Producer-owned monotonic revision for this Worker subject. pub subject_revision: u64, + /// Latest revisioned foreground state observed from the Worker. This remains + /// absent until an authoritative Worker snapshot/event has been applied. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub worker_state: Option, + /// Runtime catalog lifecycle compatibility projection; not foreground-state authority. pub state: SubscriptionWorkerState, #[serde(default)] pub has_running_internal_workers: bool, @@ -874,6 +879,7 @@ mod tests { runtime_id: None, resource_key: None, subject_revision: 0, + worker_state: None, state: SubscriptionWorkerState::Idle, has_running_internal_workers: false, workspace_id: Some("workspace-1".to_string()), diff --git a/crates/protocol/src/typescript.rs b/crates/protocol/src/typescript.rs index bdac40ca..ec5588f9 100644 --- a/crates/protocol/src/typescript.rs +++ b/crates/protocol/src/typescript.rs @@ -8,11 +8,14 @@ use crate::{ CompletionKind, ErrorCode, Event, Greeting, InFlightBlock, InFlightSnapshot, InFlightToolCallState, InternalWorkerKind, InternalWorkerRef, InternalWorkerSnapshot, InvokeKind, MemoryWorkerEvent, Method, PasteArtifactAvailability, PasteArtifactMediaType, - PasteArtifactRef, Permission, RewindSummary, RewindTarget, RewindTargetId, RunResult, - ScopeRule, Segment, SessionContentPart, SessionEntryProvenance, SessionMessageRole, - SessionSnapshot, SessionSnapshotEntry, SessionSnapshotEntryData, SessionToolAttachment, - ToolResultDisposition, TurnResult, UploadedFileAvailability, UploadedFileRef, WorkerEvent, - WorkerStatus, + PasteArtifactRef, PendingSubmissionSummary, PendingSubmissionsSnapshot, Permission, + RewindSummary, RewindTarget, RewindTargetId, RunResult, ScopeRule, Segment, SessionContentPart, + SessionEntryProvenance, SessionMessageRole, SessionSnapshot, SessionSnapshotEntry, + SessionSnapshotEntryData, SessionToolAttachment, SubmissionDisposition, ToolResultDisposition, + TurnResult, UploadedFileAvailability, UploadedFileRef, WorkerBusyState, + WorkerCommandAcknowledgement, WorkerCommandDisposition, WorkerCommandEnvelope, + WorkerCommandKind, WorkerEvent, WorkerMaintenanceState, WorkerRunState, WorkerState, + WorkerStateSnapshot, WorkerStatus, subscription::{ EventSubscriptionSelector, SubscriptionEvent, SubscriptionEventPayload, SubscriptionFrame, SubscriptionFramePayload, SubscriptionId, SubscriptionRejectionCode, SubscriptionRequest, @@ -46,6 +49,15 @@ pub fn generated_protocol_types() -> String { push_decl::(&cfg, &mut output); push_decl::(&cfg, &mut output); push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); push_decl::(&cfg, &mut output); push_decl::(&cfg, &mut output); push_decl::(&cfg, &mut output); @@ -75,6 +87,9 @@ pub fn generated_protocol_types() -> String { push_decl::(&cfg, &mut output); push_decl::(&cfg, &mut output); push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); + push_decl::(&cfg, &mut output); push_decl::(&cfg, &mut output); push_decl::(&cfg, &mut output); push_decl::(&cfg, &mut output); diff --git a/crates/session-store/src/fs_store.rs b/crates/session-store/src/fs_store.rs index a0e80a75..19a2c6d0 100644 --- a/crates/session-store/src/fs_store.rs +++ b/crates/session-store/src/fs_store.rs @@ -21,8 +21,10 @@ use crate::segment_log::LogEntry; use crate::store::{Store, StoreError}; use crate::uploaded_file::{ bind_uploaded_file, clear_uploaded_file_binding, copy_committed_uploaded_files, - delete_uncommitted_uploaded_files, delete_uploaded_file, list_uploaded_file_refs, - read_uploaded_file, read_uploaded_file_by_id, write_uploaded_file, + delete_uncommitted_uploaded_files, delete_uploaded_file, finalize_uploaded_file_binding, + list_uploaded_file_refs, pin_uploaded_file, read_uploaded_file, read_uploaded_file_by_id, + reconcile_uploaded_file_pins, release_uploaded_file_pin, uploaded_file_has_pending_owner, + write_uploaded_file, }; use crate::{ PasteArtifactLimits, SegmentId, SessionId, UploadedFileLimits, UploadedFileUploadContext, @@ -518,6 +520,61 @@ impl Store for FsStore { } } + fn pin_uploaded_file( + &self, + session_id: SessionId, + reference: &UploadedFileRef, + owner_id: &str, + ) -> Result<(), StoreError> { + let _guard = self + .append_lock + .lock() + .map_err(|_| std::io::Error::other("session store append lock was poisoned"))?; + pin_uploaded_file(&self.paste_artifact_dir(session_id), reference, owner_id) + } + + fn release_uploaded_file_pin( + &self, + session_id: SessionId, + artifact_id: &str, + owner_id: &str, + ) -> Result<(), StoreError> { + let _guard = self + .append_lock + .lock() + .map_err(|_| std::io::Error::other("session store append lock was poisoned"))?; + release_uploaded_file_pin(&self.paste_artifact_dir(session_id), artifact_id, owner_id) + } + + fn finalize_uploaded_file_binding( + &self, + session_id: SessionId, + artifact_id: &str, + source_entry_id: &str, + ) -> Result<(), StoreError> { + let _guard = self + .append_lock + .lock() + .map_err(|_| std::io::Error::other("session store append lock was poisoned"))?; + finalize_uploaded_file_binding( + &self.paste_artifact_dir(session_id), + artifact_id, + source_entry_id, + ) + } + + fn reconcile_uploaded_file_pins( + &self, + session_id: SessionId, + live_owner_ids: &[String], + ) -> Result { + let _guard = self + .append_lock + .lock() + .map_err(|_| std::io::Error::other("session store append lock was poisoned"))?; + reconcile_uploaded_file_pins(&self.paste_artifact_dir(session_id), live_owner_ids) + } + fn delete_uploaded_file( &self, session_id: SessionId, @@ -541,13 +598,18 @@ impl Store for FsStore { let Some(source_entry_id) = reference.source_entry_id.as_deref() else { continue; }; - if !self.uploaded_file_is_referenced(session_id, &reference.artifact_id)? { - clear_uploaded_file_binding(&dir, &reference.artifact_id, source_entry_id)?; - if delete_uploaded_file(&dir, &reference.artifact_id)? { - removed = removed - .checked_add(1) - .ok_or(StoreError::ArtifactQuotaExceeded)?; - } + if self.uploaded_file_is_referenced(session_id, &reference.artifact_id)? { + finalize_uploaded_file_binding(&dir, &reference.artifact_id, source_entry_id)?; + continue; + } + if uploaded_file_has_pending_owner(&dir, &reference.artifact_id)? { + continue; + } + clear_uploaded_file_binding(&dir, &reference.artifact_id, source_entry_id)?; + if delete_uploaded_file(&dir, &reference.artifact_id)? { + removed = removed + .checked_add(1) + .ok_or(StoreError::ArtifactQuotaExceeded)?; } } Ok(removed) @@ -865,6 +927,106 @@ mod tests { assert!(store.read_uploaded_file(owner, &reference).is_err()); } + #[test] + fn pending_upload_pin_survives_cleanup_until_release_or_history_binding() { + let tmp = tempfile::TempDir::new().unwrap(); + let store = FsStore::new(tmp.path()).unwrap(); + let session_id = new_session_id(); + let limits = UploadedFileLimits { + max_file_bytes: 64, + max_session_bytes: 128, + }; + let pending = store + .write_uploaded_file(session_id, "pending.txt", "text/plain", b"pending", limits) + .unwrap(); + store + .pin_uploaded_file(session_id, &pending, "submission-1") + .unwrap(); + assert!(matches!( + store.pin_uploaded_file(session_id, &pending, "submission-other"), + Err(StoreError::ArtifactAlreadyCommitted) + )); + drop(store); + let store = FsStore::new(tmp.path()).unwrap(); + assert_eq!( + store.delete_uncommitted_uploaded_files(session_id).unwrap(), + 0 + ); + assert_eq!( + store + .read_uploaded_file_by_id(session_id, &pending.artifact_id) + .unwrap() + .1, + b"pending" + ); + + let fork_session_id = new_session_id(); + assert_eq!( + store + .copy_committed_uploaded_files(session_id, fork_session_id) + .unwrap(), + 0 + ); + assert!( + store + .read_uploaded_file_by_id(fork_session_id, &pending.artifact_id) + .is_err() + ); + + let committed = store + .bind_uploaded_file(session_id, &pending, "entry-1") + .unwrap(); + assert_eq!( + store.delete_uncommitted_uploaded_files(session_id).unwrap(), + 0 + ); + assert!( + store + .read_uploaded_file_by_id(session_id, &pending.artifact_id) + .is_ok() + ); + store + .create_segment( + session_id, + new_segment_id(), + &[LogEntry::InputSegmentsCheckpoint { + ts: 1, + user_segments: vec![vec![protocol::Segment::UploadedFile { + file: committed.clone(), + }]], + }], + ) + .unwrap(); + assert_eq!( + store.delete_uncommitted_uploaded_files(session_id).unwrap(), + 0 + ); + assert!( + store + .release_uploaded_file_pin(session_id, &pending.artifact_id, "submission-1") + .is_err() + ); + + let releasable = store + .write_uploaded_file(session_id, "cancelled.txt", "text/plain", b"cancel", limits) + .unwrap(); + store + .pin_uploaded_file(session_id, &releasable, "submission-2") + .unwrap(); + store + .release_uploaded_file_pin(session_id, &releasable.artifact_id, "submission-2") + .unwrap(); + assert_eq!( + store.delete_uncommitted_uploaded_files(session_id).unwrap(), + 1 + ); + assert!( + store + .read_uploaded_file_by_id(session_id, &releasable.artifact_id) + .is_err() + ); + } + #[test] fn uploaded_file_validation_and_shared_quota_fail_closed() { let tmp = tempfile::TempDir::new().unwrap(); diff --git a/crates/session-store/src/legacy_session_log.rs b/crates/session-store/src/legacy_session_log.rs index 40dd0fd5..bb9dd1dd 100644 --- a/crates/session-store/src/legacy_session_log.rs +++ b/crates/session-store/src/legacy_session_log.rs @@ -183,6 +183,7 @@ fn canonicalize_history_entry( item, metadata: legacy_metadata(segment_id, line_index, 0), }, + extensions: Vec::new(), }, } } diff --git a/crates/session-store/src/public_snapshot.rs b/crates/session-store/src/public_snapshot.rs index f3b61754..b1793584 100644 --- a/crates/session-store/src/public_snapshot.rs +++ b/crates/session-store/src/public_snapshot.rs @@ -71,7 +71,7 @@ pub fn project_session_snapshot(session_id: SessionId, log: &[LogEntry]) -> Sess entries.push(history_entry(entry, *ts, data)); } } - LogEntry::AnnotatedSystemItem { ts, entry } => entries.push(system_entry( + LogEntry::AnnotatedSystemItem { ts, entry, .. } => entries.push(system_entry( &entry.item, entry.metadata.entry_id.0.clone(), *ts, @@ -100,7 +100,10 @@ pub fn project_session_snapshot(session_id: SessionId, log: &[LogEntry]) -> Sess } } - SessionSnapshot { entries } + SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), + entries, + } } fn extend_history( diff --git a/crates/session-store/src/segment.rs b/crates/session-store/src/segment.rs index 163b2317..75d997c2 100644 --- a/crates/session-store/src/segment.rs +++ b/crates/session-store/src/segment.rs @@ -287,6 +287,7 @@ pub fn append_system_item( LogEntry::AnnotatedSystemItem { ts: segment_log::now_millis(), entry, + extensions: Vec::new(), }, ) } diff --git a/crates/session-store/src/segment_log.rs b/crates/session-store/src/segment_log.rs index 9daf85c2..ee509cbb 100644 --- a/crates/session-store/src/segment_log.rs +++ b/crates/session-store/src/segment_log.rs @@ -112,6 +112,8 @@ pub enum LogEntry { AnnotatedSystemItem { ts: u64, entry: LoggedSystemHistoryEntry, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + extensions: Vec, }, /// Turn boundary. Records the turn count after increment. @@ -312,12 +314,19 @@ pub fn collect_state(entries: &[LogEntry]) -> RestoredState { state.annotated_history.push(entry.clone()); state.history.push(Item::from(entry.item.clone())); } - LogEntry::AnnotatedSystemItem { entry, .. } => { + LogEntry::AnnotatedSystemItem { + entry, extensions, .. + } => { state.annotated_history.push(LoggedHistoryEntry { item: LoggedItem::from(entry.item.to_history_item()), metadata: entry.metadata.clone(), }); state.history.push(entry.item.to_history_item()); + state.extensions.extend( + extensions + .iter() + .map(|extension| (extension.domain.clone(), extension.payload.clone())), + ); } LogEntry::TurnEnd { turn_count, .. } => { if let Some(active_turn_count) = &mut state.active_run_turn_count { diff --git a/crates/session-store/src/store.rs b/crates/session-store/src/store.rs index a22fd6c7..33daae39 100644 --- a/crates/session-store/src/store.rs +++ b/crates/session-store/src/store.rs @@ -226,6 +226,47 @@ pub trait Store: Send + Sync { Err(StoreError::PasteArtifactUnsupported) } + /// Retain an uploaded file while a durable pending operation owns it. + fn pin_uploaded_file( + &self, + _session_id: SessionId, + _reference: &UploadedFileRef, + _owner_id: &str, + ) -> Result<(), StoreError> { + Err(StoreError::PasteArtifactUnsupported) + } + + /// Release a pending-operation pin without changing committed ownership. + fn release_uploaded_file_pin( + &self, + _session_id: SessionId, + _artifact_id: &str, + _owner_id: &str, + ) -> Result<(), StoreError> { + Err(StoreError::PasteArtifactUnsupported) + } + + /// Complete the pending-to-history handoff after the history entry commits. + fn finalize_uploaded_file_binding( + &self, + _session_id: SessionId, + _artifact_id: &str, + _source_entry_id: &str, + ) -> Result<(), StoreError> { + Err(StoreError::PasteArtifactUnsupported) + } + + /// Clear pending-operation pins that have no owner in restored durable + /// Worker Session state. This repairs an interrupted pin-before-checkpoint + /// acceptance without disturbing live queue owners or committed history. + fn reconcile_uploaded_file_pins( + &self, + _session_id: SessionId, + _live_owner_ids: &[String], + ) -> Result { + Ok(0) + } + /// Delete an uncommitted uploaded file owned by `session_id`. fn delete_uploaded_file( &self, diff --git a/crates/session-store/src/uploaded_file.rs b/crates/session-store/src/uploaded_file.rs index 4ce37588..6583dd25 100644 --- a/crates/session-store/src/uploaded_file.rs +++ b/crates/session-store/src/uploaded_file.rs @@ -24,6 +24,12 @@ pub const DEFAULT_MAX_FILES_PER_SUBMISSION: usize = 8; pub const DEFAULT_MAX_SESSION_UPLOADED_FILES: u64 = 256; const MAX_FILE_NAME_CHARS: usize = 255; const MAX_MEDIA_TYPE_BYTES: usize = 127; +fn validate_pending_owner_id(owner_id: &str) -> Result<()> { + if owner_id.is_empty() || owner_id.len() > 256 { + return Err(StoreError::ArtifactIntegrityMismatch); + } + Ok(()) +} #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct UploadedFileLimits { @@ -59,6 +65,8 @@ struct StoredUploadedFile { #[serde(default, skip_serializing_if = "Option::is_none")] source_entry_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] + pending_owner_id: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] upload_context: Option, content_base64: String, } @@ -291,6 +299,7 @@ pub(crate) fn write_uploaded_file( byte_len, sha256: sha256.clone(), source_entry_id: None, + pending_owner_id: None, upload_context: context.cloned(), content_base64: BASE64.encode(content), }; @@ -338,6 +347,12 @@ pub(crate) fn read_uploaded_file_by_id( Ok((reference, content)) } +pub(crate) fn uploaded_file_has_pending_owner(dir: &Path, artifact_id: &str) -> Result { + let path = record_path(dir, artifact_id)?; + let stored: StoredUploadedFile = serde_json::from_slice(&fs::read(path)?)?; + Ok(stored.pending_owner_id.is_some()) +} + pub(crate) fn read_uploaded_file(dir: &Path, reference: &UploadedFileRef) -> Result> { let (stored_reference, content) = read_uploaded_file_by_id(dir, &reference.artifact_id)?; if stored_reference.file_name != reference.file_name @@ -376,6 +391,98 @@ pub(crate) fn clear_uploaded_file_binding( Ok(()) } +pub(crate) fn pin_uploaded_file( + dir: &Path, + reference: &UploadedFileRef, + owner_id: &str, +) -> Result<()> { + validate_pending_owner_id(owner_id)?; + if reference.source_entry_id.is_some() { + return Err(StoreError::ArtifactAlreadyCommitted); + } + let aggregate_lock = fs::OpenOptions::new() + .create(true) + .read(true) + .write(true) + .open(dir.join(".aggregate.lock"))?; + FileExt::lock_exclusive(&aggregate_lock)?; + let path = record_path(dir, &reference.artifact_id)?; + let mut stored: StoredUploadedFile = serde_json::from_slice(&fs::read(&path)?)?; + if stored.file_name != reference.file_name + || stored.media_type != reference.media_type + || stored.created_at_ms != reference.created_at_ms + || stored.byte_len != reference.byte_len + || stored.sha256 != reference.sha256 + { + return Err(StoreError::ArtifactIntegrityMismatch); + } + if stored.source_entry_id.is_some() { + return Err(StoreError::ArtifactAlreadyCommitted); + } + if let Some(existing_owner) = stored.pending_owner_id.as_deref() { + return if existing_owner == owner_id { + Ok(()) + } else { + Err(StoreError::ArtifactAlreadyCommitted) + }; + } + stored.pending_owner_id = Some(owner_id.to_owned()); + let temp = dir.join(format!(".{}.file.pin.tmp", reference.artifact_id)); + fs::write(&temp, serde_json::to_vec(&stored)?)?; + fs::rename(temp, path)?; + Ok(()) +} + +pub(crate) fn release_uploaded_file_pin( + dir: &Path, + artifact_id: &str, + owner_id: &str, +) -> Result<()> { + validate_pending_owner_id(owner_id)?; + let aggregate_lock = fs::OpenOptions::new() + .create(true) + .read(true) + .write(true) + .open(dir.join(".aggregate.lock"))?; + FileExt::lock_exclusive(&aggregate_lock)?; + let path = record_path(dir, artifact_id)?; + let mut stored: StoredUploadedFile = serde_json::from_slice(&fs::read(&path)?)?; + if stored.pending_owner_id.as_deref() != Some(owner_id) { + return Err(StoreError::ArtifactIntegrityMismatch); + } + stored.pending_owner_id = None; + let temp = dir.join(format!(".{artifact_id}.file.unpin.tmp")); + fs::write(&temp, serde_json::to_vec(&stored)?)?; + fs::rename(temp, path)?; + Ok(()) +} + +pub(crate) fn finalize_uploaded_file_binding( + dir: &Path, + artifact_id: &str, + source_entry_id: &str, +) -> Result<()> { + let aggregate_lock = fs::OpenOptions::new() + .create(true) + .read(true) + .write(true) + .open(dir.join(".aggregate.lock"))?; + FileExt::lock_exclusive(&aggregate_lock)?; + let path = record_path(dir, artifact_id)?; + let mut stored: StoredUploadedFile = serde_json::from_slice(&fs::read(&path)?)?; + if stored.source_entry_id.as_deref() != Some(source_entry_id) { + return Err(StoreError::ArtifactIntegrityMismatch); + } + if stored.pending_owner_id.is_none() { + return Ok(()); + } + stored.pending_owner_id = None; + let temp = dir.join(format!(".{artifact_id}.file.finalize.tmp")); + fs::write(&temp, serde_json::to_vec(&stored)?)?; + fs::rename(temp, path)?; + Ok(()) +} + pub(crate) fn bind_uploaded_file( dir: &Path, reference: &UploadedFileRef, @@ -479,6 +586,40 @@ pub(crate) fn copy_committed_uploaded_files(source_dir: &Path, target_dir: &Path Ok(copied) } +pub(crate) fn reconcile_uploaded_file_pins(dir: &Path, live_owner_ids: &[String]) -> Result { + fs::create_dir_all(dir)?; + let aggregate_lock = fs::OpenOptions::new() + .create(true) + .read(true) + .write(true) + .open(dir.join(".aggregate.lock"))?; + FileExt::lock_exclusive(&aggregate_lock)?; + let mut reconciled = 0_u64; + for entry in fs::read_dir(dir)? { + let entry = entry?; + let path = entry.path(); + let Some(file_name) = path.file_name().and_then(|name| name.to_str()) else { + continue; + }; + let Some(artifact_id) = file_name.strip_suffix(".file.json") else { + continue; + }; + let mut stored: StoredUploadedFile = serde_json::from_slice(&fs::read(&path)?)?; + let Some(owner_id) = stored.pending_owner_id.as_deref() else { + continue; + }; + if live_owner_ids.iter().any(|live| live == owner_id) { + continue; + } + stored.pending_owner_id = None; + let temp = dir.join(format!(".{artifact_id}.file.reconcile.tmp")); + fs::write(&temp, serde_json::to_vec(&stored)?)?; + fs::rename(temp, path)?; + reconciled = reconciled.saturating_add(1); + } + Ok(reconciled) +} + pub(crate) fn delete_uncommitted_uploaded_files(dir: &Path) -> Result { fs::create_dir_all(dir)?; let aggregate_lock = fs::OpenOptions::new() @@ -499,7 +640,7 @@ pub(crate) fn delete_uncommitted_uploaded_files(dir: &Path) -> Result { continue; } let stored: StoredUploadedFile = serde_json::from_slice(&fs::read(&path)?)?; - if stored.source_entry_id.is_none() { + if stored.source_entry_id.is_none() && stored.pending_owner_id.is_none() { fs::remove_file(path)?; removed = removed .checked_add(1) @@ -523,7 +664,7 @@ pub(crate) fn delete_uploaded_file(dir: &Path, artifact_id: &str) -> Result return Ok(false), Err(error) => return Err(error.into()), }; - if stored.source_entry_id.is_some() { + if stored.source_entry_id.is_some() || stored.pending_owner_id.is_some() { return Err(StoreError::ArtifactAlreadyCommitted); } match fs::remove_file(path) { diff --git a/crates/standalone/src/host.rs b/crates/standalone/src/host.rs index c4c5df73..1131bccd 100644 --- a/crates/standalone/src/host.rs +++ b/crates/standalone/src/host.rs @@ -318,7 +318,11 @@ impl StandaloneHost { } pub async fn shutdown(mut self) -> Result<(), StandaloneShutdownError> { - let _ = self.handle.send(Method::Shutdown).await; + let command = protocol::WorkerCommandEnvelope::for_snapshot( + u64::MAX, + &self.handle.shared_state.snapshot(), + ); + let _ = self.handle.send(Method::Shutdown { command }).await; let Some(shutdown) = self.shutdown.take() else { self.retain_lease(); return Err(StandaloneShutdownError::ConfirmationLost); @@ -500,7 +504,11 @@ fn active_pointer( } async fn stop_started_worker(started: BootstrappedWorker) { - let _ = started.handle.send(Method::Shutdown).await; + let command = protocol::WorkerCommandEnvelope::for_snapshot( + u64::MAX, + &started.handle.shared_state.snapshot(), + ); + let _ = started.handle.send(Method::Shutdown { command }).await; let _ = tokio::time::timeout(Duration::from_secs(2), started.shutdown).await; } diff --git a/crates/standalone/tests/host.rs b/crates/standalone/tests/host.rs index 510cc68c..45c8f620 100644 --- a/crates/standalone/tests/host.rs +++ b/crates/standalone/tests/host.rs @@ -99,7 +99,10 @@ async fn in_process_host_runs_text_and_read_tool_then_shuts_down() { let mut protocol_client = host.connect(); protocol_client - .send(&Method::run_text("read the probe")) + .send(&Method::submit_text( + protocol::new_submission_request_id(), + "read the probe", + )) .await .expect("submit input"); @@ -336,11 +339,15 @@ async fn standalone_restore_preserves_history_tasks_notifications_and_cwd_scope( let worker_id = host.worker_id(); let mut protocol_client = host.connect(); protocol_client - .send(&Method::run_text("first request")) + .send(&Method::submit_text( + protocol::new_submission_request_id(), + "first request", + )) .await?; wait_for_run_end(&mut protocol_client).await?; protocol_client .send(&Method::Notify { + notification_request_id: protocol::new_submission_request_id(), message: "persisted notification".to_string(), auto_run: true, }) @@ -394,7 +401,10 @@ async fn standalone_restore_preserves_history_tasks_notifications_and_cwd_scope( assert!(snapshot.contains("persisted notification"), "{snapshot}"); protocol_client - .send(&Method::run_text("continue after restore")) + .send(&Method::submit_text( + protocol::new_submission_request_id(), + "continue after restore", + )) .await?; wait_for_run_end(&mut protocol_client).await?; let request = second_inspection diff --git a/crates/tools/src/bash.rs b/crates/tools/src/bash.rs index fb68fcbc..433dec0c 100644 --- a/crates/tools/src/bash.rs +++ b/crates/tools/src/bash.rs @@ -118,6 +118,7 @@ impl Tool for BashTool { command: params.command, timeout_secs, output_limit: INLINE_BYTE_BUDGET, + cwd: None, spill_dir: Some(self.output_dir.clone()), tool_call_id: Some(call_id.clone()), }) diff --git a/crates/tui/src/app.rs b/crates/tui/src/app.rs index b1f4fd04..9858dd39 100644 --- a/crates/tui/src/app.rs +++ b/crates/tui/src/app.rs @@ -5,7 +5,7 @@ use std::time::{Duration, Instant}; use protocol::{ AlertLevel, AlertSource, CompletionEntry, CompletionKind, ErrorCode, Event, InFlightBlock, InFlightSnapshot, InFlightToolCallState, InternalWorkerRef, InternalWorkerSnapshot, Method, - RewindTarget, RunResult, Segment, WorkerStatus, + RewindTarget, RunResult, Segment, WorkerCommandEnvelope, WorkerStateSnapshot, WorkerStatus, }; use crate::block::{ @@ -102,23 +102,6 @@ struct RollbackSubmitState { turn_before: usize, } -#[derive(Clone)] -pub struct QueuedInput { - segments: Vec, - preview: String, -} - -impl QueuedInput { - fn new(segments: Vec) -> Self { - let preview = Segment::flatten_to_text(&segments); - Self { segments, preview } - } - - pub fn preview(&self) -> &str { - &self.preview - } -} - struct ComposerInputHistory { entries: VecDeque>, browse: Option, @@ -242,8 +225,10 @@ pub struct WorkerViewTab { pub struct App { pub worker_name: String, pub connected: bool, - /// Last controller status reported by the Worker. Drives the status line - /// and Ctrl-key routing; do not infer this solely from replayed history. + /// Latest authoritative revisioned live execution state. + pub worker_state: WorkerStateSnapshot, + next_command_id: u64, + /// Derived Runtime-catalog compatibility projection used by existing UI. pub worker_status: WorkerStatus, /// True while the Worker is in `WorkerStatus::Running`. pub running: bool, @@ -272,7 +257,7 @@ pub struct App { /// Current transient actionbar notice. Notices are local UI state only: /// they are never appended to transcript/session history or LLM context. actionbar_notice: Option, - /// Normal composer input that is submitted as `Method::Run`. + /// Normal composer input that is submitted as `Method::Submit`. pub input: InputBuffer, /// Separate command-line input. It is never submitted as a user message. pub command_input: InputBuffer, @@ -333,9 +318,8 @@ pub struct App { /// Top entry index of the task pane's visible window. Clamped on /// render so it never points past the end of the list. pub task_pane_scroll: usize, - /// TUI-local FIFO of user inputs submitted while the Worker is already running. - /// Entries have not been sent to the Worker yet, so they remain editable/cancellable locally. - queued_inputs: VecDeque, + /// Authoritative WorkerSession FIFO summary received from snapshot/live events. + pending_submissions: protocol::PendingSubmissionsSnapshot, /// TUI-local readline-style composer input history. This is intentionally /// client-side only: recalled entries are plain drafts until submitted again. input_history: ComposerInputHistory, @@ -355,6 +339,8 @@ impl App { Self { worker_name, connected: false, + worker_state: WorkerStateSnapshot::initial(1), + next_command_id: 1, worker_status: WorkerStatus::Idle, running: false, paused: false, @@ -395,7 +381,7 @@ impl App { text_selection: TextSelectionState::default(), task_pane_open: false, task_pane_scroll: 0, - queued_inputs: VecDeque::new(), + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), input_history: ComposerInputHistory::new(), input_history_store: None, pending_submit_rollback: None, @@ -763,23 +749,40 @@ impl App { if self.paused { self.input_history.cancel_browse(); self.input.clear(); - return Some(Method::Resume); + let command = self.next_command_envelope(); + return Some(Method::Resume { command }); } return None; } self.record_input_history(segments.clone()); - if self.running { - self.queued_inputs.push_back(QueuedInput::new(segments)); - self.input.clear(); - self.completion = None; - return None; - } self.input.clear(); Some(self.method_for_run(segments)) } + pub fn submit_notify_input(&mut self) -> Option { + let segments = self.input.submit_segments(); + if segments_are_blank(&segments) { + return None; + } + if segments + .iter() + .any(|segment| matches!(segment, Segment::UploadedFile { .. })) + { + self.push_error("Notify accepts text only; remove attachments or queue a Submit."); + return None; + } + let message = Segment::flatten_to_text(&segments); + self.record_input_history(segments); + self.input.clear(); + Some(Method::Notify { + notification_request_id: protocol::new_submission_request_id(), + message, + auto_run: true, + }) + } + pub fn restore_unsent_run(&mut self, method: &Method) { - let Method::Run { input } = method else { + let Method::Submit { input, .. } = method else { return; }; self.pending_submit_rollback = None; @@ -787,8 +790,9 @@ impl App { self.input.replace_with_segments(input); self.completion = None; } else { - self.queued_inputs - .push_front(QueuedInput::new(input.clone())); + self.push_error( + "Submit transport failed; current Composer was preserved and the unsent input was not queued.", + ); } } @@ -804,7 +808,10 @@ impl App { block_start: self.blocks.len(), turn_before: self.turn_index, }); - Method::Run { input: segments } + Method::Submit { + submission_request_id: protocol::new_submission_request_id(), + input: segments, + } } fn record_input_history(&mut self, segments: Vec) { @@ -825,7 +832,7 @@ impl App { } pub fn queued_input_count(&self) -> usize { - self.queued_inputs.len() + self.pending_submissions.submissions.len() } #[cfg(test)] @@ -910,36 +917,31 @@ impl App { } } + pub fn continue_pending_method(&self) -> Option { + Some(Method::ContinuePending { + expected_revision: self.pending_submissions.revision, + expected_head_id: self.pending_submissions.head_id.clone()?, + }) + } + + pub fn clear_pending_method(&self) -> Method { + Method::ClearPendingSubmissions { + expected_revision: self.pending_submissions.revision, + } + } + + pub fn cancel_pending_method(&self, submission_id: String) -> Method { + Method::CancelPendingSubmission { + submission_id, + expected_revision: self.pending_submissions.revision, + } + } + pub fn next_queued_input_preview(&self) -> Option<&str> { - self.queued_inputs.front().map(QueuedInput::preview) - } - - pub fn clear_queued_inputs(&mut self) -> usize { - let cleared = self.queued_inputs.len(); - self.queued_inputs.clear(); - cleared - } - - pub fn restore_next_queued_input_to_composer(&mut self) -> bool { - if self.queued_inputs.is_empty() { - return false; - } - if !self.input.is_empty() { - self.push_error("Composer is not empty; clear it before editing queued input."); - return false; - } - let Some(queued) = self.queued_inputs.pop_front() else { - return false; - }; - self.input_history.cancel_browse(); - self.input.replace_with_segments(&queued.segments); - self.completion = None; - true - } - - fn pop_next_queued_run(&mut self) -> Option { - let queued = self.queued_inputs.pop_front()?; - Some(self.method_for_run(queued.segments)) + self.pending_submissions + .submissions + .first() + .map(|submission| submission.submission_id.as_str()) } pub fn clear_actionbar_notice(&mut self) { @@ -1117,12 +1119,42 @@ impl App { } } + pub fn next_command_envelope(&mut self) -> WorkerCommandEnvelope { + let command_id = self + .next_command_id + .max(self.worker_state.last_command_id.saturating_add(1)); + let command = WorkerCommandEnvelope::for_snapshot(command_id, &self.worker_state); + self.next_command_id = command_id.saturating_add(1); + command + } + + fn apply_worker_state_snapshot(&mut self, snapshot: &WorkerStateSnapshot) { + match protocol::apply_worker_state_snapshot(&mut self.worker_state, snapshot) { + Ok(protocol::WorkerStateSnapshotApply::Applied) => { + self.set_worker_status(self.worker_state.catalog_status()); + } + Ok( + protocol::WorkerStateSnapshotApply::Duplicate + | protocol::WorkerStateSnapshotApply::Stale, + ) => {} + Err(error) => self.handle_error( + ErrorCode::Internal, + format!("worker state stream rejected: {error}"), + ), + } + } + pub fn handle_worker_event(&mut self, event: Event) -> Option { if self.rewind_refresh_fence && event_is_stale_after_rewind(&event) { return None; } match event { + Event::SubmissionAccepted { .. } => {} + Event::SubmissionRejected { message, .. } => self.push_error(message), + Event::PendingSubmissionsChanged { pending } => { + self.pending_submissions = pending; + } Event::UserMessage { segments } => { self.turn_index += 1; self.blocks.push(Block::TurnHeader { @@ -1148,18 +1180,14 @@ impl App { self.assistant_streaming = false; } Event::TurnStart { .. } => { - self.set_worker_status(WorkerStatus::Running); self.run_requests += 1; self.current_tool = None; self.latest_llm_wait_event = None; self.assistant_streaming = false; } - Event::InvokeStart { .. } => { - self.set_worker_status(WorkerStatus::Running); - } + Event::InvokeStart { .. } => {} // UI consumers of per-attempt LlmCall semantics remain out of scope; - // the run-level status starts at InvokeStart and TurnStart counts each - // LLM request within that run. + // authoritative run state comes only from WorkerStateSnapshot. Event::LlmCallStart { .. } | Event::LlmCallEnd { .. } => { self.latest_llm_wait_event = None; } @@ -1366,15 +1394,7 @@ impl App { output_tokens: self.run_output_tokens, }); self.pending_submit_rollback = None; - self.reset_run_state(match result { - RunResult::Paused => WorkerStatus::Paused, - RunResult::Finished | RunResult::LimitReached | RunResult::RolledBack => { - WorkerStatus::Idle - } - }); - if matches!(result, RunResult::Finished | RunResult::LimitReached) { - return self.pop_next_queued_run(); - } + self.reset_run_state(); } } Event::CompactStart { .. } => { @@ -1444,14 +1464,15 @@ impl App { Event::Snapshot { session, greeting, - status, + state, in_flight, internal_workers, } => { self.rewind_refresh_fence = false; + self.pending_submissions = session.pending_submissions.clone(); self.restore_snapshot(&session, greeting, in_flight); self.replace_internal_worker_snapshots(internal_workers); - self.set_worker_status(status); + self.apply_worker_state_snapshot(&state); } Event::InternalWorker { worker, @@ -1461,9 +1482,12 @@ impl App { Event::InternalWorkerRemoved { worker, revision } => { self.remove_internal_worker(worker, revision) } - Event::Status { status } => { + Event::WorkerState { snapshot } => { self.rewind_refresh_fence = false; - self.set_worker_status(status); + self.apply_worker_state_snapshot(&snapshot); + } + Event::CommandAcknowledged { acknowledgement } => { + self.apply_worker_state_snapshot(&acknowledgement.state); } // Command telemetry is an operational Web Console surface. The // TUI continues to render the final Bash ToolResult from history. @@ -1503,7 +1527,7 @@ impl App { }; self.completion = None; self.close_rewind_picker(); - self.reset_run_state(self.worker_status); + self.reset_run_state(); let mut message = if restored_composer { format!( "Rewound session: discarded {} log entries; restored selected input to composer.", @@ -1551,8 +1575,7 @@ impl App { None } - fn reset_run_state(&mut self, status: WorkerStatus) { - self.set_worker_status(status); + fn reset_run_state(&mut self) { self.run_requests = 0; self.run_upload_tokens = 0; self.run_output_tokens = 0; @@ -1582,7 +1605,7 @@ impl App { "Rolled back empty assistant turn; no local submitted input was available to restore." .to_owned() }; - self.reset_run_state(WorkerStatus::Idle); + self.reset_run_state(); self.blocks.push(Block::Alert { level: AlertLevel::Warn, source: AlertSource::Worker, @@ -2026,12 +2049,18 @@ impl App { self.input_mode = CommandInputMode::Composer; self.command_completion_selected = None; } - if let Some(Method::ListRewindTargets) = result.method.as_ref() { + let mut method = result.method; + if let Some(Method::Compact { .. }) = method { + method = Some(Method::Compact { + command: self.next_command_envelope(), + }); + } + if let Some(Method::ListRewindTargets) = method.as_ref() { self.completion = None; self.rewind_picker = None; self.rewind_request_pending = true; } - result.method + method } fn push_command_diagnostic(&mut self, message: impl Into) { @@ -2681,7 +2710,10 @@ mod rewind_refresh_tests { }); app.handle_worker_event(Event::RewindApplied { - session: protocol::SessionSnapshot { entries: vec![] }, + session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), + entries: vec![], + }, input: vec![Segment::text("selected rewind input")], summary: summary(3), }); @@ -2700,7 +2732,10 @@ mod rewind_refresh_tests { }); app.handle_worker_event(Event::RewindApplied { - session: protocol::SessionSnapshot { entries: vec![] }, + session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), + entries: vec![], + }, input: vec![Segment::text("rewound input")], summary: summary(1), }); @@ -2743,7 +2778,10 @@ mod rewind_refresh_tests { }); app.handle_worker_event(Event::RewindApplied { - session: protocol::SessionSnapshot { entries: vec![] }, + session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), + entries: vec![], + }, input: vec![Segment::text("rewound input")], summary: summary(2), }); @@ -2752,8 +2790,8 @@ mod rewind_refresh_tests { }); assert!(!blocks_contain(&app, "stale tail after rewind")); - app.handle_worker_event(Event::Status { - status: WorkerStatus::Idle, + app.handle_worker_event(Event::WorkerState { + snapshot: WorkerStatus::Idle.into(), }); app.handle_worker_event(Event::TextDelta { text: "new live tail after status".into(), @@ -2877,7 +2915,7 @@ mod composer_history_persistence_tests { path: "src/lib.rs".into(), }, ]); - assert!(matches!(app.submit_input(), Some(Method::Run { .. }))); + assert!(matches!(app.submit_input(), Some(Method::Submit { .. }))); let mut reloaded = App::new_with_input_history_store("test".into(), store); assert!(reloaded.browse_input_history_older()); @@ -2958,7 +2996,7 @@ mod composer_history_persistence_tests { app.insert_char(c); } match app.submit_input() { - Some(Method::Run { input }) => input, + Some(Method::Submit { input, .. }) => input, other => panic!("expected Run, got {other:?}"), } } @@ -3424,72 +3462,44 @@ mod completion_flow_tests { } #[test] - fn running_submit_is_queued_locally_and_clears_composer() { + fn running_submit_is_sent_to_the_worker_and_not_queued_locally() { let mut app = App::new("test".into()); app.set_worker_status(WorkerStatus::Running); insert_text(&mut app, "queued turn"); - assert!(app.submit_input().is_none()); + let method = app.submit_input(); - assert_eq!(app.queued_input_count(), 1); - assert_eq!(app.next_queued_input_preview(), Some("queued turn")); + assert!(matches!(method, Some(Method::Submit { .. }))); + assert_eq!(app.queued_input_count(), 0); assert_eq!(input_text(&app), ""); } #[test] - fn finished_run_auto_sends_next_queued_input() { + fn pending_submission_projection_is_worker_authoritative() { let mut app = App::new("test".into()); - app.set_worker_status(WorkerStatus::Running); - insert_text(&mut app, "next turn"); - assert!(app.submit_input().is_none()); - - let method = app.handle_worker_event(Event::RunEnd { - result: RunResult::Finished, + app.handle_worker_event(Event::PendingSubmissionsChanged { + pending: protocol::PendingSubmissionsSnapshot { + revision: 3, + notification_count: 0, + head_id: Some("submission-1".into()), + submissions: vec![protocol::PendingSubmissionSummary { + submission_id: "submission-1".into(), + accepted_at_ms: 7, + segment_count: 2, + byte_len: 42, + }], + }, }); - match method { - Some(Method::Run { input }) => { - assert_eq!(Segment::flatten_to_text(&input), "next turn"); - } - other => panic!("expected queued Run, got {other:?}"), - } - assert_eq!(app.queued_input_count(), 0); - } - - #[test] - fn limit_reached_run_auto_sends_next_queued_input() { - let mut app = App::new("test".into()); - app.set_worker_status(WorkerStatus::Running); - insert_text(&mut app, "next after limit"); - assert!(app.submit_input().is_none()); - - let method = app.handle_worker_event(Event::RunEnd { - result: RunResult::LimitReached, - }); - - match method { - Some(Method::Run { input }) => { - assert_eq!(Segment::flatten_to_text(&input), "next after limit"); - } - other => panic!("expected queued Run, got {other:?}"), - } - assert_eq!(app.queued_input_count(), 0); - } - - #[test] - fn paused_and_rolled_back_run_do_not_auto_send_queue() { - for result in [RunResult::Paused, RunResult::RolledBack] { - let mut app = App::new("test".into()); - app.set_worker_status(WorkerStatus::Running); - insert_text(&mut app, "held turn"); - assert!(app.submit_input().is_none()); - - let method = app.handle_worker_event(Event::RunEnd { result }); - - assert!(method.is_none()); - assert_eq!(app.queued_input_count(), 1); - assert_eq!(app.next_queued_input_preview(), Some("held turn")); - } + assert_eq!(app.queued_input_count(), 1); + assert_eq!(app.next_queued_input_preview(), Some("submission-1")); + assert!( + app.handle_worker_event(Event::RunEnd { + result: RunResult::Finished, + }) + .is_none() + ); + assert_eq!(app.queued_input_count(), 1); } #[test] @@ -3497,25 +3507,7 @@ mod completion_flow_tests { let mut app = App::new("test".into()); app.set_worker_status(WorkerStatus::Paused); - assert!(matches!(app.submit_input(), Some(Method::Resume))); - assert_eq!(app.queued_input_count(), 0); - } - - #[test] - fn queued_input_can_be_restored_to_composer_or_cleared() { - let mut app = App::new("test".into()); - app.set_worker_status(WorkerStatus::Running); - insert_text(&mut app, "edit me"); - assert!(app.submit_input().is_none()); - - assert!(app.restore_next_queued_input_to_composer()); - assert_eq!(app.queued_input_count(), 0); - assert_eq!(input_text(&app), "edit me"); - - app.input.clear(); - insert_text(&mut app, "clear me"); - assert!(app.submit_input().is_none()); - assert_eq!(app.clear_queued_inputs(), 1); + assert!(matches!(app.submit_input(), Some(Method::Resume { .. }))); assert_eq!(app.queued_input_count(), 0); } @@ -3530,7 +3522,7 @@ mod completion_flow_tests { app.insert_char(c); } match app.submit_input() { - Some(Method::Run { input }) => input, + Some(Method::Submit { input, .. }) => input, other => panic!("expected Run, got {other:?}"), } } @@ -3570,7 +3562,7 @@ mod completion_flow_tests { app.handle_worker_event(Event::Snapshot { greeting: test_greeting(), session: public_session(vec![session_start_value]), - status: WorkerStatus::Running, + state: test_worker_state(WorkerStatus::Running), in_flight: Default::default(), internal_workers: Vec::new(), }); @@ -3581,6 +3573,90 @@ mod completion_flow_tests { assert!(matches!(app.blocks.first(), Some(Block::Greeting(_)))); } + #[test] + fn occurrence_events_do_not_infer_foreground_worker_state() { + let mut app = App::new("test".into()); + app.handle_worker_event(Event::TurnStart { turn: 1 }); + app.handle_worker_event(Event::InvokeStart { + kind: protocol::InvokeKind::UserSend, + }); + app.handle_worker_event(Event::RunEnd { + result: RunResult::Paused, + }); + assert_eq!(app.worker_state.state, protocol::WorkerState::Idle); + assert_eq!(app.worker_status, WorkerStatus::Idle); + + let running = WorkerStateSnapshot { + execution_generation: 1, + revision: 1, + state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Running, + )), + last_command_id: 0, + }; + app.handle_worker_event(Event::WorkerState { + snapshot: running.clone(), + }); + app.handle_worker_event(Event::RunEnd { + result: RunResult::Finished, + }); + assert_eq!(app.worker_state, running); + assert_eq!(app.worker_status, WorkerStatus::Running); + } + + #[test] + fn worker_state_events_and_acknowledgements_share_monotonic_application() { + let mut app = App::new("test".into()); + let running = WorkerStateSnapshot { + execution_generation: 4, + revision: 3, + state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Running, + )), + last_command_id: 2, + }; + app.handle_worker_event(Event::WorkerState { + snapshot: running.clone(), + }); + app.handle_worker_event(Event::WorkerState { + snapshot: WorkerStateSnapshot { + revision: 2, + state: protocol::WorkerState::Idle, + ..running.clone() + }, + }); + assert_eq!(app.worker_state, running); + + let paused = WorkerStateSnapshot { + revision: 4, + state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Paused, + )), + last_command_id: 3, + ..running.clone() + }; + app.handle_worker_event(Event::CommandAcknowledged { + acknowledgement: protocol::WorkerCommandAcknowledgement { + command_id: 3, + command: protocol::WorkerCommandKind::Pause, + disposition: protocol::WorkerCommandDisposition::Accepted, + state: paused.clone(), + }, + }); + assert_eq!(app.worker_state, paused); + + app.handle_worker_event(Event::WorkerState { + snapshot: WorkerStateSnapshot { + state: protocol::WorkerState::Idle, + ..paused.clone() + }, + }); + assert_eq!(app.worker_state, paused); + assert!(app.run_error_messages.iter().any(|message| { + message.contains("conflicting worker state snapshots at generation 4 revision 4") + })); + } + #[test] fn snapshot_replaces_live_error_with_one_durable_run_error_block() { let mut app = App::new("test".into()); @@ -3588,8 +3664,8 @@ mod completion_flow_tests { code: ErrorCode::ProviderError, message: "provider unavailable".into(), }); - app.handle_worker_event(Event::Status { - status: WorkerStatus::Idle, + app.handle_worker_event(Event::WorkerState { + snapshot: WorkerStatus::Idle.into(), }); let live_errors = app @@ -3614,7 +3690,7 @@ mod completion_flow_tests { app.handle_worker_event(Event::Snapshot { greeting: test_greeting(), session: public_session(vec![serde_json::to_value(run_errored).unwrap()]), - status: WorkerStatus::Idle, + state: test_worker_state(WorkerStatus::Idle), in_flight: Default::default(), internal_workers: Vec::new(), }); @@ -3675,9 +3751,10 @@ mod completion_flow_tests { app.handle_worker_event(Event::Snapshot { greeting: test_greeting(), session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, - status: WorkerStatus::Running, + state: test_worker_state(WorkerStatus::Running), in_flight: InFlightSnapshot { blocks: vec![ InFlightBlock::Thinking { @@ -3783,6 +3860,7 @@ mod completion_flow_tests { revision, status: WorkerStatus::Idle, session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, in_flight: protocol::InFlightSnapshot::default(), @@ -4000,9 +4078,10 @@ mod completion_flow_tests { app.handle_worker_event(Event::Snapshot { greeting: test_greeting(), session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, - status: WorkerStatus::Idle, + state: test_worker_state(WorkerStatus::Idle), in_flight: Default::default(), internal_workers: Vec::new(), }); @@ -4051,9 +4130,10 @@ mod completion_flow_tests { app.handle_worker_event(Event::Snapshot { greeting: test_greeting(), session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, - status: WorkerStatus::Idle, + state: test_worker_state(WorkerStatus::Idle), in_flight: Default::default(), internal_workers: vec![InternalWorkerSnapshot { worker: InternalWorkerRef { @@ -4064,6 +4144,7 @@ mod completion_flow_tests { }, revision: 4, session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, status: WorkerStatus::Running, @@ -4200,6 +4281,13 @@ mod completion_flow_tests { .count() } + fn test_worker_state(status: WorkerStatus) -> WorkerStateSnapshot { + let mut snapshot = WorkerStateSnapshot::from(status); + snapshot.execution_generation = 1; + snapshot.revision = 1; + snapshot + } + fn test_greeting() -> protocol::Greeting { protocol::Greeting { worker_name: "test".into(), @@ -4222,10 +4310,11 @@ mod completion_flow_tests { app.handle_worker_event(Event::Snapshot { session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, greeting, - status: WorkerStatus::Idle, + state: test_worker_state(WorkerStatus::Idle), in_flight: Default::default(), internal_workers: Vec::new(), }); @@ -4424,7 +4513,7 @@ mod completion_flow_tests { app.handle_worker_event(Event::Snapshot { greeting: test_greeting(), session: public_session(assistant_item_entries), - status: WorkerStatus::Running, + state: test_worker_state(WorkerStatus::Running), in_flight: Default::default(), internal_workers: Vec::new(), }); @@ -4437,23 +4526,23 @@ mod completion_flow_tests { } #[test] - fn input_history_records_queued_inputs_and_suppresses_consecutive_duplicates() { + fn input_history_records_running_submits_and_suppresses_consecutive_duplicates() { let mut app = App::new("test".into()); app.running = true; for c in "repeat".chars() { app.insert_char(c); } - assert!(app.submit_input().is_none()); + assert!(app.submit_input().is_some()); assert_eq!(app.input_history_len(), 1); - assert_eq!(app.queued_input_count(), 1); + assert_eq!(app.queued_input_count(), 0); for c in "repeat".chars() { app.insert_char(c); } - assert!(app.submit_input().is_none()); + assert!(app.submit_input().is_some()); assert_eq!(app.input_history_len(), 1); - assert_eq!(app.queued_input_count(), 2); + assert_eq!(app.queued_input_count(), 0); app.insert_char(' '); assert!(app.submit_input().is_none()); @@ -4481,7 +4570,7 @@ mod completion_flow_tests { }, ]; app.input.replace_with_segments(&original); - assert!(matches!(app.submit_input(), Some(Method::Run { .. }))); + assert!(matches!(app.submit_input(), Some(Method::Submit { .. }))); assert!(app.browse_input_history_older()); assert_eq!(app.input.submit_segments(), original); @@ -4493,7 +4582,7 @@ mod completion_flow_tests { for c in "sent".chars() { app.insert_char(c); } - assert!(matches!(app.submit_input(), Some(Method::Run { .. }))); + assert!(matches!(app.submit_input(), Some(Method::Submit { .. }))); for c in "draft".chars() { app.insert_char(c); @@ -4511,7 +4600,7 @@ mod completion_flow_tests { for c in "sent".chars() { app.insert_char(c); } - assert!(matches!(app.submit_input(), Some(Method::Run { .. }))); + assert!(matches!(app.submit_input(), Some(Method::Submit { .. }))); assert!(app.browse_input_history_older()); assert!(app.input_history_is_browsing()); @@ -4528,17 +4617,19 @@ mod completion_flow_tests { for c in "first".chars() { app.insert_char(c); } - assert!(matches!(app.submit_input(), Some(Method::Run { .. }))); + assert!(matches!(app.submit_input(), Some(Method::Submit { .. }))); for c in "second".chars() { app.insert_char(c); } - assert!(matches!(app.submit_input(), Some(Method::Run { .. }))); + assert!(matches!(app.submit_input(), Some(Method::Submit { .. }))); assert!(app.browse_input_history_older()); assert!(app.browse_input_history_older()); let method = app.submit_input(); match method { - Some(Method::Run { input }) => assert_eq!(Segment::flatten_to_text(&input), "first"), + Some(Method::Submit { input, .. }) => { + assert_eq!(Segment::flatten_to_text(&input), "first") + } other => panic!("expected recalled run, got {other:?}"), } assert_eq!(app.input_history_len(), 3); diff --git a/crates/tui/src/backend_worker_picker.rs b/crates/tui/src/backend_worker_picker.rs index e9a37271..473ca5eb 100644 --- a/crates/tui/src/backend_worker_picker.rs +++ b/crates/tui/src/backend_worker_picker.rs @@ -321,7 +321,7 @@ fn row_line( Span::raw(" "), Span::styled( pad_column(&worker_state(worker), widths.state), - state_style(worker.state.as_str()), + state_style(worker_state_label(worker)), ), Span::raw(" "), Span::styled( @@ -341,8 +341,20 @@ fn worker_name(worker: &BackendWorkerSummary) -> &str { } } +fn worker_state_label(worker: &BackendWorkerSummary) -> &str { + match worker.worker_state.as_ref().map(|state| &state.state) { + Some(protocol::WorkerState::Idle) => "idle", + Some(protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Paused, + ))) => "paused", + Some(protocol::WorkerState::Busy(_)) => "running", + None if worker.state == "stopped" => "stopped", + None => "unknown", + } +} + fn worker_state(worker: &BackendWorkerSummary) -> String { - format!("[{}]", worker.state) + format!("[{}]", worker_state_label(worker)) } fn text_width(value: &str) -> usize { @@ -413,7 +425,15 @@ mod tests { identity: "ws".to_string(), workspace_id: Some("ws".to_string()), }, - state: "running".to_string(), + state: "idle".to_string(), + worker_state: Some(protocol::WorkerStateSnapshot { + execution_generation: 1, + revision: 1, + state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Running, + )), + last_command_id: 0, + }), last_seen_at: None, pinned: false, retention_state: String::new(), @@ -450,6 +470,7 @@ mod tests { worker.display_name = "Coder".to_string(); worker.label = "Coder · T-585".to_string(); worker.state = "stopped".to_string(); + worker.worker_state = None; worker.working_directory = Some( serde_json::from_value(serde_json::json!({ "working_directory_id": "001a06a9f0202000000", @@ -478,12 +499,19 @@ mod tests { short.label = "Coder".to_string(); short.display_name = short.label.clone(); short.state = "idle".to_string(); + short.worker_state = Some(protocol::WorkerStateSnapshot { + execution_generation: 1, + revision: 2, + state: protocol::WorkerState::Idle, + last_command_id: 0, + }); let mut long = worker("runtime-a", "worker-b", None); long.resource_key = "W-100".to_string(); long.label = "Longer worker · T-9".to_string(); long.display_name = long.label.clone(); long.state = "stopped".to_string(); + long.worker_state = None; for worker in [&mut short, &mut long] { worker.working_directory = Some( diff --git a/crates/tui/src/command.rs b/crates/tui/src/command.rs index 44004a30..ba24299e 100644 --- a/crates/tui/src/command.rs +++ b/crates/tui/src/command.rs @@ -409,7 +409,12 @@ fn compact_command(invocation: CommandInvocation<'_>) -> CommandExecution { let _ = invocation.environment; let _ = invocation.args.raw(); CommandExecution { - method: Some(Method::Compact), + method: Some(Method::Compact { + command: protocol::WorkerCommandEnvelope::for_snapshot( + 0, + &protocol::WorkerStateSnapshot::initial(1), + ), + }), diagnostics: vec![CommandDiagnostic::new("compact requested")], exit_command_mode: true, clear_input: true, @@ -483,7 +488,7 @@ mod tests { fn compact_command_returns_compact_method_not_run() { let registry = CommandRegistry::builtins(); let result = registry.dispatch("compact", &env()); - assert!(matches!(result.method, Some(Method::Compact))); + assert!(matches!(result.method, Some(Method::Compact { .. }))); assert!(result.exit_command_mode); assert!(result.clear_input); assert!(result.diagnostics[0].message.contains("compact requested")); diff --git a/crates/tui/src/console/mod.rs b/crates/tui/src/console/mod.rs index 0106b403..af041066 100644 --- a/crates/tui/src/console/mod.rs +++ b/crates/tui/src/console/mod.rs @@ -270,8 +270,8 @@ impl ConsoleConnection { async fn send(&mut self, method: &Method) -> Result<(), Box> { let mut prepared = method.clone(); let carries_attachments = - matches!(prepared, Method::Run { .. }) && !self.pending_attachments.is_empty(); - if let Method::Run { input } = &mut prepared { + matches!(prepared, Method::Submit { .. }) && !self.pending_attachments.is_empty(); + if let Method::Submit { input, .. } = &mut prepared { input.extend( self.pending_attachments .iter() @@ -569,9 +569,10 @@ async fn run_e2e_rewind_fixture( app.connected = true; app.handle_worker_event(Event::Snapshot { session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, - status: WorkerStatus::Idle, + state: WorkerStatus::Idle.into(), greeting: Greeting { worker_name: worker_name.clone(), cwd: workspace_root.display().to_string(), @@ -697,6 +698,7 @@ async fn run_e2e_rewind_fixture( if submitted_at.elapsed() >= apply_delay { app.handle_worker_event(Event::RewindApplied { session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, input: vec![Segment::text("rewind-live-refresh")], @@ -916,7 +918,7 @@ async fn run_loop( } fn attachment_command_path(method: &Method) -> Option { - let Method::Run { input } = method else { + let Method::Submit { input, .. } = method else { return None; }; let [Segment::Text { content }] = input.as_slice() else { @@ -927,7 +929,7 @@ fn attachment_command_path(method: &Method) -> Option { } fn is_clear_attachments_command(method: &Method) -> bool { - let Method::Run { input } = method else { + let Method::Submit { input, .. } = method else { return false; }; matches!( @@ -941,7 +943,7 @@ async fn send_console_method( client: &mut ConsoleConnection, method: &Method, ) -> Result<(), Box> { - if matches!(method, Method::Run { .. }) && client.has_active_uploads() { + if matches!(method, Method::Submit { .. }) && client.has_active_uploads() { app.restore_unsent_run(method); app.flash_actionbar_notice( "Attachment upload is still in progress; wait or use /clear-attachments.", @@ -953,7 +955,7 @@ async fn send_console_method( } let sends_attachments = - matches!(method, Method::Run { .. }) && !client.pending_attachments.is_empty(); + matches!(method, Method::Submit { .. }) && !client.pending_attachments.is_empty(); if let Err(error) = client.send(method).await { if sends_attachments { app.restore_unsent_run(method); @@ -1148,18 +1150,27 @@ fn handle_key(app: &mut App, key: KeyEvent) -> Option { app.clear_command_input(); Some(None) } + KeyCode::Char(c) + if c.eq_ignore_ascii_case(&'d') && alt && !ctrl && !app.is_command_mode() => + { + Some( + app.next_queued_input_preview() + .map(str::to_owned) + .map(|submission_id| app.cancel_pending_method(submission_id)), + ) + } + KeyCode::Char(c) + if c.eq_ignore_ascii_case(&'n') && alt && !ctrl && !app.is_command_mode() => + { + Some(app.submit_notify_input()) + } KeyCode::Char(c) if c.eq_ignore_ascii_case(&'q') && alt && !ctrl && !app.is_command_mode() => { - if app.restore_next_queued_input_to_composer() { - Some(app.refresh_completion()) - } else { - Some(None) - } + Some(app.continue_pending_method()) } KeyCode::Char(c) if c.eq_ignore_ascii_case(&'c') && alt && !ctrl => { - app.clear_queued_inputs(); - Some(None) + Some(Some(app.clear_pending_method())) } KeyCode::Char('c') if ctrl => Some(handle_pause_or_quit(app)), KeyCode::Char('x') if ctrl => Some(handle_cancel_or_shutdown(app)), @@ -1427,14 +1438,15 @@ fn handle_cancel_or_shutdown(app: &mut App) -> Option { WorkerStatus::Running | WorkerStatus::Paused ) { app.shutdown_confirm = None; - app.clear_queued_inputs(); - return Some(Method::Cancel); + let command = app.next_command_envelope(); + return Some(Method::Cancel { command }); } if let Some(pressed_at) = app.shutdown_confirm && pressed_at.elapsed() < CONFIRM_TIMEOUT { app.shutdown_confirm = None; - return Some(Method::Shutdown); + let command = app.next_command_envelope(); + return Some(Method::Shutdown { command }); } app.shutdown_confirm = Some(std::time::Instant::now()); app.flash_actionbar_notice( @@ -1450,8 +1462,8 @@ fn handle_cancel_or_shutdown(app: &mut App) -> Option { /// Idle / Paused → 2-tap to quit the TUI (the Worker keeps running). fn handle_pause_or_quit(app: &mut App) -> Option { if app.worker_status == WorkerStatus::Running { - app.clear_queued_inputs(); - return Some(Method::Pause); + let command = app.next_command_envelope(); + return Some(Method::Pause { command }); } if let Some(t) = app.quit_confirm && t.elapsed() < CONFIRM_TIMEOUT @@ -1476,8 +1488,8 @@ mod tests { use crate::text_selection::{HistoryViewport, SelectionRow}; use async_trait::async_trait; use protocol::{ - Event, RewindTarget, RewindTargetId, RunResult, Segment, UploadedFileAvailability, - UploadedFileRef, WorkerStatus, + Event, RewindTarget, RewindTargetId, Segment, UploadedFileAvailability, UploadedFileRef, + WorkerStatus, }; #[test] @@ -1490,7 +1502,8 @@ mod tests { #[test] fn client_local_attachment_commands_are_typed_and_do_not_send_the_path() { - let attach = Method::Run { + let attach = Method::Submit { + submission_request_id: protocol::new_submission_request_id(), input: vec![Segment::text("/attach /tmp/report.md")], }; assert_eq!( @@ -1499,7 +1512,8 @@ mod tests { ); assert!(!is_clear_attachments_command(&attach)); - let clear = Method::Run { + let clear = Method::Submit { + submission_request_id: protocol::new_submission_request_id(), input: vec![Segment::text("/clear-attachments")], }; assert!(is_clear_attachments_command(&clear)); @@ -1605,7 +1619,7 @@ mod tests { } #[tokio::test] - async fn queued_attachment_send_failure_restores_draft_without_exiting_console() { + async fn running_attachment_submit_failure_restores_draft_without_exiting_console() { let file = UploadedFileRef { artifact_id: "artifact-queued".into(), file_name: "queued.txt".into(), @@ -1631,13 +1645,10 @@ mod tests { let mut app = App::new("worker".into()); app.set_worker_status(WorkerStatus::Running); app.input.insert_str("queued inspect"); - assert!(app.submit_input().is_none()); - let method = app - .handle_worker_event(Event::RunEnd { - result: RunResult::Finished, - }) - .expect("queued run must be released"); + .submit_input() + .expect("running Submit is sent immediately"); + send_console_method(&mut app, &mut connection, &method) .await .unwrap(); @@ -1960,7 +1971,7 @@ mod tests { } #[test] - fn running_enter_queues_instead_of_sending_run() { + fn running_enter_sends_submit_to_worker() { let mut app = App::new("agent".to_string()); app.set_worker_status(WorkerStatus::Running); for c in "queued".chars() { @@ -1973,102 +1984,128 @@ mod tests { ); } - assert!(handle_key(&mut app, KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)).is_none()); + assert!(matches!( + handle_key(&mut app, KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)), + Some(Method::Submit { .. }) + )); - assert_eq!(app.queued_input_count(), 1); - assert_eq!(app.next_queued_input_preview(), Some("queued")); + assert_eq!(app.queued_input_count(), 0); assert_eq!(input_text(&app), ""); } #[test] - fn queued_input_keybindings_restore_and_clear() { - let mut app = App::new("agent".to_string()); + fn running_alt_n_sends_explicit_notify_without_implicit_submit_conversion() { + let mut app = App::new("test".into()); app.set_worker_status(WorkerStatus::Running); - for c in "edit queued".chars() { - assert!( - handle_key( - &mut app, - KeyEvent::new(KeyCode::Char(c), KeyModifiers::NONE) - ) - .is_none() - ); + for character in "progress".chars() { + app.insert_char(character); } - assert!(handle_key(&mut app, KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)).is_none()); - assert!( - handle_key( - &mut app, - KeyEvent::new(KeyCode::Char('q'), KeyModifiers::ALT) - ) - .is_none() + let method = handle_key( + &mut app, + KeyEvent::new(KeyCode::Char('n'), KeyModifiers::ALT), ); - assert_eq!(app.queued_input_count(), 0); - assert_eq!(input_text(&app), "edit queued"); - - app.input.clear(); - for c in "clear queued".chars() { - assert!( - handle_key( - &mut app, - KeyEvent::new(KeyCode::Char(c), KeyModifiers::NONE) - ) - .is_none() - ); - } - assert!(handle_key(&mut app, KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)).is_none()); - assert_eq!(app.queued_input_count(), 1); - - assert!( - handle_key( - &mut app, - KeyEvent::new(KeyCode::Char('c'), KeyModifiers::ALT) - ) - .is_none() - ); - assert_eq!(app.queued_input_count(), 0); + assert!(matches!( + method, + Some(Method::Notify { + ref message, + auto_run: true, + .. + }) if message == "progress" + )); + assert_eq!(input_text(&app), ""); } #[test] - fn pause_and_cancel_clear_queued_input() { - let mut app = App::new("agent".to_string()); - app.set_worker_status(WorkerStatus::Running); - for c in "queued".chars() { - assert!( - handle_key( - &mut app, - KeyEvent::new(KeyCode::Char(c), KeyModifiers::NONE) - ) - .is_none() - ); - } - assert!(handle_key(&mut app, KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)).is_none()); - assert_eq!(app.queued_input_count(), 1); + fn pending_queue_shortcuts_send_worker_operations() { + let mut app = App::new("test".into()); + app.handle_worker_event(Event::PendingSubmissionsChanged { + pending: protocol::PendingSubmissionsSnapshot { + revision: 2, + notification_count: 0, + head_id: Some("submission-1".into()), + submissions: vec![protocol::PendingSubmissionSummary { + submission_id: "submission-1".into(), + accepted_at_ms: 1, + segment_count: 1, + byte_len: 6, + }], + }, + }); - let pause = handle_key( + let continue_next = handle_key( &mut app, - KeyEvent::new(KeyCode::Char('c'), KeyModifiers::CONTROL), + KeyEvent::new(KeyCode::Char('q'), KeyModifiers::ALT), ); - assert!(matches!(pause, Some(Method::Pause))); - assert_eq!(app.queued_input_count(), 0); - - for c in "queued again".chars() { - assert!( - handle_key( - &mut app, - KeyEvent::new(KeyCode::Char(c), KeyModifiers::NONE) - ) - .is_none() - ); - } - assert!(handle_key(&mut app, KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)).is_none()); + assert!(matches!( + continue_next, + Some(Method::ContinuePending { + expected_revision: 2, + ref expected_head_id, + }) if expected_head_id == "submission-1" + )); assert_eq!(app.queued_input_count(), 1); let cancel = handle_key( &mut app, - KeyEvent::new(KeyCode::Char('x'), KeyModifiers::CONTROL), + KeyEvent::new(KeyCode::Char('d'), KeyModifiers::ALT), ); - assert!(matches!(cancel, Some(Method::Cancel))); - assert_eq!(app.queued_input_count(), 0); + assert!(matches!( + cancel, + Some(Method::CancelPendingSubmission { + expected_revision: 2, + ref submission_id, + }) if submission_id == "submission-1" + )); + + let clear = handle_key( + &mut app, + KeyEvent::new(KeyCode::Char('c'), KeyModifiers::ALT), + ); + assert!(matches!( + clear, + Some(Method::ClearPendingSubmissions { + expected_revision: 2 + }) + )); + assert_eq!(app.queued_input_count(), 1); + } + + #[test] + fn pause_and_cancel_preserve_authoritative_pending_queue() { + let mut app = App::new("test".into()); + app.handle_worker_event(Event::PendingSubmissionsChanged { + pending: protocol::PendingSubmissionsSnapshot { + revision: 2, + notification_count: 0, + head_id: Some("submission-1".into()), + submissions: vec![protocol::PendingSubmissionSummary { + submission_id: "submission-1".into(), + accepted_at_ms: 1, + segment_count: 1, + byte_len: 6, + }], + }, + }); + app.set_worker_status(WorkerStatus::Running); + assert!(matches!( + handle_key( + &mut app, + KeyEvent::new(KeyCode::Char('c'), KeyModifiers::CONTROL), + ), + Some(Method::Pause { .. }) + )); + assert_eq!(app.queued_input_count(), 1); + + app.set_worker_status(WorkerStatus::Running); + assert!(matches!( + handle_key( + &mut app, + KeyEvent::new(KeyCode::Char('x'), KeyModifiers::CONTROL), + ), + Some(Method::Cancel { .. }) + )); + assert_eq!(app.queued_input_count(), 1); } #[test] @@ -2080,7 +2117,7 @@ mod tests { &mut app, KeyEvent::new(KeyCode::Char('x'), KeyModifiers::CONTROL), ); - assert!(matches!(cancel, Some(Method::Cancel))); + assert!(matches!(cancel, Some(Method::Cancel { .. }))); } #[test] @@ -2102,7 +2139,7 @@ mod tests { assert!(matches!( handle_key(&mut app, ctrl_x()), - Some(Method::Shutdown) + Some(Method::Shutdown { .. }) )); assert!(app.shutdown_confirm.is_none()); } @@ -2432,7 +2469,7 @@ mod tests { } let method = handle_key(&mut app, KeyEvent::new(KeyCode::Enter, KeyModifiers::NONE)); - assert!(matches!(method, Some(protocol::Method::Compact))); + assert!(matches!(method, Some(protocol::Method::Compact { .. }))); assert!(!app.is_command_mode()); assert_eq!(input_text(&app), ""); assert_eq!(app.queued_input_count(), 0); @@ -2535,13 +2572,19 @@ mod tests { let mut app = App::new("agent".to_string()); app.handle_worker_event(Event::Snapshot { greeting: test_greeting(), - session: protocol::SessionSnapshot { entries: vec![] }, - status: WorkerStatus::Idle, + session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), + entries: vec![], + }, + state: WorkerStatus::Idle.into(), in_flight: Default::default(), internal_workers: Vec::new(), }); app.handle_worker_event(Event::RewindApplied { - session: protocol::SessionSnapshot { entries: vec![] }, + session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), + entries: vec![], + }, input: vec![Segment::Text { content: "retry this".into(), }], @@ -2562,15 +2605,21 @@ mod tests { let mut app = App::new("agent".to_string()); app.handle_worker_event(Event::Snapshot { greeting: test_greeting(), - session: protocol::SessionSnapshot { entries: vec![] }, - status: WorkerStatus::Idle, + session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), + entries: vec![], + }, + state: WorkerStatus::Idle.into(), in_flight: Default::default(), internal_workers: Vec::new(), }); type_keys(&mut app, "draft"); app.handle_worker_event(Event::RewindApplied { - session: protocol::SessionSnapshot { entries: vec![] }, + session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), + entries: vec![], + }, input: vec![Segment::Text { content: "retry this".into(), }], @@ -2697,8 +2746,8 @@ mod tests { kind: protocol::InternalWorkerKind::SubWorker, }, revision: 1, - event: Box::new(Event::Status { - status: WorkerStatus::Running, + event: Box::new(Event::WorkerState { + snapshot: WorkerStatus::Running.into(), }), }); enter_command_mode(&mut app); @@ -2813,8 +2862,8 @@ mod tests { kind: protocol::InternalWorkerKind::SubWorker, }, revision: 1, - event: Box::new(Event::Status { - status: WorkerStatus::Running, + event: Box::new(Event::WorkerState { + snapshot: WorkerStatus::Running.into(), }), }); @@ -2839,8 +2888,8 @@ mod tests { kind: protocol::InternalWorkerKind::SubWorker, }, revision: 1, - event: Box::new(Event::Status { - status: WorkerStatus::Running, + event: Box::new(Event::WorkerState { + snapshot: WorkerStatus::Running.into(), }), }); handle_key(&mut app, key(KeyCode::Tab)); @@ -2856,7 +2905,7 @@ mod tests { ); assert!(first.is_none()); - assert!(matches!(second, Some(Method::Shutdown))); + assert!(matches!(second, Some(Method::Shutdown { .. }))); assert_eq!(app.worker_status, WorkerStatus::Idle); } @@ -2878,8 +2927,8 @@ mod tests { kind: protocol::InternalWorkerKind::SubWorker, }, revision: 1, - event: Box::new(Event::Status { - status: WorkerStatus::Running, + event: Box::new(Event::WorkerState { + snapshot: WorkerStatus::Running.into(), }), }); @@ -2918,12 +2967,12 @@ mod tests { type_keys(&mut app, "first"); assert!(matches!( handle_key(&mut app, key(KeyCode::Enter)), - Some(Method::Run { .. }) + Some(Method::Submit { .. }) )); type_keys(&mut app, "second"); assert!(matches!( handle_key(&mut app, key(KeyCode::Enter)), - Some(Method::Run { .. }) + Some(Method::Submit { .. }) )); assert_eq!(input_text(&app), ""); @@ -2954,7 +3003,7 @@ mod tests { type_keys(&mut app, "sent"); assert!(matches!( handle_key(&mut app, key(KeyCode::Enter)), - Some(Method::Run { .. }) + Some(Method::Submit { .. }) )); type_keys(&mut app, "draft\nbody"); app.move_cursor_start(); diff --git a/crates/tui/src/ui.rs b/crates/tui/src/ui.rs index 35b4fcbc..58b790f1 100644 --- a/crates/tui/src/ui.rs +++ b/crates/tui/src/ui.rs @@ -1880,7 +1880,7 @@ fn actionbar_left_item(app: &App, now: Instant) -> Option<(String, Style)> { } if app.queued_input_count() > 0 { return Some(( - "Alt-q edit queued Alt-c clear queued".to_string(), + "Alt-n notify Alt-q continue Alt-d cancel queued Alt-c clear queued".to_string(), Style::default().fg(Color::DarkGray), )); } @@ -2136,9 +2136,25 @@ mod tests { use super::*; use crate::app::{ActionbarNoticeLevel, ActionbarNoticeSource, App}; use crate::block::{ToolCallBlock, ToolCallState}; - use protocol::WorkerStatus; + use protocol::Event; use std::time::{Duration, Instant}; + fn set_pending_submission(app: &mut App, id: &str) { + app.handle_worker_event(Event::PendingSubmissionsChanged { + pending: protocol::PendingSubmissionsSnapshot { + revision: 1, + notification_count: 0, + head_id: Some(id.into()), + submissions: vec![protocol::PendingSubmissionSummary { + submission_id: id.into(), + accepted_at_ms: 1, + segment_count: 1, + byte_len: 1, + }], + }, + }); + } + #[test] fn run_status_line_matches_console_metrics_and_spinner_frame() { let now = Instant::now(); @@ -2251,15 +2267,11 @@ mod tests { #[test] fn queue_status_text_includes_count_and_preview() { let mut app = App::new("test".into()); - app.set_worker_status(WorkerStatus::Running); - for c in "queued preview".chars() { - app.insert_char(c); - } - assert!(app.submit_input().is_none()); + set_pending_submission(&mut app, "submission-1"); assert_eq!( queue_status_text(&app), - Some("queued: 1 — queued preview".to_string()) + Some("queued: 1 — submission-1".to_string()) ); } @@ -2289,14 +2301,10 @@ mod tests { Some("Worker keeps running. Press Ctrl-C again to exit TUI.".into()) ); - app.set_worker_status(WorkerStatus::Running); - for c in "queued turn".chars() { - app.insert_char(c); - } - assert!(app.submit_input().is_none()); + set_pending_submission(&mut app, "submission-1"); assert_eq!( actionbar_left_item(&app, now).map(|(text, _)| text), - Some("Alt-q edit queued Alt-c clear queued".into()) + Some("Alt-n notify Alt-q continue Alt-d cancel queued Alt-c clear queued".into()) ); app.enter_command_mode(); diff --git a/crates/workdir/src/delegation.rs b/crates/workdir/src/delegation.rs deleted file mode 100644 index 17598dab..00000000 --- a/crates/workdir/src/delegation.rs +++ /dev/null @@ -1,1189 +0,0 @@ -use std::collections::HashMap; -use std::path::Path; -use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; -use std::sync::{Arc, Mutex, Weak}; - -use async_trait::async_trait; -use fs_operation::{ - EditRequest, EditResult, FsPath, GlobRequest, GlobResult, GrepRequest, GrepResult, ListRequest, - ListResult, ReadRequest, ReadResult, StatRequest, StatResult, WriteRequest, WriteResult, -}; -use tokio::sync::broadcast; - -use crate::{ - CommandEvent, CommandHandle, CommandOutput, CommandOutputRequest, CommandRequest, - CommandSnapshot, CommandStatus, Workdir, WorkdirError, WorkdirSession, - WorkdirSessionCapabilities, WorkdirSessionCapability, WorkdirSessionHandle, -}; - -#[derive(Clone, Copy, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum WorkdirDelegationPermission { - Read, - Write, -} - -#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)] -#[serde(deny_unknown_fields)] -pub struct WorkdirDelegationRule { - pub target: FsPath, - pub permission: WorkdirDelegationPermission, - pub recursive: bool, -} - -#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)] -#[serde(deny_unknown_fields)] -pub struct WorkdirDelegationRequest { - pub rules: Vec, - pub cwd: FsPath, -} - -pub struct WorkdirDelegation { - pub scoped_session: WorkdirSessionHandle, - pub capabilities: WorkdirSessionCapabilities, - validity: Arc, -} - -impl std::fmt::Debug for WorkdirDelegation { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("WorkdirDelegation") - .field("workdir", &self.scoped_session.workdir()) - .field("capabilities", &self.capabilities) - .field("active", &self.is_active()) - .finish() - } -} - -impl WorkdirDelegation { - pub fn is_active(&self) -> bool { - self.validity.is_active() - } - - pub fn release(&self) { - self.validity.active.store(false, Ordering::Release); - } -} - -impl Drop for WorkdirDelegation { - fn drop(&mut self) { - self.release(); - } -} - -pub struct AppliedWorkdirDelegation { - pub scoped_session: WorkdirSessionHandle, - _leases: Vec, -} - -impl std::fmt::Debug for AppliedWorkdirDelegation { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("AppliedWorkdirDelegation") - .field("workdir", self.scoped_session.workdir()) - .field("lease_count", &self._leases.len()) - .finish() - } -} - -pub async fn apply_delegation_chain( - source: WorkdirSessionHandle, - requests: impl IntoIterator, -) -> Result { - let mut current = source; - let mut leases = Vec::new(); - for request in requests { - let authority = if current.is_delegation_capable() { - current.clone() - } else { - delegation_capable_session(current.clone()) - }; - let lease = authority.delegate(request).await?; - current = lease.scoped_session.clone(); - leases.push(lease); - } - Ok(AppliedWorkdirDelegation { - scoped_session: current, - _leases: leases, - }) -} - -#[derive(Debug)] -struct SessionValidity { - active: AtomicBool, - parent: Option>, -} - -impl SessionValidity { - fn root() -> Arc { - Arc::new(Self { - active: AtomicBool::new(true), - parent: None, - }) - } - - fn child(parent: Arc) -> Arc { - Arc::new(Self { - active: AtomicBool::new(true), - parent: Some(parent), - }) - } - - fn is_active(&self) -> bool { - self.active.load(Ordering::Acquire) - && self.parent.as_ref().is_none_or(|parent| parent.is_active()) - } -} - -#[derive(Clone, Debug)] -struct ActiveWriteLease { - validity: Weak, - rules: Vec, -} - -struct DelegatingWorkdirSession { - source: WorkdirSessionHandle, - cwd: FsPath, - scope: Option>, - capabilities: WorkdirSessionCapabilities, - validity: Arc, - child_write_leases: Mutex>, - next_lease_id: AtomicU64, - closes_source: bool, -} - -impl std::fmt::Debug for DelegatingWorkdirSession { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("DelegatingWorkdirSession") - .field("workdir", &self.source.workdir()) - .field("scope", &self.scope) - .field("capabilities", &self.capabilities) - .field("active", &self.validity.is_active()) - .finish_non_exhaustive() - } -} - -/// Wrap a provider session with logical-path delegation and parent write gates. -pub fn delegation_capable_session(source: WorkdirSessionHandle) -> WorkdirSessionHandle { - let capabilities = source.capabilities(); - Arc::new(DelegatingWorkdirSession { - source, - cwd: FsPath::new("").expect("empty Workdir path is valid"), - scope: None, - capabilities, - validity: SessionValidity::root(), - child_write_leases: Mutex::new(HashMap::new()), - next_lease_id: AtomicU64::new(1), - closes_source: true, - }) -} - -impl DelegatingWorkdirSession { - fn ensure_active(&self) -> Result<(), WorkdirError> { - if self.validity.is_active() { - Ok(()) - } else { - Err(WorkdirError::SessionClosed) - } - } - - fn ensure_capability( - &self, - required: WorkdirSessionCapability, - operation: &'static str, - ) -> Result<(), WorkdirError> { - self.ensure_active()?; - if self.capabilities.supports(required) { - Ok(()) - } else { - Err(WorkdirError::Denied(format!( - "delegated workdir session does not permit {operation}" - ))) - } - } - - fn ensure_path( - &self, - path: &FsPath, - permission: WorkdirDelegationPermission, - ) -> Result<(), WorkdirError> { - self.ensure_active()?; - if let Some(scope) = &self.scope { - if !scope - .iter() - .any(|rule| rule_allows_path(rule, path, permission)) - { - return Err(WorkdirError::Denied(format!( - "logical workdir path `{path}` is outside the delegated {permission:?} scope" - ))); - } - } - if permission == WorkdirDelegationPermission::Write { - self.ensure_parent_write_available(path)?; - } - Ok(()) - } - - fn resolve_path(&self, path: &FsPath) -> Result { - if self.cwd.as_str().is_empty() { - return Ok(path.clone()); - } - let joined = Path::new(self.cwd.as_str()).join(path.as_str()); - let joined = joined.to_str().ok_or_else(|| { - WorkdirError::Denied("logical Workdir path is not valid UTF-8".into()) - })?; - FsPath::new(joined).map_err(|error| WorkdirError::Denied(error.to_string())) - } - - fn ensure_read( - &self, - path: &FsPath, - capability: WorkdirSessionCapability, - ) -> Result<(), WorkdirError> { - self.ensure_capability(capability, "read operations")?; - self.ensure_path(path, WorkdirDelegationPermission::Read) - } - - fn ensure_write( - &self, - path: &FsPath, - capability: WorkdirSessionCapability, - ) -> Result<(), WorkdirError> { - self.ensure_capability(capability, "write operations")?; - self.ensure_path(path, WorkdirDelegationPermission::Write) - } - - fn ensure_command(&self) -> Result<(), WorkdirError> { - self.ensure_capability(WorkdirSessionCapability::Command, "command execution") - } - - fn ensure_parent_write_available(&self, path: &FsPath) -> Result<(), WorkdirError> { - let mut leases = self - .child_write_leases - .lock() - .expect("workdir delegation lease mutex poisoned"); - leases.retain(|_, lease| lease.validity.upgrade().is_some_and(|v| v.is_active())); - if leases.values().any(|lease| { - lease.rules.iter().any(|rule| { - rule.permission == WorkdirDelegationPermission::Write - && rule_allows_path(rule, path, WorkdirDelegationPermission::Write) - }) - }) { - Err(WorkdirError::Denied(format!( - "logical workdir path `{path}` is leased to a child session" - ))) - } else { - Ok(()) - } - } - - fn validate_delegation_rules( - &self, - rules: &[WorkdirDelegationRule], - ) -> Result { - self.ensure_active()?; - if rules.is_empty() { - return Err(WorkdirError::Denied( - "workdir delegation requires at least one logical scope rule".into(), - )); - } - let writable = rules - .iter() - .any(|rule| rule.permission == WorkdirDelegationPermission::Write); - if !self.capabilities.supports(WorkdirSessionCapability::Read) - || (writable - && (!self.capabilities.supports(WorkdirSessionCapability::Write) - || !self.capabilities.supports(WorkdirSessionCapability::Edit) - || !self - .capabilities - .supports(WorkdirSessionCapability::Command))) - { - return Err(WorkdirError::Denied( - "parent workdir session cannot delegate the requested capabilities".into(), - )); - } - for requested in rules { - if let Some(scope) = &self.scope { - if !scope - .iter() - .any(|parent| rule_contains_rule(parent, requested)) - { - return Err(WorkdirError::Denied(format!( - "logical workdir scope `{}` exceeds the parent delegation", - requested.target - ))); - } - } - } - let mut delegated = vec![WorkdirSessionCapability::Read]; - for capability in [ - WorkdirSessionCapability::Glob, - WorkdirSessionCapability::Grep, - ] { - if self.capabilities.supports(capability) { - delegated.push(capability); - } - } - if writable { - delegated.push(WorkdirSessionCapability::Write); - delegated.push(WorkdirSessionCapability::Edit); - delegated.push(WorkdirSessionCapability::Command); - } - Ok(WorkdirSessionCapabilities::from_capabilities(delegated)) - } -} - -#[async_trait] -impl WorkdirSession for DelegatingWorkdirSession { - fn workdir(&self) -> &Workdir { - self.source.workdir() - } - - fn capabilities(&self) -> WorkdirSessionCapabilities { - self.capabilities - } - - fn is_delegation_capable(&self) -> bool { - true - } - - fn transports_delegation_context(&self) -> bool { - self.source.transports_delegation_context() - } - - async fn capture_delegation_source( - &self, - request: &WorkdirDelegationRequest, - ) -> Result { - self.ensure_active()?; - if self.scope.is_some() { - return Err(WorkdirError::Denied( - "scoped Workdir sessions cannot expose their provider source".into(), - )); - } - self.source.capture_delegation_source(request).await - } - - async fn delegate( - &self, - request: WorkdirDelegationRequest, - ) -> Result { - let capabilities = self.validate_delegation_rules(&request.rules)?; - if !request - .rules - .iter() - .any(|rule| rule_allows_path(rule, &request.cwd, WorkdirDelegationPermission::Read)) - { - return Err(WorkdirError::Denied(format!( - "delegated cwd `{}` is outside the delegated readable scope", - request.cwd - ))); - } - let source = self.source.capture_delegation_source(&request).await?; - let validity = SessionValidity::child(self.validity.clone()); - let id = self.next_lease_id.fetch_add(1, Ordering::Relaxed); - if request - .rules - .iter() - .any(|rule| rule.permission == WorkdirDelegationPermission::Write) - { - self.child_write_leases - .lock() - .expect("workdir delegation lease mutex poisoned") - .insert( - id, - ActiveWriteLease { - validity: Arc::downgrade(&validity), - rules: request.rules.clone(), - }, - ); - } - let child: WorkdirSessionHandle = Arc::new(DelegatingWorkdirSession { - source, - cwd: request.cwd, - scope: Some(request.rules), - capabilities, - validity: validity.clone(), - child_write_leases: Mutex::new(HashMap::new()), - next_lease_id: AtomicU64::new(1), - closes_source: false, - }); - let scoped_session: WorkdirSessionHandle = - if capabilities == WorkdirSessionCapabilities::READ_ONLY { - Arc::new(ReadOnlyWorkdirSession::new(child)) - } else { - child - }; - Ok(WorkdirDelegation { - scoped_session, - capabilities, - validity, - }) - } - - async fn stat(&self, mut request: StatRequest) -> Result { - let path = self.resolve_path(&request.path)?; - self.ensure_read(&path, WorkdirSessionCapability::Read)?; - if !self.source.transports_delegation_context() { - request.path = path; - } - self.source.stat(request).await - } - - async fn read(&self, mut request: ReadRequest) -> Result { - let path = self.resolve_path(&request.path)?; - self.ensure_read(&path, WorkdirSessionCapability::Read)?; - if !self.source.transports_delegation_context() { - request.path = path; - } - self.source.read(request).await - } - - async fn write(&self, mut request: WriteRequest) -> Result { - let path = self.resolve_path(&request.path)?; - self.ensure_write(&path, WorkdirSessionCapability::Write)?; - if !self.source.transports_delegation_context() { - request.path = path; - } - self.source.write(request).await - } - - async fn edit(&self, mut request: EditRequest) -> Result { - let path = self.resolve_path(&request.path)?; - self.ensure_write(&path, WorkdirSessionCapability::Edit)?; - if !self.source.transports_delegation_context() { - request.path = path; - } - self.source.edit(request).await - } - - async fn list(&self, mut request: ListRequest) -> Result { - let path = self.resolve_path(&request.path)?; - self.ensure_read(&path, WorkdirSessionCapability::Read)?; - if !self.source.transports_delegation_context() { - request.path = path; - } - self.source.list(request).await - } - - async fn glob(&self, mut request: GlobRequest) -> Result { - let path = self.resolve_path(&request.path)?; - self.ensure_read(&path, WorkdirSessionCapability::Glob)?; - if !self.source.transports_delegation_context() { - request.path = path; - } - self.source.glob(request).await - } - - async fn grep(&self, mut request: GrepRequest) -> Result { - let path = self.resolve_path(&request.path)?; - self.ensure_read(&path, WorkdirSessionCapability::Grep)?; - if !self.source.transports_delegation_context() { - request.path = path; - } - self.source.grep(request).await - } - - async fn start_command(&self, request: CommandRequest) -> Result { - self.ensure_command()?; - self.source.start_command(request).await - } - - async fn command_status(&self, handle: CommandHandle) -> Result { - self.ensure_command()?; - self.source.command_status(handle).await - } - - async fn command_output( - &self, - request: CommandOutputRequest, - ) -> Result { - self.ensure_command()?; - self.source.command_output(request).await - } - - async fn cancel_command(&self, handle: CommandHandle) -> Result<(), WorkdirError> { - self.ensure_command()?; - self.source.cancel_command(handle).await - } - - fn subscribe_command_events(&self) -> Option> { - self.ensure_capability(WorkdirSessionCapability::Command, "command observation") - .ok()?; - self.source.subscribe_command_events() - } - - fn command_snapshot(&self) -> Vec { - if self - .ensure_capability(WorkdirSessionCapability::Command, "command observation") - .is_err() - { - return Vec::new(); - } - self.source.command_snapshot() - } - - async fn close(&self) -> Result<(), WorkdirError> { - self.validity.active.store(false, Ordering::Release); - if self.closes_source { - self.source.close().await - } else { - Ok(()) - } - } -} - -/// A fail-closed read-only view over an already scoped delegated session. -#[derive(Debug)] -pub struct ReadOnlyWorkdirSession { - inner: WorkdirSessionHandle, -} - -impl ReadOnlyWorkdirSession { - pub fn new(inner: WorkdirSessionHandle) -> Self { - Self { inner } - } -} - -#[async_trait] -impl WorkdirSession for ReadOnlyWorkdirSession { - fn workdir(&self) -> &Workdir { - self.inner.workdir() - } - - fn capabilities(&self) -> WorkdirSessionCapabilities { - WorkdirSessionCapabilities::READ_ONLY - } - - fn is_delegation_capable(&self) -> bool { - true - } - - fn transports_delegation_context(&self) -> bool { - self.inner.transports_delegation_context() - } - - async fn delegate( - &self, - request: WorkdirDelegationRequest, - ) -> Result { - if request - .rules - .iter() - .any(|rule| rule.permission == WorkdirDelegationPermission::Write) - { - return Err(WorkdirError::Denied( - "read-only workdir session cannot delegate write access".into(), - )); - } - self.inner.delegate(request).await - } - - async fn stat(&self, request: StatRequest) -> Result { - self.inner.stat(request).await - } - - async fn read(&self, request: ReadRequest) -> Result { - self.inner.read(request).await - } - - async fn write(&self, _request: WriteRequest) -> Result { - Err(WorkdirError::Denied("read-only workdir session".into())) - } - - async fn edit(&self, _request: EditRequest) -> Result { - Err(WorkdirError::Denied("read-only workdir session".into())) - } - - async fn list(&self, request: ListRequest) -> Result { - self.inner.list(request).await - } - - async fn glob(&self, request: GlobRequest) -> Result { - self.inner.glob(request).await - } - - async fn grep(&self, request: GrepRequest) -> Result { - self.inner.grep(request).await - } - - async fn start_command(&self, _request: CommandRequest) -> Result { - Err(WorkdirError::Denied("read-only workdir session".into())) - } - - async fn command_status(&self, _handle: CommandHandle) -> Result { - Err(WorkdirError::Denied("read-only workdir session".into())) - } - - async fn command_output( - &self, - _request: CommandOutputRequest, - ) -> Result { - Err(WorkdirError::Denied("read-only workdir session".into())) - } - - async fn cancel_command(&self, _handle: CommandHandle) -> Result<(), WorkdirError> { - Err(WorkdirError::Denied("read-only workdir session".into())) - } - - async fn close(&self) -> Result<(), WorkdirError> { - self.inner.close().await - } -} - -fn rule_allows_path( - rule: &WorkdirDelegationRule, - path: &FsPath, - required: WorkdirDelegationPermission, -) -> bool { - if required == WorkdirDelegationPermission::Write - && rule.permission != WorkdirDelegationPermission::Write - { - return false; - } - path_in_rule(rule, path) -} - -fn path_in_rule(rule: &WorkdirDelegationRule, path: &FsPath) -> bool { - let target = Path::new(rule.target.as_str()); - let path = Path::new(path.as_str()); - if path == target { - return true; - } - let Ok(suffix) = path.strip_prefix(target) else { - return false; - }; - let depth = suffix.components().count(); - rule.recursive || depth <= 1 -} - -fn rule_contains_rule(parent: &WorkdirDelegationRule, child: &WorkdirDelegationRule) -> bool { - if child.permission == WorkdirDelegationPermission::Write - && parent.permission != WorkdirDelegationPermission::Write - { - return false; - } - if !path_in_rule(parent, &child.target) { - return false; - } - if parent.recursive { - return true; - } - !child.recursive && parent.target == child.target -} - -#[cfg(test)] -mod tests { - use std::fs; - - use manifest::{Permission, Scope, ScopeConfig, ScopeRule, SharedScope}; - use tempfile::TempDir; - - use super::*; - use crate::LocalWorkdirSession; - - fn fs_path(path: &str) -> FsPath { - FsPath::new(path).unwrap() - } - - fn session(root: &Path) -> WorkdirSessionHandle { - let scope = SharedScope::new( - Scope::from_config(&ScopeConfig { - allow: vec![ScopeRule { - target: root.to_path_buf(), - permission: Permission::Write, - recursive: true, - }], - deny: Vec::new(), - }) - .unwrap(), - ); - delegation_capable_session(Arc::new(LocalWorkdirSession::materialized_bound( - Workdir::new("delegation-test"), - root.to_path_buf(), - root.to_path_buf(), - scope, - WorkdirSessionCapabilities::ALL, - ))) - } - - fn request(path: &str, permission: WorkdirDelegationPermission) -> WorkdirDelegationRequest { - WorkdirDelegationRequest { - rules: vec![WorkdirDelegationRule { - target: fs_path(path), - permission, - recursive: true, - }], - cwd: fs_path(path), - } - } - - fn read(path: &str) -> ReadRequest { - ReadRequest { - path: fs_path(path), - offset: 0, - limit: 20, - max_bytes: 1024, - } - } - - fn write(path: &str, content: &str) -> WriteRequest { - WriteRequest { - path: fs_path(path), - content: content.as_bytes().to_vec(), - expected_hash: None, - } - } - - async fn run_command( - session: &WorkdirSessionHandle, - command: impl Into, - tool_call_id: impl Into, - ) -> CommandOutput { - let handle = session - .start_command(CommandRequest { - command: command.into(), - timeout_secs: 5, - output_limit: 1024, - spill_dir: None, - tool_call_id: Some(tool_call_id.into()), - }) - .await - .unwrap(); - session - .command_output(CommandOutputRequest { - handle, - cursor: 0, - limit: 1024, - wait: true, - }) - .await - .unwrap() - } - - #[tokio::test] - async fn delegation_capable_session_forwards_command_telemetry() { - let root = TempDir::new().unwrap(); - let parent = session(root.path()); - let mut events = parent - .subscribe_command_events() - .expect("delegation wrapper must preserve command observation"); - let handle = parent - .start_command(CommandRequest { - command: "printf ready; sleep 0.2; printf done".into(), - timeout_secs: 5, - output_limit: 1024, - spill_dir: None, - tool_call_id: Some("tool-delegated".into()), - }) - .await - .unwrap(); - - let first_output = loop { - let event = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv()) - .await - .expect("delegated command telemetry should not stall") - .unwrap(); - if let CommandEvent::Output { content, .. } = event { - break content; - } - }; - assert_eq!(first_output, "ready"); - let snapshots = parent.command_snapshot(); - assert_eq!(snapshots.len(), 1); - assert_eq!(snapshots[0].command_id, handle.0); - assert_eq!(snapshots[0].status, CommandStatus::Running); - assert_eq!(snapshots[0].stdout.content, "ready"); - - let output = parent - .command_output(CommandOutputRequest { - handle, - cursor: 0, - limit: 1024, - wait: true, - }) - .await - .unwrap(); - assert_eq!(output.status, CommandStatus::Completed); - assert_eq!(output.content, "readydone"); - assert!(parent.command_snapshot().is_empty()); - } - - #[test] - fn non_recursive_rule_covers_target_and_direct_children_only() { - let rule = WorkdirDelegationRule { - target: fs_path("docs"), - permission: WorkdirDelegationPermission::Read, - recursive: false, - }; - assert!(path_in_rule(&rule, &fs_path("docs"))); - assert!(path_in_rule(&rule, &fs_path("docs/readme.md"))); - assert!(!path_in_rule(&rule, &fs_path("docs/guides/start.md"))); - } - - #[tokio::test] - async fn read_only_delegation_allows_prefix_and_denies_mutation() { - let root = TempDir::new().unwrap(); - fs::create_dir_all(root.path().join("docs")).unwrap(); - fs::create_dir_all(root.path().join("secret")).unwrap(); - fs::write(root.path().join("docs/readme.md"), "visible").unwrap(); - fs::write(root.path().join("secret/key"), "hidden").unwrap(); - let parent = session(root.path()); - - let child = parent - .delegate(request("docs", WorkdirDelegationPermission::Read)) - .await - .unwrap(); - assert_eq!(child.capabilities, WorkdirSessionCapabilities::READ_ONLY); - assert_eq!( - child - .scoped_session - .read(read("readme.md")) - .await - .unwrap() - .bytes, - b"visible" - ); - assert!(matches!( - child.scoped_session.write(write("new.md", "no")).await, - Err(WorkdirError::Denied(_)) - )); - assert!( - !child - .capabilities - .supports(WorkdirSessionCapability::Command) - ); - assert!(child.scoped_session.subscribe_command_events().is_none()); - assert!(child.scoped_session.command_snapshot().is_empty()); - assert!(matches!( - child - .scoped_session - .start_command(CommandRequest { - command: "printf denied".into(), - timeout_secs: 5, - output_limit: 1024, - spill_dir: None, - tool_call_id: Some("read-only-command".into()), - }) - .await, - Err(WorkdirError::Denied(_)) - )); - } - - #[cfg(unix)] - #[tokio::test] - async fn provider_scope_denies_read_through_symlink_outside_grant() { - use std::os::unix::fs::symlink; - - let root = TempDir::new().unwrap(); - fs::create_dir_all(root.path().join("granted")).unwrap(); - fs::create_dir_all(root.path().join("secret")).unwrap(); - fs::write(root.path().join("secret/key"), "hidden").unwrap(); - symlink("../secret/key", root.path().join("granted/link")).unwrap(); - let parent = session(root.path()); - let child = parent - .delegate(request("granted", WorkdirDelegationPermission::Read)) - .await - .unwrap(); - - let result = child.scoped_session.read(read("link")).await; - assert!( - result.is_err(), - "symlink read escaped provider scope: {result:?}" - ); - } - - #[cfg(unix)] - #[tokio::test] - async fn provider_scope_denies_write_through_symlink_outside_grant() { - use std::os::unix::fs::symlink; - - let root = TempDir::new().unwrap(); - fs::create_dir_all(root.path().join("granted")).unwrap(); - fs::create_dir_all(root.path().join("secret")).unwrap(); - symlink("../secret", root.path().join("granted/outside")).unwrap(); - let parent = session(root.path()); - let child = parent - .delegate(request("granted", WorkdirDelegationPermission::Write)) - .await - .unwrap(); - - let result = child - .scoped_session - .write(write("outside/new", "forbidden")) - .await; - assert!( - result.is_err(), - "symlink write escaped provider scope: {result:?}" - ); - assert!(!root.path().join("secret/new").exists()); - } - - #[cfg(unix)] - #[tokio::test] - async fn write_delegation_rejects_symlink_target_before_lease() { - use std::os::unix::fs::symlink; - - let root = TempDir::new().unwrap(); - fs::create_dir_all(root.path().join("granted")).unwrap(); - fs::create_dir_all(root.path().join("secret")).unwrap(); - symlink("../secret", root.path().join("granted/outside")).unwrap(); - let parent = session(root.path()); - - assert!(matches!( - parent - .delegate(request( - "granted/outside", - WorkdirDelegationPermission::Write - )) - .await, - Err(WorkdirError::Denied(_)) - )); - parent - .write(write("secret/parent", "still-authoritative")) - .await - .unwrap(); - } - - #[tokio::test] - async fn write_lease_keeps_typed_parent_writes_exclusive_without_blocking_commands() { - let root = TempDir::new().unwrap(); - fs::create_dir_all(root.path().join("leased")).unwrap(); - fs::create_dir_all(root.path().join("other")).unwrap(); - let parent = session(root.path()); - let child = parent - .delegate(request("leased", WorkdirDelegationPermission::Write)) - .await - .unwrap(); - assert!( - child - .capabilities - .supports(WorkdirSessionCapability::Command) - ); - let child_output = run_command( - &child.scoped_session, - "printf child-command", - "delegated-child-command", - ) - .await; - assert_eq!(child_output.content, "child-command"); - let parent_output = run_command( - &parent, - "printf parent-write > leased/from-command; printf parent-command", - "parent-command-during-child-write", - ) - .await; - assert_eq!(parent_output.status, CommandStatus::Completed); - assert_eq!(parent_output.content, "parent-command"); - assert_eq!( - fs::read_to_string(root.path().join("leased/from-command")).unwrap(), - "parent-write" - ); - - assert!(matches!( - parent.write(write("leased/file", "parent")).await, - Err(WorkdirError::Denied(_)) - )); - parent.write(write("other/file", "parent")).await.unwrap(); - child - .scoped_session - .write(write("file", "child")) - .await - .unwrap(); - child.release(); - assert!(matches!( - child - .scoped_session - .start_command(CommandRequest { - command: "printf revoked".into(), - timeout_secs: 5, - output_limit: 1024, - spill_dir: None, - tool_call_id: Some("revoked-child-command".into()), - }) - .await, - Err(WorkdirError::SessionClosed) - )); - parent - .write(write("leased/parent", "parent")) - .await - .unwrap(); - assert!(matches!( - child.scoped_session.read(read("file")).await, - Err(WorkdirError::SessionClosed) - )); - } - - #[tokio::test] - async fn nested_delegation_is_attenuated_and_parent_revocation_cascades() { - let root = TempDir::new().unwrap(); - fs::create_dir_all(root.path().join("docs/sub")).unwrap(); - fs::create_dir_all(root.path().join("docs/peer")).unwrap(); - fs::write(root.path().join("docs/sub/a"), "a").unwrap(); - fs::write(root.path().join("docs/peer/b"), "b").unwrap(); - let root_session = session(root.path()); - let child = root_session - .delegate(request("docs", WorkdirDelegationPermission::Read)) - .await - .unwrap(); - let nested = child - .scoped_session - .delegate(request("docs/sub", WorkdirDelegationPermission::Read)) - .await - .unwrap(); - - nested.scoped_session.read(read("a")).await.unwrap(); - assert!( - child - .scoped_session - .delegate(request("other", WorkdirDelegationPermission::Read)) - .await - .is_err() - ); - assert!( - child - .scoped_session - .delegate(request("docs/sub", WorkdirDelegationPermission::Write)) - .await - .is_err() - ); - - child.release(); - assert!(matches!( - nested.scoped_session.read(read("a")).await, - Err(WorkdirError::SessionClosed) - )); - } - - #[tokio::test] - async fn nested_write_leases_do_not_block_command_capable_ancestors() { - let root = TempDir::new().unwrap(); - fs::create_dir_all(root.path().join("docs/sub")).unwrap(); - let root_session = session(root.path()); - let child = root_session - .delegate(request("docs", WorkdirDelegationPermission::Write)) - .await - .unwrap(); - let nested = child - .scoped_session - .delegate(request("docs/sub", WorkdirDelegationPermission::Write)) - .await - .unwrap(); - - for (session, label) in [ - (&root_session, "root"), - (&child.scoped_session, "child"), - (&nested.scoped_session, "nested"), - ] { - let output = run_command( - session, - format!("printf {label}"), - format!("{label}-command-during-nested-write"), - ) - .await; - assert_eq!(output.status, CommandStatus::Completed); - assert_eq!(output.content, label); - } - - assert!(matches!( - root_session.write(write("docs/root", "blocked")).await, - Err(WorkdirError::Denied(_)) - )); - assert!(matches!( - child - .scoped_session - .write(write("sub/child", "blocked")) - .await, - Err(WorkdirError::Denied(_)) - )); - nested - .scoped_session - .write(write("nested", "allowed")) - .await - .unwrap(); - - nested.release(); - child.release(); - } - - #[tokio::test] - async fn reapplied_write_delegation_chain_forwards_command_lifecycle() { - let root = TempDir::new().unwrap(); - fs::create_dir_all(root.path().join("delegated")).unwrap(); - let applied = apply_delegation_chain( - session(root.path()), - [request("delegated", WorkdirDelegationPermission::Write)], - ) - .await - .unwrap(); - - let output = run_command( - &applied.scoped_session, - "printf reapplied", - "reapplied-command", - ) - .await; - assert_eq!(output.status, CommandStatus::Completed); - assert_eq!(output.content, "reapplied"); - } - - #[tokio::test] - async fn applied_chain_cannot_replace_outer_provider_attenuation() { - let root = TempDir::new().unwrap(); - fs::create_dir_all(root.path().join("outer")).unwrap(); - fs::create_dir_all(root.path().join("outside")).unwrap(); - let result = apply_delegation_chain( - Arc::new(LocalWorkdirSession::materialized_bound( - Workdir::new("delegation-chain-test"), - root.path().to_path_buf(), - root.path().to_path_buf(), - SharedScope::new( - Scope::from_config(&ScopeConfig { - allow: vec![ScopeRule { - target: root.path().to_path_buf(), - permission: Permission::Write, - recursive: true, - }], - deny: Vec::new(), - }) - .unwrap(), - ), - WorkdirSessionCapabilities::ALL, - )), - [ - request("outer", WorkdirDelegationPermission::Read), - request("outside", WorkdirDelegationPermission::Read), - ], - ) - .await; - assert!(matches!(result, Err(WorkdirError::Denied(_)))); - } - - #[tokio::test] - async fn closing_parent_invalidates_delegated_sessions() { - let root = TempDir::new().unwrap(); - fs::create_dir_all(root.path().join("docs")).unwrap(); - fs::write(root.path().join("docs/a"), "a").unwrap(); - let parent = session(root.path()); - let child = parent - .delegate(request("docs", WorkdirDelegationPermission::Read)) - .await - .unwrap(); - - parent.close().await.unwrap(); - assert!(matches!( - parent - .start_command(CommandRequest { - command: "printf closed".into(), - timeout_secs: 5, - output_limit: 1024, - spill_dir: None, - tool_call_id: Some("closed-parent-command".into()), - }) - .await, - Err(WorkdirError::SessionClosed) - )); - assert!(matches!( - child.scoped_session.read(read("a")).await, - Err(WorkdirError::SessionClosed) - )); - } -} diff --git a/crates/workdir/src/http.rs b/crates/workdir/src/http.rs index 72733e8b..f89b608b 100644 --- a/crates/workdir/src/http.rs +++ b/crates/workdir/src/http.rs @@ -68,12 +68,10 @@ pub enum WorkdirSessionOperation { CommandCancel(CommandHandle), } -/// Wire envelope for an operation and its optional provider-enforced child scope. +/// Wire envelope for one provider operation. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct WorkdirSessionOperationRequest { - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub delegations: Vec, pub operation: WorkdirSessionOperation, } @@ -289,7 +287,7 @@ mod client { use reqwest::{Client, StatusCode, Url}; use super::*; - use crate::{Workdir, WorkdirSession, WorkdirSessionHandle}; + use crate::{Workdir, WorkdirSession}; /// Provides a fresh bearer token for each Runtime request. Backend /// implementations can mint short-lived capability tokens without making a @@ -324,7 +322,6 @@ mod client { workdir: Workdir, session_id: WorkdirSessionId, capabilities: WorkdirSessionCapabilities, - delegations: Vec, closed: AtomicBool, } @@ -377,7 +374,6 @@ mod client { workdir: Workdir::new(opened.workdir_id.as_str()), session_id: opened.session_id, capabilities: opened.capabilities, - delegations: Vec::new(), closed: AtomicBool::new(false), }) } @@ -404,10 +400,7 @@ mod client { "operations", ], )?; - let operation = WorkdirSessionOperationRequest { - delegations: self.delegations.clone(), - operation, - }; + let operation = WorkdirSessionOperationRequest { operation }; let response = self .client .post(url) @@ -436,37 +429,6 @@ mod client { self.capabilities } - fn transports_delegation_context(&self) -> bool { - true - } - - async fn capture_delegation_source( - &self, - request: &crate::WorkdirDelegationRequest, - ) -> Result { - if self.closed.load(Ordering::Acquire) { - return Err(WorkdirError::SessionClosed); - } - let mut delegations = self.delegations.clone(); - delegations.push(request.clone()); - let candidate = Arc::new(Self { - client: self.client.clone(), - base_url: self.base_url.clone(), - authorization: self.authorization.clone(), - workdir: self.workdir.clone(), - session_id: self.session_id.clone(), - capabilities: self.capabilities, - delegations, - closed: AtomicBool::new(false), - }); - candidate - .stat(StatRequest { - path: fs_operation::FsPath::new("").expect("empty Workdir path is valid"), - }) - .await?; - Ok(candidate) - } - async fn stat(&self, request: StatRequest) -> Result { match self.operate(WorkdirSessionOperation::Stat(request)).await? { WorkdirSessionOperationResult::Stat(result) => Ok(result), diff --git a/crates/workdir/src/lib.rs b/crates/workdir/src/lib.rs index 5d58da26..4cfa91c1 100644 --- a/crates/workdir/src/lib.rs +++ b/crates/workdir/src/lib.rs @@ -5,10 +5,10 @@ //! bound to one Worker. Tools consume sessions; they do not own Workdir //! materialization or cleanup. -mod delegation; pub mod http; mod local; mod operation; +mod scope; pub mod workspace; use std::path::{Path, PathBuf}; @@ -18,11 +18,6 @@ use async_trait::async_trait; use serde::{Deserialize, Serialize}; use tokio::sync::broadcast; -pub use delegation::{ - AppliedWorkdirDelegation, ReadOnlyWorkdirSession, WorkdirDelegation, - WorkdirDelegationPermission, WorkdirDelegationRequest, WorkdirDelegationRule, - apply_delegation_chain, delegation_capable_session, -}; pub use fs_operation::{ ContentHash, EditRequest, EditResult, EntryKind, FsPath as WorkdirPath, GlobRequest, GlobResult, GrepOutputMode, GrepRequest, GrepResult, ListEntry, ListRequest, ListResult, @@ -32,6 +27,10 @@ pub use local::{ LocalWorkdirSession, SymlinkInfo, WorkdirSessionResource, direct_symlink, first_symlink, }; pub use operation::*; +pub use scope::{ + ReadOnlyWorkdirSession, WorkdirScopeLease, WorkdirToolBroker, WorkdirToolScope, + WorkdirToolScopePermission, WorkdirToolScopeRule, +}; /// Persistent, opaque identity of one materialized Workdir. #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] @@ -148,39 +147,6 @@ pub trait WorkdirSession: std::fmt::Debug + Send + Sync { fn workdir(&self) -> &Workdir; fn capabilities(&self) -> WorkdirSessionCapabilities; - fn is_delegation_capable(&self) -> bool { - false - } - - /// Whether this session transports the delegation chain to another - /// provider boundary that will apply logical cwd/path resolution there. - fn transports_delegation_context(&self) -> bool { - false - } - - /// Capture a provider-specific source for a delegated child session. - /// Remote providers use this boundary to pin attachment identity without - /// exposing transport handles or host paths. - async fn capture_delegation_source( - &self, - _request: &WorkdirDelegationRequest, - ) -> Result { - Err(WorkdirError::Denied( - "workdir provider does not support delegated sessions".into(), - )) - } - - /// Attenuate this session into a revocable child lease. Only sessions - /// created with [`delegation_capable_session`] implement this operation. - async fn delegate( - &self, - _request: WorkdirDelegationRequest, - ) -> Result { - Err(WorkdirError::Denied( - "workdir session is not delegation-capable".into(), - )) - } - async fn stat(&self, request: StatRequest) -> Result; async fn read(&self, request: ReadRequest) -> Result; async fn write(&self, request: WriteRequest) -> Result; diff --git a/crates/workdir/src/local.rs b/crates/workdir/src/local.rs index 8f9556dd..94efeceb 100644 --- a/crates/workdir/src/local.rs +++ b/crates/workdir/src/local.rs @@ -18,7 +18,7 @@ use std::sync::{Arc, Mutex as StdMutex}; use std::time::{Duration, SystemTime, UNIX_EPOCH}; use async_trait::async_trait; -use manifest::{Permission, Scope, ScopeConfig, ScopeRule, SharedScope}; +use manifest::{Scope, SharedScope}; use sha2::{Digest, Sha256}; use tokio::process::Command; use tokio::sync::{Mutex, broadcast, watch}; @@ -28,10 +28,8 @@ use crate::{ CommandEvent, CommandHandle, CommandOutput, CommandOutputRequest, CommandRequest, CommandSnapshot, CommandStatus, CommandStream, CommandStreamSlice, EditRequest, EditResult, GlobRequest, GlobResult, GrepRequest, GrepResult, ListRequest, ListResult, ReadRequest, - ReadResult, StatRequest, StatResult, Workdir, WorkdirDelegationPermission, - WorkdirDelegationRequest, WorkdirError, WorkdirPath, WorkdirSession, - WorkdirSessionCapabilities, WorkdirSessionCapability, WorkdirSessionHandle, WriteRequest, - WriteResult, + ReadResult, StatRequest, StatResult, Workdir, WorkdirError, WorkdirPath, WorkdirSession, + WorkdirSessionCapabilities, WorkdirSessionCapability, WriteRequest, WriteResult, }; #[cfg(test)] use crate::{EntryKind, WriteOutcome}; @@ -558,69 +556,6 @@ impl WorkdirSession for LocalWorkdirSession { self.inner.capabilities } - async fn capture_delegation_source( - &self, - request: &WorkdirDelegationRequest, - ) -> Result { - let host_rules = request - .rules - .iter() - .map(|rule| ScopeRule { - target: self.inner.root.join(rule.target.as_str()), - permission: match rule.permission { - WorkdirDelegationPermission::Read => Permission::Read, - WorkdirDelegationPermission::Write => Permission::Write, - }, - recursive: rule.recursive, - }) - .collect::>(); - for (logical, host) in request.rules.iter().zip(&host_rules) { - if logical.permission == WorkdirDelegationPermission::Write { - let resolved = Scope::resolved_target(host) - .map_err(|error| WorkdirError::Denied(error.to_string()))?; - if resolved != host.target { - return Err(WorkdirError::Denied(format!( - "write delegation target `{}` traverses a symlink", - logical.target - ))); - } - } - } - let parent_scope = self.inner.scope.snapshot(); - for rule in &host_rules { - if !parent_scope - .allows_rule(rule) - .map_err(|error| WorkdirError::Denied(error.to_string()))? - { - return Err(WorkdirError::Denied(format!( - "delegated provider scope `{}` exceeds the parent session", - rule.target.display() - ))); - } - } - let child_scope = Scope::from_config(&ScopeConfig { - allow: host_rules, - deny: Vec::new(), - }) - .map_err(|error| WorkdirError::Denied(error.to_string()))?; - let child_cwd = self.inner.root.join(request.cwd.as_str()); - if !child_scope.is_readable(&child_cwd) - || !std::fs::metadata(&child_cwd).is_ok_and(|metadata| metadata.is_dir()) - { - return Err(WorkdirError::Denied(format!( - "delegated cwd `{}` is not a readable Workdir directory", - request.cwd - ))); - } - Ok(Arc::new(LocalWorkdirSession::materialized_bound( - self.inner.workdir.clone(), - self.inner.root.clone(), - self.inner.root.clone(), - SharedScope::new(child_scope), - self.inner.capabilities, - ))) - } - async fn stat(&self, request: StatRequest) -> Result { self.ensure_capability(WorkdirSessionCapability::Read)?; let logical = request.path.clone(); @@ -694,9 +629,20 @@ impl WorkdirSession for LocalWorkdirSession { { return Err(WorkdirError::OutOfScope(spill_dir.to_path_buf())); } + let cwd = if let Some(logical_cwd) = request.cwd.as_ref() { + let cwd = self.resolve(logical_cwd); + let scope = self.inner.scope.snapshot(); + if !scope.is_readable(&cwd) + || !std::fs::metadata(&cwd).is_ok_and(|metadata| metadata.is_dir()) + { + return Err(WorkdirError::OutOfScope(cwd)); + } + cwd + } else { + self.inner.cwd.clone() + }; let id = self.inner.next_command_id.fetch_add(1, Ordering::Relaxed); let handle = CommandHandle(format!("command-{id}")); - let cwd = self.inner.cwd.clone(); let (completion_tx, completion) = watch::channel(false); let command_id = handle.0.clone(); let telemetry = self.inner.command_telemetry.clone(); @@ -1516,6 +1462,7 @@ mod tests { command: "sleep 30".to_owned(), timeout_secs: 60, output_limit: 1024, + cwd: None, spill_dir: None, tool_call_id: None, }, @@ -2043,6 +1990,7 @@ mod tests { command: "pwd && printf provider-command".into(), timeout_secs: 5, output_limit: 4096, + cwd: None, spill_dir: None, tool_call_id: None, }, @@ -2141,6 +2089,7 @@ mod tests { command: "printf hidden".into(), timeout_secs: 5, output_limit: 1, + cwd: None, spill_dir: Some(spill.path().to_path_buf()), tool_call_id: None, }, @@ -2178,6 +2127,7 @@ mod tests { command: "i=0; while [ $i -lt 200 ]; do printf 'line-%03d\\n' \"$i\"; i=$((i+1)); done; printf 'FINAL-NEEDLE\\n'".into(), timeout_secs: 5, output_limit: 64, + cwd: None, spill_dir: Some(spill.path().to_path_buf()), tool_call_id: None, }, @@ -2224,6 +2174,7 @@ mod tests { command: "printf 'aéz'".into(), timeout_secs: 5, output_limit: 1024, + cwd: None, spill_dir: None, tool_call_id: None, }, @@ -2449,6 +2400,7 @@ mod tests { command: "printf ready; printf warning >&2; sleep 0.2; printf done".into(), timeout_secs: 5, output_limit: 1024, + cwd: None, spill_dir: None, tool_call_id: Some("tool-7".into()), }, @@ -2553,6 +2505,7 @@ mod tests { command: "sleep 30".into(), timeout_secs: 1, output_limit: 1024, + cwd: None, spill_dir: None, tool_call_id: None, }, @@ -2623,6 +2576,7 @@ mod tests { command: "sleep 30".into(), timeout_secs: 60, output_limit: 1024, + cwd: None, spill_dir: None, tool_call_id: None, }, diff --git a/crates/workdir/src/operation.rs b/crates/workdir/src/operation.rs index 67858685..5af8b54a 100644 --- a/crates/workdir/src/operation.rs +++ b/crates/workdir/src/operation.rs @@ -11,6 +11,10 @@ pub struct CommandRequest { pub command: String, pub timeout_secs: u64, pub output_limit: usize, + /// Workdir-relative command directory. Providers validate it against the + /// active session before process start. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cwd: Option, /// Provider-local directory where complete output is retained when the /// inline result exceeds `output_limit`. pub spill_dir: Option, diff --git a/crates/workdir/src/scope.rs b/crates/workdir/src/scope.rs new file mode 100644 index 00000000..131ae6cf --- /dev/null +++ b/crates/workdir/src/scope.rs @@ -0,0 +1,1944 @@ +use std::collections::{HashMap, HashSet}; +use std::path::Path; +use std::sync::atomic::{AtomicBool, AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, Weak}; + +use async_trait::async_trait; +use fs_operation::{ + EditRequest, EditResult, FsPath, GlobRequest, GlobResult, GrepRequest, GrepResult, ListRequest, + ListResult, ReadRequest, ReadResult, StatRequest, StatResult, WriteRequest, WriteResult, +}; +use tokio::sync::broadcast; + +const MAX_SCOPED_COMMANDS: usize = 16; + +use crate::{ + CommandEvent, CommandHandle, CommandOutput, CommandOutputRequest, CommandRequest, + CommandSnapshot, CommandStatus, CommandStream, Workdir, WorkdirError, WorkdirSession, + WorkdirSessionCapabilities, WorkdirSessionCapability, WorkdirSessionHandle, +}; + +#[derive(Clone, Copy, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum WorkdirToolScopePermission { + Read, + Write, +} + +#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WorkdirToolScopeRule { + pub target: FsPath, + pub permission: WorkdirToolScopePermission, + pub recursive: bool, +} + +#[derive(Clone, Debug, Eq, PartialEq, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +pub struct WorkdirToolScope { + pub rules: Vec, + pub cwd: FsPath, + pub command: bool, +} + +#[derive(Clone)] +pub struct WorkdirToolBroker { + authority: Arc, + session: WorkdirSessionHandle, + event_forwarder: Option>>>>, +} + +impl std::fmt::Debug for WorkdirToolBroker { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("WorkdirToolBroker") + .field("workdir", self.session.workdir()) + .field("capabilities", &self.session.capabilities()) + .finish_non_exhaustive() + } +} + +impl WorkdirToolBroker { + /// Own the parent Worker's active session and mediate every scoped child operation. + pub fn new(source: WorkdirSessionHandle) -> Self { + let capabilities = source.capabilities(); + let (command_events, _) = broadcast::channel(64); + let authority = Arc::new(ScopedWorkdirSession { + source, + cwd: FsPath::new("").expect("empty Workdir path is valid"), + scope: None, + capabilities, + validity: SessionValidity::root(), + child_write_leases: Mutex::new(HashMap::new()), + next_lease_id: AtomicU64::new(1), + close_lock: Arc::new(tokio::sync::Mutex::new(())), + owned_commands: Arc::new(Mutex::new(HashSet::new())), + pending_command_events: Arc::new(Mutex::new(HashMap::new())), + starting_tool_calls: Arc::new(Mutex::new(HashSet::new())), + forwarded_starts: Arc::new(Mutex::new(HashSet::new())), + forwarded_terminals: Arc::new(Mutex::new(HashSet::new())), + command_events, + closes_source: true, + #[cfg(test)] + command_start_gate: Mutex::new(None), + }); + Self { + session: authority.clone(), + authority, + event_forwarder: None, + } + } + + /// Session used only by tools registered by the owning Worker. + pub fn tool_session(&self) -> WorkdirSessionHandle { + self.session.clone() + } + + /// Create a revocable, attenuated tool route without delegating a provider session. + pub async fn scope( + &self, + request: WorkdirToolScope, + ) -> Result { + self.authority.scope(request).await + } +} + +impl std::ops::Deref for WorkdirToolBroker { + type Target = WorkdirSessionHandle; + + fn deref(&self) -> &Self::Target { + &self.session + } +} + +pub struct WorkdirScopeLease { + broker: WorkdirToolBroker, + pub capabilities: WorkdirSessionCapabilities, + validity: Arc, + cleanup_pending: Arc, + close_lock: Arc>, +} + +impl std::fmt::Debug for WorkdirScopeLease { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("WorkdirScopeLease") + .field("workdir", self.broker.session.workdir()) + .field("capabilities", &self.capabilities) + .field("active", &self.is_active()) + .finish() + } +} + +impl WorkdirScopeLease { + pub fn broker(&self) -> WorkdirToolBroker { + self.broker.clone() + } + + pub fn tool_session(&self) -> WorkdirSessionHandle { + self.broker.tool_session() + } + + pub async fn scope( + &self, + request: WorkdirToolScope, + ) -> Result { + self.broker.scope(request).await + } + + pub async fn close(&self) -> Result<(), WorkdirError> { + let _close_guard = self.close_lock.lock().await; + if !self.cleanup_pending.load(Ordering::Acquire) { + return Ok(()); + } + self.validity.active.store(false, Ordering::Release); + let command_ids = self + .broker + .authority + .owned_commands + .lock() + .expect("scoped command set mutex poisoned") + .iter() + .cloned() + .collect::>(); + let mut first_error = None; + for command_id in command_ids { + let handle = CommandHandle(command_id.clone()); + let cancel = self + .broker + .authority + .source + .cancel_command(handle.clone()) + .await; + let terminal = self + .broker + .authority + .source + .command_output(CommandOutputRequest { + handle, + cursor: 0, + limit: 1, + wait: true, + }) + .await; + match (cancel, terminal) { + (_, Ok(output)) => { + self.broker.authority.publish_terminal_if_missing( + &command_id, + output.status, + output.exit_code, + output.next_cursor.unwrap_or(output.content.len()) as u64, + &output.content, + ); + self.broker + .authority + .owned_commands + .lock() + .expect("scoped command set mutex poisoned") + .remove(&command_id); + } + (Ok(()), Err(WorkdirError::UnknownCommand(_))) + | (Err(WorkdirError::UnknownCommand(_)), Err(WorkdirError::UnknownCommand(_))) => { + self.broker.authority.publish_terminal_if_missing( + &command_id, + CommandStatus::Cancelled, + None, + 0, + "", + ); + self.broker + .authority + .owned_commands + .lock() + .expect("scoped command set mutex poisoned") + .remove(&command_id); + } + (Err(error), _) | (_, Err(error)) => { + first_error.get_or_insert(error); + } + } + } + if let Some(error) = first_error { + return Err(error); + } + tokio::task::yield_now().await; + self.finish_release(); + Ok(()) + } + + pub fn is_active(&self) -> bool { + self.validity.is_active() + } + + /// Revoke a scope whose owner has already terminalized every tool call. + /// Use [`Self::close`] when commands may still be live. + pub fn revoke(&self) { + self.finish_release(); + } + + fn finish_release(&self) { + self.validity.active.store(false, Ordering::Release); + self.cleanup_pending.store(false, Ordering::Release); + if let Some(forwarder) = &self.broker.event_forwarder + && let Some(handle) = forwarder + .lock() + .expect("scoped command forwarder mutex poisoned") + .take() + { + handle.abort(); + } + } +} + +impl std::ops::Deref for WorkdirScopeLease { + type Target = WorkdirSessionHandle; + + fn deref(&self) -> &Self::Target { + &self.broker.session + } +} + +impl Drop for WorkdirScopeLease { + fn drop(&mut self) { + self.finish_release(); + } +} + +#[derive(Debug)] +struct SessionValidity { + active: AtomicBool, + parent: Option>, +} + +impl SessionValidity { + fn root() -> Arc { + Arc::new(Self { + active: AtomicBool::new(true), + parent: None, + }) + } + + fn child(parent: Arc) -> Arc { + Arc::new(Self { + active: AtomicBool::new(true), + parent: Some(parent), + }) + } + + fn is_active(&self) -> bool { + self.active.load(Ordering::Acquire) + && self.parent.as_ref().is_none_or(|parent| parent.is_active()) + } +} + +#[derive(Clone, Debug)] +struct ActiveWriteLease { + validity: Weak, + cleanup_pending: Weak, + rules: Vec, +} + +#[cfg(test)] +struct TestCommandStartGate { + entered: tokio::sync::Notify, + release: tokio::sync::Notify, +} + +struct ScopedWorkdirSession { + source: WorkdirSessionHandle, + cwd: FsPath, + scope: Option>, + capabilities: WorkdirSessionCapabilities, + validity: Arc, + child_write_leases: Mutex>, + next_lease_id: AtomicU64, + close_lock: Arc>, + owned_commands: Arc>>, + pending_command_events: Arc>>>, + starting_tool_calls: Arc>>, + forwarded_starts: Arc>>, + forwarded_terminals: Arc>>, + command_events: broadcast::Sender, + closes_source: bool, + #[cfg(test)] + command_start_gate: Mutex>>, +} + +impl std::fmt::Debug for ScopedWorkdirSession { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ScopedWorkdirSession") + .field("workdir", &self.source.workdir()) + .field("scope", &self.scope) + .field("capabilities", &self.capabilities) + .field("active", &self.validity.is_active()) + .finish_non_exhaustive() + } +} + +impl ScopedWorkdirSession { + fn ensure_active(&self) -> Result<(), WorkdirError> { + if self.validity.is_active() { + Ok(()) + } else { + Err(WorkdirError::SessionClosed) + } + } + + fn ensure_capability( + &self, + required: WorkdirSessionCapability, + operation: &'static str, + ) -> Result<(), WorkdirError> { + self.ensure_active()?; + if self.capabilities.supports(required) { + Ok(()) + } else { + Err(WorkdirError::Denied(format!( + "scoped Workdir tools do not permit {operation}" + ))) + } + } + + fn ensure_path( + &self, + path: &FsPath, + permission: WorkdirToolScopePermission, + ) -> Result<(), WorkdirError> { + self.ensure_active()?; + if let Some(scope) = &self.scope { + if !scope + .iter() + .any(|rule| rule_allows_path(rule, path, permission)) + { + return Err(WorkdirError::Denied(format!( + "logical workdir path `{path}` is outside the scoped {permission:?} scope" + ))); + } + } + if permission == WorkdirToolScopePermission::Write { + self.ensure_parent_write_available(path)?; + } + Ok(()) + } + + fn resolve_path(&self, path: &FsPath) -> Result { + if self.cwd.as_str().is_empty() { + return Ok(path.clone()); + } + let joined = Path::new(self.cwd.as_str()).join(path.as_str()); + let joined = joined.to_str().ok_or_else(|| { + WorkdirError::Denied("logical Workdir path is not valid UTF-8".into()) + })?; + FsPath::new(joined).map_err(|error| WorkdirError::Denied(error.to_string())) + } + + fn ensure_read( + &self, + path: &FsPath, + capability: WorkdirSessionCapability, + ) -> Result<(), WorkdirError> { + self.ensure_capability(capability, "read operations")?; + self.ensure_path(path, WorkdirToolScopePermission::Read) + } + + fn ensure_write( + &self, + path: &FsPath, + capability: WorkdirSessionCapability, + ) -> Result<(), WorkdirError> { + self.ensure_capability(capability, "write operations")?; + self.ensure_path(path, WorkdirToolScopePermission::Write) + } + + fn ensure_command(&self) -> Result<(), WorkdirError> { + self.ensure_capability(WorkdirSessionCapability::Command, "command execution") + } + + fn ensure_owned_command(&self, handle: &CommandHandle) -> Result<(), WorkdirError> { + self.ensure_command()?; + if self.scope.is_none() + || self + .owned_commands + .lock() + .expect("scoped command set mutex poisoned") + .contains(&handle.0) + { + Ok(()) + } else { + Err(WorkdirError::UnknownCommand(handle.0.clone())) + } + } + + fn publish_terminal_if_missing( + &self, + command_id: &str, + status: CommandStatus, + exit_code: Option, + offset: u64, + fallback_output: &str, + ) { + let mut terminals = self + .forwarded_terminals + .lock() + .expect("forwarded terminal command mutex poisoned"); + if !terminals.insert(command_id.to_string()) { + return; + } + if !fallback_output.is_empty() { + let _ = self.command_events.send(CommandEvent::Output { + command_id: command_id.to_string(), + stream: CommandStream::Stdout, + start_offset: 0, + end_offset: fallback_output.len() as u64, + content: fallback_output.to_string(), + observed_at_ms: unix_timestamp_ms(), + }); + } + let _ = self.command_events.send(CommandEvent::Terminal { + command_id: command_id.to_string(), + status, + exit_code, + stdout_end_offset: offset, + stderr_end_offset: 0, + observed_at_ms: unix_timestamp_ms(), + }); + } + + fn ensure_parent_write_available(&self, path: &FsPath) -> Result<(), WorkdirError> { + let mut leases = self + .child_write_leases + .lock() + .expect("Workdir tool scope lease mutex poisoned"); + leases.retain(|_, lease| { + lease + .validity + .upgrade() + .is_some_and(|validity| validity.is_active()) + || lease + .cleanup_pending + .upgrade() + .is_some_and(|pending| pending.load(Ordering::Acquire)) + }); + if leases.values().any(|lease| { + lease.rules.iter().any(|rule| { + rule.permission == WorkdirToolScopePermission::Write + && rule_allows_path(rule, path, WorkdirToolScopePermission::Write) + }) + }) { + Err(WorkdirError::Denied(format!( + "logical workdir path `{path}` is leased to child Workdir tools" + ))) + } else { + Ok(()) + } + } + + async fn ensure_source_path_has_no_symlink(&self, path: &FsPath) -> Result<(), WorkdirError> { + let mut current = String::new(); + for component in Path::new(path.as_str()).components() { + let component = component.as_os_str().to_string_lossy(); + if component.is_empty() || component == "." { + continue; + } + if !current.is_empty() { + current.push('/'); + } + current.push_str(&component); + let current = FsPath::new(¤t).map_err(|error| { + WorkdirError::Denied(format!("invalid scoped Workdir path: {error}")) + })?; + match self.source.stat(StatRequest { path: current }).await { + Ok(result) if result.kind == fs_operation::EntryKind::Symlink => { + return Err(WorkdirError::Denied(format!( + "scoped Workdir path `{path}` traverses a symlink" + ))); + } + Ok(_) => {} + Err(WorkdirError::NotFound(_)) => break, + Err(error) => return Err(error), + } + } + Ok(()) + } + + async fn ensure_scope_targets_do_not_traverse_symlinks( + &self, + rules: &[WorkdirToolScopeRule], + ) -> Result<(), WorkdirError> { + for rule in rules { + self.ensure_source_path_has_no_symlink(&rule.target).await?; + } + Ok(()) + } + + async fn resolve_operation_path(&self, path: &FsPath) -> Result { + self.ensure_active()?; + let resolved = self.resolve_path(path)?; + if self.scope.is_some() { + self.ensure_source_path_has_no_symlink(&resolved).await?; + } + Ok(resolved) + } + + fn validate_scope( + &self, + rules: &[WorkdirToolScopeRule], + command: bool, + ) -> Result { + self.ensure_active()?; + if rules.is_empty() { + return Err(WorkdirError::Denied( + "workdir tool scope requires at least one logical scope rule".into(), + )); + } + let writable = rules + .iter() + .any(|rule| rule.permission == WorkdirToolScopePermission::Write); + if !self.capabilities.supports(WorkdirSessionCapability::Read) + || (writable + && (!self.capabilities.supports(WorkdirSessionCapability::Write) + || !self.capabilities.supports(WorkdirSessionCapability::Edit))) + { + return Err(WorkdirError::Denied( + "parent Workdir session cannot scope the requested capabilities".into(), + )); + } + if command { + if !writable { + return Err(WorkdirError::Denied( + "command execution requires a writable scoped path".into(), + )); + } + if !self + .capabilities + .supports(WorkdirSessionCapability::Command) + { + return Err(WorkdirError::Denied( + "parent Workdir session does not support Command".into(), + )); + } + } + for requested in rules { + if let Some(scope) = &self.scope { + if !scope + .iter() + .any(|parent| rule_contains_rule(parent, requested)) + { + return Err(WorkdirError::Denied(format!( + "logical workdir scope `{}` exceeds the parent tool scope", + requested.target + ))); + } + } + } + let mut delegated = vec![WorkdirSessionCapability::Read]; + for capability in [ + WorkdirSessionCapability::Glob, + WorkdirSessionCapability::Grep, + ] { + if self.capabilities.supports(capability) { + delegated.push(capability); + } + } + if writable { + delegated.push(WorkdirSessionCapability::Write); + delegated.push(WorkdirSessionCapability::Edit); + } + if command { + delegated.push(WorkdirSessionCapability::Command); + } + Ok(WorkdirSessionCapabilities::from_capabilities(delegated)) + } + + async fn scope( + self: &Arc, + request: WorkdirToolScope, + ) -> Result { + let capabilities = self.validate_scope(&request.rules, request.command)?; + if !request + .rules + .iter() + .any(|rule| rule_allows_path(rule, &request.cwd, WorkdirToolScopePermission::Read)) + { + return Err(WorkdirError::Denied(format!( + "scoped tool cwd `{}` is outside the readable scope", + request.cwd + ))); + } + self.ensure_scope_targets_do_not_traverse_symlinks(&request.rules) + .await?; + let validity = SessionValidity::child(self.validity.clone()); + let cleanup_pending = Arc::new(AtomicBool::new(true)); + let id = self.next_lease_id.fetch_add(1, Ordering::Relaxed); + if request + .rules + .iter() + .any(|rule| rule.permission == WorkdirToolScopePermission::Write) + { + let mut leases = self + .child_write_leases + .lock() + .expect("Workdir tool scope lease mutex poisoned"); + leases.retain(|_, lease| { + lease + .validity + .upgrade() + .is_some_and(|validity| validity.is_active()) + || lease + .cleanup_pending + .upgrade() + .is_some_and(|pending| pending.load(Ordering::Acquire)) + }); + let requested_write_rules = request + .rules + .iter() + .filter(|rule| rule.permission == WorkdirToolScopePermission::Write); + for requested in requested_write_rules { + if leases.values().any(|lease| { + lease + .rules + .iter() + .any(|active| rules_overlap(active, requested)) + }) { + return Err(WorkdirError::Denied(format!( + "scoped write path `{}` overlaps an active child scope", + requested.target + ))); + } + } + leases.insert( + id, + ActiveWriteLease { + validity: Arc::downgrade(&validity), + cleanup_pending: Arc::downgrade(&cleanup_pending), + rules: request.rules.clone(), + }, + ); + } + let owned_commands = Arc::new(Mutex::new(HashSet::new())); + let pending_command_events = Arc::new(Mutex::new(HashMap::new())); + let starting_tool_calls = Arc::new(Mutex::new(HashSet::new())); + let forwarded_starts = Arc::new(Mutex::new(HashSet::new())); + let forwarded_terminals = Arc::new(Mutex::new(HashSet::new())); + let (command_events, _) = broadcast::channel(64); + let event_forwarder = forward_owned_command_events( + self.source.subscribe_command_events(), + owned_commands.clone(), + pending_command_events.clone(), + starting_tool_calls.clone(), + forwarded_starts.clone(), + forwarded_terminals.clone(), + command_events.clone(), + ) + .map(|handle| Arc::new(Mutex::new(Some(handle)))); + let close_lock = Arc::new(tokio::sync::Mutex::new(())); + let child = Arc::new(ScopedWorkdirSession { + source: self.source.clone(), + cwd: request.cwd, + scope: Some(request.rules), + capabilities, + validity: validity.clone(), + child_write_leases: Mutex::new(HashMap::new()), + next_lease_id: AtomicU64::new(1), + close_lock: close_lock.clone(), + owned_commands, + pending_command_events, + starting_tool_calls, + forwarded_starts, + forwarded_terminals, + command_events, + closes_source: false, + #[cfg(test)] + command_start_gate: Mutex::new(None), + }); + let broker = WorkdirToolBroker { + session: child.clone(), + authority: child, + event_forwarder, + }; + Ok(WorkdirScopeLease { + broker, + capabilities, + validity, + cleanup_pending, + close_lock, + }) + } +} + +#[async_trait] +impl WorkdirSession for ScopedWorkdirSession { + fn workdir(&self) -> &Workdir { + self.source.workdir() + } + + fn capabilities(&self) -> WorkdirSessionCapabilities { + self.capabilities + } + + async fn stat(&self, mut request: StatRequest) -> Result { + let path = self.resolve_operation_path(&request.path).await?; + self.ensure_read(&path, WorkdirSessionCapability::Read)?; + request.path = path; + self.source.stat(request).await + } + + async fn read(&self, mut request: ReadRequest) -> Result { + let path = self.resolve_operation_path(&request.path).await?; + self.ensure_read(&path, WorkdirSessionCapability::Read)?; + request.path = path; + self.source.read(request).await + } + + async fn write(&self, mut request: WriteRequest) -> Result { + let path = self.resolve_operation_path(&request.path).await?; + self.ensure_write(&path, WorkdirSessionCapability::Write)?; + request.path = path; + self.source.write(request).await + } + + async fn edit(&self, mut request: EditRequest) -> Result { + let path = self.resolve_operation_path(&request.path).await?; + self.ensure_write(&path, WorkdirSessionCapability::Edit)?; + request.path = path; + self.source.edit(request).await + } + + async fn list(&self, mut request: ListRequest) -> Result { + let path = self.resolve_operation_path(&request.path).await?; + self.ensure_read(&path, WorkdirSessionCapability::Read)?; + request.path = path; + self.source.list(request).await + } + + async fn glob(&self, mut request: GlobRequest) -> Result { + let path = self.resolve_operation_path(&request.path).await?; + self.ensure_read(&path, WorkdirSessionCapability::Glob)?; + request.path = path; + self.source.glob(request).await + } + + async fn grep(&self, mut request: GrepRequest) -> Result { + let path = self.resolve_operation_path(&request.path).await?; + self.ensure_read(&path, WorkdirSessionCapability::Grep)?; + request.path = path; + self.source.grep(request).await + } + + async fn start_command( + &self, + mut request: CommandRequest, + ) -> Result { + let _admission_guard = self.close_lock.lock().await; + // Command is an explicit capability, not a typed path mutation. We + // intentionally keep an ancestor's Command capability available while + // a child holds a write scope; only typed Write/Edit operations use the + // best-effort overlapping-path guard below. + self.ensure_command()?; + #[cfg(test)] + { + let gate = self + .command_start_gate + .lock() + .expect("command start gate mutex poisoned") + .clone(); + if let Some(gate) = gate { + gate.entered.notify_one(); + gate.release.notified().await; + } + } + if self.scope.is_some() + && self + .owned_commands + .lock() + .expect("scoped command set mutex poisoned") + .len() + >= MAX_SCOPED_COMMANDS + { + return Err(WorkdirError::Unavailable(format!( + "scoped command limit of {MAX_SCOPED_COMMANDS} is reached" + ))); + } + let tool_call_id = request.tool_call_id.clone(); + if self.scope.is_some() { + request.cwd = Some(match request.cwd.as_ref() { + Some(cwd) => self.resolve_path(cwd)?, + None => self.cwd.clone(), + }); + } + if let Some(tool_call_id) = &tool_call_id { + self.starting_tool_calls + .lock() + .expect("starting tool call mutex poisoned") + .insert(tool_call_id.clone()); + } + let handle = match self.source.start_command(request).await { + Ok(handle) => handle, + Err(error) => { + if let Some(tool_call_id) = &tool_call_id { + self.starting_tool_calls + .lock() + .expect("starting tool call mutex poisoned") + .remove(tool_call_id); + } + return Err(error); + } + }; + let mut owned = self + .owned_commands + .lock() + .expect("scoped command set mutex poisoned"); + owned.insert(handle.0.clone()); + if let Some(tool_call_id) = &tool_call_id { + self.starting_tool_calls + .lock() + .expect("starting tool call mutex poisoned") + .remove(tool_call_id); + } + let pending = self + .pending_command_events + .lock() + .expect("pending scoped command event mutex poisoned") + .remove(&handle.0) + .unwrap_or_default(); + drop(owned); + if !pending + .iter() + .any(|event| matches!(event, CommandEvent::Started { .. })) + { + publish_owned_command_event( + &self.command_events, + &self.forwarded_starts, + &self.forwarded_terminals, + CommandEvent::Started { + command_id: handle.0.clone(), + tool_call_id, + observed_at_ms: unix_timestamp_ms(), + }, + ); + } + for event in pending { + publish_owned_command_event( + &self.command_events, + &self.forwarded_starts, + &self.forwarded_terminals, + event, + ); + } + Ok(handle) + } + + async fn command_status(&self, handle: CommandHandle) -> Result { + self.ensure_owned_command(&handle)?; + self.source.command_status(handle).await + } + + async fn command_output( + &self, + request: CommandOutputRequest, + ) -> Result { + self.ensure_owned_command(&request.handle)?; + let command_id = request.handle.0.clone(); + let output = self.source.command_output(request).await?; + if !matches!(output.status, CommandStatus::Running) { + self.publish_terminal_if_missing( + &command_id, + output.status, + output.exit_code, + output.next_cursor.unwrap_or(output.content.len()) as u64, + &output.content, + ); + self.owned_commands + .lock() + .expect("scoped command set mutex poisoned") + .remove(&command_id); + } + Ok(output) + } + + async fn cancel_command(&self, handle: CommandHandle) -> Result<(), WorkdirError> { + self.ensure_owned_command(&handle)?; + self.source.cancel_command(handle).await + } + + fn subscribe_command_events(&self) -> Option> { + if !self + .capabilities + .supports(WorkdirSessionCapability::Command) + { + return None; + } + if self.scope.is_none() { + self.source.subscribe_command_events() + } else { + Some(self.command_events.subscribe()) + } + } + + fn command_snapshot(&self) -> Vec { + if !self + .capabilities + .supports(WorkdirSessionCapability::Command) + { + return Vec::new(); + } + if self.scope.is_none() { + return self.source.command_snapshot(); + } + let owned = self + .owned_commands + .lock() + .expect("scoped command set mutex poisoned"); + self.source + .command_snapshot() + .into_iter() + .filter(|snapshot| owned.contains(&snapshot.command_id)) + .collect() + } + + async fn close(&self) -> Result<(), WorkdirError> { + self.validity.active.store(false, Ordering::Release); + if self.closes_source { + self.source.close().await + } else { + Ok(()) + } + } +} + +/// A fail-closed read-only view over an already scoped scoped tool route. +#[derive(Debug)] +pub struct ReadOnlyWorkdirSession { + inner: WorkdirSessionHandle, +} + +impl ReadOnlyWorkdirSession { + pub fn new(inner: WorkdirSessionHandle) -> Self { + Self { inner } + } +} + +#[async_trait] +impl WorkdirSession for ReadOnlyWorkdirSession { + fn workdir(&self) -> &Workdir { + self.inner.workdir() + } + + fn capabilities(&self) -> WorkdirSessionCapabilities { + WorkdirSessionCapabilities::READ_ONLY + } + + async fn stat(&self, request: StatRequest) -> Result { + self.inner.stat(request).await + } + + async fn read(&self, request: ReadRequest) -> Result { + self.inner.read(request).await + } + + async fn write(&self, _request: WriteRequest) -> Result { + Err(WorkdirError::Denied("read-only workdir session".into())) + } + + async fn edit(&self, _request: EditRequest) -> Result { + Err(WorkdirError::Denied("read-only workdir session".into())) + } + + async fn list(&self, request: ListRequest) -> Result { + self.inner.list(request).await + } + + async fn glob(&self, request: GlobRequest) -> Result { + self.inner.glob(request).await + } + + async fn grep(&self, request: GrepRequest) -> Result { + self.inner.grep(request).await + } + + async fn start_command(&self, _request: CommandRequest) -> Result { + Err(WorkdirError::Denied("read-only workdir session".into())) + } + + async fn command_status(&self, _handle: CommandHandle) -> Result { + Err(WorkdirError::Denied("read-only workdir session".into())) + } + + async fn command_output( + &self, + _request: CommandOutputRequest, + ) -> Result { + Err(WorkdirError::Denied("read-only workdir session".into())) + } + + async fn cancel_command(&self, _handle: CommandHandle) -> Result<(), WorkdirError> { + Err(WorkdirError::Denied("read-only workdir session".into())) + } + + async fn close(&self) -> Result<(), WorkdirError> { + self.inner.close().await + } +} + +fn forward_owned_command_events( + receiver: Option>, + owned_commands: Arc>>, + pending_command_events: Arc>>>, + starting_tool_calls: Arc>>, + forwarded_starts: Arc>>, + forwarded_terminals: Arc>>, + sender: broadcast::Sender, +) -> Option> { + let mut receiver = receiver?; + Some(tokio::spawn(async move { + loop { + let event = match receiver.recv().await { + Ok(event) => event, + Err(broadcast::error::RecvError::Lagged(_)) => continue, + Err(broadcast::error::RecvError::Closed) => break, + }; + let command_id = command_event_id(&event).to_string(); + let mut owned = owned_commands + .lock() + .expect("scoped command set mutex poisoned"); + if !owned.contains(&command_id) { + let claimed = matches!( + &event, + CommandEvent::Started { + tool_call_id: Some(tool_call_id), + .. + } if starting_tool_calls + .lock() + .expect("starting tool call mutex poisoned") + .contains(tool_call_id) + ); + if !claimed { + continue; + } + owned.insert(command_id.clone()); + pending_command_events + .lock() + .expect("pending scoped command event mutex poisoned") + .entry(command_id) + .or_default() + .push(event); + continue; + } + let mut pending = pending_command_events + .lock() + .expect("pending scoped command event mutex poisoned"); + if let Some(events) = pending.get_mut(&command_id) { + if events.len() < 64 { + events.push(event); + } + continue; + } + drop(pending); + drop(owned); + publish_owned_command_event(&sender, &forwarded_starts, &forwarded_terminals, event); + } + })) +} + +fn command_event_id(event: &CommandEvent) -> &str { + match event { + CommandEvent::Started { command_id, .. } + | CommandEvent::Output { command_id, .. } + | CommandEvent::Terminal { command_id, .. } => command_id, + } +} + +fn publish_owned_command_event( + sender: &broadcast::Sender, + forwarded_starts: &Mutex>, + forwarded_terminals: &Mutex>, + event: CommandEvent, +) { + let command_id = command_event_id(&event); + let mut terminals = forwarded_terminals + .lock() + .expect("forwarded terminal command mutex poisoned"); + match &event { + CommandEvent::Terminal { .. } if !terminals.insert(command_id.to_string()) => return, + CommandEvent::Started { .. } if terminals.contains(command_id) => return, + CommandEvent::Started { .. } + if !forwarded_starts + .lock() + .expect("forwarded command start mutex poisoned") + .insert(command_id.to_string()) => + { + return; + } + CommandEvent::Output { .. } if terminals.contains(command_id) => return, + _ => {} + } + drop(terminals); + let _ = sender.send(event); +} + +fn unix_timestamp_ms() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap_or_default() + .as_millis() + .min(u128::from(u64::MAX)) as u64 +} + +fn rules_overlap(left: &WorkdirToolScopeRule, right: &WorkdirToolScopeRule) -> bool { + left.permission == WorkdirToolScopePermission::Write + && right.permission == WorkdirToolScopePermission::Write + && (rule_allows_path(left, &right.target, WorkdirToolScopePermission::Write) + || rule_allows_path(right, &left.target, WorkdirToolScopePermission::Write)) +} + +fn rule_allows_path( + rule: &WorkdirToolScopeRule, + path: &FsPath, + required: WorkdirToolScopePermission, +) -> bool { + if required == WorkdirToolScopePermission::Write + && rule.permission != WorkdirToolScopePermission::Write + { + return false; + } + path_in_rule(rule, path) +} + +fn path_in_rule(rule: &WorkdirToolScopeRule, path: &FsPath) -> bool { + let target = Path::new(rule.target.as_str()); + let path = Path::new(path.as_str()); + if path == target { + return true; + } + let Ok(suffix) = path.strip_prefix(target) else { + return false; + }; + let depth = suffix.components().count(); + rule.recursive || depth <= 1 +} + +fn rule_contains_rule(parent: &WorkdirToolScopeRule, child: &WorkdirToolScopeRule) -> bool { + if child.permission == WorkdirToolScopePermission::Write + && parent.permission != WorkdirToolScopePermission::Write + { + return false; + } + if !path_in_rule(parent, &child.target) { + return false; + } + if parent.recursive { + return true; + } + !child.recursive && parent.target == child.target +} + +#[cfg(test)] +mod tests { + use std::fs; + + use manifest::{Permission, Scope, ScopeConfig, ScopeRule, SharedScope}; + use tempfile::TempDir; + + use super::*; + use crate::LocalWorkdirSession; + + fn fs_path(path: &str) -> FsPath { + FsPath::new(path).unwrap() + } + + fn session(root: &Path) -> WorkdirToolBroker { + let scope = SharedScope::new( + Scope::from_config(&ScopeConfig { + allow: vec![ScopeRule { + target: root.to_path_buf(), + permission: Permission::Write, + recursive: true, + }], + deny: Vec::new(), + }) + .unwrap(), + ); + WorkdirToolBroker::new(Arc::new(LocalWorkdirSession::materialized_bound( + Workdir::new("delegation-test"), + root.to_path_buf(), + root.to_path_buf(), + scope, + WorkdirSessionCapabilities::ALL, + ))) + } + + fn request(path: &str, permission: WorkdirToolScopePermission) -> WorkdirToolScope { + WorkdirToolScope { + rules: vec![WorkdirToolScopeRule { + target: fs_path(path), + permission, + recursive: true, + }], + cwd: fs_path(path), + command: permission == WorkdirToolScopePermission::Write, + } + } + + fn read(path: &str) -> ReadRequest { + ReadRequest { + path: fs_path(path), + offset: 0, + limit: 20, + max_bytes: 1024, + } + } + + fn write(path: &str, content: &str) -> WriteRequest { + WriteRequest { + path: fs_path(path), + content: content.as_bytes().to_vec(), + expected_hash: None, + } + } + + async fn run_command( + session: &WorkdirSessionHandle, + command: impl Into, + tool_call_id: impl Into, + ) -> CommandOutput { + let handle = session + .start_command(CommandRequest { + command: command.into(), + timeout_secs: 5, + output_limit: 1024, + cwd: None, + spill_dir: None, + tool_call_id: Some(tool_call_id.into()), + }) + .await + .unwrap(); + session + .command_output(CommandOutputRequest { + handle, + cursor: 0, + limit: 1024, + wait: true, + }) + .await + .unwrap() + } + + #[tokio::test] + async fn workdir_tool_broker_session_forwards_command_telemetry() { + let root = TempDir::new().unwrap(); + let parent = session(root.path()); + let mut events = parent + .subscribe_command_events() + .expect("delegation wrapper must preserve command observation"); + let handle = parent + .start_command(CommandRequest { + command: "printf ready; sleep 0.2; printf done".into(), + timeout_secs: 5, + output_limit: 1024, + cwd: None, + spill_dir: None, + tool_call_id: Some("tool-delegated".into()), + }) + .await + .unwrap(); + + let first_output = loop { + let event = tokio::time::timeout(std::time::Duration::from_secs(2), events.recv()) + .await + .expect("delegated command telemetry should not stall") + .unwrap(); + if let CommandEvent::Output { content, .. } = event { + break content; + } + }; + assert_eq!(first_output, "ready"); + let snapshots = parent.command_snapshot(); + assert_eq!(snapshots.len(), 1); + assert_eq!(snapshots[0].command_id, handle.0); + assert_eq!(snapshots[0].status, CommandStatus::Running); + assert_eq!(snapshots[0].stdout.content, "ready"); + + let output = parent + .command_output(CommandOutputRequest { + handle, + cursor: 0, + limit: 1024, + wait: true, + }) + .await + .unwrap(); + assert_eq!(output.status, CommandStatus::Completed); + assert_eq!(output.content, "readydone"); + assert!(parent.command_snapshot().is_empty()); + } + + #[tokio::test] + async fn write_scope_without_command_grant_has_no_command_capability() { + let root = TempDir::new().unwrap(); + fs::create_dir_all(root.path().join("work")).unwrap(); + let parent = session(root.path()); + let child = parent + .scope(WorkdirToolScope { + rules: vec![WorkdirToolScopeRule { + target: fs_path("work"), + permission: WorkdirToolScopePermission::Write, + recursive: true, + }], + cwd: fs_path("work"), + command: false, + }) + .await + .unwrap(); + + assert!(child.capabilities.supports(WorkdirSessionCapability::Write)); + assert!( + !child + .capabilities + .supports(WorkdirSessionCapability::Command) + ); + let error = child + .start_command(CommandRequest { + command: "pwd".into(), + timeout_secs: 5, + output_limit: 1024, + cwd: None, + spill_dir: None, + tool_call_id: None, + }) + .await + .unwrap_err(); + assert!(matches!(error, WorkdirError::Denied(_))); + } + + #[tokio::test] + async fn scoped_commands_use_child_cwd_and_do_not_leak_between_siblings() { + let root = TempDir::new().unwrap(); + fs::create_dir_all(root.path().join("one")).unwrap(); + fs::create_dir_all(root.path().join("two")).unwrap(); + let parent = session(root.path()); + let first = parent + .scope(request("one", WorkdirToolScopePermission::Write)) + .await + .unwrap(); + let second = parent + .scope(request("two", WorkdirToolScopePermission::Write)) + .await + .unwrap(); + let mut first_events = first.subscribe_command_events().unwrap(); + let mut second_events = second.subscribe_command_events().unwrap(); + + let handle = first + .start_command(CommandRequest { + command: "pwd; sleep 0.2".into(), + timeout_secs: 5, + output_limit: 4096, + cwd: None, + spill_dir: None, + tool_call_id: Some("first-command".into()), + }) + .await + .unwrap(); + assert!(matches!( + first_events.recv().await.unwrap(), + CommandEvent::Started { .. } + )); + assert!(matches!( + tokio::time::timeout(std::time::Duration::from_millis(50), second_events.recv()).await, + Err(_) + )); + assert!(matches!( + second.command_status(handle.clone()).await, + Err(WorkdirError::UnknownCommand(_)) + )); + + let output = first + .command_output(CommandOutputRequest { + handle, + cursor: 0, + limit: 4096, + wait: true, + }) + .await + .unwrap(); + let expected = root.path().join("one").to_string_lossy().into_owned(); + assert!( + output + .content + .lines() + .next() + .is_some_and(|line| line == expected) + ); + } + + #[test] + fn non_recursive_rule_covers_target_and_direct_children_only() { + let rule = WorkdirToolScopeRule { + target: fs_path("docs"), + permission: WorkdirToolScopePermission::Read, + recursive: false, + }; + assert!(path_in_rule(&rule, &fs_path("docs"))); + assert!(path_in_rule(&rule, &fs_path("docs/readme.md"))); + assert!(!path_in_rule(&rule, &fs_path("docs/guides/start.md"))); + } + + #[tokio::test] + async fn read_only_delegation_allows_prefix_and_denies_mutation() { + let root = TempDir::new().unwrap(); + fs::create_dir_all(root.path().join("docs")).unwrap(); + fs::create_dir_all(root.path().join("secret")).unwrap(); + fs::write(root.path().join("docs/readme.md"), "visible").unwrap(); + fs::write(root.path().join("secret/key"), "hidden").unwrap(); + let parent = session(root.path()); + + let child = parent + .scope(request("docs", WorkdirToolScopePermission::Read)) + .await + .unwrap(); + assert_eq!(child.capabilities, WorkdirSessionCapabilities::READ_ONLY); + assert_eq!( + child.read(read("readme.md")).await.unwrap().bytes, + b"visible" + ); + assert!(matches!( + child.write(write("new.md", "no")).await, + Err(WorkdirError::Denied(_)) + )); + assert!( + !child + .capabilities + .supports(WorkdirSessionCapability::Command) + ); + assert!(child.subscribe_command_events().is_none()); + assert!(child.command_snapshot().is_empty()); + assert!(matches!( + child + .start_command(CommandRequest { + command: "printf denied".into(), + timeout_secs: 5, + output_limit: 1024, + cwd: None, + spill_dir: None, + tool_call_id: Some("read-only-command".into()), + }) + .await, + Err(WorkdirError::Denied(_)) + )); + } + + #[cfg(unix)] + #[tokio::test] + async fn provider_scope_denies_read_through_symlink_outside_grant() { + use std::os::unix::fs::symlink; + + let root = TempDir::new().unwrap(); + fs::create_dir_all(root.path().join("granted")).unwrap(); + fs::create_dir_all(root.path().join("secret")).unwrap(); + fs::write(root.path().join("secret/key"), "hidden").unwrap(); + symlink("../secret/key", root.path().join("granted/link")).unwrap(); + let parent = session(root.path()); + let child = parent + .scope(request("granted", WorkdirToolScopePermission::Read)) + .await + .unwrap(); + + let result = child.read(read("link")).await; + assert!( + result.is_err(), + "symlink read escaped provider scope: {result:?}" + ); + } + + #[cfg(unix)] + #[tokio::test] + async fn provider_scope_denies_write_through_symlink_outside_grant() { + use std::os::unix::fs::symlink; + + let root = TempDir::new().unwrap(); + fs::create_dir_all(root.path().join("granted")).unwrap(); + fs::create_dir_all(root.path().join("secret")).unwrap(); + symlink("../secret", root.path().join("granted/outside")).unwrap(); + let parent = session(root.path()); + let child = parent + .scope(request("granted", WorkdirToolScopePermission::Write)) + .await + .unwrap(); + + let result = child.write(write("outside/new", "forbidden")).await; + assert!( + result.is_err(), + "symlink write escaped provider scope: {result:?}" + ); + assert!(!root.path().join("secret/new").exists()); + } + + #[cfg(unix)] + #[tokio::test] + async fn write_delegation_rejects_symlink_target_before_lease() { + use std::os::unix::fs::symlink; + + let root = TempDir::new().unwrap(); + fs::create_dir_all(root.path().join("granted")).unwrap(); + fs::create_dir_all(root.path().join("secret")).unwrap(); + symlink("../secret", root.path().join("granted/outside")).unwrap(); + let parent = session(root.path()); + + assert!(matches!( + parent + .scope(request( + "granted/outside", + WorkdirToolScopePermission::Write + )) + .await, + Err(WorkdirError::Denied(_)) + )); + parent + .write(write("secret/parent", "still-authoritative")) + .await + .unwrap(); + } + + #[tokio::test] + async fn write_lease_keeps_typed_parent_writes_exclusive_without_blocking_commands() { + let root = TempDir::new().unwrap(); + fs::create_dir_all(root.path().join("leased")).unwrap(); + fs::create_dir_all(root.path().join("other")).unwrap(); + let parent = session(root.path()); + let child = parent + .scope(request("leased", WorkdirToolScopePermission::Write)) + .await + .unwrap(); + assert!( + child + .capabilities + .supports(WorkdirSessionCapability::Command) + ); + let child_output = + run_command(&child, "printf child-command", "delegated-child-command").await; + assert_eq!(child_output.content, "child-command"); + let parent_output = run_command( + &parent, + "printf parent-write > leased/from-command; printf parent-command", + "parent-command-during-child-write", + ) + .await; + assert_eq!(parent_output.status, CommandStatus::Completed); + assert_eq!(parent_output.content, "parent-command"); + assert_eq!( + fs::read_to_string(root.path().join("leased/from-command")).unwrap(), + "parent-write" + ); + + assert!(matches!( + parent.write(write("leased/file", "parent")).await, + Err(WorkdirError::Denied(_)) + )); + parent.write(write("other/file", "parent")).await.unwrap(); + child.write(write("file", "child")).await.unwrap(); + child.close().await.unwrap(); + assert!(matches!( + child + .start_command(CommandRequest { + command: "printf revoked".into(), + timeout_secs: 5, + output_limit: 1024, + cwd: None, + spill_dir: None, + tool_call_id: Some("revoked-child-command".into()), + }) + .await, + Err(WorkdirError::SessionClosed) + )); + parent + .write(write("leased/parent", "parent")) + .await + .unwrap(); + assert!(matches!( + child.read(read("file")).await, + Err(WorkdirError::SessionClosed) + )); + } + + #[tokio::test] + async fn sibling_write_scopes_must_not_overlap() { + let root = TempDir::new().unwrap(); + fs::create_dir_all(root.path().join("shared/one")).unwrap(); + fs::create_dir_all(root.path().join("other")).unwrap(); + let parent = session(root.path()); + let first = parent + .scope(request("shared", WorkdirToolScopePermission::Write)) + .await + .unwrap(); + + assert!(matches!( + parent + .scope(request("shared/one", WorkdirToolScopePermission::Write)) + .await, + Err(WorkdirError::Denied(_)) + )); + let other = parent + .scope(request("other", WorkdirToolScopePermission::Write)) + .await + .unwrap(); + other.close().await.unwrap(); + first.close().await.unwrap(); + } + + #[tokio::test] + async fn fast_command_keeps_started_output_terminal_event_order() { + let root = TempDir::new().unwrap(); + fs::create_dir_all(root.path().join("work")).unwrap(); + let parent = session(root.path()); + let child = parent + .scope(request("work", WorkdirToolScopePermission::Write)) + .await + .unwrap(); + let mut events = child.subscribe_command_events().unwrap(); + + let output = run_command(&child.tool_session(), "printf fast-output", "fast-command").await; + assert_eq!(output.content, "fast-output"); + + let mut kinds = Vec::new(); + let mut streamed = String::new(); + while kinds.last().is_none_or(|kind| *kind != "terminal") { + let event = tokio::time::timeout(std::time::Duration::from_secs(1), events.recv()) + .await + .expect("fast command event timeout") + .expect("fast command event channel"); + match event { + CommandEvent::Started { .. } => kinds.push("started"), + CommandEvent::Output { content, .. } => { + kinds.push("output"); + streamed.push_str(&content); + } + CommandEvent::Terminal { .. } => kinds.push("terminal"), + } + } + assert_eq!(kinds.first(), Some(&"started")); + assert_eq!(kinds.last(), Some(&"terminal")); + assert_eq!(kinds.iter().filter(|kind| **kind == "started").count(), 1); + assert!(kinds.contains(&"output")); + assert!(streamed.contains("fast-output")); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(100), events.recv()) + .await + .is_err(), + "no provider event may follow the terminal event" + ); + assert!(child.command_snapshot().is_empty()); + child.close().await.unwrap(); + } + + #[tokio::test] + async fn scoped_command_ceiling_rejects_the_seventeenth_live_command() { + let root = TempDir::new().unwrap(); + fs::create_dir_all(root.path().join("work")).unwrap(); + let parent = session(root.path()); + let child = parent + .scope(request("work", WorkdirToolScopePermission::Write)) + .await + .unwrap(); + for index in 0..MAX_SCOPED_COMMANDS { + child + .start_command(CommandRequest { + command: "sleep 30".into(), + timeout_secs: 60, + output_limit: 1024, + cwd: None, + spill_dir: None, + tool_call_id: Some(format!("command-{index}")), + }) + .await + .unwrap(); + } + + let error = child + .start_command(CommandRequest { + command: "sleep 30".into(), + timeout_secs: 60, + output_limit: 1024, + cwd: None, + spill_dir: None, + tool_call_id: Some("command-over-limit".into()), + }) + .await + .unwrap_err(); + assert!(matches!(error, WorkdirError::Unavailable(message) if message.contains("limit"))); + child.close().await.unwrap(); + } + + #[tokio::test] + async fn close_serializes_with_inflight_command_admission() { + let root = TempDir::new().unwrap(); + fs::create_dir_all(root.path().join("work")).unwrap(); + let parent = session(root.path()); + let child = Arc::new( + parent + .scope(request("work", WorkdirToolScopePermission::Write)) + .await + .unwrap(), + ); + let gate = Arc::new(TestCommandStartGate { + entered: tokio::sync::Notify::new(), + release: tokio::sync::Notify::new(), + }); + *child.broker.authority.command_start_gate.lock().unwrap() = Some(gate.clone()); + let entered = gate.entered.notified(); + let command_child = child.clone(); + let command = tokio::spawn(async move { + command_child + .start_command(CommandRequest { + command: "sleep 30".into(), + timeout_secs: 60, + output_limit: 1024, + cwd: None, + spill_dir: None, + tool_call_id: Some("racing-command".into()), + }) + .await + }); + entered.await; + let close_child = child.clone(); + let mut close = tokio::spawn(async move { close_child.close().await }); + assert!( + tokio::time::timeout(std::time::Duration::from_millis(50), &mut close) + .await + .is_err(), + "close must wait for command admission to commit or fail" + ); + + gate.release.notify_one(); + command.await.unwrap().unwrap(); + close.await.unwrap().unwrap(); + assert!(!child.is_active()); + assert!( + child + .broker + .authority + .owned_commands + .lock() + .unwrap() + .is_empty() + ); + } + + #[tokio::test] + async fn closing_scope_cancels_and_terminalizes_owned_commands() { + let root = TempDir::new().unwrap(); + fs::create_dir_all(root.path().join("work")).unwrap(); + let parent = session(root.path()); + let child = parent + .scope(request("work", WorkdirToolScopePermission::Write)) + .await + .unwrap(); + let mut events = child.subscribe_command_events().unwrap(); + let handle = child + .start_command(CommandRequest { + command: "sleep 30; printf leaked > marker".into(), + timeout_secs: 60, + output_limit: 1024, + cwd: None, + spill_dir: None, + tool_call_id: Some("owned-command".into()), + }) + .await + .unwrap(); + assert!(matches!( + events.recv().await.unwrap(), + CommandEvent::Started { .. } + )); + + child.close().await.unwrap(); + + assert!(matches!( + parent.command_status(handle).await, + Ok(CommandStatus::Cancelled | CommandStatus::Completed | CommandStatus::Failed) + | Err(WorkdirError::UnknownCommand(_)) + )); + assert!(!root.path().join("work/marker").exists()); + let terminal = tokio::time::timeout(std::time::Duration::from_secs(1), async { + loop { + if let CommandEvent::Terminal { .. } = events.recv().await.unwrap() { + break; + } + } + }) + .await; + assert!( + terminal.is_ok(), + "scope close must publish terminal command telemetry" + ); + } + + #[tokio::test] + async fn nested_delegation_is_attenuated_and_parent_revocation_cascades() { + let root = TempDir::new().unwrap(); + fs::create_dir_all(root.path().join("docs/sub")).unwrap(); + fs::create_dir_all(root.path().join("docs/peer")).unwrap(); + fs::write(root.path().join("docs/sub/a"), "a").unwrap(); + fs::write(root.path().join("docs/peer/b"), "b").unwrap(); + let root_session = session(root.path()); + let child = root_session + .scope(request("docs", WorkdirToolScopePermission::Read)) + .await + .unwrap(); + let nested = child + .scope(request("docs/sub", WorkdirToolScopePermission::Read)) + .await + .unwrap(); + + nested.read(read("a")).await.unwrap(); + assert!( + child + .scope(request("other", WorkdirToolScopePermission::Read)) + .await + .is_err() + ); + assert!( + child + .scope(request("docs/sub", WorkdirToolScopePermission::Write)) + .await + .is_err() + ); + + child.close().await.unwrap(); + assert!(matches!( + nested.read(read("a")).await, + Err(WorkdirError::SessionClosed) + )); + } + + #[tokio::test] + async fn nested_write_leases_do_not_block_command_capable_ancestors() { + let root = TempDir::new().unwrap(); + fs::create_dir_all(root.path().join("docs/sub")).unwrap(); + let root_session = session(root.path()); + let child = root_session + .scope(request("docs", WorkdirToolScopePermission::Write)) + .await + .unwrap(); + let nested = child + .scope(request("docs/sub", WorkdirToolScopePermission::Write)) + .await + .unwrap(); + + for (session, label) in [ + (root_session.tool_session(), "root"), + (child.tool_session(), "child"), + (nested.tool_session(), "nested"), + ] { + let output = run_command( + &session, + format!("printf {label}"), + format!("{label}-command-during-nested-write"), + ) + .await; + assert_eq!(output.status, CommandStatus::Completed); + assert_eq!(output.content, label); + } + + assert!(matches!( + root_session.write(write("docs/root", "blocked")).await, + Err(WorkdirError::Denied(_)) + )); + assert!(matches!( + child.write(write("sub/child", "blocked")).await, + Err(WorkdirError::Denied(_)) + )); + nested.write(write("nested", "allowed")).await.unwrap(); + + nested.close().await.unwrap(); + child.close().await.unwrap(); + } + + #[tokio::test] + async fn closing_parent_invalidates_scoped_tools() { + let root = TempDir::new().unwrap(); + fs::create_dir_all(root.path().join("docs")).unwrap(); + fs::write(root.path().join("docs/a"), "a").unwrap(); + let parent = session(root.path()); + let child = parent + .scope(request("docs", WorkdirToolScopePermission::Read)) + .await + .unwrap(); + + parent.close().await.unwrap(); + assert!(matches!( + parent + .start_command(CommandRequest { + command: "printf closed".into(), + timeout_secs: 5, + output_limit: 1024, + cwd: None, + spill_dir: None, + tool_call_id: Some("closed-parent-command".into()), + }) + .await, + Err(WorkdirError::SessionClosed) + )); + let child_result = child.read(read("a")).await; + assert!( + matches!(child_result, Err(WorkdirError::SessionClosed)), + "child result after parent close: {child_result:?}" + ); + } +} diff --git a/crates/workdir/src/workspace.rs b/crates/workdir/src/workspace.rs index 3872bc9d..fc696f99 100644 --- a/crates/workdir/src/workspace.rs +++ b/crates/workdir/src/workspace.rs @@ -104,15 +104,5 @@ mod tests { #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] pub struct WorkspaceWorkdirSessionOperationRequest { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub expected_session_fence: Option, - #[serde(default, skip_serializing_if = "Vec::is_empty")] - pub delegations: Vec, pub operation: crate::http::WorkdirSessionOperation, } - -#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct WorkspaceWorkdirSessionFence { - pub value: String, -} diff --git a/crates/worker-runtime/src/catalog.rs b/crates/worker-runtime/src/catalog.rs index 3cc27d01..96bb2beb 100644 --- a/crates/worker-runtime/src/catalog.rs +++ b/crates/worker-runtime/src/catalog.rs @@ -307,6 +307,8 @@ pub struct WorkerSummary { pub worker_id: WorkerId, pub status: WorkerStatus, #[serde(default, skip_serializing_if = "Option::is_none")] + pub worker_state: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub workspace_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub working_directory: Option, @@ -325,6 +327,8 @@ pub struct WorkerDetail { pub worker_id: WorkerId, pub status: WorkerStatus, #[serde(default, skip_serializing_if = "Option::is_none")] + pub worker_state: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] pub workspace_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub working_directory: Option, @@ -341,6 +345,8 @@ pub struct WorkerDetail { pub struct WorkerLifecycleAck { pub worker_ref: WorkerRef, pub status: WorkerStatus, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub worker_state: Option, } #[cfg(test)] diff --git a/crates/worker-runtime/src/execution.rs b/crates/worker-runtime/src/execution.rs index 47660977..3c75dc2a 100644 --- a/crates/worker-runtime/src/execution.rs +++ b/crates/worker-runtime/src/execution.rs @@ -15,18 +15,6 @@ use std::fmt; use std::sync::Arc; use workdir::WorkdirSessionHandle; -/// Current execution-side run state for a Worker. -#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum WorkerExecutionRunState { - #[default] - Stopped, - Idle, - Busy, - Rejected, - Errored, -} - /// Execution operation that produced a result. #[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] #[serde(rename_all = "snake_case")] @@ -41,14 +29,12 @@ pub enum WorkerExecutionOperation { Cancel, } -/// Evidence that a user input reached the durable Worker session boundary. -/// -/// This is intentionally distinct from accepting a method on the Worker's -/// in-memory channel. For Flow submissions, the committed UserInput entry also -/// carries the initial Flow runtime-state extension. +/// Evidence that a Submit request reached the durable Worker session boundary. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub struct WorkerInputCommitAck { +pub struct WorkerSubmissionAck { + pub submission_request_id: String, pub submission_id: String, + pub disposition: protocol::SubmissionDisposition, } /// Typed execution result class. Results are transient operation outcomes and @@ -57,11 +43,12 @@ pub struct WorkerInputCommitAck { pub struct WorkerExecutionResult { pub operation: WorkerExecutionOperation, pub outcome: WorkerExecutionOutcome, - pub run_state: WorkerExecutionRunState, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub worker_state: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub message: Option, #[serde(default, skip_serializing_if = "Option::is_none")] - pub input_commit: Option, + pub submission: Option, } /// Backend result class for a Worker execution operation. @@ -76,31 +63,36 @@ pub enum WorkerExecutionOutcome { } impl WorkerExecutionResult { - pub fn accepted( - operation: WorkerExecutionOperation, - run_state: WorkerExecutionRunState, - ) -> Self { + pub fn accepted(operation: WorkerExecutionOperation) -> Self { Self { operation, outcome: WorkerExecutionOutcome::Accepted, - run_state, + worker_state: None, message: None, - input_commit: None, + submission: None, } } - pub fn accepted_input_committed( + pub fn with_worker_state(mut self, worker_state: protocol::WorkerStateSnapshot) -> Self { + self.worker_state = Some(worker_state); + self + } + + pub fn accepted_submission( operation: WorkerExecutionOperation, - run_state: WorkerExecutionRunState, + submission_request_id: impl Into, submission_id: impl Into, + disposition: protocol::SubmissionDisposition, ) -> Self { Self { operation, outcome: WorkerExecutionOutcome::Accepted, - run_state, + worker_state: None, message: None, - input_commit: Some(WorkerInputCommitAck { + submission: Some(WorkerSubmissionAck { + submission_request_id: submission_request_id.into(), submission_id: submission_id.into(), + disposition, }), } } @@ -109,9 +101,9 @@ impl WorkerExecutionResult { Self { operation, outcome: WorkerExecutionOutcome::Busy, - run_state: WorkerExecutionRunState::Busy, + worker_state: None, message: Some(message.into()), - input_commit: None, + submission: None, } } @@ -119,9 +111,9 @@ impl WorkerExecutionResult { Self { operation, outcome: WorkerExecutionOutcome::Rejected, - run_state: WorkerExecutionRunState::Stopped, + worker_state: None, message: Some(message.into()), - input_commit: None, + submission: None, } } @@ -129,9 +121,9 @@ impl WorkerExecutionResult { Self { operation, outcome: WorkerExecutionOutcome::Errored, - run_state: WorkerExecutionRunState::Errored, + worker_state: None, message: Some(message.into()), - input_commit: None, + submission: None, } } @@ -139,9 +131,9 @@ impl WorkerExecutionResult { Self { operation, outcome: WorkerExecutionOutcome::Unsupported, - run_state: WorkerExecutionRunState::Stopped, + worker_state: None, message: Some(message.into()), - input_commit: None, + submission: None, } } @@ -278,7 +270,6 @@ pub struct WorkerExecutionRestoreRequest { pub enum WorkerExecutionSpawnResult { Connected { handle: WorkerExecutionHandle, - run_state: WorkerExecutionRunState, working_directory: Option, }, Rejected(WorkerExecutionResult), @@ -288,12 +279,10 @@ pub enum WorkerExecutionSpawnResult { impl WorkerExecutionSpawnResult { pub fn connected( handle: WorkerExecutionHandle, - run_state: WorkerExecutionRunState, working_directory: Option, ) -> Self { Self::Connected { handle, - run_state, working_directory, } } @@ -618,14 +607,16 @@ mod tests { use super::*; #[test] - fn input_commit_ack_survives_json_round_trip() { - let result = WorkerExecutionResult::accepted_input_committed( + fn submission_ack_survives_json_round_trip() { + let result = WorkerExecutionResult::accepted_submission( WorkerExecutionOperation::Input, - WorkerExecutionRunState::Busy, + "request-1", "submission-1", + protocol::SubmissionDisposition::Started, ); let json = serde_json::to_string(&result).unwrap(); + assert!(json.contains("\"submission_request_id\":\"request-1\"")); assert!(json.contains("\"submission_id\":\"submission-1\"")); assert_eq!( serde_json::from_str::(&json).unwrap(), diff --git a/crates/worker-runtime/src/http_server.rs b/crates/worker-runtime/src/http_server.rs index 2217e973..a35eaab0 100644 --- a/crates/worker-runtime/src/http_server.rs +++ b/crates/worker-runtime/src/http_server.rs @@ -33,7 +33,7 @@ use axum::extract::rejection::{JsonRejection, QueryRejection}; #[cfg(feature = "ws-server")] use axum::extract::ws::{Message as WsMessage, WebSocket, WebSocketUpgrade}; use axum::extract::{DefaultBodyLimit, Extension, Path, Query, State}; -use axum::http::{Method, Request, StatusCode, header}; +use axum::http::{HeaderMap, Method, Request, StatusCode, header}; use axum::middleware::{self, Next}; use axum::response::{IntoResponse, Response}; use axum::routing::{delete, get, post}; @@ -66,6 +66,11 @@ use workdir::{ }; const DEFAULT_RUNTIME_HTTP_PORT: u16 = 38800; +pub const RUNTIME_HTTP_PROTOCOL_MIN_VERSION: u32 = 1; +pub const RUNTIME_HTTP_PROTOCOL_MAX_VERSION: u32 = 1; +pub const RUNTIME_HTTP_PROTOCOL_VERSION: u32 = RUNTIME_HTTP_PROTOCOL_MAX_VERSION; +pub const RUNTIME_PING_PERMISSION: &str = "runtime:ping"; +pub const RUNTIME_WORKSPACE_SCOPE_HEADER: &str = "x-yoi-workspace-id"; fn default_runtime_http_bind_addr() -> SocketAddr { SocketAddr::from(([127, 0, 0, 1], DEFAULT_RUNTIME_HTTP_PORT)) @@ -187,6 +192,7 @@ fn runtime_http_router_with_optional_auth( }; let router = Router::new() + .route("/v1/ping", get(get_runtime_ping)) .route("/v1/runtime", get(get_runtime)) .route( "/v1/config-bundles", @@ -340,6 +346,14 @@ enum RuntimeHttpWorkerStatusFilter { Stopped, } +/// `GET /v1/ping` response. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RuntimeHttpPingResponse { + pub runtime_id: String, + pub protocol_version: u32, +} + /// `GET /v1/workers` response. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct RuntimeHttpWorkersResponse { @@ -461,6 +475,48 @@ struct RuntimeWorkerEventsWsQuery { type RestResult = Result, RuntimeHttpRestError>; +async fn get_runtime_ping( + State(state): State, + Extension(auth): Extension, + headers: HeaderMap, +) -> RestResult { + let requested_workspace_id = headers + .get(RUNTIME_WORKSPACE_SCOPE_HEADER) + .and_then(|value| value.to_str().ok()) + .map(str::trim) + .filter(|value| !value.is_empty()) + .ok_or_else(|| { + RuntimeHttpRestError::new( + StatusCode::FORBIDDEN, + "runtime_ping_workspace_scope_required", + "Runtime ping requires the target Workspace scope", + ) + })?; + if requested_workspace_id != auth.workspace_id { + return Err(RuntimeHttpRestError::new( + StatusCode::FORBIDDEN, + "runtime_ping_workspace_scope_mismatch", + "Runtime ping Workspace scope does not match the authenticated capability", + )); + } + let runtime_id = state + .auth + .as_ref() + .map(|config| config.runtime_id.trim()) + .filter(|runtime_id| !runtime_id.is_empty()) + .ok_or_else(|| { + RuntimeHttpRestError::new( + StatusCode::SERVICE_UNAVAILABLE, + "runtime_ping_identity_unavailable", + "Runtime ping identity is not configured", + ) + })?; + Ok(Json(RuntimeHttpPingResponse { + runtime_id: runtime_id.to_string(), + protocol_version: RUNTIME_HTTP_PROTOCOL_VERSION, + })) +} + async fn get_runtime( State(state): State, ) -> RestResult { @@ -718,8 +774,7 @@ async fn run_workdir_session_operation( .ok_or_else(RuntimeHttpWorkdirError::not_found)?; record.session.clone() }; - let applied = workdir::apply_delegation_chain(source, request.delegations).await?; - let session = applied.scoped_session.as_ref(); + let session = source.as_ref(); let operation = request.operation; let result = match operation { @@ -1184,10 +1239,12 @@ async fn worker_protocol_ws( auth: Option>, Path(worker_id): Path, Query(query): Query, + headers: HeaderMap, ws: WebSocketUpgrade, ) -> Result { let worker_ref = worker_ref_for(&state.runtime, worker_id)?; let scope = auth_workspace_scope(&state, auth.as_ref())?; + let input_source = authenticated_protocol_input_source(&headers)?; match scope.as_ref() { Some(scope) => state .runtime @@ -1198,17 +1255,89 @@ async fn worker_protocol_ws( .map_err(RuntimeHttpRestError::runtime)?; Ok(ws .on_upgrade(move |socket| { - worker_protocol_ws_session(state.runtime, scope, worker_ref, query, socket) + worker_protocol_ws_session( + state.runtime, + scope, + worker_ref, + query, + input_source, + socket, + ) }) .into_response()) } +#[cfg(feature = "ws-server")] +fn authenticated_protocol_input_source( + headers: &HeaderMap, +) -> Result, RuntimeHttpRestError> { + let Some(value) = headers.get(protocol::AUTHENTICATED_ACCOUNT_ID_HEADER) else { + return Ok(None); + }; + let account_id = value.to_str().map_err(|_| { + RuntimeHttpRestError::new( + StatusCode::BAD_REQUEST, + "authenticated_input_source_invalid", + "authenticated Worker input source is invalid", + ) + })?; + if account_id.trim().is_empty() || account_id.len() > 128 { + return Err(RuntimeHttpRestError::new( + StatusCode::BAD_REQUEST, + "authenticated_input_source_invalid", + "authenticated Worker input source is invalid", + )); + } + Ok(Some(protocol::AuthenticatedInputSource::Account { + account_id: account_id.to_owned(), + })) +} + +#[cfg(feature = "ws-server")] +fn authorize_runtime_protocol_method( + method: protocol::Method, + transport_source: Option<&protocol::AuthenticatedInputSource>, +) -> protocol::Method { + match method { + protocol::Method::SubmitTracked { + submission_request_id, + input, + .. + } => protocol::Method::SubmitTracked { + source: transport_source.cloned().unwrap_or_else(|| { + protocol::AuthenticatedInputSource::Backend { + operation_id: submission_request_id.clone(), + } + }), + submission_request_id, + input, + }, + protocol::Method::NotifyTracked { + notification_request_id, + message, + auto_run, + .. + } => protocol::Method::NotifyTracked { + source: transport_source.cloned().unwrap_or_else(|| { + protocol::AuthenticatedInputSource::Backend { + operation_id: notification_request_id.clone(), + } + }), + notification_request_id, + message, + auto_run, + }, + other => other, + } +} + #[cfg(feature = "ws-server")] async fn worker_protocol_ws_session( runtime: Runtime, scope: Option, worker_ref: WorkerRef, query: RuntimeWorkerEventsWsQuery, + input_source: Option, mut socket: WebSocket, ) { let mut cursor = match query.cursor.as_deref() { @@ -1291,6 +1420,8 @@ async fn worker_protocol_ws_session( match inbound { Some(Ok(WsMessage::Text(text))) => match decode_method(&text) { Ok(method) => { + let method = + authorize_runtime_protocol_method(method, input_source.as_ref()); let result = match scope.as_ref() { Some(scope) => { runtime.send_protocol_method_scoped(scope, &worker_ref, method) @@ -1767,6 +1898,9 @@ fn auth_workspace_scope( } fn required_runtime_permission(method: &Method, path: &str) -> Option<&'static str> { + if path == "/v1/ping" && *method == Method::GET { + return Some(RUNTIME_PING_PERMISSION); + } if path == "/v1/runtime" { return None; } @@ -2071,8 +2205,8 @@ mod tests { }; use crate::execution::{ WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation, - WorkerExecutionRestoreRequest, WorkerExecutionResult, WorkerExecutionRunState, - WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult, + WorkerExecutionRestoreRequest, WorkerExecutionResult, WorkerExecutionSpawnRequest, + WorkerExecutionSpawnResult, }; use crate::management::RuntimeOptions; use axum::body::to_bytes; @@ -2080,10 +2214,143 @@ mod tests { use manifest::{Scope, SharedScope}; use tower::ServiceExt; use workdir::{ - GrepOutputMode, GrepRequest, LocalWorkdirSession, ReadRequest, StatRequest, Workdir, - WorkdirPath, WorkdirSessionCapabilities, + GrepOutputMode, GrepRequest, LocalWorkdirSession, StatRequest, Workdir, WorkdirPath, + WorkdirSessionCapabilities, }; + #[tokio::test] + async fn ping_requires_scoped_permission_and_returns_versioned_identity() { + let runtime = Runtime::new_memory(); + let (auth, signer) = auth_config_and_signer(); + let app = runtime_http_router_with_auth(runtime, None, auth); + let token = + token_for_workspace_with_permissions(&signer, "workspace-a", [RUNTIME_PING_PERMISSION]); + let request = Request::builder() + .method(Method::GET) + .uri("/v1/ping") + .header(header::AUTHORIZATION, format!("Bearer {token}")) + .header(RUNTIME_WORKSPACE_SCOPE_HEADER, "workspace-a") + .body(Body::empty()) + .unwrap(); + + let response = app.clone().oneshot(request).await.unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + assert_eq!( + serde_json::from_slice::(&body).unwrap(), + RuntimeHttpPingResponse { + runtime_id: "runtime-test".to_string(), + protocol_version: RUNTIME_HTTP_PROTOCOL_VERSION, + } + ); + + let wrong_scope_token = + token_for_workspace_with_permissions(&signer, "workspace-a", [RUNTIME_PING_PERMISSION]); + let wrong_scope_request = Request::builder() + .method(Method::GET) + .uri("/v1/ping") + .header(header::AUTHORIZATION, format!("Bearer {wrong_scope_token}")) + .header(RUNTIME_WORKSPACE_SCOPE_HEADER, "workspace-b") + .body(Body::empty()) + .unwrap(); + assert_eq!( + app.oneshot(wrong_scope_request).await.unwrap().status(), + StatusCode::FORBIDDEN + ); + } + + #[tokio::test] + async fn ping_rejects_token_without_ping_permission() { + let runtime = Runtime::new_memory(); + let (auth, signer) = auth_config_and_signer(); + let app = runtime_http_router_with_auth(runtime, None, auth); + let missing_credential = Request::builder() + .method(Method::GET) + .uri("/v1/ping") + .header(RUNTIME_WORKSPACE_SCOPE_HEADER, "workspace-a") + .body(Body::empty()) + .unwrap(); + assert_eq!( + app.clone() + .oneshot(missing_credential) + .await + .unwrap() + .status(), + StatusCode::UNAUTHORIZED + ); + + let token = token_for_workspace_with_permissions(&signer, "workspace-a", ["workers:read"]); + let request = Request::builder() + .method(Method::GET) + .uri("/v1/ping") + .header(header::AUTHORIZATION, format!("Bearer {token}")) + .header(RUNTIME_WORKSPACE_SCOPE_HEADER, "workspace-a") + .body(Body::empty()) + .unwrap(); + + assert_eq!( + app.oneshot(request).await.unwrap().status(), + StatusCode::FORBIDDEN + ); + } + + #[test] + fn runtime_protocol_replaces_serialized_tracked_source() { + let wire = serde_json::to_string(&protocol::Method::SubmitTracked { + submission_request_id: "request-1".into(), + input: vec![protocol::Segment::text("hello")], + source: protocol::AuthenticatedInputSource::Account { + account_id: "forged".into(), + }, + }) + .unwrap(); + let decoded: protocol::Method = serde_json::from_str(&wire).unwrap(); + assert!(matches!( + decoded, + protocol::Method::SubmitTracked { + source: protocol::AuthenticatedInputSource::UntrustedWire, + .. + } + )); + assert!(matches!( + authorize_runtime_protocol_method(decoded, None), + protocol::Method::SubmitTracked { + source: protocol::AuthenticatedInputSource::Backend { operation_id }, + .. + } if operation_id == "request-1" + )); + } + + #[test] + fn runtime_protocol_uses_transport_authenticated_account_source() { + let mut headers = HeaderMap::new(); + headers.insert( + protocol::AUTHENTICATED_ACCOUNT_ID_HEADER, + "account-1".parse().unwrap(), + ); + let source = authenticated_protocol_input_source(&headers) + .unwrap() + .expect("account source header must resolve"); + let wire = serde_json::to_string(&protocol::Method::NotifyTracked { + notification_request_id: "notification-1".into(), + message: "hello".into(), + auto_run: true, + source: protocol::AuthenticatedInputSource::Account { + account_id: "forged".into(), + }, + }) + .unwrap(); + let decoded: protocol::Method = serde_json::from_str(&wire).unwrap(); + + assert!(matches!( + authorize_runtime_protocol_method(decoded, Some(&source)), + protocol::Method::NotifyTracked { + source: protocol::AuthenticatedInputSource::Account { account_id }, + .. + } if account_id == "account-1" + )); + } + #[test] fn attachment_routes_require_worker_input_permission() { assert_eq!( @@ -2502,16 +2769,6 @@ mod tests { async fn workdir_session_operations_enforce_owner_and_close_terminally() { let temp = tempfile::tempdir().expect("tempdir"); std::fs::write(temp.path().join("hello.txt"), "hello").expect("write fixture"); - #[cfg(unix)] - { - use std::os::unix::fs::symlink; - std::fs::create_dir(temp.path().join("granted")).expect("granted directory"); - std::fs::write(temp.path().join("granted/visible"), "visible") - .expect("visible fixture"); - std::fs::create_dir(temp.path().join("secret")).expect("secret directory"); - std::fs::write(temp.path().join("secret/key"), "hidden").expect("secret fixture"); - symlink("../secret/key", temp.path().join("granted/link")).expect("symlink fixture"); - } let scope = SharedScope::new(Scope::writable(temp.path()).expect("scope")); let session: WorkdirSessionHandle = Arc::new(LocalWorkdirSession::materialized_bound( Workdir::new("wd-1"), @@ -2545,7 +2802,6 @@ mod tests { expires_at: u64::MAX, }; let operation = WorkdirSessionOperationRequest { - delegations: Vec::new(), operation: WorkdirSessionOperation::Stat(StatRequest { path: WorkdirPath::new("hello.txt").expect("logical path"), }), @@ -2562,7 +2818,6 @@ mod tests { assert!(matches!(result, WorkdirSessionOperationResult::Stat(_))); let grep = WorkdirSessionOperationRequest { - delegations: Vec::new(), operation: WorkdirSessionOperation::Grep(GrepRequest { pattern: "hello".into(), path: WorkdirPath::new("hello.txt").unwrap(), @@ -2585,78 +2840,7 @@ mod tests { ) .await .expect("grep direct file through provider operation"); - match result { - WorkdirSessionOperationResult::Grep(result) => { - assert_eq!(result.match_count, 1); - assert_eq!(result.matched_files, 1); - assert!(result.output.starts_with("hello.txt\n")); - assert!(result.output.contains("> 1 │ hello")); - } - other => panic!("unexpected workdir grep result: {other:?}"), - } - - #[cfg(unix)] - { - let delegated_visible = WorkdirSessionOperationRequest { - delegations: vec![workdir::WorkdirDelegationRequest { - rules: vec![workdir::WorkdirDelegationRule { - target: WorkdirPath::new("granted").unwrap(), - permission: workdir::WorkdirDelegationPermission::Read, - recursive: true, - }], - cwd: WorkdirPath::new("granted").unwrap(), - }], - operation: WorkdirSessionOperation::Read(ReadRequest { - path: WorkdirPath::new("visible").unwrap(), - offset: 0, - limit: 20, - max_bytes: 1024, - }), - }; - let visible = run_workdir_session_operation( - State(state.clone()), - Path("session-1".to_string()), - Some(Extension(auth.clone())), - Ok(Json(delegated_visible)), - ) - .await - .expect("non-root delegated cwd should resolve once") - .0; - assert!(matches!( - visible, - WorkdirSessionOperationResult::Read(result) if result.bytes == b"visible" - )); - - let delegated_read = WorkdirSessionOperationRequest { - delegations: vec![workdir::WorkdirDelegationRequest { - rules: vec![workdir::WorkdirDelegationRule { - target: WorkdirPath::new("granted").unwrap(), - permission: workdir::WorkdirDelegationPermission::Read, - recursive: true, - }], - cwd: WorkdirPath::new("granted").unwrap(), - }], - operation: WorkdirSessionOperation::Read(ReadRequest { - path: WorkdirPath::new("link").unwrap(), - offset: 0, - limit: 20, - max_bytes: 1024, - }), - }; - let error = run_workdir_session_operation( - State(state.clone()), - Path("session-1".to_string()), - Some(Extension(auth.clone())), - Ok(Json(delegated_read)), - ) - .await - .expect_err("provider must reject delegated symlink escape"); - assert_ne!(error.status, StatusCode::OK); - assert_eq!( - std::fs::read_to_string(temp.path().join("secret/key")).unwrap(), - "hidden" - ); - } + assert!(matches!(result, WorkdirSessionOperationResult::Grep(_))); let wrong_owner = RuntimeAuthContext { workspace_id: "workspace-b".to_string(), @@ -2711,7 +2895,6 @@ mod tests { fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult { WorkerExecutionSpawnResult::Connected { handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()), - run_state: WorkerExecutionRunState::Idle, working_directory: request .working_directory .as_ref() @@ -2725,7 +2908,6 @@ mod tests { ) -> WorkerExecutionSpawnResult { WorkerExecutionSpawnResult::Connected { handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()), - run_state: WorkerExecutionRunState::Idle, working_directory: request.previous_working_directory, } } @@ -2735,25 +2917,20 @@ mod tests { _handle: &WorkerExecutionHandle, input: WorkerInput, ) -> WorkerExecutionResult { - if let Some(submission_id) = input.submission_id { - WorkerExecutionResult::accepted_input_committed( + if let Some(submission_id) = input.submission_request_id { + WorkerExecutionResult::accepted_submission( WorkerExecutionOperation::Input, - WorkerExecutionRunState::Idle, + submission_id.clone(), submission_id, + protocol::SubmissionDisposition::Started, ) } else { - WorkerExecutionResult::accepted( - WorkerExecutionOperation::Input, - WorkerExecutionRunState::Idle, - ) + WorkerExecutionResult::accepted(WorkerExecutionOperation::Input) } } fn stop_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult { - WorkerExecutionResult::accepted( - WorkerExecutionOperation::Stop, - WorkerExecutionRunState::Stopped, - ) + WorkerExecutionResult::accepted(WorkerExecutionOperation::Stop) } } @@ -3025,8 +3202,7 @@ mod ws_tests { }; use crate::execution::{ WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation, - WorkerExecutionResult, WorkerExecutionRunState, WorkerExecutionSpawnRequest, - WorkerExecutionSpawnResult, + WorkerExecutionResult, WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult, }; use crate::management::RuntimeOptions; use futures::{SinkExt, StreamExt}; @@ -3046,7 +3222,6 @@ mod ws_tests { fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult { WorkerExecutionSpawnResult::Connected { handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()), - run_state: WorkerExecutionRunState::Idle, working_directory: request .working_directory .as_ref() @@ -3059,29 +3234,49 @@ mod ws_tests { _handle: &WorkerExecutionHandle, input: WorkerInput, ) -> WorkerExecutionResult { - if let Some(submission_id) = input.submission_id { - WorkerExecutionResult::accepted_input_committed( + if let Some(submission_id) = input.submission_request_id { + WorkerExecutionResult::accepted_submission( WorkerExecutionOperation::Input, - WorkerExecutionRunState::Idle, + submission_id.clone(), submission_id, + protocol::SubmissionDisposition::Started, ) } else { - WorkerExecutionResult::accepted( - WorkerExecutionOperation::Input, - WorkerExecutionRunState::Idle, - ) + WorkerExecutionResult::accepted(WorkerExecutionOperation::Input) } } + fn worker_snapshot(&self, handle: &WorkerExecutionHandle) -> Option { + Some(protocol::Event::Snapshot { + session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), + entries: Vec::new(), + }, + greeting: protocol::Greeting { + worker_name: handle.worker_ref().worker_id.to_string(), + cwd: String::new(), + provider: "ws-test".to_string(), + model: "ws-test".to_string(), + scope_summary: "WebSocket test execution snapshot".to_string(), + tools: Vec::new(), + context_window: 0, + context_tokens: 0, + }, + state: protocol::WorkerStateSnapshot::initial(1), + in_flight: protocol::InFlightSnapshot { + blocks: Vec::new(), + commands: Vec::new(), + }, + internal_workers: Vec::new(), + }) + } + fn dispatch_method( &self, _handle: &WorkerExecutionHandle, _method: protocol::Method, ) -> WorkerExecutionResult { - WorkerExecutionResult::accepted( - WorkerExecutionOperation::ProtocolMethod, - WorkerExecutionRunState::Idle, - ) + WorkerExecutionResult::accepted(WorkerExecutionOperation::ProtocolMethod) } } @@ -3292,16 +3487,16 @@ mod ws_tests { runtime .observe_worker_event( &other.worker_ref, - protocol::Event::Status { - status: protocol::WorkerStatus::Running, + protocol::Event::WorkerState { + snapshot: protocol::WorkerStatus::Running.into(), }, ) .unwrap(); runtime .observe_worker_event( &worker_ref, - protocol::Event::Status { - status: protocol::WorkerStatus::Running, + protocol::Event::WorkerState { + snapshot: protocol::WorkerStatus::Running.into(), }, ) .unwrap(); @@ -3316,7 +3511,16 @@ mod ws_tests { .. }) if delivered_subscription_id == subscription_id && worker.worker_id.as_str() == worker_ref.worker_id.to_string() - && worker.state == protocol::subscription::SubscriptionWorkerState::Running + && worker.state == protocol::subscription::SubscriptionWorkerState::Idle + && matches!( + worker.worker_state, + Some(protocol::WorkerStateSnapshot { + state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Running + )), + .. + }) + ) )); let unsubscribe_request_id = diff --git a/crates/worker-runtime/src/interaction.rs b/crates/worker-runtime/src/interaction.rs index d76c675d..52e0168e 100644 --- a/crates/worker-runtime/src/interaction.rs +++ b/crates/worker-runtime/src/interaction.rs @@ -25,10 +25,10 @@ impl WorkerInputKind { pub struct WorkerInput { pub kind: WorkerInputKind, pub content: String, - /// Runtime-generated correlation id. This is never accepted from public - /// JSON input and is consumed only by the execution backend. - #[serde(skip)] - pub submission_id: Option, + /// Authenticated client-generated idempotency key. Runtime generates one + /// only for trusted internal callers that omit it. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub submission_request_id: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub segments: Option>, } @@ -38,7 +38,7 @@ impl WorkerInput { Self { kind: WorkerInputKind::User, content: content.into(), - submission_id: None, + submission_request_id: None, segments: None, } } @@ -47,7 +47,7 @@ impl WorkerInput { Self { kind: WorkerInputKind::Notify, content: content.into(), - submission_id: None, + submission_request_id: None, segments: None, } } @@ -57,6 +57,21 @@ impl WorkerInput { mod tests { use super::WorkerInput; + #[test] + fn submission_request_id_round_trips_for_authenticated_client_retry() { + let input: WorkerInput = serde_json::from_value(serde_json::json!({ + "kind": "user", + "content": "message", + "submission_request_id": "request-1" + })) + .unwrap(); + assert_eq!(input.submission_request_id.as_deref(), Some("request-1")); + assert_eq!( + serde_json::to_value(input).unwrap()["submission_request_id"], + "request-1" + ); + } + #[test] fn notify_is_an_operation_and_legacy_system_kind_is_rejected() { assert_eq!( @@ -78,4 +93,7 @@ mod tests { pub struct WorkerInteractionAck { pub worker_ref: WorkerRef, pub status: WorkerStatus, + /// Present for User Submit and absent for non-Submit interactions. + #[serde(skip_serializing_if = "Option::is_none")] + pub submission: Option, } diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index fd5a0b16..e44e9f7f 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -13,8 +13,8 @@ use crate::error::RuntimeError; use crate::execution::WorkerExecutionRestoreRequest; use crate::execution::{ WorkerExecutionBackend, WorkerExecutionBackendRef, WorkerExecutionHandle, - WorkerExecutionOperation, WorkerExecutionResult, WorkerExecutionRunState, - WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult, + WorkerExecutionOperation, WorkerExecutionResult, WorkerExecutionSpawnRequest, + WorkerExecutionSpawnResult, }; #[cfg(feature = "fs-store")] use crate::fs_store::{ @@ -700,6 +700,7 @@ impl Runtime { worker_ref: worker_ref.clone(), worker_id: worker_id.clone(), status: WorkerStatus::Stopped, + worker_state: None, workspace_id: scope.map(|scope| scope.workspace_id.clone()), request: durable_request, run_generation: 1, @@ -725,12 +726,11 @@ impl Runtime { }; let spawn_result = backend.spawn_worker(spawn_request); - let (handle, run_state, working_directory) = match spawn_result { + let (handle, working_directory) = match spawn_result { WorkerExecutionSpawnResult::Connected { handle, - run_state, working_directory, - } => (handle, run_state, working_directory), + } => (handle, working_directory), WorkerExecutionSpawnResult::Rejected(result) | WorkerExecutionSpawnResult::Errored(result) => { self.rollback_failed_create(&worker_ref)?; @@ -748,8 +748,12 @@ impl Runtime { let state = self.lock()?; state.worker(&worker_ref)?.request.initial_input.clone() } { - let expected_submission_id = Uuid::now_v7().to_string(); - initial_input.submission_id = Some(expected_submission_id.clone()); + let expected_submission_id = initial_input + .submission_request_id + .clone() + .filter(|request_id| !request_id.trim().is_empty()) + .unwrap_or_else(|| Uuid::now_v7().to_string()); + initial_input.submission_request_id = Some(expected_submission_id.clone()); let dispatch_result = backend.dispatch_input(&handle, initial_input.clone()); if !dispatch_result.is_accepted() { let _ = backend.stop_worker(&handle); @@ -763,9 +767,9 @@ impl Runtime { }); } let has_commit_ack = dispatch_result - .input_commit + .submission .as_ref() - .is_some_and(|ack| ack.submission_id == expected_submission_id); + .is_some_and(|ack| ack.submission_request_id == expected_submission_id); if !has_commit_ack { let _ = backend.stop_worker(&handle); self.rollback_failed_create(&worker_ref)?; @@ -781,11 +785,9 @@ impl Runtime { result, }); } - let initial_run_state = dispatch_result.run_state; let detail = self.commit_created_worker( &worker_ref, handle, - initial_run_state, working_directory, dispatch_result, )?; @@ -795,9 +797,8 @@ impl Runtime { self.commit_created_worker( &worker_ref, handle, - run_state, working_directory, - WorkerExecutionResult::accepted(WorkerExecutionOperation::Spawn, run_state), + WorkerExecutionResult::accepted(WorkerExecutionOperation::Spawn), ) } } @@ -1082,13 +1083,12 @@ impl Runtime { match backend.restore_worker(request) { WorkerExecutionSpawnResult::Connected { handle, - run_state, working_directory, } => { self.commit_restored_worker_execution( worker_ref, handle, - run_state, + WorkerStatus::Idle, working_directory, )?; self.worker_detail(worker_ref) @@ -1146,13 +1146,18 @@ impl Runtime { mut input: WorkerInput, ) -> Result { validate_worker_input(&input)?; - let expected_submission_id = if input.kind == WorkerInputKind::User { - let submission_id = Uuid::now_v7().to_string(); - input.submission_id = Some(submission_id.clone()); - Some(submission_id) - } else { - None - }; + let expected_submission_id = + if matches!(input.kind, WorkerInputKind::User | WorkerInputKind::Notify) { + let submission_id = input + .submission_request_id + .clone() + .filter(|request_id| !request_id.trim().is_empty()) + .unwrap_or_else(|| Uuid::now_v7().to_string()); + input.submission_request_id = Some(submission_id.clone()); + Some(submission_id) + } else { + None + }; self.ensure_worker_execution(worker_ref)?; let (backend, handle) = { let state = self.lock()?; @@ -1191,13 +1196,13 @@ impl Runtime { } if let Some(expected_submission_id) = expected_submission_id && dispatch_result - .input_commit + .submission .as_ref() - .is_none_or(|ack| ack.submission_id != expected_submission_id) + .is_none_or(|ack| ack.submission_request_id != expected_submission_id) { let result = WorkerExecutionResult::rejected( WorkerExecutionOperation::Input, - "execution backend did not acknowledge the committed Runtime submission id", + "execution backend did not acknowledge the committed Runtime submission request id", ); self.record_execution_result(worker_ref, result.clone())?; return Err(RuntimeError::WorkerExecutionRejected { @@ -1209,10 +1214,13 @@ impl Runtime { }); } + let submission = dispatch_result.submission.clone(); let mut state = self.lock()?; state.ensure_running()?; let worker = state.worker_mut(worker_ref)?; - worker.status = worker_status_from_run_state(dispatch_result.run_state); + if let Some(snapshot) = dispatch_result.worker_state.as_ref() { + let _ = worker.apply_worker_state(snapshot); + } let status = worker.status; #[cfg(feature = "ws-server")] if let Some(payload) = input_protocol_event(&input) { @@ -1225,6 +1233,7 @@ impl Runtime { Ok(WorkerInteractionAck { worker_ref: worker_ref.clone(), status, + submission, }) } @@ -1420,7 +1429,7 @@ impl Runtime { let entries = self.worker_completions(worker_ref, kind, &prefix)?; return Ok(vec![Event::Completions { kind, entries }]); } - if matches!(&method, Method::Shutdown) { + if matches!(&method, Method::Shutdown { .. }) { self.stop_worker(worker_ref, Some("worker protocol shutdown".to_string()))?; return Ok(Vec::new()); } @@ -1470,16 +1479,19 @@ impl Runtime { &self, worker_ref: &WorkerRef, handle: WorkerExecutionHandle, - run_state: WorkerExecutionRunState, working_directory: Option, - _result: WorkerExecutionResult, + result: WorkerExecutionResult, ) -> Result { let mut state = self.lock()?; let detail = { let worker = state.worker_mut(worker_ref)?; worker.execution_handle = Some(handle); worker.execution_bound = true; - worker.status = worker_status_from_run_state(run_state); + worker.status = WorkerStatus::Idle; + worker.worker_state = None; + if let Some(snapshot) = result.worker_state.as_ref() { + let _ = worker.apply_worker_state(snapshot); + } worker.restore_intent = restore_intent_for_status(worker.status); worker.working_directory = working_directory; worker.detail() @@ -1507,16 +1519,26 @@ impl Runtime { worker_ref: &WorkerRef, result: WorkerExecutionResult, ) -> Result<(), RuntimeError> { - let mut state = self.lock()?; - if result.is_accepted() { - let status = worker_status_from_run_state(result.run_state); - let worker = state.worker_mut(worker_ref)?; - worker.status = status; - worker.restore_intent = restore_intent_for_status(status); - state.publish_worker_upsert(worker_ref.worker_id)?; - state.persist_runtime_snapshot()?; - state.persist_worker(&worker_ref.worker_id)?; + // Accepted dispatch without a state snapshot is transport evidence only; + // the revisioned protocol stream remains live authority. Test/detached + // backends may return an exact full snapshot as their acknowledgement. + if !result.is_accepted() { + return Ok(()); } + let Some(snapshot) = result.worker_state else { + return Ok(()); + }; + let mut state = self.lock()?; + let worker = state.worker_mut(worker_ref)?; + let applied = worker + .apply_worker_state(&snapshot) + .is_ok_and(|result| matches!(result, protocol::WorkerStateSnapshotApply::Applied)); + if !applied { + return Ok(()); + } + state.publish_worker_upsert(worker_ref.worker_id)?; + state.persist_runtime_snapshot()?; + state.persist_worker(&worker_ref.worker_id)?; Ok(()) } @@ -1604,20 +1626,26 @@ impl Runtime { worker_ref: &WorkerRef, reason: Option, ) -> Result { - let current = { + { let state = self.lock()?; state.ensure_running()?; - state.worker(worker_ref)?.status - }; - if matches!(current, WorkerStatus::Idle | WorkerStatus::Stopped) { - return Ok(WorkerLifecycleAck { - worker_ref: worker_ref.clone(), - status: current, - }); + if state.worker(worker_ref)?.status == WorkerStatus::Stopped { + return Ok(WorkerLifecycleAck { + worker_ref: worker_ref.clone(), + status: WorkerStatus::Stopped, + worker_state: None, + }); + } } self.dispatch_lifecycle_to_backend(worker_ref, WorkerExecutionOperation::Cancel)?; let _ = reason; - self.transition_worker_preserving_execution(worker_ref, WorkerStatus::Idle) + let state = self.lock()?; + let worker = state.worker(worker_ref)?; + Ok(WorkerLifecycleAck { + worker_ref: worker_ref.clone(), + status: worker.status, + worker_state: worker.worker_state.clone(), + }) } /// Delete a non-running Worker through a workspace-scoped Runtime authorization context. @@ -1704,26 +1732,9 @@ impl Runtime { return Ok(snapshot); } } - Ok(protocol::Event::Snapshot { - session: protocol::SessionSnapshot { - entries: Vec::new(), - }, - greeting: protocol::Greeting { - worker_name: worker_ref.worker_id.to_string(), - cwd: String::new(), - provider: "worker-runtime".to_string(), - model: "worker-runtime".to_string(), - scope_summary: "runtime worker observation".to_string(), - tools: Vec::new(), - context_window: 0, - context_tokens: 0, - }, - status: protocol::WorkerStatus::Idle, - in_flight: protocol::InFlightSnapshot { - blocks: Vec::new(), - commands: Vec::new(), - }, - internal_workers: Vec::new(), + Err(RuntimeError::WorkerExecutionUnavailable { + worker_id: worker_ref.worker_id, + message: "authoritative Worker snapshot is unavailable".to_string(), }) } @@ -1762,12 +1773,13 @@ impl Runtime { ) -> Result { let mut state = self.lock()?; state.ensure_worker_ref(worker_ref)?; - let status_changed = state.project_protocol_event_to_status(worker_ref, &payload); + let worker_state_changed = + state.project_protocol_event_to_worker_state(worker_ref, &payload); let activity_changed = state.project_internal_worker_activity(worker_ref, &payload); - if status_changed || activity_changed { + if worker_state_changed || activity_changed { state.publish_worker_upsert(worker_ref.worker_id)?; } - if status_changed { + if worker_state_changed { state.persist_runtime_snapshot()?; state.persist_worker(&worker_ref.worker_id)?; } @@ -1803,26 +1815,6 @@ impl Runtime { Ok(()) } - fn transition_worker_preserving_execution( - &self, - worker_ref: &WorkerRef, - status: WorkerStatus, - ) -> Result { - let mut state = self.lock()?; - state.ensure_running()?; - let worker = state.worker_mut(worker_ref)?; - worker.status = status; - worker.restore_intent = restore_intent_for_status(status); - let status = worker.status; - state.publish_worker_upsert(worker_ref.worker_id)?; - state.persist_runtime_snapshot()?; - state.persist_worker(&worker_ref.worker_id)?; - Ok(WorkerLifecycleAck { - worker_ref: worker_ref.clone(), - status, - }) - } - fn transition_worker( &self, worker_ref: &WorkerRef, @@ -1834,6 +1826,7 @@ impl Runtime { let worker = state.worker_mut(worker_ref)?; worker.status = status; + worker.worker_state = None; worker.restore_intent = restore_intent_for_status(status); worker.execution_handle = None; worker.internal_workers.clear(); @@ -1844,6 +1837,7 @@ impl Runtime { Ok(WorkerLifecycleAck { worker_ref: worker_ref.clone(), status, + worker_state: None, }) } @@ -1956,12 +1950,11 @@ impl Runtime { match backend.restore_worker(request) { WorkerExecutionSpawnResult::Connected { handle, - run_state, working_directory, } => self.commit_restored_worker_execution( &candidate.worker_ref, handle, - run_state, + WorkerStatus::Idle, working_directory, )?, WorkerExecutionSpawnResult::Rejected(result) @@ -1978,7 +1971,7 @@ impl Runtime { &self, worker_ref: &WorkerRef, handle: WorkerExecutionHandle, - run_state: WorkerExecutionRunState, + status: WorkerStatus, working_directory: Option, ) -> Result<(), RuntimeError> { let mut state = self.lock()?; @@ -1987,7 +1980,7 @@ impl Runtime { let worker = state.worker_mut(worker_ref)?; worker.execution_handle = Some(handle); worker.execution_bound = true; - worker.status = worker_status_from_run_state(run_state); + worker.status = status; worker.restore_intent = restore_intent_for_status(worker.status); worker.working_directory = working_directory; } @@ -2269,6 +2262,7 @@ impl RuntimeState { worker_ref: worker.worker_ref, worker_id: worker.worker_id, status: worker.status, + worker_state: None, workspace_id: worker.workspace_id, request: worker.request, run_generation, @@ -2598,6 +2592,7 @@ impl RuntimeState { .get(&worker.worker_id) .copied() .unwrap_or(0), + worker_state: worker.worker_state.clone(), state: subscription_worker_state(worker.status), has_running_internal_workers: worker .internal_workers @@ -2855,7 +2850,7 @@ impl RuntimeState { ) { match event { protocol::Event::Snapshot { - status, + state, internal_workers, .. } => { @@ -2863,7 +2858,7 @@ impl RuntimeState { statuses.insert( worker.session_id.clone(), InternalWorkerActivity { - status: *status, + status: state.catalog_status(), parent_session_id: worker.parent_session_id.clone(), }, ); @@ -2876,26 +2871,17 @@ impl RuntimeState { event, .. } => Self::project_internal_worker_event(statuses, nested_worker, event), - protocol::Event::Status { status } => { - statuses.insert( - worker.session_id.clone(), - InternalWorkerActivity { - status: *status, - parent_session_id: worker.parent_session_id.clone(), + protocol::Event::WorkerState { snapshot } + | protocol::Event::CommandAcknowledged { + acknowledgement: + protocol::WorkerCommandAcknowledgement { + state: snapshot, .. }, - ); - } - protocol::Event::RunEnd { result } => { - let status = match result { - protocol::RunResult::Paused => protocol::WorkerStatus::Paused, - protocol::RunResult::Finished - | protocol::RunResult::LimitReached - | protocol::RunResult::RolledBack => protocol::WorkerStatus::Idle, - }; + } => { statuses.insert( worker.session_id.clone(), InternalWorkerActivity { - status, + status: snapshot.catalog_status(), parent_session_id: worker.parent_session_id.clone(), }, ); @@ -2942,7 +2928,7 @@ impl RuntimeState { Self::update_internal_worker_activity(&mut worker.internal_workers, event) } - fn project_protocol_event_to_status( + fn project_protocol_event_to_worker_state( &mut self, worker_ref: &WorkerRef, event: &protocol::Event, @@ -2950,38 +2936,26 @@ impl RuntimeState { let Some(worker) = self.workers.get_mut(&worker_ref.worker_id) else { return false; }; - let next_status = match event { - protocol::Event::Status { - status: protocol::WorkerStatus::Running, - } => Some(WorkerStatus::Running), - protocol::Event::Status { - status: protocol::WorkerStatus::Idle, - } => Some(WorkerStatus::Idle), - protocol::Event::Status { - status: protocol::WorkerStatus::Paused, - } => Some(WorkerStatus::Paused), - protocol::Event::Snapshot { status, .. } => match status { - protocol::WorkerStatus::Running => Some(WorkerStatus::Running), - protocol::WorkerStatus::Idle => Some(WorkerStatus::Idle), - protocol::WorkerStatus::Paused => Some(WorkerStatus::Paused), - protocol::WorkerStatus::Stopped => Some(WorkerStatus::Stopped), - }, - protocol::Event::RunEnd { result } => match result { - protocol::RunResult::Finished | protocol::RunResult::RolledBack => { - Some(WorkerStatus::Idle) - } - protocol::RunResult::Paused => Some(WorkerStatus::Paused), - protocol::RunResult::LimitReached => Some(WorkerStatus::Idle), - }, - _ => None, + let incoming = match event { + protocol::Event::WorkerState { snapshot } + | protocol::Event::Snapshot { + state: snapshot, .. + } + | protocol::Event::CommandAcknowledged { + acknowledgement: + protocol::WorkerCommandAcknowledgement { + state: snapshot, .. + }, + } => snapshot, + _ => return false, }; - if let Some(next_status) = next_status { - let changed = worker.status != next_status; - worker.status = next_status; - worker.restore_intent = restore_intent_for_status(next_status); - changed - } else { - false + match worker.apply_worker_state(incoming) { + Ok(protocol::WorkerStateSnapshotApply::Applied) => true, + Ok( + protocol::WorkerStateSnapshotApply::Duplicate + | protocol::WorkerStateSnapshotApply::Stale, + ) + | Err(_) => false, } } } @@ -2997,6 +2971,7 @@ struct WorkerRecord { worker_ref: WorkerRef, worker_id: WorkerId, status: WorkerStatus, + worker_state: Option, workspace_id: Option, request: CreateWorkerRequest, run_generation: u64, @@ -3008,6 +2983,19 @@ struct WorkerRecord { } impl WorkerRecord { + fn apply_worker_state( + &mut self, + incoming: &protocol::WorkerStateSnapshot, + ) -> Result { + match self.worker_state.as_mut() { + Some(current) => protocol::apply_worker_state_snapshot(current, incoming), + None => { + self.worker_state = Some(incoming.clone()); + Ok(protocol::WorkerStateSnapshotApply::Applied) + } + } + } + fn belongs_to_workspace(&self, workspace_id: &str) -> bool { self.workspace_id.as_deref() == Some(workspace_id) } @@ -3017,6 +3005,7 @@ impl WorkerRecord { worker_ref: self.worker_ref.clone(), worker_id: self.worker_id, status: self.status, + worker_state: self.worker_state.clone(), workspace_id: self.workspace_id.clone(), working_directory: self.working_directory.clone(), profile: self.request.profile.clone(), @@ -3031,6 +3020,7 @@ impl WorkerRecord { worker_ref: self.worker_ref.clone(), worker_id: self.worker_id, status: self.status, + worker_state: self.worker_state.clone(), workspace_id: self.workspace_id.clone(), working_directory: self.working_directory.clone(), profile: self.request.profile.clone(), @@ -3069,16 +3059,6 @@ fn restore_intent_for_status(status: WorkerStatus) -> WorkerRestoreIntent { } } -fn worker_status_from_run_state(run_state: WorkerExecutionRunState) -> WorkerStatus { - match run_state { - WorkerExecutionRunState::Idle => WorkerStatus::Idle, - WorkerExecutionRunState::Busy => WorkerStatus::Running, - WorkerExecutionRunState::Stopped - | WorkerExecutionRunState::Rejected - | WorkerExecutionRunState::Errored => WorkerStatus::Stopped, - } -} - fn repository_resource_error(error: BackendResourceError) -> RuntimeError { let (code, message) = match error { BackendResourceError::Expired => ( @@ -3250,17 +3230,9 @@ fn validate_worker_input(input: &WorkerInput) -> Result<(), RuntimeError> { #[cfg(feature = "ws-server")] fn input_protocol_event(input: &WorkerInput) -> Option { match input.kind { - WorkerInputKind::User => Some(protocol::Event::UserMessage { - segments: input.segments.clone().unwrap_or_else(|| { - vec![protocol::Segment::Text { - content: input.content.clone(), - }] - }), - }), - // The committed `SystemItem::Notification` is the sole agent-visible - // and Console-visible authority for Notify. A synthetic observation - // here would display the same notification twice. - WorkerInputKind::Notify => None, + // Submit is projected only after the Worker commits UserInput. Queued + // payloads must never become model- or client-visible history early. + WorkerInputKind::User | WorkerInputKind::Notify => None, WorkerInputKind::Compact | WorkerInputKind::ListRewindTargets | WorkerInputKind::RegisterPeer => Some(protocol::Event::SystemItem { @@ -3300,7 +3272,7 @@ mod tests { }; use crate::execution::{ WorkerExecutionBackend, WorkerExecutionContext, WorkerExecutionHandle, - WorkerExecutionRestoreRequest, WorkerExecutionRunState, + WorkerExecutionRestoreRequest, }; use crate::working_directory::WorkingDirectoryDiagnostic; use async_trait::async_trait; @@ -3309,6 +3281,14 @@ mod tests { use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, Mutex}; + fn test_command() -> protocol::WorkerCommandEnvelope { + protocol::WorkerCommandEnvelope { + command_id: 1, + expected_execution_generation: 1, + expected_worker_state_revision: 0, + } + } + #[test] fn repository_resource_failures_keep_typed_credential_diagnostics() { let cases = [ @@ -3355,7 +3335,9 @@ mod tests { protocol::Event::InternalWorker { worker, revision: 1, - event: Box::new(protocol::Event::Status { status }), + event: Box::new(protocol::Event::WorkerState { + snapshot: status.into(), + }), } } @@ -3435,6 +3417,7 @@ mod tests { ); let snapshot = protocol::Event::Snapshot { session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, greeting: protocol::Greeting { @@ -3447,7 +3430,7 @@ mod tests { context_window: 0, context_tokens: 0, }, - status: protocol::WorkerStatus::Idle, + state: protocol::WorkerStatus::Idle.into(), in_flight: protocol::InFlightSnapshot::default(), internal_workers: Vec::new(), }; @@ -3475,7 +3458,7 @@ mod tests { let input = WorkerInput { kind: WorkerInputKind::User, content: String::new(), - submission_id: None, + submission_request_id: None, segments: Some(vec![protocol::Segment::Flow { selector: "builtin:coder-review".to_string(), }]), @@ -3488,7 +3471,7 @@ mod tests { let input = WorkerInput { kind: WorkerInputKind::User, content: String::new(), - submission_id: None, + submission_request_id: None, segments: Some(Vec::new()), }; assert!(matches!( @@ -3503,7 +3486,7 @@ mod tests { request.initial_input = Some(WorkerInput { kind: WorkerInputKind::User, content: String::new(), - submission_id: None, + submission_request_id: None, segments: Some(vec![protocol::Segment::Flow { selector: "builtin:coder-review".to_string(), }]), @@ -3962,7 +3945,6 @@ mod tests { .insert(request.worker_ref.worker_id.clone(), request.context); WorkerExecutionSpawnResult::Connected { handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()), - run_state: WorkerExecutionRunState::Idle, working_directory: request .working_directory .as_ref() @@ -3992,7 +3974,6 @@ mod tests { .insert(request.worker_ref.worker_id.clone(), request.context); WorkerExecutionSpawnResult::Connected { handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()), - run_state: WorkerExecutionRunState::Idle, working_directory: request .working_directory .as_ref() @@ -4005,7 +3986,7 @@ mod tests { _handle: &WorkerExecutionHandle, input: WorkerInput, ) -> WorkerExecutionResult { - let submission_id = input.submission_id.clone(); + let submission_id = input.submission_request_id.clone(); self.dispatched_inputs.lock().unwrap().push(input); let mut result = self .dispatch_result @@ -4013,35 +3994,30 @@ mod tests { .unwrap() .clone() .unwrap_or_else(|| { - WorkerExecutionResult::accepted_input_committed( + WorkerExecutionResult::accepted_submission( WorkerExecutionOperation::Input, - WorkerExecutionRunState::Idle, + "request-test", "test-submission", + protocol::SubmissionDisposition::Started, ) }); if !self .preserve_commit_ack_submission_id .load(Ordering::SeqCst) && let (Some(ack), Some(submission_id)) = - (result.input_commit.as_mut(), submission_id) + (result.submission.as_mut(), submission_id) { - ack.submission_id = submission_id; + ack.submission_request_id = submission_id; } result } fn stop_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult { - WorkerExecutionResult::accepted( - WorkerExecutionOperation::Stop, - WorkerExecutionRunState::Stopped, - ) + WorkerExecutionResult::accepted(WorkerExecutionOperation::Stop) } fn cancel_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult { - WorkerExecutionResult::accepted( - WorkerExecutionOperation::Cancel, - WorkerExecutionRunState::Stopped, - ) + WorkerExecutionResult::accepted(WorkerExecutionOperation::Cancel) } #[cfg(feature = "ws-server")] @@ -4367,7 +4343,9 @@ mod tests { .send_protocol_method_scoped( &scope("workspace-a", "server-a"), &workspace_b.worker_ref, - Method::Shutdown, + Method::Shutdown { + command: test_command(), + }, ) .unwrap_err(); assert!(matches!( @@ -4715,12 +4693,13 @@ mod tests { } #[test] - fn create_worker_uses_committed_input_ack_run_state() { + fn create_worker_does_not_infer_state_from_started_submission_ack() { let (runtime, backend) = runtime_and_backend(); - backend.set_dispatch_result(WorkerExecutionResult::accepted_input_committed( + backend.set_dispatch_result(WorkerExecutionResult::accepted_submission( WorkerExecutionOperation::Input, - WorkerExecutionRunState::Idle, + "request-test", "test-submission", + protocol::SubmissionDisposition::Started, )); let mut request = task_request("committed initial input is already idle"); request.initial_input = Some(WorkerInput::user("start the ticket")); @@ -4728,16 +4707,79 @@ mod tests { let detail = runtime.create_worker(request).unwrap(); assert_eq!(detail.status, WorkerStatus::Idle); + assert_eq!(detail.worker_state, None); } #[test] - fn create_worker_rejects_mismatched_input_commit_acknowledgement() { + fn runtime_applies_only_newer_worker_state_snapshots() { + let (runtime, _) = runtime_and_backend(); + let detail = runtime + .create_worker(task_request("state ordering")) + .unwrap(); + let running = protocol::WorkerStateSnapshot { + execution_generation: 7, + revision: 3, + state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Running, + )), + last_command_id: 2, + }; + assert!({ + let mut state = runtime.lock().unwrap(); + state.project_protocol_event_to_worker_state( + &detail.worker_ref, + &protocol::Event::WorkerState { + snapshot: running.clone(), + }, + ) + }); + assert_eq!( + runtime + .worker_detail(&detail.worker_ref) + .unwrap() + .worker_state, + Some(running.clone()) + ); + + assert!({ + let mut state = runtime.lock().unwrap(); + !state.project_protocol_event_to_worker_state( + &detail.worker_ref, + &protocol::Event::WorkerState { + snapshot: protocol::WorkerStateSnapshot { + revision: 2, + state: protocol::WorkerState::Idle, + ..running.clone() + }, + }, + ) + }); + assert!({ + let mut state = runtime.lock().unwrap(); + !state.project_protocol_event_to_worker_state( + &detail.worker_ref, + &protocol::Event::WorkerState { + snapshot: protocol::WorkerStateSnapshot { + state: protocol::WorkerState::Idle, + ..running.clone() + }, + }, + ) + }); + let after = runtime.worker_detail(&detail.worker_ref).unwrap(); + assert_eq!(after.status, WorkerStatus::Idle); + assert_eq!(after.worker_state, Some(running)); + } + + #[test] + fn create_worker_rejects_mismatched_submission_acknowledgement() { let (runtime, backend) = runtime_and_backend(); backend.preserve_commit_ack_submission_id(); - backend.set_dispatch_result(WorkerExecutionResult::accepted_input_committed( + backend.set_dispatch_result(WorkerExecutionResult::accepted_submission( WorkerExecutionOperation::Input, - WorkerExecutionRunState::Busy, + "request-test", "forged-submission", + protocol::SubmissionDisposition::Started, )); let mut request = task_request("mismatched initial input commit ack"); request.initial_input = Some(WorkerInput::user("start the ticket")); @@ -4759,7 +4801,6 @@ mod tests { let (runtime, backend) = runtime_and_backend(); backend.set_dispatch_result(WorkerExecutionResult::accepted( WorkerExecutionOperation::Input, - WorkerExecutionRunState::Busy, )); let mut request = task_request("missing initial input commit ack"); request.initial_input = Some(WorkerInput::user("start the ticket")); @@ -4866,6 +4907,7 @@ mod tests { &detail.worker_ref, protocol::Event::Snapshot { session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: vec![protocol::SessionSnapshotEntry { entry_id: "restored-log-entry".to_owned(), timestamp: 1, @@ -4886,7 +4928,7 @@ mod tests { context_window: 128, context_tokens: 64, }, - status: protocol::WorkerStatus::Running, + state: protocol::WorkerStatus::Running.into(), in_flight: protocol::InFlightSnapshot { blocks: Vec::new(), commands: Vec::new(), @@ -4902,18 +4944,38 @@ mod tests { protocol::Event::Snapshot { session, greeting, - status, + state, .. } => { assert_eq!(session.entries.len(), 1); assert_eq!(session.entries[0].entry_id, "restored-log-entry"); assert_eq!(greeting.worker_name, "live-worker"); - assert_eq!(status, protocol::WorkerStatus::Running); + assert_eq!(state.catalog_status(), protocol::WorkerStatus::Running); } other => panic!("expected snapshot, got {other:?}"), } } + #[cfg(feature = "ws-server")] + #[test] + fn observation_snapshot_fails_closed_when_backend_snapshot_is_unavailable() { + let runtime = runtime_with_backend(); + let detail = runtime + .create_worker(task_request("snapshot unavailable")) + .unwrap(); + + assert!(matches!( + runtime + .worker_observation_snapshot(&detail.worker_ref) + .unwrap_err(), + RuntimeError::WorkerExecutionUnavailable { + worker_id, + message, + } if worker_id == detail.worker_ref.worker_id + && message == "authoritative Worker snapshot is unavailable" + )); + } + struct InputOnlyBackend; impl WorkerExecutionBackend for InputOnlyBackend { @@ -4924,7 +4986,6 @@ mod tests { fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult { WorkerExecutionSpawnResult::Connected { handle: WorkerExecutionHandle::new(request.worker_ref, self.backend_id()), - run_state: WorkerExecutionRunState::Idle, working_directory: request .working_directory .as_ref() @@ -4937,10 +4998,13 @@ mod tests { _handle: &WorkerExecutionHandle, input: WorkerInput, ) -> WorkerExecutionResult { - WorkerExecutionResult::accepted_input_committed( + WorkerExecutionResult::accepted_submission( WorkerExecutionOperation::Input, - WorkerExecutionRunState::Idle, - input.submission_id.expect("Runtime submission id"), + "request-test", + input + .submission_request_id + .expect("Runtime submission request id"), + protocol::SubmissionDisposition::Started, ) } } @@ -4977,7 +5041,12 @@ mod tests { .unwrap(); runtime - .send_protocol_method(&detail.worker_ref, Method::Shutdown) + .send_protocol_method( + &detail.worker_ref, + Method::Shutdown { + command: test_command(), + }, + ) .unwrap(); assert_eq!( @@ -4993,7 +5062,12 @@ mod tests { .create_worker(task_request("restore explicitly")) .unwrap(); runtime - .send_protocol_method(&detail.worker_ref, Method::Shutdown) + .send_protocol_method( + &detail.worker_ref, + Method::Shutdown { + command: test_command(), + }, + ) .unwrap(); assert!(matches!( @@ -5009,10 +5083,9 @@ mod tests { assert_eq!(*backend.restore_count.lock().unwrap(), 1); assert_eq!(*backend.run_generations.lock().unwrap(), vec![1, 2]); - assert_eq!( - runtime.worker_detail(&detail.worker_ref).unwrap().status, - WorkerStatus::Idle - ); + let restored = runtime.worker_detail(&detail.worker_ref).unwrap(); + assert_eq!(restored.status, WorkerStatus::Idle); + assert_eq!(restored.worker_state, None); } #[test] @@ -5030,7 +5103,7 @@ mod tests { request.initial_input = Some(WorkerInput { kind: WorkerInputKind::User, content: String::new(), - submission_id: None, + submission_request_id: None, segments: Some(vec![ protocol::Segment::Flow { selector: "builtin:coder-review".to_string(), @@ -5068,7 +5141,7 @@ mod tests { let input = WorkerInput { kind: WorkerInputKind::User, content: String::new(), - submission_id: None, + submission_request_id: None, segments: Some(vec![protocol::Segment::Flow { selector: "builtin:coder-review".to_string(), }]), @@ -5083,11 +5156,11 @@ mod tests { assert_eq!(dispatched[0].kind, input.kind); assert_eq!(dispatched[0].content, input.content); assert_eq!(dispatched[0].segments, input.segments); - let submission_id = dispatched[0] - .submission_id + let submission_request_id = dispatched[0] + .submission_request_id .as_deref() - .expect("Runtime submission id"); - Uuid::parse_str(submission_id).expect("submission id UUID"); + .expect("Runtime submission request id"); + Uuid::parse_str(submission_request_id).expect("submission request id UUID"); } #[cfg(feature = "ws-server")] @@ -5113,11 +5186,7 @@ mod tests { let observations = runtime .read_worker_observation_events(&detail.worker_ref, WorkerObservationCursor::zero()) .unwrap(); - assert_eq!(observations.len(), 1); - assert!(matches!( - observations[0].payload, - protocol::Event::UserMessage { .. } - )); + assert!(observations.is_empty()); runtime .observe_worker_event( @@ -5135,8 +5204,8 @@ mod tests { let observations = runtime .read_worker_observation_events(&detail.worker_ref, WorkerObservationCursor::zero()) .unwrap(); - assert_eq!(observations.len(), 2); - let protocol::Event::SystemItem { item } = &observations[1].payload else { + assert_eq!(observations.len(), 1); + let protocol::Event::SystemItem { item } = &observations[0].payload else { panic!("committed notification observation must be a system item"); }; assert_eq!(item["kind"], "notification"); diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index 95d0c941..20b05aed 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -10,8 +10,8 @@ use std::collections::HashMap; use std::future::Future; use std::path::{Path, PathBuf}; -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Mutex, mpsc}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Mutex, RwLock, mpsc}; use std::time::Duration; use crate::auth::{ @@ -25,8 +25,8 @@ use crate::catalog::{ }; use crate::execution::{ WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation, - WorkerExecutionRestoreRequest, WorkerExecutionResult, WorkerExecutionRunState, - WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult, + WorkerExecutionRestoreRequest, WorkerExecutionResult, WorkerExecutionSpawnRequest, + WorkerExecutionSpawnResult, }; use crate::identity::WorkerRef; use crate::interaction::{WorkerInput, WorkerInputKind}; @@ -38,8 +38,29 @@ use crate::working_directory::{ WorkingDirectoryBinding, WorkingDirectoryDiagnostic, WorkingDirectoryMaterializer, }; use async_trait::async_trait; -use protocol::{ErrorCode, Event, Method, Segment, WorkerStatus}; -use session_store::{CombinedStore, LogEntry, WorkerAggregateStore, WorkerSessionStore}; +#[cfg(test)] +use protocol::WorkerStatus; +use protocol::{Event, Method, Segment, WorkerCommandEnvelope}; + +static NEXT_INTERNAL_COMMAND_ID: AtomicU64 = AtomicU64::new(1); + +fn next_internal_command( + state: &RwLock, +) -> Result { + let snapshot = state + .read() + .map_err(|_| "worker state lock is poisoned".to_string())? + .clone(); + let floor = snapshot.last_command_id.saturating_add(1); + let command_id = NEXT_INTERNAL_COMMAND_ID + .fetch_update(Ordering::AcqRel, Ordering::Acquire, |current| { + Some(current.max(floor).saturating_add(1)) + }) + .unwrap_or(floor) + .max(floor); + Ok(WorkerCommandEnvelope::for_snapshot(command_id, &snapshot)) +} +use session_store::{CombinedStore, WorkerAggregateStore, WorkerSessionStore}; #[cfg(test)] use session_store::{FsStore, FsWorkerStore}; use tokio::runtime::Runtime; @@ -57,11 +78,10 @@ use worker::feature::builtin::{ #[cfg(feature = "ws-server")] use worker::ipc::protocol_session::{live_log_entry_event, subscribe_worker_protocol_session}; use worker::{ - PreparedWorker, PromptCatalogSource, SegmentLogSink, WORKER_INPUT_SUBMISSION_EXTENSION_DOMAIN, - Worker, WorkerBootstrap, WorkerBootstrapError, WorkerBootstrapLayout, - WorkerControllerTransport, WorkerError, WorkerFilesystemAuthority, WorkerHandle, - WorkerSharedState, WorkerWorkspaceContext, WorkspaceClient, WorkspaceId, - bash_output_dir_for_worker_id, + PreparedWorker, PromptCatalogSource, SegmentLogSink, Worker, WorkerBootstrap, + WorkerBootstrapError, WorkerBootstrapLayout, WorkerControllerTransport, WorkerError, + WorkerFilesystemAuthority, WorkerHandle, WorkerSharedState, WorkerWorkspaceContext, + WorkspaceClient, WorkspaceId, bash_output_dir_for_worker_id, }; const DEFAULT_BACKEND_ID: &str = "worker-crate"; @@ -70,17 +90,6 @@ const RUNTIME_TASK_TIMEOUT: Duration = Duration::from_secs(10); // returns a typed execution error instead of leaving the outer waiter to time out. const USER_INPUT_COMMIT_TIMEOUT: Duration = Duration::from_secs(9); -fn user_input_has_submission(entry: &LogEntry, submission_id: &str) -> bool { - let extensions = match entry { - LogEntry::AnnotatedUserInput { extensions, .. } => extensions, - _ => return false, - }; - extensions.iter().any(|extension| { - extension.domain == WORKER_INPUT_SUBMISSION_EXTENSION_DOMAIN - && extension.payload["submission_id"].as_str() == Some(submission_id) - }) -} - pub struct RuntimeWorkerController { pub handle: WorkerHandle, pub shutdown: Arc>>, @@ -184,7 +193,7 @@ impl WorkerObservationProvider for RuntimeGrantedWorkerObservationProvider { }, display_name: grant.worker_id.clone(), relation: "granted_peer".to_string(), - status: format!("{:?}", state.get_status()).to_lowercase(), + status: format!("{:?}", state.catalog_status()).to_lowercase(), }); } subjects.sort_by(|left, right| left.subject.cmp(&right.subject)); @@ -1186,10 +1195,11 @@ impl RuntimeWorkerFactory for ProfileRuntimeWorkerFactory { } } +#[derive(Clone)] struct RuntimeWorkerExecution { handle: WorkerHandle, shutdown: Arc>>, - busy: Arc, + worker_state: Arc>, workspace_client: Option>, } @@ -1287,7 +1297,7 @@ where ) -> Result< ( WorkerHandle, - Arc, + Arc>, Option>, ), WorkerExecutionResult, @@ -1313,7 +1323,7 @@ where .map(|execution| { ( execution.handle.clone(), - execution.busy.clone(), + execution.worker_state.clone(), execution.workspace_client.clone(), ) }) @@ -1330,7 +1340,6 @@ where operation: WorkerExecutionOperation, worker: WorkerHandle, method: Method, - accepted_run_state: WorkerExecutionRunState, ) -> WorkerExecutionResult { self.run_on_adapter_runtime(async move { worker @@ -1338,130 +1347,77 @@ where .await .map_err(|err| format!("failed to send Worker method: {err}")) }) - .map(|_| WorkerExecutionResult::accepted(operation, accepted_run_state)) + .map(|_| WorkerExecutionResult::accepted(operation)) .unwrap_or_else(|message| WorkerExecutionResult::errored(operation, message)) } - fn send_user_input_and_wait_for_commit( + fn send_submit_and_wait_for_acceptance( &self, operation: WorkerExecutionOperation, worker: WorkerHandle, method: Method, - submission_id: String, - accepted_run_state: WorkerExecutionRunState, + submission_request_id: String, ) -> WorkerExecutionResult { - let acknowledged_submission_id = submission_id.clone(); + let request_id = submission_request_id.clone(); self.run_on_adapter_runtime(async move { - // Subscribe before enqueueing the input so the acknowledgement cannot - // race with a fast Worker commit. The opaque submission id is stored in - // the same UserInput entry as the transformed Flow input and its state. - let (_, mut committed_entries) = worker.sink.subscribe_with_snapshot(); - let committed_probe = worker.clone(); + // Subscribe before enqueueing so a fast durable acceptance cannot + // race the Runtime acknowledgement. let mut events = worker.subscribe(); worker .send(method) .await .map_err(|err| format!("failed to send Worker method: {err}"))?; - let timeout_probe = committed_probe.clone(); - let timeout_submission_id = submission_id.clone(); - let acknowledgement = tokio::time::timeout(USER_INPUT_COMMIT_TIMEOUT, async move { - let input_was_committed = || { - committed_probe - .committed_entries() - .iter() - .any(|entry| user_input_has_submission(entry, &submission_id)) - }; + tokio::time::timeout(USER_INPUT_COMMIT_TIMEOUT, async move { loop { - tokio::select! { - entry = committed_entries.recv() => { - match entry { - Ok(entry) if user_input_has_submission(&entry, &submission_id) => { - return Ok(()); - } - Ok(_) => {} - Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped)) => { - if input_was_committed() { - return Ok(()); - } - return Err(format!( - "worker input commit acknowledgement lagged by {skipped} entry event(s)" - )); - } - Err(tokio::sync::broadcast::error::RecvError::Closed) => { - if input_was_committed() { - return Ok(()); - } - return Err( - "worker entry stream closed before user input was committed" - .to_string(), - ); - } - } + match events.recv().await { + Ok(Event::SubmissionAccepted { + submission_request_id, + submission_id, + disposition, + }) if submission_request_id == request_id => { + return Ok((submission_id, disposition)); } - event = events.recv() => { - match event { - Ok(Event::Error { message, .. }) => { - if input_was_committed() { - return Ok(()); - } - return Err(format!( - "worker rejected user input before session commit: {message}" - )); - } - Ok(Event::Shutdown) => { - if input_was_committed() { - return Ok(()); - } - return Err( - "worker shut down before user input was committed".to_string() - ); - } - Ok(_) => {} - Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped)) => { - if input_was_committed() { - return Ok(()); - } - return Err(format!( - "worker input commit acknowledgement lagged by {skipped} protocol event(s)" - )); - } - Err(tokio::sync::broadcast::error::RecvError::Closed) => { - if input_was_committed() { - return Ok(()); - } - return Err( - "worker event stream closed before user input was committed" - .to_string(), - ); - } - } + Ok(Event::SubmissionRejected { + submission_request_id, + message, + }) if submission_request_id == request_id => { + return Err(format!("worker rejected Submit: {message}")); + } + Ok(Event::Error { message, .. }) => { + return Err(format!( + "worker rejected Submit before durable acceptance: {message}" + )); + } + Ok(Event::Shutdown) => { + return Err( + "worker shut down before Submit was durably accepted".to_string() + ); + } + Ok(_) => {} + Err(tokio::sync::broadcast::error::RecvError::Lagged(skipped)) => { + return Err(format!( + "worker Submit acknowledgement lagged by {skipped} protocol event(s)" + )); + } + Err(tokio::sync::broadcast::error::RecvError::Closed) => { + return Err( + "worker event stream closed before Submit was durably accepted" + .to_string(), + ); } } } }) - .await; - - match acknowledgement { - Ok(result) => result, - Err(_) => { - if timeout_probe - .committed_entries() - .iter() - .any(|entry| user_input_has_submission(entry, &timeout_submission_id)) - { - Ok(()) - } else { - Err("timed out waiting for worker user input commit".to_string()) - } - } - } + .await + .map_err(|_| "timed out waiting for durable Worker Submit acceptance".to_string())? }) - .map(|_| { - WorkerExecutionResult::accepted_input_committed( + .map(|(submission_id, disposition)| { + WorkerExecutionResult::accepted_submission( operation, - accepted_run_state, - acknowledged_submission_id, + submission_request_id, + submission_id, + disposition, ) }) .unwrap_or_else(|message| WorkerExecutionResult::errored(operation, message)) @@ -1477,41 +1433,31 @@ where working_directory: Option, workspace_client: Option>, ) -> WorkerExecutionSpawnResult { - let busy = Arc::new(AtomicBool::new(false)); + let worker_state = Arc::new(RwLock::new(handle.shared_state.snapshot())); #[cfg(feature = "ws-server")] { let streams = subscribe_worker_protocol_session(&handle); let mut events = streams.events; let mut entry_events = streams.log_entries; - let bridge_busy = busy.clone(); + let bridge_worker_state = worker_state.clone(); if let Err(message) = self.spawn_on_adapter_runtime(async move { loop { tokio::select! { event = events.recv() => { match event { - Ok(event) => { - let next_busy = match &event { - Event::InvokeStart { .. } - | Event::Status { - status: WorkerStatus::Running, - } => Some(true), - Event::RunEnd { .. } - | Event::Error { - code: ErrorCode::NotPaused, - .. + Ok(mut event) => { + match apply_protocol_worker_state(&bridge_worker_state, &mut event) { + Ok(true) => { + let _ = bridge_context.publish_protocol_event(event); } - | Event::Status { - status: - WorkerStatus::Idle - | WorkerStatus::Paused - | WorkerStatus::Stopped, + Ok(false) => {} + Err(message) => { + let _ = bridge_context.publish_protocol_event(Event::Error { + code: protocol::ErrorCode::Internal, + message: format!("worker state stream rejected: {message}"), + }); + break; } - | Event::Shutdown => Some(false), - _ => None, - }; - let _ = bridge_context.publish_protocol_event(event); - if let Some(next_busy) = next_busy { - bridge_busy.store(next_busy, Ordering::SeqCst); } } Err(broadcast::error::RecvError::Lagged(_)) => continue, @@ -1556,14 +1502,13 @@ where RuntimeWorkerExecution { handle, shutdown, - busy, + worker_state, workspace_client, }, ); WorkerExecutionSpawnResult::Connected { handle: WorkerExecutionHandle::new(worker_ref, self.backend_id()), - run_state: WorkerExecutionRunState::Idle, working_directory: working_directory.map(|binding| binding.status()), } } @@ -1579,43 +1524,28 @@ impl Drop for WorkerRuntimeExecutionBackend { } } -fn method_starts_turn(method: &Method) -> bool { - matches!( - method, - Method::Run { .. } - | Method::RunTracked { .. } - | Method::Notify { auto_run: true, .. } - | Method::Resume - | Method::Compact - ) -} - -fn method_can_start_turn_from_status(method: &Method, status: WorkerStatus) -> bool { - match method { - Method::Resume => matches!(status, WorkerStatus::Idle | WorkerStatus::Paused), - _ => status == WorkerStatus::Idle, - } -} - -fn accepted_notify_run_state(status: WorkerStatus, auto_run: bool) -> WorkerExecutionRunState { - match status { - WorkerStatus::Running => WorkerExecutionRunState::Busy, - WorkerStatus::Idle if auto_run => WorkerExecutionRunState::Busy, - WorkerStatus::Idle | WorkerStatus::Paused | WorkerStatus::Stopped => { - WorkerExecutionRunState::Idle +fn apply_protocol_worker_state( + current: &Arc>, + event: &mut Event, +) -> Result { + let (incoming, replace_stale) = match event { + Event::WorkerState { snapshot } => (snapshot, false), + Event::Snapshot { state, .. } => (state, true), + Event::CommandAcknowledged { acknowledgement } => (&mut acknowledgement.state, true), + _ => return Ok(true), + }; + let mut current = current + .write() + .map_err(|_| "worker state projection lock is poisoned".to_string())?; + match protocol::apply_worker_state_snapshot(&mut current, incoming) { + Ok(protocol::WorkerStateSnapshotApply::Applied) + | Ok(protocol::WorkerStateSnapshotApply::Duplicate) => Ok(true), + Ok(protocol::WorkerStateSnapshotApply::Stale) if replace_stale => { + *incoming = current.clone(); + Ok(true) } - } -} - -fn accepted_run_state_for_method(method: &Method) -> WorkerExecutionRunState { - match method { - Method::Run { .. } - | Method::RunTracked { .. } - | Method::Notify { auto_run: true, .. } - | Method::Resume - | Method::Compact => WorkerExecutionRunState::Busy, - Method::Shutdown => WorkerExecutionRunState::Stopped, - _ => WorkerExecutionRunState::Idle, + Ok(protocol::WorkerStateSnapshotApply::Stale) => Ok(false), + Err(error) => Err(error.to_string()), } } @@ -1944,7 +1874,7 @@ where handle: &WorkerExecutionHandle, input: WorkerInput, ) -> WorkerExecutionResult { - let (worker, busy, _workspace_client) = match self.get_execution(handle) { + let (worker, worker_state, _workspace_client) = match self.get_execution(handle) { Ok(execution) => execution, Err(mut result) => { result.operation = WorkerExecutionOperation::Input; @@ -1953,65 +1883,65 @@ where }; if input.kind == WorkerInputKind::Notify { - let status = worker.shared_state.get_status(); - let accepted_run_state = accepted_notify_run_state(status, true); - let claimed_here = status == WorkerStatus::Idle - && busy - .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) - .is_ok(); - let result = self.send_method( + let notification_request_id = input + .submission_request_id + .unwrap_or_else(protocol::new_submission_request_id); + return self.send_method( WorkerExecutionOperation::Input, worker, - Method::Notify { + Method::NotifyTracked { + notification_request_id: notification_request_id.clone(), message: input.content, auto_run: true, + source: protocol::AuthenticatedInputSource::Backend { + operation_id: notification_request_id, + }, }, - accepted_run_state, ); - if claimed_here && result.outcome != crate::execution::WorkerExecutionOutcome::Accepted - { - busy.store(false, Ordering::SeqCst); - } - return result; } - if worker.shared_state.get_status() != WorkerStatus::Idle - || busy - .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) - .is_err() - { - return WorkerExecutionResult::busy( + if input.kind == WorkerInputKind::Compact { + let command = match next_internal_command(&worker_state) { + Ok(command) => command, + Err(error) => { + return WorkerExecutionResult::errored(WorkerExecutionOperation::Input, error); + } + }; + return self.send_method( WorkerExecutionOperation::Input, - "Worker is already running; runtime adapter v0 does not queue input", + worker, + Method::Compact { command }, ); } - let (method, submission_id) = match input.kind { + let (method, submission_request_id) = match input.kind { WorkerInputKind::User => { let Some(submission_id) = input - .submission_id + .submission_request_id .filter(|submission_id| !submission_id.trim().is_empty()) else { - busy.store(false, Ordering::SeqCst); return WorkerExecutionResult::rejected( WorkerExecutionOperation::Input, "Runtime user input is missing its internal submission id", ); }; ( - Method::RunTracked { + Method::SubmitTracked { + submission_request_id: submission_id.clone(), input: input.segments.unwrap_or_else(|| { vec![Segment::text(input.content.trim().to_string())] }), - submission_id: submission_id.clone(), + source: protocol::AuthenticatedInputSource::Backend { + operation_id: submission_id.clone(), + }, }, Some(submission_id), ) } WorkerInputKind::Notify => { - unreachable!("Notify input is dispatched before the turn-start busy guard") + unreachable!("Notify input is dispatched before ordinary input mapping") } - WorkerInputKind::Compact => (Method::Compact, None), + WorkerInputKind::Compact => unreachable!("compact input is dispatched above"), WorkerInputKind::ListRewindTargets => (Method::ListRewindTargets, None), WorkerInputKind::RegisterPeer => ( Method::RegisterPeer { @@ -2020,37 +1950,18 @@ where None, ), }; - let accepted_run_state = match method { - Method::Run { .. } - | Method::RunTracked { .. } - | Method::Notify { .. } - | Method::Compact => WorkerExecutionRunState::Busy, - _ => WorkerExecutionRunState::Idle, - }; - let accepted_is_idle = accepted_run_state == WorkerExecutionRunState::Idle; - let waits_for_user_input_commit = submission_id.is_some(); + let waits_for_submission_acceptance = submission_request_id.is_some(); - let result = if waits_for_user_input_commit { - self.send_user_input_and_wait_for_commit( + if waits_for_submission_acceptance { + self.send_submit_and_wait_for_acceptance( WorkerExecutionOperation::Input, worker, method, - submission_id.expect("tracked Run has submission id"), - accepted_run_state, + submission_request_id.expect("Submit must have a submission request id"), ) } else { - self.send_method( - WorkerExecutionOperation::Input, - worker, - method, - accepted_run_state, - ) - }; - if accepted_is_idle || result.outcome != crate::execution::WorkerExecutionOutcome::Accepted - { - busy.store(false, Ordering::SeqCst); + self.send_method(WorkerExecutionOperation::Input, worker, method) } - result } fn upload_file( @@ -2092,10 +2003,7 @@ where } }; match worker.delete_uploaded_file(artifact_id) { - Ok(_) => WorkerExecutionResult::accepted( - WorkerExecutionOperation::DeleteUploadedFile, - WorkerExecutionRunState::Idle, - ), + Ok(_) => WorkerExecutionResult::accepted(WorkerExecutionOperation::DeleteUploadedFile), Err(error) => WorkerExecutionResult::rejected( WorkerExecutionOperation::DeleteUploadedFile, format!("uploaded_file_delete_rejected: {error}"), @@ -2108,7 +2016,7 @@ where handle: &WorkerExecutionHandle, method: Method, ) -> WorkerExecutionResult { - let (worker, busy, _workspace_client) = match self.get_execution(handle) { + let (worker, _worker_state, _workspace_client) = match self.get_execution(handle) { Ok(execution) => execution, Err(mut result) => { result.operation = WorkerExecutionOperation::ProtocolMethod; @@ -2116,55 +2024,7 @@ where } }; - if let Method::Notify { auto_run, .. } = &method { - let auto_run = *auto_run; - let status = worker.shared_state.get_status(); - let accepted_run_state = accepted_notify_run_state(status, auto_run); - let claimed_here = status == WorkerStatus::Idle - && auto_run - && busy - .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) - .is_ok(); - let result = self.send_method( - WorkerExecutionOperation::ProtocolMethod, - worker, - method, - accepted_run_state, - ); - if claimed_here && result.outcome != crate::execution::WorkerExecutionOutcome::Accepted - { - busy.store(false, Ordering::SeqCst); - } - return result; - } - - let starts_turn = method_starts_turn(&method); - if starts_turn - && (!method_can_start_turn_from_status(&method, worker.shared_state.get_status()) - || busy - .compare_exchange(false, true, Ordering::SeqCst, Ordering::SeqCst) - .is_err()) - { - return WorkerExecutionResult::busy( - WorkerExecutionOperation::ProtocolMethod, - "Worker is already running; runtime adapter v0 does not queue protocol methods", - ); - } - - let accepted_run_state = accepted_run_state_for_method(&method); - let accepted_is_idle = accepted_run_state == WorkerExecutionRunState::Idle; - let result = self.send_method( - WorkerExecutionOperation::ProtocolMethod, - worker, - method, - accepted_run_state, - ); - if (starts_turn && accepted_is_idle) - || (starts_turn && result.outcome != crate::execution::WorkerExecutionOutcome::Accepted) - { - busy.store(false, Ordering::SeqCst); - } - result + self.send_method(WorkerExecutionOperation::ProtocolMethod, worker, method) } fn stop_worker(&self, handle: &WorkerExecutionHandle) -> WorkerExecutionResult { @@ -2179,7 +2039,7 @@ where ); } let execution = match self.workers.lock() { - Ok(mut workers) => workers.remove(handle.worker_ref()), + Ok(workers) => workers.get(handle.worker_ref()).cloned(), Err(_) => { return WorkerExecutionResult::errored( WorkerExecutionOperation::Stop, @@ -2195,48 +2055,73 @@ where }; let artifact_cleanup = execution.handle.clone(); let shutdown = execution.shutdown.clone(); + let command = match next_internal_command(&execution.worker_state) { + Ok(command) => command, + Err(error) => { + return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, error); + } + }; let result = self.send_method( WorkerExecutionOperation::Stop, - execution.handle, - Method::Shutdown, - WorkerExecutionRunState::Stopped, + execution.handle.clone(), + Method::Shutdown { command }, ); if result.outcome != crate::execution::WorkerExecutionOutcome::Accepted { return result; } - match self.run_on_adapter_runtime(async move { - let receiver = shutdown.lock().await.take(); - if let Some(receiver) = receiver { - receiver - .await - .map_err(|_| "Worker shutdown completion channel closed".to_string())?; + let shutdown_wait = self.run_on_adapter_runtime(async move { + let mut guard = shutdown.lock().await; + let Some(mut receiver) = guard.take() else { + return Ok(()); + }; + match tokio::time::timeout(Duration::from_secs(5), &mut receiver).await { + Ok(Ok(())) => Ok(()), + Ok(Err(_)) => Err("Worker shutdown completion channel closed".to_string()), + Err(_) => { + *guard = Some(receiver); + Err("Worker shutdown confirmation timed out; stop remains retryable".into()) + } } - Ok(()) - }) { - Ok(()) => match artifact_cleanup.delete_uncommitted_uploaded_files() { - Ok(_) => result, - Err(error) => WorkerExecutionResult::errored( - WorkerExecutionOperation::Stop, - format!("uploaded_file_cleanup_failed: {error}"), - ), - }, - Err(message) => WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message), + }); + if let Err(message) = shutdown_wait { + return WorkerExecutionResult::errored(WorkerExecutionOperation::Stop, message); + } + if let Err(error) = artifact_cleanup.delete_uncommitted_uploaded_files() { + return WorkerExecutionResult::errored( + WorkerExecutionOperation::Stop, + format!("uploaded_file_cleanup_failed: {error}"), + ); + } + match self.workers.lock() { + Ok(mut workers) => { + workers.remove(handle.worker_ref()); + result + } + Err(_) => WorkerExecutionResult::errored( + WorkerExecutionOperation::Stop, + "worker adapter registry lock is poisoned after shutdown", + ), } } fn cancel_worker(&self, handle: &WorkerExecutionHandle) -> WorkerExecutionResult { - let (worker, _busy, _workspace_client) = match self.get_execution(handle) { + let (worker, worker_state, _workspace_client) = match self.get_execution(handle) { Ok(execution) => execution, Err(mut result) => { result.operation = WorkerExecutionOperation::Cancel; return result; } }; + let command = match next_internal_command(&worker_state) { + Ok(command) => command, + Err(error) => { + return WorkerExecutionResult::errored(WorkerExecutionOperation::Cancel, error); + } + }; self.send_method( WorkerExecutionOperation::Cancel, worker, - Method::Cancel, - WorkerExecutionRunState::Idle, + Method::Cancel { command }, ) } @@ -2301,6 +2186,79 @@ mod tests { use manifest::{Scope, WorkerManifest}; use session_store::{LogEntry, WorkerMetadataStore}; + fn test_command() -> WorkerCommandEnvelope { + WorkerCommandEnvelope { + command_id: 1, + expected_execution_generation: 1, + expected_worker_state_revision: 0, + } + } + + fn adapter_command( + backend: &WorkerRuntimeExecutionBackend, + worker_ref: &WorkerRef, + ) -> WorkerCommandEnvelope { + let workers = backend.workers.lock().unwrap(); + let state = workers + .get(worker_ref) + .expect("worker execution") + .worker_state + .read() + .unwrap() + .clone(); + WorkerCommandEnvelope::for_snapshot(state.last_command_id.saturating_add(1), &state) + } + + #[test] + fn protocol_bridge_applies_state_and_acknowledgement_monotonically() { + let running = protocol::WorkerStateSnapshot { + execution_generation: 4, + revision: 3, + state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Running, + )), + last_command_id: 2, + }; + let current = Arc::new(RwLock::new(running.clone())); + let mut stale = Event::WorkerState { + snapshot: protocol::WorkerStateSnapshot { + revision: 2, + state: protocol::WorkerState::Idle, + ..running.clone() + }, + }; + assert!(!apply_protocol_worker_state(¤t, &mut stale).unwrap()); + assert_eq!(*current.read().unwrap(), running); + + let paused = protocol::WorkerStateSnapshot { + revision: 4, + state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Paused, + )), + last_command_id: 3, + ..running.clone() + }; + let mut acknowledgement = Event::CommandAcknowledged { + acknowledgement: protocol::WorkerCommandAcknowledgement { + command_id: 3, + command: protocol::WorkerCommandKind::Pause, + disposition: protocol::WorkerCommandDisposition::Accepted, + state: paused.clone(), + }, + }; + assert!(apply_protocol_worker_state(¤t, &mut acknowledgement).unwrap()); + assert_eq!(*current.read().unwrap(), paused); + + let mut conflict = Event::WorkerState { + snapshot: protocol::WorkerStateSnapshot { + state: protocol::WorkerState::Idle, + ..paused.clone() + }, + }; + assert!(apply_protocol_worker_state(¤t, &mut conflict).is_err()); + assert_eq!(*current.read().unwrap(), paused); + } + #[test] fn workspace_prompt_projection_notification_advances_shared_cache() { let cache = WorkspacePromptProjectionCache::default(); @@ -2447,46 +2405,6 @@ mod tests { assert_eq!(after_restore_workspace_id.as_deref(), Some("workspace-a")); } - #[test] - fn notify_run_state_allows_running_worker_inbox_delivery() { - assert_eq!( - accepted_notify_run_state(WorkerStatus::Running, true), - WorkerExecutionRunState::Busy - ); - assert_eq!( - accepted_notify_run_state(WorkerStatus::Idle, true), - WorkerExecutionRunState::Busy - ); - assert_eq!( - accepted_notify_run_state(WorkerStatus::Idle, false), - WorkerExecutionRunState::Idle - ); - assert_eq!( - accepted_notify_run_state(WorkerStatus::Paused, true), - WorkerExecutionRunState::Idle - ); - } - - #[test] - fn resume_turn_claim_accepts_paused_and_idle_but_not_running_status() { - assert!(method_can_start_turn_from_status( - &Method::Resume, - WorkerStatus::Paused - )); - assert!(method_can_start_turn_from_status( - &Method::Resume, - WorkerStatus::Idle - )); - assert!(!method_can_start_turn_from_status( - &Method::Resume, - WorkerStatus::Running - )); - assert!(!method_can_start_turn_from_status( - &Method::Compact, - WorkerStatus::Paused - )); - } - #[derive(Clone)] enum MockResponse { Complete(Vec), @@ -2687,28 +2605,53 @@ mod tests { .collect() } + fn wait_for_adapter_command( + backend: &WorkerRuntimeExecutionBackend, + worker_ref: &WorkerRef, + expected_command_id: u64, + ) { + let deadline = std::time::Instant::now() + Duration::from_secs(5); + loop { + let observed = { + let workers = backend.workers.lock().unwrap(); + workers + .get(worker_ref) + .expect("live Worker execution") + .worker_state + .read() + .unwrap() + .last_command_id + }; + if observed >= expected_command_id { + return; + } + assert!( + std::time::Instant::now() < deadline, + "timed out waiting for adapter command {expected_command_id}; last observed={observed}", + ); + std::thread::sleep(Duration::from_millis(10)); + } + } + fn wait_for_adapter_state( backend: &WorkerRuntimeExecutionBackend, worker_ref: &WorkerRef, expected_status: WorkerStatus, - expected_busy: bool, ) { let deadline = std::time::Instant::now() + Duration::from_secs(5); loop { let observed = { let workers = backend.workers.lock().unwrap(); let execution = workers.get(worker_ref).expect("live Worker execution"); - ( - execution.handle.shared_state.get_status(), - execution.busy.load(Ordering::SeqCst), - ) + let projected = execution.worker_state.read().unwrap().catalog_status(); + (execution.handle.shared_state.catalog_status(), projected) }; - if observed == (expected_status, expected_busy) { + if observed == (expected_status, expected_status) { return; } assert!( std::time::Instant::now() < deadline, - "timed out waiting for adapter state {expected_status:?}, busy={expected_busy}; last observed status={:?}, busy={}", + "timed out waiting for adapter state {expected_status:?}; last observed controller={:?}, projected={:?}", observed.0, observed.1, ); @@ -3211,13 +3154,19 @@ mod tests { .expect("in-process restore must not bind the overlong Unix socket path"); assert_eq!( - controller.handle.shared_state.get_status(), + controller.handle.shared_state.catalog_status(), WorkerStatus::Idle ); assert!(!socket_path.exists()); assert!(run_dir.join("worker.out.log").is_file()); assert!(run_dir.join("worker.err.log").is_file()); - controller.handle.send(Method::Shutdown).await.unwrap(); + controller + .handle + .send(Method::Shutdown { + command: test_command(), + }) + .await + .unwrap(); if let Some(receiver) = controller.shutdown.lock().await.take() { receiver.await.unwrap(); } @@ -3331,7 +3280,9 @@ mod tests { backend .run_on_adapter_runtime(async move { handle - .send(Method::Shutdown) + .send(Method::Shutdown { + command: test_command(), + }) .await .map_err(|error| error.to_string())?; if let Some(receiver) = shutdown.lock().await.take() { @@ -3400,6 +3351,60 @@ mod tests { ); } + #[test] + fn running_worker_accepts_a_second_submit_as_queued() { + let client = MockClient::sequential(vec![MockResponse::Hang(vec![])]); + let runtime_base = tempfile::tempdir().unwrap(); + let cwd = tempfile::tempdir().unwrap(); + let store = tempfile::tempdir().unwrap(); + let factory = MockFactory { + client, + runtime_base: runtime_base.path().to_path_buf(), + cwd: cwd.path().to_path_buf(), + store_dir: store.path().join("sessions"), + worker_metadata_dir: store.path().join("workers"), + observed_cwds: Arc::new(Mutex::new(Vec::new())), + observed_workspace_clients: Arc::new(Mutex::new(Vec::new())), + }; + let backend = Arc::new(WorkerRuntimeExecutionBackend::new(factory).unwrap()); + let runtime = + EmbeddedRuntime::with_execution_backend(RuntimeOptions::default(), backend).unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let detail = runtime + .create_worker(create_request("queued-submit")) + .unwrap(); + + let mut first_input = WorkerInput::user("first"); + first_input.submission_request_id = Some("request-first".into()); + let first = runtime + .send_input(&detail.worker_ref, first_input.clone()) + .unwrap(); + assert_eq!( + first.submission.as_ref().map(|ack| ack.disposition), + Some(protocol::SubmissionDisposition::Started) + ); + let retry = runtime.send_input(&detail.worker_ref, first_input).unwrap(); + assert_eq!(retry.submission, first.submission); + let mut conflicting_retry = WorkerInput::user("different"); + conflicting_retry.submission_request_id = Some("request-first".into()); + assert!( + runtime + .send_input(&detail.worker_ref, conflicting_retry) + .is_err(), + "same request id with a different payload must fail" + ); + + let mut second_input = WorkerInput::user("second"); + second_input.submission_request_id = Some("request-second".into()); + let second = runtime + .send_input(&detail.worker_ref, second_input) + .unwrap(); + assert_eq!( + second.submission.as_ref().map(|ack| ack.disposition), + Some(protocol::SubmissionDisposition::Queued) + ); + } + #[test] fn create_with_initial_input_returns_after_session_commit() { let client = MockClient::new(simple_text_events()); @@ -3449,8 +3454,10 @@ mod tests { }; extensions .iter() - .find(|extension| extension.domain == WORKER_INPUT_SUBMISSION_EXTENSION_DOMAIN) - .and_then(|extension| extension.payload["submission_id"].as_str()) + .find(|extension| extension.domain == "worker.pending_activations.v1") + .and_then(|extension| { + extension.payload["receipts"][0]["submission_id"].as_str() + }) }) .expect("committed input submission id"); uuid::Uuid::parse_str(submission_id).expect("opaque submission id is a UUID"); @@ -3600,6 +3607,7 @@ mod tests { #[test] #[cfg(feature = "ws-server")] + #[serial_test::serial(worker_allocation)] fn adapter_resumes_paused_turn_once_and_preserves_idle_not_paused_error() { let hanging_events = || simple_text_events().into_iter().take(2).collect::>(); let client = MockClient::sequential(vec![ @@ -3632,62 +3640,91 @@ mod tests { runtime .send_input(&detail.worker_ref, WorkerInput::user("pause and resume")) .expect("start initial turn"); - wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Running, true); + wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Running); - let running_resume = runtime - .send_protocol_method(&detail.worker_ref, Method::Resume) - .expect_err("Resume while Running must be rejected"); - assert!( - running_resume - .to_string() - .contains("does not queue protocol methods"), - "unexpected Running Resume error: {running_resume}" - ); + let running_resume = adapter_command(&backend, &detail.worker_ref); + runtime + .send_protocol_method( + &detail.worker_ref, + Method::Resume { + command: running_resume, + }, + ) + .expect("running Resume is forwarded for controller admission"); + wait_for_adapter_command(&backend, &detail.worker_ref, running_resume.command_id); runtime - .send_protocol_method(&detail.worker_ref, Method::Pause) + .send_protocol_method( + &detail.worker_ref, + Method::Pause { + command: adapter_command(&backend, &detail.worker_ref), + }, + ) .expect("pause initial turn"); - wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Paused, false); + wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Paused); runtime - .send_protocol_method(&detail.worker_ref, Method::Resume) + .send_protocol_method( + &detail.worker_ref, + Method::Resume { + command: adapter_command(&backend, &detail.worker_ref), + }, + ) .expect("resume paused turn"); - wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Running, true); + wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Running); - let duplicate_resume = runtime - .send_protocol_method(&detail.worker_ref, Method::Resume) - .expect_err("duplicate Resume must be rejected"); - assert!( - duplicate_resume - .to_string() - .contains("does not queue protocol methods"), - "unexpected duplicate Resume error: {duplicate_resume}" - ); + let duplicate_resume = adapter_command(&backend, &detail.worker_ref); + runtime + .send_protocol_method( + &detail.worker_ref, + Method::Resume { + command: duplicate_resume, + }, + ) + .expect("duplicate Resume is forwarded for controller admission"); + wait_for_adapter_command(&backend, &detail.worker_ref, duplicate_resume.command_id); runtime - .send_protocol_method(&detail.worker_ref, Method::Pause) + .send_protocol_method( + &detail.worker_ref, + Method::Pause { + command: adapter_command(&backend, &detail.worker_ref), + }, + ) .expect("pause resumed turn"); - wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Paused, false); + wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Paused); runtime - .send_protocol_method(&detail.worker_ref, Method::Resume) + .send_protocol_method( + &detail.worker_ref, + Method::Resume { + command: adapter_command(&backend, &detail.worker_ref), + }, + ) .expect("resume paused turn a second time"); - wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Idle, false); + wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Idle); assert_eq!(call_count.load(Ordering::SeqCst), 3); + let idle_resume = adapter_command(&backend, &detail.worker_ref); runtime - .send_protocol_method(&detail.worker_ref, Method::Resume) + .send_protocol_method( + &detail.worker_ref, + Method::Resume { + command: idle_resume, + }, + ) .expect("Idle Resume preserves controller NotPaused semantics"); - wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Idle, false); + wait_for_adapter_command(&backend, &detail.worker_ref, idle_resume.command_id); + wait_for_adapter_state(&backend, &detail.worker_ref, WorkerStatus::Idle); let events = runtime .read_worker_observation_events(&detail.worker_ref, WorkerObservationCursor::zero()) .expect("read protocol events"); assert!(events.iter().any(|event| { matches!( &event.payload, - Event::Error { - code: protocol::ErrorCode::NotPaused, - .. - } + Event::CommandAcknowledged { acknowledgement } + if acknowledgement.command == protocol::WorkerCommandKind::Resume + && acknowledgement.disposition + == protocol::WorkerCommandDisposition::InvalidState ) })); assert_eq!(call_count.load(Ordering::SeqCst), 3); diff --git a/crates/worker/Cargo.toml b/crates/worker/Cargo.toml index 87437d17..87046834 100644 --- a/crates/worker/Cargo.toml +++ b/crates/worker/Cargo.toml @@ -20,6 +20,7 @@ protocol = { workspace = true, features = ["json-schema"] } client = { workspace = true } serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } +sha2 = { workspace = true } reqwest = { version = "0.13", default-features = false, features = ["blocking", "native-tls"] } thiserror = { workspace = true } tokio = { workspace = true, features = ["fs", "io-util", "macros", "net", "process", "rt-multi-thread", "signal", "sync", "time"] } diff --git a/crates/worker/examples/worker_protocol.rs b/crates/worker/examples/worker_protocol.rs index 1e2c618f..cd00370a 100644 --- a/crates/worker/examples/worker_protocol.rs +++ b/crates/worker/examples/worker_protocol.rs @@ -101,7 +101,10 @@ async fn main() -> Result<(), Box> { // Send a run method handle - .send(Method::run_text("What is the capital of France?")) + .send(Method::submit_text( + protocol::new_submission_request_id(), + "What is the capital of France?", + )) .await?; // Wait for completion diff --git a/crates/worker/src/controller.rs b/crates/worker/src/controller.rs index 70584074..1655a387 100644 --- a/crates/worker/src/controller.rs +++ b/crates/worker/src/controller.rs @@ -1,3 +1,4 @@ +use std::collections::VecDeque; use std::path::{Path, PathBuf}; use std::sync::Arc; use std::sync::atomic::Ordering; @@ -5,7 +6,7 @@ use std::sync::atomic::Ordering; use agen::EngineError; use agen::llm_client::client::LlmClient; use session_store::WorkerMetadataStore; -use session_store::{LogEntry, SessionExtension, Store}; +use session_store::{LogEntry, Store}; use tokio::sync::{broadcast, mpsc, oneshot}; use crate::discovery::WorkerDiscovery; @@ -16,23 +17,21 @@ use crate::ipc::notify_buffer::NotifyBuffer; use crate::ipc::server::SocketServer; use crate::runtime::dir::RuntimeDir; use crate::segment_log_sink::SegmentLogSink; -use crate::shared_state::WorkerSharedState; +use crate::shared_state::{WorkerCommandAdmission, WorkerSharedState}; use crate::shutdown_after_idle::{ ShutdownAfterIdleRequest, TicketIntakeReadyShutdownHook, is_ticket_intake_role, take_shutdown_request_after_status, }; use crate::spawn::registry::SpawnedWorkerRegistry; use crate::spawn::tool::sub_worker_spawn_tool; -use crate::worker::{ - SystemItemCommitter, WORKER_INPUT_SUBMISSION_EXTENSION_DOMAIN, Worker, WorkerError, - WorkerRunResult, -}; +use crate::worker::{SystemItemCommitter, Worker, WorkerError, WorkerRunResult}; use protocol::{ AlertLevel, AlertSource, CommandEvent as ProtocolCommandEvent, CommandSnapshot as ProtocolCommandSnapshot, CommandStatus as ProtocolCommandStatus, CommandStream as ProtocolCommandStream, CommandStreamSlice as ProtocolCommandStreamSlice, - ErrorCode, Event, Method, RewindTargetId, RunResult, Segment, TurnResult, UploadedFileRef, - WorkerStatus, + ErrorCode, Event, Method, RewindTargetId, RunResult, TurnResult, UploadedFileRef, + WorkerBusyState, WorkerCommandAcknowledgement, WorkerCommandDisposition, WorkerCommandEnvelope, + WorkerCommandKind, WorkerMaintenanceState, WorkerRunState, WorkerState, WorkerStatus, }; use workdir::{ CommandEvent as WorkdirCommandEvent, CommandSnapshot as WorkdirCommandSnapshot, @@ -58,6 +57,7 @@ pub struct WorkerHandle { spawned_registry: Arc, artifact_store: Arc, session_id: session_store::SessionId, + pending_activations: Arc>, } impl WorkerHandle { @@ -131,10 +131,17 @@ impl WorkerHandle { let in_flight = snapshot_from_guard(&in_flight_guard); (entries, entry_rx, in_flight) }; + let mut session = + session_store::public_snapshot::project_current_session_snapshot(&entries); + session.pending_submissions = self + .pending_activations + .lock() + .expect("pending activation state poisoned") + .snapshot(); let event = Event::Snapshot { - session: session_store::public_snapshot::project_current_session_snapshot(&entries), + session, greeting: self.shared_state.greeting.clone(), - status: self.shared_state.get_status(), + state: self.shared_state.snapshot(), in_flight, internal_workers: self.spawned_registry.internal_worker_snapshots(), }; @@ -174,15 +181,101 @@ impl WorkerHandle { } } +fn command_admission_disposition( + admission: WorkerCommandAdmission, +) -> Result<(), WorkerCommandDisposition> { + match admission { + WorkerCommandAdmission::Accepted => Ok(()), + WorkerCommandAdmission::Retry | WorkerCommandAdmission::StaleCommandId => { + Err(WorkerCommandDisposition::StaleCommandId) + } + WorkerCommandAdmission::Conflict => Err(WorkerCommandDisposition::Conflict), + WorkerCommandAdmission::ExecutionGenerationMismatch => { + Err(WorkerCommandDisposition::StaleExecutionGeneration) + } + WorkerCommandAdmission::StateRevisionMismatch => { + Err(WorkerCommandDisposition::StaleWorkerStateRevision) + } + } +} + +fn validate_command( + envelope: WorkerCommandEnvelope, + kind: WorkerCommandKind, + shared_state: &WorkerSharedState, +) -> Result<(), WorkerCommandDisposition> { + command_admission_disposition(shared_state.admit_command(envelope, kind, true)) +} + +fn validate_shutdown_command( + envelope: WorkerCommandEnvelope, + shared_state: &WorkerSharedState, +) -> Result<(), WorkerCommandDisposition> { + match shared_state.admit_command(envelope, WorkerCommandKind::Shutdown, false) { + WorkerCommandAdmission::Accepted | WorkerCommandAdmission::Retry => Ok(()), + admission => command_admission_disposition(admission), + } +} + +fn acknowledge_command( + working_event_tx: &broadcast::Sender, + shared_state: &WorkerSharedState, + command_id: u64, + command: WorkerCommandKind, + disposition: WorkerCommandDisposition, +) { + shared_state.complete_command(command_id, command, disposition); + let _ = working_event_tx.send(Event::CommandAcknowledged { + acknowledgement: WorkerCommandAcknowledgement { + command_id, + command, + disposition, + state: shared_state.snapshot(), + }, + }); +} + +fn reject_invalid_command_state( + working_event_tx: &broadcast::Sender, + shared_state: &WorkerSharedState, + envelope: WorkerCommandEnvelope, + command: WorkerCommandKind, +) { + acknowledge_command( + working_event_tx, + shared_state, + envelope.command_id, + command, + WorkerCommandDisposition::InvalidState, + ); +} + +async fn set_controller_state( + shared_state: &Arc, + runtime_dir: &RuntimeDir, + working_event_tx: &broadcast::Sender, + state: WorkerState, +) -> protocol::WorkerStateSnapshot { + let snapshot = shared_state.transition(state); + let _ = runtime_dir.write_status(shared_state).await; + let _ = working_event_tx.send(Event::WorkerState { + snapshot: snapshot.clone(), + }); + snapshot +} + async fn set_controller_status( shared_state: &Arc, runtime_dir: &RuntimeDir, working_event_tx: &broadcast::Sender, status: WorkerStatus, ) { - shared_state.set_status(status); - let _ = runtime_dir.write_status(shared_state).await; - let _ = working_event_tx.send(Event::Status { status }); + let state = match status { + WorkerStatus::Idle | WorkerStatus::Stopped => WorkerState::Idle, + WorkerStatus::Running => WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)), + WorkerStatus::Paused => WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused)), + }; + set_controller_state(shared_state, runtime_dir, working_event_tx, state).await; } async fn finish_controller_run( @@ -213,21 +306,143 @@ async fn finish_controller_run( /// `Worker::*` entry point — `RunForNotification` carries none because /// `worker.run_for_notification()` drains the NotifyBuffer on its own. enum PendingRun { - Run(Vec), - RunTracked { - input: Vec, - extension: SessionExtension, - }, + Submit(crate::worker::PendingSubmission), /// Self-initiated turn kicked from the notify buffer. The carried /// `InvokeKind` is the trigger that flipped the Worker from IDLE /// (Notify or WorkerEvent) and is recorded by the Invoke marker /// committed at the start of `worker.run_for_notification`. - RunForNotification(protocol::InvokeKind), + RunForNotification { + invoke_kind: protocol::InvokeKind, + notification_request_id: Option, + }, Resume, } +fn resolved_input_source( + pending_submissions: &crate::worker::PendingSubmissionHandle, + source: &protocol::AuthenticatedInputSource, +) -> (String, session_store::LoggedSessionHistoryOrigin) { + if matches!(source, protocol::AuthenticatedInputSource::UntrustedWire) { + return ( + pending_submissions.direct_client_namespace(), + session_store::LoggedSessionHistoryOrigin::LegacyUnknown, + ); + } + ( + source.namespace(), + crate::worker::authenticated_input_provenance(source), + ) +} + +fn durable_parent_notification_target( + pending_submissions: crate::worker::PendingSubmissionHandle, + notify_buffer: NotifyBuffer, +) -> crate::spawn::tool::ParentNotificationTarget { + crate::spawn::tool::ParentNotificationTarget::Durable(Arc::new(move |method| { + let Method::NotifyTracked { + notification_request_id, + message, + auto_run, + source, + } = method + else { + return; + }; + let (source_namespace, provenance) = resolved_input_source(&pending_submissions, &source); + match pending_submissions.accept_notification_from_source( + notification_request_id.clone(), + message, + source_namespace.clone(), + provenance, + auto_run, + ) { + Ok(_) if !auto_run => { + stage_pending_notification( + &pending_submissions, + ¬ify_buffer, + &source_namespace, + ¬ification_request_id, + ); + } + Ok(_) => {} + Err(error) => { + tracing::warn!(%error, "failed to durably accept SubWorker notification"); + } + } + })) +} + +fn stage_pending_notification( + pending_submissions: &crate::worker::PendingSubmissionHandle, + notify_buffer: &NotifyBuffer, + source_namespace: &str, + notification_request_id: &str, +) -> bool { + let Some(notification) = + pending_submissions.prepare_notification(source_namespace, notification_request_id) + else { + return false; + }; + let extension = pending_submissions.notification_activation_extension(); + notify_buffer.push_durable_notify( + notification.message, + notification.auto_run, + notification.provenance, + extension, + ); + true +} + +fn stage_oldest_passive_notification( + pending_submissions: &crate::worker::PendingSubmissionHandle, + notify_buffer: &NotifyBuffer, +) -> bool { + pending_submissions + .next_passive_notification_identity() + .is_some_and(|(source_namespace, request_id)| { + stage_pending_notification( + pending_submissions, + notify_buffer, + &source_namespace, + &request_id, + ) + }) +} + +fn prepare_pending_run( + pending_submissions: &crate::worker::PendingSubmissionHandle, + notify_buffer: &NotifyBuffer, + fence: Option<(u64, &str)>, +) -> Result, crate::worker::PendingSubmissionError> { + let staged_passive_notification = pending_submissions.activating_passive_notification_id(); + Ok(match pending_submissions.prepare_next_activation(fence)? { + Some(crate::worker::PendingActivation::Submission(submission)) => { + if staged_passive_notification.is_some() { + let extension = pending_submissions.notification_activation_extension(); + debug_assert!(notify_buffer.replace_durable_notification_extension(extension)); + } + Some(PendingRun::Submit(submission)) + } + Some(crate::worker::PendingActivation::Notification(notification)) => { + let extension = pending_submissions.notification_activation_extension(); + let notification_request_id = notification.notification_request_id.clone(); + notify_buffer.push_durable_notify( + notification.message, + notification.auto_run, + notification.provenance, + extension, + ); + Some(PendingRun::RunForNotification { + invoke_kind: protocol::InvokeKind::Notify, + notification_request_id: Some(notification_request_id), + }) + } + None => None, + }) +} + impl PendingRun { - /// Whether this turn was kicked off by the parent (via `Method::Run` + /// Whether this turn was kicked off by the parent (via `Method::Submit` /// or `Method::Resume`). Used by [`drive_turn`] to gate upward /// `WorkerEvent::TurnEnded` / `WorkerEvent::Errored` reports so the parent /// only sees completion signals for work it actually delegated. @@ -235,16 +450,12 @@ impl PendingRun { /// notify buffer (Notify / inbound WorkerEvent) and stays silent. fn is_parent_originated(&self) -> bool { match self { - PendingRun::Run(_) | PendingRun::RunTracked { .. } | PendingRun::Resume => true, - PendingRun::RunForNotification(_) => false, + PendingRun::Submit(_) | PendingRun::Resume => true, + PendingRun::RunForNotification { .. } => false, } } } -fn should_auto_run_notification(status: WorkerStatus, auto_run: bool) -> bool { - auto_run && status == WorkerStatus::Idle -} - // --------------------------------------------------------------------------- // WorkerController — actor that owns a Worker // --------------------------------------------------------------------------- @@ -514,6 +725,7 @@ impl WorkerController { runtime_base.to_path_buf(), spawned_registry.clone(), Some(method_tx.downgrade()), + None, ) .await?; if let Some(session) = fs_for_view.as_ref() { @@ -537,12 +749,24 @@ impl WorkerController { // === 4. Initial runtime files + WorkerSharedState + WorkerHandle + // SocketServer === let manifest_toml = toml::to_string_pretty(worker.manifest()).unwrap_or_default(); + worker + .recover_unfinished_compaction() + .await + .map_err(|error| std::io::Error::other(error.to_string()))?; let greeting = build_greeting(&worker); - let shared_state = Arc::new(WorkerSharedState::new( + let execution_generation = runtime_dir + .path() + .file_name() + .and_then(|name| name.to_str()) + .and_then(|name| name.parse::().ok()) + .filter(|generation| *generation > 0) + .unwrap_or(1); + let shared_state = Arc::new(WorkerSharedState::new_with_generation( worker.manifest().worker.name.clone(), worker.segment_id(), manifest_toml.clone(), greeting, + execution_generation, )); if let Some(fs_for_view) = fs_for_view { shared_state.set_fs_view(crate::fs_view::WorkerFsView::new(fs_for_view)); @@ -552,6 +776,7 @@ impl WorkerController { let artifact_store: Arc = Arc::new(worker.store().clone()); let session_id = worker.session_id(); + let pending_activations = worker.pending_activation_state(); let handle = WorkerHandle { method_tx, working_event_tx: working_event_tx.clone(), @@ -563,6 +788,7 @@ impl WorkerController { spawned_registry: spawned_registry.clone(), artifact_store, session_id, + pending_activations, }; let socket_server = match transport { @@ -911,6 +1137,7 @@ pub(crate) async fn register_worker_tools( runtime_base: PathBuf, spawned_registry: Arc, parent_method_tx: Option>, + inherited_workdir_tool_broker: Option, ) -> std::io::Result> where C: LlmClient + Clone + 'static, @@ -919,21 +1146,26 @@ where // Worker-immutable snapshots taken before the mutable worker borrow // below so the worker borrow doesn't conflict with reads on `worker`. let feature_config = worker.manifest().feature.clone(); + let mut workdir_tool_broker = inherited_workdir_tool_broker; if feature_config.manage_workdir.enabled && worker.workdir_session().is_none() { let workspace_client = worker.workspace_client_handle(); - worker.bind_workdir_session(Some(workdir::delegation_capable_session( + let broker = workdir::WorkdirToolBroker::new( crate::feature::builtin::manage_workdir::WorkspaceAttachedWorkdirSession::handle( workspace_client, ), - ))); - } - if feature_config.sub_worker.enabled + ); + worker.bind_workdir_session(Some(broker.tool_session())); + workdir_tool_broker = Some(broker); + } else if workdir_tool_broker.is_none() && let Some(existing) = worker.workdir_session().cloned() - && !existing.is_delegation_capable() { - worker.bind_workdir_session(Some(workdir::delegation_capable_session(existing))); + let broker = workdir::WorkdirToolBroker::new(existing); + worker.bind_workdir_session(Some(broker.tool_session())); + workdir_tool_broker = Some(broker); } - let worker_workdir = worker.workdir_session().cloned(); + let worker_workdir = workdir_tool_broker + .as_ref() + .map(workdir::WorkdirToolBroker::tool_session); let local_filesystem = worker.local_working_directory().cloned(); let local_workspace_root = local_filesystem.as_ref().map(|local| local.root.clone()); let task_feature = worker.task_feature(); @@ -942,11 +1174,17 @@ where let spawner_name = worker.manifest().worker.name.clone(); let spawner_manifest = worker.manifest().clone(); let spawner_workspace_context = worker.workspace_context_handle(); - let parent_notifications = parent_method_tx - .map(crate::spawn::tool::ParentNotificationTarget::Controller) - .unwrap_or_else(|| { - crate::spawn::tool::ParentNotificationTarget::Buffer(worker.notify_buffer_handle()) - }); + let pending_submissions = worker.pending_submission_handle(); + let notify_buffer = worker.notify_buffer_handle(); + let durable_parent_notifications = + durable_parent_notification_target(pending_submissions.clone(), notify_buffer.clone()); + let parent_notifications = match parent_method_tx { + Some(sender) => crate::spawn::tool::ParentNotificationTarget::with_controller_fallback( + sender, + durable_parent_notifications, + ), + None => durable_parent_notifications, + }; let prompts = worker.prompts().clone(); let paste_store = worker.store().clone(); let paste_session_id = worker.session_id(); @@ -1094,8 +1332,17 @@ where "manage Workdir tools require Backend Workspace API authority", )); } + let shutdown_registry = spawned_registry.clone(); + let reopen_registry = spawned_registry.clone(); feature_registry.add_module( - crate::feature::builtin::manage_workdir::manage_workdir_feature(workspace_client), + crate::feature::builtin::manage_workdir::ManageWorkdirFeature::with_child_lifecycle( + workspace_client, + Arc::new(move || { + let child_registry = shutdown_registry.clone(); + Box::pin(async move { child_registry.shutdown_internal().await }) + }), + Arc::new(move || reopen_registry.reopen_internal()), + ), ); } if feature_config.workspace_worker_discovery.enabled { @@ -1157,7 +1404,6 @@ where } let host_worker_observation_provider = worker.worker_observation_provider(); - let source_workdir_session = worker.workdir_session().cloned(); { let workspace_client = worker.workspace_client_handle(); let engine = worker.engine_mut(); @@ -1199,7 +1445,7 @@ where runtime_base.clone(), bash_output_dir.clone(), spawner_workspace_root, - source_workdir_session, + workdir_tool_broker, spawned_registry.clone(), spawner_manifest, prompts, @@ -1289,10 +1535,22 @@ async fn controller_loop( discovery_cwd, spawned_registry.clone(), ); - let mut pending: Option = None; + let pending_submissions = worker.pending_submission_handle(); + stage_oldest_passive_notification(&pending_submissions, ¬ify_buffer); + let mut pending = match prepare_pending_run(&pending_submissions, ¬ify_buffer, None) { + Ok(pending) => pending, + Err(error) => { + let _ = working_event_tx.send(Event::Error { + code: ErrorCode::Internal, + message: error.to_string(), + }); + None + } + }; - loop { - // Top-of-iteration: if an event handler staged a run, fire it + let mut deferred_methods = VecDeque::new(); + + 'controller: loop { // here so the status flip → drive_turn → finish sequence lives // in one place, regardless of which Method caused it. if let Some(run) = pending.take() { @@ -1306,8 +1564,8 @@ async fn controller_loop( // interrupted/error turn from being carried into the next snapshot. worker.clear_in_flight_events(); let parent_originated = run.is_parent_originated(); - let user_input_run = matches!(&run, PendingRun::Run(_) | PendingRun::RunTracked { .. }); - if !user_input_run { + let user_input_submit = matches!(&run, PendingRun::Submit(_)); + if !user_input_submit { set_controller_status( &shared_state, &runtime_dir, @@ -1316,38 +1574,25 @@ async fn controller_loop( ) .await; } - let (mut new_status, shutdown) = match run { - PendingRun::Run(input) => { + let notification_request_id = match &run { + PendingRun::RunForNotification { + notification_request_id, + .. + } => notification_request_id.clone(), + _ => None, + }; + let passive_notification_request_id = + pending_submissions.activating_passive_notification_id(); + let (mut new_status, shutdown, may_drain_pending) = match run { + PendingRun::Submit(submission) => { let (input_commit_tx, input_commit_rx) = oneshot::channel(); + let committed_submission = submission.clone(); + let extension = pending_submissions.activation_extension(); drive_turn( worker.run_with_input_extensions_and_commit_hook( - input, - Vec::new(), - move || { - let _ = input_commit_tx.send(()); - }, - ), - &mut method_rx, - &working_event_tx, - &cancel_tx, - &pause_tx, - &shared_state, - &runtime_dir, - Some(input_commit_rx), - ¬ify_buffer, - self_parent_socket.as_ref(), - &spawner_name, - &spawned_registry, - parent_originated, - ) - .await - } - PendingRun::RunTracked { input, extension } => { - let (input_commit_tx, input_commit_rx) = oneshot::channel(); - drive_turn( - worker.run_with_input_extensions_and_commit_hook( - input, + submission.input, vec![extension], + submission.provenance, move || { let _ = input_commit_tx.send(()); }, @@ -1358,8 +1603,9 @@ async fn controller_loop( &pause_tx, &shared_state, &runtime_dir, - Some(input_commit_rx), + Some((input_commit_rx, committed_submission)), ¬ify_buffer, + &pending_submissions, self_parent_socket.as_ref(), &spawner_name, &spawned_registry, @@ -1367,9 +1613,9 @@ async fn controller_loop( ) .await } - PendingRun::RunForNotification(kind) => { + PendingRun::RunForNotification { invoke_kind, .. } => { drive_turn( - worker.run_for_notification(kind), + worker.run_for_notification(invoke_kind), &mut method_rx, &working_event_tx, &cancel_tx, @@ -1378,6 +1624,7 @@ async fn controller_loop( &runtime_dir, None, ¬ify_buffer, + &pending_submissions, self_parent_socket.as_ref(), &spawner_name, &spawned_registry, @@ -1396,6 +1643,7 @@ async fn controller_loop( &runtime_dir, None, ¬ify_buffer, + &pending_submissions, self_parent_socket.as_ref(), &spawner_name, &spawned_registry, @@ -1404,10 +1652,35 @@ async fn controller_loop( .await } }; - if !shutdown && new_status == WorkerStatus::Idle && notify_buffer.has_auto_run_pending() + if let Some(notification_request_id) = + notification_request_id.or(passive_notification_request_id) { - pending = Some(PendingRun::RunForNotification(protocol::InvokeKind::Notify)); - new_status = WorkerStatus::Running; + pending_submissions.finish_notification_activation(¬ification_request_id); + stage_oldest_passive_notification(&pending_submissions, ¬ify_buffer); + } + + if !shutdown && may_drain_pending && new_status == WorkerStatus::Idle { + match prepare_pending_run(&pending_submissions, ¬ify_buffer, None) { + Ok(Some(next)) => { + pending = Some(next); + new_status = WorkerStatus::Running; + } + Ok(None) => { + if notify_buffer.has_auto_run_pending() { + pending = Some(PendingRun::RunForNotification { + invoke_kind: protocol::InvokeKind::Notify, + notification_request_id: None, + }); + new_status = WorkerStatus::Running; + } + } + Err(error) => { + let _ = working_event_tx.send(Event::Error { + code: ErrorCode::Internal, + message: error.to_string(), + }); + } + } } finish_controller_run( &mut worker, @@ -1428,149 +1701,497 @@ async fn controller_loop( continue; } - let method = match method_rx.recv().await { - Some(m) => m, - None => break, + let method = if let Some(method) = deferred_methods.pop_front() { + method + } else { + match method_rx.recv().await { + Some(method) => method, + None => break, + } }; match method { - Method::Run { input } => { - if shared_state.get_status() == WorkerStatus::Running { - // Defensive: the inner select! inside drive_turn - // already rejects `Run` while a turn is live, so - // this branch is only reachable across a race window - // around status flips. - let _ = working_event_tx.send(Event::Error { - code: ErrorCode::AlreadyRunning, - message: "Worker is already executing a turn".into(), - }); - continue; - } - // Stage the run without a speculative user-message echo. - // `Worker::run` validates the input, commits - // `LogEntry::AnnotatedUserInput`, and the session-log sink turns that - // committed entry into the live `Event::UserMessage`. That - // keeps every client ordered against `SegmentStart` replay and - // makes persisted history the single source of visible user - // input. Paused→Run cleanup (orphan tool_result closure + - // interrupt system note) is applied inside `Worker::run` itself - // when the worker's `last_run_interrupted` flag is set. - pending = Some(PendingRun::Run(input)); - } - - Method::RunTracked { + Method::Submit { + submission_request_id, input, - submission_id, } => { - // Runtime-correlated submissions retain their opaque id in the - // same durable UserInput record used for Flow state. - let extension = SessionExtension::new( - WORKER_INPUT_SUBMISSION_EXTENSION_DOMAIN, - serde_json::json!({ "submission_id": submission_id }), - ); - pending = Some(PendingRun::RunTracked { input, extension }); + let request_id = submission_request_id.clone(); + match pending_submissions.accept_from_source( + submission_request_id, + input, + pending_submissions.direct_client_namespace(), + session_store::LoggedSessionHistoryOrigin::LegacyUnknown, + true, + ) { + Ok(acceptance) => { + if let Some(activation) = acceptance.activation { + pending = Some(PendingRun::Submit(activation)); + } else { + let _ = working_event_tx.send(Event::SubmissionAccepted { + submission_request_id: acceptance.submission_request_id, + submission_id: acceptance.submission_id, + disposition: acceptance.disposition, + }); + } + } + Err(error) => { + let _ = working_event_tx.send(Event::SubmissionRejected { + submission_request_id: request_id, + message: error.to_string(), + }); + } + } } - - Method::Notify { message, auto_run } => { - // Client-side live echo is delivered as `Event::SystemItem` - // once the interceptor commits the corresponding - // `LogEntry::AnnotatedSystemItem` entry — drained out of the - // notify buffer + broadcast through the sink. No - // separate echo here. - worker.push_notify(message, auto_run); - // RUNNING: the in-flight turn drains the buffer at its next - // pending_history_appends; if an auto-run notification remains - // at turn end, the Controller stages a follow-up notification - // turn. Paused notifications remain queued until Resume/Run. - // IDLE: `auto_run` notifications stage RunForNotification; - // weak progress notices stay queued until an explicit run. - if should_auto_run_notification(shared_state.get_status(), auto_run) { - pending = Some(PendingRun::RunForNotification(protocol::InvokeKind::Notify)); + Method::SubmitTracked { + submission_request_id, + input, + source, + } => { + let request_id = submission_request_id.clone(); + let (source_namespace, provenance) = + resolved_input_source(&pending_submissions, &source); + match pending_submissions.accept_from_source( + submission_request_id, + input, + source_namespace, + provenance, + true, + ) { + Ok(acceptance) => { + if let Some(activation) = acceptance.activation { + pending = Some(PendingRun::Submit(activation)); + } else { + let _ = working_event_tx.send(Event::SubmissionAccepted { + submission_request_id: acceptance.submission_request_id, + submission_id: acceptance.submission_id, + disposition: acceptance.disposition, + }); + } + } + Err(error) => { + let _ = working_event_tx.send(Event::SubmissionRejected { + submission_request_id: request_id, + message: error.to_string(), + }); + } } } - Method::Resume => { - if shared_state.get_status() != WorkerStatus::Paused { + Method::Notify { + notification_request_id, + message, + auto_run, + } => { + let request_id = notification_request_id.clone(); + let source_namespace = pending_submissions.direct_client_namespace(); + match pending_submissions.accept_notification_from_source( + notification_request_id, + message, + source_namespace.clone(), + session_store::LoggedSessionHistoryOrigin::LegacyUnknown, + auto_run, + ) { + Ok(_) if auto_run => { + match prepare_pending_run(&pending_submissions, ¬ify_buffer, None) { + Ok(Some(next)) => pending = Some(next), + Ok(None) => {} + Err(error) => { + let _ = working_event_tx.send(Event::Error { + code: ErrorCode::Internal, + message: error.to_string(), + }); + } + } + } + Ok(_) => { + stage_pending_notification( + &pending_submissions, + ¬ify_buffer, + &source_namespace, + &request_id, + ); + } + Err(error) => { + let _ = working_event_tx.send(Event::Error { + code: ErrorCode::InvalidRequest, + message: error.to_string(), + }); + } + } + } + + Method::NotifyTracked { + notification_request_id, + message, + auto_run, + source, + } => { + let request_id = notification_request_id.clone(); + let (source_namespace, provenance) = + resolved_input_source(&pending_submissions, &source); + match pending_submissions.accept_notification_from_source( + notification_request_id, + message, + source_namespace.clone(), + provenance, + auto_run, + ) { + Ok(_) if auto_run => { + match prepare_pending_run(&pending_submissions, ¬ify_buffer, None) { + Ok(Some(next)) => pending = Some(next), + Ok(None) => {} + Err(error) => { + let _ = working_event_tx.send(Event::Error { + code: ErrorCode::Internal, + message: error.to_string(), + }); + } + } + } + Ok(_) => { + stage_pending_notification( + &pending_submissions, + ¬ify_buffer, + &source_namespace, + &request_id, + ); + } + Err(error) => { + let _ = working_event_tx.send(Event::Error { + code: ErrorCode::InvalidRequest, + message: error.to_string(), + }); + } + } + } + + Method::ListPendingSubmissions => { + let _ = working_event_tx.send(Event::PendingSubmissionsChanged { + pending: pending_submissions.snapshot(), + }); + } + Method::CancelPendingSubmission { + submission_id, + expected_revision, + } => match pending_submissions.cancel(&submission_id, expected_revision) { + Ok(pending_snapshot) => { + let _ = working_event_tx.send(Event::PendingSubmissionsChanged { + pending: pending_snapshot, + }); + } + Err(error) => { let _ = working_event_tx.send(Event::Error { - code: ErrorCode::NotPaused, - message: "Worker is not paused".into(), + code: ErrorCode::InvalidRequest, + message: error.to_string(), + }); + } + }, + Method::ClearPendingSubmissions { expected_revision } => { + match pending_submissions.clear(expected_revision) { + Ok(pending_snapshot) => { + let _ = working_event_tx.send(Event::PendingSubmissionsChanged { + pending: pending_snapshot, + }); + } + Err(error) => { + let _ = working_event_tx.send(Event::Error { + code: ErrorCode::InvalidRequest, + message: error.to_string(), + }); + } + } + } + Method::ContinuePending { + expected_revision, + expected_head_id, + } => { + if shared_state.catalog_status() != WorkerStatus::Idle { + let _ = working_event_tx.send(Event::Error { + code: ErrorCode::InvalidRequest, + message: "ContinuePending requires an idle Worker; Resume or Cancel a paused run first".into(), }); continue; } + match prepare_pending_run( + &pending_submissions, + ¬ify_buffer, + Some((expected_revision, &expected_head_id)), + ) { + Ok(Some(next)) => pending = Some(next), + Ok(None) => { + let _ = working_event_tx.send(Event::Error { + code: ErrorCode::InvalidRequest, + message: "pending activation queue is empty".into(), + }); + } + Err(error) => { + let _ = working_event_tx.send(Event::Error { + code: ErrorCode::Internal, + message: error.to_string(), + }); + } + } + } + Method::Resume { command } => { + if let Err(disposition) = + validate_command(command, WorkerCommandKind::Resume, &shared_state) + { + acknowledge_command( + &working_event_tx, + &shared_state, + command.command_id, + WorkerCommandKind::Resume, + disposition, + ); + continue; + } + if !matches!( + shared_state.snapshot().state, + WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused)) + ) { + reject_invalid_command_state( + &working_event_tx, + &shared_state, + command, + WorkerCommandKind::Resume, + ); + continue; + } + set_controller_state( + &shared_state, + &runtime_dir, + &working_event_tx, + WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)), + ) + .await; + acknowledge_command( + &working_event_tx, + &shared_state, + command.command_id, + WorkerCommandKind::Resume, + WorkerCommandDisposition::Accepted, + ); pending = Some(PendingRun::Resume); } - Method::Cancel => match shared_state.get_status() { - WorkerStatus::Paused => match worker.cancel_paused_turn() { + Method::Cancel { command } => { + if let Err(disposition) = + validate_command(command, WorkerCommandKind::Cancel, &shared_state) + { + acknowledge_command( + &working_event_tx, + &shared_state, + command.command_id, + WorkerCommandKind::Cancel, + disposition, + ); + continue; + } + if !matches!( + shared_state.snapshot().state, + WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused)) + ) { + reject_invalid_command_state( + &working_event_tx, + &shared_state, + command, + WorkerCommandKind::Cancel, + ); + continue; + } + set_controller_state( + &shared_state, + &runtime_dir, + &working_event_tx, + WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Cancelling)), + ) + .await; + acknowledge_command( + &working_event_tx, + &shared_state, + command.command_id, + WorkerCommandKind::Cancel, + WorkerCommandDisposition::Accepted, + ); + match worker.cancel_paused_turn() { Ok(()) => { worker.clear_in_flight_events(); - set_controller_status( + set_controller_state( &shared_state, &runtime_dir, &working_event_tx, - WorkerStatus::Idle, + WorkerState::Idle, ) .await; } Err(error) => { + set_controller_state( + &shared_state, + &runtime_dir, + &working_event_tx, + WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused)), + ) + .await; let _ = working_event_tx.send(Event::Error { code: worker_error_code(&error), message: error.to_string(), }); } - }, - WorkerStatus::Idle | WorkerStatus::Stopped => { - let _ = working_event_tx.send(Event::Error { - code: ErrorCode::NotRunning, - message: "Worker is not running".into(), - }); - } - WorkerStatus::Running => { - // Running turns receive Cancel through drive_turn; this is - // only reachable across a defensive race window. - let _ = cancel_tx.try_send(()); - } - }, - - Method::Pause => { - // Already paused → idempotent no-op. Otherwise the - // Worker is Idle (Running turns go through `drive_turn`, - // not this outer match), so there is nothing to pause. - if shared_state.get_status() != WorkerStatus::Paused { - let _ = working_event_tx.send(Event::Error { - code: ErrorCode::NotRunning, - message: "Worker is not running".into(), - }); } } - Method::Compact => match shared_state.get_status() { - WorkerStatus::Idle => { - if let Err(error) = worker.manual_compact().await { - let _ = working_event_tx.send(Event::Error { - code: worker_error_code(&error), - message: error.to_string(), - }); - } + Method::Pause { command } => { + if let Err(disposition) = + validate_command(command, WorkerCommandKind::Pause, &shared_state) + { + acknowledge_command( + &working_event_tx, + &shared_state, + command.command_id, + WorkerCommandKind::Pause, + disposition, + ); + } else { + reject_invalid_command_state( + &working_event_tx, + &shared_state, + command, + WorkerCommandKind::Pause, + ); } - WorkerStatus::Paused => { - let _ = working_event_tx.send(Event::Error { - code: ErrorCode::InvalidRequest, - message: "Cannot compact while the Worker is paused; resume or start a fresh turn first" - .into(), - }); - } - WorkerStatus::Running | WorkerStatus::Stopped => { - let _ = working_event_tx.send(Event::Error { - code: ErrorCode::AlreadyRunning, - message: - "Worker is already executing a turn; compact can only run while idle" - .into(), - }); - } - }, + } - Method::ListRewindTargets => match shared_state.get_status() { + Method::Compact { command } => { + if let Err(disposition) = + validate_command(command, WorkerCommandKind::Compact, &shared_state) + { + acknowledge_command( + &working_event_tx, + &shared_state, + command.command_id, + WorkerCommandKind::Compact, + disposition, + ); + continue; + } + if !matches!(shared_state.snapshot().state, WorkerState::Idle) { + reject_invalid_command_state( + &working_event_tx, + &shared_state, + command, + WorkerCommandKind::Compact, + ); + continue; + } + set_controller_state( + &shared_state, + &runtime_dir, + &working_event_tx, + WorkerState::Busy(WorkerBusyState::Maintenance( + WorkerMaintenanceState::Compacting, + )), + ) + .await; + acknowledge_command( + &working_event_tx, + &shared_state, + command.command_id, + WorkerCommandKind::Compact, + WorkerCommandDisposition::Accepted, + ); + let (cancel_tx, cancel_rx) = tokio::sync::watch::channel(false); + let mut shutdown_after_compaction = false; + let result = { + let mut compact = Box::pin(worker.manual_compact_with_cancel(cancel_rx)); + loop { + tokio::select! { + result = &mut compact => break result, + method = method_rx.recv() => { + match method { + Some(Method::Cancel { command }) => { + if let Err(disposition) = validate_command( + command, + WorkerCommandKind::Cancel, + &shared_state, + ) { + acknowledge_command( + &working_event_tx, + &shared_state, + command.command_id, + WorkerCommandKind::Cancel, + disposition, + ); + continue; + } + acknowledge_command( + &working_event_tx, + &shared_state, + command.command_id, + WorkerCommandKind::Cancel, + WorkerCommandDisposition::Accepted, + ); + let _ = cancel_tx.send(true); + } + Some(Method::Shutdown { command }) => { + if let Err(disposition) = + validate_shutdown_command(command, &shared_state) + { + acknowledge_command( + &working_event_tx, + &shared_state, + command.command_id, + WorkerCommandKind::Shutdown, + disposition, + ); + continue; + } + shutdown_after_compaction = true; + acknowledge_command( + &working_event_tx, + &shared_state, + command.command_id, + WorkerCommandKind::Shutdown, + WorkerCommandDisposition::Accepted, + ); + let _ = cancel_tx.send(true); + } + Some(method) => deferred_methods.push_back(method), + None => { + shutdown_after_compaction = true; + let _ = cancel_tx.send(true); + } + } + } + } + } + }; + if !matches!( + result, + Err(WorkerError::Store(_)) + | Err(WorkerError::WorkerStore(_)) + | Err(WorkerError::InvalidState(_)) + ) { + set_controller_state( + &shared_state, + &runtime_dir, + &working_event_tx, + WorkerState::Idle, + ) + .await; + } + if let Err(error) = result { + let _ = working_event_tx.send(Event::Error { + code: worker_error_code(&error), + message: error.to_string(), + }); + } + if shutdown_after_compaction { + let _ = working_event_tx.send(Event::Shutdown); + break 'controller; + } + } + + Method::ListRewindTargets => match shared_state.catalog_status() { WorkerStatus::Idle | WorkerStatus::Paused => { emit_rewind_targets(&worker, &working_event_tx) } @@ -1586,7 +2207,7 @@ async fn controller_loop( Method::RewindTo { target, expected_head_entries, - } => match shared_state.get_status() { + } => match shared_state.catalog_status() { WorkerStatus::Idle => { if apply_rewind( &mut worker, @@ -1597,10 +2218,8 @@ async fn controller_loop( .await { worker.clear_in_flight_events(); - shared_state.set_status(WorkerStatus::Idle); - let _ = working_event_tx.send(Event::Status { - status: WorkerStatus::Idle, - }); + let snapshot = shared_state.transition(WorkerState::Idle); + let _ = working_event_tx.send(Event::WorkerState { snapshot }); } } WorkerStatus::Paused => { @@ -1619,7 +2238,26 @@ async fn controller_loop( } }, - Method::Shutdown => { + Method::Shutdown { command } => { + // Shutdown ignores the state-revision fence but remains bound to the + // current execution generation and command payload identity. + if let Err(disposition) = validate_shutdown_command(command, &shared_state) { + acknowledge_command( + &working_event_tx, + &shared_state, + command.command_id, + WorkerCommandKind::Shutdown, + disposition, + ); + continue; + } + acknowledge_command( + &working_event_tx, + &shared_state, + command.command_id, + WorkerCommandKind::Shutdown, + WorkerCommandDisposition::Accepted, + ); let _ = working_event_tx.send(Event::Shutdown); break; } @@ -1701,10 +2339,11 @@ async fn controller_loop( // Auto-kick a turn if the Worker is idle so the // notification is not stranded. Matches the // `Method::Notify` idle path. - if shared_state.get_status() == WorkerStatus::Idle { - pending = Some(PendingRun::RunForNotification( - protocol::InvokeKind::WorkerEvent, - )); + if shared_state.catalog_status() == WorkerStatus::Idle { + pending = Some(PendingRun::RunForNotification { + invoke_kind: protocol::InvokeKind::WorkerEvent, + notification_request_id: None, + }); } } } @@ -1720,7 +2359,16 @@ async fn controller_loop( // Memory/Workdir teardown so they cannot observe a partially closed Worker. worker.stop_feature_runtime("controller shutdown").await; - if let Some(session) = worker.workdir_session() + let child_cleanup_succeeded = match spawned_registry.shutdown_internal().await { + Ok(()) => true, + Err(error) => { + tracing::warn!(%error, "Internal SubWorker cleanup failed before Workdir shutdown"); + false + } + }; + + if child_cleanup_succeeded + && let Some(session) = worker.workdir_session() && let Err(error) = session.close().await { tracing::warn!(%error, "Workdir session close failed"); @@ -1787,12 +2435,12 @@ async fn handle_inbound_worker_event( /// as `Errored` — only the worker-execution `Err` branch below fires. /// /// `parent_originated` further restricts both upward reports to turns -/// the parent actually delegated (`Method::Run` / `Method::Resume`). +/// the parent actually delegated (`Method::Submit` / `Method::Resume`). /// `Method::Notify` / inbound `WorkerEvent` auto-kicks complete silently /// so the parent's history does not get flooded with child-internal /// turn boundaries. #[allow(clippy::too_many_arguments)] -async fn drive_turn( +async fn drive_turn( worker_future: F, method_rx: &mut mpsc::Receiver, working_event_tx: &broadcast::Sender, @@ -1800,15 +2448,17 @@ async fn drive_turn( pause_tx: &mpsc::Sender<()>, shared_state: &Arc, runtime_dir: &RuntimeDir, - mut input_commit_rx: Option>, + mut input_commit: Option<(oneshot::Receiver<()>, crate::worker::PendingSubmission)>, notify_buffer: &NotifyBuffer, + pending_submissions: &crate::worker::PendingSubmissionHandle, parent_socket: Option<&PathBuf>, self_name: &str, spawned_registry: &Arc, parent_originated: bool, -) -> (WorkerStatus, bool) +) -> (WorkerStatus, bool, bool) where F: std::future::Future>, + St: Store + Clone, { tokio::pin!(worker_future); let mut shutdown_requested = false; @@ -1821,13 +2471,25 @@ where // Running snapshot contract deterministic even for immediate clients. biased; committed = async { - input_commit_rx + input_commit .as_mut() + .map(|(receiver, _)| receiver) .expect("input commit receiver guarded by select condition") .await - }, if input_commit_rx.is_some() => { - input_commit_rx = None; + }, if input_commit.is_some() => { + let submission = input_commit.take().map(|(_, submission)| submission); if committed.is_ok() { + if let Some(submission) = submission { + pending_submissions.finish_activation(&submission.submission_id); + let _ = working_event_tx.send(Event::SubmissionAccepted { + submission_request_id: submission.submission_request_id, + submission_id: submission.submission_id, + disposition: protocol::SubmissionDisposition::Started, + }); + let _ = working_event_tx.send(Event::PendingSubmissionsChanged { + pending: pending_submissions.snapshot(), + }); + } set_controller_status( shared_state, runtime_dir, @@ -1835,11 +2497,33 @@ where WorkerStatus::Running, ) .await; + } else if let Some(submission) = submission { + pending_submissions.abort_activation(submission); } } result = &mut worker_future => { + if let Some((mut receiver, submission)) = input_commit.take() { + match receiver.try_recv() { + Ok(()) => { + pending_submissions.finish_activation(&submission.submission_id); + let _ = working_event_tx.send(Event::SubmissionAccepted { + submission_request_id: submission.submission_request_id, + submission_id: submission.submission_id, + disposition: protocol::SubmissionDisposition::Started, + }); + let _ = working_event_tx.send(Event::PendingSubmissionsChanged { + pending: pending_submissions.snapshot(), + }); + } + Err(_) => pending_submissions.abort_activation(submission), + } + } return match result { Ok(r) => { + let may_drain_pending = matches!( + &r, + WorkerRunResult::Finished | WorkerRunResult::LimitReached + ); let (status, run_result) = match r { WorkerRunResult::Finished if pause_requested => { (WorkerStatus::Paused, RunResult::Paused) @@ -1850,7 +2534,7 @@ where WorkerRunResult::RolledBack => (WorkerStatus::Idle, RunResult::RolledBack), WorkerRunResult::Interrupted { .. } if pause_requested => { let _ = working_event_tx.send(Event::RunEnd { result: RunResult::Paused }); - return (WorkerStatus::Paused, shutdown_requested); + return (WorkerStatus::Paused, shutdown_requested, false); } WorkerRunResult::Interrupted { code, message } => { let _ = working_event_tx.send(Event::Error { @@ -1866,7 +2550,7 @@ where }, ); } - return (WorkerStatus::Idle, shutdown_requested); + return (WorkerStatus::Idle, shutdown_requested, false); } }; let _ = working_event_tx.send(Event::RunEnd { result: run_result }); @@ -1878,7 +2562,7 @@ where }, ); } - (status, shutdown_requested) + (status, shutdown_requested, may_drain_pending) } Err(WorkerError::Engine(EngineError::Cancelled)) if pause_requested => { // User-initiated Pause. Report the transition to @@ -1887,7 +2571,7 @@ where // that channel is reserved for worker runtime // failures, not deliberate interruptions. let _ = working_event_tx.send(Event::RunEnd { result: RunResult::Paused }); - (WorkerStatus::Paused, shutdown_requested) + (WorkerStatus::Paused, shutdown_requested, false) } Err(e) => { let code = worker_error_code(&e); @@ -1905,41 +2589,342 @@ where }, ); } - (WorkerStatus::Idle, shutdown_requested) + (WorkerStatus::Idle, shutdown_requested, false) } }; } - method = method_rx.recv() => { + method = method_rx.recv(), if input_commit.is_none() => { match method { - Some(Method::Cancel) => { + Some(Method::Cancel { command }) => { + if let Err(disposition) = + validate_command(command, WorkerCommandKind::Cancel, shared_state) + { + acknowledge_command( + working_event_tx, + shared_state, + command.command_id, + WorkerCommandKind::Cancel, + disposition, + ); + continue; + } + if !matches!( + shared_state.snapshot().state, + WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)) + ) { + reject_invalid_command_state( + working_event_tx, + shared_state, + command, + WorkerCommandKind::Cancel, + ); + continue; + } + set_controller_state( + shared_state, + runtime_dir, + working_event_tx, + WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Cancelling)), + ) + .await; + acknowledge_command( + working_event_tx, + shared_state, + command.command_id, + WorkerCommandKind::Cancel, + WorkerCommandDisposition::Accepted, + ); let _ = cancel_tx.try_send(()); } - Some(Method::Pause) => { + Some(Method::Pause { command }) => { + if let Err(disposition) = + validate_command(command, WorkerCommandKind::Pause, shared_state) + { + acknowledge_command( + working_event_tx, + shared_state, + command.command_id, + WorkerCommandKind::Pause, + disposition, + ); + continue; + } + if !matches!( + shared_state.snapshot().state, + WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)) + ) { + reject_invalid_command_state( + working_event_tx, + shared_state, + command, + WorkerCommandKind::Pause, + ); + continue; + } pause_requested = true; + set_controller_state( + shared_state, + runtime_dir, + working_event_tx, + WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Pausing)), + ) + .await; + acknowledge_command( + working_event_tx, + shared_state, + command.command_id, + WorkerCommandKind::Pause, + WorkerCommandDisposition::Accepted, + ); let _ = pause_tx.try_send(()); } - Some(Method::Shutdown) => { + Some(Method::Shutdown { command }) => { + if let Err(disposition) = validate_shutdown_command(command, shared_state) { + acknowledge_command( + working_event_tx, + shared_state, + command.command_id, + WorkerCommandKind::Shutdown, + disposition, + ); + continue; + } shutdown_requested = true; + set_controller_state( + shared_state, + runtime_dir, + working_event_tx, + WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Cancelling)), + ) + .await; + acknowledge_command( + working_event_tx, + shared_state, + command.command_id, + WorkerCommandKind::Shutdown, + WorkerCommandDisposition::Accepted, + ); let _ = cancel_tx.try_send(()); } - Some(Method::Run { .. } | Method::RunTracked { .. } | Method::Resume) => { + Some(Method::Submit { + submission_request_id, + input, + }) => { + let request_id = submission_request_id.clone(); + match pending_submissions.accept_from_source( + submission_request_id, + input, + pending_submissions.direct_client_namespace(), + session_store::LoggedSessionHistoryOrigin::LegacyUnknown, + false, + ) { + Ok(acceptance) => { + let _ = working_event_tx.send(Event::SubmissionAccepted { + submission_request_id: acceptance.submission_request_id, + submission_id: acceptance.submission_id, + disposition: acceptance.disposition, + }); + let _ = working_event_tx.send(Event::PendingSubmissionsChanged { + pending: pending_submissions.snapshot(), + }); + } + Err(error) => { + let _ = working_event_tx.send(Event::SubmissionRejected { + submission_request_id: request_id, + message: error.to_string(), + }); + } + } + } + Some(Method::SubmitTracked { + submission_request_id, + input, + source, + }) => { + let request_id = submission_request_id.clone(); + let (source_namespace, provenance) = + resolved_input_source(pending_submissions, &source); + match pending_submissions.accept_from_source( + submission_request_id, + input, + source_namespace, + provenance, + false, + ) { + Ok(acceptance) => { + let _ = working_event_tx.send(Event::SubmissionAccepted { + submission_request_id: acceptance.submission_request_id, + submission_id: acceptance.submission_id, + disposition: acceptance.disposition, + }); + let _ = working_event_tx.send(Event::PendingSubmissionsChanged { + pending: pending_submissions.snapshot(), + }); + } + Err(error) => { + let _ = working_event_tx.send(Event::SubmissionRejected { + submission_request_id: request_id, + message: error.to_string(), + }); + } + } + } + Some(Method::Resume { command }) => { + if let Err(disposition) = + validate_command(command, WorkerCommandKind::Resume, shared_state) + { + acknowledge_command( + working_event_tx, + shared_state, + command.command_id, + WorkerCommandKind::Resume, + disposition, + ); + } else { + reject_invalid_command_state( + working_event_tx, + shared_state, + command, + WorkerCommandKind::Resume, + ); + } + } + Some(Method::ContinuePending { .. }) => { let _ = working_event_tx.send(Event::Error { code: ErrorCode::AlreadyRunning, message: "Worker is already executing a turn".into(), }); } - Some(Method::Compact | Method::ListRewindTargets | Method::RewindTo { .. }) => { + Some(Method::ListPendingSubmissions) => { + let _ = working_event_tx.send(Event::PendingSubmissionsChanged { + pending: pending_submissions.snapshot(), + }); + } + Some(Method::CancelPendingSubmission { + submission_id, + expected_revision, + }) => { + match pending_submissions.cancel(&submission_id, expected_revision) { + Ok(pending) => { + let _ = working_event_tx.send(Event::PendingSubmissionsChanged { pending }); + } + Err(error) => { + let _ = working_event_tx.send(Event::Error { + code: ErrorCode::InvalidRequest, + message: error.to_string(), + }); + } + } + } + Some(Method::ClearPendingSubmissions { expected_revision }) => { + match pending_submissions.clear(expected_revision) { + Ok(pending) => { + let _ = working_event_tx.send(Event::PendingSubmissionsChanged { pending }); + } + Err(error) => { + let _ = working_event_tx.send(Event::Error { + code: ErrorCode::InvalidRequest, + message: error.to_string(), + }); + } + } + } + Some(Method::Compact { command }) => { + if let Err(disposition) = + validate_command(command, WorkerCommandKind::Compact, shared_state) + { + acknowledge_command( + working_event_tx, + shared_state, + command.command_id, + WorkerCommandKind::Compact, + disposition, + ); + } else { + reject_invalid_command_state( + working_event_tx, + shared_state, + command, + WorkerCommandKind::Compact, + ); + } + } + Some(Method::ListRewindTargets | Method::RewindTo { .. }) => { let _ = working_event_tx.send(Event::Error { code: ErrorCode::AlreadyRunning, message: "Worker is already executing a turn; rewind/compact can only run while idle or paused" .into(), }); } - Some(Method::Notify { message, auto_run }) => { - // Live echo arrives via `Event::SystemItem` once - // the in-flight turn's next `pending_history_appends` - // drains this entry through the interceptor. - notify_buffer.push_notify(message, auto_run); + Some(Method::Notify { + notification_request_id, + message, + auto_run, + }) => { + let request_id = notification_request_id.clone(); + let source_namespace = pending_submissions.direct_client_namespace(); + match pending_submissions.accept_notification_from_source( + notification_request_id, + message, + source_namespace.clone(), + session_store::LoggedSessionHistoryOrigin::LegacyUnknown, + auto_run, + ) { + Ok(_) if !auto_run => { + stage_pending_notification( + &pending_submissions, + notify_buffer, + &source_namespace, + &request_id, + ); + } + Ok(_) => {} + Err(error) => { + let _ = working_event_tx.send(Event::Error { + code: ErrorCode::InvalidRequest, + message: error.to_string(), + }); + } + } + let _ = working_event_tx.send(Event::PendingSubmissionsChanged { + pending: pending_submissions.snapshot(), + }); + } + Some(Method::NotifyTracked { + notification_request_id, + message, + auto_run, + source, + }) => { + let request_id = notification_request_id.clone(); + let (source_namespace, provenance) = + resolved_input_source(pending_submissions, &source); + match pending_submissions.accept_notification_from_source( + notification_request_id, + message, + source_namespace.clone(), + provenance, + auto_run, + ) { + Ok(_) if !auto_run => { + stage_pending_notification( + &pending_submissions, + notify_buffer, + &source_namespace, + &request_id, + ); + } + Ok(_) => {} + Err(error) => { + let _ = working_event_tx.send(Event::Error { + code: ErrorCode::InvalidRequest, + message: error.to_string(), + }); + } + } + let _ = working_event_tx.send(Event::PendingSubmissionsChanged { + pending: pending_submissions.snapshot(), + }); } Some(Method::ListCompletions { .. }) => {} Some(Method::ListWorkers | Method::RestoreWorker { .. } | Method::RegisterPeer { .. }) => { @@ -1967,8 +2952,8 @@ where } None => { let _ = cancel_tx.try_send(()); - shared_state.set_status(WorkerStatus::Idle); - return (WorkerStatus::Idle, false); + shared_state.transition(WorkerState::Idle); + return (WorkerStatus::Idle, false, false); } } } @@ -2117,6 +3102,113 @@ mod tests { use tempfile::TempDir; use tokio::net::UnixListener; + #[test] + fn no_controller_parent_notification_uses_durable_pending_authority() { + let temp = TempDir::new().unwrap(); + let pending = + crate::worker::PendingSubmissionHandle::for_test(&temp.path().join("sessions")); + let target = durable_parent_notification_target(pending.clone(), NotifyBuffer::new()); + let (wire_namespace, wire_provenance) = + resolved_input_source(&pending, &protocol::AuthenticatedInputSource::UntrustedWire); + assert_eq!(wire_namespace, pending.direct_client_namespace()); + assert!(matches!( + wire_provenance, + session_store::LoggedSessionHistoryOrigin::LegacyUnknown + )); + + target.notify("child-session".into(), "completed".into(), true); + + let snapshot = pending.snapshot(); + assert_eq!(snapshot.notification_count, 1); + assert!(snapshot.head_id.is_some()); + let notify_buffer = NotifyBuffer::new(); + assert!(matches!( + prepare_pending_run(&pending, ¬ify_buffer, None).unwrap(), + Some(PendingRun::RunForNotification { + notification_request_id: Some(_), + .. + }) + )); + assert!(notify_buffer.has_auto_run_pending()); + } + + #[test] + fn restored_mixed_activations_preserve_global_fifo_order() { + let temp = TempDir::new().unwrap(); + let pending = + crate::worker::PendingSubmissionHandle::for_test(&temp.path().join("submit-first")); + pending + .accept( + "submit-first".into(), + vec![protocol::Segment::Text { + content: "queued submit".into(), + }], + false, + ) + .unwrap(); + pending + .accept_notification("notify-second".into(), "newer notification".into(), true) + .unwrap(); + let notify_buffer = NotifyBuffer::new(); + + assert!(matches!( + prepare_pending_run(&pending, ¬ify_buffer, None).unwrap(), + Some(PendingRun::Submit(_)) + )); + + let pending = + crate::worker::PendingSubmissionHandle::for_test(&temp.path().join("notify-first")); + pending + .accept_notification("notify-first".into(), "older notification".into(), true) + .unwrap(); + pending + .accept( + "submit-second".into(), + vec![protocol::Segment::Text { + content: "newer submit".into(), + }], + false, + ) + .unwrap(); + let notify_buffer = NotifyBuffer::new(); + + assert!(matches!( + prepare_pending_run(&pending, ¬ify_buffer, None).unwrap(), + Some(PendingRun::RunForNotification { + notification_request_id: Some(request_id), + .. + }) if request_id == "notify-first" + )); + + let pending = + crate::worker::PendingSubmissionHandle::for_test(&temp.path().join("passive-first")); + pending + .accept_notification("passive-first".into(), "passive notification".into(), false) + .unwrap(); + pending + .accept( + "submit-after-passive".into(), + vec![protocol::Segment::Text { + content: "queued after passive".into(), + }], + false, + ) + .unwrap(); + let snapshot = pending.snapshot(); + let head_id = snapshot.head_id.clone().expect("queued Submit is the head"); + let notify_buffer = NotifyBuffer::new(); + assert!(stage_oldest_passive_notification(&pending, ¬ify_buffer)); + assert!(matches!( + prepare_pending_run( + &pending, + ¬ify_buffer, + Some((snapshot.revision + 1, &head_id)), + ) + .unwrap(), + Some(PendingRun::Submit(_)) + )); + } + #[test] fn image_attachment_gate_requires_vision_and_supported_openai_scheme() { let openai = manifest::ModelManifest { @@ -2133,21 +3225,16 @@ mod tests { #[test] fn pending_run_parent_origin_table() { - assert!(PendingRun::Run(Vec::new()).is_parent_originated()); assert!(PendingRun::Resume.is_parent_originated()); assert!( - !PendingRun::RunForNotification(protocol::InvokeKind::Notify).is_parent_originated() + !PendingRun::RunForNotification { + invoke_kind: protocol::InvokeKind::Notify, + notification_request_id: None, + } + .is_parent_originated() ); } - #[test] - fn notification_auto_run_gate_only_allows_idle_auto_run() { - assert!(should_auto_run_notification(WorkerStatus::Idle, true)); - assert!(!should_auto_run_notification(WorkerStatus::Idle, false)); - assert!(!should_auto_run_notification(WorkerStatus::Running, true)); - assert!(!should_auto_run_notification(WorkerStatus::Paused, true)); - } - struct DriveTurnEnv { // Held to keep the channel alive; without this `method_rx.recv()` // would observe channel-closed and confuse the select! arm. @@ -2160,6 +3247,7 @@ mod tests { _pause_rx: mpsc::Receiver<()>, shared_state: Arc, notify_buffer: NotifyBuffer, + pending_submissions: crate::worker::PendingSubmissionHandle, spawned_registry: Arc, parent_socket_path: PathBuf, runtime_dir: Arc, @@ -2193,6 +3281,8 @@ mod tests { }, )); let notify_buffer = NotifyBuffer::new(); + let pending_submissions = + crate::worker::PendingSubmissionHandle::for_test(&temp.path().join("pending-sessions")); let spawned_registry = SpawnedWorkerRegistry::new(runtime_dir.clone()); let parent_socket_path = temp.path().join("parent.sock"); @@ -2206,6 +3296,7 @@ mod tests { _pause_rx: pause_rx, shared_state, notify_buffer, + pending_submissions, spawned_registry, parent_socket_path, runtime_dir, @@ -2224,6 +3315,7 @@ mod tests { writer .write(&Event::Snapshot { session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, greeting: protocol::Greeting { @@ -2236,7 +3328,7 @@ mod tests { context_window: 200_000, context_tokens: 0, }, - status: WorkerStatus::Idle, + state: WorkerStatus::Idle.into(), in_flight: Default::default(), internal_workers: Vec::new(), }) @@ -2258,7 +3350,7 @@ mod tests { let recv = tokio::spawn(recv_worker_event(listener, Duration::from_secs(2))); let worker_future = async { Ok::<_, WorkerError>(WorkerRunResult::Finished) }; - let (status, shutdown) = drive_turn( + let (status, shutdown, _) = drive_turn( worker_future, &mut env.method_rx, &env.working_event_tx, @@ -2268,6 +3360,7 @@ mod tests { &env.runtime_dir, None, &env.notify_buffer, + &env.pending_submissions, Some(&env.parent_socket_path), "child-worker", &env.spawned_registry, @@ -2291,9 +3384,17 @@ mod tests { async fn pause_waits_for_run_boundary_and_uses_safe_pause_channel() { let mut env = make_env().await; let method_tx = env._method_tx.clone(); + env.shared_state + .transition(WorkerState::Busy(WorkerBusyState::Run( + WorkerRunState::Running, + ))); + let command = WorkerCommandEnvelope::for_snapshot(1, &env.shared_state.snapshot()); tokio::spawn(async move { tokio::time::sleep(Duration::from_millis(10)).await; - method_tx.send(Method::Pause).await.expect("send pause"); + method_tx + .send(Method::Pause { command }) + .await + .expect("send pause"); }); let worker_future = async { @@ -2301,7 +3402,7 @@ mod tests { Ok::<_, WorkerError>(WorkerRunResult::Finished) }; let started_at = std::time::Instant::now(); - let (status, shutdown) = drive_turn( + let (status, shutdown, _) = drive_turn( worker_future, &mut env.method_rx, &env.working_event_tx, @@ -2311,6 +3412,7 @@ mod tests { &env.runtime_dir, None, &env.notify_buffer, + &env.pending_submissions, None, "child-worker", &env.spawned_registry, @@ -2331,7 +3433,7 @@ mod tests { let listener = UnixListener::bind(&env.parent_socket_path).expect("bind listener"); let worker_future = async { Ok::<_, WorkerError>(WorkerRunResult::Finished) }; - let (status, _) = drive_turn( + let (status, _, _) = drive_turn( worker_future, &mut env.method_rx, &env.working_event_tx, @@ -2341,6 +3443,7 @@ mod tests { &env.runtime_dir, None, &env.notify_buffer, + &env.pending_submissions, Some(&env.parent_socket_path), "child-worker", &env.spawned_registry, @@ -2369,7 +3472,7 @@ mod tests { "boom from test".into(), ))) }; - let (status, _) = drive_turn( + let (status, _, _) = drive_turn( worker_future, &mut env.method_rx, &env.working_event_tx, @@ -2379,6 +3482,7 @@ mod tests { &env.runtime_dir, None, &env.notify_buffer, + &env.pending_submissions, Some(&env.parent_socket_path), "child-worker", &env.spawned_registry, @@ -2413,7 +3517,7 @@ mod tests { "boom from notify".into(), ))) }; - let (status, _) = drive_turn( + let (status, _, _) = drive_turn( worker_future, &mut env.method_rx, &env.working_event_tx, @@ -2423,6 +3527,7 @@ mod tests { &env.runtime_dir, None, &env.notify_buffer, + &env.pending_submissions, Some(&env.parent_socket_path), "child-worker", &env.spawned_registry, @@ -2455,7 +3560,7 @@ mod tests { tokio::time::sleep(Duration::from_millis(50)).await; Ok::<_, WorkerError>(WorkerRunResult::Finished) }; - let (status, shutdown) = drive_turn( + let (status, shutdown, _) = drive_turn( worker_future, &mut env.method_rx, &env.working_event_tx, @@ -2465,6 +3570,7 @@ mod tests { &env.runtime_dir, None, &env.notify_buffer, + &env.pending_submissions, Some(&env.parent_socket_path), "parent", &env.spawned_registry, @@ -2494,7 +3600,7 @@ mod tests { tokio::time::sleep(Duration::from_millis(50)).await; Ok::<_, WorkerError>(WorkerRunResult::Finished) }; - let (status, shutdown) = drive_turn( + let (status, shutdown, _) = drive_turn( worker_future, &mut env.method_rx, &env.working_event_tx, @@ -2504,6 +3610,7 @@ mod tests { &env.runtime_dir, None, &env.notify_buffer, + &env.pending_submissions, Some(&env.parent_socket_path), "parent", &env.spawned_registry, @@ -2521,6 +3628,7 @@ mod tests { let mut env = make_env().await; env._method_tx .send(Method::Notify { + notification_request_id: protocol::new_submission_request_id(), message: "continue".into(), auto_run: true, }) @@ -2531,7 +3639,7 @@ mod tests { tokio::time::sleep(Duration::from_millis(50)).await; Ok::<_, WorkerError>(WorkerRunResult::Finished) }; - let (status, shutdown) = drive_turn( + let (status, shutdown, _) = drive_turn( worker_future, &mut env.method_rx, &env.working_event_tx, @@ -2541,6 +3649,7 @@ mod tests { &env.runtime_dir, None, &env.notify_buffer, + &env.pending_submissions, Some(&env.parent_socket_path), "parent", &env.spawned_registry, @@ -2550,16 +3659,21 @@ mod tests { assert_eq!(status, WorkerStatus::Idle); assert!(!shutdown); - assert_eq!(env.notify_buffer.len(), 1); - assert!(env.notify_buffer.has_auto_run_pending()); + assert_eq!(env.notify_buffer.len(), 0); + assert_eq!(env.pending_submissions.snapshot().notification_count, 1); } #[tokio::test] async fn compact_method_is_rejected_while_running() { let mut env = make_env().await; let mut events = env.working_event_tx.subscribe(); + env.shared_state + .transition(WorkerState::Busy(WorkerBusyState::Run( + WorkerRunState::Running, + ))); + let command = WorkerCommandEnvelope::for_snapshot(1, &env.shared_state.snapshot()); env._method_tx - .send(Method::Compact) + .send(Method::Compact { command }) .await .expect("send compact"); @@ -2567,7 +3681,7 @@ mod tests { tokio::time::sleep(Duration::from_millis(50)).await; Ok::<_, WorkerError>(WorkerRunResult::Finished) }; - let (status, shutdown) = drive_turn( + let (status, shutdown, _) = drive_turn( worker_future, &mut env.method_rx, &env.working_event_tx, @@ -2577,6 +3691,7 @@ mod tests { &env.runtime_dir, None, &env.notify_buffer, + &env.pending_submissions, Some(&env.parent_socket_path), "child-worker", &env.spawned_registry, @@ -2591,11 +3706,127 @@ mod tests { .expect("event timeout") .expect("event"); match event { - Event::Error { code, message } => { - assert_eq!(code, ErrorCode::AlreadyRunning); - assert!(message.contains("compact"), "got message: {message}"); + Event::CommandAcknowledged { acknowledgement } => { + assert_eq!(acknowledgement.command, WorkerCommandKind::Compact); + assert_eq!( + acknowledgement.disposition, + WorkerCommandDisposition::InvalidState + ); + assert!(matches!( + acknowledgement.state.state, + WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)) + )); } - other => panic!("expected compact rejection error, got {other:?}"), + other => panic!("expected compact rejection acknowledgement, got {other:?}"), } } + + #[test] + fn command_admission_rejects_stale_generation_revision_and_order() { + let shared = WorkerSharedState::new_with_generation( + "worker".into(), + session_store::new_segment_id(), + String::new(), + protocol::Greeting { + worker_name: "worker".into(), + cwd: "/tmp".into(), + provider: "test".into(), + model: "test".into(), + scope_summary: String::new(), + tools: Vec::new(), + context_window: 1, + context_tokens: 0, + }, + 9, + ); + assert_eq!( + validate_command( + WorkerCommandEnvelope { + command_id: 1, + expected_execution_generation: 8, + expected_worker_state_revision: 0, + }, + WorkerCommandKind::Pause, + &shared, + ), + Err(WorkerCommandDisposition::StaleExecutionGeneration) + ); + assert_eq!( + validate_command( + WorkerCommandEnvelope { + command_id: 2, + expected_execution_generation: 9, + expected_worker_state_revision: 1, + }, + WorkerCommandKind::Pause, + &shared, + ), + Err(WorkerCommandDisposition::StaleWorkerStateRevision) + ); + assert!( + validate_command( + WorkerCommandEnvelope { + command_id: 1, + expected_execution_generation: 9, + expected_worker_state_revision: 0, + }, + WorkerCommandKind::Pause, + &shared, + ) + .is_ok() + ); + assert_eq!( + validate_command( + WorkerCommandEnvelope { + command_id: 1, + expected_execution_generation: 9, + expected_worker_state_revision: 0, + }, + WorkerCommandKind::Pause, + &shared, + ), + Err(WorkerCommandDisposition::StaleCommandId) + ); + assert_eq!( + validate_command( + WorkerCommandEnvelope { + command_id: 1, + expected_execution_generation: 9, + expected_worker_state_revision: 0, + }, + WorkerCommandKind::Cancel, + &shared, + ), + Err(WorkerCommandDisposition::Conflict) + ); + assert!( + validate_command( + WorkerCommandEnvelope { + command_id: 2, + expected_execution_generation: 9, + expected_worker_state_revision: 1, + }, + WorkerCommandKind::Pause, + &shared, + ) + .is_ok() + ); + } + + #[test] + fn controller_shutdown_orders_child_cleanup_before_workdir_close() { + let source = include_str!("controller.rs"); + let shutdown_start = source + .rfind("worker.stop_feature_runtime(\"controller shutdown\")") + .expect("controller shutdown block"); + let shutdown = &source[shutdown_start..]; + let children = shutdown + .find("spawned_registry.shutdown_internal().await") + .expect("Internal SubWorker cleanup"); + let workdir = shutdown + .find("session.close().await") + .expect("parent Workdir close"); + assert!(children < workdir); + assert!(shutdown.contains("if child_cleanup_succeeded")); + } } diff --git a/crates/worker/src/discovery.rs b/crates/worker/src/discovery.rs index 32da104d..844d8c52 100644 --- a/crates/worker/src/discovery.rs +++ b/crates/worker/src/discovery.rs @@ -779,10 +779,10 @@ async fn probe_socket(socket_path: &Path) -> LiveInfo { loop { match tokio::time::timeout(PROBE_TIMEOUT, reader.next::()).await { Ok(Ok(Some(Event::Snapshot { - status: snapshot_status, + state: snapshot_state, .. }))) => { - status = Some(snapshot_status); + status = Some(snapshot_state.catalog_status()); break; } Ok(Ok(Some(Event::Alert(_)))) => continue, @@ -1012,7 +1012,19 @@ async fn send_peer_notify(socket_path: &Path, message: String) -> io::Result<()> } async fn send_notify(socket_path: &Path, message: String, auto_run: bool) -> io::Result<()> { - connect_and_send(socket_path, &Method::Notify { message, auto_run }).await + let notification_request_id = protocol::new_submission_request_id(); + connect_and_send( + socket_path, + &Method::NotifyTracked { + notification_request_id: notification_request_id.clone(), + message, + auto_run, + source: protocol::AuthenticatedInputSource::Backend { + operation_id: notification_request_id, + }, + }, + ) + .await } fn json_content(value: &T) -> Result { @@ -1482,6 +1494,7 @@ mod tests { writer .write(&Event::Snapshot { session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, greeting: protocol::Greeting { @@ -1494,7 +1507,7 @@ mod tests { context_window: 0, context_tokens: 0, }, - status: WorkerStatus::Idle, + state: WorkerStatus::Idle.into(), in_flight: Default::default(), internal_workers: Vec::new(), }) @@ -1517,6 +1530,7 @@ mod tests { writer .write(&Event::Snapshot { session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, greeting: protocol::Greeting { @@ -1529,14 +1543,17 @@ mod tests { context_window: 0, context_tokens: 0, }, - status: WorkerStatus::Idle, + state: WorkerStatus::Idle.into(), in_flight: Default::default(), internal_workers: Vec::new(), }) .await .unwrap(); let method = reader.next::().await.unwrap().unwrap(); - if let Method::Notify { message, auto_run } = method { + if let Method::NotifyTracked { + message, auto_run, .. + } = method + { assert!(auto_run); tx.send(message).await.unwrap(); } else { @@ -1608,6 +1625,7 @@ mod tests { writer .write(&Event::Snapshot { session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, greeting: protocol::Greeting { @@ -1620,7 +1638,7 @@ mod tests { context_window: 0, context_tokens: 0, }, - status: WorkerStatus::Idle, + state: WorkerStatus::Idle.into(), in_flight: Default::default(), internal_workers: Vec::new(), }) @@ -1634,6 +1652,7 @@ mod tests { writer .write(&Event::Snapshot { session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, greeting: protocol::Greeting { @@ -1646,14 +1665,17 @@ mod tests { context_window: 0, context_tokens: 0, }, - status: WorkerStatus::Idle, + state: WorkerStatus::Idle.into(), in_flight: Default::default(), internal_workers: Vec::new(), }) .await .unwrap(); let method = reader.next::().await.unwrap().unwrap(); - if let Method::Notify { message, auto_run } = method { + if let Method::NotifyTracked { + message, auto_run, .. + } = method + { assert!(!auto_run); tx.send(message).await.unwrap(); } else { @@ -1738,6 +1760,7 @@ mod tests { writer .write(&Event::Snapshot { session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: Vec::new(), }, greeting: protocol::Greeting { @@ -1750,7 +1773,7 @@ mod tests { context_window: 0, context_tokens: 0, }, - status: WorkerStatus::Paused, + state: WorkerStatus::Paused.into(), in_flight: Default::default(), internal_workers: Vec::new(), }) @@ -1790,6 +1813,8 @@ mod tests { let _ = writer .write(&Event::Snapshot { session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default( + ), entries: Vec::new(), }, greeting: protocol::Greeting { @@ -1802,7 +1827,7 @@ mod tests { context_window: 0, context_tokens: 0, }, - status: WorkerStatus::Idle, + state: WorkerStatus::Idle.into(), in_flight: Default::default(), internal_workers: Vec::new(), }) diff --git a/crates/worker/src/feature/builtin/manage_workdir.rs b/crates/worker/src/feature/builtin/manage_workdir.rs index f00093eb..eb950bbc 100644 --- a/crates/worker/src/feature/builtin/manage_workdir.rs +++ b/crates/worker/src/feature/builtin/manage_workdir.rs @@ -5,6 +5,8 @@ //! endpoints, credentials, materializer handles, and operation sessions stay //! behind [`WorkspaceClient`]. +use std::future::Future; +use std::pin::Pin; use std::sync::Arc; use agen::tool::{Tool, ToolDefinition, ToolError, ToolExecutionContext, ToolMeta, ToolOutput}; @@ -12,7 +14,7 @@ use async_trait::async_trait; use serde::{Deserialize, Serialize}; use serde_json::json; use workdir::http::{WorkdirSessionOperation, WorkdirSessionOperationResult}; -use workdir::workspace::{WorkspaceWorkdirSessionFence, WorkspaceWorkdirSessionOperationRequest}; +use workdir::workspace::WorkspaceWorkdirSessionOperationRequest; use workdir::{ CommandHandle, CommandOutput, CommandOutputRequest, CommandRequest, CommandStatus, EditRequest, EditResult, GlobRequest, GlobResult, GrepRequest, GrepResult, ListRequest, ListResult, @@ -52,16 +54,48 @@ const LIST_DESCRIPTION: &str = "List persistent Workdirs in the current Workspac const CREATE_DESCRIPTION: &str = "Materialize a persistent Workdir on a selected Runtime from a Workspace repository and optional selector. This does not change this Worker's attachment; use WorkdirAttach explicitly after creation."; const ATTACH_DESCRIPTION: &str = "Attach this Worker to one existing Workdir. The Backend enforces one active Workdir per Worker and one active Worker per Workdir, then opens an ephemeral operation session."; const DETACH_DESCRIPTION: &str = "Detach this Worker from its active Workdir and release Workdir occupancy. Any ephemeral operation session is closed."; +pub(crate) type BeforeWorkdirRelease = + Arc Pin> + Send>> + Send + Sync>; +pub(crate) type AfterWorkdirAttach = Arc; + const DELETE_DESCRIPTION: &str = "Request removal of one persistent Workdir by id through durable Backend Workspace authority. The input includes only the Workdir id and a bounded reason. The result reports removed, retained, or attention_required without exposing operation-table or provider internals."; -#[derive(Clone, Debug)] +#[derive(Clone)] pub struct ManageWorkdirFeature { client: Arc, + before_workdir_release: Option, + after_workdir_attach: Option, +} + +impl std::fmt::Debug for ManageWorkdirFeature { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("ManageWorkdirFeature") + .field("client_kind", &self.client.kind()) + .field("release_guard", &self.before_workdir_release.is_some()) + .finish() + } } impl ManageWorkdirFeature { pub fn new(client: Arc) -> Self { - Self { client } + Self { + client, + before_workdir_release: None, + after_workdir_attach: None, + } + } + + pub(crate) fn with_child_lifecycle( + client: Arc, + before_workdir_release: BeforeWorkdirRelease, + after_workdir_attach: AfterWorkdirAttach, + ) -> Self { + Self { + client, + before_workdir_release: Some(before_workdir_release), + after_workdir_attach: Some(after_workdir_attach), + } } } @@ -81,7 +115,10 @@ impl FeatureModule for ManageWorkdirFeature { } fn install(&self, context: &mut FeatureInstallContext<'_>) -> Result<(), FeatureInstallError> { - let backend = WorkspaceHttpWorkdirBackend::new(self.client.clone()); + let backend = WorkspaceHttpWorkdirBackend::new(self.client.clone()).with_child_lifecycle( + self.before_workdir_release.clone(), + self.after_workdir_attach.clone(), + ); for (name, definition) in [ ( LIST_TOOL, @@ -142,9 +179,21 @@ impl FeatureModule for ManageWorkdirFeature { } } -#[derive(Clone, Debug)] +#[derive(Clone)] struct WorkspaceHttpWorkdirBackend { client: Arc, + before_workdir_release: Option, + after_workdir_attach: Option, +} + +impl std::fmt::Debug for WorkspaceHttpWorkdirBackend { + fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + formatter + .debug_struct("WorkspaceHttpWorkdirBackend") + .field("client_kind", &self.client.kind()) + .field("release_guard", &self.before_workdir_release.is_some()) + .finish() + } } /// Worker-local Workdir handle whose operation authority remains in the Workspace Backend. @@ -156,8 +205,6 @@ struct WorkspaceHttpWorkdirBackend { pub struct WorkspaceAttachedWorkdirSession { client: Arc, workdir: Workdir, - expected_session_fence: Option, - delegations: Vec, } impl WorkspaceAttachedWorkdirSession { @@ -165,8 +212,6 @@ impl WorkspaceAttachedWorkdirSession { Arc::new(Self { client, workdir: Workdir::new("workspace-attachment"), - expected_session_fence: None, - delegations: Vec::new(), }) } @@ -183,16 +228,13 @@ impl WorkspaceAttachedWorkdirSession { "/api/w/{}/workers/self/workdir-session/operations", encode_path_segment(workspace_id) ), - serde_json::to_string(&WorkspaceWorkdirSessionOperationRequest { - expected_session_fence: self.expected_session_fence.clone(), - delegations: self.delegations.clone(), - operation, - }) - .map_err(|error| { - WorkdirError::Transport(format!( - "failed to encode Workspace Workdir operation: {error}" - )) - })?, + serde_json::to_string(&WorkspaceWorkdirSessionOperationRequest { operation }).map_err( + |error| { + WorkdirError::Transport(format!( + "failed to encode Workspace Workdir operation: {error}" + )) + }, + )?, ); let response = self .client @@ -241,59 +283,6 @@ impl WorkdirSession for WorkspaceAttachedWorkdirSession { WorkdirSessionCapabilities::ALL } - fn transports_delegation_context(&self) -> bool { - true - } - - async fn capture_delegation_source( - &self, - request: &workdir::WorkdirDelegationRequest, - ) -> Result { - let expected_session_fence = if let Some(fence) = &self.expected_session_fence { - fence.clone() - } else { - let workspace_id = self.client.workspace_id().ok_or_else(|| { - WorkdirError::Unavailable("Workspace identity is unavailable".to_string()) - })?; - let response = self - .client - .execute(WorkspaceRequest { - method: WorkspaceRequestMethod::Get, - path: format!( - "/api/w/{}/workers/self/workdir-session/fence", - encode_path_segment(workspace_id) - ), - body: None, - }) - .map_err(|error| { - WorkdirError::Unavailable(format!( - "failed to capture Workdir attachment fence: {error}" - )) - })?; - let fence: WorkspaceWorkdirSessionFence = serde_json::from_str(&response.body) - .map_err(|error| { - WorkdirError::Unavailable(format!( - "invalid Workdir attachment fence response: {error}" - )) - })?; - fence.value - }; - let mut delegations = self.delegations.clone(); - delegations.push(request.clone()); - let candidate = Arc::new(Self { - client: self.client.clone(), - workdir: self.workdir.clone(), - expected_session_fence: Some(expected_session_fence), - delegations, - }); - candidate - .stat(StatRequest { - path: workdir::WorkdirPath::new("").expect("empty Workdir path is valid"), - }) - .await?; - Ok(candidate) - } - async fn stat(&self, request: StatRequest) -> Result { match self.operate(WorkdirSessionOperation::Stat(request))? { WorkdirSessionOperationResult::Stat(result) => Ok(result), @@ -387,7 +376,21 @@ impl WorkdirSession for WorkspaceAttachedWorkdirSession { impl WorkspaceHttpWorkdirBackend { fn new(client: Arc) -> Self { - Self { client } + Self { + client, + before_workdir_release: None, + after_workdir_attach: None, + } + } + + fn with_child_lifecycle( + mut self, + before_workdir_release: Option, + after_workdir_attach: Option, + ) -> Self { + self.before_workdir_release = before_workdir_release; + self.after_workdir_attach = after_workdir_attach; + self } fn workspace_id(&self) -> Result<&str, ToolError> { @@ -565,11 +568,26 @@ impl Tool for WorkspaceHttpWorkdirTool { parse_input::(input_json)?, ctx.call_id.to_string(), ), - WorkdirOperation::Attach => self - .backend - .attach(parse_input::(input_json)?), + WorkdirOperation::Attach => { + let result = self + .backend + .attach(parse_input::(input_json)?); + if result.is_ok() + && let Some(after_attach) = &self.backend.after_workdir_attach + { + after_attach(); + } + result + } WorkdirOperation::Detach => { let _input = parse_input::(input_json)?; + if let Some(before_release) = &self.backend.before_workdir_release { + before_release().await.map_err(|error| { + ToolError::ExecutionFailed(format!( + "stop Internal SubWorkers before Workdir detach: {error}" + )) + })?; + } self.backend.detach() } WorkdirOperation::Delete => self @@ -765,6 +783,7 @@ struct WorkdirDeleteInput { #[cfg(test)] mod tests { use std::sync::Mutex; + use std::sync::atomic::{AtomicUsize, Ordering}; use super::*; use crate::feature::{FeatureModule, FeatureRegistryBuilder}; @@ -1155,6 +1174,7 @@ mod tests { command: "true".to_string(), timeout_secs: 120, output_limit: 1024, + cwd: None, spill_dir: Some("/worker-local/bash-output".into()), tool_call_id: Some("call-1".to_string()), }) @@ -1178,83 +1198,6 @@ mod tests { ); } - #[tokio::test] - async fn delegated_attached_session_carries_captured_fence_on_operations() { - let client = Arc::new(RecordingWorkspaceClient::new(vec![ - response(json!({"value": "attachment-fence"})), - response(json!({ - "operation": "stat", - "result": {"path": "", "kind": "directory", "size": 0} - })), - response(json!({ - "operation": "stat", - "result": {"path": "visible.txt", "kind": "file", "size": 8} - })), - ])); - let parent = workdir::delegation_capable_session(WorkspaceAttachedWorkdirSession::handle( - client.clone(), - )); - let delegation = parent - .delegate(workdir::WorkdirDelegationRequest { - rules: vec![workdir::WorkdirDelegationRule { - target: workdir::WorkdirPath::new("").unwrap(), - permission: workdir::WorkdirDelegationPermission::Read, - recursive: false, - }], - cwd: workdir::WorkdirPath::new("").unwrap(), - }) - .await - .unwrap(); - delegation - .scoped_session - .stat(StatRequest { - path: workdir::WorkdirPath::new("visible.txt").unwrap(), - }) - .await - .unwrap(); - - let requests = client.requests(); - assert_eq!(requests.len(), 3); - assert_eq!( - requests[0].path, - "/api/w/workspace%2Ftest/workers/self/workdir-session/fence" - ); - let body: serde_json::Value = - serde_json::from_str(requests[2].body.as_deref().unwrap()).unwrap(); - assert_eq!(body["expected_session_fence"], "attachment-fence"); - assert_eq!(body["operation"]["operation"], "stat"); - assert_eq!(body["delegations"][0]["rules"][0]["target"], ""); - } - - #[tokio::test] - async fn attached_provider_rejection_happens_before_delegation_is_returned() { - let client = Arc::new(RecordingWorkspaceClient::new(vec![ - response(json!({"value": "attachment-fence"})), - response(json!({"error": "provider rejected delegated write target"})), - ])); - let parent = workdir::delegation_capable_session(WorkspaceAttachedWorkdirSession::handle( - client.clone(), - )); - let result = parent - .delegate(workdir::WorkdirDelegationRequest { - rules: vec![workdir::WorkdirDelegationRule { - target: workdir::WorkdirPath::new("linked-target").unwrap(), - permission: workdir::WorkdirDelegationPermission::Write, - recursive: true, - }], - cwd: workdir::WorkdirPath::new("linked-target").unwrap(), - }) - .await; - - assert!(result.is_err(), "provider rejection must fail before lease"); - let requests = client.requests(); - assert_eq!(requests.len(), 2); - let validation: serde_json::Value = - serde_json::from_str(requests[1].body.as_deref().unwrap()).unwrap(); - assert_eq!(validation["operation"]["operation"], "stat"); - assert_eq!(validation["delegations"].as_array().unwrap().len(), 1); - } - #[tokio::test] async fn attached_session_preserves_typed_provider_validation_error() { let client = Arc::new(RecordingWorkspaceClient::new(vec![error_response( @@ -1298,73 +1241,52 @@ mod tests { } #[tokio::test] - async fn nested_attached_session_preserves_full_delegation_chain() { + async fn scoped_broker_operations_carry_no_child_context() { let client = Arc::new(RecordingWorkspaceClient::new(vec![ - response(json!({"value": "attachment-fence"})), response(json!({ "operation": "stat", - "result": {"path": "", "kind": "directory", "size": 0} + "result": {"path": "visible.txt", "kind": "file", "size": 8} })), response(json!({ "operation": "stat", - "result": {"path": "nested", "kind": "directory", "size": 0} - })), - response(json!({ - "operation": "stat", - "result": {"path": "nested/file", "kind": "file", "size": 1} + "result": {"path": "visible.txt", "kind": "file", "size": 8} })), ])); - let parent = workdir::delegation_capable_session(WorkspaceAttachedWorkdirSession::handle( + let broker = workdir::WorkdirToolBroker::new(WorkspaceAttachedWorkdirSession::handle( client.clone(), )); - let outer = parent - .delegate(workdir::WorkdirDelegationRequest { - rules: vec![workdir::WorkdirDelegationRule { + let scoped = broker + .scope(workdir::WorkdirToolScope { + rules: vec![workdir::WorkdirToolScopeRule { target: workdir::WorkdirPath::new("").unwrap(), - permission: workdir::WorkdirDelegationPermission::Read, + permission: workdir::WorkdirToolScopePermission::Read, recursive: true, }], cwd: workdir::WorkdirPath::new("").unwrap(), + command: false, }) .await .unwrap(); - let nested = outer - .scoped_session - .delegate(workdir::WorkdirDelegationRequest { - rules: vec![workdir::WorkdirDelegationRule { - target: workdir::WorkdirPath::new("nested").unwrap(), - permission: workdir::WorkdirDelegationPermission::Read, - recursive: true, - }], - cwd: workdir::WorkdirPath::new("nested").unwrap(), - }) - .await - .unwrap(); - nested - .scoped_session + scoped .stat(StatRequest { - path: workdir::WorkdirPath::new("file").unwrap(), + path: workdir::WorkdirPath::new("visible.txt").unwrap(), }) .await .unwrap(); let requests = client.requests(); - assert_eq!(requests.len(), 4); - let outer_validation: serde_json::Value = - serde_json::from_str(requests[1].body.as_deref().unwrap()).unwrap(); - let nested_validation: serde_json::Value = - serde_json::from_str(requests[2].body.as_deref().unwrap()).unwrap(); - assert_eq!(outer_validation["delegations"].as_array().unwrap().len(), 1); - assert_eq!( - nested_validation["delegations"].as_array().unwrap().len(), - 2 - ); - let body: serde_json::Value = - serde_json::from_str(requests[3].body.as_deref().unwrap()).unwrap(); - assert_eq!(body["delegations"].as_array().unwrap().len(), 2); - assert_eq!(body["delegations"][0]["rules"][0]["target"], ""); - assert_eq!(body["delegations"][1]["rules"][0]["target"], "nested"); - assert_eq!(body["operation"]["request"]["path"], "file"); + assert_eq!(requests.len(), 2); + for request in requests { + assert_eq!( + request.path, + "/api/w/workspace%2Ftest/workers/self/workdir-session/operations" + ); + let body: serde_json::Value = + serde_json::from_str(request.body.as_deref().unwrap()).unwrap(); + assert!(body.get("delegations").is_none()); + assert!(body.get("child").is_none()); + assert!(body.get("expected_session_fence").is_none()); + } } #[test] @@ -1416,4 +1338,86 @@ mod tests { assert!(client.requests().is_empty()); assert!(parse_input::(r#"{"path":"/tmp"}"#).is_err()); } + + #[tokio::test] + async fn detach_stops_internal_subworkers_before_backend_release() { + let client = Arc::new(RecordingWorkspaceClient::new(vec![response(json!({ + "workspace_id": "workspace/test", + "workdir_id": "wd-attached", + "attached": false + }))])); + let cleanup_calls = Arc::new(AtomicUsize::new(0)); + let cleanup_calls_for_guard = cleanup_calls.clone(); + let before_release: BeforeWorkdirRelease = Arc::new(move || { + let cleanup_calls = cleanup_calls_for_guard.clone(); + Box::pin(async move { + cleanup_calls.fetch_add(1, Ordering::SeqCst); + Ok(()) + }) + }); + let tool = WorkspaceHttpWorkdirTool { + backend: WorkspaceHttpWorkdirBackend::new(client.clone()) + .with_child_lifecycle(Some(before_release), None), + operation: WorkdirOperation::Detach, + }; + + tool.execute("{}", ToolExecutionContext::default()) + .await + .unwrap(); + + assert_eq!(cleanup_calls.load(Ordering::SeqCst), 1); + assert_eq!(client.requests().len(), 1); + assert_eq!( + client.requests()[0].path, + "/api/w/workspace%2Ftest/workers/self/workdir-attachment" + ); + } + + #[tokio::test] + async fn detach_does_not_release_backend_when_child_cleanup_fails() { + let client = Arc::new(RecordingWorkspaceClient::new(Vec::new())); + let before_release: BeforeWorkdirRelease = + Arc::new(|| Box::pin(async { Err(std::io::Error::other("child cleanup failed")) })); + let tool = WorkspaceHttpWorkdirTool { + backend: WorkspaceHttpWorkdirBackend::new(client.clone()) + .with_child_lifecycle(Some(before_release), None), + operation: WorkdirOperation::Detach, + }; + + let error = tool + .execute("{}", ToolExecutionContext::default()) + .await + .unwrap_err(); + + assert!(error.to_string().contains("stop Internal SubWorkers")); + assert!(client.requests().is_empty()); + } + + #[tokio::test] + async fn successful_attach_reopens_internal_subworker_admission() { + let client = Arc::new(RecordingWorkspaceClient::new(vec![response(json!({ + "workspace_id": "workspace/test", + "workdir_id": "wd-attached", + "attached": true + }))])); + let reopen_calls = Arc::new(AtomicUsize::new(0)); + let reopen_calls_for_hook = reopen_calls.clone(); + let after_attach: AfterWorkdirAttach = Arc::new(move || { + reopen_calls_for_hook.fetch_add(1, Ordering::SeqCst); + }); + let tool = WorkspaceHttpWorkdirTool { + backend: WorkspaceHttpWorkdirBackend::new(client) + .with_child_lifecycle(None, Some(after_attach)), + operation: WorkdirOperation::Attach, + }; + + tool.execute( + r#"{"workdir_id":"wd-attached"}"#, + ToolExecutionContext::default(), + ) + .await + .unwrap(); + + assert_eq!(reopen_calls.load(Ordering::SeqCst), 1); + } } diff --git a/crates/worker/src/feature/builtin/worker_observation.rs b/crates/worker/src/feature/builtin/worker_observation.rs index 2dc059f6..bda05e09 100644 --- a/crates/worker/src/feature/builtin/worker_observation.rs +++ b/crates/worker/src/feature/builtin/worker_observation.rs @@ -803,7 +803,10 @@ mod tests { .collect(); Ok(WorkerSessionCapture { segment_id: "segment".to_string(), - session: protocol::SessionSnapshot { entries }, + session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), + entries, + }, }) } } diff --git a/crates/worker/src/internal_worker.rs b/crates/worker/src/internal_worker.rs index ec7ef586..0f553102 100644 --- a/crates/worker/src/internal_worker.rs +++ b/crates/worker/src/internal_worker.rs @@ -295,6 +295,38 @@ impl InternalWorkerSessionStatus { } } +fn send_internal_worker_state( + event_tx: &broadcast::Sender, + state_revision: &std::sync::atomic::AtomicU64, + status: InternalWorkerSessionStatus, +) { + let state = match status { + InternalWorkerSessionStatus::Idle + | InternalWorkerSessionStatus::Stopped + | InternalWorkerSessionStatus::Failed => protocol::WorkerState::Idle, + InternalWorkerSessionStatus::Paused => protocol::WorkerState::Busy( + protocol::WorkerBusyState::Run(protocol::WorkerRunState::Paused), + ), + InternalWorkerSessionStatus::Running => protocol::WorkerState::Busy( + protocol::WorkerBusyState::Run(protocol::WorkerRunState::Running), + ), + InternalWorkerSessionStatus::Stopping => protocol::WorkerState::Busy( + protocol::WorkerBusyState::Run(protocol::WorkerRunState::Cancelling), + ), + }; + let revision = state_revision + .fetch_add(1, std::sync::atomic::Ordering::AcqRel) + .saturating_add(1); + let _ = event_tx.send(Event::WorkerState { + snapshot: protocol::WorkerStateSnapshot { + execution_generation: 1, + revision, + last_command_id: 0, + state, + }, + }); +} + fn classify_internal_turn_result( result: Result, ) -> (InternalWorkerSessionStatus, Option) { @@ -351,6 +383,7 @@ pub(crate) struct InternalWorkerSessionSnapshot { pub(crate) struct InternalWorkerSessionHandle { command_tx: tokio::sync::mpsc::Sender, status: Arc, + state_revision: Arc, store: EphemeralSessionStore, session_id: SessionId, segment_id: SegmentId, @@ -400,6 +433,10 @@ impl InternalWorkerSessionHandle { self.in_flight.text_delta(block_id, text.to_owned()); } + fn emit_worker_state(&self, status: InternalWorkerSessionStatus) { + send_internal_worker_state(&self.event_tx, &self.state_revision, status); + } + pub(crate) fn protocol_snapshot(&self) -> InternalWorkerSessionSnapshot { let (entries, in_flight) = { let guard = self.in_flight.snapshot_guard(); @@ -473,9 +510,7 @@ impl InternalWorkerSessionHandle { }); return Err(InternalWorkerSessionError::Unavailable); } - let _ = self.event_tx.send(Event::Status { - status: WorkerStatus::Running, - }); + self.emit_worker_state(InternalWorkerSessionStatus::Running); Ok(()) } @@ -709,7 +744,7 @@ pub(crate) fn prepare_internal_worker_from_spec( } Box::pin(prepare_internal_worker_session( - worker, store, visibility, None, None, + worker, store, visibility, None, None, None, )) .await }) @@ -746,13 +781,16 @@ pub(crate) async fn prepare_internal_worker_session( visibility: InternalWorkerVisibility, child_registry: Option>, on_turn_end: Option>, + command_event_broker: Option, ) -> Result { let (event_tx, _event_rx) = broadcast::channel(256); let sink = worker.sink(); spawn_internal_log_event_bridge(sink.clone(), event_tx.clone()); let alerter = Alerter::new(event_tx.clone()); let in_flight = InFlightEvents::new(event_tx.clone()); - if let Some(session) = worker.workdir_session() { + if let Some(broker) = command_event_broker.as_ref() { + wire_workdir_command_events(&broker.tool_session(), &in_flight); + } else if let Some(session) = worker.workdir_session() { wire_workdir_command_events(session, &in_flight); } let actor_in_flight = in_flight.clone(); @@ -767,11 +805,13 @@ pub(crate) async fn prepare_internal_worker_session( let status = Arc::new(std::sync::atomic::AtomicU8::new( InternalWorkerSessionStatus::Idle.encode(), )); + let state_revision = Arc::new(std::sync::atomic::AtomicU64::new(0)); let state_changed = Arc::new(tokio::sync::Notify::new()); let last_error = Arc::new(Mutex::new(None)); let handle = InternalWorkerSessionHandle { command_tx, status: status.clone(), + state_revision: state_revision.clone(), store, session_id, segment_id, @@ -807,19 +847,11 @@ pub(crate) async fn prepare_internal_worker_session( message, }); } - let protocol_status = match turn_status { - InternalWorkerSessionStatus::Idle => WorkerStatus::Idle, - InternalWorkerSessionStatus::Paused => WorkerStatus::Paused, - InternalWorkerSessionStatus::Stopped - | InternalWorkerSessionStatus::Failed => WorkerStatus::Stopped, - InternalWorkerSessionStatus::Running - | InternalWorkerSessionStatus::Stopping => { - unreachable!("run completion cannot remain active") - } - }; - let _ = event_tx.send(Event::Status { - status: protocol_status, - }); + send_internal_worker_state( + &event_tx, + &state_revision, + turn_status, + ); if let Some(callback) = &on_turn_end { callback(turn_status); } @@ -861,9 +893,11 @@ pub(crate) async fn prepare_internal_worker_session( InternalWorkerSessionStatus::Stopped.encode(), std::sync::atomic::Ordering::Release, ); - let _ = event_tx.send(Event::Status { - status: WorkerStatus::Stopped, - }); + send_internal_worker_state( + &event_tx, + &state_revision, + InternalWorkerSessionStatus::Stopped, + ); let _ = event_tx.send(Event::Shutdown); state_changed.notify_waiters(); if let Some(done) = stop_done { @@ -887,6 +921,7 @@ pub(crate) async fn spawn_prepared_internal_worker_session( InternalWorkerVisibility::ServicePrivate, None, on_turn_end, + None, ) .await?; handle.send(input).await?; @@ -1114,6 +1149,7 @@ pub(crate) fn test_internal_worker_session( status: Arc::new(std::sync::atomic::AtomicU8::new( InternalWorkerSessionStatus::Idle.encode(), )), + state_revision: Arc::new(std::sync::atomic::AtomicU64::new(0)), store, session_id, segment_id, diff --git a/crates/worker/src/ipc/interceptor.rs b/crates/worker/src/ipc/interceptor.rs index 30bf4e12..86f26230 100644 --- a/crates/worker/src/ipc/interceptor.rs +++ b/crates/worker/src/ipc/interceptor.rs @@ -176,12 +176,23 @@ impl WorkerInterceptor { /// `Item::system_message`s reach the worker via /// `ContinueWith` / `pending_history_appends`, so on-disk order /// matches worker-history order. - fn commit_system_items(&self, items: &[SystemItem]) -> Result<(), session_store::StoreError> { + fn commit_system_items_with_extensions( + &self, + items: &[( + SystemItem, + Vec, + Option, + )], + ) -> Result<(), session_store::StoreError> { let Some(writer) = self.log_writer.as_ref() else { return Ok(()); }; - for item in items { - let entry = writer.commit_system_item(item.clone())?; + for (item, extensions, history_provenance) in items { + let entry = writer.commit_system_item_with_extensions( + item.clone(), + extensions.clone(), + history_provenance.clone(), + )?; self.pending_committed_history .lock() .expect("pending committed history poisoned") @@ -190,6 +201,16 @@ impl WorkerInterceptor { Ok(()) } + fn commit_system_items(&self, items: &[SystemItem]) -> Result<(), session_store::StoreError> { + self.commit_system_items_with_extensions( + &items + .iter() + .cloned() + .map(|item| (item, Vec::new(), None)) + .collect::>(), + ) + } + fn current_turn_index(&self) -> usize { self.next_turn_index .load(Ordering::Relaxed) @@ -327,7 +348,11 @@ impl Interceptor for WorkerInterceptor { projection_digest: projection.catalog_digest.clone(), logical_name: "internal.notify_wrapper".to_string(), }; - let mut system_items: Vec = Vec::with_capacity(drained.len()); + let mut system_items: Vec<( + SystemItem, + Vec, + Option, + )> = Vec::with_capacity(drained.len()); let mut items: Vec = Vec::with_capacity(drained.len()); for entry in &drained { let system_item = match build_system_item_with_provenance( @@ -345,9 +370,9 @@ impl Interceptor for WorkerInterceptor { } }; items.push(system_item.to_history_item()); - system_items.push(system_item); + system_items.push((system_item, entry.extensions(), entry.history_provenance())); } - if let Err(error) = self.commit_system_items(&system_items) { + if let Err(error) = self.commit_system_items_with_extensions(&system_items) { self.pending_notifies.requeue_front(drained); return Err(InterceptorError::new( InterceptorErrorCategory::Dependency, diff --git a/crates/worker/src/ipc/notify_buffer.rs b/crates/worker/src/ipc/notify_buffer.rs index 7b97e4ed..2fd2ca03 100644 --- a/crates/worker/src/ipc/notify_buffer.rs +++ b/crates/worker/src/ipc/notify_buffer.rs @@ -25,7 +25,7 @@ use std::collections::VecDeque; use std::sync::{Arc, Mutex}; use protocol::WorkerEvent; -use session_store::SystemItem; +use session_store::{LoggedSessionHistoryOrigin, SessionExtension, SystemItem}; use tracing::warn; use crate::prompt::catalog::{CatalogError, PromptCatalog}; @@ -41,8 +41,33 @@ const CAPACITY: usize = 128; /// is available. #[derive(Debug, Clone)] pub enum PendingNotify { - Notify { message: String, auto_run: bool }, - WorkerEvent { event: WorkerEvent }, + Notify { + message: String, + auto_run: bool, + extensions: Vec, + history_provenance: Option, + }, + WorkerEvent { + event: WorkerEvent, + }, +} + +impl PendingNotify { + pub(crate) fn extensions(&self) -> Vec { + match self { + PendingNotify::Notify { extensions, .. } => extensions.clone(), + PendingNotify::WorkerEvent { .. } => Vec::new(), + } + } + + pub(crate) fn history_provenance(&self) -> Option { + match self { + PendingNotify::Notify { + history_provenance, .. + } => history_provenance.clone(), + PendingNotify::WorkerEvent { .. } => None, + } + } } /// Shared, mutex-guarded buffer of pending entries. @@ -62,7 +87,46 @@ impl NotifyBuffer { /// oldest entry is dropped and a `tracing::warn` is emitted — the /// caller should never hit this in normal operation. pub fn push_notify(&self, message: String, auto_run: bool) { - self.push_entry(PendingNotify::Notify { message, auto_run }); + self.push_entry(PendingNotify::Notify { + message, + auto_run, + extensions: Vec::new(), + history_provenance: None, + }); + } + + pub fn push_durable_notify( + &self, + message: String, + auto_run: bool, + history_provenance: LoggedSessionHistoryOrigin, + extension: SessionExtension, + ) { + self.push_entry(PendingNotify::Notify { + message, + auto_run, + extensions: vec![extension], + history_provenance: Some(history_provenance), + }); + } + + pub(crate) fn replace_durable_notification_extension( + &self, + extension: SessionExtension, + ) -> bool { + let mut queue = self.inner.lock().expect("notify buffer poisoned"); + let Some(extensions) = queue.iter_mut().rev().find_map(|pending| match pending { + PendingNotify::Notify { + auto_run: false, + extensions, + .. + } if !extensions.is_empty() => Some(extensions), + _ => None, + }) else { + return false; + }; + *extensions = vec![extension]; + true } /// Push a typed worker-event entry onto the queue. @@ -202,6 +266,8 @@ mod tests { let entry = PendingNotify::Notify { message: "hello".into(), auto_run: false, + extensions: Vec::new(), + history_provenance: None, }; let catalog = PromptCatalog::builtins_only().unwrap(); let item = build_system_item(&entry, &catalog).unwrap(); diff --git a/crates/worker/src/lib.rs b/crates/worker/src/lib.rs index d5f7f8d1..c83ef56a 100644 --- a/crates/worker/src/lib.rs +++ b/crates/worker/src/lib.rs @@ -57,9 +57,9 @@ pub use session_history::{ }; pub use shared_state::WorkerSharedState; pub use worker::{ - LocalWorkingDirectory, WORKER_INPUT_SUBMISSION_EXTENSION_DOMAIN, Worker, WorkerError, - WorkerFilesystemAuthority, WorkerRunResult, WorkerWorkspaceContext, WorkspaceClient, - WorkspaceClientError, WorkspaceId, WorkspaceIdError, WorkspacePromptCatalogResolution, - WorkspaceRequest, WorkspaceRequestMethod, WorkspaceResponse, WorkspaceWorkerDiscoveryRequest, - apply_worker_manifest, marker_workspace_client, unavailable_workspace_client, + LocalWorkingDirectory, Worker, WorkerError, WorkerFilesystemAuthority, WorkerRunResult, + WorkerWorkspaceContext, WorkspaceClient, WorkspaceClientError, WorkspaceId, WorkspaceIdError, + WorkspacePromptCatalogResolution, WorkspaceRequest, WorkspaceRequestMethod, WorkspaceResponse, + WorkspaceWorkerDiscoveryRequest, apply_worker_manifest, marker_workspace_client, + unavailable_workspace_client, }; diff --git a/crates/worker/src/runtime/dir.rs b/crates/worker/src/runtime/dir.rs index ad462f39..37d8d99e 100644 --- a/crates/worker/src/runtime/dir.rs +++ b/crates/worker/src/runtime/dir.rs @@ -197,7 +197,6 @@ pub fn default_base() -> Result { mod tests { use super::*; use crate::shared_state::WorkerSharedState; - use protocol::WorkerStatus; fn test_state() -> WorkerSharedState { WorkerSharedState::new( @@ -247,7 +246,9 @@ mod tests { let rt = RuntimeDir::create(tmp.path(), "my-worker").await.unwrap(); let state = test_state(); - state.set_status(WorkerStatus::Running); + state.transition(protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Running, + ))); rt.write_status(&state).await.unwrap(); let content = std::fs::read_to_string(rt.path().join("status.json")).unwrap(); diff --git a/crates/worker/src/segment_log_sink.rs b/crates/worker/src/segment_log_sink.rs index 91c2792b..20233983 100644 --- a/crates/worker/src/segment_log_sink.rs +++ b/crates/worker/src/segment_log_sink.rs @@ -291,6 +291,7 @@ mod tests { prompt_provenance: None, }, ), + extensions: Vec::new(), } } diff --git a/crates/worker/src/shared_state.rs b/crates/worker/src/shared_state.rs index 20563691..810b59f4 100644 --- a/crates/worker/src/shared_state.rs +++ b/crates/worker/src/shared_state.rs @@ -1,28 +1,49 @@ -use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{OnceLock, RwLock}; +use std::collections::VecDeque; +use std::sync::{ + OnceLock, RwLock, + atomic::{AtomicBool, Ordering}, +}; -use protocol::WorkerStatus; +use protocol::{ + WorkerBusyState, WorkerCommandDisposition, WorkerCommandEnvelope, WorkerCommandKind, + WorkerMaintenanceState, WorkerRunState, WorkerState, WorkerStateSnapshot, WorkerStatus, +}; use serde_json::json; use session_store::SegmentId; use crate::fs_view::WorkerFsView; +const COMPLETED_COMMAND_RETENTION: usize = 256; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct AcceptedWorkerCommand { + envelope: WorkerCommandEnvelope, + kind: WorkerCommandKind, + disposition: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum WorkerCommandAdmission { + Accepted, + Retry, + Conflict, + StaleCommandId, + ExecutionGenerationMismatch, + StateRevisionMismatch, +} + /// Shared state between WorkerController and runtime directory. /// -/// Controller updates this in-memory; RuntimeDir writes the status -/// snapshot to disk. Wrapped in `Arc` for sharing. -/// -/// History and typed user-segment mirrors used to live here so the -/// IPC layer could answer `Method::GetHistory`. Those reads now go -/// directly through the session-log sink (`Event::Snapshot` + -/// live events), so this struct holds only status, identity, -/// greeting, and filesystem completion lookup hubs. +/// `WorkerStateSnapshot` is the sole live execution-state authority. Runtime +/// catalog status remains a separate lifecycle projection because `Stopped` +/// describes the execution handle rather than a live controller state. pub struct WorkerSharedState { pub worker_name: String, pub segment_id: SegmentId, pub manifest_toml: String, pub greeting: protocol::Greeting, - pub status: RwLock, + state: RwLock, + accepted_commands: RwLock>, /// Worker-from-the-inside view of the filesystem. Set once in /// `WorkerController::start` after the local WorkdirSession provider is /// materialised, and read from the IPC server layer to answer @@ -38,13 +59,24 @@ impl WorkerSharedState { segment_id: SegmentId, manifest_toml: String, greeting: protocol::Greeting, + ) -> Self { + Self::new_with_generation(worker_name, segment_id, manifest_toml, greeting, 1) + } + + pub fn new_with_generation( + worker_name: String, + segment_id: SegmentId, + manifest_toml: String, + greeting: protocol::Greeting, + execution_generation: u64, ) -> Self { Self { worker_name, segment_id, manifest_toml, greeting, - status: RwLock::new(WorkerStatus::Idle), + state: RwLock::new(WorkerStateSnapshot::initial(execution_generation)), + accepted_commands: RwLock::new(VecDeque::new()), fs_view: OnceLock::new(), flow_transition_enabled: AtomicBool::new(false), } @@ -70,21 +102,140 @@ impl WorkerSharedState { self.flow_transition_enabled.load(Ordering::Acquire) } - pub fn set_status(&self, status: WorkerStatus) { - if let Ok(mut s) = self.status.write() { - *s = status; + pub fn transition(&self, state: WorkerState) -> WorkerStateSnapshot { + let mut snapshot = self + .state + .write() + .expect("worker state lock poisoned; refusing an inferred fallback state"); + if snapshot.state != state { + snapshot.revision = snapshot.revision.saturating_add(1); + snapshot.state = state; + } + snapshot.clone() + } + + pub(crate) fn admit_command( + &self, + envelope: WorkerCommandEnvelope, + kind: WorkerCommandKind, + require_state_revision: bool, + ) -> WorkerCommandAdmission { + let mut snapshot = self + .state + .write() + .expect("worker state lock poisoned; refusing command admission"); + let mut accepted = self + .accepted_commands + .write() + .expect("worker command ledger lock poisoned; refusing command admission"); + if let Some(existing) = accepted + .iter() + .find(|accepted| accepted.envelope.command_id == envelope.command_id) + { + return if existing.envelope == envelope && existing.kind == kind { + WorkerCommandAdmission::Retry + } else { + WorkerCommandAdmission::Conflict + }; + } + if envelope.expected_execution_generation != snapshot.execution_generation { + return WorkerCommandAdmission::ExecutionGenerationMismatch; + } + if require_state_revision && envelope.expected_worker_state_revision != snapshot.revision { + return WorkerCommandAdmission::StateRevisionMismatch; + } + if envelope.command_id <= snapshot.last_command_id { + return WorkerCommandAdmission::StaleCommandId; + } + + snapshot.last_command_id = envelope.command_id; + snapshot.revision = snapshot.revision.saturating_add(1); + accepted.push_back(AcceptedWorkerCommand { + envelope, + kind, + disposition: None, + }); + WorkerCommandAdmission::Accepted + } + + pub(crate) fn complete_command( + &self, + command_id: u64, + kind: WorkerCommandKind, + disposition: WorkerCommandDisposition, + ) { + if !matches!( + disposition, + WorkerCommandDisposition::Accepted | WorkerCommandDisposition::InvalidState + ) { + return; + } + let mut accepted = self + .accepted_commands + .write() + .expect("worker command ledger lock poisoned; refusing command completion"); + if let Some(command) = accepted + .iter_mut() + .find(|command| command.envelope.command_id == command_id && command.kind == kind) + { + command.disposition.get_or_insert(disposition); + } + while accepted + .iter() + .filter(|command| command.disposition.is_some()) + .count() + > COMPLETED_COMMAND_RETENTION + { + let Some(index) = accepted + .iter() + .position(|command| command.disposition.is_some()) + else { + break; + }; + accepted.remove(index); } } - pub fn get_status(&self) -> WorkerStatus { - self.status.read().map(|s| *s).unwrap_or(WorkerStatus::Idle) + #[cfg(test)] + pub(crate) fn command_result( + &self, + command_id: u64, + ) -> Option> { + self.accepted_commands + .read() + .expect("worker command ledger lock poisoned") + .iter() + .find(|command| command.envelope.command_id == command_id) + .map(|command| command.disposition) } - /// Serialize status as JSON. + pub fn snapshot(&self) -> WorkerStateSnapshot { + self.state + .read() + .expect("worker state lock poisoned; refusing an inferred fallback state") + .clone() + } + + /// Runtime catalog projection. This must not be used as live command + /// admission authority. + pub fn catalog_status(&self) -> WorkerStatus { + match self.snapshot().state { + WorkerState::Idle => WorkerStatus::Idle, + WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused)) => WorkerStatus::Paused, + WorkerState::Busy(WorkerBusyState::Run(_)) + | WorkerState::Busy(WorkerBusyState::Maintenance(WorkerMaintenanceState::Compacting)) => { + WorkerStatus::Running + } + } + } + + /// Serialize the runtime-directory lifecycle projection as JSON while + /// retaining the full state snapshot for diagnostics and reconnects. pub fn status_json(&self) -> String { - let status = self.get_status(); + let snapshot = self.snapshot(); json!({ - "state": status, + "state": self.catalog_status(), + "worker_state": snapshot, "segment_id": self.segment_id.to_string(), "worker_name": self.worker_name, }) @@ -97,11 +248,12 @@ mod tests { use super::*; fn test_state() -> WorkerSharedState { - WorkerSharedState::new( + WorkerSharedState::new_with_generation( "test-worker".into(), session_store::new_segment_id(), "[engine]\nname = \"test-worker\"".into(), test_greeting(), + 7, ) } @@ -119,36 +271,82 @@ mod tests { } #[test] - fn initial_status_is_idle() { + fn initial_snapshot_is_idle() { let state = test_state(); - assert_eq!(state.get_status(), WorkerStatus::Idle); + assert_eq!(state.snapshot(), WorkerStateSnapshot::initial(7)); + assert_eq!(state.catalog_status(), WorkerStatus::Idle); } #[test] - fn set_and_get_status() { + fn transitions_increment_revision_only_when_state_changes() { let state = test_state(); - state.set_status(WorkerStatus::Running); - assert_eq!(state.get_status(), WorkerStatus::Running); - state.set_status(WorkerStatus::Paused); - assert_eq!(state.get_status(), WorkerStatus::Paused); + let running = WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Running)); + let snapshot = state.transition(running.clone()); + assert_eq!(snapshot.revision, 1); + assert_eq!(snapshot.state, running); + assert_eq!(state.transition(running).revision, 1); + + let paused = WorkerState::Busy(WorkerBusyState::Run(WorkerRunState::Paused)); + let snapshot = state.transition(paused.clone()); + assert_eq!(snapshot.revision, 2); + assert_eq!(snapshot.state, paused); + assert_eq!(state.catalog_status(), WorkerStatus::Paused); } #[test] - fn status_json_contains_fields() { + fn accepted_command_identity_advances_revision_and_detects_reuse_conflicts() { let state = test_state(); - let json = state.status_json(); - let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); - assert_eq!(parsed["state"], "idle"); + let envelope = WorkerCommandEnvelope { + command_id: 9, + expected_execution_generation: 7, + expected_worker_state_revision: 0, + }; + assert_eq!( + state.admit_command(envelope, WorkerCommandKind::Pause, true), + WorkerCommandAdmission::Accepted + ); + assert_eq!( + state.snapshot(), + WorkerStateSnapshot { + execution_generation: 7, + revision: 1, + last_command_id: 9, + state: WorkerState::Idle, + } + ); + assert_eq!(state.command_result(9), Some(None)); + state.complete_command( + 9, + WorkerCommandKind::Pause, + WorkerCommandDisposition::Accepted, + ); + assert_eq!( + state.command_result(9), + Some(Some(WorkerCommandDisposition::Accepted)) + ); + assert_eq!( + state.admit_command(envelope, WorkerCommandKind::Pause, true), + WorkerCommandAdmission::Retry + ); + assert_eq!( + state.admit_command(envelope, WorkerCommandKind::Cancel, true), + WorkerCommandAdmission::Conflict + ); + assert_eq!(state.snapshot().revision, 1); + } + + #[test] + fn status_json_contains_full_snapshot_and_catalog_projection() { + let state = test_state(); + state.transition(WorkerState::Busy(WorkerBusyState::Maintenance( + WorkerMaintenanceState::Compacting, + ))); + let parsed: serde_json::Value = serde_json::from_str(&state.status_json()).unwrap(); + assert_eq!(parsed["state"], "running"); + assert_eq!(parsed["worker_state"]["execution_generation"], 7); + assert_eq!(parsed["worker_state"]["revision"], 1); + assert_eq!(parsed["worker_state"]["state"]["kind"], "busy"); assert_eq!(parsed["worker_name"], "test-worker"); assert!(parsed["segment_id"].is_string()); } - - #[test] - fn status_json_reflects_changes() { - let state = test_state(); - state.set_status(WorkerStatus::Running); - let json = state.status_json(); - let parsed: serde_json::Value = serde_json::from_str(&json).unwrap(); - assert_eq!(parsed["state"], "running"); - } } diff --git a/crates/worker/src/spawn/comm_tools.rs b/crates/worker/src/spawn/comm_tools.rs index 27e6845f..28efb9a6 100644 --- a/crates/worker/src/spawn/comm_tools.rs +++ b/crates/worker/src/spawn/comm_tools.rs @@ -72,6 +72,7 @@ mod tests { fn snapshot(entries: Vec) -> Event { Event::Snapshot { session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), entries: entries .into_iter() .enumerate() @@ -96,7 +97,7 @@ mod tests { context_window: 200_000, context_tokens: 0, }, - status: WorkerStatus::Idle, + state: WorkerStatus::Idle.into(), in_flight: Default::default(), internal_workers: Vec::new(), } @@ -136,10 +137,16 @@ mod tests { ], ); - connect_and_send(&socket, &Method::Shutdown).await.unwrap(); + let method = Method::Shutdown { + command: protocol::WorkerCommandEnvelope::for_snapshot( + 1, + &protocol::WorkerStateSnapshot::initial(1), + ), + }; + connect_and_send(&socket, &method).await.unwrap(); let method = received.await.unwrap().expect("expected method"); - assert!(matches!(method, Method::Shutdown)); + assert!(matches!(method, Method::Shutdown { .. })); } #[tokio::test] diff --git a/crates/worker/src/spawn/registry.rs b/crates/worker/src/spawn/registry.rs index 62fec8d4..fd946e54 100644 --- a/crates/worker/src/spawn/registry.rs +++ b/crates/worker/src/spawn/registry.rs @@ -12,7 +12,7 @@ use std::collections::{BTreeMap, HashSet}; use std::io; use std::sync::{ Arc, Mutex, - atomic::{AtomicBool, AtomicU64, Ordering}, + atomic::{AtomicBool, AtomicU64, AtomicUsize, Ordering}, }; use std::time::Instant; @@ -23,9 +23,9 @@ use protocol::{Event, InternalWorkerKind, InternalWorkerRef, InternalWorkerSnaps use session_store::{ LoggedItem, WorkerMetadataStore, WorkerReclaimedChild, WorkerSpawnedChild, WorkerStoreError, }; -use tokio::sync::broadcast; +use tokio::sync::{Notify, broadcast}; use tracing::warn; -use workdir::WorkdirDelegation; +use workdir::WorkdirScopeLease; use crate::internal_worker::{InternalWorkerSessionHandle, InternalWorkerVisibility}; use crate::runtime::dir::{RuntimeDir, SpawnedWorkerRecord}; @@ -68,10 +68,11 @@ pub(crate) struct SubWorkerStopSummary { pub(crate) struct InternalSpawnedWorkerRecord { pub worker_name: String, pub scope_delegated: Vec, - pub workdir_delegation: Arc, + pub workdir_tool_scope: Arc, #[cfg(test)] pub installed_tools: Arc<[String]>, pub session: InternalWorkerSessionHandle, + pub child_registry: Arc, change_tracker: Option, started_at: Instant, stop_lock: Arc>, @@ -86,18 +87,20 @@ impl InternalSpawnedWorkerRecord { pub(crate) fn new( worker_name: String, scope_delegated: Vec, - workdir_delegation: WorkdirDelegation, + workdir_tool_scope: WorkdirScopeLease, #[cfg(test)] installed_tools: Vec, session: InternalWorkerSessionHandle, + child_registry: Arc, change_tracker: Option, ) -> Self { Self { worker_name, scope_delegated, - workdir_delegation: Arc::new(workdir_delegation), + workdir_tool_scope: Arc::new(workdir_tool_scope), #[cfg(test)] installed_tools: installed_tools.into(), session, + child_registry, change_tracker, started_at: Instant::now(), stop_lock: Arc::new(tokio::sync::Mutex::new(())), @@ -235,18 +238,56 @@ pub(crate) struct InternalSpawnReservation { } impl InternalSpawnReservation { - pub(crate) fn commit(mut self, record: InternalSpawnedWorkerRecord) -> io::Result<()> { - if record.worker_name != self.worker_name { - return Err(io::Error::new( + pub(crate) async fn commit(mut self, record: InternalSpawnedWorkerRecord) -> io::Result<()> { + let rejection = if record.worker_name != self.worker_name { + Some(io::Error::new( io::ErrorKind::InvalidInput, "internal SubWorker reservation name does not match record name", - )); + )) + } else { + match self.registry.internal_records.lock() { + Ok(mut records) => { + if self.registry.internal_shutting_down.load(Ordering::Acquire) { + Some(io::Error::new( + io::ErrorKind::Interrupted, + "internal SubWorker registry is shutting down", + )) + } else { + records.push(record.clone()); + None + } + } + Err(_) => Some(io::Error::other( + "internal spawned-worker registry lock poisoned", + )), + } + }; + if let Some(error) = rejection { + let mut cleanup_failures = Vec::new(); + if let Err(cleanup) = record.session.stop().await { + cleanup_failures.push(format!("stop rejected Internal SubWorker: {cleanup}")); + } + if let Err(cleanup) = Box::pin(record.child_registry.shutdown_internal()).await { + cleanup_failures.push(format!( + "stop rejected Internal SubWorker descendants: {cleanup}" + )); + } + if let Err(cleanup) = record.workdir_tool_scope.close().await { + cleanup_failures.push(format!( + "close rejected Internal SubWorker Workdir tools: {cleanup}" + )); + } + if cleanup_failures.is_empty() { + return Err(error); + } + self.registry + .internal_spawn_cleanup_failed + .store(true, Ordering::Release); + return Err(io::Error::other(format!( + "{error}; {}", + cleanup_failures.join("; ") + ))); } - self.registry - .internal_records - .lock() - .map_err(|_| io::Error::other("internal spawned-worker registry lock poisoned"))? - .push(record.clone()); self.registry.start_protocol_forwarding(record); self.committed = true; Ok(()) @@ -260,6 +301,10 @@ impl Drop for InternalSpawnReservation { names.remove(&self.worker_name); } } + self.registry + .pending_internal_spawns + .fetch_sub(1, Ordering::AcqRel); + self.registry.pending_internal_notify.notify_waiters(); } } @@ -267,6 +312,10 @@ pub struct SpawnedWorkerRegistry { internal_records: std::sync::Mutex>, service_records: std::sync::Mutex>, internal_names: std::sync::Mutex>, + internal_shutting_down: AtomicBool, + pending_internal_spawns: AtomicUsize, + pending_internal_notify: Notify, + internal_spawn_cleanup_failed: AtomicBool, parent_scope: Option, parent_protocol: Mutex, String)>>, } @@ -283,6 +332,10 @@ impl SpawnedWorkerRegistry { internal_records: std::sync::Mutex::new(Vec::new()), service_records: std::sync::Mutex::new(Vec::new()), internal_names: std::sync::Mutex::new(HashSet::new()), + internal_shutting_down: AtomicBool::new(false), + pending_internal_spawns: AtomicUsize::new(0), + pending_internal_notify: Notify::new(), + internal_spawn_cleanup_failed: AtomicBool::new(false), parent_scope: None, parent_protocol: Mutex::new(None), }) @@ -294,6 +347,10 @@ impl SpawnedWorkerRegistry { internal_records: std::sync::Mutex::new(Vec::new()), service_records: std::sync::Mutex::new(Vec::new()), internal_names: std::sync::Mutex::new(HashSet::new()), + internal_shutting_down: AtomicBool::new(false), + pending_internal_spawns: AtomicUsize::new(0), + pending_internal_notify: Notify::new(), + internal_spawn_cleanup_failed: AtomicBool::new(false), parent_scope: None, parent_protocol: Mutex::new(None), }) @@ -304,6 +361,10 @@ impl SpawnedWorkerRegistry { internal_records: std::sync::Mutex::new(Vec::new()), service_records: std::sync::Mutex::new(Vec::new()), internal_names: std::sync::Mutex::new(HashSet::new()), + internal_shutting_down: AtomicBool::new(false), + pending_internal_spawns: AtomicUsize::new(0), + pending_internal_notify: Notify::new(), + internal_spawn_cleanup_failed: AtomicBool::new(false), parent_scope: Some(parent_scope), parent_protocol: Mutex::new(None), }) @@ -383,6 +444,10 @@ impl SpawnedWorkerRegistry { internal_records: std::sync::Mutex::new(Vec::new()), service_records: std::sync::Mutex::new(Vec::new()), internal_names: std::sync::Mutex::new(HashSet::new()), + internal_shutting_down: AtomicBool::new(false), + pending_internal_spawns: AtomicUsize::new(0), + pending_internal_notify: Notify::new(), + internal_spawn_cleanup_failed: AtomicBool::new(false), parent_scope, parent_protocol: Mutex::new(None), }), @@ -394,6 +459,16 @@ impl SpawnedWorkerRegistry { self: &Arc, worker_name: String, ) -> io::Result { + let records = self + .internal_records + .lock() + .map_err(|_| io::Error::other("internal Worker registry lock poisoned"))?; + if self.internal_shutting_down.load(Ordering::Acquire) { + return Err(io::Error::new( + io::ErrorKind::Interrupted, + "internal SubWorker registry is shutting down", + )); + } let mut names = self .internal_names .lock() @@ -404,7 +479,9 @@ impl SpawnedWorkerRegistry { format!("spawned worker `{worker_name}` is already registered"), )); } + self.pending_internal_spawns.fetch_add(1, Ordering::AcqRel); drop(names); + drop(records); Ok(InternalSpawnReservation { registry: Arc::clone(self), worker_name, @@ -679,18 +756,11 @@ impl SpawnedWorkerRegistry { .unwrap_or_default() } - pub(crate) fn reclaim_internal_scope(&self, worker_name: &str) -> io::Result { - let record = self.get_internal(worker_name).ok_or_else(|| { - io::Error::new(io::ErrorKind::NotFound, "internal SubWorker not found") - })?; - self.reclaim_record_scope(&record) - } - fn reclaim_record_scope(&self, record: &InternalSpawnedWorkerRecord) -> io::Result { if !record.claim_scope_reclaim() { return Ok(false); } - record.workdir_delegation.release(); + record.workdir_tool_scope.revoke(); let result = if let Some(parent_scope) = &self.parent_scope { parent_scope .update(|current| current.with_removed_deny_rules(delegated_write_rules(record))) @@ -705,6 +775,58 @@ impl SpawnedWorkerRegistry { result } + pub(crate) async fn close_internal_scope(&self, name: &str) -> io::Result { + let Some(record) = self.get_internal(name) else { + return Ok(false); + }; + Box::pin(record.child_registry.shutdown_internal()).await?; + record + .workdir_tool_scope + .close() + .await + .map_err(|error| io::Error::other(error.to_string()))?; + self.reclaim_record_scope(&record) + } + + pub(crate) async fn shutdown_internal(&self) -> io::Result<()> { + let names = { + let records = self + .internal_records + .lock() + .map_err(|_| io::Error::other("internal Worker registry lock poisoned"))?; + self.internal_shutting_down.store(true, Ordering::Release); + records + .iter() + .map(|record| record.worker_name.clone()) + .collect::>() + }; + loop { + let notified = self.pending_internal_notify.notified(); + if self.pending_internal_spawns.load(Ordering::Acquire) == 0 { + break; + } + notified.await; + } + let mut first_error = None; + for name in names { + if let Err(error) = self.remove_internal(&name).await { + first_error.get_or_insert(error); + } + } + if first_error.is_none() && self.internal_spawn_cleanup_failed.load(Ordering::Acquire) { + first_error = Some(io::Error::other( + "an in-flight Internal SubWorker failed cleanup during shutdown", + )); + } + first_error.map_or(Ok(()), Err) + } + + pub(crate) fn reopen_internal(&self) { + self.internal_shutting_down.store(false, Ordering::Release); + self.internal_spawn_cleanup_failed + .store(false, Ordering::Release); + } + /// Stop one direct Internal SubWorker and discard its registry/scope state. /// /// The child actor must acknowledge its stop before the registry is removed. @@ -731,6 +853,12 @@ impl SpawnedWorkerRegistry { .stop() .await .map_err(|error| io::Error::other(error.to_string()))?; + Box::pin(record.child_registry.shutdown_internal()).await?; + record + .workdir_tool_scope + .close() + .await + .map_err(|error| io::Error::other(error.to_string()))?; let summary = record.stop_summary(); self.reclaim_record_scope(&record)?; let removed = @@ -966,7 +1094,7 @@ mod tests { deny: Vec::new(), }) .unwrap(); - let source = workdir::delegation_capable_session(Arc::new( + let source = workdir::WorkdirToolBroker::new(Arc::new( workdir::LocalWorkdirSession::materialized_bound( workdir::Workdir::new("registry-test"), root.clone(), @@ -976,13 +1104,14 @@ mod tests { ), )); let delegation = source - .delegate(workdir::WorkdirDelegationRequest { - rules: vec![workdir::WorkdirDelegationRule { + .scope(workdir::WorkdirToolScope { + rules: vec![workdir::WorkdirToolScopeRule { target: workdir::WorkdirPath::new("").unwrap(), - permission: workdir::WorkdirDelegationPermission::Read, + permission: workdir::WorkdirToolScopePermission::Read, recursive: true, }], cwd: workdir::WorkdirPath::new("").unwrap(), + command: false, }) .await .unwrap(); @@ -993,6 +1122,7 @@ mod tests { delegation, Vec::new(), session, + registry(), None, ), sender, @@ -1230,6 +1360,143 @@ mod tests { } } + #[tokio::test] + async fn parent_shutdown_stops_all_internal_workers_before_returning() { + let registry = registry(); + for name in ["first", "second"] { + let (record, _events) = record(name, InternalWorkerVisibility::ParentClient).await; + record + .session + .force_status(InternalWorkerSessionStatus::Running); + install_record(®istry, record); + } + + registry.shutdown_internal().await.unwrap(); + + assert!(registry.list_internal().is_empty()); + assert!(registry.get_internal("first").is_none()); + assert!(registry.get_internal("second").is_none()); + } + + #[tokio::test] + async fn shutdown_rejects_new_reservations_until_reopened() { + let registry = registry(); + registry.shutdown_internal().await.unwrap(); + assert!(registry.reserve_internal_name("late-child".into()).is_err()); + + registry.reopen_internal(); + let reservation = registry.reserve_internal_name("late-child".into()).unwrap(); + drop(reservation); + } + + #[tokio::test] + async fn concurrent_commit_and_shutdown_leave_no_live_internal_worker() { + let registry = registry(); + let reservation = registry + .reserve_internal_name("racing-child".into()) + .unwrap(); + let (record, _events) = + record("racing-child", InternalWorkerVisibility::ParentClient).await; + let scope = record.workdir_tool_scope.clone(); + let barrier = Arc::new(tokio::sync::Barrier::new(2)); + let commit_barrier = barrier.clone(); + let commit = tokio::spawn(async move { + commit_barrier.wait().await; + reservation.commit(record).await + }); + let shutdown_registry = registry.clone(); + let shutdown = tokio::spawn(async move { + barrier.wait().await; + shutdown_registry.shutdown_internal().await + }); + + let commit = commit.await.unwrap(); + shutdown.await.unwrap().unwrap(); + if let Err(error) = commit { + assert_eq!(error.kind(), io::ErrorKind::Interrupted); + } + + assert!(registry.list_internal().is_empty()); + assert!(!scope.is_active()); + } + + #[tokio::test] + async fn shutdown_fences_a_reservation_that_has_not_committed() { + let registry = registry(); + let reservation = registry + .reserve_internal_name("racing-child".into()) + .unwrap(); + let (record, _events) = + record("racing-child", InternalWorkerVisibility::ParentClient).await; + + let mut shutdown = { + let registry = registry.clone(); + tokio::spawn(async move { registry.shutdown_internal().await }) + }; + while !registry.internal_shutting_down.load(Ordering::Acquire) { + tokio::task::yield_now().await; + } + assert!( + tokio::time::timeout(std::time::Duration::from_millis(50), &mut shutdown) + .await + .is_err(), + "shutdown must wait for the pending spawn to roll back" + ); + let error = reservation.commit(record).await.unwrap_err(); + assert_eq!(error.kind(), io::ErrorKind::Interrupted); + shutdown.await.unwrap().unwrap(); + } + + #[tokio::test] + async fn rejected_spawn_cleanup_failure_keeps_shutdown_failed_closed() { + let registry = registry(); + let reservation = registry + .reserve_internal_name("cleanup-failure".into()) + .unwrap(); + let (record, _events) = + record("cleanup-failure", InternalWorkerVisibility::ParentClient).await; + record.session.force_stop_failure(); + let shutdown = { + let registry = registry.clone(); + tokio::spawn(async move { registry.shutdown_internal().await }) + }; + while !registry.internal_shutting_down.load(Ordering::Acquire) { + tokio::task::yield_now().await; + } + + let error = reservation.commit(record).await.unwrap_err(); + assert!( + error + .to_string() + .contains("stop rejected Internal SubWorker") + ); + let shutdown_error = shutdown.await.unwrap().unwrap_err(); + assert!( + shutdown_error + .to_string() + .contains("failed cleanup during shutdown") + ); + assert!(registry.internal_shutting_down.load(Ordering::Acquire)); + } + + #[tokio::test] + async fn shutdown_recursively_stops_grandchildren_before_parent_scope_release() { + let registry = registry(); + let (child, _child_events) = record("child", InternalWorkerVisibility::ParentClient).await; + let child_registry = child.child_registry.clone(); + let (grandchild, _grandchild_events) = + record("grandchild", InternalWorkerVisibility::ParentClient).await; + let grandchild_scope = grandchild.workdir_tool_scope.clone(); + install_record(&child_registry, grandchild); + install_record(®istry, child); + + registry.shutdown_internal().await.unwrap(); + + assert!(registry.list_internal().is_empty()); + assert!(child_registry.list_internal().is_empty()); + assert!(!grandchild_scope.is_active()); + } + #[tokio::test] async fn running_worker_is_stopped_before_removal() { let registry = registry(); diff --git a/crates/worker/src/spawn/tool.rs b/crates/worker/src/spawn/tool.rs index e6b8f872..6ec7c3cc 100644 --- a/crates/worker/src/spawn/tool.rs +++ b/crates/worker/src/spawn/tool.rs @@ -22,8 +22,7 @@ use manifest::{ use serde::Deserialize; use tokio::sync::mpsc; use workdir::{ - WorkdirDelegationPermission, WorkdirDelegationRequest, WorkdirDelegationRule, WorkdirPath, - WorkdirSessionHandle, + WorkdirToolBroker, WorkdirToolScope, WorkdirToolScopePermission, WorkdirToolScopeRule, }; use crate::PromptCatalogSource; @@ -58,12 +57,15 @@ struct SubWorkerSpawnInput { /// a host path and grants no authority. When omitted, the Workdir root is used. #[serde(default)] cwd: Option, - /// First message sent to the spawned SubWorker via `Method::Run`. + /// First message sent to the spawned SubWorker via `Method::Submit`. task: String, /// Allow rules delegated to the spawned SubWorker. Must be a subset of the /// spawner's explicit delegation authority; direct tool scope alone is not /// sufficient. Omit `recursive` for normal workspace/worktree delegation; it defaults to true. scope: Vec, + /// Explicitly grant command execution through the parent-owned Workdir tool broker. + #[serde(default)] + command: bool, /// Binds an actual read-only builtin Reviewer child to the current Merge Request candidate. /// Review capability material is generated by the trusted spawn layer. #[serde(default)] @@ -219,33 +221,50 @@ fn parse_spawn_profile_selector(raw: Option<&str>) -> Result), - Buffer(crate::ipc::notify_buffer::NotifyBuffer), + Controller { + sender: mpsc::WeakSender, + fallback: Arc, + }, + Durable(Arc), } impl ParentNotificationTarget { - fn notify(&self, message: String, auto_run: bool) { + pub(crate) fn with_controller_fallback( + sender: mpsc::WeakSender, + fallback: ParentNotificationTarget, + ) -> Self { + let ParentNotificationTarget::Durable(fallback) = fallback else { + unreachable!("controller fallback must use durable pending authority"); + }; + Self::Controller { sender, fallback } + } + + pub(crate) fn notify(&self, child_session_id: String, message: String, auto_run: bool) { + let method = Method::NotifyTracked { + notification_request_id: protocol::new_submission_request_id(), + message, + auto_run, + source: protocol::AuthenticatedInputSource::SubWorker { + session_id: child_session_id, + }, + }; match self { - Self::Controller(parent_method_tx) => { - let Some(parent_method_tx) = parent_method_tx.upgrade() else { - tracing::warn!( - "parent Worker controller closed before Internal SubWorker completion notification" - ); + Self::Controller { sender, fallback } => { + let Some(parent_method_tx) = sender.upgrade() else { + fallback(method); return; }; + let fallback = fallback.clone(); tokio::spawn(async move { - if let Err(error) = parent_method_tx - .send(Method::Notify { message, auto_run }) - .await - { + if let Err(error) = parent_method_tx.send(method).await { tracing::warn!( - %error, - "failed to notify parent Worker about Internal SubWorker completion" + "failed to notify parent Controller; using durable pending authority" ); + fallback(error.0); } }); } - Self::Buffer(parent_notifies) => parent_notifies.push_notify(message, auto_run), + Self::Durable(notify) => notify(method), } } } @@ -267,8 +286,8 @@ pub struct SubWorkerSpawnTool { workspace_root: PathBuf, /// Directory the spawned SubWorker's tools should use when the LLM did not /// override it. Defaults to the spawner's cwd. - /// Active provider-backed Workdir session from which child leases are captured. - source_workdir_session: Option, + /// Parent-owned broker for scoped Workdir tool execution. + workdir_tool_broker: Option, /// Parent-owned in-memory registry shared by the five SubWorker tools. registry: Arc, /// Spawner's resolved Manifest. `profile = "inherit"` derives the @@ -295,7 +314,7 @@ impl SubWorkerSpawnTool { runtime_base: PathBuf, bash_output_dir: PathBuf, workspace_root: PathBuf, - source_workdir_session: Option, + workdir_tool_broker: Option, registry: Arc, spawner_manifest: WorkerManifest, prompt_loader: PromptCatalogSource, @@ -308,7 +327,7 @@ impl SubWorkerSpawnTool { runtime_base, bash_output_dir, workspace_root, - source_workdir_session, + workdir_tool_broker, registry, spawner_manifest, prompt_loader, @@ -341,6 +360,11 @@ fn validate_reviewer_handoff(input: &SubWorkerSpawnInput) -> Result<(), ToolErro "Merge Request Reviewer SubWorkers must include writable delegated scope".to_string(), )); } + if !input.command { + return Err(ToolError::InvalidArgument( + "Merge Request Reviewer SubWorkers require an explicit command grant".to_string(), + )); + } Ok(()) } @@ -370,7 +394,7 @@ impl Tool for SubWorkerSpawnTool { .reserve_internal_name(input.name.clone()) .map_err(|error| ToolError::InvalidArgument(error.to_string()))?; - let mut workdir_rules = parse_workdir_scope(&input.scope)?; + let workdir_rules = parse_workdir_scope(&input.scope)?; let child_bash_output_dir = self.bash_output_dir.join("sub-workers").join(&input.name); tokio::fs::create_dir_all(&child_bash_output_dir) .await @@ -380,28 +404,15 @@ impl Tool for SubWorkerSpawnTool { child_bash_output_dir.display() )) })?; - let source_workdir_session = - require_active_workdir_session(self.source_workdir_session.as_ref())?; - let transports_delegation_context = source_workdir_session.transports_delegation_context(); - // Provider-transported sessions resolve every delegation rule in the - // receiving Workdir namespace. The Bash spill directory instead belongs - // to this Worker host, so forwarding it would widen the request with a - // foreign absolute path and fail the provider's existing scope check. - if !transports_delegation_context { - workdir_rules.push(WorkdirDelegationRule { - target: WorkdirPath::new_scoped(child_bash_output_dir.to_string_lossy()) - .map_err(|error| ToolError::ExecutionFailed(error.to_string()))?, - permission: WorkdirDelegationPermission::Read, - recursive: true, - }); - } - let delegation_request = workdir_delegation_request(input.cwd.as_deref(), workdir_rules)?; - let workdir_delegation = source_workdir_session - .delegate(delegation_request) + let workdir_tool_broker = require_workdir_tool_broker(self.workdir_tool_broker.as_ref())?; + let tool_scope = workdir_tool_scope(input.cwd.as_deref(), workdir_rules, input.command)?; + let workdir_scope = workdir_tool_broker + .scope(tool_scope) .await .map_err(|error| { - ToolError::InvalidArgument(format!("delegate Workdir session: {error}")) + ToolError::InvalidArgument(format!("scope parent-owned Workdir tools: {error}")) })?; + let child_workdir_tool_broker = workdir_scope.broker(); let spawn_selector = parse_spawn_profile_selector(input.profile.as_deref()).map_err(|msg| { @@ -490,7 +501,6 @@ impl Tool for SubWorkerSpawnTool { ) .await .map_err(|error| ToolError::ExecutionFailed(format!("build Internal Worker: {error}")))?; - child.bind_workdir_session(Some(workdir_delegation.scoped_session.clone())); child .add_scope_rules([ScopeRule { target: child_bash_output_dir.clone(), @@ -510,6 +520,7 @@ impl Tool for SubWorkerSpawnTool { self.runtime_base.clone(), child_registry.clone(), None, + Some(child_workdir_tool_broker.clone()), ) .await .map_err(|error| { @@ -538,20 +549,24 @@ impl Tool for SubWorkerSpawnTool { InternalWorkerSessionStatus::Failed | InternalWorkerSessionStatus::Stopped ) { if let Some(registry) = registry.upgrade() { - if let Err(error) = registry.reclaim_internal_scope(&child_name) { - tracing::warn!( - child_name, - %error, - "failed to reclaim delegated scope after Internal SubWorker failure" - ); - } + let child_name = child_name.clone(); + tokio::spawn(async move { + if let Err(error) = registry.close_internal_scope(&child_name).await { + tracing::warn!( + child_name, + %error, + "failed to close parent-owned Workdir tools after Internal SubWorker failure" + ); + } + }); } } let message = format!( "SubWorker `{child_name}` turn ended with status {status:?}. Inspect its committed session with worker-observation tools before making completion decisions." ); - parent_notifications.notify(message, true); + parent_notifications.notify(child_name.clone(), message, true); })), + Some(child_workdir_tool_broker.clone()), ) .await; let session = session_result.map_err(|error| { @@ -602,15 +617,19 @@ impl Tool for SubWorkerSpawnTool { ), body.to_string(), ); - let response = self - .workspace_context - .client() - .execute(request) - .map_err(|error| { - ToolError::ExecutionFailed(format!("register review capability: {error}")) - })?; + let response = match self.workspace_context.client().execute(request) { + Ok(response) => response, + Err(error) => { + let _ = session.stop().await; + let _ = workdir_scope.close().await; + return Err(ToolError::ExecutionFailed(format!( + "register review capability: {error}" + ))); + } + }; if !response.is_success() { let _ = session.stop().await; + let _ = workdir_scope.close().await; return Err(ToolError::ExecutionFailed(format!( "register review capability failed with status {}: {}", response.status, response.body @@ -621,14 +640,14 @@ impl Tool for SubWorkerSpawnTool { let record = crate::spawn::registry::InternalSpawnedWorkerRecord::new( input.name.clone(), scope_allow, - workdir_delegation, + workdir_scope, #[cfg(test)] installed_tools, session.clone(), + child_registry, child_change_tracker, ); - if let Err(error) = name_reservation.commit(record) { - let _ = session.stop().await; + if let Err(error) = name_reservation.commit(record).await { return Err(ToolError::ExecutionFailed(format!( "register Internal Worker session: {error}" ))); @@ -674,18 +693,18 @@ fn logical_workdir_path(value: &str, field: &str) -> Result { }) } -fn parse_workdir_scope(rules: &[ScopeRuleInput]) -> Result, ToolError> { +fn parse_workdir_scope(rules: &[ScopeRuleInput]) -> Result, ToolError> { if rules.is_empty() { return Err(ToolError::InvalidArgument("scope must not be empty".into())); } rules .iter() .map(|rule| { - Ok(WorkdirDelegationRule { + Ok(WorkdirToolScopeRule { target: logical_workdir_path(&rule.target, "scope.target")?, permission: match rule.permission { - PermissionInput::Read => WorkdirDelegationPermission::Read, - PermissionInput::Write => WorkdirDelegationPermission::Write, + PermissionInput::Read => WorkdirToolScopePermission::Read, + PermissionInput::Write => WorkdirToolScopePermission::Write, }, recursive: rule.recursive, }) @@ -693,22 +712,24 @@ fn parse_workdir_scope(rules: &[ScopeRuleInput]) -> Result, - rules: Vec, -) -> Result { - Ok(WorkdirDelegationRequest { + rules: Vec, + command: bool, +) -> Result { + Ok(WorkdirToolScope { rules, cwd: logical_workdir_path(cwd.unwrap_or("."), "cwd")?, + command, }) } -fn require_active_workdir_session( - session: Option<&WorkdirSessionHandle>, -) -> Result<&WorkdirSessionHandle, ToolError> { - session.ok_or_else(|| { +fn require_workdir_tool_broker( + broker: Option<&WorkdirToolBroker>, +) -> Result<&WorkdirToolBroker, ToolError> { + broker.ok_or_else(|| { ToolError::InvalidArgument( - "SubWorkerSpawn requires an active Workdir session; attach a Workdir before delegating filesystem access" + "SubWorkerSpawn requires parent-owned Workdir tools; attach a Workdir before granting filesystem access" .to_string(), ) }) @@ -946,7 +967,7 @@ pub(crate) fn sub_worker_spawn_tool( runtime_base: PathBuf, bash_output_dir: PathBuf, workspace_root: PathBuf, - source_workdir_session: Option, + workdir_tool_broker: Option, registry: Arc, spawner_manifest: WorkerManifest, prompts: Arc>, @@ -958,7 +979,7 @@ pub(crate) fn sub_worker_spawn_tool( runtime_base, bash_output_dir, workspace_root, - source_workdir_session, + workdir_tool_broker, registry, spawner_manifest, prompts, @@ -972,7 +993,7 @@ fn sub_worker_spawn_tool_impl( runtime_base: PathBuf, bash_output_dir: PathBuf, workspace_root: PathBuf, - source_workdir_session: Option, + workdir_tool_broker: Option, registry: Arc, spawner_manifest: WorkerManifest, prompts: Arc>, @@ -1004,7 +1025,7 @@ fn sub_worker_spawn_tool_impl( runtime_base.clone(), bash_output_dir.clone(), workspace_root.clone(), - source_workdir_session.clone(), + workdir_tool_broker.clone(), registry.clone(), spawner_manifest.clone(), prompts.load_full().source(), @@ -1037,12 +1058,12 @@ mod tests { }; #[test] - fn missing_active_workdir_session_fails_deterministically() { - let error = require_active_workdir_session(None).unwrap_err(); + fn missing_parent_workdir_tool_broker_fails_deterministically() { + let error = require_workdir_tool_broker(None).unwrap_err(); assert!(matches!( error, ToolError::InvalidArgument(message) - if message.contains("requires an active Workdir session") + if message.contains("requires parent-owned Workdir tools") )); } @@ -1079,6 +1100,7 @@ mod tests { let valid: SubWorkerSpawnInput = serde_json::from_value(serde_json::json!({ "name":"reviewer","task":"review","profile":"builtin:reviewer", "scope":[{"target":"work","permission":"write"}], + "command":true, "review":{"ticket_id":"T1"} })) .unwrap(); @@ -1134,12 +1156,41 @@ enabled = false #[tokio::test] async fn parent_controller_notification_target_does_not_keep_channel_open() { let (parent_method_tx, mut parent_method_rx) = mpsc::channel(1); - let target = ParentNotificationTarget::Controller(parent_method_tx.downgrade()); + let captured = Arc::new(std::sync::Mutex::new(false)); + let captured_for_fallback = captured.clone(); + let target = ParentNotificationTarget::with_controller_fallback( + parent_method_tx.downgrade(), + ParentNotificationTarget::Durable(Arc::new(move |_| { + *captured_for_fallback.lock().unwrap() = true; + })), + ); drop(parent_method_tx); assert!(parent_method_rx.recv().await.is_none()); - target.notify("late completion".to_string(), true); + target.notify("child-session".into(), "late completion".to_string(), true); + assert!(*captured.lock().unwrap()); + } + + #[test] + fn durable_parent_notification_target_preserves_child_source() { + let captured = Arc::new(std::sync::Mutex::new(None)); + let captured_for_target = captured.clone(); + let target = ParentNotificationTarget::Durable(Arc::new(move |method| { + *captured_for_target.lock().unwrap() = Some(method); + })); + + target.notify("child-session".into(), "completed".into(), true); + + assert!(matches!( + captured.lock().unwrap().take(), + Some(Method::NotifyTracked { + message, + auto_run: true, + source: protocol::AuthenticatedInputSource::SubWorker { session_id }, + .. + }) if session_id == "child-session" && message == "completed" + )); } #[tokio::test] @@ -1173,7 +1224,7 @@ enabled = false let fail_requests = Arc::new(AtomicBool::new(false)); let prompt_loader = PromptCatalogSource::builtins_only(); let (parent_method_tx, mut parent_method_rx) = mpsc::channel(8); - let source_workdir_session = workdir::delegation_capable_session(Arc::new( + let workdir_tool_broker = workdir::WorkdirToolBroker::new(Arc::new( workdir::LocalWorkdirSession::materialized_bound( workdir::Workdir::new("test-workdir"), workspace_root.clone(), @@ -1185,11 +1236,14 @@ enabled = false let tool = SubWorkerSpawnTool::new( "parent".into(), workspace_context, - ParentNotificationTarget::Controller(parent_method_tx.downgrade()), + ParentNotificationTarget::with_controller_fallback( + parent_method_tx.downgrade(), + ParentNotificationTarget::Durable(Arc::new(|_| {})), + ), runtime.path().to_path_buf(), bash_output_dir.clone(), workspace_root.clone(), - Some(source_workdir_session), + Some(workdir_tool_broker), registry.clone(), manifest.clone(), prompt_loader, @@ -1212,7 +1266,8 @@ enabled = false "target": ".", "permission": "write", "recursive": true - }] + }], + "command": true }); assert!(spawner_scope.snapshot().is_writable(&workspace_root)); @@ -1247,15 +1302,6 @@ enabled = false let record = registry .get_internal("reviewer-child") .expect("Internal reviewer registry record"); - let child_bash_output_dir = bash_output_dir.join("sub-workers").join("reviewer-child"); - record - .workdir_delegation - .scoped_session - .stat(workdir::StatRequest { - path: WorkdirPath::new_scoped(child_bash_output_dir.to_string_lossy()).unwrap(), - }) - .await - .expect("local child retains read scope for its Bash output directory"); for required in ["Read", "Write", "Edit", "Glob", "Grep", "Bash"] { assert!( record.installed_tools.iter().any(|name| name == required), @@ -1282,10 +1328,13 @@ enabled = false .expect("parent method channel remains open"); assert!(matches!( completion, - Method::Notify { + Method::NotifyTracked { message, auto_run: true, - } if message.contains("SubWorker `reviewer-child` turn ended with status Idle") + source: protocol::AuthenticatedInputSource::SubWorker { session_id }, + .. + } if session_id == "reviewer-child" + && message.contains("SubWorker `reviewer-child` turn ended with status Idle") )); assert!(!runtime.path().join("reviewer-child/sock").exists()); @@ -1371,7 +1420,7 @@ enabled = false "Stopped terminal child must release its delegated Workdir session" ); assert!( - !record.workdir_delegation.is_active(), + !record.workdir_tool_scope.is_active(), "stopped child must revoke cloned scoped sessions" ); assert!(registry.get_internal("reviewer-child").is_some()); @@ -1426,7 +1475,7 @@ enabled = false Arc::new(AvailableWorkspaceClient), ); let remote_client = Arc::new(StrictRemoteWorkdirWorkspaceClient::default()); - let source_workdir_session = workdir::delegation_capable_session( + let workdir_tool_broker = workdir::WorkdirToolBroker::new( WorkspaceAttachedWorkdirSession::handle(remote_client.clone()), ); let calls = Arc::new(AtomicUsize::new(0)); @@ -1434,11 +1483,14 @@ enabled = false let tool = SubWorkerSpawnTool::new( "parent".into(), workspace_context, - ParentNotificationTarget::Controller(parent_method_tx.downgrade()), + ParentNotificationTarget::with_controller_fallback( + parent_method_tx.downgrade(), + ParentNotificationTarget::Durable(Arc::new(|_| {})), + ), runtime.path().to_path_buf(), bash_output_dir.clone(), workspace_root.clone(), - Some(source_workdir_session), + Some(workdir_tool_broker), registry.clone(), manifest, PromptCatalogSource::builtins_only(), @@ -1478,51 +1530,12 @@ enabled = false record.session.wait_until_idle().await, crate::internal_worker::InternalWorkerSessionStatus::Idle ); + assert!(record.installed_tools.iter().any(|tool| tool == "Write")); + assert!(!record.installed_tools.iter().any(|tool| tool == "Bash")); assert_eq!(calls.load(Ordering::SeqCst), 1); - assert_eq!( - remote_client - .foreign_scope_rejections - .load(Ordering::SeqCst), - 0 - ); - let child_bash_output_dir = bash_output_dir.join("sub-workers").join("remote-child"); - assert!(child_bash_output_dir.is_dir()); - for required in ["Read", "Write", "Edit", "Glob", "Grep", "Bash"] { - assert!( - record.installed_tools.iter().any(|name| name == required), - "remote write-scoped child is missing {required}: {:?}", - record.installed_tools - ); - } - - let remote_requests = remote_client.requests(); - let operate_requests = remote_requests - .iter() - .filter(|request| request.body.is_some()) - .collect::>(); - assert_eq!( - operate_requests.len(), - 1, - "remote requests: {remote_requests:?}" - ); - let operation_body: serde_json::Value = serde_json::from_str( - operate_requests[0] - .body - .as_deref() - .expect("remote operation body"), - ) - .unwrap(); - let rules = operation_body["delegations"][0]["rules"] - .as_array() - .expect("delegation rules"); - assert_eq!(rules.len(), 1, "remote operation body: {operation_body}"); - assert_eq!(rules[0]["target"], ""); assert!( - !operation_body.to_string().contains( - child_bash_output_dir - .to_str() - .expect("UTF-8 test output directory") - ) + remote_client.requests().is_empty(), + "spawning a child must not open or delegate a provider Workdir session" ); } @@ -1534,6 +1547,7 @@ enabled = false .and_then(serde_json::Value::as_object) .expect("schema properties"); assert!(properties.contains_key("cwd"), "schema: {schema}"); + assert!(properties.contains_key("command"), "schema: {schema}"); let required = schema .get("required") .and_then(serde_json::Value::as_array) @@ -1663,7 +1677,6 @@ enabled = false #[derive(Debug, Default)] struct StrictRemoteWorkdirWorkspaceClient { requests: Mutex>, - foreign_scope_rejections: AtomicUsize, } impl StrictRemoteWorkdirWorkspaceClient { @@ -1695,59 +1708,10 @@ enabled = false self.requests .lock() .expect("remote Workdir request lock") - .push(request.clone()); - if request.path.ends_with("/fence") { - return Ok(WorkspaceResponse { - status: 200, - body: serde_json::json!({ "value": "remote-fence-1" }).to_string(), - }); - } - - let body: serde_json::Value = serde_json::from_str( - request - .body - .as_deref() - .ok_or_else(|| WorkspaceClientError::Request("missing request body".into()))?, - ) - .map_err(|error| WorkspaceClientError::Request(error.to_string()))?; - let has_foreign_scope = body - .get("delegations") - .and_then(serde_json::Value::as_array) - .into_iter() - .flatten() - .flat_map(|delegation| { - delegation - .get("rules") - .and_then(serde_json::Value::as_array) - .into_iter() - .flatten() - }) - .filter_map(|rule| rule.get("target").and_then(serde_json::Value::as_str)) - .any(|target| Path::new(target).is_absolute()); - if has_foreign_scope { - self.foreign_scope_rejections.fetch_add(1, Ordering::SeqCst); - return Ok(WorkspaceResponse { - status: 403, - body: serde_json::json!({ - "code": "out_of_scope", - "message": "Worker-host path is outside the remote Workdir namespace" - }) - .to_string(), - }); - } - - Ok(WorkspaceResponse { - status: 200, - body: serde_json::json!({ - "operation": "stat", - "result": { - "path": "", - "kind": "directory", - "size": 0 - } - }) - .to_string(), - }) + .push(request); + Err(WorkspaceClientError::Request( + "SubWorker spawn must not call the remote Workdir provider".into(), + )) } } diff --git a/crates/worker/src/worker.rs b/crates/worker/src/worker.rs index 55d041ff..93ec40ac 100644 --- a/crates/worker/src/worker.rs +++ b/crates/worker/src/worker.rs @@ -1,3 +1,4 @@ +use std::collections::VecDeque; #[cfg(test)] use std::path::Path; use std::path::PathBuf; @@ -68,6 +69,243 @@ const LARGE_PASTE_INLINE_MAX_BYTES: usize = 32 * 1024; const WORKER_ORCHESTRATION_INSTRUCTION_ID: &str = "worker.orchestration"; const WORKER_ORCHESTRATION_PROMPT_REF: &str = "common.worker_orchestration"; const FEATURE_HOOK_CHAIN_TIMEOUT: Duration = Duration::from_secs(30); +const SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN: &str = "worker.pending_activations.v1"; +const MAX_PENDING_SUBMISSIONS: usize = 32; +const MAX_PENDING_SUBMISSION_BYTES: u64 = 1024 * 1024; +const MAX_PENDING_ARTIFACT_REFS: usize = 64; +const MAX_ACTIVATION_REQUEST_ID_BYTES: usize = 128; +const MAX_SUBMISSION_RECEIPTS: usize = 128; + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub(crate) struct PendingSubmission { + pub(crate) submission_request_id: String, + source_namespace: String, + pub(crate) submission_id: String, + payload_digest: String, + accepted_at_ms: u64, + activation_sequence: u64, + pub(crate) provenance: WorkerHistoryProvenance, + #[serde(default)] + was_queued: bool, + pub(crate) input: Vec, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +struct SubmissionReceipt { + submission_request_id: String, + source_namespace: String, + submission_id: String, + payload_digest: String, + disposition: protocol::SubmissionDisposition, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub(crate) struct PendingNotification { + pub(crate) notification_request_id: String, + source_namespace: String, + pub(crate) message: String, + payload_digest: String, + pub(crate) auto_run: bool, + accepted_at_ms: u64, + activation_sequence: u64, + pub(crate) provenance: WorkerHistoryProvenance, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +struct NotificationReceipt { + notification_request_id: String, + source_namespace: String, + payload_digest: String, + auto_run: bool, +} + +#[derive(Debug, Clone, Default, serde::Serialize, serde::Deserialize)] +pub(crate) struct PendingActivationState { + revision: u64, + next_activation_sequence: u64, + /// A prepared activation remains in checkpoints until the same atomic + /// UserInput record commits the clearing checkpoint. Restore puts it back + /// at the FIFO head. + activating: Option, + activating_notification: Option, + pending: VecDeque, + pending_notifications: VecDeque, + receipts: VecDeque, + notification_receipts: VecDeque, +} + +impl PendingActivationState { + pub(crate) fn snapshot(&self) -> protocol::PendingSubmissionsSnapshot { + let pending_notification = self + .pending_notifications + .iter() + .find(|notification| notification.auto_run); + let head_id = match (self.pending.front(), pending_notification) { + (Some(submission), Some(notification)) + if notification.activation_sequence < submission.activation_sequence => + { + Some(notification_head_id( + ¬ification.source_namespace, + ¬ification.notification_request_id, + )) + } + (Some(submission), _) => Some(submission.submission_id.clone()), + (None, Some(notification)) => Some(notification_head_id( + ¬ification.source_namespace, + ¬ification.notification_request_id, + )), + (None, None) => None, + }; + protocol::PendingSubmissionsSnapshot { + revision: self.revision, + notification_count: u32::try_from(self.pending_notifications.len()).unwrap_or(u32::MAX), + head_id, + submissions: self + .pending + .iter() + .map(|pending| protocol::PendingSubmissionSummary { + submission_id: pending.submission_id.clone(), + accepted_at_ms: pending.accepted_at_ms, + segment_count: u32::try_from(pending.input.len()).unwrap_or(u32::MAX), + byte_len: submission_payload_len(&pending.input), + }) + .collect(), + } + } + + fn live_artifact_pin_owner_ids(&self) -> Vec { + self.activating + .iter() + .chain(self.pending.iter()) + .map(|submission| submission.submission_id.clone()) + .collect() + } + + fn remember_notification_receipt(&mut self, receipt: NotificationReceipt) { + self.notification_receipts.push_back(receipt); + while self.notification_receipts.len() > MAX_SUBMISSION_RECEIPTS { + self.notification_receipts.pop_front(); + } + } + + fn remember_receipt(&mut self, receipt: SubmissionReceipt) { + self.receipts.push_back(receipt); + while self.receipts.len() > MAX_SUBMISSION_RECEIPTS { + self.receipts.pop_front(); + } + } +} + +fn notification_payload_digest(message: &str, auto_run: bool) -> String { + use sha2::Digest as _; + let mut hasher = sha2::Sha256::new(); + hasher.update(if auto_run { + &b"auto\0"[..] + } else { + &b"deferred\0"[..] + }); + hasher.update(message.as_bytes()); + hasher + .finalize() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +pub(crate) fn authenticated_input_provenance( + source: &protocol::AuthenticatedInputSource, +) -> WorkerHistoryProvenance { + match source { + protocol::AuthenticatedInputSource::UntrustedWire => WorkerHistoryProvenance::LegacyUnknown, + protocol::AuthenticatedInputSource::Account { account_id } => { + WorkerHistoryProvenance::HumanInput { + account_id: account_id.clone(), + } + } + protocol::AuthenticatedInputSource::Worker { + runtime_id, + worker_id, + } => WorkerHistoryProvenance::WorkerInput { + actor: session_store::LoggedWorkerSubject { + workspace_id: None, + runtime_id: Some(runtime_id.clone()), + worker_id: worker_id.clone(), + }, + }, + protocol::AuthenticatedInputSource::SubWorker { session_id } => { + WorkerHistoryProvenance::WorkerInput { + actor: session_store::LoggedWorkerSubject { + workspace_id: None, + runtime_id: None, + worker_id: session_id.clone(), + }, + } + } + protocol::AuthenticatedInputSource::Backend { operation_id } => { + WorkerHistoryProvenance::BackendInstruction { + operation_id: Some(operation_id.clone()), + } + } + } +} + +fn notification_head_id(source_namespace: &str, request_id: &str) -> String { + use sha2::Digest as _; + let mut hasher = sha2::Sha256::new(); + hasher.update(source_namespace.as_bytes()); + hasher.update(b"\0"); + hasher.update(request_id.as_bytes()); + format!( + "notification:{}", + hasher + .finalize() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::() + ) +} + +fn submission_payload_len(input: &[Segment]) -> u64 { + serde_json::to_vec(input) + .map(|bytes| u64::try_from(bytes.len()).unwrap_or(u64::MAX)) + .unwrap_or(u64::MAX) +} + +fn submission_payload_digest(input: &[Segment]) -> String { + use sha2::Digest as _; + sha2::Sha256::digest(serde_json::to_vec(input).unwrap_or_default()) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() +} + +fn submission_uploaded_file_refs( + input: &[Segment], +) -> impl Iterator { + input.iter().filter_map(|segment| match segment { + Segment::UploadedFile { file } => Some(file), + _ => None, + }) +} + +fn submission_artifact_ref_count(input: &[Segment]) -> usize { + input + .iter() + .filter(|segment| { + matches!( + segment, + Segment::PasteArtifact { .. } | Segment::UploadedFile { .. } + ) + }) + .count() +} + +fn pending_activation_extension(state: &PendingActivationState) -> SessionExtension { + SessionExtension { + domain: SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN.to_owned(), + payload: serde_json::to_value(state).expect("pending activation state must serialize"), + } +} fn hook_run_exit(exit: &EngineRunExit) -> RunCommittedExit { match exit { @@ -970,18 +1208,740 @@ where } } -/// Type-erased commit handle for the interceptor. Lets the -/// interceptor commit `SystemItem`s without being generic over the +#[derive(Debug, Clone)] +pub(crate) enum PendingActivation { + Submission(PendingSubmission), + Notification(PendingNotification), +} + +#[derive(Debug, Clone)] +pub(crate) struct SubmissionAcceptance { + pub(crate) submission_request_id: String, + pub(crate) submission_id: String, + pub(crate) disposition: protocol::SubmissionDisposition, + pub(crate) activation: Option, +} + +#[derive(Debug, thiserror::Error)] +pub(crate) enum PendingSubmissionError { + #[error("submission_request_id must not be empty")] + EmptyRequestId, + #[error("activation request id exceeds {MAX_ACTIVATION_REQUEST_ID_BYTES} bytes")] + RequestIdLimit, + #[error("submission input must contain at least one typed segment")] + EmptyInput, + #[error("submission request id was already used with a different payload")] + IdempotencyConflict, + #[error("pending submission queue is full (maximum {MAX_PENDING_SUBMISSIONS})")] + CountLimit, + #[error("pending submission bytes exceed {MAX_PENDING_SUBMISSION_BYTES}")] + ByteLimit, + #[error("pending submission artifact references exceed {MAX_PENDING_ARTIFACT_REFS}")] + ArtifactLimit, + #[error("pending queue revision conflict: expected {expected}, current {current}")] + RevisionConflict { expected: u64, current: u64 }, + #[error("pending queue head conflict: expected {expected}, current {current:?}")] + HeadConflict { + expected: String, + current: Option, + }, + #[error("pending submission not found: {0}")] + NotFound(String), + #[error("pending submission state persistence failed: {0}")] + Store(#[from] StoreError), +} + +#[derive(Clone)] +pub(crate) struct PendingSubmissionHandle { + state: Arc>, + writer: LogWriterHandle, +} + +impl PendingSubmissionHandle +where + St: Store + Clone, +{ + fn validate_fence( + state: &PendingActivationState, + expected_revision: u64, + expected_head_id: Option<&str>, + ) -> Result<(), PendingSubmissionError> { + if state.revision != expected_revision { + return Err(PendingSubmissionError::RevisionConflict { + expected: expected_revision, + current: state.revision, + }); + } + if let Some(expected) = expected_head_id { + let current = state.snapshot().head_id; + if current.as_deref() != Some(expected) { + return Err(PendingSubmissionError::HeadConflict { + expected: expected.to_owned(), + current, + }); + } + } + Ok(()) + } + + fn persist_locked(&self, state: &PendingActivationState) -> Result<(), PendingSubmissionError> { + self.writer.append_entry_locked(LogEntry::Extension { + ts: segment_log::now_millis(), + domain: SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN.to_owned(), + payload: serde_json::to_value(state).expect("pending activation state must serialize"), + })?; + Ok(()) + } + + fn pin_submission_files( + &self, + pending: &PendingSubmission, + ) -> Result<(), PendingSubmissionError> { + let session_id = self.writer.state.location().session_id; + let mut pinned = Vec::new(); + for reference in submission_uploaded_file_refs(&pending.input) { + if pinned.iter().any(|existing: &protocol::UploadedFileRef| { + existing.artifact_id == reference.artifact_id + }) { + continue; + } + if let Err(pin_error) = + self.writer + .store + .pin_uploaded_file(session_id, reference, &pending.submission_id) + { + let mut rollback_error = None; + for acquired in pinned.iter().rev() { + if let Err(error) = self.writer.store.release_uploaded_file_pin( + session_id, + &acquired.artifact_id, + &pending.submission_id, + ) { + rollback_error.get_or_insert(error); + } + } + return Err(rollback_error.unwrap_or(pin_error).into()); + } + pinned.push(reference.clone()); + } + Ok(()) + } + + fn release_submission_files( + &self, + pending: &PendingSubmission, + ) -> Result<(), PendingSubmissionError> { + let session_id = self.writer.state.location().session_id; + let mut released = Vec::new(); + let mut first_error = None; + for reference in submission_uploaded_file_refs(&pending.input) { + if released + .iter() + .any(|artifact_id: &String| artifact_id == &reference.artifact_id) + { + continue; + } + if let Err(error) = self.writer.store.release_uploaded_file_pin( + session_id, + &reference.artifact_id, + &pending.submission_id, + ) { + first_error.get_or_insert(error); + } + released.push(reference.artifact_id.clone()); + } + match first_error { + Some(error) => Err(error.into()), + None => Ok(()), + } + } + + #[cfg(test)] + pub(crate) fn accept( + &self, + submission_request_id: String, + input: Vec, + activate_now: bool, + ) -> Result { + self.accept_from_source( + submission_request_id, + input, + self.direct_client_namespace(), + WorkerHistoryProvenance::LegacyUnknown, + activate_now, + ) + } + + pub(crate) fn accept_from_source( + &self, + submission_request_id: String, + input: Vec, + source_namespace: String, + provenance: WorkerHistoryProvenance, + activate_now: bool, + ) -> Result { + if submission_request_id.trim().is_empty() { + return Err(PendingSubmissionError::EmptyRequestId); + } + if submission_request_id.len() > MAX_ACTIVATION_REQUEST_ID_BYTES { + return Err(PendingSubmissionError::RequestIdLimit); + } + if input.is_empty() { + return Err(PendingSubmissionError::EmptyInput); + } + let payload_digest = submission_payload_digest(&input); + let _append_guard = self + .writer + .state + .append_lock + .lock() + .expect("segment append lock poisoned"); + let mut current = self + .state + .lock() + .expect("pending activation state poisoned"); + let original = current.clone(); + if let Some(receipt) = current.receipts.iter().find(|receipt| { + receipt.submission_request_id == submission_request_id + && receipt.source_namespace == source_namespace + }) { + if receipt.payload_digest != payload_digest { + return Err(PendingSubmissionError::IdempotencyConflict); + } + return Ok(SubmissionAcceptance { + submission_request_id, + submission_id: receipt.submission_id.clone(), + disposition: receipt.disposition, + activation: None, + }); + } + + let submission_id = uuid::Uuid::now_v7().to_string(); + let pending = PendingSubmission { + submission_request_id: submission_request_id.clone(), + source_namespace: source_namespace.clone(), + submission_id: submission_id.clone(), + payload_digest: payload_digest.clone(), + accepted_at_ms: segment_log::now_millis(), + activation_sequence: current.next_activation_sequence, + provenance, + was_queued: !activate_now, + input, + }; + current.next_activation_sequence = current.next_activation_sequence.saturating_add(1); + let disposition = if activate_now { + protocol::SubmissionDisposition::Started + } else { + protocol::SubmissionDisposition::Queued + }; + current.remember_receipt(SubmissionReceipt { + submission_request_id: submission_request_id.clone(), + source_namespace, + submission_id: submission_id.clone(), + payload_digest, + disposition, + }); + current.revision = current.revision.saturating_add(1); + + if activate_now { + current.activating = Some(pending.clone()); + } else { + let count = current + .pending + .len() + .saturating_add(current.pending_notifications.len()) + .saturating_add(1); + if count > MAX_PENDING_SUBMISSIONS { + *current = original; + return Err(PendingSubmissionError::CountLimit); + } + let bytes = current + .pending + .iter() + .map(|pending| submission_payload_len(&pending.input)) + .sum::() + .saturating_add( + current + .pending_notifications + .iter() + .map(|pending| u64::try_from(pending.message.len()).unwrap_or(u64::MAX)) + .sum::(), + ) + .saturating_add(submission_payload_len(&pending.input)); + if bytes > MAX_PENDING_SUBMISSION_BYTES { + *current = original; + return Err(PendingSubmissionError::ByteLimit); + } + let artifact_refs = current + .pending + .iter() + .map(|pending| submission_artifact_ref_count(&pending.input)) + .sum::() + .saturating_add(submission_artifact_ref_count(&pending.input)); + if artifact_refs > MAX_PENDING_ARTIFACT_REFS { + *current = original; + return Err(PendingSubmissionError::ArtifactLimit); + } + current.pending.push_back(pending.clone()); + } + if !activate_now { + if let Err(error) = self.pin_submission_files(&pending) { + *current = original; + return Err(error); + } + if let Err(error) = self.persist_locked(¤t) { + let _ = self.release_submission_files(&pending); + *current = original; + return Err(error); + } + } + Ok(SubmissionAcceptance { + submission_request_id, + submission_id, + disposition, + activation: activate_now.then_some(pending), + }) + } + + #[cfg(test)] + pub(crate) fn accept_notification( + &self, + notification_request_id: String, + message: String, + auto_run: bool, + ) -> Result { + self.accept_notification_from_source( + notification_request_id, + message, + self.direct_client_namespace(), + WorkerHistoryProvenance::LegacyUnknown, + auto_run, + ) + } + + pub(crate) fn accept_notification_from_source( + &self, + notification_request_id: String, + message: String, + source_namespace: String, + provenance: WorkerHistoryProvenance, + auto_run: bool, + ) -> Result { + if notification_request_id.trim().is_empty() { + return Err(PendingSubmissionError::EmptyRequestId); + } + if notification_request_id.len() > MAX_ACTIVATION_REQUEST_ID_BYTES { + return Err(PendingSubmissionError::RequestIdLimit); + } + let payload_digest = notification_payload_digest(&message, auto_run); + let _append_guard = self + .writer + .state + .append_lock + .lock() + .expect("segment append lock poisoned"); + let mut state = self + .state + .lock() + .expect("pending activation state poisoned"); + if let Some(receipt) = state.notification_receipts.iter().find(|receipt| { + receipt.notification_request_id == notification_request_id + && receipt.source_namespace == source_namespace + }) { + if receipt.payload_digest != payload_digest || receipt.auto_run != auto_run { + return Err(PendingSubmissionError::IdempotencyConflict); + } + return Ok(false); + } + if state + .pending + .len() + .saturating_add(state.pending_notifications.len()) + >= MAX_PENDING_SUBMISSIONS + { + return Err(PendingSubmissionError::CountLimit); + } + let queued_bytes = state + .pending + .iter() + .map(|pending| submission_payload_len(&pending.input)) + .sum::() + .saturating_add( + state + .pending_notifications + .iter() + .map(|pending| u64::try_from(pending.message.len()).unwrap_or(u64::MAX)) + .sum::(), + ) + .saturating_add(u64::try_from(message.len()).unwrap_or(u64::MAX)); + if queued_bytes > MAX_PENDING_SUBMISSION_BYTES { + return Err(PendingSubmissionError::ByteLimit); + } + let original = state.clone(); + let activation_sequence = state.next_activation_sequence; + state.next_activation_sequence = state.next_activation_sequence.saturating_add(1); + state.pending_notifications.push_back(PendingNotification { + notification_request_id: notification_request_id.clone(), + source_namespace: source_namespace.clone(), + message, + payload_digest: payload_digest.clone(), + auto_run, + accepted_at_ms: segment_log::now_millis(), + activation_sequence, + provenance, + }); + state.remember_notification_receipt(NotificationReceipt { + notification_request_id, + source_namespace, + payload_digest, + auto_run, + }); + state.revision = state.revision.saturating_add(1); + if let Err(error) = self.persist_locked(&state) { + *state = original; + return Err(error); + } + Ok(true) + } + + pub(crate) fn activating_passive_notification_id(&self) -> Option { + self.state + .lock() + .expect("pending activation state poisoned") + .activating_notification + .as_ref() + .filter(|notification| !notification.auto_run) + .map(|notification| notification.notification_request_id.clone()) + } + + pub(crate) fn next_passive_notification_identity(&self) -> Option<(String, String)> { + self.state + .lock() + .expect("pending activation state poisoned") + .pending_notifications + .iter() + .find(|notification| !notification.auto_run) + .map(|notification| { + ( + notification.source_namespace.clone(), + notification.notification_request_id.clone(), + ) + }) + } + + pub(crate) fn prepare_notification( + &self, + source_namespace: &str, + notification_request_id: &str, + ) -> Option { + let mut state = self + .state + .lock() + .expect("pending activation state poisoned"); + if state.activating_notification.is_some() { + return None; + } + let index = state + .pending_notifications + .iter() + .position(|notification| { + notification.notification_request_id == notification_request_id + && notification.source_namespace == source_namespace + })?; + let notification = state + .pending_notifications + .remove(index) + .expect("located pending notification must exist"); + state.activating_notification = Some(notification.clone()); + state.revision = state.revision.saturating_add(1); + Some(notification) + } + + pub(crate) fn prepare_next_activation( + &self, + fence: Option<(u64, &str)>, + ) -> Result, PendingSubmissionError> { + let _append_guard = self + .writer + .state + .append_lock + .lock() + .expect("segment append lock poisoned"); + let mut state = self + .state + .lock() + .expect("pending activation state poisoned"); + if let Some((expected_revision, expected_head_id)) = fence { + Self::validate_fence(&state, expected_revision, Some(expected_head_id))?; + } + if state.activating.is_some() + || state + .activating_notification + .as_ref() + .is_some_and(|notification| notification.auto_run) + { + return Ok(None); + } + let has_staged_passive_notification = state.activating_notification.is_some(); + let submission_sequence = state.pending.front().map(|item| item.activation_sequence); + let notification_index = if has_staged_passive_notification { + None + } else { + state + .pending_notifications + .iter() + .position(|item| item.auto_run) + }; + let notification_sequence = notification_index + .and_then(|index| state.pending_notifications.get(index)) + .map(|item| item.activation_sequence); + if notification_sequence.is_some() + && (submission_sequence.is_none() || notification_sequence < submission_sequence) + { + let notification = state + .pending_notifications + .remove(notification_index.expect("notification sequence came from an item")) + .expect("notification sequence came from an existing item"); + state.activating_notification = Some(notification.clone()); + state.revision = state.revision.saturating_add(1); + return Ok(Some(PendingActivation::Notification(notification))); + } + if submission_sequence.is_some() { + let pending = state + .pending + .pop_front() + .expect("submission sequence came from queue head"); + state.activating = Some(pending.clone()); + state.revision = state.revision.saturating_add(1); + return Ok(Some(PendingActivation::Submission(pending))); + } + Ok(None) + } + + pub(crate) fn abort_activation(&self, pending: PendingSubmission) { + let _append_guard = self + .writer + .state + .append_lock + .lock() + .expect("segment append lock poisoned"); + let mut state = self + .state + .lock() + .expect("pending activation state poisoned"); + if state.activating.as_ref().map(|item| &item.submission_id) != Some(&pending.submission_id) + { + return; + } + state.activating = None; + if pending.was_queued { + state.pending.push_front(pending.clone()); + } else { + state + .receipts + .retain(|receipt| receipt.submission_id != pending.submission_id); + } + state.revision = state.revision.saturating_add(1); + if let Err(error) = self.persist_locked(&state) { + tracing::error!(error = %error, "failed to persist aborted pending activation"); + } + } + + pub(crate) fn activation_extension(&self) -> SessionExtension { + let state = self + .state + .lock() + .expect("pending activation state poisoned"); + let mut committed = state.clone(); + if let Some(activating) = &committed.activating + && let Some(receipt) = committed + .receipts + .iter_mut() + .find(|receipt| receipt.submission_id == activating.submission_id) + { + receipt.disposition = protocol::SubmissionDisposition::Started; + } + committed.activating = None; + committed.revision = committed.revision.saturating_add(1); + pending_activation_extension(&committed) + } + + pub(crate) fn finish_activation(&self, submission_id: &str) { + let mut state = self + .state + .lock() + .expect("pending activation state poisoned"); + if state + .activating + .as_ref() + .map(|item| item.submission_id.as_str()) + == Some(submission_id) + { + if let Some(receipt) = state + .receipts + .iter_mut() + .find(|receipt| receipt.submission_id == submission_id) + { + receipt.disposition = protocol::SubmissionDisposition::Started; + } + state.activating = None; + state.revision = state.revision.saturating_add(1); + } + } + + pub(crate) fn notification_activation_extension(&self) -> SessionExtension { + let state = self + .state + .lock() + .expect("pending activation state poisoned"); + let mut committed = state.clone(); + committed.activating = None; + committed.activating_notification = None; + committed.revision = committed.revision.saturating_add(1); + pending_activation_extension(&committed) + } + + pub(crate) fn finish_notification_activation(&self, notification_request_id: &str) { + let mut state = self + .state + .lock() + .expect("pending activation state poisoned"); + if state + .activating_notification + .as_ref() + .map(|item| item.notification_request_id.as_str()) + == Some(notification_request_id) + { + state.activating_notification = None; + state.revision = state.revision.saturating_add(1); + } + } + + pub(crate) fn direct_client_namespace(&self) -> String { + format!("direct:{}", self.writer.state.location().session_id) + } + + pub(crate) fn snapshot(&self) -> protocol::PendingSubmissionsSnapshot { + self.state + .lock() + .expect("pending activation state poisoned") + .snapshot() + } + + fn reconcile_uploaded_file_pins(&self) -> Result { + let live_owner_ids = self + .state + .lock() + .expect("pending activation state poisoned") + .live_artifact_pin_owner_ids(); + Ok(self + .writer + .store + .reconcile_uploaded_file_pins(self.writer.state.session_id(), &live_owner_ids)?) + } + + pub(crate) fn cancel( + &self, + submission_id: &str, + expected_revision: u64, + ) -> Result { + let _append_guard = self + .writer + .state + .append_lock + .lock() + .expect("segment append lock poisoned"); + let mut state = self + .state + .lock() + .expect("pending activation state poisoned"); + Self::validate_fence(&state, expected_revision, None)?; + let original = state.clone(); + let Some(index) = state + .pending + .iter() + .position(|pending| pending.submission_id == submission_id) + else { + return Err(PendingSubmissionError::NotFound(submission_id.to_owned())); + }; + let removed = state + .pending + .remove(index) + .expect("located pending submission must exist"); + state.revision = state.revision.saturating_add(1); + if let Err(error) = self.persist_locked(&state) { + *state = original; + return Err(error); + } + self.release_submission_files(&removed)?; + Ok(state.snapshot()) + } + + pub(crate) fn clear( + &self, + expected_revision: u64, + ) -> Result { + let _append_guard = self + .writer + .state + .append_lock + .lock() + .expect("segment append lock poisoned"); + let mut state = self + .state + .lock() + .expect("pending activation state poisoned"); + Self::validate_fence(&state, expected_revision, None)?; + let original = state.clone(); + let removed = state.pending.drain(..).collect::>(); + state.pending_notifications.clear(); + state.revision = state.revision.saturating_add(1); + if let Err(error) = self.persist_locked(&state) { + *state = original; + return Err(error); + } + for pending in &removed { + self.release_submission_files(pending)?; + } + Ok(state.snapshot()) + } +} + +impl PendingSubmissionHandle { + #[cfg(test)] + pub(crate) fn for_test(root: &std::path::Path) -> Self { + let store = session_store::FsStore::new(root).expect("test session store"); + let session_id = session_store::new_session_id(); + let segment_id = session_store::new_segment_id(); + store + .create_segment(session_id, segment_id, &[]) + .expect("test session segment"); + Self { + state: Arc::new(Mutex::new(PendingActivationState::default())), + writer: LogWriterHandle { + store, + state: SegmentState::new(session_id, segment_id, 0), + sink: SegmentLogSink::new(), + in_flight: None, + }, + } + } +} + +/// Type-erased commit handle for the interceptor. Lets the interceptor commit `SystemItem`s without being generic over the /// concrete `Store` type. pub trait SystemItemCommitter: Send + Sync { fn commit_log_entry(&self, entry: LogEntry) -> Result<(), StoreError>; - fn commit_system_item( + fn commit_system_item_with_extensions( &self, item: SystemItem, + extensions: Vec, + history_provenance: Option, ) -> Result, StoreError> { let metadata = new_history_metadata( - WorkerHistoryProvenance::BackendInstruction { operation_id: None }, + history_provenance + .unwrap_or(WorkerHistoryProvenance::BackendInstruction { operation_id: None }), None, ); let history_item = item.to_history_item(); @@ -991,6 +1951,7 @@ pub trait SystemItemCommitter: Send + Sync { item, metadata: metadata.clone(), }, + extensions, })?; Ok(HistoryEntry::new(history_item, metadata)) } @@ -1027,8 +1988,6 @@ where } } -pub const WORKER_INPUT_SUBMISSION_EXTENSION_DOMAIN: &str = "worker.input-submission.v1"; - #[derive(Clone)] struct PreparedFlowProjection { selector: String, @@ -1049,6 +2008,7 @@ pub struct WorkerSession { session_id: SessionId, revision: u64, history: History, + pending_activations: Arc>, } impl WorkerSession { @@ -1058,9 +2018,39 @@ impl WorkerSession { session_id, revision, history: History::from_entries(entries), + pending_activations: Arc::new(Mutex::new(PendingActivationState::default())), } } + fn restore_pending_activations(&mut self, extensions: &[(String, serde_json::Value)]) { + let Some(payload) = extensions.iter().rev().find_map(|(domain, payload)| { + (domain == SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN).then_some(payload) + }) else { + return; + }; + if let Ok(mut state) = serde_json::from_value::(payload.clone()) { + if let Some(activating) = state.activating.take() { + state.pending.push_front(activating); + state.revision = state.revision.saturating_add(1); + } + if let Some(activating) = state.activating_notification.take() { + state.pending_notifications.push_front(activating); + state.revision = state.revision.saturating_add(1); + } + *self + .pending_activations + .lock() + .expect("pending activation state poisoned") = state; + } + } + + pub fn pending_submissions(&self) -> protocol::PendingSubmissionsSnapshot { + self.pending_activations + .lock() + .expect("pending activation state poisoned") + .snapshot() + } + pub fn session_id(&self) -> SessionId { self.session_id } @@ -1314,6 +2304,21 @@ impl Worker { } } + pub(crate) fn pending_activation_state(&self) -> Arc> { + self.session.pending_activations.clone() + } + + pub(crate) fn pending_submission_handle(&self) -> PendingSubmissionHandle { + PendingSubmissionHandle { + state: self.session.pending_activations.clone(), + writer: self.log_writer_handle(), + } + } + + pub fn pending_submissions(&self) -> protocol::PendingSubmissionsSnapshot { + self.session.pending_submissions() + } + /// Attach a type-erased system-item commit handle. The controller /// calls this once during spawn so the interceptor can commit /// `SystemItem`s directly without owning a generic store handle. @@ -1675,6 +2680,7 @@ impl Worker { }, metadata: skill_metadata.clone(), }, + extensions: Vec::new(), })?; let history_entry = HistoryEntry::new(agen::Item::system_message(body), skill_metadata); let mut annotate = history_annotator( @@ -1965,6 +2971,30 @@ impl Worker { .truncate(loc.session_id, loc.segment_id, truncate_entries)?; self.segment_state.set_entries_written(truncate_entries); self.sink.truncate_silent(truncate_entries); + let pending_state = self + .session + .pending_activations + .lock() + .expect("pending activation state poisoned") + .clone(); + if !pending_state.pending.is_empty() + || !pending_state.pending_notifications.is_empty() + || pending_state.activating.is_some() + || pending_state.activating_notification.is_some() + || !pending_state.receipts.is_empty() + || !pending_state.notification_receipts.is_empty() + { + let checkpoint = LogEntry::Extension { + ts: segment_log::now_millis(), + domain: SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN.to_owned(), + payload: serde_json::to_value(&pending_state).map_err(|error| { + RewindError::Invalid(format!( + "serialize pending submissions during rewind: {error}" + )) + })?, + }; + self.commit_entry(checkpoint)?; + } let history_entries = restore_history_entries(loc.session_id, loc.segment_id, &retained) .map_err(|error| RewindError::Invalid(error.into()))?; @@ -2534,7 +3564,7 @@ impl Worker { /// Convenience: run with a single `Segment::Text`. /// /// Equivalent to `run(vec![Segment::text(s)])`. The dumb-client - /// counterpart of [`protocol::Method::run_text`]; primarily for + /// counterpart of [`protocol::Method::submit_text`]; primarily for /// tests and tools that have only a string in hand. pub async fn run_text(&mut self, s: impl Into) -> Result where @@ -2785,8 +3815,13 @@ impl Worker { where St: Clone + 'static, { - self.run_with_input_extensions_and_commit_hook(input, input_extensions, || {}) - .await + self.run_with_input_extensions_and_commit_hook( + input, + input_extensions, + WorkerHistoryProvenance::LegacyUnknown, + || {}, + ) + .await } /// Run user input and invoke `on_input_committed` only after the annotated @@ -2797,6 +3832,7 @@ impl Worker { &mut self, input: Vec, mut input_extensions: Vec, + input_provenance: WorkerHistoryProvenance, on_input_committed: F, ) -> Result where @@ -2844,8 +3880,12 @@ impl Worker { trigger: protocol::InvokeKind::UserSend, })?; - let projected_input = - self.projected_input_history(&input, flow_projection.as_ref(), &projected_entry_ids); + let projected_input = self.projected_input_history( + &input, + flow_projection.as_ref(), + &projected_entry_ids, + &input_provenance, + ); // Persist original typed segments together with the exact ordered // model-visible item+origin projection before any entry becomes live. @@ -2858,6 +3898,11 @@ impl Worker { .map(to_logged_history_entry) .collect(), })?; + self.finalize_uploaded_segment_bindings( + &input, + &projected_entry_ids, + flow_projection.is_some(), + ); if let Some(state) = pending_flow_state { *self .flow_runtime_state @@ -3051,6 +4096,7 @@ impl Worker { }, metadata: interrupt_metadata.clone(), }, + extensions: Vec::new(), })?; let interrupt_entry = HistoryEntry::new(agen::Item::system_message(system_note), interrupt_metadata); @@ -3079,6 +4125,36 @@ impl Worker { Ok(()) } + fn finalize_uploaded_segment_bindings( + &self, + input: &[Segment], + projected_entry_ids: &[SessionHistoryEntryId], + one_entry_per_segment: bool, + ) { + for (index, segment) in input.iter().enumerate() { + let Segment::UploadedFile { file } = segment else { + continue; + }; + let entry_index = if one_entry_per_segment { index } else { 0 }; + let source_entry_id = projected_entry_ids + .get(entry_index) + .expect("projected input id exists for every uploaded file") + .0 + .as_str(); + if let Err(error) = self.store.finalize_uploaded_file_binding( + self.session_id(), + &file.artifact_id, + source_entry_id, + ) { + tracing::warn!( + artifact_id = %file.artifact_id, + error = %error, + "deferred uploaded file pin finalization to cleanup reconciliation" + ); + } + } + } + fn materialize_large_pastes( &self, input: &mut [Segment], @@ -3150,6 +4226,7 @@ impl Worker { input: &[Segment], flow_projection: Option<&PreparedFlowProjection>, entry_ids: &[SessionHistoryEntryId], + provenance: &WorkerHistoryProvenance, ) -> Vec> { if let Some(flow) = flow_projection { return input @@ -3170,10 +4247,7 @@ impl Worker { other => history_entry_with_id( Item::user_message(Segment::flatten_to_text(std::slice::from_ref(other))), entry_id.clone(), - // Current public submit transport does not carry a - // trusted account/Worker subject envelope. Fail closed - // instead of promoting role=user to HumanInput. - WorkerHistoryProvenance::LegacyUnknown, + provenance.clone(), ), }) .collect(); @@ -3185,7 +4259,7 @@ impl Worker { .first() .expect("projected Worker input always has one entry id") .clone(), - WorkerHistoryProvenance::LegacyUnknown, + provenance.clone(), )] } @@ -3497,15 +4571,6 @@ impl Worker { Ok(()) } - fn persist_and_send_compact_done( - &mut self, - lifecycle: CompactionLifecycle, - ) -> Result<(), WorkerError> { - self.persist_compaction_lifecycle(&lifecycle)?; - self.send_event(Event::CompactDone { lifecycle }); - Ok(()) - } - fn persist_and_send_compact_failed( &mut self, lifecycle: CompactionLifecycle, @@ -3650,7 +4715,97 @@ impl Worker { Ok(rewrite_guard) } + /// Terminalize and clean up any compaction that was left active by the + /// previous controller generation. This runs before the restored + /// controller publishes its first Idle state. + pub async fn recover_unfinished_compaction(&mut self) -> Result<(), WorkerError> { + let (entries, _) = self.sink.subscribe_with_snapshot(); + let latest_payload = entries.iter().rev().find_map(|entry| match entry { + LogEntry::Extension { + domain, payload, .. + } if domain == COMPACTION_EXTENSION_DOMAIN => Some(payload.clone()), + _ => None, + }); + let Some(payload) = latest_payload else { + return Ok(()); + }; + #[derive(serde::Deserialize)] + #[serde(deny_unknown_fields)] + struct CompactionLifecycleWire { + schema_version: u32, + compaction_id: String, + revision: u64, + #[serde(default)] + internal_worker: Option, + state: CompactionLifecycleState, + started_at_ms: u64, + #[serde(default)] + ended_at_ms: Option, + #[serde(default)] + summary: Option, + #[serde(default)] + error: Option, + #[serde(default)] + new_segment_id: Option, + } + let wire: CompactionLifecycleWire = serde_json::from_value(payload).map_err(|error| { + WorkerError::InvalidState(format!("decode compaction lifecycle: {error}")) + })?; + if !matches!(wire.schema_version, 2 | 3) { + return Err(WorkerError::InvalidState(format!( + "unsupported compaction lifecycle schema version {}", + wire.schema_version + ))); + } + let mut lifecycle = CompactionLifecycle { + schema_version: wire.schema_version, + compaction_id: wire.compaction_id, + revision: wire.revision, + internal_worker: wire.internal_worker, + state: wire.state, + started_at_ms: wire.started_at_ms, + ended_at_ms: wire.ended_at_ms, + summary: wire.summary, + error: wire.error, + new_segment_id: wire.new_segment_id, + }; + match lifecycle.state { + CompactionLifecycleState::Running => { + lifecycle.schema_version = 3; + lifecycle.revision = lifecycle.revision.saturating_add(1); + lifecycle.state = CompactionLifecycleState::Interrupted; + lifecycle.ended_at_ms = Some(segment_log::now_millis()); + lifecycle.error = + Some("worker execution restarted before compaction completed".into()); + self.persist_compaction_lifecycle(&lifecycle)?; + self.send_event(Event::CompactFailed { + lifecycle: lifecycle.clone(), + }); + self.release_compaction_service(&lifecycle).await; + } + CompactionLifecycleState::Interrupted => { + self.release_compaction_service(&lifecycle).await; + } + CompactionLifecycleState::Done | CompactionLifecycleState::Failed => {} + } + Ok(()) + } + pub async fn manual_compact(&mut self) -> Result { + self.manual_compact_inner(None).await + } + + pub async fn manual_compact_with_cancel( + &mut self, + cancel: tokio::sync::watch::Receiver, + ) -> Result { + self.manual_compact_inner(Some(cancel)).await + } + + async fn manual_compact_inner( + &mut self, + mut cancel: Option>, + ) -> Result { if self.manifest.compaction.is_none() { let message = "manual compact is unavailable because [compaction] is not configured".to_string(); @@ -3690,7 +4845,7 @@ impl Worker { return Ok(ManualCompactResult::Skipped { message }); } - match self.compact(retained).await { + match self.compact_with_cancel(retained, cancel.take()).await { Ok(new_segment_id) => { info!(new_segment_id = %new_segment_id, "Manual compaction succeeded"); if let Some(ref state) = state { @@ -3863,11 +5018,19 @@ impl Worker { /// Runs one parent-owned observable compaction service and returns the new /// Segment ID. Lifecycle revisions are committed before they are broadcast. pub async fn compact(&mut self, retained_tokens: u64) -> Result { + self.compact_with_cancel(retained_tokens, None).await + } + + async fn compact_with_cancel( + &mut self, + retained_tokens: u64, + mut cancel: Option>, + ) -> Result { let _rewrite_guard = self .prepare_session_rewrite(SessionRewriteKind::Compact) .await?; let mut lifecycle = CompactionLifecycle { - schema_version: 2, + schema_version: 3, compaction_id: uuid::Uuid::now_v7().to_string(), revision: 1, internal_worker: None, @@ -3879,16 +5042,25 @@ impl Worker { new_segment_id: None, }; self.persist_and_send_compact_start(lifecycle.clone())?; - match self.compact_impl(retained_tokens, &mut lifecycle).await { - Ok((new_segment_id, summary)) => { - lifecycle.revision = lifecycle.revision.saturating_add(1); - lifecycle.state = CompactionLifecycleState::Done; - lifecycle.ended_at_ms = Some(segment_log::now_millis()); - lifecycle.summary = Some(summary); - lifecycle.new_segment_id = Some(new_segment_id.to_string()); - let terminal = self.persist_and_send_compact_done(lifecycle.clone()); + let outcome = if let Some(cancel) = cancel.as_mut() { + tokio::select! { + biased; + changed = cancel.changed() => { + let _ = changed; + Err(WorkerError::CompactCancelled) + } + result = self.compact_impl(retained_tokens, &mut lifecycle) => result, + } + } else { + self.compact_impl(retained_tokens, &mut lifecycle).await + }; + match outcome { + Ok((new_segment_id, _summary)) => { + debug_assert_eq!(lifecycle.state, CompactionLifecycleState::Done); + self.send_event(Event::CompactDone { + lifecycle: lifecycle.clone(), + }); self.release_compaction_service(&lifecycle).await; - terminal?; Ok(new_segment_id) } Err(error) => { @@ -4437,6 +5609,22 @@ impl Worker { { initial_entries.push(checkpoint); } + initial_entries.push(LogEntry::Extension { + ts: segment_log::now_millis(), + domain: SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN.to_owned(), + payload: serde_json::to_value( + &*self + .session + .pending_activations + .lock() + .expect("pending activation state poisoned"), + ) + .map_err(|error| { + WorkerError::InvalidState(format!( + "serialize pending submissions during compaction: {error}" + )) + })?, + }); if let Some(flow_state) = self .flow_runtime_state .lock() @@ -4453,6 +5641,24 @@ impl Worker { })?, }); } + // Commit the terminal lifecycle in the same atomic replacement-segment + // creation as the rewritten history. Restore can therefore never see a + // replacement segment without the Done fact for the compaction that + // created it. + lifecycle.revision = lifecycle.revision.saturating_add(1); + lifecycle.state = CompactionLifecycleState::Done; + lifecycle.ended_at_ms = Some(segment_log::now_millis()); + lifecycle.summary = Some(summary_text.clone()); + lifecycle.new_segment_id = Some(new_segment_id.to_string()); + initial_entries.push(LogEntry::Extension { + ts: segment_log::now_millis(), + domain: COMPACTION_EXTENSION_DOMAIN.to_string(), + payload: serde_json::to_value(&*lifecycle).map_err(|error| { + WorkerError::InvalidState(format!( + "serialize terminal compaction lifecycle: {error}" + )) + })?, + }); self.store .create_segment(old_loc.session_id, new_segment_id, &initial_entries)?; self.segment_state.set_location(SegmentLocation { @@ -5257,6 +6463,12 @@ where history_persistence_wired: false, log_writer: None, }; + worker + .session + .restore_pending_activations(&state.extensions); + worker + .pending_submission_handle() + .reconcile_uploaded_file_pins()?; worker.apply_permissions_from_manifest(); worker.apply_prune_from_manifest(); worker.write_worker_metadata_active(SegmentLocation { @@ -7202,8 +8414,15 @@ mod build_summary_prompt_tests { serde_json::to_value(&state).unwrap(), ); let projected_ids = vec![SessionHistoryEntryId::new(), SessionHistoryEntryId::new()]; - let projected = - worker.projected_input_history(&segments, projection.as_ref(), &projected_ids); + let input_provenance = WorkerHistoryProvenance::HumanInput { + account_id: "account-1".into(), + }; + let projected = worker.projected_input_history( + &segments, + projection.as_ref(), + &projected_ids, + &input_provenance, + ); worker .commit_entry(LogEntry::AnnotatedUserInput { ts: segment_log::now_millis(), @@ -7230,6 +8449,7 @@ mod build_summary_prompt_tests { projected[0].annotation.origin, WorkerHistoryProvenance::FlowInstruction { .. } )); + assert_eq!(projected[1].annotation.origin, input_provenance); assert_eq!(state.instance.definition_revision, 3); assert_eq!(state.instance.current_state.as_str(), "implement"); assert_eq!(workspace_client.requests.lock().unwrap().len(), 1); @@ -7312,7 +8532,12 @@ mod build_summary_prompt_tests { .delete_uploaded_file(worker.session_id(), &file.artifact_id), Err(StoreError::ArtifactAlreadyCommitted) )); - let projected = worker.projected_input_history(&input, None, &[entry_id]); + let projected = worker.projected_input_history( + &input, + None, + &[entry_id], + &WorkerHistoryProvenance::LegacyUnknown, + ); let text = projected[0].item.as_text().unwrap(); assert!(text.contains("notes.md")); assert!(text.contains(&file.artifact_id)); @@ -7394,7 +8619,12 @@ mod build_summary_prompt_tests { if retained.source_entry_id == artifact.source_entry_id )); - let history = worker.projected_input_history(&input, None, &[entry_id]); + let history = worker.projected_input_history( + &input, + None, + &[entry_id], + &WorkerHistoryProvenance::LegacyUnknown, + ); assert!(!history[0].item.as_text().unwrap().contains("終端")); append_test_entry( &worker, @@ -7670,6 +8900,47 @@ mod build_summary_prompt_tests { assert_eq!(worker.history()[0].as_text().unwrap(), "first message"); } + #[tokio::test] + async fn rewind_preserves_notification_only_pending_activation_checkpoint() { + let (_dir, mut worker) = rewind_test_worker().await; + append_user_turn(&worker, 10, "first message"); + append_user_turn(&worker, 20, "second message"); + worker + .pending_submission_handle() + .accept_notification("notification-1".into(), "keep me".into(), true) + .unwrap(); + let (head_entries, targets) = worker.list_rewind_targets().unwrap(); + + worker + .rewind_to(targets.last().unwrap().id.clone(), head_entries) + .await + .unwrap(); + + let location = worker.segment_state.location(); + let entries = worker + .store + .read_all(location.session_id, location.segment_id) + .unwrap(); + let restored: PendingActivationState = entries + .iter() + .rev() + .find_map(|entry| match entry { + LogEntry::Extension { + domain, payload, .. + } if domain == SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN => { + serde_json::from_value(payload.clone()).ok() + } + _ => None, + }) + .unwrap(); + assert_eq!(restored.pending_notifications.len(), 1); + assert_eq!(restored.notification_receipts.len(), 1); + assert_eq!( + restored.pending_notifications[0].notification_request_id, + "notification-1" + ); + } + #[tokio::test] async fn annotated_history_rewind_commits_authoritative_prefix() { let (_dir, mut worker) = rewind_test_worker().await; @@ -8515,6 +9786,551 @@ mod build_summary_prompt_tests { ); } + #[test] + fn submission_retry_identity_is_scoped_to_authenticated_source_and_keeps_provenance() { + let temp = tempfile::tempdir().unwrap(); + let handle = PendingSubmissionHandle::for_test(temp.path()); + let input = vec![Segment::text("same request")]; + let account_a = WorkerHistoryProvenance::HumanInput { + account_id: "account-a".into(), + }; + let account_b = WorkerHistoryProvenance::HumanInput { + account_id: "account-b".into(), + }; + let first = handle + .accept_from_source( + "request-1".into(), + input.clone(), + "account:account-a".into(), + account_a.clone(), + false, + ) + .unwrap(); + let replay = handle + .accept_from_source( + "request-1".into(), + input.clone(), + "account:account-a".into(), + account_a.clone(), + false, + ) + .unwrap(); + let other_source = handle + .accept_from_source( + "request-1".into(), + input, + "account:account-b".into(), + account_b.clone(), + false, + ) + .unwrap(); + + assert_eq!(replay.submission_id, first.submission_id); + assert_ne!(other_source.submission_id, first.submission_id); + let state = handle.state.lock().unwrap(); + assert_eq!(state.pending[0].provenance, account_a); + assert_eq!(state.pending[1].provenance, account_b); + } + + #[test] + fn restore_reconciliation_clears_interrupted_acceptance_pin_for_retry() { + let temp = tempfile::tempdir().unwrap(); + let handle = PendingSubmissionHandle::for_test(temp.path()); + let session_id = handle.writer.state.session_id(); + let segment_id = handle.writer.state.location().segment_id; + let limits = session_store::UploadedFileLimits { + max_file_bytes: 1024, + max_session_bytes: 2048, + }; + let file = handle + .writer + .store + .write_uploaded_file(session_id, "retry.txt", "text/plain", b"retry", limits) + .unwrap(); + handle + .writer + .store + .pin_uploaded_file(session_id, &file, "interrupted-before-checkpoint") + .unwrap(); + drop(handle); + let handle = PendingSubmissionHandle { + state: Arc::new(Mutex::new(PendingActivationState::default())), + writer: LogWriterHandle { + store: session_store::FsStore::new(temp.path()).unwrap(), + state: SegmentState::new(session_id, segment_id, 0), + sink: SegmentLogSink::new(), + in_flight: None, + }, + }; + + assert_eq!(handle.reconcile_uploaded_file_pins().unwrap(), 1); + let accepted = handle + .accept( + "request-after-restore".into(), + vec![Segment::UploadedFile { file: file.clone() }], + false, + ) + .unwrap(); + assert!(!accepted.submission_id.is_empty()); + assert_eq!(handle.reconcile_uploaded_file_pins().unwrap(), 0); + assert!(matches!( + handle + .writer + .store + .delete_uploaded_file(session_id, &file.artifact_id), + Err(StoreError::ArtifactAlreadyCommitted) + )); + } + + #[test] + fn rejected_submission_rolls_back_uploaded_file_pins_acquired_before_conflict() { + let temp = tempfile::tempdir().unwrap(); + let handle = PendingSubmissionHandle::for_test(temp.path()); + let session_id = handle.writer.state.session_id(); + let limits = session_store::UploadedFileLimits { + max_file_bytes: 1024, + max_session_bytes: 2048, + }; + let first = handle + .writer + .store + .write_uploaded_file(session_id, "first.txt", "text/plain", b"first", limits) + .unwrap(); + let second = handle + .writer + .store + .write_uploaded_file(session_id, "second.txt", "text/plain", b"second", limits) + .unwrap(); + handle + .writer + .store + .pin_uploaded_file(session_id, &second, "other-submission") + .unwrap(); + + assert!( + handle + .accept( + "request-partial-pin".into(), + vec![ + Segment::UploadedFile { + file: first.clone(), + }, + Segment::UploadedFile { + file: second.clone(), + }, + ], + false, + ) + .is_err() + ); + assert!(handle.snapshot().submissions.is_empty()); + assert!( + handle + .writer + .store + .delete_uploaded_file(session_id, &first.artifact_id) + .unwrap() + ); + assert!(matches!( + handle + .writer + .store + .delete_uploaded_file(session_id, &second.artifact_id), + Err(StoreError::ArtifactAlreadyCommitted) + )); + handle + .writer + .store + .release_uploaded_file_pin(session_id, &second.artifact_id, "other-submission") + .unwrap(); + assert!( + handle + .writer + .store + .delete_uploaded_file(session_id, &second.artifact_id) + .unwrap() + ); + } + + #[test] + fn queued_submission_pins_uploaded_file_until_cancelled() { + let temp = tempfile::tempdir().unwrap(); + let handle = PendingSubmissionHandle::for_test(temp.path()); + let session_id = handle.writer.state.session_id(); + let reference = handle + .writer + .store + .write_uploaded_file( + session_id, + "queued.txt", + "text/plain", + b"queued artifact", + session_store::UploadedFileLimits { + max_file_bytes: 1024, + max_session_bytes: 2048, + }, + ) + .unwrap(); + let accepted = handle + .accept( + "artifact-request".into(), + vec![Segment::UploadedFile { + file: reference.clone(), + }], + false, + ) + .unwrap(); + + assert_eq!( + handle + .writer + .store + .delete_uncommitted_uploaded_files(session_id) + .unwrap(), + 0 + ); + assert!( + handle + .writer + .store + .read_uploaded_file_by_id(session_id, &reference.artifact_id) + .is_ok() + ); + + handle + .cancel(&accepted.submission_id, handle.snapshot().revision) + .unwrap(); + assert_eq!( + handle + .writer + .store + .delete_uncommitted_uploaded_files(session_id) + .unwrap(), + 1 + ); + } + + #[test] + fn pending_submission_queue_is_durable_idempotent_and_bounded() { + let temp = tempfile::tempdir().unwrap(); + let handle = PendingSubmissionHandle::for_test(temp.path()); + let input = vec![Segment::text("queued")]; + let accepted = handle + .accept("request-1".into(), input.clone(), false) + .unwrap(); + assert_eq!( + accepted.disposition, + protocol::SubmissionDisposition::Queued + ); + assert_eq!(handle.snapshot().submissions.len(), 1); + + let replay = handle + .accept("request-1".into(), input.clone(), false) + .unwrap(); + assert_eq!(replay.submission_id, accepted.submission_id); + assert!(replay.activation.is_none()); + assert_eq!(handle.snapshot().submissions.len(), 1); + assert!(matches!( + handle.accept("request-1".into(), vec![Segment::text("different")], false), + Err(PendingSubmissionError::IdempotencyConflict) + )); + + let entries = handle + .writer + .store + .read_all( + handle.writer.state.session_id(), + handle.writer.state.segment_id(), + ) + .unwrap(); + let payload = entries + .iter() + .rev() + .find_map(|entry| match entry { + LogEntry::Extension { + domain, payload, .. + } if domain == SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN => { + Some(payload.clone()) + } + _ => None, + }) + .unwrap(); + let restored: PendingActivationState = serde_json::from_value(payload).unwrap(); + assert_eq!(restored.pending.len(), 1); + assert_eq!(restored.pending[0].submission_id, accepted.submission_id); + + let fence = handle.snapshot(); + assert!(matches!( + handle.cancel(&accepted.submission_id, fence.revision.saturating_sub(1)), + Err(PendingSubmissionError::RevisionConflict { .. }) + )); + assert!(matches!( + handle.prepare_next_activation(Some((fence.revision, "wrong-head"))), + Err(PendingSubmissionError::HeadConflict { .. }) + )); + assert_eq!(handle.snapshot(), fence); + + let snapshot = handle + .cancel(&accepted.submission_id, handle.snapshot().revision) + .unwrap(); + assert!(snapshot.submissions.is_empty()); + assert!(matches!( + handle.cancel(&accepted.submission_id, handle.snapshot().revision), + Err(PendingSubmissionError::NotFound(_)) + )); + + for index in 0..MAX_PENDING_SUBMISSIONS { + handle + .accept( + format!("limit-{index}"), + vec![Segment::text(format!("value-{index}"))], + false, + ) + .unwrap(); + } + assert!(matches!( + handle.accept("over-limit".into(), vec![Segment::text("too much")], false), + Err(PendingSubmissionError::CountLimit) + )); + assert_eq!(handle.snapshot().submissions.len(), MAX_PENDING_SUBMISSIONS); + let cleared = handle.clear(handle.snapshot().revision).unwrap(); + assert!(cleared.submissions.is_empty()); + assert_eq!(cleared.notification_count, 0); + } + + #[test] + fn durable_notification_commits_authenticated_history_provenance() { + let temp = tempfile::tempdir().unwrap(); + let handle = PendingSubmissionHandle::for_test(temp.path()); + let provenance = WorkerHistoryProvenance::HumanInput { + account_id: "account-1".into(), + }; + let committed = handle + .writer + .commit_system_item_with_extensions( + SystemItem::Notification { + message: "notice".into(), + body: "notice".into(), + prompt_provenance: None, + }, + Vec::new(), + Some(provenance.clone()), + ) + .unwrap(); + assert_eq!(committed.annotation.origin, provenance); + } + + #[test] + fn notification_retry_identity_is_scoped_to_authenticated_source() { + let temp = tempfile::tempdir().unwrap(); + let handle = PendingSubmissionHandle::for_test(temp.path()); + let account_a = WorkerHistoryProvenance::HumanInput { + account_id: "account-a".into(), + }; + let account_b = WorkerHistoryProvenance::HumanInput { + account_id: "account-b".into(), + }; + assert!( + handle + .accept_notification_from_source( + "request-1".into(), + "notice".into(), + "account:account-a".into(), + account_a.clone(), + false, + ) + .unwrap() + ); + assert!( + !handle + .accept_notification_from_source( + "request-1".into(), + "notice".into(), + "account:account-a".into(), + account_a.clone(), + false, + ) + .unwrap() + ); + assert!( + handle + .accept_notification_from_source( + "request-1".into(), + "notice".into(), + "account:account-b".into(), + account_b.clone(), + false, + ) + .unwrap() + ); + let state = handle.state.lock().unwrap(); + assert_eq!(state.pending_notifications[0].provenance, account_a); + assert_eq!(state.pending_notifications[1].provenance, account_b); + } + + #[test] + fn notification_and_submit_share_activation_order_and_notification_dedupes() { + let temp = tempfile::tempdir().unwrap(); + let handle = PendingSubmissionHandle::for_test(temp.path()); + assert!( + handle + .accept_notification("notification-1".into(), "notice".into(), true) + .unwrap() + ); + assert!( + !handle + .accept_notification("notification-1".into(), "notice".into(), true) + .unwrap() + ); + assert!(matches!( + handle.accept_notification("notification-1".into(), "different".into(), true), + Err(PendingSubmissionError::IdempotencyConflict) + )); + assert!(matches!( + handle.accept_notification("notification-1".into(), "notice".into(), false), + Err(PendingSubmissionError::IdempotencyConflict) + )); + handle + .accept("request-1".into(), vec![Segment::text("submit")], false) + .unwrap(); + + let first = handle.prepare_next_activation(None).unwrap().unwrap(); + assert!(matches!( + first, + PendingActivation::Notification(PendingNotification { ref message, .. }) + if message == "notice" + )); + let committed = handle.notification_activation_extension(); + let committed_state: PendingActivationState = + serde_json::from_value(committed.payload).unwrap(); + assert!(committed_state.pending_notifications.is_empty()); + assert!(committed_state.activating_notification.is_none()); + handle.finish_notification_activation("notification-1"); + let second = handle.prepare_next_activation(None).unwrap().unwrap(); + assert!(matches!(second, PendingActivation::Submission(_))); + } + + #[test] + fn restoring_an_in_flight_activation_requeues_it_at_the_fifo_head() { + let mut session = WorkerSession::new(session_store::new_session_id(), Vec::new()); + let state = PendingActivationState { + revision: 4, + next_activation_sequence: 3, + activating: Some(PendingSubmission { + submission_request_id: "request-1".into(), + source_namespace: "direct:test".into(), + submission_id: "submission-1".into(), + payload_digest: submission_payload_digest(&[Segment::text("first")]), + accepted_at_ms: 1, + activation_sequence: 0, + provenance: WorkerHistoryProvenance::LegacyUnknown, + was_queued: false, + input: vec![Segment::text("first")], + }), + activating_notification: Some(PendingNotification { + notification_request_id: "notification-1".into(), + source_namespace: "account:account-1".into(), + message: "deferred notice".into(), + payload_digest: notification_payload_digest("deferred notice", false), + auto_run: false, + accepted_at_ms: 3, + activation_sequence: 2, + provenance: WorkerHistoryProvenance::HumanInput { + account_id: "account-1".into(), + }, + }), + pending: VecDeque::from([PendingSubmission { + submission_request_id: "request-2".into(), + source_namespace: "direct:test".into(), + submission_id: "submission-2".into(), + payload_digest: submission_payload_digest(&[Segment::text("second")]), + accepted_at_ms: 2, + activation_sequence: 1, + provenance: WorkerHistoryProvenance::LegacyUnknown, + was_queued: true, + input: vec![Segment::text("second")], + }]), + pending_notifications: VecDeque::new(), + receipts: VecDeque::new(), + notification_receipts: VecDeque::from([NotificationReceipt { + notification_request_id: "notification-1".into(), + source_namespace: "account:account-1".into(), + payload_digest: notification_payload_digest("deferred notice", false), + auto_run: false, + }]), + }; + session.restore_pending_activations(&[( + SESSION_PENDING_ACTIVATIONS_EXTENSION_DOMAIN.into(), + serde_json::to_value(state).unwrap(), + )]); + let state = session + .pending_activations + .lock() + .expect("pending activation state poisoned"); + assert!(state.activating.is_none()); + assert_eq!(state.pending.len(), 2); + assert_eq!(state.pending[0].submission_id, "submission-1"); + assert_eq!(state.pending[1].submission_id, "submission-2"); + assert!(state.activating_notification.is_none()); + assert_eq!(state.pending_notifications.len(), 1); + assert!(!state.pending_notifications[0].auto_run); + assert!(matches!( + state.pending_notifications[0].provenance, + WorkerHistoryProvenance::HumanInput { ref account_id } if account_id == "account-1" + )); + assert_eq!(state.notification_receipts.len(), 1); + } + + #[tokio::test] + async fn restore_terminalizes_running_compaction_before_idle_publication() { + let (_dir, mut worker) = rewind_test_worker().await; + let lifecycle = CompactionLifecycle { + schema_version: 3, + compaction_id: "compact-before-restart".into(), + revision: 1, + internal_worker: None, + state: CompactionLifecycleState::Running, + started_at_ms: segment_log::now_millis(), + ended_at_ms: None, + summary: None, + error: None, + new_segment_id: None, + }; + worker.persist_compaction_lifecycle(&lifecycle).unwrap(); + + worker.recover_unfinished_compaction().await.unwrap(); + + let (entries, _) = worker.sink.subscribe_with_snapshot(); + let restored = entries.iter().rev().find_map(|entry| match entry { + LogEntry::Extension { + domain, payload, .. + } if domain == COMPACTION_EXTENSION_DOMAIN => { + serde_json::from_value::(payload.clone()).ok() + } + _ => None, + }); + let restored = restored.expect("terminal compaction lifecycle"); + assert_eq!(restored.state, CompactionLifecycleState::Interrupted); + assert_eq!(restored.revision, 2); + assert!( + restored + .error + .as_deref() + .is_some_and(|error| error.contains("restarted")) + ); + + let mut future = lifecycle; + future.schema_version = 4; + future.compaction_id = "future-compaction".into(); + worker.persist_compaction_lifecycle(&future).unwrap(); + let error = worker.recover_unfinished_compaction().await.unwrap_err(); + assert!( + error + .to_string() + .contains("unsupported compaction lifecycle schema version 4") + ); + } + fn minimal_manifest() -> WorkerManifest { let toml_str = r#" [worker] diff --git a/crates/worker/tests/compact_events_test.rs b/crates/worker/tests/compact_events_test.rs index 47fbd351..1521b909 100644 --- a/crates/worker/tests/compact_events_test.rs +++ b/crates/worker/tests/compact_events_test.rs @@ -72,6 +72,41 @@ impl LlmClient for MockClient { } } +#[derive(Clone)] +struct BlockingCompactClient { + calls: Arc, +} + +impl BlockingCompactClient { + fn new() -> Self { + Self { + calls: Arc::new(AtomicUsize::new(0)), + } + } +} + +#[async_trait] +impl LlmClient for BlockingCompactClient { + fn clone_boxed(&self) -> Box { + Box::new(self.clone()) + } + + async fn stream( + &self, + _request: Request, + ) -> Result> + Send>>, ClientError> + { + let call = self.calls.fetch_add(1, Ordering::SeqCst); + if call == 0 { + Ok(Box::pin(futures::stream::iter( + single_text_events("seed").into_iter().map(Ok), + ))) + } else { + Ok(Box::pin(futures::stream::pending())) + } + } +} + fn single_text_events(text: &str) -> Vec { vec![ LlmEvent::text_block_start(0), @@ -156,10 +191,10 @@ target = "./" permission = "write" "#; -async fn make_worker_with_manifest( - manifest_toml: &str, - client: MockClient, -) -> Worker { +async fn make_worker_with_manifest(manifest_toml: &str, client: C) -> Worker +where + C: LlmClient + Clone + Send + Sync + 'static, +{ let manifest = worker::WorkerManifest::from_toml(manifest_toml).unwrap(); let store_tmp = tempfile::tempdir().unwrap(); @@ -614,12 +649,144 @@ async fn pre_run_compact_failure_broadcasts_start_and_failed() { ); } +#[tokio::test] +async fn manual_compact_cancel_terminalizes_before_returning_idle() { + let worker = + make_worker_with_manifest(POST_RUN_MANIFEST_TOML, BlockingCompactClient::new()).await; + let runtime_tmp = tempfile::tempdir().unwrap(); + let bash_output_dir = runtime_tmp.path().join("bash-output"); + let (handle, shutdown_receiver) = + WorkerController::spawn(worker, runtime_tmp.path(), &bash_output_dir) + .await + .unwrap(); + let mut rx = handle.subscribe(); + + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "seed history", + )) + .await + .expect("send seed run"); + loop { + if matches!( + tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv()) + .await + .expect("timeout waiting for seed run") + .expect("event"), + Event::RunEnd { + result: RunResult::Finished + } + ) { + break; + } + } + + let compact = protocol::WorkerCommandEnvelope::for_snapshot(1, &handle.shared_state.snapshot()); + handle + .send(Method::Compact { command: compact }) + .await + .expect("send compact"); + loop { + if matches!( + tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv()) + .await + .expect("timeout waiting for compact start") + .expect("event"), + Event::CompactStart { .. } + ) { + break; + } + } + + let cancel = protocol::WorkerCommandEnvelope::for_snapshot(2, &handle.shared_state.snapshot()); + handle + .send(Method::Cancel { command: cancel }) + .await + .expect("send compact cancel"); + let mut saw_interrupted = false; + let mut saw_idle = false; + while !(saw_interrupted && saw_idle) { + match tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv()) + .await + .expect("timeout waiting for compact cancellation") + .expect("event") + { + Event::CompactFailed { lifecycle } + if lifecycle.state == protocol::CompactionLifecycleState::Interrupted => + { + saw_interrupted = true; + } + Event::WorkerState { snapshot } + if snapshot.catalog_status() == protocol::WorkerStatus::Idle => + { + assert!( + saw_interrupted, + "Idle must follow durable Interrupted evidence" + ); + saw_idle = true; + } + _ => {} + } + } + + let compact = protocol::WorkerCommandEnvelope::for_snapshot(3, &handle.shared_state.snapshot()); + handle + .send(Method::Compact { command: compact }) + .await + .expect("send second compact"); + loop { + if matches!( + tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv()) + .await + .expect("timeout waiting for second compact start") + .expect("event"), + Event::CompactStart { .. } + ) { + break; + } + } + let shutdown = + protocol::WorkerCommandEnvelope::for_snapshot(4, &handle.shared_state.snapshot()); + handle + .send(Method::Shutdown { command: shutdown }) + .await + .expect("send shutdown during compact"); + let mut interrupted_before_shutdown = false; + loop { + match tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv()) + .await + .expect("timeout waiting for shutdown") + .expect("event") + { + Event::CompactFailed { lifecycle } + if lifecycle.state == protocol::CompactionLifecycleState::Interrupted => + { + interrupted_before_shutdown = true; + } + Event::Shutdown => { + assert!( + interrupted_before_shutdown, + "shutdown must await terminal compaction evidence" + ); + break; + } + _ => {} + } + } + tokio::time::timeout(std::time::Duration::from_secs(2), shutdown_receiver) + .await + .expect("controller shutdown timeout") + .expect("shutdown confirmation"); +} + #[tokio::test] async fn controller_compact_method_emits_start_and_done() { let client = MockClient::new(vec![ text_events_with_usage("hi", 1000), write_summary_tool_use_events("manual-summary", "manual compact summary"), single_text_events("done"), + single_text_events("follow-up"), ]); let worker = make_worker_with_manifest(POST_RUN_MANIFEST_TOML, client).await; let runtime_tmp = tempfile::tempdir().unwrap(); @@ -630,7 +797,10 @@ async fn controller_compact_method_emits_start_and_done() { let mut rx = handle.subscribe(); handle - .send(Method::run_text("seed history")) + .send(Method::submit_text( + protocol::new_submission_request_id(), + "seed history", + )) .await .expect("send run"); loop { @@ -646,7 +816,11 @@ async fn controller_compact_method_emits_start_and_done() { } } - handle.send(Method::Compact).await.expect("send compact"); + let command = protocol::WorkerCommandEnvelope::for_snapshot(1, &handle.shared_state.snapshot()); + handle + .send(Method::Compact { command }) + .await + .expect("send compact"); let mut saw_start = false; loop { match tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv()) @@ -667,5 +841,30 @@ async fn controller_compact_method_emits_start_and_done() { } assert!(saw_start, "manual compact should emit CompactStart"); - let _ = handle.send(Method::Shutdown).await; + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "run after compact", + )) + .await + .expect("send follow-up run"); + loop { + match tokio::time::timeout(std::time::Duration::from_secs(2), rx.recv()) + .await + .expect("timeout waiting for follow-up run") + .expect("event") + { + Event::RunEnd { + result: RunResult::Finished, + } => break, + _ => {} + } + } + assert_eq!( + handle.shared_state.catalog_status(), + protocol::WorkerStatus::Idle, + "successful manual compaction must release the execution fence" + ); + let command = protocol::WorkerCommandEnvelope::for_snapshot(2, &handle.shared_state.snapshot()); + let _ = handle.send(Method::Shutdown { command }).await; } diff --git a/crates/worker/tests/controller_test.rs b/crates/worker/tests/controller_test.rs index 143c1da9..ba1fada3 100644 --- a/crates/worker/tests/controller_test.rs +++ b/crates/worker/tests/controller_test.rs @@ -1,5 +1,5 @@ use std::pin::Pin; -use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering}; use std::sync::{Arc, Mutex}; use agen::Engine; @@ -25,6 +25,15 @@ use worker::{ type TestStore = CombinedStore; +static NEXT_COMMAND_ID: AtomicU64 = AtomicU64::new(1); + +fn worker_command(handle: &WorkerHandle) -> protocol::WorkerCommandEnvelope { + protocol::WorkerCommandEnvelope::for_snapshot( + NEXT_COMMAND_ID.fetch_add(1, Ordering::Relaxed), + &handle.shared_state.snapshot(), + ) +} + /// Reconstruct a worker-history-like `Vec` from the live session /// log mirror held by the Worker's broadcast sink. Replaces the previous /// `WorkerSharedState.history()` test helper now that the mirror lives in @@ -313,7 +322,12 @@ async fn controller_grants_read_scope_for_exact_bash_output_directory() { })); assert!(!handle.runtime_dir.path().join("bash-output").exists()); - handle.send(Method::Shutdown).await.unwrap(); + handle + .send(Method::Shutdown { + command: worker_command(&handle), + }) + .await + .unwrap(); shutdown_rx.await.unwrap(); } @@ -332,6 +346,7 @@ async fn shutdown_closes_bound_workdir_session() { command: "sleep 30".to_owned(), timeout_secs: 60, output_limit: 1024, + cwd: None, spill_dir: None, tool_call_id: None, }) @@ -345,7 +360,12 @@ async fn shutdown_closes_bound_workdir_session() { WorkerController::spawn(worker, runtime_base.path(), &bash_output_dir) .await .unwrap(); - handle.send(Method::Shutdown).await.unwrap(); + handle + .send(Method::Shutdown { + command: worker_command(&handle), + }) + .await + .unwrap(); tokio::time::timeout(std::time::Duration::from_secs(5), shutdown_rx) .await .expect("controller should shut down") @@ -376,6 +396,7 @@ async fn controller_projects_workdir_command_events_and_snapshot_state() { command: "printf ready; sleep 0.3; printf done".to_owned(), timeout_secs: 5, output_limit: 1024, + cwd: None, spill_dir: None, tool_call_id: Some("tool-command-1".into()), }) @@ -459,7 +480,12 @@ async fn controller_projects_workdir_command_events_and_snapshot_state() { !durable_history.contains("ready") && !durable_history.contains("done"), "operational command chunks must not be appended to Worker history: {durable_history}" ); - handle.send(Method::Shutdown).await.unwrap(); + handle + .send(Method::Shutdown { + command: worker_command(&handle), + }) + .await + .unwrap(); } #[tokio::test] @@ -484,6 +510,7 @@ async fn controller_refreshes_command_snapshot_after_high_output_provider_lag() .to_owned(), timeout_secs: 10, output_limit: 1024, + cwd: None, spill_dir: None, tool_call_id: Some("tool-high-output".into()), }) @@ -530,7 +557,12 @@ async fn controller_refreshes_command_snapshot_after_high_output_provider_lag() .await .unwrap(); assert_eq!(output.status, workdir::CommandStatus::Cancelled); - handle.send(Method::Shutdown).await.unwrap(); + handle + .send(Method::Shutdown { + command: worker_command(&handle), + }) + .await + .unwrap(); } #[tokio::test] @@ -560,6 +592,7 @@ async fn controller_startup_failure_closes_bound_workdir_session() { command: "printf unreachable".to_owned(), timeout_secs: 5, output_limit: 1024, + cwd: None, spill_dir: None, tool_call_id: None, }) @@ -571,13 +604,13 @@ async fn controller_startup_failure_closes_bound_workdir_session() { async fn wait_for_status(handle: &WorkerHandle, status: WorkerStatus) { let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2); loop { - if handle.shared_state.get_status() == status { + if handle.shared_state.catalog_status() == status { return; } assert!( tokio::time::Instant::now() < deadline, "timed out waiting for status {status:?}; current={:?}", - handle.shared_state.get_status() + handle.shared_state.catalog_status() ); tokio::time::sleep(std::time::Duration::from_millis(10)).await; } @@ -617,7 +650,13 @@ async fn feature_flags_default_to_core_tool_surface_only() { let worker = make_worker(client).await; let handle = spawn_controller(worker).await; - handle.send(Method::run_text("Hello")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "Hello", + )) + .await + .unwrap(); wait_for_status(&handle, WorkerStatus::Idle).await; let request = wait_for_captured_request(&client_for_assert).await; @@ -672,7 +711,13 @@ permission = "write" let worker = make_worker_with_pwd_and_manifest(client, manifest).await.0; let handle = spawn_controller(worker).await; - handle.send(Method::run_text("Hello")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "Hello", + )) + .await + .unwrap(); wait_for_status(&handle, WorkerStatus::Idle).await; let request = wait_for_captured_request(&client_for_assert).await; @@ -758,7 +803,13 @@ permission = "write" let worker = make_worker_with_pwd_and_manifest(client, &manifest).await.0; let handle = spawn_controller(worker).await; - handle.send(Method::run_text("Hello")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "Hello", + )) + .await + .unwrap(); wait_for_status(&handle, WorkerStatus::Idle).await; let request = wait_for_captured_request(&client_for_assert).await; @@ -826,7 +877,13 @@ async fn builtin_orchestrator_exposes_worker_remove_and_workdir_delete() { .await; let handle = spawn_controller(worker).await; - handle.send(Method::run_text("Hello")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "Hello", + )) + .await + .unwrap(); wait_for_status(&handle, WorkerStatus::Idle).await; let request = wait_for_captured_request(&client_for_assert).await; let installed = request_tool_names(&request); @@ -875,7 +932,13 @@ permission = "write" .0; let handle = spawn_controller(worker).await; - handle.send(Method::run_text("Hello")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "Hello", + )) + .await + .unwrap(); wait_for_status(&handle, WorkerStatus::Idle).await; let request = wait_for_captured_request(&client_for_assert).await; @@ -928,7 +991,13 @@ permission = "write" ) .await; let handle = spawn_controller(worker).await; - handle.send(Method::run_text("Hello")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "Hello", + )) + .await + .unwrap(); wait_for_status(&handle, WorkerStatus::Idle).await; let request = wait_for_captured_request(&client_for_assert).await; let names = request_tool_names(&request); @@ -975,7 +1044,13 @@ async fn run_end_returns_to_idle_without_busy_status() { let handle = spawn_controller(worker).await; let mut rx = handle.subscribe(); - handle.send(Method::run_text("Hello")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "Hello", + )) + .await + .unwrap(); let mut saw_run_end = false; let mut saw_idle_status = false; @@ -987,7 +1062,8 @@ async fn run_end_returns_to_idle_without_busy_status() { Ok(Event::RunEnd { result: protocol::RunResult::Finished }) => { saw_run_end = true; } - Ok(Event::Status { status: WorkerStatus::Idle }) if saw_run_end => { + Ok(Event::WorkerState { snapshot }) + if saw_run_end && snapshot.catalog_status() == WorkerStatus::Idle => { saw_idle_status = true; break; } @@ -1004,7 +1080,7 @@ async fn run_end_returns_to_idle_without_busy_status() { saw_idle_status, "expected idle status immediately after RunEnd" ); - assert_eq!(handle.shared_state.get_status(), WorkerStatus::Idle); + assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle); } #[tokio::test] @@ -1017,7 +1093,13 @@ async fn provider_stream_error_records_run_errored() { let handle = spawn_controller(worker).await; let mut rx = handle.subscribe(); - handle.send(Method::run_text("ping")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "ping", + )) + .await + .unwrap(); assert!( drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!( @@ -1066,16 +1148,17 @@ async fn snapshot_includes_user_input_for_in_flight_turn() { let mut events = handle.subscribe(); handle - .send(Method::run_text("hello in-flight")) + .send(Method::submit_text( + protocol::new_submission_request_id(), + "hello in-flight", + )) .await .unwrap(); tokio::time::timeout(std::time::Duration::from_secs(2), async { loop { if matches!( events.recv().await, - Ok(Event::Status { - status: WorkerStatus::Running, - }) + Ok(Event::WorkerState { snapshot }) if snapshot.catalog_status() == WorkerStatus::Running ) { break; } @@ -1131,7 +1214,13 @@ async fn attach_snapshot_includes_current_status() { let worker = make_worker(client).await; let handle = spawn_controller(worker).await; - handle.send(Method::run_text("Hello")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "Hello", + )) + .await + .unwrap(); wait_for_status(&handle, WorkerStatus::Running).await; let stream = tokio::net::UnixStream::connect(handle.runtime_dir.socket_path()) @@ -1144,8 +1233,8 @@ async fn attach_snapshot_includes_current_status() { loop { let event = reader.next::().await.unwrap().unwrap(); match event { - Event::Snapshot { status, .. } => { - assert_eq!(status, WorkerStatus::Running); + Event::Snapshot { state, .. } => { + assert_eq!(state.catalog_status(), WorkerStatus::Running); return; } Event::Alert(_) => continue, @@ -1160,7 +1249,7 @@ async fn shared_state_starts_idle() { let worker = make_worker(client).await; let handle = spawn_controller(worker).await; - assert_eq!(handle.shared_state.get_status(), WorkerStatus::Idle); + assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle); } #[tokio::test] @@ -1169,12 +1258,18 @@ async fn run_updates_shared_state_to_idle_after_completion() { let worker = make_worker(client).await; let handle = spawn_controller(worker).await; - handle.send(Method::run_text("Hello")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "Hello", + )) + .await + .unwrap(); // Wait for the run to complete tokio::time::sleep(std::time::Duration::from_millis(100)).await; - assert_eq!(handle.shared_state.get_status(), WorkerStatus::Idle); + assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle); } #[tokio::test] @@ -1183,7 +1278,13 @@ async fn run_populates_history() { let worker = make_worker(client).await; let handle = spawn_controller(worker).await; - handle.send(Method::run_text("Hello")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "Hello", + )) + .await + .unwrap(); tokio::time::sleep(std::time::Duration::from_millis(100)).await; @@ -1201,7 +1302,13 @@ async fn events_are_broadcast() { let handle = spawn_controller(worker).await; let mut rx = handle.subscribe(); - handle.send(Method::run_text("Hello")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "Hello", + )) + .await + .unwrap(); let mut saw_turn_start = false; let mut saw_text_delta = false; @@ -1236,10 +1343,8 @@ async fn events_are_broadcast() { } #[tokio::test] -async fn double_run_returns_error() { - // Keep the first turn in-flight until the test drops the handle. A - // finite stream can finish before the second Method reaches the - // controller in the full test suite, making this assertion racy. +async fn submit_while_running_is_durably_queued() { + // Keep the first turn in-flight until the second Submit is accepted. let events = vec![ LlmEvent::text_block_start(0), LlmEvent::text_delta(0, "slow..."), @@ -1249,45 +1354,87 @@ async fn double_run_returns_error() { let handle = spawn_controller(worker).await; let mut rx = handle.subscribe(); - // Send first run and wait until the controller has entered Running. - handle.send(Method::run_text("first")).await.unwrap(); + handle + .send(Method::submit_text("request-first", "first")) + .await + .unwrap(); wait_for_status(&handle, WorkerStatus::Running).await; + handle + .send(Method::submit_text("request-second", "second")) + .await + .unwrap(); - // Now the second run must be rejected by drive_turn's live Method arm. - handle.send(Method::run_text("second")).await.unwrap(); - - // Look for the error event - let mut saw_already_running = false; let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2); - loop { - tokio::select! { - event = rx.recv() => { - match event { - Ok(Event::Error { code, .. }) => { - if code == worker::ErrorCode::AlreadyRunning { - saw_already_running = true; - break; - } - } - Err(_) => break, - _ => {} - } + let mut accepted = None; + let mut pending_snapshot = None; + while tokio::time::Instant::now() < deadline { + match tokio::time::timeout(std::time::Duration::from_millis(100), rx.recv()).await { + Ok(Ok(Event::SubmissionAccepted { + submission_request_id, + disposition, + .. + })) if submission_request_id == "request-second" => accepted = Some(disposition), + Ok(Ok(Event::PendingSubmissionsChanged { pending })) + if pending.submissions.len() == 1 => + { + pending_snapshot = Some(pending) } - _ = tokio::time::sleep_until(deadline) => break, + Ok(Ok(Event::Error { code, message })) if code == worker::ErrorCode::AlreadyRunning => { + panic!("Submit was busy-rejected: {message}") + } + _ => {} + } + if accepted.is_some() && pending_snapshot.is_some() { + break; } } - assert!(saw_already_running, "should see already_running error"); + assert_eq!(accepted, Some(protocol::SubmissionDisposition::Queued)); + let pending_snapshot = pending_snapshot.expect("pending snapshot"); + assert_eq!(pending_snapshot.submissions.len(), 1); + handle + .send(Method::Pause { + command: worker_command(&handle), + }) + .await + .unwrap(); + wait_for_status(&handle, WorkerStatus::Paused).await; + handle + .send(Method::ContinuePending { + expected_revision: pending_snapshot.revision, + expected_head_id: pending_snapshot.head_id.expect("pending head"), + }) + .await + .unwrap(); + let rejection = tokio::time::timeout(std::time::Duration::from_secs(1), async { + loop { + if let Ok(Event::Error { code, message }) = rx.recv().await + && code == worker::ErrorCode::InvalidRequest + && message.contains("requires an idle Worker") + { + break message; + } + } + }) + .await + .expect("paused ContinuePending rejection"); + assert!(rejection.contains("Resume or Cancel")); + assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Paused); } #[tokio::test] -async fn resume_without_pause_returns_error() { +async fn resume_without_pause_returns_invalid_state_acknowledgement() { let client = MockClient::new(simple_text_events()); let worker = make_worker(client).await; let handle = spawn_controller(worker).await; let mut rx = handle.subscribe(); - handle.send(Method::Resume).await.unwrap(); + handle + .send(Method::Resume { + command: worker_command(&handle), + }) + .await + .unwrap(); let mut saw_not_paused = false; let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(1); @@ -1295,7 +1442,10 @@ async fn resume_without_pause_returns_error() { tokio::select! { event = rx.recv() => { match event { - Ok(Event::Error { code, .. }) if code == worker::ErrorCode::NotPaused => { + Ok(Event::CommandAcknowledged { acknowledgement }) + if acknowledgement.command == protocol::WorkerCommandKind::Resume + && acknowledgement.disposition + == protocol::WorkerCommandDisposition::InvalidState => { saw_not_paused = true; break; } @@ -1307,17 +1457,22 @@ async fn resume_without_pause_returns_error() { } } - assert!(saw_not_paused, "should see not_paused error"); + assert!(saw_not_paused, "should see invalid-state acknowledgement"); } #[tokio::test] -async fn cancel_without_run_returns_error() { +async fn cancel_without_run_returns_invalid_state_acknowledgement() { let client = MockClient::new(simple_text_events()); let worker = make_worker(client).await; let handle = spawn_controller(worker).await; let mut rx = handle.subscribe(); - handle.send(Method::Cancel).await.unwrap(); + handle + .send(Method::Cancel { + command: worker_command(&handle), + }) + .await + .unwrap(); let mut saw_not_running = false; let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(1); @@ -1325,7 +1480,10 @@ async fn cancel_without_run_returns_error() { tokio::select! { event = rx.recv() => { match event { - Ok(Event::Error { code, .. }) if code == worker::ErrorCode::NotRunning => { + Ok(Event::CommandAcknowledged { acknowledgement }) + if acknowledgement.command == protocol::WorkerCommandKind::Cancel + && acknowledgement.disposition + == protocol::WorkerCommandDisposition::InvalidState => { saw_not_running = true; break; } @@ -1337,7 +1495,7 @@ async fn cancel_without_run_returns_error() { } } - assert!(saw_not_running, "should see not_running error"); + assert!(saw_not_running, "should see invalid-state acknowledgement"); } #[tokio::test] @@ -1365,7 +1523,8 @@ async fn run_with_paste_segment_inlines_content_and_emits_typed_user_message() { protocol::Segment::text(" thanks"), ]; handle - .send(Method::Run { + .send(Method::Submit { + submission_request_id: protocol::new_submission_request_id(), input: segments.clone(), }) .await @@ -1437,7 +1596,13 @@ async fn run_with_resolvable_file_ref_attaches_system_message_after_user() { path: "notes.md".into(), }, ]; - handle.send(Method::Run { input: segments }).await.unwrap(); + handle + .send(Method::Submit { + submission_request_id: protocol::new_submission_request_id(), + input: segments, + }) + .await + .unwrap(); // Wait for the turn to complete. let mut rx = handle.subscribe(); @@ -1485,7 +1650,8 @@ async fn run_with_file_ref_uses_manifest_file_upload_limit() { let handle = spawn_controller(worker).await; handle - .send(Method::Run { + .send(Method::Submit { + submission_request_id: protocol::new_submission_request_id(), input: vec![protocol::Segment::FileRef { path: "long.txt".into(), }], @@ -1538,7 +1704,13 @@ async fn run_with_unresolved_segment_emits_alert_and_placeholder() { path: "src/lib.rs".into(), }, ]; - handle.send(Method::Run { input: segments }).await.unwrap(); + handle + .send(Method::Submit { + submission_request_id: protocol::new_submission_request_id(), + input: segments, + }) + .await + .unwrap(); let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2); let mut saw_alert_for_file_ref = false; @@ -1586,6 +1758,7 @@ async fn notify_while_idle_auto_starts_turn_and_injects_system_message() { handle .send(Method::Notify { + notification_request_id: protocol::new_submission_request_id(), message: "turn finished".into(), auto_run: true, }) @@ -1626,6 +1799,19 @@ async fn notify_while_idle_auto_starts_turn_and_injects_system_message() { saw_notify_in_mirror, "Method::Notify should commit a SystemItem::Notification entry; mirror = {entries:?}" ); + let queue_checkpoint_is_atomic = entries.iter().any(|entry| match entry { + LogEntry::AnnotatedSystemItem { extensions, .. } => extensions.iter().any(|extension| { + extension.domain == "worker.pending_activations.v1" + && extension.payload["pending_notifications"] + .as_array() + .is_some_and(Vec::is_empty) + }), + _ => false, + }); + assert!( + queue_checkpoint_is_atomic, + "notification history and queue claim must share one log entry" + ); // Exactly one request was made; it must contain the formatted // notification as one of the items (committed to history by @@ -1671,23 +1857,33 @@ async fn notify_while_idle_with_auto_run_false_waits_for_explicit_run() { let client_for_assert = client.clone(); let worker = make_worker(client).await; let handle = spawn_controller(worker).await; + let notification_request_id = protocol::new_submission_request_id(); - handle - .send(Method::Notify { - message: "progress snapshot".into(), - auto_run: false, - }) - .await - .unwrap(); + for _ in 0..2 { + handle + .send(Method::Notify { + notification_request_id: notification_request_id.clone(), + message: "progress snapshot".into(), + auto_run: false, + }) + .await + .unwrap(); + } tokio::time::sleep(std::time::Duration::from_millis(100)).await; - assert_eq!(handle.shared_state.get_status(), WorkerStatus::Idle); + assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle); assert!( client_for_assert.captured_requests().is_empty(), "weak Notify must not stage RunForNotification while idle" ); - handle.send(Method::run_text("continue")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "continue", + )) + .await + .unwrap(); let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2); loop { if !client_for_assert.captured_requests().is_empty() { @@ -1772,7 +1968,7 @@ async fn worker_event_turn_ended_while_idle_auto_starts_turn_and_injects_system_ saw_worker_event_in_mirror, "Method::WorkerEvent should commit a SystemItem::WorkerEvent entry" ); - assert_eq!(handle.shared_state.get_status(), WorkerStatus::Idle); + assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle); let requests = client_for_assert.captured_requests(); assert_eq!( @@ -1835,7 +2031,7 @@ async fn worker_event_scope_sub_delegated_while_idle_stays_control_plane_only() tokio::time::sleep(std::time::Duration::from_millis(100)).await; assert_eq!( - handle.shared_state.get_status(), + handle.shared_state.catalog_status(), WorkerStatus::Idle, "control-plane ScopeSubDelegated must not auto-start the parent LLM" ); @@ -1867,9 +2063,16 @@ async fn notify_while_running_does_not_emit_already_running_error() { let handle = spawn_controller(worker).await; let mut rx = handle.subscribe(); - handle.send(Method::run_text("start")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "start", + )) + .await + .unwrap(); handle .send(Method::Notify { + notification_request_id: protocol::new_submission_request_id(), message: "ping".into(), auto_run: true, }) @@ -1902,6 +2105,71 @@ async fn notify_while_running_does_not_emit_already_running_error() { wait_for_status(&handle, WorkerStatus::Idle).await; } +#[tokio::test] +async fn weak_notify_while_running_is_deduped_and_survives_until_next_submit() { + let client = MockClient::sequential(vec![ + MockResponse::Hang(Vec::new()), + MockResponse::Complete(simple_text_events()), + ]); + let client_for_assert = client.clone(); + let worker = make_worker(client).await; + let handle = spawn_controller(worker).await; + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "first", + )) + .await + .unwrap(); + wait_for_status(&handle, WorkerStatus::Running).await; + + let notification_request_id = protocol::new_submission_request_id(); + for _ in 0..2 { + handle + .send(Method::Notify { + notification_request_id: notification_request_id.clone(), + message: "durable weak notice".into(), + auto_run: false, + }) + .await + .unwrap(); + } + handle + .send(Method::Cancel { + command: worker_command(&handle), + }) + .await + .unwrap(); + wait_for_status(&handle, WorkerStatus::Idle).await; + + let mut rx = handle.subscribe(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "second", + )) + .await + .unwrap(); + tokio::time::timeout(std::time::Duration::from_secs(2), async { + loop { + if matches!(rx.recv().await, Ok(Event::TurnEnd { .. })) { + break; + } + } + }) + .await + .expect("second submit completes"); + + let requests = client_for_assert.captured_requests(); + let notice_count = requests[1] + .items + .iter() + .filter_map(|item| item.as_text()) + .filter(|text| text.contains("durable weak notice")) + .count(); + assert_eq!(notice_count, 1); +} + #[tokio::test] async fn status_json_reflects_worker_name() { let client = MockClient::new(simple_text_events()); @@ -1936,7 +2204,13 @@ async fn socket_run_receives_events() { let mut writer = JsonLineWriter::new(writer); // Send run method via socket - writer.write(&Method::run_text("Hello")).await.unwrap(); + writer + .write(&Method::submit_text( + protocol::new_submission_request_id(), + "Hello", + )) + .await + .unwrap(); // Collect events let mut saw_turn_start = false; @@ -2243,7 +2517,13 @@ async fn pause_then_resume_transitions_and_preserves_history_consistency() { let handle = spawn_controller(worker).await; let mut rx = handle.subscribe(); - handle.send(Method::run_text("hello")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "hello", + )) + .await + .unwrap(); // Wait for the partial text_delta to confirm the first stream is // live before we pause. @@ -2256,7 +2536,12 @@ async fn pause_then_resume_transitions_and_preserves_history_consistency() { "text_delta should arrive before pause" ); - handle.send(Method::Pause).await.unwrap(); + handle + .send(Method::Pause { + command: worker_command(&handle), + }) + .await + .unwrap(); // The controller emits RunEnd { Paused } when the // EngineError::Cancelled is translated under pause_requested. @@ -2272,9 +2557,14 @@ async fn pause_then_resume_transitions_and_preserves_history_consistency() { ); tokio::time::sleep(std::time::Duration::from_millis(50)).await; - assert_eq!(handle.shared_state.get_status(), WorkerStatus::Paused); + assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Paused); - handle.send(Method::Resume).await.unwrap(); + handle + .send(Method::Resume { + command: worker_command(&handle), + }) + .await + .unwrap(); assert!( drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!( @@ -2288,7 +2578,7 @@ async fn pause_then_resume_transitions_and_preserves_history_consistency() { ); tokio::time::sleep(std::time::Duration::from_millis(50)).await; - assert_eq!(handle.shared_state.get_status(), WorkerStatus::Idle); + assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Idle); // History consistency: exactly [user "hello", assistant // "resumed output"]. No artifacts from the aborted stream @@ -2332,7 +2622,7 @@ async fn pause_then_resume_transitions_and_preserves_history_consistency() { assert!(!has_tool_call, "no orphan tool_call in history"); } -/// Paused with an orphan `tool_use` in history + a fresh `Method::Run` +/// Paused with an orphan `tool_use` in history + a fresh `Method::Submit` /// must produce a wire-valid next LLM request: the orphan is closed /// with a synthetic `tool_result`, a system note is inserted, and the /// new user input is appended. @@ -2369,7 +2659,13 @@ async fn paused_then_run_closes_orphan_tool_use_for_next_request() { let handle = spawn_controller(worker).await; let mut rx = handle.subscribe(); - handle.send(Method::run_text("first")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "first", + )) + .await + .unwrap(); // Wait for ToolCallDone — the ToolCall is committed to history // right before the Engine enters tool execution and pends. @@ -2382,7 +2678,12 @@ async fn paused_then_run_closes_orphan_tool_use_for_next_request() { "tool_call_done should arrive before pause" ); - handle.send(Method::Pause).await.unwrap(); + handle + .send(Method::Pause { + command: worker_command(&handle), + }) + .await + .unwrap(); assert!( drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!( e, @@ -2394,13 +2695,19 @@ async fn paused_then_run_closes_orphan_tool_use_for_next_request() { "expected RunEnd::Paused" ); tokio::time::sleep(std::time::Duration::from_millis(50)).await; - assert_eq!(handle.shared_state.get_status(), WorkerStatus::Paused); + assert_eq!(handle.shared_state.catalog_status(), WorkerStatus::Paused); // New user input while Paused → `Worker::run` observes // `last_run_interrupted` and runs its interrupt-prep step, which // closes the orphan + injects a system note before the fresh user // message. - handle.send(Method::run_text("new request")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "new request", + )) + .await + .unwrap(); assert!( drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!( e, @@ -2531,7 +2838,13 @@ async fn paused_cancel_abandons_resume_and_next_input_is_fresh_run() { let handle = spawn_controller(worker).await; let mut rx = handle.subscribe(); - handle.send(Method::run_text("first")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "first", + )) + .await + .unwrap(); assert!( drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!( e, @@ -2541,7 +2854,12 @@ async fn paused_cancel_abandons_resume_and_next_input_is_fresh_run() { "tool_call_done should arrive before pause" ); - handle.send(Method::Pause).await.unwrap(); + handle + .send(Method::Pause { + command: worker_command(&handle), + }) + .await + .unwrap(); assert!( drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!( e, @@ -2554,7 +2872,12 @@ async fn paused_cancel_abandons_resume_and_next_input_is_fresh_run() { ); wait_for_status(&handle, WorkerStatus::Paused).await; - handle.send(Method::Cancel).await.unwrap(); + handle + .send(Method::Cancel { + command: worker_command(&handle), + }) + .await + .unwrap(); wait_for_status(&handle, WorkerStatus::Idle).await; let (entries_after_cancel, _rx_after_cancel) = handle.sink.subscribe_with_snapshot(); assert!( @@ -2580,17 +2903,22 @@ async fn paused_cancel_abandons_resume_and_next_input_is_fresh_run() { "paused cancel must not resume or start another LLM request" ); - handle.send(Method::Resume).await.unwrap(); + handle + .send(Method::Resume { + command: worker_command(&handle), + }) + .await + .unwrap(); assert!( drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!( e, - Event::Error { - code: worker::ErrorCode::NotPaused, - .. - } + Event::CommandAcknowledged { acknowledgement } + if acknowledgement.command == protocol::WorkerCommandKind::Resume + && acknowledgement.disposition + == protocol::WorkerCommandDisposition::InvalidState )) .await, - "resume after paused cancel should be rejected as not paused" + "resume after paused cancel should receive invalid-state acknowledgement" ); assert_eq!( client_for_assert.captured_requests().len(), @@ -2599,7 +2927,10 @@ async fn paused_cancel_abandons_resume_and_next_input_is_fresh_run() { ); handle - .send(Method::run_text("fresh request")) + .send(Method::submit_text( + protocol::new_submission_request_id(), + "fresh request", + )) .await .unwrap(); assert!( @@ -2688,9 +3019,20 @@ async fn empty_turn_cancel_rolls_back_submit_entries_and_emits_signal() { let handle = spawn_controller(worker).await; let mut rx = handle.subscribe(); - handle.send(Method::run_text("rollback me")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "rollback me", + )) + .await + .unwrap(); wait_for_status(&handle, WorkerStatus::Running).await; - handle.send(Method::Cancel).await.unwrap(); + handle + .send(Method::Cancel { + command: worker_command(&handle), + }) + .await + .unwrap(); assert!( drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!( @@ -2721,11 +3063,19 @@ async fn empty_turn_pause_rolls_back_and_snapshot_does_not_restore_input() { let mut rx = handle.subscribe(); handle - .send(Method::run_text("pause rollback")) + .send(Method::submit_text( + protocol::new_submission_request_id(), + "pause rollback", + )) .await .unwrap(); wait_for_status(&handle, WorkerStatus::Running).await; - handle.send(Method::Pause).await.unwrap(); + handle + .send(Method::Pause { + command: worker_command(&handle), + }) + .await + .unwrap(); assert!( drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!( @@ -2755,7 +3105,13 @@ async fn empty_turn_rollback_removes_only_the_most_recent_turn() { let handle = spawn_controller(worker).await; let mut rx = handle.subscribe(); - handle.send(Method::run_text("first kept")).await.unwrap(); + handle + .send(Method::submit_text( + protocol::new_submission_request_id(), + "first kept", + )) + .await + .unwrap(); assert!( drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!( e, @@ -2769,11 +3125,19 @@ async fn empty_turn_rollback_removes_only_the_most_recent_turn() { wait_for_status(&handle, WorkerStatus::Idle).await; handle - .send(Method::run_text("second rolled back")) + .send(Method::submit_text( + protocol::new_submission_request_id(), + "second rolled back", + )) .await .unwrap(); wait_for_status(&handle, WorkerStatus::Running).await; - handle.send(Method::Cancel).await.unwrap(); + handle + .send(Method::Cancel { + command: worker_command(&handle), + }) + .await + .unwrap(); assert!( drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!( e, @@ -2816,7 +3180,10 @@ async fn pause_after_assistant_token_does_not_rollback() { let mut rx = handle.subscribe(); handle - .send(Method::run_text("keep this turn")) + .send(Method::submit_text( + protocol::new_submission_request_id(), + "keep this turn", + )) .await .unwrap(); assert!( @@ -2827,7 +3194,12 @@ async fn pause_after_assistant_token_does_not_rollback() { .await, "assistant token should be visible before pause" ); - handle.send(Method::Pause).await.unwrap(); + handle + .send(Method::Pause { + command: worker_command(&handle), + }) + .await + .unwrap(); assert!( drain_until(&mut rx, std::time::Duration::from_secs(2), |e| matches!( diff --git a/crates/workspace-api/src/lib.rs b/crates/workspace-api/src/lib.rs index b77db77a..e52b45e8 100644 --- a/crates/workspace-api/src/lib.rs +++ b/crates/workspace-api/src/lib.rs @@ -539,6 +539,8 @@ pub enum WorkspaceAuthConfig { pub struct WorkspacePermissionSummary { pub manage_repositories: bool, pub manage_secrets: bool, + pub manage_runtimes: bool, + pub delete_workspace: bool, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] @@ -605,6 +607,341 @@ pub struct WorkspaceMetadataMutationResponse { pub diagnostics: Vec, } +pub const WORKSPACE_DELETION_MAX_OPERATION_ID_BYTES: usize = 128; +pub const WORKSPACE_DELETION_MAX_REVISION_BYTES: usize = 128; +pub const WORKSPACE_DELETION_MAX_CONFIRMATION_BYTES: usize = 256; +pub const WORKSPACE_DELETION_MAX_BLOCKERS: usize = 1024; +pub const WORKSPACE_DELETION_MAX_CHILD_OPERATION_IDS: usize = 4096; +pub const WORKSPACE_DELETION_MAX_RESOURCE_VALUE_BYTES: usize = 128; +pub const WORKSPACE_DELETION_MAX_BLOCKER_MESSAGE_BYTES: usize = 512; + +fn deserialize_workspace_deletion_operation_id<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + let value = String::deserialize(deserializer)?; + if value.len() > WORKSPACE_DELETION_MAX_OPERATION_ID_BYTES { + return Err(serde::de::Error::custom( + "Workspace deletion operation_id is too long", + )); + } + Ok(value) +} + +fn deserialize_workspace_deletion_revision<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + let value = String::deserialize(deserializer)?; + if value.len() > WORKSPACE_DELETION_MAX_REVISION_BYTES { + return Err(serde::de::Error::custom( + "Workspace deletion revision is too long", + )); + } + Ok(value) +} + +fn deserialize_workspace_deletion_confirmation<'de, D>(deserializer: D) -> Result +where + D: serde::Deserializer<'de>, +{ + let value = String::deserialize(deserializer)?; + if value.len() > WORKSPACE_DELETION_MAX_CONFIRMATION_BYTES { + return Err(serde::de::Error::custom( + "Workspace deletion confirmation is too long", + )); + } + Ok(value) +} + +fn deserialize_workspace_deletion_resource_value<'de, D>( + deserializer: D, +) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + let value = Option::::deserialize(deserializer)?; + if value + .as_ref() + .is_some_and(|value| value.len() > WORKSPACE_DELETION_MAX_RESOURCE_VALUE_BYTES) + { + return Err(serde::de::Error::custom( + "Workspace deletion resource value is too long", + )); + } + Ok(value) +} + +fn deserialize_workspace_deletion_blocker_message<'de, D>( + deserializer: D, +) -> Result +where + D: serde::Deserializer<'de>, +{ + let value = String::deserialize(deserializer)?; + if value.len() > WORKSPACE_DELETION_MAX_BLOCKER_MESSAGE_BYTES { + return Err(serde::de::Error::custom( + "Workspace deletion blocker message is too long", + )); + } + Ok(value) +} + +fn deserialize_workspace_deletion_blockers<'de, D>( + deserializer: D, +) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + let value = Vec::::deserialize(deserializer)?; + if value.len() > WORKSPACE_DELETION_MAX_BLOCKERS { + return Err(serde::de::Error::custom( + "too many Workspace deletion blockers", + )); + } + Ok(value) +} + +fn deserialize_workspace_deletion_child_operation_ids<'de, D>( + deserializer: D, +) -> Result, D::Error> +where + D: serde::Deserializer<'de>, +{ + let value = Vec::::deserialize(deserializer)?; + if value.len() > WORKSPACE_DELETION_MAX_CHILD_OPERATION_IDS { + return Err(serde::de::Error::custom( + "too many Workspace deletion child operations", + )); + } + if value + .iter() + .any(|operation_id| operation_id.len() > WORKSPACE_DELETION_MAX_OPERATION_ID_BYTES) + { + return Err(serde::de::Error::custom( + "Workspace deletion child operation_id is too long", + )); + } + Ok(value) +} + +/// Lifecycle state for one durable Workspace deletion operation. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(rename_all = "snake_case")] +pub enum WorkspaceDeletionState { + Queued, + Running, + Blocked, + Failed, + Succeeded, +} + +/// Stable category explaining why Workspace deletion cannot currently advance. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(rename_all = "snake_case")] +pub enum WorkspaceDeletionBlockerKind { + LastAccessibleWorkspace, + RevisionConflict, + DirtyWorkdir, + WorkerRemovalBlocked, + WorkdirRemovalBlocked, + RetentionHold, + CleanupUnavailable, +} + +/// One bounded, user-actionable blocker returned by preflight or execution. +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +pub struct WorkspaceDeletionBlocker { + pub kind: WorkspaceDeletionBlockerKind, + pub resource_kind: Option, + pub resource_key: Option, + pub message: String, +} + +/// Workspace-owned resources summarized before destructive confirmation. +#[derive(Debug, Clone, Copy, Default, Serialize, Deserialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] +pub struct WorkspaceDeletionResourceCounts { + #[cfg_attr(feature = "typescript", ts(type = "number"))] + pub workers: u64, + #[cfg_attr(feature = "typescript", ts(type = "number"))] + pub workdirs: u64, + #[cfg_attr(feature = "typescript", ts(type = "number"))] + pub repositories: u64, + #[cfg_attr(feature = "typescript", ts(type = "number"))] + pub runtime_bindings: u64, + #[cfg_attr(feature = "typescript", ts(type = "number"))] + pub secrets: u64, + #[cfg_attr(feature = "typescript", ts(type = "number"))] + pub artifacts: u64, +} + +/// Owner-only impact preview for deleting one Workspace. +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +pub struct WorkspaceDeletionPreflightResponse { + pub workspace_id: String, + pub display_name: String, + /// Opaque persisted Workspace metadata revision used as a CAS fence. + pub expected_revision: String, + pub can_delete: bool, + pub resources: WorkspaceDeletionResourceCounts, + pub blockers: Vec, +} + +/// Idempotent request to start or resume Workspace deletion. +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +pub struct WorkspaceDeletionRequest { + pub operation_id: String, + pub expected_revision: String, + pub confirmation: String, +} + +/// Durable deletion operation projection used by request responses and polling. +#[derive(Debug, Clone, Serialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +pub struct WorkspaceDeletionOperationResponse { + pub operation_id: String, + pub workspace_id: String, + pub display_name: String, + pub state: WorkspaceDeletionState, + pub resources: WorkspaceDeletionResourceCounts, + pub child_operation_ids: Vec, + pub blockers: Vec, + pub failure_category: Option, + pub created_at: String, + pub updated_at: String, + pub completed_at: Option, +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct WorkspaceDeletionBlockerWire { + kind: WorkspaceDeletionBlockerKind, + #[serde(deserialize_with = "deserialize_workspace_deletion_resource_value")] + resource_kind: Option, + #[serde(deserialize_with = "deserialize_workspace_deletion_resource_value")] + resource_key: Option, + #[serde(deserialize_with = "deserialize_workspace_deletion_blocker_message")] + message: String, +} + +impl<'de> Deserialize<'de> for WorkspaceDeletionBlocker { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let wire = WorkspaceDeletionBlockerWire::deserialize(deserializer)?; + Ok(Self { + kind: wire.kind, + resource_kind: wire.resource_kind, + resource_key: wire.resource_key, + message: wire.message, + }) + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct WorkspaceDeletionPreflightResponseWire { + workspace_id: String, + display_name: String, + #[serde(deserialize_with = "deserialize_workspace_deletion_revision")] + expected_revision: String, + can_delete: bool, + resources: WorkspaceDeletionResourceCounts, + #[serde(deserialize_with = "deserialize_workspace_deletion_blockers")] + blockers: Vec, +} + +impl<'de> Deserialize<'de> for WorkspaceDeletionPreflightResponse { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let wire = WorkspaceDeletionPreflightResponseWire::deserialize(deserializer)?; + Ok(Self { + workspace_id: wire.workspace_id, + display_name: wire.display_name, + expected_revision: wire.expected_revision, + can_delete: wire.can_delete, + resources: wire.resources, + blockers: wire.blockers, + }) + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct WorkspaceDeletionRequestWire { + #[serde(deserialize_with = "deserialize_workspace_deletion_operation_id")] + operation_id: String, + #[serde(deserialize_with = "deserialize_workspace_deletion_revision")] + expected_revision: String, + #[serde(deserialize_with = "deserialize_workspace_deletion_confirmation")] + confirmation: String, +} + +impl<'de> Deserialize<'de> for WorkspaceDeletionRequest { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let wire = WorkspaceDeletionRequestWire::deserialize(deserializer)?; + Ok(Self { + operation_id: wire.operation_id, + expected_revision: wire.expected_revision, + confirmation: wire.confirmation, + }) + } +} + +#[derive(Deserialize)] +#[serde(deny_unknown_fields)] +struct WorkspaceDeletionOperationResponseWire { + #[serde(deserialize_with = "deserialize_workspace_deletion_operation_id")] + operation_id: String, + workspace_id: String, + display_name: String, + state: WorkspaceDeletionState, + resources: WorkspaceDeletionResourceCounts, + #[serde(deserialize_with = "deserialize_workspace_deletion_child_operation_ids")] + child_operation_ids: Vec, + #[serde(deserialize_with = "deserialize_workspace_deletion_blockers")] + blockers: Vec, + failure_category: Option, + created_at: String, + updated_at: String, + completed_at: Option, +} + +impl<'de> Deserialize<'de> for WorkspaceDeletionOperationResponse { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let wire = WorkspaceDeletionOperationResponseWire::deserialize(deserializer)?; + Ok(Self { + operation_id: wire.operation_id, + workspace_id: wire.workspace_id, + display_name: wire.display_name, + state: wire.state, + resources: wire.resources, + child_operation_ids: wire.child_operation_ids, + blockers: wire.blockers, + failure_category: wire.failure_category, + created_at: wire.created_at, + updated_at: wire.updated_at, + completed_at: wire.completed_at, + }) + } +} + /// Read-only Profile catalog projected from one active Workspace config revision. #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] @@ -1135,6 +1472,7 @@ pub struct ObjectiveLinkTicketRequest { } #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[serde(rename_all = "snake_case")] pub enum RuntimeSourceKind { EmbeddedWorkerRuntime, @@ -1142,6 +1480,7 @@ pub enum RuntimeSourceKind { } #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[serde(rename_all = "snake_case")] pub enum RuntimeSourceStatus { Active, @@ -1149,6 +1488,7 @@ pub enum RuntimeSourceStatus { } #[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[serde(rename_all = "snake_case")] pub enum RuntimeIdentityAuthority { RuntimeRegistryProjection, @@ -1156,6 +1496,8 @@ pub enum RuntimeIdentityAuthority { } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] pub struct RuntimeSourceSummary { pub kind: RuntimeSourceKind, pub status: RuntimeSourceStatus, @@ -1164,6 +1506,7 @@ pub struct RuntimeSourceSummary { } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] pub struct RuntimeSummary { pub runtime_id: String, pub label: String, @@ -1180,6 +1523,8 @@ pub struct RuntimeSummary { } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] pub struct RuntimeManagementSummary { pub built_in: bool, pub config_managed: bool, @@ -1189,12 +1534,124 @@ pub struct RuntimeManagementSummary { } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] pub struct WorkspaceRuntimeResource { #[serde(flatten)] pub runtime: RuntimeSummary, pub management: RuntimeManagementSummary, } +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(rename_all = "snake_case")] +pub enum RuntimeTrustKeyStatus { + Unconfigured, + Active, + Revoked, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] +pub struct RuntimeTrustKeyState { + pub status: RuntimeTrustKeyStatus, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub fingerprint: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[cfg_attr(feature = "typescript", ts(type = "number | null"))] + pub revision: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub created_at: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub updated_at: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub revoked_at: Option, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(rename_all = "snake_case")] +pub enum RuntimeTrustAuditAction { + Created, + Replaced, + Reactivated, + Revoked, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] +pub struct RuntimeTrustAuditEntry { + pub action: RuntimeTrustAuditAction, + pub actor_account_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub old_fingerprint: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub new_fingerprint: Option, + #[cfg_attr(feature = "typescript", ts(type = "number"))] + pub revision: u64, + pub at: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] +pub struct WorkspaceRuntimeDetail { + pub workspace_id: String, + pub runtime: WorkspaceRuntimeResource, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub endpoint: Option, + pub trust_key: RuntimeTrustKeyState, + #[serde(default)] + pub recent_audit: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] +pub struct RuntimeTrustKeyRevealResponse { + pub public_key: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] +pub struct PutRuntimeTrustKeyRequest { + pub public_key: String, + #[serde(default)] + #[cfg_attr(feature = "typescript", ts(type = "number | null"))] + pub expected_revision: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] +pub struct RevokeRuntimeTrustKeyRequest { + #[cfg_attr(feature = "typescript", ts(type = "number"))] + pub expected_revision: u64, +} + +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(rename_all = "snake_case")] +pub enum RuntimeTrustConflictKind { + StaleRevision, + FingerprintInUse, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] +pub struct RuntimeTrustConflictResponse { + pub error: RuntimeTrustConflictKind, + pub message: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[cfg_attr(feature = "typescript", ts(type = "number"))] + pub current_revision: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub current_fingerprint: Option, +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] #[serde(deny_unknown_fields)] pub struct CreateRemoteRuntimeRequest { @@ -1205,16 +1662,39 @@ pub struct CreateRemoteRuntimeRequest { } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(rename_all = "snake_case")] +pub enum RuntimeConnectionTestStatus { + Compatible, + Failed, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(rename_all = "snake_case")] +pub enum RuntimeConnectionTestFailureKind { + Authentication, + Authorization, + NetworkUnreachable, + Timeout, + TlsOrTransport, + MalformedResponse, + ProtocolVersionMismatch, + RuntimeIdentityMismatch, + Configuration, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] pub struct RuntimeConnectionTestResponse { pub workspace_id: String, pub runtime_id: String, pub checked_at: String, - pub state: String, - pub protocol_version: Option, - pub compatibility_basis: String, - #[serde(default)] - pub capabilities: Vec, - pub health_result: String, + pub status: RuntimeConnectionTestStatus, + pub failure_kind: Option, + pub expected_protocol_version: u32, + pub actual_protocol_version: Option, #[serde(default)] pub diagnostics: Vec, } @@ -1424,7 +1904,12 @@ pub struct WorkerSummary { #[serde(default)] pub tags: Vec, pub workspace: WorkerWorkspaceSummary, + /// Runtime catalog lifecycle compatibility state. Live foreground state, when + /// available, is carried separately in `worker_state`. pub state: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + #[cfg_attr(feature = "typescript", ts(optional))] + pub worker_state: Option, pub last_seen_at: Option, #[serde(default)] pub pinned: bool, @@ -2348,6 +2833,13 @@ pub fn catalog_typescript() -> String { WorkspaceCreateResponse::decl(&config), WorkspaceAuthConfig::decl(&config), WorkspacePermissionSummary::decl(&config), + WorkspaceDeletionState::decl(&config), + WorkspaceDeletionBlockerKind::decl(&config), + WorkspaceDeletionBlocker::decl(&config), + WorkspaceDeletionResourceCounts::decl(&config), + WorkspaceDeletionPreflightResponse::decl(&config), + WorkspaceDeletionRequest::decl(&config), + WorkspaceDeletionOperationResponse::decl(&config), DiagnosticSeverity::decl(&config), Diagnostic::decl(&config), WorkspaceExtensionPointState::decl(&config), @@ -2371,6 +2863,26 @@ pub fn catalog_typescript() -> String { RepositoryListResponse::decl(&config), RepositoryDetailResponse::decl(&config), RepositoryLogResponse::decl(&config), + RuntimeSourceKind::decl(&config), + RuntimeSourceStatus::decl(&config), + RuntimeIdentityAuthority::decl(&config), + RuntimeSourceSummary::decl(&config), + RuntimeSummary::decl(&config), + RuntimeManagementSummary::decl(&config), + WorkspaceRuntimeResource::decl(&config), + RuntimeTrustKeyStatus::decl(&config), + RuntimeTrustKeyState::decl(&config), + RuntimeTrustAuditAction::decl(&config), + RuntimeTrustAuditEntry::decl(&config), + WorkspaceRuntimeDetail::decl(&config), + RuntimeTrustKeyRevealResponse::decl(&config), + PutRuntimeTrustKeyRequest::decl(&config), + RevokeRuntimeTrustKeyRequest::decl(&config), + RuntimeTrustConflictKind::decl(&config), + RuntimeTrustConflictResponse::decl(&config), + RuntimeConnectionTestStatus::decl(&config), + RuntimeConnectionTestFailureKind::decl(&config), + RuntimeConnectionTestResponse::decl(&config), ] .map(|declaration| format!("export {declaration}")); @@ -2981,6 +3493,81 @@ mod tests { } } + #[test] + fn workspace_deletion_wire_contract_is_closed_and_typed() { + let preflight = WorkspaceDeletionPreflightResponse { + workspace_id: "workspace-test".to_string(), + display_name: "Test".to_string(), + expected_revision: "revision-7".to_string(), + can_delete: true, + resources: WorkspaceDeletionResourceCounts { + workers: 2, + workdirs: 1, + repositories: 1, + runtime_bindings: 1, + secrets: 0, + artifacts: 3, + }, + blockers: Vec::new(), + }; + let value = serde_json::to_value(&preflight).unwrap(); + assert_eq!( + serde_json::from_value::(value.clone()).unwrap(), + preflight + ); + let mut stale = value.as_object().unwrap().clone(); + stale.insert("revision".to_string(), serde_json::json!(7)); + assert!( + serde_json::from_value::(stale.into()).is_err() + ); + + assert!( + serde_json::from_value::(serde_json::json!({ + "operation_id": "delete-test", + "expected_revision": "revision-7", + "confirmation": "Test", + "workspace_id": "caller-controlled" + })) + .is_err() + ); + assert!( + serde_json::from_value::(serde_json::json!({ + "operation_id": "x".repeat(WORKSPACE_DELETION_MAX_OPERATION_ID_BYTES + 1), + "expected_revision": "revision-7", + "confirmation": "Test" + })) + .is_err() + ); + assert!( + serde_json::from_value::(serde_json::json!({ + "operation_id": "delete-test", + "workspace_id": "workspace-test", + "display_name": "Test", + "state": "blocked", + "resources": { + "workers": 0, + "workdirs": 0, + "repositories": 0, + "runtime_bindings": 0, + "secrets": 0, + "artifacts": 0 + }, + "child_operation_ids": [], + "blockers": (0..=WORKSPACE_DELETION_MAX_BLOCKERS).map(|_| serde_json::json!({ + "kind": "cleanup_unavailable", + "resource_kind": null, + "resource_key": null, + "message": "blocked" + })).collect::>(), + "failure_category": null, + "created_at": "1", + "updated_at": "1", + "completed_at": null + })) + .is_err() + ); + } + #[test] fn workspace_and_repository_response_shapes_round_trip() { let workspace = serde_json::json!({ @@ -2996,7 +3583,9 @@ mod tests { }}, "permissions": { "manage_repositories": true, - "manage_secrets": true + "manage_secrets": true, + "manage_runtimes": true, + "delete_workspace": true }, "extension_points": { "store": "sqlite", @@ -3060,6 +3649,102 @@ mod tests { assert!(serde_json::from_value::(stale).is_err()); } + #[test] + fn runtime_detail_and_trust_mutations_are_closed_and_typed() { + let detail = serde_json::json!({ + "workspace_id": "workspace-test", + "runtime": { + "runtime_id": "runtime-test", + "label": "Runtime Test", + "kind": "remote_http", + "status": "active", + "source": { + "kind": "remote_http", + "status": "active", + "identity_authority": "runtime_registry_projection", + "note": "active" + }, + "host_ids": [], + "worker_creation_available": true, + "os": "linux", + "arch": "x86_64", + "diagnostics": [], + "management": { + "built_in": false, + "config_managed": true, + "removable": true, + "endpoint_configured": true, + "token_ref_configured": false + } + }, + "endpoint": "https://runtime.example", + "trust_key": { + "status": "active", + "fingerprint": "SHA256:test", + "revision": 2, + "created_at": "2026-09-01T12:00:00Z", + "updated_at": "2026-09-01T13:00:00Z" + }, + "recent_audit": [{ + "action": "replaced", + "actor_account_id": "account-owner", + "old_fingerprint": "SHA256:old", + "new_fingerprint": "SHA256:test", + "revision": 2, + "at": "2026-09-01T13:00:00Z" + }] + }); + let parsed: WorkspaceRuntimeDetail = serde_json::from_value(detail.clone()).unwrap(); + assert_eq!(serde_json::to_value(parsed).unwrap(), detail); + + let mut unknown = detail; + unknown["trust_key"]["private_key"] = serde_json::json!("forbidden"); + assert!(serde_json::from_value::(unknown).is_err()); + assert!( + serde_json::from_value::(serde_json::json!({ + "public_key": "yoi-ed25519-pub:v1:key", + "private_key": "forbidden" + })) + .is_err() + ); + assert!( + serde_json::from_value::(serde_json::json!({ + "public_key": "key", + "expected_revision": 1, + "replace": true + })) + .is_err() + ); + assert!( + serde_json::from_value::(serde_json::json!({ + "expected_revision": 1, + "delete_runtime": true + })) + .is_err() + ); + } + + #[test] + fn runtime_connection_test_response_is_closed_and_typed() { + let compatible = serde_json::json!({ + "workspace_id": "workspace-test", + "runtime_id": "runtime-test", + "checked_at": "2026-09-01T12:00:00Z", + "status": "compatible", + "failure_kind": null, + "expected_protocol_version": 1, + "actual_protocol_version": 1, + "diagnostics": [] + }); + let parsed: RuntimeConnectionTestResponse = + serde_json::from_value(compatible.clone()).unwrap(); + assert_eq!(serde_json::to_value(parsed).unwrap(), compatible); + + let mut unknown = compatible; + unknown["capabilities"] = serde_json::json!(["shell"]); + assert!(serde_json::from_value::(unknown).is_err()); + } + #[cfg(feature = "typescript")] #[test] fn generated_catalog_typescript_keeps_public_wrappers_and_nullability() { @@ -3080,6 +3765,9 @@ mod tests { assert!(output.contains( "export type WorkspaceProfileSourceProvenance = \"project_profile_source_tree\"" )); + assert!(output.contains("export type RuntimeConnectionTestResponse =")); + assert!(output.contains("status: RuntimeConnectionTestStatus")); + assert!(output.contains("failure_kind: RuntimeConnectionTestFailureKind | null")); assert!(!output.contains("repository_key: string, display_name")); } diff --git a/crates/workspace-server/src/config.rs b/crates/workspace-server/src/config.rs index af52b7d0..ec7ce820 100644 --- a/crates/workspace-server/src/config.rs +++ b/crates/workspace-server/src/config.rs @@ -5,12 +5,10 @@ use std::{fs, io}; use serde::{Deserialize, Serialize}; use url::Url; -use crate::hosts::RemoteRuntimeConfig; use crate::identity::WorkspaceIdentity; use crate::server::{AuthConfig, ServerConfig}; use crate::{Error, Result}; -pub const BACKEND_RUNTIMES_CONFIG_FILE_NAME: &str = "runtimes.toml"; pub const SERVER_HOST_CONFIG_FILE_NAME: &str = "server.toml"; const DEFAULT_LISTEN: &str = "127.0.0.1:8787"; const DEFAULT_BROWSER_PUBLIC_URL: &str = "http://localhost:5173"; @@ -51,31 +49,6 @@ fn default_browser_public_url() -> String { DEFAULT_BROWSER_PUBLIC_URL.to_string() } -#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct BackendRuntimesConfigFile { - #[serde(default)] - pub runtimes: WorkspaceBackendRuntimesConfig, -} - -#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct WorkspaceBackendRuntimesConfig { - #[serde(default)] - pub remote: Vec, -} - -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -#[serde(deny_unknown_fields)] -pub struct RemoteRuntimeConfigFile { - pub id: String, - pub endpoint: String, - #[serde(default)] - pub display_name: Option, - #[serde(default)] - pub token_ref: Option, -} - #[derive(Clone)] pub struct ResolvedWorkspaceBackendConfig { pub server: ServerConfig, @@ -124,80 +97,11 @@ impl ServerHostConfigFile { } } -impl BackendRuntimesConfigFile { - pub fn path_for_config_dir(config_dir: impl AsRef) -> PathBuf { - config_dir.as_ref().join(BACKEND_RUNTIMES_CONFIG_FILE_NAME) - } - - pub fn default_path() -> Option { - manifest::paths::config_dir().map(Self::path_for_config_dir) - } - - pub fn load_default() -> Result { - match Self::default_path() { - Some(path) => Self::load_from_path(path), - None => Ok(Self::default()), - } - } - - pub fn load_from_config_dir(config_dir: impl AsRef) -> Result { - Self::load_from_path(Self::path_for_config_dir(config_dir)) - } - - pub fn load_from_path(path: impl AsRef) -> Result { - let path = path.as_ref(); - match fs::read_to_string(path) { - Ok(raw) => Self::parse_str(&raw, path), - Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(Self::default()), - Err(error) => Err(Error::Io(error)), - } - } - - pub fn write_default(&self) -> Result { - let path = Self::default_path().ok_or_else(|| { - Error::Config( - "YOI_CONFIG_DIR, YOI_HOME, XDG_CONFIG_HOME, or HOME is required to write Backend runtimes config" - .to_string(), - ) - })?; - self.write_to_path(&path)?; - Ok(path) - } - - pub fn write_to_config_dir(&self, config_dir: impl AsRef) -> Result<()> { - self.write_to_path(Self::path_for_config_dir(config_dir)) - } - - pub fn write_to_path(&self, path: impl AsRef) -> Result<()> { - let path = path.as_ref(); - if let Some(parent) = path.parent() { - fs::create_dir_all(parent)?; - } - let raw = toml::to_string_pretty(self).map_err(|error| { - Error::Config(format!( - "failed to serialize Backend runtimes config: {error}" - )) - })?; - fs::write(path, raw)?; - Ok(()) - } - - pub fn parse_str(raw: &str, path: impl AsRef) -> Result { - toml::from_str(raw).map_err(|error| { - Error::Config(format!( - "failed to parse Backend runtimes config `{}`: {error}", - path.as_ref().display() - )) - }) - } -} - impl ResolvedWorkspaceBackendConfig { pub fn local_dev( workspace_root: impl AsRef, identity: WorkspaceIdentity, host_config: &ServerHostConfigFile, - runtime_config: &BackendRuntimesConfigFile, ) -> Result { let workspace_root = workspace_root.as_ref(); let data_root = ServerConfig::default_workspace_backend_data_root(&identity.workspace_id); @@ -208,12 +112,7 @@ impl ResolvedWorkspaceBackendConfig { server.database_path = database_path.clone(); server.embedded_runtime_store_root = data_root.join("embedded-runtime"); server.max_records = DEFAULT_MAX_RECORDS; - server.remote_runtime_sources = runtime_config - .runtimes - .remote - .iter() - .map(resolve_remote_runtime) - .collect::>>()?; + server.remote_runtime_sources = Vec::new(); server.auth = AuthConfig::Passkey { rp_id: browser_rp_id, origin: browser_public_url.clone(), @@ -252,26 +151,6 @@ fn normalize_required_string(field: &str, value: &str) -> Result { Ok(trimmed.to_string()) } -pub(crate) fn resolve_remote_runtime( - config: &RemoteRuntimeConfigFile, -) -> Result { - if let Some(token_ref) = config.token_ref.as_deref() { - return Err(Error::Config(format!( - "remote runtime `{}` uses token_ref `{token_ref}`, but secret ref resolution is not implemented for Backend runtime settings yet", - config.id - ))); - } - Ok(RemoteRuntimeConfig::new( - config.id.clone(), - config - .display_name - .clone() - .unwrap_or_else(|| config.id.clone()), - config.endpoint.clone(), - None, - )) -} - fn resolve_browser_public_url(value: &str) -> Result<(String, String)> { let value = normalize_required_string("browser.public_url", value)?; let url = Url::parse(&value).map_err(|error| { @@ -314,22 +193,19 @@ mod tests { } } - fn resolved_with_runtimes( - runtimes: &BackendRuntimesConfigFile, - ) -> ResolvedWorkspaceBackendConfig { + fn resolved() -> ResolvedWorkspaceBackendConfig { let dir = tempfile::tempdir().unwrap(); ResolvedWorkspaceBackendConfig::local_dev( dir.path(), identity(), &ServerHostConfigFile::default(), - runtimes, ) .unwrap() } #[test] fn default_settings_resolve_without_a_repository_file() { - let resolved = resolved_with_runtimes(&BackendRuntimesConfigFile::default()); + let resolved = resolved(); assert_eq!(resolved.listen, "127.0.0.1:8787".parse().unwrap()); let AuthConfig::Passkey { @@ -354,7 +230,7 @@ mod tests { #[test] fn backend_base_url_is_explicit_and_normalized() { let listen = "127.0.0.1:48787".parse().unwrap(); - let resolved = resolved_with_runtimes(&BackendRuntimesConfigFile::default()) + let resolved = resolved() .with_listen(listen) .with_backend_base_url("http://127.0.0.1:48787/"); @@ -376,7 +252,6 @@ mod tests { tempfile::tempdir().unwrap().path(), identity(), &host_config, - &BackendRuntimesConfigFile::default(), ) .unwrap(); @@ -407,7 +282,6 @@ mod tests { tempfile::tempdir().unwrap().path(), identity(), &host_config, - &BackendRuntimesConfigFile::default(), ); let error = match result { Ok(_) => panic!("expected {value} to be rejected"), @@ -446,92 +320,8 @@ mod tests { } #[test] - fn backend_runtimes_config_loads_from_config_dir() { - let dir = tempfile::tempdir().unwrap(); - let config = BackendRuntimesConfigFile { - runtimes: WorkspaceBackendRuntimesConfig { - remote: vec![RemoteRuntimeConfigFile { - id: "arc".to_string(), - endpoint: "http://127.0.0.1:38800".to_string(), - display_name: Some("arc".to_string()), - token_ref: None, - }], - }, - }; - config.write_to_config_dir(dir.path()).unwrap(); - let loaded = BackendRuntimesConfigFile::load_from_config_dir(dir.path()).unwrap(); - assert_eq!(loaded, config); - assert_eq!( - BackendRuntimesConfigFile::path_for_config_dir(dir.path()), - dir.path().join("runtimes.toml") - ); - } - - #[test] - fn backend_runtimes_config_is_the_only_runtime_source() { - let runtime_config = BackendRuntimesConfigFile::parse_str( - r#" -[[runtimes.remote]] -id = "arc" -endpoint = "http://xdg.example.test" -display_name = "xdg arc" -"#, - "runtimes.toml", - ) - .unwrap(); - let resolved = resolved_with_runtimes(&runtime_config); - assert_eq!(resolved.server.remote_runtime_sources.len(), 1); - assert_eq!(resolved.server.remote_runtime_sources[0].runtime_id, "arc"); - assert_eq!( - resolved.server.remote_runtime_sources[0].base_url.as_str(), - "http://xdg.example.test" - ); - } - - #[test] - fn token_value_field_is_not_in_runtime_schema() { - let error = BackendRuntimesConfigFile::parse_str( - r#" -[[runtimes.remote]] -id = "remote" -endpoint = "http://127.0.0.1:8790" -token = "secret" -"#, - "runtimes.toml", - ) - .unwrap_err(); - assert!( - error.to_string().contains("unknown field"), - "unexpected error: {error}" - ); - } - - #[test] - fn token_ref_fails_closed_until_secret_resolution_exists() { - let runtime_config = BackendRuntimesConfigFile::parse_str( - r#" -[[runtimes.remote]] -id = "remote" -endpoint = "http://127.0.0.1:8790" -token_ref = "local:remote-token" -"#, - "runtimes.toml", - ) - .unwrap(); - let error = match ResolvedWorkspaceBackendConfig::local_dev( - tempfile::tempdir().unwrap().path(), - identity(), - &ServerHostConfigFile::default(), - &runtime_config, - ) { - Ok(_) => panic!("token_ref should fail closed until secret resolution exists"), - Err(error) => error, - }; - assert!( - error - .to_string() - .contains("secret ref resolution is not implemented"), - "unexpected error: {error}" - ); + fn local_host_config_does_not_supply_runtime_authority() { + let resolved = resolved(); + assert!(resolved.server.remote_runtime_sources.is_empty()); } } diff --git a/crates/workspace-server/src/hosts.rs b/crates/workspace-server/src/hosts.rs index 7f530a8d..a6b065b2 100644 --- a/crates/workspace-server/src/hosts.rs +++ b/crates/workspace-server/src/hosts.rs @@ -9,7 +9,9 @@ use serde::de::DeserializeOwned; use serde::{Deserialize, Serialize}; use sha2::{Digest, Sha256}; use std::{ + error::Error as _, future::Future, + io::Read as _, path::PathBuf, pin::Pin, sync::{Arc, RwLock}, @@ -34,17 +36,17 @@ use worker_runtime::config_bundle::{ ConfigBundleMetadata, ConfigBundleProvenance, ConfigProfileDescriptor, }; use worker_runtime::error::RuntimeError as EmbeddedRuntimeError; -#[cfg(test)] -use worker_runtime::execution::WorkerExecutionRunState; use worker_runtime::fs_store::FsRuntimeStoreOptions; use worker_runtime::http_server::{ + RUNTIME_PING_PERMISSION, RUNTIME_WORKSPACE_SCOPE_HEADER, RuntimeHttpConfigBundleAvailabilityResponse, RuntimeHttpConfigBundleSyncRequest, - RuntimeHttpErrorResponse, RuntimeHttpRepositoryAccessResponse, RuntimeHttpSummaryResponse, - RuntimeHttpUploadedFileDeleteResponse, RuntimeHttpUploadedFileResponse, - RuntimeHttpWorkerCompletionsRequest, RuntimeHttpWorkerCompletionsResponse, - RuntimeHttpWorkerDeleteResponse, RuntimeHttpWorkerInputResponse, - RuntimeHttpWorkerLifecycleRequest, RuntimeHttpWorkerLifecycleResponse, - RuntimeHttpWorkerResponse, RuntimeHttpWorkerWorkspaceApiRequest, RuntimeHttpWorkersResponse, + RuntimeHttpErrorResponse, RuntimeHttpPingResponse, RuntimeHttpRepositoryAccessResponse, + RuntimeHttpSummaryResponse, RuntimeHttpUploadedFileDeleteResponse, + RuntimeHttpUploadedFileResponse, RuntimeHttpWorkerCompletionsRequest, + RuntimeHttpWorkerCompletionsResponse, RuntimeHttpWorkerDeleteResponse, + RuntimeHttpWorkerInputResponse, RuntimeHttpWorkerLifecycleRequest, + RuntimeHttpWorkerLifecycleResponse, RuntimeHttpWorkerResponse, + RuntimeHttpWorkerWorkspaceApiRequest, RuntimeHttpWorkersResponse, RuntimeHttpWorkingDirectoriesResponse, RuntimeHttpWorkingDirectoryResponse, RuntimeHttpWorkspacePromptProjectionRequest, RuntimeHttpWorkspacePromptProjectionResponse, }; @@ -60,10 +62,11 @@ use worker_runtime::retention::{ WorkerRetentionExecutionRequest, WorkerRetentionExecutionResult, WorkerRetentionInventory, }; -pub(crate) const EMBEDDED_RUNTIME_ID: &str = "embedded-worker-runtime"; +pub const EMBEDDED_RUNTIME_ID: &str = "embedded-worker-runtime"; const EMBEDDED_HOST_KIND: &str = "embedded-worker-runtime-host"; const REMOTE_HOST_KIND: &str = "remote-worker-runtime-host"; const MAX_DIAGNOSTICS: usize = 16; +const MAX_RUNTIME_PING_RESPONSE_BYTES: usize = 8 * 1024; const MAX_HOST_SCAN: usize = 256; const MAX_IDENTIFIER_LEN: usize = 120; const ID_DIGEST_HEX_LEN: usize = 16; @@ -240,7 +243,10 @@ pub struct WorkerSummary { #[serde(default)] pub tags: Vec, pub workspace: WorkerWorkspaceSummary, + /// Runtime catalog lifecycle compatibility state. pub state: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub worker_state: Option, pub last_seen_at: Option, #[serde(default)] pub pinned: bool, @@ -332,6 +338,7 @@ pub(crate) fn workspace_worker_summary( workspace_id: summary.workspace.workspace_id, }, state: summary.state, + worker_state: summary.worker_state, last_seen_at: summary.last_seen_at, pinned: summary.pinned, retention_state: summary.retention_state, @@ -533,7 +540,7 @@ fn initial_worker_input(segments: &[Segment]) -> Option { Some(EmbeddedWorkerInput { kind: EmbeddedWorkerInputKind::User, content: Segment::flatten_to_text(segments), - submission_id: None, + submission_request_id: None, segments: Some(segments.to_vec()), }) } @@ -760,11 +767,50 @@ fn default_worker_input_kind() -> WorkerInputKind { WorkerInputKind::User } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum RuntimePingFailureKind { + Authentication, + Authorization, + NetworkUnreachable, + Timeout, + TlsOrTransport, + MalformedResponse, + Configuration, + Unsupported, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RuntimePingFailure { + pub kind: RuntimePingFailureKind, + pub diagnostic: RuntimeDiagnostic, +} + +impl RuntimePingFailure { + fn new( + kind: RuntimePingFailureKind, + code: impl Into, + message: impl Into, + ) -> Self { + Self { + kind, + diagnostic: diagnostic(code, DiagnosticSeverity::Error, message.into()), + } + } +} + pub trait WorkspaceWorkerRuntime: Send + Sync { fn runtime_id(&self) -> &str; fn runtime_summary(&self, limit: usize) -> RuntimeSummary; + fn ping(&self) -> Result { + Err(RuntimePingFailure::new( + RuntimePingFailureKind::Unsupported, + "runtime_ping_unsupported", + "Runtime connection testing is unavailable for this Runtime provider", + )) + } + fn list_hosts(&self, limit: usize) -> RuntimeList; fn list_workers(&self, limit: usize) -> RuntimeList; @@ -1126,15 +1172,19 @@ pub enum RuntimeRegistryUnregisterResult { }, } +type RuntimeBindingGate = Arc bool + Send + Sync>; + #[derive(Clone)] pub struct RuntimeRegistry { runtimes: Arc>>>, + runtime_binding_gate: Arc>>, } impl RuntimeRegistry { pub fn new(runtimes: Vec>) -> Self { Self { runtimes: Arc::new(RwLock::new(runtimes)), + runtime_binding_gate: Arc::new(RwLock::new(None)), } } @@ -1142,6 +1192,27 @@ impl RuntimeRegistry { Self::new(vec![Arc::new(embedded_runtime)]) } + pub fn set_runtime_binding_gate(&self, gate: F) + where + F: Fn(&str) -> bool + Send + Sync + 'static, + { + *self + .runtime_binding_gate + .write() + .expect("runtime binding gate lock poisoned") = Some(Arc::new(gate)); + } + + fn runtime_binding_is_active(&self, runtime_id: &str) -> bool { + if runtime_id == EMBEDDED_RUNTIME_ID { + return true; + } + self.runtime_binding_gate + .read() + .expect("runtime binding gate lock poisoned") + .as_ref() + .is_none_or(|gate| gate(runtime_id)) + } + pub fn register(&self, runtime: R) where R: WorkspaceWorkerRuntime + 'static, @@ -1357,12 +1428,7 @@ impl RuntimeRegistry { &self, projection: worker::WorkspacePromptProjection, ) -> Vec { - let runtimes = self - .runtimes - .read() - .map(|runtimes| runtimes.clone()) - .unwrap_or_default(); - runtimes + self.runtimes_snapshot() .into_iter() .filter_map(|runtime| { runtime @@ -1791,17 +1857,34 @@ impl RuntimeRegistry { }) } + pub fn ping(&self, runtime_id: &str) -> Result { + let runtime = self.runtime(runtime_id).map_err(|_| { + RuntimePingFailure::new( + RuntimePingFailureKind::Configuration, + "runtime_ping_registration_unavailable", + "Registered Runtime binding is unavailable", + ) + })?; + runtime.ping() + } + fn runtimes_snapshot(&self) -> Vec> { self.runtimes .read() .expect("runtime registry lock poisoned") - .clone() + .iter() + .filter(|runtime| self.runtime_binding_is_active(runtime.runtime_id())) + .cloned() + .collect() } fn runtime( &self, runtime_id: &str, ) -> Result, RuntimeRegistryError> { + if !self.runtime_binding_is_active(runtime_id) { + return Err(RuntimeRegistryError::UnknownRuntime(runtime_id.to_string())); + } self.runtimes .read() .expect("runtime registry lock poisoned") @@ -1919,6 +2002,7 @@ impl EmbeddedWorkerRuntime { workspace_id: summary.workspace_id.clone(), }, state: embedded_worker_status_label(summary.status).to_string(), + worker_state: summary.worker_state.clone(), last_seen_at: None, pinned: false, retention_state: "transient".to_string(), @@ -1958,6 +2042,7 @@ impl EmbeddedWorkerRuntime { workspace_id: detail.workspace_id.clone(), }, state: embedded_worker_status_label(detail.status).to_string(), + worker_state: detail.worker_state.clone(), last_seen_at: None, pinned: false, retention_state: "transient".to_string(), @@ -2625,7 +2710,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime { WorkerInputKind::RegisterPeer => EmbeddedWorkerInputKind::RegisterPeer, }, content: request.content, - submission_id: None, + submission_request_id: None, segments: request.segments, }; match self.runtime.send_input(&worker_ref, input) { @@ -2901,6 +2986,49 @@ pub struct RemoteWorkerRuntime { async_http: AsyncHttpClient, } +fn remote_runtime_ping_transport_failure(error: reqwest::Error) -> RuntimePingFailure { + if error.is_timeout() { + return RuntimePingFailure::new( + RuntimePingFailureKind::Timeout, + "runtime_ping_timeout", + "Runtime ping timed out", + ); + } + let mut source = error.source(); + let mut tls_error = false; + while let Some(current) = source { + let message = current.to_string().to_ascii_lowercase(); + if message.contains("tls") + || message.contains("certificate") + || message.contains("unknownissuer") + || message.contains("handshake") + { + tls_error = true; + break; + } + source = current.source(); + } + if tls_error { + return RuntimePingFailure::new( + RuntimePingFailureKind::TlsOrTransport, + "runtime_ping_tls_failed", + "Runtime TLS connection failed", + ); + } + if error.is_connect() { + return RuntimePingFailure::new( + RuntimePingFailureKind::NetworkUnreachable, + "runtime_ping_network_unreachable", + "Runtime could not be reached", + ); + } + RuntimePingFailure::new( + RuntimePingFailureKind::TlsOrTransport, + "runtime_ping_transport_failed", + "Runtime ping transport failed", + ) +} + fn all_remote_runtime_permissions() -> Vec { [ "workers:list", @@ -3049,14 +3177,18 @@ impl RemoteWorkerRuntime { self.send_json(path, self.http.delete(self.endpoint(path))) } - fn runtime_capability_token(&self, path: &str) -> Option { + fn runtime_capability_token_with_permissions( + &self, + path: &str, + permissions: Vec, + ) -> Option { let auth = self.auth.as_ref()?; let signer = CapabilityTokenSigner::new(&auth.server_id, &auth.server_private_key); let claims = capability_claims( &auth.server_id, &self.runtime_id, &self.workspace_id, - all_remote_runtime_permissions(), + permissions, 300, ) .map_err(|error| { @@ -3079,6 +3211,82 @@ impl RemoteWorkerRuntime { .ok() } + fn runtime_capability_token(&self, path: &str) -> Option { + self.runtime_capability_token_with_permissions(path, all_remote_runtime_permissions()) + } + + fn ping_http(&self) -> Result { + const PATH: &str = "/v1/ping"; + let workspace_id = self.workspace_id.clone(); + let bearer_token = self.bearer_token.clone(); + let capability_token = self.runtime_capability_token_with_permissions( + PATH, + vec![RUNTIME_PING_PERMISSION.to_string()], + ); + let request = self + .http + .get(self.endpoint(PATH)) + .header(RUNTIME_WORKSPACE_SCOPE_HEADER, &workspace_id); + run_blocking_http(move || { + let request = match capability_token.as_deref().or(bearer_token.as_deref()) { + Some(token) => request.header(AUTHORIZATION, format!("Bearer {token}")), + None => request, + }; + let response = request + .send() + .map_err(remote_runtime_ping_transport_failure)?; + match response.status() { + StatusCode::UNAUTHORIZED => { + return Err(RuntimePingFailure::new( + RuntimePingFailureKind::Authentication, + "runtime_ping_authentication_failed", + "Runtime rejected the connection-test credential", + )); + } + StatusCode::FORBIDDEN => { + return Err(RuntimePingFailure::new( + RuntimePingFailureKind::Authorization, + "runtime_ping_authorization_failed", + "Runtime rejected the connection-test scope or permission", + )); + } + status if !status.is_success() => { + return Err(RuntimePingFailure::new( + RuntimePingFailureKind::TlsOrTransport, + "runtime_ping_http_failed", + "Runtime ping returned an unsuccessful HTTP response", + )); + } + _ => {} + } + let mut body = Vec::new(); + response + .take((MAX_RUNTIME_PING_RESPONSE_BYTES + 1) as u64) + .read_to_end(&mut body) + .map_err(|_| { + RuntimePingFailure::new( + RuntimePingFailureKind::TlsOrTransport, + "runtime_ping_response_read_failed", + "Runtime ping response could not be read", + ) + })?; + if body.len() > MAX_RUNTIME_PING_RESPONSE_BYTES { + return Err(RuntimePingFailure::new( + RuntimePingFailureKind::MalformedResponse, + "runtime_ping_response_too_large", + "Runtime ping response exceeded the allowed size", + )); + } + serde_json::from_slice::(&body).map_err(|_| { + RuntimePingFailure::new( + RuntimePingFailureKind::MalformedResponse, + "runtime_ping_malformed_response", + "Runtime ping returned an unrecognized response", + ) + }) + }) + } + fn send_json(&self, path: &str, request: RequestBuilder) -> Result where T: DeserializeOwned + Send + 'static, @@ -3138,6 +3346,7 @@ impl RemoteWorkerRuntime { workspace_id: summary.workspace_id.clone(), }, state: embedded_worker_status_label(summary.status).to_string(), + worker_state: summary.worker_state.clone(), last_seen_at: None, pinned: false, retention_state: "transient".to_string(), @@ -3181,6 +3390,7 @@ impl RemoteWorkerRuntime { workspace_id: detail.workspace_id.clone(), }, state: embedded_worker_status_label(detail.status).to_string(), + worker_state: detail.worker_state.clone(), last_seen_at: None, pinned: false, retention_state: "transient".to_string(), @@ -3266,6 +3476,10 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime { } } + fn ping(&self) -> Result { + self.ping_http() + } + fn list_hosts(&self, limit: usize) -> RuntimeList { if limit == 0 { return RuntimeList::new(Vec::new(), Vec::new()); @@ -3726,7 +3940,7 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime { WorkerInputKind::RegisterPeer => EmbeddedWorkerInputKind::RegisterPeer, }, content: request.content, - submission_id: None, + submission_request_id: None, segments: request.segments, }; match self.post_json::<_, RuntimeHttpWorkerInputResponse>( @@ -4524,6 +4738,7 @@ pub fn placeholder_worker(host_id: impl Into) -> WorkerSummary { workspace_id: None, }, state: "unsupported".to_string(), + worker_state: None, last_seen_at: None, pinned: false, retention_state: "transient".to_string(), @@ -4562,7 +4777,7 @@ mod tests { use super::*; use serde_json::json; use std::collections::HashMap; - use std::io::{Read as _, Write as _}; + use std::io::Write as _; use std::net::TcpListener; use std::sync::{Arc, Mutex}; use std::thread; @@ -4962,7 +5177,6 @@ mod tests { request.worker_ref, self.backend_id(), ), - run_state: WorkerExecutionRunState::Idle, working_directory: request .working_directory .as_ref() @@ -4987,12 +5201,12 @@ mod tests { "missing test context", ); }; - let submission_id = input.submission_id.clone(); + let submission_request_id = input.submission_request_id.clone(); let content = input.content; std::thread::spawn(move || { std::thread::sleep(std::time::Duration::from_millis(10)); - let _ = context.publish_protocol_event(protocol::Event::Status { - status: protocol::WorkerStatus::Running, + let _ = context.publish_protocol_event(protocol::Event::WorkerState { + snapshot: protocol::WorkerStatus::Running.into(), }); let _ = context.publish_protocol_event(protocol::Event::TextDone { text: format!("echo: {content}"), @@ -5000,20 +5214,20 @@ mod tests { let _ = context.publish_protocol_event(protocol::Event::RunEnd { result: protocol::RunResult::Finished, }); - let _ = context.publish_protocol_event(protocol::Event::Status { - status: protocol::WorkerStatus::Idle, + let _ = context.publish_protocol_event(protocol::Event::WorkerState { + snapshot: protocol::WorkerStatus::Idle.into(), }); }); - if let Some(submission_id) = submission_id { - worker_runtime::execution::WorkerExecutionResult::accepted_input_committed( + if let Some(submission_request_id) = submission_request_id { + worker_runtime::execution::WorkerExecutionResult::accepted_submission( worker_runtime::execution::WorkerExecutionOperation::Input, - WorkerExecutionRunState::Busy, - submission_id, + submission_request_id, + uuid::Uuid::now_v7().to_string(), + protocol::SubmissionDisposition::Started, ) } else { worker_runtime::execution::WorkerExecutionResult::accepted( worker_runtime::execution::WorkerExecutionOperation::Input, - WorkerExecutionRunState::Busy, ) } } @@ -5046,6 +5260,7 @@ mod tests { workspace_id: None, }, state: "available".to_string(), + worker_state: None, last_seen_at: None, pinned: false, retention_state: "transient".to_string(), @@ -5165,6 +5380,64 @@ mod tests { assert_eq!(from_runtime_a.label, "worker from runtime a"); } + #[test] + fn registry_gate_rejects_cached_runtime_immediately_after_binding_revocation() { + let remote = + FixtureRuntime::with_worker("runtime-a", "host-a", "worker-a", "worker from runtime a"); + let remote_observed = remote.observed_prompt_revisions.clone(); + let embedded = FixtureRuntime::with_worker( + EMBEDDED_RUNTIME_ID, + "embedded-host", + "embedded-worker", + "embedded worker", + ); + let embedded_observed = embedded.observed_prompt_revisions.clone(); + let registry = RuntimeRegistry::new(vec![Arc::new(remote), Arc::new(embedded)]); + let active = Arc::new(Mutex::new(true)); + let gate_state = active.clone(); + registry.set_runtime_binding_gate(move |_| { + *gate_state.lock().expect("gate state lock poisoned") + }); + assert_eq!(registry.list_runtimes(10).items.len(), 2); + assert!( + registry + .worker(&RuntimeWorkerRef::new("runtime-a", "worker-a")) + .is_ok() + ); + + *active.lock().expect("gate state lock poisoned") = false; + assert_eq!(registry.list_runtimes(10).items.len(), 1); + assert!(matches!( + registry.worker(&RuntimeWorkerRef::new("runtime-a", "worker-a")), + Err(RuntimeRegistryError::UnknownRuntime(runtime_id)) if runtime_id == "runtime-a" + )); + + let catalog = worker::EffectivePromptCatalog::new( + std::collections::BTreeMap::from([( + "default".to_string(), + "workspace prompt".to_string(), + )]), + 12, + "schema", + "toolchain", + ) + .unwrap(); + let projection = worker::WorkspacePromptProjection::new( + "workspace-a", + "source-12", + catalog.catalog_digest.clone(), + catalog, + ) + .unwrap(); + assert!( + registry + .observe_workspace_prompt_projection(projection) + .is_empty() + ); + assert!(remote_observed.lock().unwrap().is_empty()); + assert_eq!(*embedded_observed.lock().unwrap(), vec![12]); + } + #[test] fn registry_broadcasts_workspace_prompt_projection_revisions() { let runtime = @@ -5707,6 +5980,30 @@ mod tests { assert_eq!(runtime.runtime_id(), "remote:async-init"); } + #[test] + fn remote_runtime_ping_classifies_unreachable_without_endpoint_leak() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + drop(listener); + let runtime = RemoteWorkerRuntime::new( + RemoteRuntimeConfig::new( + "remote:unreachable", + "Remote Unreachable", + endpoint.clone(), + Some("secret-token".to_string()), + ), + "workspace-test".to_string(), + "http://127.0.0.1:8787".to_string(), + ) + .unwrap(); + + let failure = runtime.ping().unwrap_err(); + assert_eq!(failure.kind, RuntimePingFailureKind::NetworkUnreachable); + assert_eq!(failure.diagnostic.code, "runtime_ping_network_unreachable"); + assert!(!failure.diagnostic.message.contains(&endpoint)); + assert!(!format!("{failure:?}").contains("secret-token")); + } + #[test] fn remote_runtime_registry_routes_commands_without_browser_secret_leaks() { let worker_id = EmbeddedWorkerId::from_legacy_u64(1).to_string(); diff --git a/crates/workspace-server/src/latest_schema.sql b/crates/workspace-server/src/latest_schema.sql index 038dd530..7a7c2199 100644 --- a/crates/workspace-server/src/latest_schema.sql +++ b/crates/workspace-server/src/latest_schema.sql @@ -433,15 +433,37 @@ CREATE TABLE ticket_worker_assignments ( (principal_kind != 'worker' AND runtime_id IS NULL AND worker_id IS NULL AND principal_id IS NOT NULL AND length(trim(principal_id)) > 0) ) ); -CREATE TABLE trusted_runtime_records ( - runtime_id TEXT PRIMARY KEY, +CREATE TABLE workspace_runtime_bindings ( + workspace_id TEXT NOT NULL, + runtime_id TEXT NOT NULL, display_name TEXT NOT NULL, base_url TEXT NOT NULL, public_key TEXT NOT NULL, + public_key_fingerprint TEXT NOT NULL, + binding_revision INTEGER NOT NULL DEFAULT 1 CHECK (binding_revision > 0), created_at TEXT NOT NULL, updated_at TEXT NOT NULL, - revoked_at TEXT -, workspace_id TEXT REFERENCES workspaces(workspace_id) ON DELETE RESTRICT); + revoked_at TEXT, + PRIMARY KEY (workspace_id, runtime_id), + UNIQUE (workspace_id, public_key_fingerprint), + FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE RESTRICT +); +CREATE TABLE workspace_runtime_binding_audit ( + workspace_id TEXT NOT NULL, + runtime_id TEXT NOT NULL, + actor_account_id TEXT NOT NULL, + action TEXT NOT NULL CHECK (action IN ('created', 'replaced', 'reactivated', 'revoked')), + old_fingerprint TEXT, + new_fingerprint TEXT, + binding_revision INTEGER NOT NULL CHECK (binding_revision > 0), + at TEXT NOT NULL, + PRIMARY KEY (workspace_id, runtime_id, binding_revision), + FOREIGN KEY(workspace_id, runtime_id) + REFERENCES workspace_runtime_bindings(workspace_id, runtime_id) ON DELETE RESTRICT, + FOREIGN KEY(actor_account_id) REFERENCES accounts(account_id) ON DELETE RESTRICT +); +CREATE INDEX idx_workspace_runtime_binding_audit_recent + ON workspace_runtime_binding_audit(workspace_id, runtime_id, binding_revision DESC); CREATE TABLE typed_ticket_artifacts ( workspace_id TEXT NOT NULL, ticket_id TEXT NOT NULL, relative_path TEXT NOT NULL, content BLOB NOT NULL, PRIMARY KEY (workspace_id, ticket_id, relative_path), @@ -637,7 +659,7 @@ CREATE TABLE worker_create_reservations ( worker_id TEXT NOT NULL, runtime_id TEXT NOT NULL, create_fingerprint TEXT NOT NULL, - state TEXT NOT NULL CHECK (state IN ('reserved', 'created')), + state TEXT NOT NULL CHECK (state IN ('reserved', 'created', 'removed')), created_at TEXT NOT NULL, updated_at TEXT NOT NULL, request_fingerprint TEXT, memory_settings_revision INTEGER, memory_language TEXT, PRIMARY KEY (workspace_id, allocation_key), @@ -651,11 +673,12 @@ CREATE TABLE worker_diagnostics_archives ( FOREIGN KEY(operation_id) REFERENCES worker_removal_operations(operation_id), FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE); CREATE TABLE worker_mutation_source_proof_jtis ( + workspace_id TEXT NOT NULL, runtime_id TEXT NOT NULL, jti TEXT NOT NULL, expires_at INTEGER NOT NULL, consumed_at TEXT NOT NULL, - PRIMARY KEY (runtime_id, jti) + PRIMARY KEY (workspace_id, runtime_id, jti) ); CREATE TABLE worker_orphan_diagnostics ( diagnostic_id TEXT PRIMARY KEY, workspace_id TEXT NOT NULL, runtime_id TEXT NOT NULL, worker_id TEXT NOT NULL, @@ -857,8 +880,8 @@ CREATE INDEX idx_ticket_worker_assignments_principal ON ticket_worker_assignments(workspace_id, role, principal_kind, principal_id, runtime_id, worker_id, assigned_at DESC); CREATE INDEX idx_ticket_worker_assignments_ticket ON ticket_worker_assignments(workspace_id, ticket_id, role, assigned_at DESC); -CREATE INDEX idx_trusted_runtime_records_workspace - ON trusted_runtime_records(workspace_id, revoked_at, runtime_id); +CREATE INDEX idx_workspace_runtime_bindings_workspace + ON workspace_runtime_bindings(workspace_id, revoked_at, runtime_id); CREATE INDEX idx_typed_ticket_relations_workspace_target ON typed_ticket_relations(workspace_id, target, at DESC); CREATE INDEX idx_typed_tickets_workspace_state_updated @@ -917,6 +940,28 @@ CREATE UNIQUE INDEX worker_workdir_links_active_worker_unique WHERE unlinked_at IS NULL; CREATE INDEX worker_workdir_links_workdir ON worker_workdir_links(workspace_id, workdir_id); +CREATE TABLE workspace_deletion_operations ( + operation_id TEXT PRIMARY KEY, + request_fingerprint TEXT NOT NULL, + workspace_id TEXT NOT NULL, + workspace_display_name TEXT NOT NULL, + workspace_revision TEXT NOT NULL, + owner_account_id TEXT NOT NULL, + actor_account_id TEXT NOT NULL, + state TEXT NOT NULL CHECK(state IN ('queued', 'running', 'blocked', 'failed', 'succeeded')), + resource_counts_json TEXT NOT NULL, + child_operation_ids_json TEXT NOT NULL, + blockers_json TEXT NOT NULL, + failure_category TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + completed_at TEXT, + FOREIGN KEY(owner_account_id) REFERENCES accounts(account_id) ON DELETE RESTRICT, + FOREIGN KEY(actor_account_id) REFERENCES accounts(account_id) ON DELETE RESTRICT +); +CREATE INDEX workspace_deletion_operations_workspace_recent + ON workspace_deletion_operations(workspace_id, created_at DESC); + CREATE TRIGGER seed_worker_retention_policy_after_workspace_insert AFTER INSERT ON workspaces BEGIN INSERT INTO workspace_worker_retention_policy_revisions (workspace_id,policy_id,revision,session_disposition,metadata_disposition,archive_retention_kind,archive_retention_seconds,diagnostics_disposition,diagnostics_retention_seconds,created_at) diff --git a/crates/workspace-server/src/lib.rs b/crates/workspace-server/src/lib.rs index 87d47955..f95318f2 100644 --- a/crates/workspace-server/src/lib.rs +++ b/crates/workspace-server/src/lib.rs @@ -33,6 +33,7 @@ pub mod workdir_create_operations; mod workdir_removal; pub mod worker_source; pub mod workspace_catalog; +mod workspace_deletion; mod workspace_subscription; pub use authority::{ @@ -40,7 +41,7 @@ pub use authority::{ ObjectiveAuthority, SqliteWorkspaceAuthority, TicketAuthority, TicketMergeRevisionSource, WorkspaceAuthority, }; -pub use config::{BackendRuntimesConfigFile, ResolvedWorkspaceBackendConfig, ServerHostConfigFile}; +pub use config::{ResolvedWorkspaceBackendConfig, ServerHostConfigFile}; pub use identity::{WORKSPACE_IDENTITY_RELATIVE_PATH, WorkspaceIdentity}; pub use records::{ObjectiveDetail, ObjectiveSummary, TicketDetail, TicketSummary}; pub use repositories::{ConfiguredRepository, RepositoryLogRead, RepositoryRegistryReader}; @@ -118,6 +119,17 @@ pub enum Error { WorkspacePermissionDenied(String), #[error("Workspace config update conflict: {0}")] WorkspaceConfigConflict(String), + #[error("Runtime binding conflict: {0}")] + RuntimeBindingConflict(String), + #[error("Runtime binding revision conflict: expected {expected:?}, current {actual:?}")] + RuntimeBindingRevisionConflict { + expected: Option, + actual: Option, + }, + #[error("Runtime public key fingerprint is already bound in this Workspace: {fingerprint}")] + RuntimeBindingFingerprintConflict { fingerprint: String }, + #[error("Runtime binding was not found for {runtime_id}")] + RuntimeBindingNotFound { runtime_id: String }, #[error("Repository conflict: {0}")] RepositoryConflict(String), #[error("Registry inconsistency: {0}")] diff --git a/crates/workspace-server/src/main.rs b/crates/workspace-server/src/main.rs index a20981e7..882d4436 100644 --- a/crates/workspace-server/src/main.rs +++ b/crates/workspace-server/src/main.rs @@ -9,10 +9,10 @@ use serde::{Deserialize, Serialize}; use tokio::net::TcpListener; use worker_runtime::auth::{RuntimeIdentityMaterial, decode_public_key}; use yoi_workspace_server::hosts::{RemoteRuntimeAuthConfig, RemoteRuntimeConfig}; -use yoi_workspace_server::store::{SqliteWorkspaceStore, TrustedRuntimeRecord}; +use yoi_workspace_server::store::{SqliteWorkspaceStore, WorkspaceRuntimeBinding}; use yoi_workspace_server::{ - BackendRuntimesConfigFile, ControlPlaneStore, ResolvedWorkspaceBackendConfig, ServerConfig, - ServerHostConfigFile, WorkspaceIdentity, WorkspaceRecord, serve_workspace_catalog, + ControlPlaneStore, ResolvedWorkspaceBackendConfig, ServerConfig, ServerHostConfigFile, + WorkspaceIdentity, WorkspaceRecord, serve_workspace_catalog, }; #[derive(Debug)] @@ -315,40 +315,48 @@ fn run_trust_runtime_command(args: Vec) -> Result<(), Box + "created", + yoi_workspace_server::store::WorkspaceRuntimeBindingUpsert::Unchanged => + "unchanged", + yoi_workspace_server::store::WorkspaceRuntimeBindingUpsert::Replaced => + "replaced", + } + ); println!("server_db={}", database_path.display()); Ok(()) } "list" => { + let mut workspace_id = None; let mut json = false; let mut include_revoked = false; while let Some(arg) = args.pop_front() { let (flag, inline_value) = split_flag_value(arg)?; match flag.as_str() { + "--workspace-id" => { + workspace_id = Some(take_value(&flag, inline_value, &mut args)?) + } "--json" => { ensure_no_inline_value(&flag, inline_value.as_deref())?; json = true; @@ -364,17 +372,20 @@ fn run_trust_runtime_command(args: Vec) -> Result<(), Box) -> Result<(), Box { + let mut workspace_id = None; let mut runtime_id = None; while let Some(arg) = args.pop_front() { let (flag, inline_value) = split_flag_value(arg)?; match flag.as_str() { + "--workspace-id" => { + workspace_id = Some(take_value(&flag, inline_value, &mut args)?) + } "--runtime-id" => { runtime_id = Some(take_value(&flag, inline_value, &mut args)?) } @@ -396,11 +411,14 @@ fn run_trust_runtime_command(args: Vec) -> Result<(), Box) -> Result<(), Box Result<(), Box> { - if store - .list_trusted_runtimes(true)? - .iter() - .any(|runtime| runtime.runtime_id == runtime_id) - && !replace - { - return Err(Box::new(CliError(format!( - "trusted runtime `{runtime_id}` already exists; pass --replace to update it" - )))); - } - Ok(()) -} - fn split_flag_value(arg: String) -> Result<(String, Option), CliError> { if let Some((flag, value)) = arg.split_once('=') { if flag.is_empty() { @@ -543,16 +543,11 @@ async fn run_serve(options: ServeOptions) -> Result<(), Box ServerHostConfigFile::load_from_path(path)?, None => ServerHostConfigFile::load_default()?, }; - let runtime_config = BackendRuntimesConfigFile::load_default()?; - let mut resolved = ResolvedWorkspaceBackendConfig::local_dev( - &workspace_root, - identity, - &host_config, - &runtime_config, - )?; + let mut resolved = + ResolvedWorkspaceBackendConfig::local_dev(&workspace_root, identity, &host_config)?; resolved.database_path = database_path.clone(); resolved.server.database_path = database_path.clone(); - append_trusted_runtime_sources(store.as_ref(), &mut resolved.server.remote_runtime_sources)?; + append_workspace_runtime_sources(store.as_ref(), &mut resolved.server.remote_runtime_sources)?; if let Some(listen) = options.listen { resolved = resolved.with_listen(listen); } @@ -572,22 +567,38 @@ async fn run_serve(options: ServeOptions) -> Result<(), Box, ) -> Result<(), Box> { + let workspaces = store.list_workspaces()?; + let bindings = workspaces + .iter() + .map(|workspace| { + store + .list_workspace_runtime_bindings(&workspace.workspace_id, false) + .map(|bindings| { + bindings + .into_iter() + .filter(|binding| { + binding.runtime_id != yoi_workspace_server::hosts::EMBEDDED_RUNTIME_ID + }) + .collect::>() + }) + }) + .collect::, _>>()? + .into_iter() + .flatten() + .collect::>(); let Some(server_identity) = read_server_identity_file(&server_identity_path())? else { - if !store.list_trusted_runtimes(false)?.is_empty() { + if !bindings.is_empty() { return Err(Box::new(CliError( - "trusted runtimes are registered but server identity is not initialized; run `yoi-server identity init`".to_string(), + "Runtime bindings are registered but server identity is not initialized; run `yoi-server identity init`".to_string(), ))); } return Ok(()); }; - for runtime in store.list_trusted_runtimes(false)? { - let Some(workspace_id) = runtime.workspace_id.clone() else { - continue; - }; + for runtime in bindings { let auth = RemoteRuntimeAuthConfig { server_id: server_identity.identity.identity_id.clone(), server_private_key: server_identity.identity.private_key.clone(), @@ -598,9 +609,12 @@ fn append_trusted_runtime_sources( runtime.base_url, None, ) - .with_workspace_id(workspace_id) + .with_workspace_id(runtime.workspace_id.clone()) .with_auth(auth); - remote_runtime_sources.retain(|existing| existing.runtime_id != runtime.runtime_id); + remote_runtime_sources.retain(|existing| { + existing.workspace_id.as_deref() != Some(runtime.workspace_id.as_str()) + || existing.runtime_id != runtime.runtime_id + }); remote_runtime_sources.push(remote); } Ok(()) @@ -731,7 +745,7 @@ fn parse_listen(value: &str) -> Result { fn print_help() { println!( - "yoi-server\n\nUsage:\n yoi-server identity init --server-id [--replace]\n yoi-server identity show [--json]\n yoi-server trust-runtime add --runtime-id --workspace-id --base-url --public-key [--display-name ] [--replace]\n yoi-server trust-runtime list [--json] [--include-revoked]\n yoi-server trust-runtime revoke --runtime-id \n yoi-server skills [OPTIONS]\n yoi-server serve [OPTIONS]\n\nOptions:\n -h, --help Print help" + "yoi-server\n\nUsage:\n yoi-server identity init --server-id [--replace]\n yoi-server identity show [--json]\n yoi-server trust-runtime add --runtime-id --workspace-id --base-url --public-key [--display-name ] [--replace]\n yoi-server trust-runtime list --workspace-id [--json] [--include-revoked]\n yoi-server trust-runtime revoke --workspace-id --runtime-id \n yoi-server skills [OPTIONS]\n yoi-server serve [OPTIONS]\n\nOptions:\n -h, --help Print help" ); } @@ -743,7 +757,7 @@ fn print_skills_help() { fn print_serve_help() { println!( - "yoi-server serve\n\nUsage:\n yoi-server serve [OPTIONS]\n\nDescription:\n Serves Workspaces recorded in the Yoi server DB. Host-level deployment settings are loaded from the explicit --config path or the canonical XDG yoi/server.toml path, and runtime sources are loaded from XDG runtimes.toml.\n\nOptions:\n --listen Listen address (default 127.0.0.1:8787)\n --config Host-level Server config path\n -h, --help Print help" + "yoi-server serve\n\nUsage:\n yoi-server serve [OPTIONS]\n\nDescription:\n Serves Workspaces recorded in the Yoi server DB. Host-level deployment settings are loaded from the explicit --config path or the canonical XDG yoi/server.toml path, and Runtime bindings are loaded from the Server DB.\n\nOptions:\n --listen Listen address (default 127.0.0.1:8787)\n --config Host-level Server config path\n -h, --help Print help" ); } @@ -823,30 +837,52 @@ mod tests { } #[test] - fn trusted_runtime_add_requires_replace_for_existing_record() { + fn runtime_binding_requires_explicit_replace_for_changed_authority() { let temp = tempfile::tempdir().unwrap(); - let store = SqliteWorkspaceStore::open(temp.path().join("server.db")).unwrap(); + let path = temp.path().join("server.db"); + let store = SqliteWorkspaceStore::open(&path).unwrap(); + rusqlite::Connection::open(&path) + .unwrap() + .execute_batch( + "INSERT INTO accounts(account_id, kind, handle, display_name, created_at, updated_at) + VALUES ('owner', 'user', 'owner', 'Owner', '1', '1'); + INSERT INTO workspaces(workspace_id, owner_account_id, display_name, state, created_at, updated_at) + VALUES ('workspace-a', 'owner', 'Workspace A', 'active', '1', '1');", + ) + .unwrap(); let public_key = RuntimeIdentityMaterial::generate("runtime-a") .unwrap() .public_key; + let binding = WorkspaceRuntimeBinding { + workspace_id: "workspace-a".to_string(), + runtime_id: "runtime-a".to_string(), + display_name: "Runtime A".to_string(), + base_url: "http://127.0.0.1:18080".to_string(), + public_key, + public_key_fingerprint: String::new(), + binding_revision: 1, + created_at: "2026-07-26T00:00:00Z".to_string(), + updated_at: "2026-07-26T00:00:00Z".to_string(), + revoked_at: None, + }; store - .upsert_trusted_runtime(&TrustedRuntimeRecord { - runtime_id: "runtime-a".to_string(), - workspace_id: None, - display_name: "Runtime A".to_string(), - base_url: "http://127.0.0.1:18080".to_string(), - public_key, - created_at: "2026-07-26T00:00:00Z".to_string(), - updated_at: "2026-07-26T00:00:00Z".to_string(), - revoked_at: None, - }) + .upsert_workspace_runtime_binding(binding.clone(), false) .unwrap(); - - let error = ensure_trusted_runtime_replace_allowed(&store, "runtime-a", false).unwrap_err(); - assert_eq!( - error.to_string(), - "trusted runtime `runtime-a` already exists; pass --replace to update it" + assert!(matches!( + store + .upsert_workspace_runtime_binding(binding.clone(), false) + .unwrap(), + yoi_workspace_server::store::WorkspaceRuntimeBindingUpsert::Unchanged + )); + let mut changed = binding; + changed.base_url = "http://127.0.0.1:18081".to_string(); + assert!( + store + .upsert_workspace_runtime_binding(changed.clone(), false) + .is_err() ); - ensure_trusted_runtime_replace_allowed(&store, "runtime-a", true).unwrap(); + store + .upsert_workspace_runtime_binding(changed, true) + .unwrap(); } } diff --git a/crates/workspace-server/src/retention.rs b/crates/workspace-server/src/retention.rs index 474abeac..19c22ccb 100644 --- a/crates/workspace-server/src/retention.rs +++ b/crates/workspace-server/src/retention.rs @@ -464,6 +464,7 @@ impl SqliteWorkspaceStore { tx.execute("INSERT OR IGNORE INTO worker_tombstones(workspace_id,runtime_id,worker_id,display_name,profile,worker_created_at,removed_at,archive_id,policy_id,policy_revision,operation_id) VALUES(?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11)",params![workspace_id,plan.worker.runtime_id,plan.worker.worker_id,worker.display_name,worker.profile,worker.created_at,now,plan.archive_id,plan.policy_id,plan.policy_revision,operation_id])?; } let deleted=tx.execute("DELETE FROM worker_registry WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND updated_at=?4",params![workspace_id,plan.worker.runtime_id,plan.worker.worker_id,plan.worker_revision])?; + tx.execute("UPDATE worker_create_reservations SET state='removed',updated_at=?4 WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND state='created'",params![workspace_id,plan.worker.runtime_id,plan.worker.worker_id,now])?; if deleted!=1{return Err(StoreError::InvalidInput(format!("stale:{}:removal fence changed",plan.plan_id)));} tx.execute("UPDATE worker_removal_operations SET state='succeeded',failure_category=NULL,updated_at=?1 WHERE operation_id=?2",params![now,operation_id])?; tx.execute("INSERT OR IGNORE INTO worker_retention_audit_events(event_id,operation_id,workspace_id,event_kind,detail,created_at) VALUES(?1,?2,?3,'worker_removed',?4,?5)",params![stable("wre",operation_id),operation_id,workspace_id,format!("runtime_id={} worker_id={} session={} metadata={} diagnostics={}",plan.worker.runtime_id,plan.worker.worker_id,sess(plan.session_disposition),meta(plan.metadata_disposition),diag(plan.diagnostics_disposition)),now])?; @@ -1166,6 +1167,7 @@ mod tests { s.with_conn(|conn| { conn.execute("INSERT INTO typed_tickets(workspace_id,ticket_id,slug,title,status,kind,priority,body,workflow_state,workflow_state_explicit) VALUES('w','ticket-old','ticket-old','Old Ticket','open','task','normal','','planning',1)", [])?; conn.execute("INSERT INTO worker_registry(workspace_id,worker_id,runtime_id,display_name,profile,retention_state,created_at,updated_at) VALUES('w','1','r','old worker','builtin:coder','normal','created','rev1')", [])?; + conn.execute("INSERT INTO worker_create_reservations(workspace_id,allocation_key,worker_id,runtime_id,create_fingerprint,state,created_at,updated_at) VALUES('w','allocation-old',?1,'r','fingerprint','created','created','created')", [worker_id().to_string()])?; conn.execute("INSERT INTO ticket_worker_assignments(workspace_id,ticket_id,assignment_id,runtime_id,worker_id,assigned_by,assigned_at) VALUES('w','ticket-old','assignment-old','r','1','test','t')", [])?; conn.execute("DELETE FROM worker_registry WHERE workspace_id='w' AND runtime_id='r' AND worker_id='1'", [])?; conn.execute("DELETE FROM typed_tickets WHERE workspace_id='w' AND ticket_id='ticket-old'", [])?; @@ -1212,6 +1214,15 @@ mod tests { WorkerRemovalPlanState::Succeeded ); assert!(s.worker_tombstone("w", &p.worker).unwrap().is_some()); + let reservation_state: String = s.with_conn(|conn| { + conn.query_row( + "SELECT state FROM worker_create_reservations WHERE workspace_id='w' AND allocation_key='allocation-old'", + [], + |row| row.get(0), + ) + .map_err(StoreError::from) + }).unwrap(); + assert_eq!(reservation_state, "removed"); assert_eq!( s.commit_worker_removal("w", &p.operation_id, &p.input_fingerprint, &result) .unwrap() diff --git a/crates/workspace-server/src/runtime_subscription_tests.rs b/crates/workspace-server/src/runtime_subscription_tests.rs index a42c0c02..ab5d65ac 100644 --- a/crates/workspace-server/src/runtime_subscription_tests.rs +++ b/crates/workspace-server/src/runtime_subscription_tests.rs @@ -6,7 +6,7 @@ use worker_runtime::catalog::{ }; use worker_runtime::execution::{ WorkerExecutionBackend, WorkerExecutionHandle, WorkerExecutionOperation, WorkerExecutionResult, - WorkerExecutionRunState, WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult, + WorkerExecutionSpawnRequest, WorkerExecutionSpawnResult, }; use worker_runtime::identity::WorkerId; use worker_runtime::profile_archive::{ProfileSourceArchiveRef, ProfileSourceGraphSummary}; @@ -22,7 +22,6 @@ impl WorkerExecutionBackend for TestExecutionBackend { fn spawn_worker(&self, request: WorkerExecutionSpawnRequest) -> WorkerExecutionSpawnResult { WorkerExecutionSpawnResult::connected( WorkerExecutionHandle::new(request.worker_ref, self.backend_id()), - WorkerExecutionRunState::Idle, None, ) } @@ -32,25 +31,20 @@ impl WorkerExecutionBackend for TestExecutionBackend { _handle: &WorkerExecutionHandle, input: worker_runtime::interaction::WorkerInput, ) -> WorkerExecutionResult { - if let Some(submission_id) = input.submission_id { - WorkerExecutionResult::accepted_input_committed( + if let Some(submission_request_id) = input.submission_request_id { + WorkerExecutionResult::accepted_submission( WorkerExecutionOperation::Input, - WorkerExecutionRunState::Busy, - submission_id, + submission_request_id, + uuid::Uuid::now_v7().to_string(), + protocol::SubmissionDisposition::Started, ) } else { - WorkerExecutionResult::accepted( - WorkerExecutionOperation::Input, - WorkerExecutionRunState::Busy, - ) + WorkerExecutionResult::accepted(WorkerExecutionOperation::Input) } } fn stop_worker(&self, _handle: &WorkerExecutionHandle) -> WorkerExecutionResult { - WorkerExecutionResult::accepted( - WorkerExecutionOperation::Stop, - WorkerExecutionRunState::Stopped, - ) + WorkerExecutionResult::accepted(WorkerExecutionOperation::Stop) } } @@ -197,8 +191,8 @@ async fn equal_downstream_selectors_share_one_upstream_subscription() { runtime .observe_worker_event( &worker.worker_ref, - protocol::Event::Status { - status: protocol::WorkerStatus::Running, + protocol::Event::WorkerState { + snapshot: protocol::WorkerStatus::Running.into(), }, ) .unwrap(); @@ -208,7 +202,16 @@ async fn equal_downstream_selectors_share_one_upstream_subscription() { BrokerSubscriptionEvent::Event { payload: SubscriptionEventPayload::WorkerUpserted { ref worker }, .. - } if worker.state == SubscriptionWorkerState::Running + } if worker.state == SubscriptionWorkerState::Idle + && matches!( + worker.worker_state, + Some(protocol::WorkerStateSnapshot { + state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Running + )), + .. + }) + ) )); } let mut late = broker.subscribe("runtime-test", selector.clone()).unwrap(); @@ -218,7 +221,18 @@ async fn equal_downstream_selectors_share_one_upstream_subscription() { assert!(matches!( snapshot, SubscriptionSnapshot::Workers { workers } - if workers.iter().any(|worker| worker.state == SubscriptionWorkerState::Running) + if workers.iter().any(|worker| { + worker.state == SubscriptionWorkerState::Idle + && matches!( + worker.worker_state, + Some(protocol::WorkerStateSnapshot { + state: protocol::WorkerState::Busy( + protocol::WorkerBusyState::Run(protocol::WorkerRunState::Running) + ), + .. + }) + ) + }) )); drop(late); @@ -337,14 +351,24 @@ async fn embedded_runtime_uses_in_process_subscription_source() { runtime .observe_worker_event( &worker.worker_ref, - protocol::Event::Status { - status: protocol::WorkerStatus::Running, + protocol::Event::WorkerState { + snapshot: protocol::WorkerStatus::Running.into(), }, ) .unwrap(); assert!(matches!(next_event(&mut subscription).await, - BrokerSubscriptionEvent::Event { payload: SubscriptionEventPayload::WorkerUpserted { worker }, .. } - if worker.runtime_id.as_deref() == Some("embedded-worker-runtime") && worker.state == SubscriptionWorkerState::Running)); + BrokerSubscriptionEvent::Event { payload: SubscriptionEventPayload::WorkerUpserted { worker }, .. } + if worker.runtime_id.as_deref() == Some("embedded-worker-runtime") + && worker.state == SubscriptionWorkerState::Idle + && matches!( + worker.worker_state, + Some(protocol::WorkerStateSnapshot { + state: protocol::WorkerState::Busy(protocol::WorkerBusyState::Run( + protocol::WorkerRunState::Running + )), + .. + }) + ))); let mut late = broker .subscribe( "embedded-worker-runtime", @@ -357,7 +381,18 @@ async fn embedded_runtime_uses_in_process_subscription_source() { assert!(matches!( snapshot, SubscriptionSnapshot::Workers { workers } - if workers.iter().any(|worker| worker.state == SubscriptionWorkerState::Running) + if workers.iter().any(|worker| { + worker.state == SubscriptionWorkerState::Idle + && matches!( + worker.worker_state, + Some(protocol::WorkerStateSnapshot { + state: protocol::WorkerState::Busy( + protocol::WorkerBusyState::Run(protocol::WorkerRunState::Running) + ), + .. + }) + ) + }) )); runtime diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 3599f26d..04f2f22c 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -1,7 +1,7 @@ use std::collections::{BTreeMap, HashMap, HashSet}; use std::path::{Component, Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::{Arc, Mutex, Weak}; +use std::sync::{Arc, Mutex, RwLock, Weak}; use axum::body::Bytes; use axum::extract::ws::{Message as WsMessage, WebSocket, WebSocketUpgrade}; @@ -48,11 +48,14 @@ use workdir::http::{ }; use workdir::workspace::{ MaterializerKind, WorkingDirectoryCleanupTarget, WorkingDirectoryOccupancy, - WorkingDirectoryStatusKind, WorkingDirectorySummary, WorkspaceWorkdirSessionFence, - WorkspaceWorkdirSessionOperationRequest, + WorkingDirectoryStatusKind, WorkingDirectorySummary, WorkspaceWorkdirSessionOperationRequest, }; use workdir::{CommandHandle, WorkdirSessionHandle}; use worker::feature::builtin::{WorkerObservationSubject, WorkerObservationSubjectRef}; +use worker_runtime::http_server::{ + RUNTIME_HTTP_PROTOCOL_MAX_VERSION, RUNTIME_HTTP_PROTOCOL_MIN_VERSION, + RUNTIME_HTTP_PROTOCOL_VERSION, +}; use worker_runtime::resource::{BackendResourceError, BackendResourceFetchRequest}; use worker_runtime::worker_backend::{ProfileRuntimeWorkerFactory, WorkerRuntimeExecutionBackend}; use workspace_api::{ @@ -70,23 +73,29 @@ use workspace_api::{ PasskeyLoginCompleteRequest, PasskeyLoginOptionsRequest, PasskeyLoginOptionsResponse, PasskeyRegistrationCompleteRequest, PasskeyRegistrationOptionsRequest, PasskeyRegistrationOptionsResponse, ProfileSettingsResponse, PutRepositorySshHostTrustRequest, - RepositoryAccessProjection, RepositoryDetailResponse, RepositoryListResponse, - RepositoryLogResponse, RepositorySshCredential, RepositorySshHostTrust, RequestActor, - RotateRepositorySshCredentialRequest, RuntimeConnectionTestResponse, RuntimeManagementSummary, - TICKET_ORCHESTRATION_PLANS_QUERY_PATH, TICKET_RELATIONS_QUERY_PATH, - UpdateWorkspaceMetadataRequest, WhoamiResponse, WorkerLaunchOptionsResponse, - WorkerLaunchProfileCandidate, WorkerLaunchRuntimeOption, WorkerLaunchWorkerSummary, + PutRuntimeTrustKeyRequest, RepositoryAccessProjection, RepositoryDetailResponse, + RepositoryListResponse, RepositoryLogResponse, RepositorySshCredential, RepositorySshHostTrust, + RequestActor, RevokeRuntimeTrustKeyRequest, RotateRepositorySshCredentialRequest, + RuntimeConnectionTestFailureKind, RuntimeConnectionTestResponse, RuntimeConnectionTestStatus, + RuntimeManagementSummary, RuntimeTrustAuditAction, RuntimeTrustAuditEntry, + RuntimeTrustConflictKind, RuntimeTrustConflictResponse, RuntimeTrustKeyRevealResponse, + RuntimeTrustKeyState, RuntimeTrustKeyStatus, TICKET_ORCHESTRATION_PLANS_QUERY_PATH, + TICKET_RELATIONS_QUERY_PATH, UpdateWorkspaceMetadataRequest, WhoamiResponse, + WorkerLaunchOptionsResponse, WorkerLaunchProfileCandidate, WorkerLaunchRuntimeOption, + WorkerLaunchWorkerSummary, WorkingDirectoryCreateRequest as BrowserWorkingDirectoryCreateRequest, WorkingDirectoryCreateResponse as BrowserWorkingDirectoryCreateResponse, WorkingDirectoryDetailResponse as BrowserWorkingDirectoryDetailResponse, WorkingDirectoryListResponse as BrowserWorkingDirectoryListResponse, WorkingDirectoryRemovalDisposition, WorkingDirectoryRemovalRequest, WorkingDirectoryRemovalResponse, WorkingDirectoryRepositoryOption, - WorkspaceCatalogListResponse, WorkspaceCreateResponse, WorkspaceExtensionPointState, - WorkspaceExtensionPoints, WorkspaceMetadataMutationResponse, WorkspaceMetadataSettingsResponse, - WorkspacePermissionSummary, WorkspaceRepositoryRecord, WorkspaceResponse, - WorkspaceRuntimeResource, WorkspaceSummary, WorkspaceWorkerDiscoveryItem, - WorkspaceWorkerDiscoveryPage, WorkspaceWorkerSubject, + WorkspaceCatalogListResponse, WorkspaceCreateResponse, WorkspaceDeletionBlocker, + WorkspaceDeletionBlockerKind, WorkspaceDeletionOperationResponse, + WorkspaceDeletionPreflightResponse, WorkspaceDeletionRequest, WorkspaceDeletionState, + WorkspaceExtensionPointState, WorkspaceExtensionPoints, WorkspaceMetadataMutationResponse, + WorkspaceMetadataSettingsResponse, WorkspacePermissionSummary, WorkspaceRepositoryRecord, + WorkspaceResponse, WorkspaceRuntimeDetail, WorkspaceRuntimeResource, WorkspaceSummary, + WorkspaceWorkerDiscoveryItem, WorkspaceWorkerDiscoveryPage, WorkspaceWorkerSubject, }; use crate::auth::{ @@ -102,19 +111,18 @@ use crate::companion::{ CompanionCancelRequest, CompanionConsole, CompanionMessageRequest, CompanionMessageResponse, CompanionStatusResponse, CompanionTranscriptProjection, }; -use crate::config::{BackendRuntimesConfigFile, RemoteRuntimeConfigFile, resolve_remote_runtime}; use crate::config_source::ConfigCommitRequest; use crate::hosts::{ ConfigBundleCheckResult, ConfigBundleSyncResult, DiagnosticSeverity, EMBEDDED_RUNTIME_ID, EmbeddedWorkerRuntime, HostSummary, RemoteRuntimeConfig, RemoteWorkerRuntime, - RuntimeDiagnostic, RuntimeRegistry, RuntimeRegistryError, RuntimeRegistryUnregisterResult, - TicketWorkerRole, WorkerCapabilitySummary, WorkerCompletionsRequest, WorkerCompletionsResult, - WorkerControlOperation, WorkerCreateBinding, WorkerImplementationSummary, WorkerInputKind, - WorkerInputRequest, WorkerInputResult, WorkerLifecycleRequest, WorkerLifecycleResult, - WorkerOperationState, WorkerRestoreResult, WorkerSpawnAcceptanceRequirement, WorkerSpawnIntent, - WorkerSpawnRequest, WorkerSpawnResult, WorkerSpawnWorkingDirectoryRequest, WorkerSummary, - WorkerTicketAssignmentRequest, WorkerWorkspaceSummary, worker_spawn_create_fingerprint, - workspace_worker_summary, + RuntimeDiagnostic, RuntimePingFailureKind, RuntimeRegistry, RuntimeRegistryError, + RuntimeRegistryUnregisterResult, TicketWorkerRole, WorkerCapabilitySummary, + WorkerCompletionsRequest, WorkerCompletionsResult, WorkerControlOperation, WorkerCreateBinding, + WorkerImplementationSummary, WorkerInputKind, WorkerInputRequest, WorkerInputResult, + WorkerLifecycleRequest, WorkerLifecycleResult, WorkerOperationState, WorkerRestoreResult, + WorkerSpawnAcceptanceRequirement, WorkerSpawnIntent, WorkerSpawnRequest, WorkerSpawnResult, + WorkerSpawnWorkingDirectoryRequest, WorkerSummary, WorkerTicketAssignmentRequest, + WorkerWorkspaceSummary, worker_spawn_create_fingerprint, workspace_worker_summary, }; use crate::identity::WorkspaceIdentity; use crate::memory_backend::execute_memory_backend_operation_with_authority; @@ -149,13 +157,14 @@ use crate::store::{ RepositoryRecord, TicketAssignmentPrincipal, TicketAssignmentRole, TicketCoderAssignmentRecord, TicketRoleAssignmentRecord, UserRecord, WorkdirCreateOperationRecord, WorkdirRegistryRecord, WorkerControlGrantRecord, WorkerRegistryRecord, WorkerWorkdirLinkRecord, WorkspaceRecord, - WorkspaceResourceKind, + WorkspaceResourceKind, WorkspaceRuntimeBinding, WorkspaceRuntimeBindingAuditRecord, }; use crate::workdir_removal::{ WorkdirRemovalAttemptOwner, WorkdirRemovalDisposition, WorkdirRemovalOperation, WorkdirRemovalOperationState, workdir_removal_intent, }; use crate::workspace_catalog::{WorkspaceCatalogService, WorkspaceCreateRequest}; +use crate::workspace_deletion::WorkspaceDeletionStore; use crate::{Error, Result}; use worker_runtime::catalog::{ ConfigBundleRef, ProfileSelector, RepositoryMaterializationContext, RepositoryRefObservation, @@ -164,11 +173,7 @@ use worker_runtime::catalog::{ WorkingDirectoryRepository, WorkingDirectoryRequest, WorkspaceApiRef, }; use worker_runtime::config_bundle::ConfigBundle; -use worker_runtime::http_server::{ - MAX_WORKER_FILE_UPLOAD_BYTES, RuntimeHttpConfigBundleAvailabilityResponse, - RuntimeHttpConfigBundlesResponse, RuntimeHttpSummaryResponse, RuntimeHttpWorkerResponse, - RuntimeHttpWorkersResponse, -}; +use worker_runtime::http_server::MAX_WORKER_FILE_UPLOAD_BYTES; use worker_runtime::identity::{RuntimeWorkerRef, WorkerId}; const EMBEDDED_WORKER_RUNTIME_ID: &str = "embedded-worker-runtime"; @@ -189,7 +194,6 @@ pub struct ServerConfig { pub repositories: Vec, pub runtime_event_sources: Vec, pub remote_runtime_sources: Vec, - pub runtime_config_path: Option, pub backend_base_url: Option, } @@ -217,7 +221,6 @@ impl ServerConfig { repositories: Vec::new(), runtime_event_sources: Vec::new(), remote_runtime_sources: Vec::new(), - runtime_config_path: BackendRuntimesConfigFile::default_path(), backend_base_url: None, } } @@ -355,7 +358,6 @@ static EMBEDDED_RUNTIME_REQUEST_IDENTITY: std::sync::LazyLock< struct WorkdirCommandSession { source: WorkdirSessionHandle, provider_handle: CommandHandle, - delegations: Vec, } enum RegisteredWorkdirSession { @@ -398,7 +400,6 @@ impl WorkdirSessionRegistry { worker: RuntimeWorkerRef, source: WorkdirSessionHandle, provider_handle: CommandHandle, - delegations: Vec, ) -> CommandHandle { let external_handle = loop { let candidate = CommandHandle(Uuid::now_v7().to_string()); @@ -414,7 +415,6 @@ impl WorkdirSessionRegistry { WorkdirCommandSession { source, provider_handle, - delegations, }, ); external_handle @@ -587,6 +587,7 @@ pub struct WorkspaceApi { prompt_projection_cache: crate::prompt_settings::WorkspacePromptProjectionCache, authority: SqliteWorkspaceAuthority, runtime: Arc, + runtime_binding_expectations: Arc>>, companion: Arc, orchestrator_spawn_lock: Arc>, orchestrator_attention_fingerprint: Arc>>, @@ -746,6 +747,17 @@ impl WorkspaceWorkerRemoveExecutor { )); } + self.execute_target_removal(&runtime, &target, reason, None) + .await + } + + async fn execute_target_removal( + &self, + runtime: &RuntimeRegistry, + target: &RuntimeWorkerRef, + reason: &str, + parent_workspace_deletion_operation_id: Option<&str>, + ) -> std::result::Result { let remove_lock = { let mut locks = self .worker_remove_locks @@ -793,6 +805,14 @@ impl WorkspaceWorkerRemoveExecutor { } else { prepared }; + if let Some(parent_operation_id) = parent_workspace_deletion_operation_id { + self.store + .append_workspace_deletion_child_operation( + parent_operation_id, + &prepared.plan.operation_id, + ) + .map_err(|error| error.to_string())?; + } if close_worker_workdir_sessions(&self.workdir_sessions, &target) .await .is_err() @@ -832,7 +852,7 @@ impl WorkspaceWorkerRemoveExecutor { )); } return self - .resume_worker_retention(&runtime, &target, prepared) + .resume_worker_retention(runtime, &target, prepared) .await; } @@ -889,6 +909,14 @@ impl WorkspaceWorkerRemoveExecutor { Err(error) => return Ok(worker_retention_error_response(error)), }; + if let Some(parent_operation_id) = parent_workspace_deletion_operation_id { + self.store + .append_workspace_deletion_child_operation( + parent_operation_id, + &prepared.plan.operation_id, + ) + .map_err(|error| error.to_string())?; + } if close_worker_workdir_sessions(&self.workdir_sessions, &target) .await .is_err() @@ -998,6 +1026,21 @@ pub struct WorkspaceServerApi { store: Arc, catalog: WorkspaceCatalogService, routers: Arc>>, + apis: Arc>>, + mutation_locks: Arc>>>>, + running_deletions: Arc>>, + hook_handles: Arc>>, +} + +async fn workspace_mutation_lock( + locks: &Arc>>>>, + workspace_id: &str, +) -> Arc> { + let mut locks = locks.lock().await; + locks + .entry(workspace_id.to_string()) + .or_insert_with(|| Arc::new(AsyncMutex::new(()))) + .clone() } impl WorkspaceServerApi { @@ -1007,13 +1050,21 @@ impl WorkspaceServerApi { catalog: WorkspaceCatalogService::new(store.clone()), store, routers: Arc::new(AsyncMutex::new(HashMap::new())), + apis: Arc::new(AsyncMutex::new(HashMap::new())), + mutation_locks: Arc::new(AsyncMutex::new(HashMap::new())), + running_deletions: Arc::new(AsyncMutex::new(HashSet::new())), + hook_handles: Arc::new(AsyncMutex::new(HashMap::new())), } } - async fn router_for_workspace(&self, workspace_id: &str) -> Result> { - let mut routers = self.routers.lock().await; - if let Some(router) = routers.get(workspace_id) { - return Ok(Some(router.clone())); + async fn mutation_lock(&self, workspace_id: &str) -> Arc> { + workspace_mutation_lock(&self.mutation_locks, workspace_id).await + } + + async fn api_for_workspace(&self, workspace_id: &str) -> Result> { + let mut apis = self.apis.lock().await; + if let Some(api) = apis.get(workspace_id).cloned() { + return Ok(Some(api)); } let Some(workspace) = self.store.get_workspace(workspace_id).await? else { return Ok(None); @@ -1023,7 +1074,252 @@ impl WorkspaceServerApi { .template .for_catalog_workspace(&workspace, repositories)?; let api = WorkspaceApi::new(config, self.store.clone()).await?; - tokio::spawn(run_orchestrator_turn_end_hook(api.clone())); + apis.insert(workspace_id.to_string(), api.clone()); + Ok(Some(api)) + } + + async fn schedule_workspace_deletion(&self, operation_id: String) { + let mut running = self.running_deletions.lock().await; + if !running.insert(operation_id.clone()) { + return; + } + drop(running); + let api = self.clone(); + tokio::spawn(async move { + if api.execute_workspace_deletion(&operation_id).await.is_err() { + let operation = api + .store + .workspace_deletion_operation_for_recovery(&operation_id) + .ok() + .flatten(); + let child_operation_ids = operation + .as_ref() + .map(|operation| operation.child_operation_ids.as_slice()) + .unwrap_or_default(); + let blockers = operation + .as_ref() + .map(|operation| operation.blockers.as_slice()) + .unwrap_or_default(); + let _ = api.store.update_workspace_deletion_operation( + &operation_id, + WorkspaceDeletionState::Failed, + child_operation_ids, + blockers, + Some("workspace_deletion_execution_failed"), + ); + } + api.running_deletions.lock().await.remove(&operation_id); + }); + } + + async fn recover_workspace_deletions(&self) -> Result<()> { + for operation_id in self.store.resumable_workspace_deletion_operation_ids()? { + self.schedule_workspace_deletion(operation_id).await; + } + Ok(()) + } + + async fn workspace_deletion_preflight( + &self, + actor_account_id: &str, + workspace_id: &str, + ) -> Result { + let mut preflight = self + .store + .workspace_deletion_preflight(actor_account_id, workspace_id)?; + let Some(api) = self.api_for_workspace(workspace_id).await? else { + return Err(Error::InvalidInput("Workspace does not exist".to_string())); + }; + for registry_worker in self + .store + .list_worker_registry(workspace_id, i64::MAX as usize)? + { + let worker_key = registry_worker.display_name; + match api.runtime.worker(®istry_worker.worker) { + Ok(worker) if worker.state == "stopped" && worker.singleton_key.is_none() => {} + Ok(worker) => preflight.blockers.push(WorkspaceDeletionBlocker { + kind: if worker.singleton_key.is_some() { + WorkspaceDeletionBlockerKind::RetentionHold + } else { + WorkspaceDeletionBlockerKind::WorkerRemovalBlocked + }, + resource_kind: Some("worker".to_string()), + resource_key: Some(worker_key), + message: if worker.singleton_key.is_some() { + "Internal or singleton Workers must be released by their owning service first." + .to_string() + } else { + "Stop running, restoring, or otherwise active Workers before deleting the Workspace." + .to_string() + }, + }), + Err(_) => preflight.blockers.push(WorkspaceDeletionBlocker { + kind: WorkspaceDeletionBlockerKind::CleanupUnavailable, + resource_kind: Some("worker".to_string()), + resource_key: Some(worker_key), + message: "Worker state is unavailable; retry after Runtime state is healthy." + .to_string(), + }), + } + } + crate::workspace_deletion::bound_workspace_deletion_blockers(&mut preflight.blockers); + preflight.can_delete = preflight.blockers.is_empty(); + Ok(preflight) + } + + async fn execute_workspace_deletion( + &self, + operation_id: &str, + ) -> Result { + let prior = self + .store + .workspace_deletion_operation_for_recovery(operation_id)? + .ok_or_else(|| Error::InvalidInput("Workspace deletion operation".to_string()))?; + let operation = self.store.update_workspace_deletion_operation( + operation_id, + WorkspaceDeletionState::Running, + &prior.child_operation_ids, + &[], + None, + )?; + let api = self + .api_for_workspace(&operation.workspace_id) + .await? + .ok_or_else(|| Error::InvalidInput("Workspace no longer exists".to_string()))?; + + let mut child_operation_ids = operation.child_operation_ids.clone(); + let mut blockers = Vec::new(); + for worker in self + .store + .list_worker_registry(&operation.workspace_id, i64::MAX as usize)? + { + let worker_key = worker.display_name.clone(); + let target = worker.worker; + let response = WorkspaceWorkerRemoveExecutor::new(&api) + .execute_target_removal( + api.runtime.as_ref(), + &target, + "Workspace deletion", + Some(operation_id), + ) + .await + .map_err(Error::Store)?; + child_operation_ids = self + .store + .workspace_deletion_operation_for_recovery(operation_id)? + .ok_or_else(|| { + Error::Store("Workspace deletion operation disappeared".to_string()) + })? + .child_operation_ids; + if response.status != 200 { + blockers.push(WorkspaceDeletionBlocker { + kind: WorkspaceDeletionBlockerKind::WorkerRemovalBlocked, + resource_kind: Some("worker".to_string()), + resource_key: Some(worker_key.clone()), + message: "Worker retention or removal policy blocked deletion.".to_string(), + }); + } + } + + if blockers.is_empty() { + for workdir in self + .store + .list_workdir_registry(&operation.workspace_id, i64::MAX as usize)? + { + match execute_workdir_removal_for_workspace_deletion( + &api, + &workdir.workdir_id, + operation_id, + ) { + Ok(child) => { + child_operation_ids = self + .store + .append_workspace_deletion_child_operation( + operation_id, + &child.operation_id, + )? + .child_operation_ids; + if child.state != WorkdirRemovalOperationState::Completed + || child.disposition != Some(WorkdirRemovalDisposition::Removed) + { + let dirty = + child.failure_category.as_deref() == Some("dirty_or_unknown"); + blockers.push(WorkspaceDeletionBlocker { + kind: if dirty { + WorkspaceDeletionBlockerKind::DirtyWorkdir + } else { + WorkspaceDeletionBlockerKind::WorkdirRemovalBlocked + }, + resource_kind: Some("workdir".to_string()), + resource_key: Some(workdir.workdir_id), + message: if dirty { + "Workdir is dirty or its cleanliness is unknown. Clean it and refresh status before retrying deletion." + .to_string() + } else { + "Workdir removal did not complete; retry the Workspace deletion operation." + .to_string() + }, + }); + } + } + Err(error) => blockers.push(WorkspaceDeletionBlocker { + kind: WorkspaceDeletionBlockerKind::WorkdirRemovalBlocked, + resource_kind: Some("workdir".to_string()), + resource_key: Some(workdir.workdir_id), + message: format!("Workdir removal failed: {error}"), + }), + } + } + } + + if !blockers.is_empty() { + return self.store.update_workspace_deletion_operation( + operation_id, + WorkspaceDeletionState::Blocked, + &child_operation_ids, + &blockers, + None, + ); + } + let completed = self.store.finalize_workspace_deletion(operation_id)?; + self.routers.lock().await.remove(&completed.workspace_id); + if let Some(handle) = self + .hook_handles + .lock() + .await + .remove(&completed.workspace_id) + { + handle.abort(); + } + self.apis.lock().await.remove(&completed.workspace_id); + self.mutation_locks + .lock() + .await + .remove(&completed.workspace_id); + Ok(completed) + } + + async fn router_for_workspace(&self, workspace_id: &str) -> Result> { + if let Some(router) = self.routers.lock().await.get(workspace_id).cloned() { + return Ok(Some(router)); + } + let Some(api) = self.api_for_workspace(workspace_id).await? else { + return Ok(None); + }; + let mut routers = self.routers.lock().await; + if let Some(router) = routers.get(workspace_id).cloned() { + return Ok(Some(router)); + } + let Some(workspace) = self.store.get_workspace(workspace_id).await? else { + return Ok(None); + }; + if workspace.state == "active" { + let hook = tokio::spawn(run_orchestrator_turn_end_hook(api.clone())); + self.hook_handles + .lock() + .await + .insert(workspace_id.to_string(), hook.abort_handle()); + } let router = build_inner_router(api); routers.insert(workspace_id.to_string(), router.clone()); Ok(Some(router)) @@ -1120,6 +1416,106 @@ async fn create_server_workspace( (status, Json(workspace_create_response(created))).into_response() } +async fn preflight_server_workspace_deletion( + State(api): State, + AxumPath(workspace_id): AxumPath, + headers: HeaderMap, +) -> Response { + let actor_account_id = match resolve_server_actor(&api, &headers).await { + Ok(Some(actor)) => actor.account_id, + Ok(None) => return forbidden_server_response("Workspace deletion requires its owner"), + Err(error) => return server_error_response(error), + }; + match api + .workspace_deletion_preflight(&actor_account_id, &workspace_id) + .await + { + Ok(preflight) => Json(preflight).into_response(), + Err(error) => server_error_response(error), + } +} + +async fn start_server_workspace_deletion( + State(api): State, + AxumPath(workspace_id): AxumPath, + headers: HeaderMap, + Json(request): Json, +) -> Response { + let actor_account_id = match resolve_server_actor(&api, &headers).await { + Ok(Some(actor)) => actor.account_id, + Ok(None) => return forbidden_server_response("Workspace deletion requires its owner"), + Err(error) => return server_error_response(error), + }; + let mutation_lock = api.mutation_lock(&workspace_id).await; + let _mutation_guard = mutation_lock.lock().await; + let existing = match api + .store + .workspace_deletion_operation(&actor_account_id, &request.operation_id) + { + Ok(existing) => existing, + Err(error) => return server_error_response(error), + }; + if existing.is_none() { + let preflight = match api + .workspace_deletion_preflight(&actor_account_id, &workspace_id) + .await + { + Ok(preflight) => preflight, + Err(error) => return server_error_response(error), + }; + if !preflight.can_delete { + return (StatusCode::CONFLICT, Json(preflight)).into_response(); + } + } + let reservation = + match api + .store + .reserve_workspace_deletion(&actor_account_id, &workspace_id, &request) + { + Ok(reservation) => reservation, + Err(error) => return server_error_response(error), + }; + if let Some(handle) = api.hook_handles.lock().await.remove(&workspace_id) { + handle.abort(); + } + let operation = reservation.operation; + if operation.state != WorkspaceDeletionState::Succeeded { + api.schedule_workspace_deletion(request.operation_id).await; + } + let status = if operation.state == WorkspaceDeletionState::Succeeded { + StatusCode::OK + } else { + StatusCode::ACCEPTED + }; + (status, Json(operation)).into_response() +} + +async fn get_server_workspace_deletion( + State(api): State, + AxumPath(operation_id): AxumPath, + headers: HeaderMap, +) -> Response { + let actor_account_id = match resolve_server_actor(&api, &headers).await { + Ok(Some(actor)) => actor.account_id, + Ok(None) => { + return forbidden_server_response("Workspace deletion status requires its owner"); + } + Err(error) => return server_error_response(error), + }; + match api + .store + .workspace_deletion_operation(&actor_account_id, &operation_id) + { + Ok(Some(operation)) => Json(operation).into_response(), + Ok(None) => ( + StatusCode::NOT_FOUND, + "Workspace deletion operation not found", + ) + .into_response(), + Err(error) => server_error_response(error), + } +} + fn workspace_summary(record: WorkspaceRecord) -> WorkspaceSummary { WorkspaceSummary { workspace_id: record.workspace_id, @@ -1219,6 +1615,22 @@ async fn authorize_scoped_workspace_request( .await .map_err(|_| StatusCode::UNAUTHORIZED.into_response())?; request.extensions_mut().insert(source); + if !matches!( + *request.method(), + Method::GET | Method::HEAD | Method::OPTIONS + ) && !api + .store + .get_workspace(workspace_id) + .await + .map_err(server_error_response)? + .is_some_and(|workspace| workspace.state == "active") + { + return Err(( + StatusCode::CONFLICT, + "Workspace is deleting and no longer accepts mutations", + ) + .into_response()); + } return Ok(()); } @@ -1245,6 +1657,20 @@ async fn authorize_scoped_workspace_request( } } request.extensions_mut().insert(actor); + if mutating + && !api + .store + .get_workspace(workspace_id) + .await + .map_err(server_error_response)? + .is_some_and(|workspace| workspace.state == "active") + { + return Err(( + StatusCode::CONFLICT, + "Workspace is deleting and no longer accepts mutations", + ) + .into_response()); + } Ok(()) } @@ -1299,6 +1725,23 @@ async fn authorize_workspace_api_request( return StatusCode::UNAUTHORIZED.into_response(); }; request.extensions_mut().insert(source); + if !matches!( + *request.method(), + Method::GET | Method::HEAD | Method::OPTIONS + ) && !api + .store + .get_workspace(&workspace_id) + .await + .ok() + .flatten() + .is_some_and(|workspace| workspace.state == "active") + { + return ( + StatusCode::CONFLICT, + "Workspace is deleting and no longer accepts mutations", + ) + .into_response(); + } return next.run(request).await; } @@ -1332,6 +1775,21 @@ async fn authorize_workspace_api_request( } } request.extensions_mut().insert(actor); + if mutating + && !api + .store + .get_workspace(&workspace_id) + .await + .ok() + .flatten() + .is_some_and(|workspace| workspace.state == "active") + { + return ( + StatusCode::CONFLICT, + "Workspace is deleting and no longer accepts mutations", + ) + .into_response(); + } next.run(request).await } @@ -1374,6 +1832,15 @@ async fn dispatch_workspace_request( ) -> Response { let path = request.uri().path().to_owned(); let workspace_id = scoped_workspace_id(&path); + let _mutation_guard = if let Some(workspace_id) = workspace_id + && !matches!( + *request.method(), + Method::GET | Method::HEAD | Method::OPTIONS + ) { + Some(api.mutation_lock(workspace_id).await.lock_owned().await) + } else { + None + }; if let Some(workspace_id) = workspace_id && (path.starts_with("/api/w/") || path.starts_with("/api/runtime/v1/workspaces/")) && let Err(response) = @@ -1479,11 +1946,20 @@ pub async fn build_workspace_server_router( }); let api = WorkspaceServerApi::new(template, store); api.preload().await?; + api.recover_workspace_deletions().await?; let catalog = Router::new() .route( "/api/workspaces", get(list_server_workspaces).post(create_server_workspace), ) + .route( + "/api/workspaces/{workspace_id}/deletion", + get(preflight_server_workspace_deletion).post(start_server_workspace_deletion), + ) + .route( + "/api/workspace-deletions/{operation_id}", + get(get_server_workspace_deletion), + ) .fallback(dispatch_workspace_request) .with_state(api.clone()); Ok(auth @@ -1561,18 +2037,29 @@ impl WorkspaceApi { pub async fn new(config: ServerConfig, store: Arc) -> Result { let resource_broker = BackendResourceBroker::default(); + if store.get_workspace(&config.workspace_id).await?.is_none() { + return Err(crate::Error::Config(format!( + "Workspace {} is not registered in the Server DB", + config.workspace_id + ))); + } let embedded_identity = (*EMBEDDED_RUNTIME_REQUEST_IDENTITY).clone(); store - .upsert_trusted_runtime_record(&crate::store::TrustedRuntimeRecord { - runtime_id: EMBEDDED_RUNTIME_ID.to_owned(), - workspace_id: None, - display_name: "Embedded Runtime".to_owned(), - base_url: "in-process://embedded".to_owned(), - public_key: embedded_identity.public_key.clone(), - created_at: config.workspace_created_at.clone(), - updated_at: config.workspace_created_at.clone(), - revoked_at: None, - }) + .upsert_workspace_runtime_binding_record( + crate::store::WorkspaceRuntimeBinding { + workspace_id: config.workspace_id.clone(), + runtime_id: EMBEDDED_RUNTIME_ID.to_owned(), + display_name: "Embedded Runtime".to_owned(), + base_url: "in-process://embedded".to_owned(), + public_key: embedded_identity.public_key.clone(), + public_key_fingerprint: String::new(), + binding_revision: 1, + created_at: config.workspace_created_at.clone(), + updated_at: config.workspace_created_at.clone(), + revoked_at: None, + }, + true, + ) .await?; let embedded_audience = format!("embedded:{}", config.workspace_id); let worker_remove_dispatcher = Arc::new( @@ -1597,14 +2084,68 @@ impl WorkspaceApi { "failed to initialize embedded Worker backend: {err}" )) })?; - Self::new_with_execution_backend_and_broker( + let runtime_binding_store = store.clone(); + let configured_runtime_endpoints = config + .remote_runtime_sources + .iter() + .filter_map(|source| { + (source.workspace_id.as_deref() == Some(config.workspace_id.as_str())) + .then(|| (source.runtime_id.clone(), source.base_url.clone())) + }) + .collect::>(); + let expected_runtime_bindings = store + .list_workspace_runtime_bindings(&config.workspace_id, false) + .await? + .into_iter() + .filter(|binding| binding.runtime_id != EMBEDDED_RUNTIME_ID) + .filter(|binding| { + configured_runtime_endpoints.get(&binding.runtime_id) == Some(&binding.base_url) + }) + .map(|binding| { + ( + (binding.workspace_id.clone(), binding.runtime_id.clone()), + binding, + ) + }) + .collect::>(); + let workspace_id = config.workspace_id.clone(); + let api = Self::new_with_execution_backend_and_broker( config, store, Arc::new(execution_backend), resource_broker, Some(worker_remove_dispatcher), ) - .await + .await?; + *api.runtime_binding_expectations + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) = expected_runtime_bindings; + let runtime_binding_expectations = Arc::clone(&api.runtime_binding_expectations); + api.runtime.set_runtime_binding_gate(move |runtime_id| { + runtime_binding_expectations + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .get(&(workspace_id.clone(), runtime_id.to_string())) + .is_some_and(|expected| { + runtime_binding_store + .workspace_runtime_binding_matches(expected) + .unwrap_or(false) + }) + }); + let active_expectations = api + .runtime_binding_expectations + .read() + .unwrap_or_else(std::sync::PoisonError::into_inner); + for source in &api.config.remote_runtime_sources { + if !active_expectations + .contains_key(&(api.config.workspace_id.clone(), source.runtime_id.clone())) + { + api.runtime_subscription_broker + .unregister_runtime(&source.runtime_id); + } + } + drop(active_expectations); + Ok(api) } #[cfg(test)] @@ -1709,6 +2250,7 @@ impl WorkspaceApi { config, store, runtime, + runtime_binding_expectations: Arc::new(RwLock::new(HashMap::new())), companion, orchestrator_spawn_lock: Arc::new(std::sync::Mutex::new(())), orchestrator_attention_fingerprint: Arc::new(Mutex::new(None)), @@ -2586,10 +3128,6 @@ fn build_inner_router(api: WorkspaceApi) -> Router { post(scoped_attach_current_worker_workdir) .delete(scoped_detach_current_worker_workdir), ) - .route( - "/api/w/{workspace_id}/workers/self/workdir-session/fence", - get(scoped_current_worker_workdir_session_fence), - ) .route( "/api/w/{workspace_id}/workers/self/workdir-session/operations", post(scoped_execute_current_worker_workdir_operation), @@ -2609,7 +3147,13 @@ fn build_inner_router(api: WorkspaceApi) -> Router { ) .route( "/api/w/{workspace_id}/runtimes/{runtime_id}", - delete(scoped_delete_remote_runtime), + get(scoped_get_runtime_detail).delete(scoped_delete_remote_runtime), + ) + .route( + "/api/w/{workspace_id}/runtimes/{runtime_id}/trust-key", + get(scoped_reveal_runtime_trust_key) + .put(scoped_put_runtime_trust_key) + .delete(scoped_revoke_runtime_trust_key), ) .route( "/api/w/{workspace_id}/runtimes/{runtime_id}/connection-tests", @@ -7341,46 +7885,11 @@ async fn scoped_detach_current_worker_workdir( })) } -async fn scoped_current_worker_workdir_session_fence( - State(api): State, - AxumPath(path): AxumPath, - headers: HeaderMap, -) -> ApiResult> { - validate_workspace_scope(&api, &path.workspace_id)?; - let worker = current_worker_identity(&api, &path.workspace_id, &headers)?; - let session_lock = current_worker_session_lock(&api, &worker); - let _session_guard = session_lock.lock().await; - let link = current_worker_active_attachment(&api, &worker)?; - Ok(Json(WorkspaceWorkdirSessionFence { - value: current_worker_workdir_session_fence(&link), - })) -} - -fn current_worker_workdir_session_fence(link: &WorkerWorkdirLinkRecord) -> String { - format!("v1:{}\0{}", link.workdir_id, link.linked_at) -} - -fn validate_current_worker_workdir_session_fence( - link: &WorkerWorkdirLinkRecord, - expected: Option<&str>, -) -> Result<()> { - if expected.is_some_and(|expected| expected != current_worker_workdir_session_fence(link)) { - Err(Error::WorkdirAttachmentConflict( - "delegated Workdir session attachment changed".to_string(), - )) - } else { - Ok(()) - } -} - fn validated_current_worker_attachment( api: &WorkspaceApi, worker: &RuntimeWorkerRef, - expected_session_fence: Option<&str>, ) -> ApiResult { - let link = current_worker_active_attachment(api, worker)?; - validate_current_worker_workdir_session_fence(&link, expected_session_fence)?; - Ok(link) + current_worker_active_attachment(api, worker) } #[derive(Debug)] @@ -7435,23 +7944,13 @@ async fn scoped_execute_current_worker_workdir_operation( ) -> std::result::Result, WorkdirOperationApiError> { validate_workspace_scope(&api, &path.workspace_id)?; let worker = current_worker_identity(&api, &path.workspace_id, &headers)?; - let expected_session_fence = request.expected_session_fence; - let delegations = request.delegations; let result = match request.operation { WorkdirSessionOperation::CommandStart(command) => { let session_lock = current_worker_session_lock(&api, &worker); let _session_guard = session_lock.lock().await; - let link = validated_current_worker_attachment( - &api, - &worker, - expected_session_fence.as_deref(), - )?; + let link = validated_current_worker_attachment(&api, &worker)?; let source = open_current_worker_workdir_session_locked(&api, &worker, &link).await?; - let applied = - apply_current_worker_delegations(&worker, source.clone(), delegations.clone()) - .await?; - let provider_handle = applied - .scoped_session + let provider_handle = source .start_command(command) .await .map_err(|error| current_worker_workdir_operation_error(&worker, error))?; @@ -7470,58 +7969,32 @@ async fn scoped_execute_current_worker_workdir_operation( .workdir_sessions .lock() .expect("Workdir session registry lock poisoned") - .register_command( - worker.clone(), - registered_source, - provider_handle, - delegations, - ); + .register_command(worker.clone(), registered_source, provider_handle); WorkdirSessionOperationResult::CommandStart(external_handle) } WorkdirSessionOperation::CommandStatus(external_handle) => { - let (session, provider_handle) = current_worker_command_session( - &api, - &worker, - &external_handle, - &delegations, - expected_session_fence.as_deref(), - ) - .await?; + let (session, provider_handle) = + current_worker_command_session(&api, &worker, &external_handle)?; session - .scoped_session .command_status(provider_handle) .await .map(WorkdirSessionOperationResult::CommandStatus) .map_err(|error| current_worker_workdir_operation_error(&worker, error))? } WorkdirSessionOperation::CommandOutput(mut output) => { - let (session, provider_handle) = current_worker_command_session( - &api, - &worker, - &output.handle, - &delegations, - expected_session_fence.as_deref(), - ) - .await?; + let (session, provider_handle) = + current_worker_command_session(&api, &worker, &output.handle)?; output.handle = provider_handle; session - .scoped_session .command_output(output) .await .map(WorkdirSessionOperationResult::CommandOutput) .map_err(|error| current_worker_workdir_operation_error(&worker, error))? } WorkdirSessionOperation::CommandCancel(external_handle) => { - let (session, provider_handle) = current_worker_command_session( - &api, - &worker, - &external_handle, - &delegations, - expected_session_fence.as_deref(), - ) - .await?; + let (session, provider_handle) = + current_worker_command_session(&api, &worker, &external_handle)?; session - .scoped_session .cancel_command(provider_handle) .await .map(|()| WorkdirSessionOperationResult::CommandCancel) @@ -7536,14 +8009,9 @@ async fn scoped_execute_current_worker_workdir_operation( | WorkdirSessionOperation::Grep(_)) => { let session_lock = current_worker_session_lock(&api, &worker); let _session_guard = session_lock.lock().await; - let link = validated_current_worker_attachment( - &api, - &worker, - expected_session_fence.as_deref(), - )?; + let link = validated_current_worker_attachment(&api, &worker)?; let source = open_current_worker_workdir_session_locked(&api, &worker, &link).await?; - let applied = apply_current_worker_delegations(&worker, source, delegations).await?; - execute_workdir_session_operation(&applied.scoped_session, operation) + execute_workdir_session_operation(&source, operation) .await .map_err(|error| current_worker_workdir_operation_error(&worker, error))? } @@ -7551,29 +8019,12 @@ async fn scoped_execute_current_worker_workdir_operation( Ok(Json(result)) } -async fn apply_current_worker_delegations( - worker: &RuntimeWorkerRef, - source: WorkdirSessionHandle, - delegations: Vec, -) -> Result { - workdir::apply_delegation_chain(source, delegations) - .await - .map_err(|error| Error::RuntimeOperationFailed { - runtime_id: worker.runtime_id.clone(), - code: "workdir_session_delegation_failed".to_string(), - message: error.to_string(), - }) -} - -async fn current_worker_command_session( +fn current_worker_command_session( api: &WorkspaceApi, worker: &RuntimeWorkerRef, external_handle: &CommandHandle, - delegations: &[workdir::WorkdirDelegationRequest], - expected_session_fence: Option<&str>, -) -> std::result::Result<(workdir::AppliedWorkdirDelegation, CommandHandle), WorkdirOperationApiError> -{ - let _link = validated_current_worker_attachment(api, worker, expected_session_fence)?; +) -> std::result::Result<(WorkdirSessionHandle, CommandHandle), WorkdirOperationApiError> { + let _link = validated_current_worker_attachment(api, worker)?; let command = api .workdir_sessions .lock() @@ -7585,15 +8036,7 @@ async fn current_worker_command_session( workdir::WorkdirError::UnknownCommand(external_handle.0.clone()), )) })?; - if command.delegations != delegations { - return Err(Error::WorkdirAttachmentConflict( - "command lifecycle delegation differs from CommandStart".to_string(), - ) - .into()); - } - let session = - apply_current_worker_delegations(worker, command.source, command.delegations).await?; - Ok((session, command.provider_handle)) + Ok((command.source, command.provider_handle)) } fn current_worker_workdir_operation_error( @@ -8528,22 +8971,22 @@ async fn scoped_list_runtimes( AxumPath(path): AxumPath, ) -> ApiResult>> { validate_workspace_scope(&api, &path.workspace_id)?; - let runtime_config = load_backend_runtimes_config_for_settings(&api)?; - Ok(Json(workspace_runtime_resources_response( - &api, - &runtime_config, - ))) + Ok(Json( + workspace_runtime_resources_response(&api, &api.config.workspace_id).await?, + )) } async fn scoped_workspace_protocol_ws( State(api): State, + Extension(actor): Extension, AxumPath(workspace_id): AxumPath, ws: axum::extract::ws::WebSocketUpgrade, ) -> std::result::Result { validate_workspace_scope(&api, &workspace_id).map_err(|error| error.into_response())?; + let input_source = authenticated_browser_input_source(&actor); Ok(ws .on_upgrade(move |socket| { - crate::workspace_subscription::serve_workspace_subscription(api, socket) + crate::workspace_subscription::serve_workspace_subscription(api, socket, input_source) }) .into_response()) } @@ -9249,7 +9692,7 @@ async fn scoped_capture_worker_observation_session( return Err(ApiError::from(Error::UnknownWorker { worker: target })); } - let mut connection = connect_workspace_worker_protocol(&api, &target).await?; + let mut connection = connect_workspace_worker_protocol(&api, &target, None).await?; let event = tokio::time::timeout(std::time::Duration::from_secs(10), connection.events.recv()) .await .map_err(|_| { @@ -10421,6 +10864,41 @@ fn execute_workdir_removal( .map(|operation| workdir_removal_response(&operation)) } +fn execute_workdir_removal_for_workspace_deletion( + api: &WorkspaceApi, + working_directory_id: &str, + parent_operation_id: &str, +) -> Result { + let source_actor = format!("workspace-deletion:{parent_operation_id}"); + let reason = "Workspace deletion"; + let lock = workdir_removal_execution_lock(api, working_directory_id)?; + let _guard = lock + .lock() + .map_err(|_| Error::Store("Workdir removal lock was poisoned".to_string()))?; + let operation = if let Some(existing) = + api.config_store.find_workdir_removal_operation_by_intent( + api.workspace_id(), + working_directory_id, + &source_actor, + reason, + )? { + existing + } else { + let workdir = api + .config_store + .get_workdir_registry(api.workspace_id(), working_directory_id)? + .ok_or_else(|| { + Error::InvalidInput(format!("Unknown Workdir `{working_directory_id}`")) + })?; + let intent = workdir_removal_intent(&workdir, &source_actor, reason)?; + api.config_store + .reserve_workdir_removal_operation(&intent)? + }; + api.config_store + .append_workspace_deletion_child_operation(parent_operation_id, &operation.operation_id)?; + execute_reserved_workdir_removal(api, operation, false) +} + fn recover_workdir_removals(api: &WorkspaceApi) -> Result<()> { for operation in api .config_store @@ -10919,11 +11397,249 @@ async fn scoped_create_remote_runtime( create_remote_runtime(State(api), Json(request)).await } +async fn scoped_get_runtime_detail( + State(api): State, + AxumPath(path): AxumPath, +) -> ApiResult> { + validate_workspace_scope(&api, &path.workspace_id)?; + Ok(Json( + workspace_runtime_detail(&api, &path.workspace_id, &path.runtime_id).await?, + )) +} + +async fn scoped_reveal_runtime_trust_key( + State(api): State, + AxumPath(path): AxumPath, + Extension(actor): Extension, +) -> ApiResult> { + validate_workspace_scope(&api, &path.workspace_id)?; + require_workspace_owner( + &api, + &path.workspace_id, + &actor, + "Runtime public key reveal", + ) + .await?; + if path.runtime_id == EMBEDDED_WORKER_RUNTIME_ID { + return Err(settings_bad_request( + "embedded_runtime_trust_managed_internally", + "the embedded Runtime trust key is managed by Server identity authority", + )); + } + let binding = api + .store + .get_workspace_runtime_binding(&path.workspace_id, &path.runtime_id) + .await? + .ok_or_else(|| Error::RuntimeBindingNotFound { + runtime_id: path.runtime_id.clone(), + })?; + Ok(Json(RuntimeTrustKeyRevealResponse { + public_key: binding.public_key, + })) +} + +async fn scoped_put_runtime_trust_key( + State(api): State, + AxumPath(path): AxumPath, + Extension(actor): Extension, + Json(request): Json, +) -> std::result::Result { + validate_workspace_scope(&api, &path.workspace_id)?; + require_workspace_owner(&api, &path.workspace_id, &actor, "Runtime trust changes").await?; + let actor_account_id = actor.account_id.clone(); + if path.runtime_id == EMBEDDED_WORKER_RUNTIME_ID { + return Err(settings_bad_request( + "embedded_runtime_trust_managed_internally", + "the embedded Runtime trust key is managed by Server identity authority", + )); + } + if request.expected_revision == Some(0) { + return Err(settings_bad_request( + "invalid_runtime_binding_revision", + "expected_revision must be greater than zero when provided", + )); + } + if request.public_key.len() > 16 * 1024 { + return Err(settings_bad_request( + "runtime_public_key_too_large", + "public_key must be at most 16384 bytes", + )); + } + let existing = api + .store + .get_workspace_runtime_binding(&path.workspace_id, &path.runtime_id) + .await?; + let source = api + .config + .remote_runtime_sources + .iter() + .find(|source| { + source.runtime_id == path.runtime_id + && source.workspace_id.as_deref() == Some(path.workspace_id.as_str()) + }) + .cloned(); + if let (Some(binding), Some(source)) = (&existing, &source) + && binding.base_url != source.base_url + { + return Err(settings_bad_request( + "runtime_endpoint_mismatch", + "the persisted Runtime endpoint no longer matches Server Runtime configuration; reconcile the endpoint before changing trust", + )); + } + let (display_name, base_url) = if let Some(binding) = &existing { + (binding.display_name.clone(), binding.base_url.clone()) + } else if let Some(source) = &source { + (source.display_name.clone(), source.base_url.clone()) + } else { + return Err(Error::UnknownRuntime(path.runtime_id.clone()).into()); + }; + let now = Utc::now().to_rfc3339(); + let record = WorkspaceRuntimeBinding { + workspace_id: path.workspace_id.clone(), + runtime_id: path.runtime_id.clone(), + display_name, + base_url, + public_key: request.public_key, + public_key_fingerprint: String::new(), + binding_revision: 1, + created_at: existing + .as_ref() + .map_or_else(|| now.clone(), |binding| binding.created_at.clone()), + updated_at: now, + revoked_at: None, + }; + let mutation = api + .store + .put_workspace_runtime_binding_key(record, request.expected_revision, &actor_account_id) + .await; + let (_, binding) = match mutation { + Ok(result) => result, + Err(error) => { + if let Some(response) = runtime_trust_conflict_response(&api, &path, &error).await { + return Ok(response); + } + return Err(error.into()); + } + }; + api.runtime_binding_expectations + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .insert( + (path.workspace_id.clone(), path.runtime_id.clone()), + binding, + ); + if let Some(source) = source { + api.runtime_subscription_broker + .register_remote_runtime(source); + } + Ok( + Json(workspace_runtime_detail(&api, &path.workspace_id, &path.runtime_id).await?) + .into_response(), + ) +} + +async fn scoped_revoke_runtime_trust_key( + State(api): State, + AxumPath(path): AxumPath, + Extension(actor): Extension, + Json(request): Json, +) -> std::result::Result { + validate_workspace_scope(&api, &path.workspace_id)?; + require_workspace_owner(&api, &path.workspace_id, &actor, "Runtime trust changes").await?; + let actor_account_id = actor.account_id.clone(); + if path.runtime_id == EMBEDDED_WORKER_RUNTIME_ID { + return Err(settings_bad_request( + "embedded_runtime_trust_managed_internally", + "the embedded Runtime trust key is managed by Server identity authority", + )); + } + if request.expected_revision == 0 { + return Err(settings_bad_request( + "invalid_runtime_binding_revision", + "expected_revision must be greater than zero", + )); + } + let now = Utc::now().to_rfc3339(); + let mutation = api + .store + .revoke_workspace_runtime_binding_key( + &path.workspace_id, + &path.runtime_id, + request.expected_revision, + &actor_account_id, + &now, + ) + .await; + let _ = match mutation { + Ok(result) => result, + Err(error) => { + if let Some(response) = runtime_trust_conflict_response(&api, &path, &error).await { + return Ok(response); + } + return Err(error.into()); + } + }; + api.runtime_binding_expectations + .write() + .unwrap_or_else(std::sync::PoisonError::into_inner) + .remove(&(path.workspace_id.clone(), path.runtime_id.clone())); + api.runtime_subscription_broker + .unregister_runtime(&path.runtime_id); + Ok( + Json(workspace_runtime_detail(&api, &path.workspace_id, &path.runtime_id).await?) + .into_response(), + ) +} + +async fn runtime_trust_conflict_response( + api: &WorkspaceApi, + path: &ScopedRuntimePath, + error: &Error, +) -> Option { + let kind = match error { + Error::RuntimeBindingRevisionConflict { .. } => RuntimeTrustConflictKind::StaleRevision, + Error::RuntimeBindingFingerprintConflict { .. } => { + RuntimeTrustConflictKind::FingerprintInUse + } + _ => return None, + }; + let current = api + .store + .get_workspace_runtime_binding(&path.workspace_id, &path.runtime_id) + .await + .ok() + .flatten(); + Some( + ( + StatusCode::CONFLICT, + Json(RuntimeTrustConflictResponse { + error: kind, + message: match kind { + RuntimeTrustConflictKind::StaleRevision => { + "the Runtime trust binding changed; reload before retrying".to_string() + } + RuntimeTrustConflictKind::FingerprintInUse => { + "the public key is already bound to another Runtime in this Workspace" + .to_string() + } + }, + current_revision: current.as_ref().map(|binding| binding.binding_revision), + current_fingerprint: current + .as_ref() + .map(|binding| binding.public_key_fingerprint.clone()), + }), + ) + .into_response(), + ) +} + async fn scoped_delete_remote_runtime( State(api): State, AxumPath(path): AxumPath, + Extension(actor): Extension, ) -> ApiResult { validate_workspace_scope(&api, &path.workspace_id)?; + require_workspace_owner(&api, &path.workspace_id, &actor, "Runtime removal").await?; delete_remote_runtime(State(api), AxumPath(path.runtime_id)).await } @@ -11458,6 +12174,7 @@ async fn scoped_cancel_runtime_worker( async fn scoped_worker_protocol_ws( ws: WebSocketUpgrade, State(api): State, + Extension(actor): Extension, AxumPath(path): AxumPath, ) -> Response { if let Err(err) = validate_workspace_scope(&api, &path.workspace_id) { @@ -11465,6 +12182,7 @@ async fn scoped_worker_protocol_ws( } worker_protocol_ws( State(api), + Extension(actor), AxumPath((path.worker.runtime_id, path.worker.worker_id)), ws, ) @@ -12174,6 +12892,8 @@ async fn get_workspace( permissions: WorkspacePermissionSummary { manage_repositories: is_owner, manage_secrets: is_owner, + manage_runtimes: is_owner, + delete_workspace: is_owner, }, extension_points: WorkspaceExtensionPoints { store: "sqlite".to_string(), @@ -12391,11 +13111,10 @@ async fn list_workers( } async fn create_remote_runtime( - State(api): State, + State(_api): State, Json(request): Json, ) -> ApiResult<(StatusCode, Json)> { validate_runtime_connection_request(&request)?; - let mut runtime_config = load_backend_runtimes_config_for_settings(&api)?; let id = request.runtime_id.trim().to_string(); if id == EMBEDDED_WORKER_RUNTIME_ID { return Err(settings_bad_request( @@ -12413,54 +13132,10 @@ async fn create_remote_runtime( "remote Runtime token_ref persistence is not supported", )); } - if runtime_config - .runtimes - .remote - .iter() - .any(|remote| remote.id == id) - { - return Err(settings_bad_request( - "remote_runtime_already_exists", - "a remote Runtime with that id already exists", - )); - } - let remote_config = RemoteRuntimeConfigFile { - id: id.clone(), - endpoint: request.endpoint.trim().to_string(), - display_name: request - .display_name - .as_deref() - .map(str::trim) - .filter(|value| !value.is_empty()) - .map(ToOwned::to_owned), - token_ref: None, - }; - let active_config = remote_runtime_config_from_file(&remote_config).map_err(|diagnostic| { - ApiError::with_diagnostics( - Error::RuntimeOperationFailed { - runtime_id: remote_config.id.clone(), - code: diagnostic.code.clone(), - message: diagnostic.message.clone(), - }, - vec![diagnostic], - ) - })?; - let active_runtime = RemoteWorkerRuntime::new( - active_config, - api.config.workspace_id.clone(), - api.config - .backend_base_url - .clone() - .unwrap_or_else(|| "http://127.0.0.1:8787".to_string()), - ) - .map(|host| host.with_resource_broker(api.resource_broker.clone())) - .map_err(|err| err.into_error())?; - runtime_config.runtimes.remote.push(remote_config); - write_backend_runtimes_config_for_settings(&api, &runtime_config)?; - api.runtime.register_or_replace(active_runtime); - let resource = workspace_runtime_resource_by_id(&api, &runtime_config, &id) - .ok_or_else(|| Error::UnknownRuntime(id.clone()))?; - Ok((StatusCode::CREATED, Json(resource))) + Err(settings_bad_request( + "runtime_public_key_required", + "remote Runtime registration requires an authenticated public key; configure it from the Runtime detail page after the Runtime endpoint is registered", + )) } async fn delete_remote_runtime( @@ -12473,14 +13148,16 @@ async fn delete_remote_runtime( "the embedded Runtime is built in and cannot be deleted", )); } - let mut runtime_config = load_backend_runtimes_config_for_settings(&api)?; - let before = runtime_config.runtimes.remote.len(); - runtime_config - .runtimes - .remote - .retain(|remote| remote.id != runtime_id); - if before == runtime_config.runtimes.remote.len() { - return Err(Error::UnknownRuntime(runtime_id).into()); + let binding = api + .store + .get_workspace_runtime_binding(&api.config.workspace_id, &runtime_id) + .await? + .ok_or_else(|| Error::UnknownRuntime(runtime_id.clone()))?; + if binding.revoked_at.is_none() { + return Err(Error::RuntimeBindingConflict( + "runtime trust is still active; revoke this Workspace's trust key with an expected revision before removing the inactive registration".to_string(), + ) + .into()); } match api .runtime @@ -12510,7 +13187,6 @@ async fn delete_remote_runtime( )); } } - write_backend_runtimes_config_for_settings(&api, &runtime_config)?; Ok(StatusCode::NO_CONTENT) } @@ -12518,14 +13194,37 @@ async fn test_runtime_connection( State(api): State, AxumPath(runtime_id): AxumPath, ) -> ApiResult> { - let runtime_config = load_backend_runtimes_config_for_settings(&api)?; - let remote = runtime_config - .runtimes - .remote - .iter() - .find(|remote| remote.id == runtime_id) + let runtime_id = runtime_id.trim().to_string(); + if runtime_id.is_empty() { + return Err(Error::InvalidRuntimeIdentifier { + kind: "runtime".to_string(), + value: runtime_id, + } + .into()); + } + api.store + .get_workspace_runtime_binding(api.workspace_id(), &runtime_id) + .await? + .filter(|binding| binding.revoked_at.is_none()) .ok_or_else(|| Error::UnknownRuntime(runtime_id.clone()))?; - Ok(Json(test_remote_runtime_config(&api, remote).await)) + + let checked_at = Utc::now().to_rfc3339(); + let runtime = api.runtime.clone(); + let ping_runtime_id = runtime_id.clone(); + let ping = tokio::task::spawn_blocking(move || runtime.ping(&ping_runtime_id)) + .await + .map_err(|_| Error::RuntimeOperationFailed { + runtime_id: runtime_id.clone(), + code: "runtime_connection_test_unavailable".to_string(), + message: "Runtime connection test could not be completed".to_string(), + })?; + + Ok(Json(runtime_connection_test_response( + api.workspace_id(), + &runtime_id, + checked_at, + ping, + ))) } async fn get_worker_launch_options( @@ -13877,8 +14576,45 @@ async fn cancel_runtime_worker( Ok(Json(result)) } +fn authenticated_browser_input_source(actor: &RequestActor) -> protocol::AuthenticatedInputSource { + protocol::AuthenticatedInputSource::Account { + account_id: actor.account_id.clone(), + } +} + +pub(crate) fn authorize_browser_worker_method( + method: protocol::Method, + source: &protocol::AuthenticatedInputSource, +) -> std::result::Result { + match method { + protocol::Method::Submit { + submission_request_id, + input, + } => Ok(protocol::Method::SubmitTracked { + submission_request_id, + input, + source: source.clone(), + }), + protocol::Method::Notify { + notification_request_id, + message, + auto_run, + } => Ok(protocol::Method::NotifyTracked { + notification_request_id, + message, + auto_run, + source: source.clone(), + }), + protocol::Method::SubmitTracked { .. } | protocol::Method::NotifyTracked { .. } => { + Err("authenticated Worker input source is server-owned") + } + other => Ok(other), + } +} + async fn worker_protocol_ws( State(api): State, + Extension(actor): Extension, AxumPath((runtime_id, worker_id)): AxumPath<(String, String)>, ws: WebSocketUpgrade, ) -> impl IntoResponse { @@ -13902,7 +14638,8 @@ async fn worker_protocol_ws( .into_response(); } }; - ws.on_upgrade(move |socket| worker_protocol_ws_session(source, socket)) + let input_source = authenticated_browser_input_source(&actor); + ws.on_upgrade(move |socket| worker_protocol_ws_session(source, socket, input_source)) } pub(crate) struct WorkspaceWorkerProtocolConnection { @@ -13913,6 +14650,7 @@ pub(crate) struct WorkspaceWorkerProtocolConnection { pub(crate) async fn connect_workspace_worker_protocol( api: &WorkspaceApi, worker: &RuntimeWorkerRef, + input_source: Option<&protocol::AuthenticatedInputSource>, ) -> Result { let source = match api.observation_proxy.source(worker) { Ok(source) => source, @@ -13929,15 +14667,39 @@ pub(crate) async fn connect_workspace_worker_protocol( } }; match source { - RuntimeObservationSource::RemoteWs(config) => connect_remote_worker_protocol(config).await, + RuntimeObservationSource::RemoteWs(config) => { + connect_remote_worker_protocol(config, input_source).await + } RuntimeObservationSource::Embedded(source) => { connect_embedded_worker_protocol(source).await } } } +fn insert_authenticated_input_source_header( + headers: &mut HeaderMap, + input_source: Option<&protocol::AuthenticatedInputSource>, +) -> Result<()> { + let Some(input_source) = input_source else { + return Ok(()); + }; + let protocol::AuthenticatedInputSource::Account { account_id } = input_source else { + return Err(Error::Config( + "remote Worker protocol transport supports only Account input source".into(), + )); + }; + headers.insert( + protocol::AUTHENTICATED_ACCOUNT_ID_HEADER, + account_id.parse().map_err(|error| { + Error::Config(format!("invalid authenticated Account identity: {error}")) + })?, + ); + Ok(()) +} + async fn connect_remote_worker_protocol( config: RuntimeObservationSourceConfig, + input_source: Option<&protocol::AuthenticatedInputSource>, ) -> Result { let mut request = config .endpoint @@ -13952,6 +14714,7 @@ async fn connect_remote_worker_protocol( })?, ); } + insert_authenticated_input_source_header(request.headers_mut(), input_source)?; let (socket, _) = connect_async(request) .await @@ -14029,13 +14792,17 @@ async fn connect_embedded_worker_protocol( Ok(WorkspaceWorkerProtocolConnection { methods, events }) } -async fn worker_protocol_ws_session(source: RuntimeObservationSource, socket: WebSocket) { +async fn worker_protocol_ws_session( + source: RuntimeObservationSource, + socket: WebSocket, + input_source: protocol::AuthenticatedInputSource, +) { match source { RuntimeObservationSource::RemoteWs(config) => { - remote_worker_protocol_ws_session(config, socket).await; + remote_worker_protocol_ws_session(config, socket, input_source).await; } RuntimeObservationSource::Embedded(source) => { - embedded_worker_protocol_ws_session(source, socket).await; + embedded_worker_protocol_ws_session(source, socket, input_source).await; } } } @@ -14043,6 +14810,7 @@ async fn worker_protocol_ws_session(source: RuntimeObservationSource, socket: We async fn remote_worker_protocol_ws_session( config: RuntimeObservationSourceConfig, socket: WebSocket, + input_source: protocol::AuthenticatedInputSource, ) { let mut request = match config.endpoint.clone().into_client_request() { Ok(request) => request, @@ -14070,6 +14838,16 @@ async fn remote_worker_protocol_ws_session( } } } + if let Err(error) = + insert_authenticated_input_source_header(request.headers_mut(), Some(&input_source)) + { + let mut socket = socket; + let event = protocol_error_event(format!( + "failed to build authenticated Account identity header: {error}" + )); + let _ = send_protocol_event(&mut socket, &event).await; + return; + } let (upstream, _) = match connect_async(request).await { Ok(connection) => connection, @@ -14091,14 +14869,33 @@ async fn remote_worker_protocol_ws_session( inbound = client_stream.next() => { match inbound { Some(Ok(WsMessage::Text(text))) => { - if upstream_sink.send(TungsteniteMessage::Text(text.to_string().into())).await.is_err() { + let method = match protocol::stream::decode_method(text.as_ref()) { + Ok(method) => match authorize_browser_worker_method(method, &input_source) { + Ok(method) => method, + Err(message) => { + if let Ok(event) = protocol::stream::encode_event(&protocol_error_event(message)) { + let _ = client_sink.send(WsMessage::Text(event.into())).await; + } + break; + } + }, + Err(error) => { + if let Ok(event) = protocol::stream::encode_event(&protocol_error_event(error.to_string())) { + let _ = client_sink.send(WsMessage::Text(event.into())).await; + } + break; + } + }; + let Ok(method) = protocol::stream::encode_method(&method) else { break }; + if upstream_sink.send(TungsteniteMessage::Text(method.into())).await.is_err() { break; } } - Some(Ok(WsMessage::Binary(binary))) => { - if upstream_sink.send(TungsteniteMessage::Binary(binary.to_vec().into())).await.is_err() { - break; + Some(Ok(WsMessage::Binary(_))) => { + if let Ok(event) = protocol::stream::encode_event(&protocol_error_event("binary Worker methods are not accepted")) { + let _ = client_sink.send(WsMessage::Text(event.into())).await; } + break; } Some(Ok(WsMessage::Close(_))) | None => { let _ = upstream_sink.send(TungsteniteMessage::Close(None)).await; @@ -14154,6 +14951,7 @@ async fn remote_worker_protocol_ws_session( async fn embedded_worker_protocol_ws_session( source: crate::observation::EmbeddedRuntimeObservationSource, mut socket: WebSocket, + input_source: protocol::AuthenticatedInputSource, ) { let mut upstream = match RuntimeObservationClient::connect(&RuntimeObservationSource::Embedded( source.clone(), @@ -14173,24 +14971,32 @@ async fn embedded_worker_protocol_ws_session( inbound = socket.next() => { match inbound { Some(Ok(WsMessage::Text(text))) => match decode_method(&text) { - Ok(method) => match source.runtime.send_protocol_method(&source.worker_ref, method) { - Ok(events) => { - for event in events { + Ok(method) => match authorize_browser_worker_method(method, &input_source) { + Ok(method) => match source.runtime.send_protocol_method(&source.worker_ref, method) { + Ok(events) => { + for event in events { + if !send_protocol_event(&mut socket, &event).await { + return; + } + } + } + Err(error) => { + let event = protocol_error_event(error.to_string()); if !send_protocol_event(&mut socket, &event).await { return; } } - } - Err(error) => { - let event = protocol_error_event(error.to_string()); - if !send_protocol_event(&mut socket, &event).await { - return; - } + }, + Err(message) => { + let event = protocol_error_event(message); + let _ = send_protocol_event(&mut socket, &event).await; + return; } }, Err(error) => { - let event = - protocol_error_event(format!("malformed protocol method frame: {error}")); + let event = protocol_error_event(format!( + "malformed protocol method frame: {error}" + )); if !send_protocol_event(&mut socket, &event).await { return; } @@ -14384,92 +15190,49 @@ fn workers_response( }) } -fn load_backend_runtimes_config_for_settings( +async fn workspace_runtime_resources_response( api: &WorkspaceApi, -) -> ApiResult { - api.config - .runtime_config_path - .as_ref() - .map(BackendRuntimesConfigFile::load_from_path) - .transpose() - .map_err(|error| { - Error::Config(format!( - "failed to read Backend runtimes config for Runtime connections: {}", - sanitize_backend_error(&error.to_string()) - )) - .into() - }) - .map(|config| config.unwrap_or_default()) -} - -fn write_backend_runtimes_config_for_settings( - api: &WorkspaceApi, - runtime_config: &BackendRuntimesConfigFile, -) -> ApiResult<()> { - let path = api.config.runtime_config_path.as_ref().ok_or_else(|| { - Error::Config( - "Backend runtimes config path is unavailable; set YOI_CONFIG_DIR, YOI_HOME, XDG_CONFIG_HOME, or HOME" - .to_string(), - ) - })?; - runtime_config.write_to_path(path).map_err(|error| { - Error::Config(format!( - "failed to write Backend runtimes config for Runtime connections: {}", - sanitize_backend_error(&error.to_string()) - )) - .into() - }) -} - -fn workspace_runtime_resources_response( - api: &WorkspaceApi, - runtime_config: &BackendRuntimesConfigFile, -) -> workspace_api::ListResponse { + workspace_id: &str, +) -> ApiResult> { let limit = api.config.max_records.min(200); let runtimes = api.runtime.list_runtimes(limit); + let bindings = api + .store + .list_workspace_runtime_bindings(workspace_id, true) + .await?; let mut items = runtimes .items .into_iter() .map(|runtime| { - let remote = runtime_config - .runtimes - .remote + let binding = bindings .iter() - .find(|remote| remote.id == runtime.runtime_id); + .find(|binding| binding.runtime_id == runtime.runtime_id); let built_in = runtime.runtime_id == EMBEDDED_WORKER_RUNTIME_ID; WorkspaceRuntimeResource { runtime: runtime.into(), management: RuntimeManagementSummary { built_in, - config_managed: remote.is_some(), - removable: remote.is_some() && !built_in, - endpoint_configured: remote - .is_some_and(|remote| !remote.endpoint.trim().is_empty()), - token_ref_configured: remote.is_some_and(|remote| { - remote - .token_ref - .as_deref() - .is_some_and(|value| !value.trim().is_empty()) - }), + config_managed: binding.is_some(), + removable: binding.is_some() && !built_in, + endpoint_configured: binding + .is_some_and(|binding| !binding.base_url.trim().is_empty()), + token_ref_configured: false, }, } }) .collect::>(); - for remote in &runtime_config.runtimes.remote { + for binding in &bindings { if items .iter() - .any(|resource| resource.runtime.runtime_id == remote.id) + .any(|resource| resource.runtime.runtime_id == binding.runtime_id) { continue; } items.push(WorkspaceRuntimeResource { runtime: workspace_api::RuntimeSummary { - runtime_id: remote.id.clone(), - label: remote - .display_name - .clone() - .unwrap_or_else(|| remote.id.clone()), + runtime_id: binding.runtime_id.clone(), + label: binding.display_name.clone(), kind: "remote_http".to_string(), status: "unavailable".to_string(), source: workspace_api::RuntimeSourceSummary { @@ -14477,7 +15240,7 @@ fn workspace_runtime_resources_response( status: workspace_api::RuntimeSourceStatus::Reserved, identity_authority: workspace_api::RuntimeIdentityAuthority::ServerRuntimeConfiguration, - note: "The configured Runtime is not present in the active Runtime registry." + note: "The registered Runtime is not present in the active Runtime registry." .to_string(), }, host_ids: Vec::new(), @@ -14486,9 +15249,9 @@ fn workspace_runtime_resources_response( arch: String::new(), diagnostics: vec![ settings_diagnostic( - "configured_runtime_unavailable", + "registered_runtime_unavailable", DiagnosticSeverity::Warning, - "The configured Runtime is not present in the active Runtime registry.", + "The registered Runtime is not present in the active Runtime registry.", ) .into(), ], @@ -14497,33 +15260,143 @@ fn workspace_runtime_resources_response( built_in: false, config_managed: true, removable: true, - endpoint_configured: !remote.endpoint.trim().is_empty(), - token_ref_configured: remote - .token_ref - .as_deref() - .is_some_and(|value| !value.trim().is_empty()), + endpoint_configured: !binding.base_url.trim().is_empty(), + token_ref_configured: false, }, }); } - workspace_api::ListResponse { - workspace_id: api.config.workspace_id.clone(), + Ok(workspace_api::ListResponse { + workspace_id: workspace_id.to_string(), limit, items, - source: "workspace-runtime-resources".to_string(), + source: "workspace-runtime-bindings".to_string(), diagnostics: runtimes.diagnostics.into_iter().map(Into::into).collect(), - } + }) } -fn workspace_runtime_resource_by_id( +async fn workspace_runtime_detail( api: &WorkspaceApi, - runtime_config: &BackendRuntimesConfigFile, + workspace_id: &str, runtime_id: &str, -) -> Option { - workspace_runtime_resources_response(api, runtime_config) +) -> ApiResult { + let binding = api + .store + .get_workspace_runtime_binding(workspace_id, runtime_id) + .await?; + let mut resource = workspace_runtime_resources_response(api, workspace_id) + .await? .items .into_iter() - .find(|resource| resource.runtime.runtime_id == runtime_id) + .find(|resource| resource.runtime.runtime_id == runtime_id); + if resource.is_none() { + resource = binding.as_ref().map(|binding| WorkspaceRuntimeResource { + runtime: workspace_api::RuntimeSummary { + runtime_id: binding.runtime_id.clone(), + label: binding.display_name.clone(), + kind: "remote_http".to_string(), + status: "unavailable".to_string(), + source: workspace_api::RuntimeSourceSummary { + kind: workspace_api::RuntimeSourceKind::RemoteHttp, + status: workspace_api::RuntimeSourceStatus::Reserved, + identity_authority: + workspace_api::RuntimeIdentityAuthority::ServerRuntimeConfiguration, + note: "The Runtime trust binding is not active in the Runtime registry." + .to_string(), + }, + host_ids: Vec::new(), + worker_creation_available: false, + os: String::new(), + arch: String::new(), + diagnostics: Vec::new(), + }, + management: RuntimeManagementSummary { + built_in: false, + config_managed: true, + removable: false, + endpoint_configured: !binding.base_url.trim().is_empty(), + token_ref_configured: false, + }, + }); + } + let mut resource = resource.ok_or_else(|| Error::UnknownRuntime(runtime_id.to_string()))?; + if let Some(binding) = &binding { + resource.management.config_managed = true; + resource.management.endpoint_configured = !binding.base_url.trim().is_empty(); + } + let endpoint = binding + .as_ref() + .map(|binding| binding.base_url.clone()) + .or_else(|| { + api.config + .remote_runtime_sources + .iter() + .find(|source| { + source.runtime_id == runtime_id + && source.workspace_id.as_deref() == Some(workspace_id) + }) + .map(|source| source.base_url.clone()) + }); + let trust_key = binding.as_ref().map_or( + RuntimeTrustKeyState { + status: RuntimeTrustKeyStatus::Unconfigured, + fingerprint: None, + revision: None, + created_at: None, + updated_at: None, + revoked_at: None, + }, + |binding| RuntimeTrustKeyState { + status: if binding.revoked_at.is_some() { + RuntimeTrustKeyStatus::Revoked + } else { + RuntimeTrustKeyStatus::Active + }, + fingerprint: Some(binding.public_key_fingerprint.clone()), + revision: Some(binding.binding_revision), + created_at: Some(binding.created_at.clone()), + updated_at: Some(binding.updated_at.clone()), + revoked_at: binding.revoked_at.clone(), + }, + ); + let recent_audit = api + .store + .list_workspace_runtime_binding_audit(workspace_id, runtime_id, 20) + .await? + .into_iter() + .map(project_runtime_trust_audit) + .collect::>>()?; + Ok(WorkspaceRuntimeDetail { + workspace_id: workspace_id.to_string(), + runtime: resource, + endpoint, + trust_key, + recent_audit, + }) +} + +fn project_runtime_trust_audit( + record: WorkspaceRuntimeBindingAuditRecord, +) -> Result { + let action = match record.action.as_str() { + "created" => RuntimeTrustAuditAction::Created, + "replaced" => RuntimeTrustAuditAction::Replaced, + "reactivated" => RuntimeTrustAuditAction::Reactivated, + "revoked" => RuntimeTrustAuditAction::Revoked, + other => { + return Err(Error::Store(format!( + "unsupported Runtime trust audit action {other}" + ))); + } + }; + Ok(RuntimeTrustAuditEntry { + action, + actor_account_id: record.actor_account_id, + old_fingerprint: record.old_fingerprint, + new_fingerprint: record.new_fingerprint, + revision: record.binding_revision, + at: record.at, + }) } fn validate_runtime_connection_request(request: &CreateRemoteRuntimeRequest) -> ApiResult<()> { @@ -14569,450 +15442,125 @@ fn validate_public_runtime_id(runtime_id: &str) -> ApiResult<()> { Ok(()) } -fn remote_runtime_config_from_file( - remote: &RemoteRuntimeConfigFile, +#[cfg(test)] +fn remote_runtime_config_from_binding( + binding: &crate::store::WorkspaceRuntimeBinding, ) -> std::result::Result { - resolve_remote_runtime(remote).map_err(|err| { - settings_diagnostic( - "remote_runtime_apply_failed", - DiagnosticSeverity::Error, - err.to_string(), - ) - }) + let remote = RemoteRuntimeConfig::new( + binding.runtime_id.clone(), + binding.display_name.clone(), + binding.base_url.clone(), + None, + ) + .with_workspace_id(binding.workspace_id.clone()); + Ok(remote) } -async fn test_remote_runtime_config( - api: &WorkspaceApi, - remote: &RemoteRuntimeConfigFile, -) -> RuntimeConnectionTestResponse { - let checked_at = Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true); - if remote - .token_ref - .as_deref() - .is_some_and(|value| !value.trim().is_empty()) - { - return RuntimeConnectionTestResponse { - workspace_id: api.config.workspace_id.clone(), - runtime_id: remote.id.clone(), - checked_at, - state: "rejected".to_string(), - protocol_version: None, - compatibility_basis: "not_checked_token_ref_unsupported".to_string(), - capabilities: Vec::new(), - health_result: "not_checked".to_string(), - diagnostics: vec![settings_diagnostic( - "remote_runtime_token_ref_unsupported", - DiagnosticSeverity::Error, - "Remote Runtime test cannot use token_ref in v0; no token or secret value was exposed to the Browser.", - ) - .into()], - }; - } - - let client = match reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(5)) - .build() - { - Ok(client) => client, - Err(_) => { - return remote_runtime_test_failed( - api, - remote, - checked_at, - "remote_runtime_test_client_unavailable", - "Remote Runtime test client could not be initialized.", - ); - } - }; - - let mut observation = RuntimeCompatibilityObservation::default(); - let summary_url = match remote_probe_url(remote, "/v1/runtime") { - Ok(url) => url, - Err(diagnostic) => { - return remote_runtime_test_failed( - api, - remote, - checked_at, - diagnostic.code, - diagnostic.message, - ); - } - }; - - let summary_payload = - match probe_remote_json(&client, summary_url, "runtime.summary", "Runtime summary").await { - Ok(payload) => payload, - Err(diagnostic) => { - return remote_runtime_test_failed( - api, - remote, - checked_at, - diagnostic.code, - diagnostic.message, - ); - } - }; - let protocol_version = summary_payload - .get("protocol_version") - .and_then(|value| value.as_str()) - .map(ToOwned::to_owned); - let summary = match serde_json::from_value::(summary_payload) { - Ok(summary) => summary, - Err(_) => { - return remote_runtime_test_failed( - api, - remote, - checked_at, - "remote_runtime_malformed_summary", - "Remote Runtime summary responded, but the payload was not recognized.", - ); - } - }; - observation.available( - "runtime.summary", - "Connected: /v1/runtime responded with a recognized worker-runtime summary.", - ); - - let workers_url = match remote_probe_url(remote, "/v1/workers") { - Ok(url) => url, - Err(diagnostic) => { - observation.incompatible("workers.list", diagnostic); - String::new() - } - }; - let workers = if workers_url.is_empty() { - None - } else { - match probe_remote_json(&client, workers_url, "workers.list", "Worker list").await { - Ok(payload) => match serde_json::from_value::(payload) { - Ok(workers) => { - observation.available( - "workers.list", - "Verified: /v1/workers responded with a recognized worker list.", - ); - Some(workers) - } - Err(_) => { - observation.incompatible( - "workers.list", - settings_diagnostic( - "remote_runtime_workers_malformed", - DiagnosticSeverity::Error, - "Remote Runtime worker list responded, but the payload was not recognized.", - ), - ); - None - } - }, - Err(diagnostic) => { - observation.incompatible("workers.list", diagnostic); - None - } - } - }; - - if let Some(worker) = workers.as_ref().and_then(|workers| workers.workers.first()) { - let path = format!( - "/v1/workers/{}", - encode_path_segment(&worker.worker_id.to_string()) - ); - match remote_probe_url(remote, &path) { - Ok(url) => match probe_remote_json(&client, url, "workers.detail", "Worker detail").await { - Ok(payload) => match serde_json::from_value::(payload) { - Ok(_) => observation.available( - "workers.detail", - "Verified: worker detail responded for an existing worker reported by the remote Runtime.", - ), - Err(_) => observation.incompatible( - "workers.detail", - settings_diagnostic( - "remote_runtime_worker_detail_malformed", - DiagnosticSeverity::Error, - "Remote Runtime worker detail responded, but the payload was not recognized.", - ), - ), - }, - Err(diagnostic) => observation.incompatible("workers.detail", diagnostic), - }, - Err(diagnostic) => observation.incompatible("workers.detail", diagnostic), - } - } else { - observation.unknown( - "workers.detail", - "No connection problem found. Worker detail was not checked because the remote Runtime reported no workers during the lightweight probe.", - ); - } - - observation.available( - "workers.events_ws.construct", - "Verified: worker event websocket URL can be constructed from the configured HTTP(S) Runtime endpoint. The lightweight test does not open a websocket stream.", - ); - - let bundles_url = match remote_probe_url(remote, "/v1/config-bundles") { - Ok(url) => url, - Err(diagnostic) => { - observation.incompatible("config_bundles.list", diagnostic); - String::new() - } - }; - let bundles = if bundles_url.is_empty() { - None - } else { - match probe_remote_json( - &client, - bundles_url, - "config_bundles.list", - "Config-bundle list", - ) - .await - { - Ok(payload) => { - match serde_json::from_value::(payload) { - Ok(bundles) => { - observation.available( - "config_bundles.list", - "Verified: /v1/config-bundles responded with a recognized config-bundle list.", - ); - Some(bundles) - } - Err(_) => { - observation.incompatible( - "config_bundles.list", - settings_diagnostic( - "remote_runtime_config_bundles_malformed", - DiagnosticSeverity::Error, - "Remote Runtime config-bundle list responded, but the payload was not recognized.", - ), - ); - None - } - } - } - Err(diagnostic) => { - observation.incompatible("config_bundles.list", diagnostic); - None - } - } - }; - - if let Some(bundle) = bundles.as_ref().and_then(|bundles| bundles.bundles.first()) { - let path = format!( - "/v1/config-bundles/{}/availability?digest={}", - encode_path_segment(&bundle.id), - encode_path_segment(&bundle.digest) - ); - match remote_probe_url(remote, &path) { - Ok(url) => match probe_remote_json( - &client, - url, - "config_bundles.availability", - "Config-bundle availability", - ) - .await - { - Ok(payload) => { - match serde_json::from_value::(payload) - { - Ok(_) => observation.available( - "config_bundles.availability", - "Verified: config-bundle availability was confirmed for an advertised bundle.", - ), - Err(_) => observation.incompatible( - "config_bundles.availability", - settings_diagnostic( - "remote_runtime_config_bundle_availability_malformed", - DiagnosticSeverity::Error, - "Remote Runtime config-bundle availability responded, but the payload was not recognized.", - ), - ), - } - } - Err(diagnostic) => { - observation.incompatible("config_bundles.availability", diagnostic) - } - }, - Err(diagnostic) => observation.incompatible("config_bundles.availability", diagnostic), - } - } else { - observation.unknown( - "config_bundles.availability", - "No connection problem found. Config-bundle availability was not checked because the remote Runtime advertised no bundles during the lightweight probe.", - ); - } - - if summary.runtime.worker_creation_available { - observation.available( - "workers.spawn", - "Verified: /v1/runtime reports worker creation is enabled by a Runtime execution backend. The lightweight test does not create a worker.", - ); - } else { - observation.incompatible( - "workers.spawn", - settings_diagnostic( - "remote_runtime_worker_creation_unavailable", - DiagnosticSeverity::Error, - "Connected to the Runtime, but worker creation is unavailable because this Runtime process has no execution backend attached.", - ), - ); - } - observation.unknown( - "workers.input_dispatch", - "No connection problem found. Worker input dispatch was not checked because this lightweight test does not send model-visible input as a side effect.", - ); - observation.unknown( - "config_bundles.sync", - "No connection problem found. Config-bundle sync was not checked because this lightweight test does not upload bundles as a side effect.", - ); - - RuntimeConnectionTestResponse { - workspace_id: api.config.workspace_id.clone(), - runtime_id: remote.id.clone(), - checked_at, - state: observation.state().to_string(), - protocol_version, - compatibility_basis: "Connected to /v1/runtime and verified non-side-effecting worker-runtime HTTP endpoints. No incompatible operation was found; warning items below are unproven optional or side-effecting checks, not connection failures.".to_string(), - capabilities: observation.capabilities, - health_result: format!( - "connected=true; runtime_status={:?}; available={}; incompatible={}; warnings={}", - summary.runtime.status, - observation.available_count, - observation.incompatible_count, - observation.unknown_count - ), - diagnostics: observation - .diagnostics - .into_iter() - .map(Into::into) - .collect(), - } -} - -fn remote_runtime_test_failed( - api: &WorkspaceApi, - remote: &RemoteRuntimeConfigFile, +fn runtime_connection_test_response( + workspace_id: &str, + runtime_id: &str, checked_at: String, - code: impl Into, - message: impl Into, + ping: std::result::Result< + worker_runtime::http_server::RuntimeHttpPingResponse, + crate::hosts::RuntimePingFailure, + >, +) -> RuntimeConnectionTestResponse { + match ping { + Ok(ping) if ping.runtime_id != runtime_id => runtime_connection_test_failure( + workspace_id, + runtime_id, + checked_at, + RuntimeConnectionTestFailureKind::RuntimeIdentityMismatch, + None, + RuntimeDiagnostic::new( + "runtime_ping_identity_mismatch", + "error", + "Runtime ping identity does not match the registered Runtime", + ), + ), + Ok(ping) + if !(RUNTIME_HTTP_PROTOCOL_MIN_VERSION..=RUNTIME_HTTP_PROTOCOL_MAX_VERSION) + .contains(&ping.protocol_version) => + { + let code = if ping.protocol_version > RUNTIME_HTTP_PROTOCOL_MAX_VERSION { + "runtime_ping_protocol_newer" + } else { + "runtime_ping_protocol_older" + }; + runtime_connection_test_failure( + workspace_id, + runtime_id, + checked_at, + RuntimeConnectionTestFailureKind::ProtocolVersionMismatch, + Some(ping.protocol_version), + RuntimeDiagnostic::new( + code, + "error", + "Runtime protocol version is incompatible with this Server", + ), + ) + } + Ok(ping) => RuntimeConnectionTestResponse { + workspace_id: workspace_id.to_string(), + runtime_id: runtime_id.to_string(), + checked_at, + status: RuntimeConnectionTestStatus::Compatible, + failure_kind: None, + expected_protocol_version: RUNTIME_HTTP_PROTOCOL_VERSION, + actual_protocol_version: Some(ping.protocol_version), + diagnostics: Vec::new(), + }, + Err(failure) => runtime_connection_test_failure( + workspace_id, + runtime_id, + checked_at, + match failure.kind { + RuntimePingFailureKind::Authentication => { + RuntimeConnectionTestFailureKind::Authentication + } + RuntimePingFailureKind::Authorization => { + RuntimeConnectionTestFailureKind::Authorization + } + RuntimePingFailureKind::NetworkUnreachable => { + RuntimeConnectionTestFailureKind::NetworkUnreachable + } + RuntimePingFailureKind::Timeout => RuntimeConnectionTestFailureKind::Timeout, + RuntimePingFailureKind::TlsOrTransport => { + RuntimeConnectionTestFailureKind::TlsOrTransport + } + RuntimePingFailureKind::MalformedResponse => { + RuntimeConnectionTestFailureKind::MalformedResponse + } + RuntimePingFailureKind::Configuration | RuntimePingFailureKind::Unsupported => { + RuntimeConnectionTestFailureKind::Configuration + } + }, + None, + failure.diagnostic, + ), + } +} + +fn runtime_connection_test_failure( + workspace_id: &str, + runtime_id: &str, + checked_at: String, + failure_kind: RuntimeConnectionTestFailureKind, + actual_protocol_version: Option, + diagnostic: RuntimeDiagnostic, ) -> RuntimeConnectionTestResponse { RuntimeConnectionTestResponse { - workspace_id: api.config.workspace_id.clone(), - runtime_id: remote.id.clone(), + workspace_id: workspace_id.to_string(), + runtime_id: runtime_id.to_string(), checked_at, - state: "failed".to_string(), - protocol_version: None, - compatibility_basis: "worker-runtime lightweight HTTP compatibility probes".to_string(), - capabilities: Vec::new(), - health_result: "failed".to_string(), - diagnostics: vec![settings_diagnostic(code, DiagnosticSeverity::Error, message).into()], + status: RuntimeConnectionTestStatus::Failed, + failure_kind: Some(failure_kind), + expected_protocol_version: RUNTIME_HTTP_PROTOCOL_VERSION, + actual_protocol_version, + diagnostics: vec![diagnostic.into()], } } -#[derive(Default)] -struct RuntimeCompatibilityObservation { - capabilities: Vec, - diagnostics: Vec, - available_count: usize, - incompatible_count: usize, - unknown_count: usize, -} - -impl RuntimeCompatibilityObservation { - fn available(&mut self, operation: &str, message: impl Into) { - self.available_count += 1; - self.capabilities.push(format!("{operation}:available")); - self.diagnostics.push(settings_diagnostic( - format!("{operation}.available"), - DiagnosticSeverity::Info, - message, - )); - } - - fn unknown(&mut self, operation: &str, message: impl Into) { - self.unknown_count += 1; - self.capabilities.push(format!("{operation}:unknown")); - self.diagnostics.push(settings_diagnostic( - format!("{operation}.unknown"), - DiagnosticSeverity::Warning, - message, - )); - } - - fn incompatible(&mut self, operation: &str, diagnostic: RuntimeDiagnostic) { - self.incompatible_count += 1; - self.capabilities.push(format!("{operation}:incompatible")); - self.diagnostics.push(diagnostic); - } - - fn state(&self) -> &'static str { - if self.incompatible_count > 0 { - "incompatible" - } else { - "compatible" - } - } -} - -fn remote_probe_url( - remote: &RemoteRuntimeConfigFile, - path: &str, -) -> std::result::Result { - let endpoint = remote.endpoint.trim(); - if !(endpoint.starts_with("http://") || endpoint.starts_with("https://")) { - return Err(settings_diagnostic( - "remote_runtime_endpoint_invalid", - DiagnosticSeverity::Error, - "Configured remote Runtime endpoint is not an absolute HTTP(S) URL.", - )); - } - Ok(format!("{}{}", endpoint.trim_end_matches('/'), path)) -} - -async fn probe_remote_json( - client: &reqwest::Client, - url: String, - operation: &'static str, - label: &'static str, -) -> std::result::Result { - let response = client.get(url).send().await.map_err(|error| { - let (code, message) = if error.is_timeout() { - ( - format!("{operation}.timeout"), - format!("Remote Runtime probe for {label} timed out."), - ) - } else if error.is_connect() { - ( - format!("{operation}.connect_failed"), - format!("Remote Runtime probe for {label} could not connect."), - ) - } else { - ( - format!("{operation}.request_failed"), - format!("Remote Runtime probe for {label} failed before a response was received."), - ) - }; - settings_diagnostic(code, DiagnosticSeverity::Error, message) - })?; - - if !response.status().is_success() { - return Err(settings_diagnostic( - format!("{operation}.http_status"), - DiagnosticSeverity::Error, - format!( - "Remote Runtime probe for {label} returned HTTP status {}.", - response.status().as_u16() - ), - )); - } - - response.json::().await.map_err(|_| { - settings_diagnostic( - format!("{operation}.malformed_json"), - DiagnosticSeverity::Error, - format!("Remote Runtime probe for {label} returned an unrecognized JSON payload."), - ) - }) -} - fn worker_launch_options_response(api: &WorkspaceApi) -> ApiResult { let runtimes = api .runtime @@ -15218,6 +15766,7 @@ fn worker_summary_from_registry(record: &WorkerRegistryRecord) -> WorkerSummary singleton_key: None, tags: Vec::new(), state: "missing".to_string(), + worker_state: None, last_seen_at: Some(record.updated_at.clone()), pinned: record.retention_state == "pinned", retention_state: record.retention_state.clone(), @@ -16083,12 +16632,14 @@ fn worker_create_not_accepted_error( } fn settings_bad_request(code: &'static str, message: &'static str) -> ApiError { - Error::RuntimeOperationFailed { - runtime_id: "workspace-backend".to_string(), - code: code.to_string(), - message: message.to_string(), - } - .into() + ApiError::with_diagnostics( + Error::InvalidInput(format!("{code}: {message}")), + vec![settings_diagnostic( + code, + DiagnosticSeverity::Error, + message, + )], + ) } fn settings_diagnostic( @@ -16400,6 +16951,9 @@ impl IntoResponse for ApiError { Error::TicketAssignmentConflict(_) | Error::WorkdirAttachmentConflict(_) | Error::WorkspaceConfigConflict(_) + | Error::RuntimeBindingConflict(_) + | Error::RuntimeBindingRevisionConflict { .. } + | Error::RuntimeBindingFingerprintConflict { .. } | Error::RepositoryConflict(_) => StatusCode::CONFLICT, Error::WorkerSourceIdentity(_) | Error::InvalidInput(_) => StatusCode::BAD_REQUEST, Error::InvalidRuntimeIdentifier { .. } | Error::ReservedWorkerName(_) => { @@ -16428,6 +16982,7 @@ impl IntoResponse for ApiError { | Error::UnknownRuntime(_) | Error::UnknownWorker { .. } | Error::UnknownRepository(_) + | Error::RuntimeBindingNotFound { .. } | Error::WorkspaceIdMismatch => StatusCode::NOT_FOUND, Error::RuntimeOperationFailed { code, .. } if code == "skill_not_found" => { StatusCode::NOT_FOUND @@ -16539,7 +17094,6 @@ impl IntoResponse for ApiError { #[cfg(test)] mod tests { use super::*; - use crate::config::WorkspaceBackendRuntimesConfig; use axum::body::{Body, to_bytes}; use axum::http::Request; use futures::{SinkExt, StreamExt}; @@ -16565,9 +17119,71 @@ mod tests { use crate::store::{ AccountRecord, ApiTokenRecord, BrowserSessionRecord, MemoryDocumentRecord, MemoryStagingRecord, ObjectiveRecord, ObjectiveResourceRecord, ObjectiveTicketLinkRecord, - SqliteWorkspaceStore, TrustedRuntimeRecord, UserRecord, WorkspaceRecord, + SqliteWorkspaceStore, UserRecord, WorkspaceRecord, WorkspaceRuntimeBinding, }; + #[tokio::test] + async fn workspace_mutation_gate_serializes_deletion_with_active_mutations() { + let locks = Arc::new(AsyncMutex::new(HashMap::new())); + let active_mutation = workspace_mutation_lock(&locks, "workspace-a").await; + let deletion = workspace_mutation_lock(&locks, "workspace-a").await; + assert!(Arc::ptr_eq(&active_mutation, &deletion)); + + let active_guard = active_mutation.lock_owned().await; + let (acquired_tx, mut acquired_rx) = tokio::sync::oneshot::channel(); + let waiter = tokio::spawn(async move { + let _deletion_guard = deletion.lock_owned().await; + let _ = acquired_tx.send(()); + }); + tokio::task::yield_now().await; + assert!(matches!( + acquired_rx.try_recv(), + Err(tokio::sync::oneshot::error::TryRecvError::Empty) + )); + drop(active_guard); + acquired_rx.await.expect("deletion acquires after mutation"); + waiter.await.expect("waiter joins"); + } + + #[test] + fn workspace_deletion_execution_is_server_owned_and_polling_is_read_only() { + let source = include_str!("server.rs"); + let start = handler_source(source, "start_server_workspace_deletion"); + assert!(start.contains("schedule_workspace_deletion")); + assert!(!start.contains("execute_workspace_deletion(&request")); + let poll = handler_source(source, "get_server_workspace_deletion"); + assert!(!poll.contains("execute_workspace_deletion")); + assert!(source.contains("api.recover_workspace_deletions().await?")); + } + + #[test] + fn workspace_deletion_checkpoints_child_operations_before_external_cleanup() { + let source = include_str!("server.rs"); + let worker_start = source + .find("async fn execute_target_removal") + .expect("Worker removal helper"); + let worker_source = &source[worker_start..]; + let checkpoint = worker_source + .find("append_workspace_deletion_child_operation") + .expect("Worker child checkpoint"); + let cleanup = worker_source + .find("close_worker_workdir_sessions") + .expect("Worker cleanup side effect"); + assert!(checkpoint < cleanup); + + let workdir_start = source + .find("fn execute_workdir_removal_for_workspace_deletion") + .expect("Workdir removal helper"); + let workdir_source = &source[workdir_start..]; + let checkpoint = workdir_source + .find("append_workspace_deletion_child_operation") + .expect("Workdir child checkpoint"); + let cleanup = workdir_source + .find("execute_reserved_workdir_removal") + .expect("Workdir cleanup side effect"); + assert!(checkpoint < cleanup); + } + fn handler_source<'a>(source: &'a str, name: &str) -> &'a str { let start = source .find(&format!("async fn {name}")) @@ -16580,6 +17196,48 @@ mod tests { &tail[..end] } + #[test] + fn browser_worker_methods_receive_server_owned_account_source() { + let source = protocol::AuthenticatedInputSource::Account { + account_id: "account-1".into(), + }; + let method = authorize_browser_worker_method( + protocol::Method::Submit { + submission_request_id: "request-1".into(), + input: vec![protocol::Segment::text("hello")], + }, + &source, + ) + .unwrap(); + assert!(matches!( + method, + protocol::Method::SubmitTracked { + source: protocol::AuthenticatedInputSource::Account { ref account_id }, + .. + } if account_id == "account-1" + )); + assert!(authorize_browser_worker_method(method, &source).is_err()); + } + + #[test] + fn remote_worker_protocol_header_preserves_authenticated_account_source() { + let mut headers = HeaderMap::new(); + insert_authenticated_input_source_header( + &mut headers, + Some(&protocol::AuthenticatedInputSource::Account { + account_id: "account-1".into(), + }), + ) + .unwrap(); + + assert_eq!( + headers + .get(protocol::AUTHENTICATED_ACCOUNT_ID_HEADER) + .unwrap(), + "account-1" + ); + } + #[test] fn merge_request_http_paths_observe_refs_through_runtime_provider_authority() { let source = include_str!("server.rs"); @@ -16775,6 +17433,7 @@ mod tests { command: "printf ready; sleep 30".to_string(), timeout_secs: 60, output_limit: 4096, + cwd: None, spill_dir: None, tool_call_id: Some("tool-call-command-session".to_string()), }) @@ -16784,12 +17443,8 @@ mod tests { let mut registry = WorkdirSessionRegistry::default(); registry.insert_attachment(worker.clone(), source.clone()); let registered_source = registry.remove_attachment(&worker).unwrap(); - let external_handle = registry.register_command( - worker.clone(), - registered_source, - provider_handle.clone(), - Vec::new(), - ); + let external_handle = + registry.register_command(worker.clone(), registered_source, provider_handle.clone()); assert_ne!(external_handle, provider_handle); let refreshed: WorkdirSessionHandle = Arc::new(workdir::LocalWorkdirSession::new( @@ -16926,16 +17581,21 @@ mod tests { }); SqliteWorkspaceStore::open(&api.config.database_path) .unwrap() - .upsert_trusted_runtime(&TrustedRuntimeRecord { - runtime_id: runtime_id.to_owned(), - workspace_id: Some(api.workspace_id().to_owned()), - display_name: runtime_id.to_owned(), - base_url: "https://runtime.test".to_owned(), - public_key: identity.public_key.clone(), - created_at: "2026-01-01T00:00:00Z".to_owned(), - updated_at: "2026-01-01T00:00:00Z".to_owned(), - revoked_at: None, - }) + .upsert_workspace_runtime_binding( + WorkspaceRuntimeBinding { + workspace_id: api.workspace_id().to_owned(), + runtime_id: runtime_id.to_owned(), + display_name: runtime_id.to_owned(), + base_url: "https://runtime.test".to_owned(), + public_key: identity.public_key.clone(), + public_key_fingerprint: String::new(), + binding_revision: 1, + created_at: "2026-01-01T00:00:00Z".to_owned(), + updated_at: "2026-01-01T00:00:00Z".to_owned(), + revoked_at: None, + }, + false, + ) .unwrap(); } @@ -18763,7 +19423,6 @@ mod tests { request.worker_ref, self.backend_id(), ), - run_state: worker_runtime::execution::WorkerExecutionRunState::Idle, working_directory, } } @@ -18779,17 +19438,43 @@ mod tests { .push((handle.worker_ref().clone(), method)); worker_runtime::execution::WorkerExecutionResult::accepted( worker_runtime::execution::WorkerExecutionOperation::ProtocolMethod, - worker_runtime::execution::WorkerExecutionRunState::Idle, ) } + fn worker_snapshot( + &self, + handle: &worker_runtime::execution::WorkerExecutionHandle, + ) -> Option { + Some(protocol::Event::Snapshot { + session: protocol::SessionSnapshot { + pending_submissions: protocol::PendingSubmissionsSnapshot::default(), + entries: Vec::new(), + }, + greeting: protocol::Greeting { + worker_name: handle.worker_ref().worker_id.to_string(), + cwd: String::new(), + provider: "deterministic-workspace-server-test".to_string(), + model: "deterministic-workspace-server-test".to_string(), + scope_summary: "test execution snapshot".to_string(), + tools: Vec::new(), + context_window: 0, + context_tokens: 0, + }, + state: protocol::WorkerStateSnapshot::initial(1), + in_flight: protocol::InFlightSnapshot { + blocks: Vec::new(), + commands: Vec::new(), + }, + internal_workers: Vec::new(), + }) + } + fn stop_worker( &self, _handle: &worker_runtime::execution::WorkerExecutionHandle, ) -> worker_runtime::execution::WorkerExecutionResult { worker_runtime::execution::WorkerExecutionResult::accepted( worker_runtime::execution::WorkerExecutionOperation::Stop, - worker_runtime::execution::WorkerExecutionRunState::Stopped, ) } @@ -18799,7 +19484,6 @@ mod tests { ) -> worker_runtime::execution::WorkerExecutionResult { worker_runtime::execution::WorkerExecutionResult::accepted( worker_runtime::execution::WorkerExecutionOperation::Cancel, - worker_runtime::execution::WorkerExecutionRunState::Stopped, ) } @@ -18825,7 +19509,7 @@ mod tests { .get(handle.worker_ref()) .cloned() .expect("execution context"); - let submission_id = input.submission_id.clone(); + let submission_request_id = input.submission_request_id.clone(); let content = input.content.clone(); std::thread::spawn(move || { std::thread::sleep(std::time::Duration::from_millis(25)); @@ -18833,17 +19517,19 @@ mod tests { text: format!("server companion echoed: {content}"), }); }); - if let Some(submission_id) = submission_id { - worker_runtime::execution::WorkerExecutionResult::accepted_input_committed( + if let Some(submission_request_id) = submission_request_id { + worker_runtime::execution::WorkerExecutionResult::accepted_submission( worker_runtime::execution::WorkerExecutionOperation::Input, - worker_runtime::execution::WorkerExecutionRunState::Idle, - submission_id, + submission_request_id, + uuid::Uuid::now_v7().to_string(), + protocol::SubmissionDisposition::Started, ) + .with_worker_state(protocol::WorkerStateSnapshot::initial(1)) } else { worker_runtime::execution::WorkerExecutionResult::accepted( worker_runtime::execution::WorkerExecutionOperation::Input, - worker_runtime::execution::WorkerExecutionRunState::Idle, ) + .with_worker_state(protocol::WorkerStateSnapshot::initial(1)) } } } @@ -18901,13 +19587,59 @@ mod tests { ); } + #[test] + fn catalog_runtime_sources_are_scoped_by_workspace_and_runtime_id() { + let mut base = test_server_config(tempfile::tempdir().unwrap().path()); + base.remote_runtime_sources = vec![ + RemoteRuntimeConfig::new("shared", "A", "https://a.runtime.test", None) + .with_workspace_id("workspace-a"), + RemoteRuntimeConfig::new("shared", "B", "https://b.runtime.test", None) + .with_workspace_id("workspace-b"), + ]; + let workspace = WorkspaceRecord { + workspace_id: "workspace-b".to_string(), + display_name: "Workspace B".to_string(), + state: "active".to_string(), + owner_account_id: "owner-account".to_string(), + created_at: "1".to_string(), + updated_at: "1".to_string(), + }; + let source = workspace_api::RepositorySource { + kind: workspace_api::RepositorySourceKind::Https, + uri: "https://example.test/org/repository.git".to_string(), + }; + let repositories = vec![RepositoryRecord { + workspace_id: "workspace-b".to_string(), + repository_id: "main".to_string(), + repository_key: "main".to_string(), + kind: "git".to_string(), + provider: Some("git".to_string()), + source_fingerprint: crate::repository_source::repository_source_fingerprint(&source), + source, + default_ref: Some("main".to_string()), + source_revision: 1, + observed_status: workspace_api::RepositoryObservedStatus::Unverified, + observed_at: None, + created_at: "1".to_string(), + updated_at: "1".to_string(), + }]; + let scoped = base + .for_catalog_workspace(&workspace, repositories) + .unwrap(); + assert_eq!(scoped.remote_runtime_sources.len(), 1); + assert_eq!(scoped.remote_runtime_sources[0].runtime_id, "shared"); + assert_eq!( + scoped.remote_runtime_sources[0].base_url, + "https://b.runtime.test" + ); + } + fn test_server_config(workspace_root: impl Into) -> ServerConfig { let workspace_root = workspace_root.into(); let store_root = workspace_root.join(".test-embedded-runtime-store"); let mut config = ServerConfig::local_dev(workspace_root.clone(), test_identity()) .with_embedded_runtime_store_root(store_root); config.database_path = workspace_root.join(".test-yoi-server.db"); - config.runtime_config_path = Some(workspace_root.join(".test-config/runtimes.toml")); let source = workspace_api::RepositorySource { kind: workspace_api::RepositorySourceKind::LocalPath, uri: workspace_root.display().to_string(), @@ -22379,6 +23111,192 @@ mod tests { assert_eq!(detail.provenance.id, "workspace:triage-errors"); } + #[tokio::test] + async fn runtime_trust_management_is_owner_only_revisioned_and_redacted() { + let temp = tempfile::tempdir().unwrap(); + let api = test_api(temp.path()).await; + let owner_account_id = format!("account-{TEST_WORKSPACE_ID}"); + let owner = RequestActor { + user_id: "owner-user".to_string(), + account_id: owner_account_id.clone(), + handle: "owner".to_string(), + display_name: "Owner".to_string(), + auth_method: ActorAuthMethod::BrowserSession, + }; + let non_owner = RequestActor { + user_id: "other-user".to_string(), + account_id: "other-account".to_string(), + handle: "other".to_string(), + display_name: "Other".to_string(), + auth_method: ActorAuthMethod::ApiToken, + }; + let first = worker_runtime::auth::RuntimeIdentityMaterial::generate("runtime-a").unwrap(); + let second = worker_runtime::auth::RuntimeIdentityMaterial::generate("runtime-a").unwrap(); + let third = worker_runtime::auth::RuntimeIdentityMaterial::generate("runtime-a").unwrap(); + let now = Utc::now().to_rfc3339(); + api.store + .put_workspace_runtime_binding_key( + WorkspaceRuntimeBinding { + workspace_id: TEST_WORKSPACE_ID.to_string(), + runtime_id: "runtime-a".to_string(), + display_name: "Runtime A".to_string(), + base_url: "https://runtime.example".to_string(), + public_key: first.public_key, + public_key_fingerprint: String::new(), + binding_revision: 1, + created_at: now.clone(), + updated_at: now, + revoked_at: None, + }, + None, + &owner_account_id, + ) + .await + .unwrap(); + + let Json(detail) = scoped_get_runtime_detail( + State(api.clone()), + AxumPath(ScopedRuntimePath { + workspace_id: TEST_WORKSPACE_ID.to_string(), + runtime_id: "runtime-a".to_string(), + }), + ) + .await + .unwrap(); + assert_eq!(detail.trust_key.revision, Some(1)); + assert!(detail.trust_key.fingerprint.is_some()); + let Json(revealed) = scoped_reveal_runtime_trust_key( + State(api.clone()), + AxumPath(ScopedRuntimePath { + workspace_id: TEST_WORKSPACE_ID.to_string(), + runtime_id: "runtime-a".to_string(), + }), + Extension(owner.clone()), + ) + .await + .unwrap(); + assert!(revealed.public_key.starts_with("yoi-ed25519-pub:v1:")); + let denied_reveal = scoped_reveal_runtime_trust_key( + State(api.clone()), + AxumPath(ScopedRuntimePath { + workspace_id: TEST_WORKSPACE_ID.to_string(), + runtime_id: "runtime-a".to_string(), + }), + Extension(non_owner.clone()), + ) + .await + .unwrap_err(); + assert_eq!( + denied_reveal.into_response().status(), + StatusCode::FORBIDDEN + ); + + let response = scoped_put_runtime_trust_key( + State(api.clone()), + AxumPath(ScopedRuntimePath { + workspace_id: TEST_WORKSPACE_ID.to_string(), + runtime_id: "runtime-a".to_string(), + }), + Extension(owner.clone()), + Json(PutRuntimeTrustKeyRequest { + public_key: second.public_key, + expected_revision: Some(1), + }), + ) + .await + .unwrap(); + assert_eq!(response.status(), StatusCode::OK); + let body = axum::body::to_bytes(response.into_body(), usize::MAX) + .await + .unwrap(); + let detail: WorkspaceRuntimeDetail = serde_json::from_slice(&body).unwrap(); + assert_eq!(detail.trust_key.revision, Some(2)); + assert_eq!( + detail.recent_audit[0].action, + RuntimeTrustAuditAction::Replaced + ); + + let stale = scoped_put_runtime_trust_key( + State(api.clone()), + AxumPath(ScopedRuntimePath { + workspace_id: TEST_WORKSPACE_ID.to_string(), + runtime_id: "runtime-a".to_string(), + }), + Extension(owner.clone()), + Json(PutRuntimeTrustKeyRequest { + public_key: third.public_key, + expected_revision: Some(1), + }), + ) + .await + .unwrap(); + assert_eq!(stale.status(), StatusCode::CONFLICT); + let body = axum::body::to_bytes(stale.into_body(), usize::MAX) + .await + .unwrap(); + let conflict: RuntimeTrustConflictResponse = serde_json::from_slice(&body).unwrap(); + assert_eq!(conflict.error, RuntimeTrustConflictKind::StaleRevision); + assert_eq!(conflict.current_revision, Some(2)); + + let denied = scoped_revoke_runtime_trust_key( + State(api.clone()), + AxumPath(ScopedRuntimePath { + workspace_id: TEST_WORKSPACE_ID.to_string(), + runtime_id: "runtime-a".to_string(), + }), + Extension(non_owner), + Json(RevokeRuntimeTrustKeyRequest { + expected_revision: 2, + }), + ) + .await + .unwrap_err(); + assert_eq!(denied.into_response().status(), StatusCode::FORBIDDEN); + + let revoked = scoped_revoke_runtime_trust_key( + State(api.clone()), + AxumPath(ScopedRuntimePath { + workspace_id: TEST_WORKSPACE_ID.to_string(), + runtime_id: "runtime-a".to_string(), + }), + Extension(owner), + Json(RevokeRuntimeTrustKeyRequest { + expected_revision: 2, + }), + ) + .await + .unwrap(); + assert_eq!(revoked.status(), StatusCode::OK); + let binding = api + .store + .get_workspace_runtime_binding(TEST_WORKSPACE_ID, "runtime-a") + .await + .unwrap() + .unwrap(); + assert_eq!(binding.binding_revision, 3); + assert!(binding.revoked_at.is_some()); + let listed = workspace_runtime_resources_response(&api, TEST_WORKSPACE_ID) + .await + .unwrap(); + let listed_runtime = listed + .items + .iter() + .find(|resource| resource.runtime.runtime_id == "runtime-a") + .expect("revoked binding must remain listed"); + assert!(listed_runtime.management.config_managed); + let detail = workspace_runtime_detail(&api, TEST_WORKSPACE_ID, "runtime-a") + .await + .unwrap(); + assert_eq!(detail.trust_key.status, RuntimeTrustKeyStatus::Revoked); + assert!(detail.runtime.management.config_managed); + assert!( + !api.runtime_binding_expectations + .read() + .unwrap() + .contains_key(&(TEST_WORKSPACE_ID.to_string(), "runtime-a".to_string())) + ); + } + #[tokio::test] async fn repository_secret_management_is_owner_only() { let temp = tempfile::tempdir().unwrap(); @@ -22426,6 +23344,16 @@ mod tests { test_api_with_recording_backend(workspace_root).await.0 } + fn test_owner_actor() -> RequestActor { + RequestActor { + user_id: "owner-user".to_string(), + account_id: format!("account-{TEST_WORKSPACE_ID}"), + handle: "owner".to_string(), + display_name: "Owner".to_string(), + auth_method: ActorAuthMethod::BrowserSession, + } + } + fn test_repository_id(api: &WorkspaceApi) -> String { api.store .get_repository_by_key(TEST_WORKSPACE_ID, "test-repository") @@ -22906,17 +23834,21 @@ mod tests { }) .await .unwrap(); - let trust = crate::store::TrustedRuntimeRecord { + let trust = crate::store::WorkspaceRuntimeBinding { + workspace_id: TEST_WORKSPACE_ID.to_string(), runtime_id: "runtime-remote".to_string(), - workspace_id: Some(TEST_WORKSPACE_ID.to_string()), display_name: "Remote Runtime".to_string(), base_url: "https://runtime.invalid".to_string(), public_key: identity.public_key.clone(), + public_key_fingerprint: String::new(), + binding_revision: 1, created_at: "2026-08-11T00:00:00Z".to_string(), updated_at: "2026-08-11T00:00:00Z".to_string(), revoked_at: None, }; - store.upsert_trusted_runtime(&trust).unwrap(); + store + .upsert_workspace_runtime_binding(trust.clone(), false) + .unwrap(); let api = WorkspaceApi::new_with_execution_backend( config, Arc::new(store), @@ -23179,7 +24111,9 @@ mod tests { let mut revoked = trust; revoked.revoked_at = Some("2026-08-11T00:01:00Z".to_string()); let authority = SqliteWorkspaceStore::open(api.config.database_path.clone()).unwrap(); - authority.upsert_trusted_runtime(&revoked).unwrap(); + authority + .upsert_workspace_runtime_binding(revoked, true) + .unwrap(); let revoked_token = signer .issue_worker_remove( "server-main", @@ -23190,16 +24124,20 @@ mod tests { 60, ) .unwrap(); - assert!(matches!( - crate::worker_source::verify_worker_remove_source( - &api, - crate::worker_source::PresentedWorkerMutationSourceProof::Remote(&revoked_token), - "runtime-target", - "target-worker", - ) - .await, - Err(crate::worker_source::WorkerMutationSourceProofError::RevokedRuntimeTrust) - )); + let revoked_result = crate::worker_source::verify_worker_remove_source( + &api, + crate::worker_source::PresentedWorkerMutationSourceProof::Remote(&revoked_token), + "runtime-target", + "target-worker", + ) + .await; + assert!( + matches!( + revoked_result, + Err(crate::worker_source::WorkerMutationSourceProofError::RevokedRuntimeTrust) + ), + "unexpected revoked trust result: {revoked_result:?}" + ); } fn seed_worker_source_member(api: &WorkspaceApi, runtime_id: &str, worker_id: &str) { @@ -23503,30 +24441,6 @@ mod tests { assert_eq!(response.status(), StatusCode::BAD_REQUEST); } - #[test] - fn delegated_workdir_session_fence_rejects_reattached_link() { - let first = WorkerWorkdirLinkRecord { - workspace_id: "workspace-a".to_string(), - worker: workdir::workspace::RuntimeWorkerRef::new("runtime-a", "worker-a"), - workdir_id: "workdir-a".to_string(), - role: "primary".to_string(), - linked_at: "2026-01-01T00:00:00Z".to_string(), - unlinked_at: None, - }; - let expected = current_worker_workdir_session_fence(&first); - assert!(validate_current_worker_workdir_session_fence(&first, None).is_ok()); - assert!(validate_current_worker_workdir_session_fence(&first, Some(&expected)).is_ok()); - - let reattached = WorkerWorkdirLinkRecord { - linked_at: "2026-01-01T00:00:01Z".to_string(), - ..first - }; - assert!(matches!( - validate_current_worker_workdir_session_fence(&reattached, Some(&expected)), - Err(Error::WorkdirAttachmentConflict(_)) - )); - } - #[tokio::test] async fn backend_workdir_session_proxy_executes_typed_operations() { use manifest::Scope; @@ -24337,6 +25251,91 @@ mod tests { axum::serve(listener, proxy).await } + async fn runtime_ping_stub( + status: StatusCode, + body: serde_json::Value, + ) -> (String, tokio::task::JoinHandle<()>) { + async fn ping( + State((status, body)): State<(StatusCode, serde_json::Value)>, + headers: HeaderMap, + ) -> (StatusCode, Json) { + assert_eq!( + headers + .get(worker_runtime::http_server::RUNTIME_WORKSPACE_SCOPE_HEADER) + .and_then(|value| value.to_str().ok()), + Some(TEST_WORKSPACE_ID) + ); + assert!( + headers + .get(axum::http::header::AUTHORIZATION) + .and_then(|value| value.to_str().ok()) + .is_some_and(|value| value.starts_with("Bearer ")) + ); + (status, Json(body)) + } + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0") + .await + .expect("bind ping stub"); + let base_url = format!("http://{}", listener.local_addr().expect("ping stub addr")); + let app = Router::new() + .route("/v1/ping", axum::routing::get(ping)) + .with_state((status, body)); + let server = tokio::spawn(async move { + axum::serve(listener, app).await.expect("serve ping stub"); + }); + (base_url, server) + } + + async fn test_app_with_remote_runtime( + workspace_root: impl Into, + runtime_id: &str, + endpoint: String, + ) -> Router { + let api = test_api(workspace_root).await; + api.store + .upsert_workspace_runtime_binding_record( + WorkspaceRuntimeBinding { + workspace_id: TEST_WORKSPACE_ID.to_string(), + runtime_id: runtime_id.to_string(), + display_name: "Probe Runtime".to_string(), + base_url: endpoint.clone(), + public_key: RuntimeIdentityMaterial::generate(runtime_id) + .unwrap() + .public_key, + public_key_fingerprint: String::new(), + binding_revision: 1, + created_at: "1".to_string(), + updated_at: "1".to_string(), + revoked_at: None, + }, + false, + ) + .await + .unwrap(); + api.runtime.register_or_replace( + RemoteWorkerRuntime::new( + RemoteRuntimeConfig { + runtime_id: runtime_id.to_string(), + workspace_id: Some(TEST_WORKSPACE_ID.to_string()), + display_name: "Probe Runtime".to_string(), + base_url: endpoint, + bearer_token: Some("test-connection-token".to_string()), + auth: None, + cached_worker_creation_available: true, + cached_os: "linux".to_string(), + cached_arch: "x86_64".to_string(), + cached_status: "active".to_string(), + timeout: std::time::Duration::from_secs(2), + }, + TEST_WORKSPACE_ID.to_string(), + "http://127.0.0.1:1".to_string(), + ) + .unwrap(), + ); + build_inner_router(api) + } + async fn test_app(workspace_root: impl Into) -> Router { build_inner_router(test_api(workspace_root).await) } @@ -24431,7 +25430,7 @@ mod tests { .await; assert!(matches!( result, - Err(crate::worker_source::WorkerMutationSourceProofError::WrongWorkspace) + Err(crate::worker_source::WorkerMutationSourceProofError::RevokedRuntimeTrust) )); } @@ -24596,6 +25595,7 @@ mod tests { workspace_id: Some(TEST_WORKSPACE_ID.to_string()), }, state: "idle".to_string(), + worker_state: None, last_seen_at: None, pinned: false, retention_state: "normal".to_string(), @@ -24692,6 +25692,7 @@ mod tests { workspace_id: Some(TEST_WORKSPACE_ID.to_string()), }, state: "idle".to_string(), + worker_state: None, last_seen_at: None, pinned: false, retention_state: "normal".to_string(), @@ -25413,7 +26414,35 @@ mod tests { #[tokio::test] async fn runtime_rest_resource_create_list_and_delete_apply_live_registry() { let dir = tempfile::tempdir().unwrap(); - let app = test_app(dir.path()).await; + let api = test_api(dir.path()).await; + let store = api.store.clone(); + let identity = RuntimeIdentityMaterial::generate("team-runtime").unwrap(); + let binding = WorkspaceRuntimeBinding { + workspace_id: TEST_WORKSPACE_ID.to_string(), + runtime_id: "team-runtime".to_string(), + display_name: "Team Runtime".to_string(), + base_url: "https://runtime.example.invalid".to_string(), + public_key: identity.public_key, + public_key_fingerprint: String::new(), + binding_revision: 1, + created_at: "1".to_string(), + updated_at: "1".to_string(), + revoked_at: None, + }; + store + .upsert_workspace_runtime_binding_record(binding.clone(), false) + .await + .unwrap(); + let remote = remote_runtime_config_from_binding(&binding).unwrap(); + api.runtime.register_or_replace( + RemoteWorkerRuntime::new( + remote, + TEST_WORKSPACE_ID.to_string(), + "http://127.0.0.1:8787".to_string(), + ) + .unwrap(), + ); + let app = build_inner_router(api.clone()).layer(Extension(test_owner_actor())); let runtimes_uri = format!("/api/w/{TEST_WORKSPACE_ID}/runtimes"); let initial = get_json(app.clone(), &runtimes_uri).await; @@ -25442,34 +26471,44 @@ mod tests { ) .await; - let added = request_json( + let rejected = request_json( app.clone(), "POST", &runtimes_uri, Some(serde_json::json!({ - "runtime_id": "team-runtime", - "display_name": "Team Runtime", - "endpoint": "https://runtime.example.invalid" + "runtime_id": "keyless-runtime", + "display_name": "Keyless Runtime", + "endpoint": "https://keyless.runtime.invalid" })), - StatusCode::CREATED, + StatusCode::BAD_REQUEST, ) .await; - assert_eq!(added["runtime_id"], "team-runtime"); + assert!( + rejected["message"] + .as_str() + .unwrap_or_default() + .contains("runtime_public_key_required") + ); + let listed = get_json(app.clone(), &runtimes_uri).await; + let added = listed["items"] + .as_array() + .unwrap() + .iter() + .find(|runtime| runtime["runtime_id"] == "team-runtime") + .unwrap(); assert_eq!(added["management"]["config_managed"], true); assert_eq!(added["management"]["endpoint_configured"], true); let projected = serde_json::to_string(&added).unwrap(); assert!(!projected.contains("runtime.example.invalid")); - let persisted = BackendRuntimesConfigFile::load_from_path( - dir.path().join(".test-config/runtimes.toml"), - ) - .unwrap(); - assert_eq!(persisted.runtimes.remote.len(), 1); - assert_eq!(persisted.runtimes.remote[0].id, "team-runtime"); - assert_eq!( - persisted.runtimes.remote[0].endpoint, - "https://runtime.example.invalid" - ); + let persisted = store + .get_workspace_runtime_binding(TEST_WORKSPACE_ID, "team-runtime") + .await + .unwrap() + .unwrap(); + assert_eq!(persisted.runtime_id, "team-runtime"); + assert_eq!(persisted.base_url, "https://runtime.example.invalid"); + assert!(persisted.revoked_at.is_none()); let launch_options = get_json(app.clone(), "/api/workers/launch-options").await; let runtimes = launch_options["runtimes"].as_array().unwrap(); @@ -25484,6 +26523,16 @@ mod tests { .expect("team runtime launch option"); assert_eq!(team_runtime["working_directory_required"], true); + api.store + .revoke_workspace_runtime_binding_key( + TEST_WORKSPACE_ID, + "team-runtime", + 1, + &format!("account-{TEST_WORKSPACE_ID}"), + &Utc::now().to_rfc3339(), + ) + .await + .unwrap(); let deleted = request_json( app.clone(), "DELETE", @@ -25501,11 +26550,12 @@ mod tests { .iter() .any(|runtime| runtime["runtime_id"] == "team-runtime") ); - let persisted = BackendRuntimesConfigFile::load_from_path( - dir.path().join(".test-config/runtimes.toml"), - ) - .unwrap(); - assert!(persisted.runtimes.remote.is_empty()); + let persisted = store + .get_workspace_runtime_binding(TEST_WORKSPACE_ID, "team-runtime") + .await + .unwrap() + .unwrap(); + assert!(persisted.revoked_at.is_some()); } #[tokio::test(flavor = "multi_thread")] @@ -25523,20 +26573,45 @@ mod tests { }); let dir = tempfile::tempdir().unwrap(); - let app = test_app(dir.path()).await; - let added = request_json( - app.clone(), - "POST", - &format!("/api/w/{TEST_WORKSPACE_ID}/runtimes"), - Some(serde_json::json!({ - "runtime_id": "busy-runtime", - "display_name": "Busy Runtime", - "endpoint": format!("http://{runtime_addr}") - })), - StatusCode::CREATED, - ) - .await; - assert_eq!(added["runtime_id"], "busy-runtime"); + let endpoint = format!("http://{runtime_addr}"); + let api = test_api(dir.path()).await; + let binding = WorkspaceRuntimeBinding { + workspace_id: TEST_WORKSPACE_ID.to_string(), + runtime_id: "busy-runtime".to_string(), + display_name: "Busy Runtime".to_string(), + base_url: endpoint, + public_key: RuntimeIdentityMaterial::generate("busy-runtime") + .unwrap() + .public_key, + public_key_fingerprint: String::new(), + binding_revision: 1, + created_at: "1".to_string(), + updated_at: "1".to_string(), + revoked_at: None, + }; + api.store + .upsert_workspace_runtime_binding_record(binding.clone(), false) + .await + .unwrap(); + api.runtime.register_or_replace( + RemoteWorkerRuntime::new( + remote_runtime_config_from_binding(&binding).unwrap(), + TEST_WORKSPACE_ID.to_string(), + "http://127.0.0.1:8787".to_string(), + ) + .unwrap(), + ); + api.store + .revoke_workspace_runtime_binding_key( + TEST_WORKSPACE_ID, + "busy-runtime", + 1, + &format!("account-{TEST_WORKSPACE_ID}"), + &Utc::now().to_rfc3339(), + ) + .await + .unwrap(); + let app = build_inner_router(api).layer(Extension(test_owner_actor())); let workers = get_json(app.clone(), "/api/workers").await; assert!( workers["items"] @@ -25568,131 +26643,143 @@ mod tests { .iter() .any(|diagnostic| { diagnostic["code"] == "remote_runtime_delete_blocked" }) ); - let persisted = BackendRuntimesConfigFile::load_from_path( - dir.path().join(".test-config/runtimes.toml"), - ) - .unwrap(); - assert_eq!(persisted.runtimes.remote.len(), 1); } - #[tokio::test(flavor = "multi_thread")] - async fn runtime_connection_test_reports_compatible_with_unknown_warnings_without_endpoint_leak() - { - let (runtime, _worker_ref) = runtime_with_worker(); - let runtime_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let runtime_addr = runtime_listener.local_addr().unwrap(); - tokio::spawn({ - let runtime = runtime.clone(); - async move { - serve_runtime_http_with_injected_test_auth(runtime, runtime_listener) - .await - .unwrap() - } - }); - + async fn run_runtime_connection_test( + body: serde_json::Value, + status: StatusCode, + ) -> serde_json::Value { + let (endpoint, _server) = runtime_ping_stub(status, body).await; let dir = tempfile::tempdir().unwrap(); - let endpoint = format!("http://{runtime_addr}"); - BackendRuntimesConfigFile { - runtimes: WorkspaceBackendRuntimesConfig { - remote: vec![RemoteRuntimeConfigFile { - id: "probe-runtime".to_string(), - endpoint: endpoint.clone(), - display_name: Some("Probe Runtime".to_string()), - token_ref: None, - }], - }, - } - .write_to_path(dir.path().join(".test-config/runtimes.toml")) - .unwrap(); - let app = test_app(dir.path()).await; - - let response = post_json( + let app = test_app_with_remote_runtime(dir.path(), "probe-runtime", endpoint).await; + post_json( app, &format!("/api/w/{TEST_WORKSPACE_ID}/runtimes/probe-runtime/connection-tests"), serde_json::json!({}), ) - .await; - assert_eq!(response["state"], "compatible"); - let capabilities = response["capabilities"].as_array().unwrap(); - assert!( - capabilities - .iter() - .any(|value| value == "runtime.summary:available") - ); - assert!( - capabilities - .iter() - .any(|value| value == "workers.list:available") - ); - assert!( - capabilities - .iter() - .any(|value| value == "workers.spawn:available") - ); - assert!( - response["diagnostics"] - .as_array() - .unwrap() - .iter() - .any(|diagnostic| { diagnostic["code"] == "workers.spawn.available" }) - ); - let projected = serde_json::to_string(&response).unwrap(); - assert!(!projected.contains(&endpoint)); - assert!(!projected.contains(&runtime_addr.to_string())); - assert_eq!(response["protocol_version"], serde_json::Value::Null); + .await } #[tokio::test(flavor = "multi_thread")] - async fn runtime_connection_test_marks_missing_execution_backend_incompatible() { - let runtime = - worker_runtime::Runtime::with_options(worker_runtime::RuntimeOptions::default()); - let runtime_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); - let runtime_addr = runtime_listener.local_addr().unwrap(); - tokio::spawn(async move { - serve_runtime_http_with_injected_test_auth(runtime, runtime_listener) - .await - .unwrap() - }); - - let dir = tempfile::tempdir().unwrap(); - let endpoint = format!("http://{runtime_addr}"); - BackendRuntimesConfigFile { - runtimes: WorkspaceBackendRuntimesConfig { - remote: vec![RemoteRuntimeConfigFile { - id: "control-only-runtime".to_string(), - display_name: Some("Control-only Runtime".to_string()), - endpoint, - token_ref: None, - }], - }, - } - .write_to_path(dir.path().join(".test-config/runtimes.toml")) - .unwrap(); - let app = test_app(dir.path()).await; - - let response = post_json( - app, - &format!("/api/w/{TEST_WORKSPACE_ID}/runtimes/control-only-runtime/connection-tests"), - serde_json::json!({}), + async fn runtime_connection_test_reports_exact_compatible_protocol() { + let response = run_runtime_connection_test( + serde_json::json!({ + "runtime_id": "probe-runtime", + "protocol_version": RUNTIME_HTTP_PROTOCOL_VERSION, + }), + StatusCode::OK, ) .await; - assert_eq!(response["state"], "incompatible"); - assert!( - response["capabilities"] - .as_array() - .unwrap() - .iter() - .any(|value| { value == "workers.spawn:incompatible" }) + + assert_eq!(response["status"], "compatible"); + assert_eq!(response["failure_kind"], serde_json::Value::Null); + assert_eq!( + response["expected_protocol_version"], + RUNTIME_HTTP_PROTOCOL_VERSION ); - assert!( - response["diagnostics"] - .as_array() - .unwrap() - .iter() - .any(|diagnostic| { - diagnostic["code"] == "remote_runtime_worker_creation_unavailable" - }) + assert_eq!( + response["actual_protocol_version"], + RUNTIME_HTTP_PROTOCOL_VERSION ); + assert_eq!(response["diagnostics"], serde_json::json!([])); + let projected = serde_json::to_string(&response).unwrap(); + assert!(!projected.contains("Bearer")); + assert!(!projected.contains("public_key")); + } + + #[tokio::test(flavor = "multi_thread")] + async fn runtime_connection_test_rejects_newer_protocol() { + let newer = RUNTIME_HTTP_PROTOCOL_MAX_VERSION + 1; + let response = run_runtime_connection_test( + serde_json::json!({ + "runtime_id": "probe-runtime", + "protocol_version": newer, + }), + StatusCode::OK, + ) + .await; + + assert_eq!(response["status"], "failed"); + assert_eq!(response["failure_kind"], "protocol_version_mismatch"); + assert_eq!(response["actual_protocol_version"], newer); + assert_eq!( + response["diagnostics"][0]["code"], + "runtime_ping_protocol_newer" + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn runtime_connection_test_rejects_older_protocol() { + let older = RUNTIME_HTTP_PROTOCOL_MIN_VERSION.saturating_sub(1); + let response = run_runtime_connection_test( + serde_json::json!({ + "runtime_id": "probe-runtime", + "protocol_version": older, + }), + StatusCode::OK, + ) + .await; + + assert_eq!(response["status"], "failed"); + assert_eq!(response["failure_kind"], "protocol_version_mismatch"); + assert_eq!(response["actual_protocol_version"], older); + assert_eq!( + response["diagnostics"][0]["code"], + "runtime_ping_protocol_older" + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn runtime_connection_test_classifies_authentication_failure() { + let response = run_runtime_connection_test( + serde_json::json!({"error": "credential details must not escape"}), + StatusCode::UNAUTHORIZED, + ) + .await; + + assert_eq!(response["status"], "failed"); + assert_eq!(response["failure_kind"], "authentication"); + assert_eq!(response["actual_protocol_version"], serde_json::Value::Null); + let projected = serde_json::to_string(&response).unwrap(); + assert!(!projected.contains("credential details")); + } + + #[tokio::test(flavor = "multi_thread")] + async fn runtime_connection_test_rejects_malformed_ping_response() { + let response = run_runtime_connection_test( + serde_json::json!({ + "runtime_id": "probe-runtime", + "protocol_version": "not-a-number", + "unexpected": true, + }), + StatusCode::OK, + ) + .await; + + assert_eq!(response["status"], "failed"); + assert_eq!(response["failure_kind"], "malformed_response"); + assert_eq!( + response["diagnostics"][0]["code"], + "runtime_ping_malformed_response" + ); + } + + #[tokio::test(flavor = "multi_thread")] + async fn runtime_connection_test_rejects_runtime_identity_mismatch() { + let response = run_runtime_connection_test( + serde_json::json!({ + "runtime_id": "different-runtime", + "protocol_version": RUNTIME_HTTP_PROTOCOL_VERSION, + }), + StatusCode::OK, + ) + .await; + + assert_eq!(response["status"], "failed"); + assert_eq!(response["failure_kind"], "runtime_identity_mismatch"); + assert_eq!(response["actual_protocol_version"], serde_json::Value::Null); + let projected = serde_json::to_string(&response).unwrap(); + assert!(!projected.contains("different-runtime")); } #[tokio::test] @@ -27223,6 +28310,16 @@ mod tests { (runtime, worker_ref, endpoint) } + fn test_browser_request_actor() -> RequestActor { + RequestActor { + user_id: "test-user".into(), + account_id: format!("account-{TEST_WORKSPACE_ID}"), + handle: "test".into(), + display_name: "Test".into(), + auth_method: ActorAuthMethod::BrowserSession, + } + } + async fn spawn_workspace_proxy( source: RuntimeObservationSourceConfig, ) -> (String, tempfile::TempDir) { @@ -27241,11 +28338,8 @@ mod tests { .unwrap(); let app_listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let app_addr = app_listener.local_addr().unwrap(); - tokio::spawn(async move { - axum::serve(app_listener, build_inner_router(api)) - .await - .unwrap() - }); + let app = build_inner_router(api).layer(Extension(test_browser_request_actor())); + tokio::spawn(async move { axum::serve(app_listener, app).await.unwrap() }); ( format!("ws://{app_addr}/api/runtimes/{runtime_id}/workers/{worker_id}/protocol/ws"), dir, @@ -27257,7 +28351,8 @@ mod tests { let dir = tempfile::tempdir().unwrap(); let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); - let app = build_inner_router(test_api(dir.path()).await); + let app = build_inner_router(test_api(dir.path()).await) + .layer(Extension(test_browser_request_actor())); let server = tokio::spawn(async move { let _ = axum::serve(listener, app).await; }); @@ -27305,7 +28400,7 @@ mod tests { let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); let address = listener.local_addr().unwrap(); - let app = build_inner_router(api); + let app = build_inner_router(api).layer(Extension(test_browser_request_actor())); let server = tokio::spawn(async move { let _ = axum::serve(listener, app).await; }); @@ -27488,7 +28583,13 @@ mod tests { protocol::subscription::SubscriptionFramePayload::WorkerProtocol( protocol::subscription::SubscriptionWorkerProtocolMethod { subscription_id: second_protocol_subscription_id, - method: protocol::Method::Resume, + method: protocol::Method::Resume { + command: protocol::WorkerCommandEnvelope { + command_id: 1, + expected_execution_generation: 1, + expected_worker_state_revision: 0, + }, + }, }, ), ); @@ -27505,7 +28606,7 @@ mod tests { .iter() .any(|(worker_ref, method)| { worker_ref.worker_id.to_string() == worker_id - && matches!(method, protocol::Method::Resume) + && matches!(method, protocol::Method::Resume { .. }) }) { break; @@ -27518,7 +28619,7 @@ mod tests { let protocol_methods = execution_backend.protocol_methods(); assert!(protocol_methods.iter().any(|(worker_ref, method)| { worker_ref.worker_id.to_string() == worker_id - && matches!(method, protocol::Method::Resume) + && matches!(method, protocol::Method::Resume { .. }) })); server.abort(); } diff --git a/crates/workspace-server/src/store.rs b/crates/workspace-server/src/store.rs index efca949f..bd934788 100644 --- a/crates/workspace-server/src/store.rs +++ b/crates/workspace-server/src/store.rs @@ -1,3 +1,4 @@ +use std::collections::BTreeSet; use std::path::Path; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -12,9 +13,12 @@ use uuid::Uuid; use worker_runtime::identity::{RuntimeWorkerRef, WorkerId}; use workspace_api::{RepositoryObservedStatus, RepositorySource}; +use crate::workspace_deletion::WorkspaceDeletionStore; use crate::{Error, Result}; -const LATEST_SCHEMA_VERSION: i64 = 50; +const PREVIOUS_SCHEMA_VERSION: i64 = 52; +const LATEST_SCHEMA_VERSION: i64 = 53; +const WORKSPACE_DELETION_MIGRATION_NAME: &str = "durable Workspace deletion operations"; const MIGRATIONS: &[Migration] = &[Migration { version: LATEST_SCHEMA_VERSION, @@ -96,17 +100,47 @@ pub struct WorkspaceBootstrapResult { } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct TrustedRuntimeRecord { +pub struct WorkspaceRuntimeBinding { + pub workspace_id: String, pub runtime_id: String, - pub workspace_id: Option, pub display_name: String, pub base_url: String, pub public_key: String, + pub public_key_fingerprint: String, + pub binding_revision: u64, pub created_at: String, pub updated_at: String, pub revoked_at: Option, } +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WorkspaceRuntimeBindingUpsert { + Created, + Unchanged, + Replaced, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum WorkspaceRuntimeBindingMutation { + Created, + Unchanged, + Replaced, + Reactivated, + Revoked, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct WorkspaceRuntimeBindingAuditRecord { + pub workspace_id: String, + pub runtime_id: String, + pub actor_account_id: String, + pub action: String, + pub old_fingerprint: Option, + pub new_fingerprint: Option, + pub binding_revision: u64, + pub at: String, +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct AccountRecord { pub account_id: String, @@ -513,7 +547,7 @@ impl WorkspaceResourceKind { } #[async_trait] -pub trait ControlPlaneStore: Send + Sync { +pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore { async fn schema_version(&self) -> Result; fn resource_key( &self, @@ -533,10 +567,52 @@ pub trait ControlPlaneStore: Send + Sync { &self, record: &WorkspaceBootstrapRecord, ) -> Result; - async fn get_trusted_runtime(&self, runtime_id: &str) -> Result>; - async fn upsert_trusted_runtime_record(&self, record: &TrustedRuntimeRecord) -> Result<()>; + fn workspace_runtime_binding_matches(&self, expected: &WorkspaceRuntimeBinding) + -> Result; + async fn get_workspace_runtime_binding( + &self, + workspace_id: &str, + runtime_id: &str, + ) -> Result>; + async fn list_workspace_runtime_bindings( + &self, + workspace_id: &str, + include_revoked: bool, + ) -> Result>; + async fn upsert_workspace_runtime_binding_record( + &self, + record: WorkspaceRuntimeBinding, + replace: bool, + ) -> Result; + async fn revoke_workspace_runtime_binding_record( + &self, + workspace_id: &str, + runtime_id: &str, + revoked_at: &str, + ) -> Result; + async fn put_workspace_runtime_binding_key( + &self, + record: WorkspaceRuntimeBinding, + expected_revision: Option, + actor_account_id: &str, + ) -> Result<(WorkspaceRuntimeBindingMutation, WorkspaceRuntimeBinding)>; + async fn revoke_workspace_runtime_binding_key( + &self, + workspace_id: &str, + runtime_id: &str, + expected_revision: u64, + actor_account_id: &str, + revoked_at: &str, + ) -> Result<(WorkspaceRuntimeBindingMutation, WorkspaceRuntimeBinding)>; + async fn list_workspace_runtime_binding_audit( + &self, + workspace_id: &str, + runtime_id: &str, + limit: usize, + ) -> Result>; async fn consume_worker_mutation_source_jti( &self, + workspace_id: &str, runtime_id: &str, jti: &str, expires_at: u64, @@ -1058,6 +1134,18 @@ impl SqliteWorkspaceStore { f(&mut conn) } + pub(crate) fn with_transaction( + &self, + f: impl FnOnce(&rusqlite::Transaction<'_>) -> Result, + ) -> Result { + self.with_conn_mut(|conn| { + let tx = conn.transaction()?; + let value = f(&tx)?; + tx.commit()?; + Ok(value) + }) + } + pub(crate) fn get_workspace_memory_settings( &self, workspace_id: &str, @@ -1176,7 +1264,7 @@ impl SqliteWorkspaceStore { let existing = tx .query_row( "SELECT worker_id, runtime_id, request_fingerprint, create_fingerprint, \ - memory_settings_revision, memory_language \ + memory_settings_revision, memory_language, state \ FROM worker_create_reservations \ WHERE workspace_id = ?1 AND allocation_key = ?2", params![workspace_id, allocation_key], @@ -1188,11 +1276,17 @@ impl SqliteWorkspaceStore { row.get::<_, String>(3)?, row.get::<_, Option>(4)?, row.get::<_, Option>(5)?, + row.get::<_, String>(6)?, )) }, ) .optional()?; - if let Some((worker_id, reserved_runtime_id, stored_request_fingerprint, create_fingerprint, revision, language)) = existing { + if let Some((worker_id, reserved_runtime_id, stored_request_fingerprint, create_fingerprint, revision, language, state)) = existing { + if state == "removed" { + return Err(Error::InvalidInput(format!( + "Worker create allocation {allocation_key} was terminally removed" + ))); + } if reserved_runtime_id != runtime_id || stored_request_fingerprint.as_deref() != Some(request_fingerprint) { @@ -1303,7 +1397,7 @@ impl SqliteWorkspaceStore { let changed = conn.execute( "UPDATE worker_create_reservations \ SET state = 'created', updated_at = ?3 \ - WHERE workspace_id = ?1 AND worker_id = ?2", + WHERE workspace_id = ?1 AND worker_id = ?2 AND state IN ('reserved', 'created')", params![ workspace_id, worker_id.to_string(), @@ -1337,62 +1431,454 @@ impl SqliteWorkspaceStore { }) } - pub fn upsert_trusted_runtime(&self, record: &TrustedRuntimeRecord) -> Result<()> { + pub fn list_workspace_runtime_bindings( + &self, + workspace_id: &str, + include_revoked: bool, + ) -> Result> { + validate_identifier("workspace_id", workspace_id)?; self.with_conn(|conn| { - conn.execute( - r#"INSERT INTO trusted_runtime_records ( - runtime_id, workspace_id, display_name, base_url, public_key, created_at, updated_at, revoked_at - ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8) - ON CONFLICT(runtime_id) DO UPDATE SET - display_name = excluded.display_name, - base_url = excluded.base_url, - public_key = excluded.public_key, - updated_at = excluded.updated_at, - revoked_at = excluded.revoked_at"#, + let sql = if include_revoked { + r#"SELECT workspace_id, runtime_id, display_name, base_url, public_key, + public_key_fingerprint, binding_revision, created_at, updated_at, revoked_at + FROM workspace_runtime_bindings + WHERE workspace_id = ?1 + ORDER BY runtime_id ASC"# + } else { + r#"SELECT workspace_id, runtime_id, display_name, base_url, public_key, + public_key_fingerprint, binding_revision, created_at, updated_at, revoked_at + FROM workspace_runtime_bindings + WHERE workspace_id = ?1 AND revoked_at IS NULL + ORDER BY runtime_id ASC"# + }; + let mut stmt = conn.prepare(sql)?; + let rows = stmt.query_map(params![workspace_id], read_workspace_runtime_binding)?; + rows.collect::, _>>() + .map_err(Error::from) + }) + } + + pub fn get_workspace_runtime_binding( + &self, + workspace_id: &str, + runtime_id: &str, + ) -> Result> { + validate_identifier("workspace_id", workspace_id)?; + validate_identifier("runtime_id", runtime_id)?; + self.with_conn(|conn| { + conn.query_row( + r#"SELECT workspace_id, runtime_id, display_name, base_url, public_key, + public_key_fingerprint, binding_revision, created_at, updated_at, revoked_at + FROM workspace_runtime_bindings + WHERE workspace_id = ?1 AND runtime_id = ?2"#, + params![workspace_id, runtime_id], + read_workspace_runtime_binding, + ) + .optional() + .map_err(Error::from) + }) + } + + pub fn upsert_workspace_runtime_binding( + &self, + mut record: WorkspaceRuntimeBinding, + replace: bool, + ) -> Result { + validate_identifier("workspace_id", &record.workspace_id)?; + validate_identifier("runtime_id", &record.runtime_id)?; + validate_non_empty("runtime display_name", &record.display_name)?; + validate_runtime_base_url(&record.base_url)?; + normalize_workspace_runtime_binding_key(&mut record)?; + self.with_conn_mut(|conn| { + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let existing = tx + .query_row( + r#"SELECT workspace_id, runtime_id, display_name, base_url, public_key, + public_key_fingerprint, binding_revision, created_at, updated_at, revoked_at + FROM workspace_runtime_bindings + WHERE workspace_id = ?1 AND runtime_id = ?2"#, + params![record.workspace_id, record.runtime_id], + read_workspace_runtime_binding, + ) + .optional()?; + if let Some(existing) = existing { + let exact_active_match = existing.revoked_at.is_none() + && record.revoked_at.is_none() + && existing.display_name == record.display_name + && existing.base_url == record.base_url + && existing.public_key == record.public_key + && existing.public_key_fingerprint == record.public_key_fingerprint; + if exact_active_match { + tx.commit()?; + return Ok(WorkspaceRuntimeBindingUpsert::Unchanged); + } + if !replace { + return Err(Error::RuntimeBindingConflict(format!( + "binding {}/{} already exists with different endpoint, trust, or lifecycle state; retry with explicit replacement", + record.workspace_id, record.runtime_id + ))); + } + tx.execute( + r#"UPDATE workspace_runtime_bindings + SET display_name = ?3, base_url = ?4, public_key = ?5, + public_key_fingerprint = ?6, binding_revision = binding_revision + 1, + updated_at = ?7, revoked_at = ?8 + WHERE workspace_id = ?1 AND runtime_id = ?2"#, + params![ + record.workspace_id, + record.runtime_id, + record.display_name, + record.base_url, + record.public_key, + record.public_key_fingerprint, + record.updated_at, + record.revoked_at, + ], + ) + .map_err(map_runtime_binding_write_error)?; + tx.commit()?; + return Ok(WorkspaceRuntimeBindingUpsert::Replaced); + } + tx.execute( + r#"INSERT INTO workspace_runtime_bindings ( + workspace_id, runtime_id, display_name, base_url, public_key, + public_key_fingerprint, binding_revision, created_at, updated_at, revoked_at + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, 1, ?7, ?8, ?9)"#, params![ - record.runtime_id, record.workspace_id, + record.runtime_id, record.display_name, record.base_url, record.public_key, + record.public_key_fingerprint, record.created_at, record.updated_at, record.revoked_at, ], - )?; - Ok(()) + ) + .map_err(map_runtime_binding_write_error)?; + tx.commit()?; + Ok(WorkspaceRuntimeBindingUpsert::Created) }) } - pub fn list_trusted_runtimes( + pub fn revoke_workspace_runtime_binding( &self, - include_revoked: bool, - ) -> Result> { - self.with_conn(|conn| { - let sql = if include_revoked { - r#"SELECT runtime_id, workspace_id, display_name, base_url, public_key, created_at, updated_at, revoked_at - FROM trusted_runtime_records ORDER BY runtime_id ASC"# - } else { - r#"SELECT runtime_id, workspace_id, display_name, base_url, public_key, created_at, updated_at, revoked_at - FROM trusted_runtime_records WHERE revoked_at IS NULL ORDER BY runtime_id ASC"# - }; - let mut stmt = conn.prepare(sql)?; - let rows = stmt.query_map([], read_trusted_runtime_record)?; - rows.collect::, _>>().map_err(Error::from) - }) - } - - pub fn revoke_trusted_runtime(&self, runtime_id: &str, revoked_at: &str) -> Result { + workspace_id: &str, + runtime_id: &str, + revoked_at: &str, + ) -> Result { + validate_identifier("workspace_id", workspace_id)?; + validate_identifier("runtime_id", runtime_id)?; + validate_non_empty("revoked_at", revoked_at)?; self.with_conn(|conn| { let changed = conn.execute( - r#"UPDATE trusted_runtime_records - SET revoked_at = ?2, updated_at = ?2 - WHERE runtime_id = ?1 AND revoked_at IS NULL"#, - params![runtime_id, revoked_at], + r#"UPDATE workspace_runtime_bindings + SET revoked_at = ?3, updated_at = ?3, + binding_revision = binding_revision + 1 + WHERE workspace_id = ?1 AND runtime_id = ?2 AND revoked_at IS NULL"#, + params![workspace_id, runtime_id, revoked_at], )?; Ok(changed > 0) }) } + + pub fn put_workspace_runtime_binding_key( + &self, + mut record: WorkspaceRuntimeBinding, + expected_revision: Option, + actor_account_id: &str, + ) -> Result<(WorkspaceRuntimeBindingMutation, WorkspaceRuntimeBinding)> { + validate_identifier("workspace_id", &record.workspace_id)?; + validate_identifier("runtime_id", &record.runtime_id)?; + validate_identifier("actor_account_id", actor_account_id)?; + validate_non_empty("runtime display_name", &record.display_name)?; + validate_runtime_base_url(&record.base_url)?; + validate_non_empty("updated_at", &record.updated_at)?; + normalize_workspace_runtime_binding_key(&mut record)?; + self.with_conn_mut(|conn| { + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let existing = tx + .query_row( + r#"SELECT workspace_id, runtime_id, display_name, base_url, public_key, + public_key_fingerprint, binding_revision, created_at, updated_at, revoked_at + FROM workspace_runtime_bindings + WHERE workspace_id = ?1 AND runtime_id = ?2"#, + params![record.workspace_id, record.runtime_id], + read_workspace_runtime_binding, + ) + .optional()?; + + if let Some(existing) = existing { + if existing.revoked_at.is_none() + && existing.public_key == record.public_key + && existing.public_key_fingerprint == record.public_key_fingerprint + { + tx.commit()?; + return Ok((WorkspaceRuntimeBindingMutation::Unchanged, existing)); + } + if expected_revision != Some(existing.binding_revision) { + return Err(Error::RuntimeBindingRevisionConflict { + expected: expected_revision, + actual: Some(existing.binding_revision), + }); + } + let fingerprint_owner = tx + .query_row( + r#"SELECT runtime_id FROM workspace_runtime_bindings + WHERE workspace_id = ?1 AND public_key_fingerprint = ?2 + AND runtime_id != ?3"#, + params![ + record.workspace_id, + record.public_key_fingerprint, + record.runtime_id + ], + |row| row.get::<_, String>(0), + ) + .optional()?; + if fingerprint_owner.is_some() { + return Err(Error::RuntimeBindingFingerprintConflict { + fingerprint: record.public_key_fingerprint, + }); + } + let action = if existing.revoked_at.is_some() { + WorkspaceRuntimeBindingMutation::Reactivated + } else { + WorkspaceRuntimeBindingMutation::Replaced + }; + let action_name = match action { + WorkspaceRuntimeBindingMutation::Reactivated => "reactivated", + WorkspaceRuntimeBindingMutation::Replaced => "replaced", + _ => unreachable!("action is selected above"), + }; + let next_revision = existing.binding_revision.checked_add(1).ok_or_else(|| { + Error::Store("Runtime binding revision overflow".to_string()) + })?; + tx.execute( + r#"UPDATE workspace_runtime_bindings + SET public_key = ?3, public_key_fingerprint = ?4, + binding_revision = ?5, updated_at = ?6, revoked_at = NULL + WHERE workspace_id = ?1 AND runtime_id = ?2"#, + params![ + record.workspace_id, + record.runtime_id, + record.public_key, + record.public_key_fingerprint, + next_revision, + record.updated_at, + ], + )?; + insert_workspace_runtime_binding_audit( + &tx, + &record.workspace_id, + &record.runtime_id, + actor_account_id, + action_name, + Some(&existing.public_key_fingerprint), + Some(&record.public_key_fingerprint), + next_revision, + &record.updated_at, + )?; + let updated = tx.query_row( + r#"SELECT workspace_id, runtime_id, display_name, base_url, public_key, + public_key_fingerprint, binding_revision, created_at, updated_at, revoked_at + FROM workspace_runtime_bindings + WHERE workspace_id = ?1 AND runtime_id = ?2"#, + params![record.workspace_id, record.runtime_id], + read_workspace_runtime_binding, + )?; + tx.commit()?; + return Ok((action, updated)); + } + + if expected_revision.is_some() { + return Err(Error::RuntimeBindingRevisionConflict { + expected: expected_revision, + actual: None, + }); + } + let fingerprint_owner = tx + .query_row( + r#"SELECT runtime_id FROM workspace_runtime_bindings + WHERE workspace_id = ?1 AND public_key_fingerprint = ?2"#, + params![record.workspace_id, record.public_key_fingerprint], + |row| row.get::<_, String>(0), + ) + .optional()?; + if fingerprint_owner.is_some() { + return Err(Error::RuntimeBindingFingerprintConflict { + fingerprint: record.public_key_fingerprint, + }); + } + record.binding_revision = 1; + record.revoked_at = None; + tx.execute( + r#"INSERT INTO workspace_runtime_bindings ( + workspace_id, runtime_id, display_name, base_url, public_key, + public_key_fingerprint, binding_revision, created_at, updated_at, revoked_at + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, 1, ?7, ?8, NULL)"#, + params![ + record.workspace_id, + record.runtime_id, + record.display_name, + record.base_url, + record.public_key, + record.public_key_fingerprint, + record.created_at, + record.updated_at, + ], + )?; + insert_workspace_runtime_binding_audit( + &tx, + &record.workspace_id, + &record.runtime_id, + actor_account_id, + "created", + None, + Some(&record.public_key_fingerprint), + 1, + &record.updated_at, + )?; + tx.commit()?; + Ok((WorkspaceRuntimeBindingMutation::Created, record)) + }) + } + + pub fn revoke_workspace_runtime_binding_key( + &self, + workspace_id: &str, + runtime_id: &str, + expected_revision: u64, + actor_account_id: &str, + revoked_at: &str, + ) -> Result<(WorkspaceRuntimeBindingMutation, WorkspaceRuntimeBinding)> { + validate_identifier("workspace_id", workspace_id)?; + validate_identifier("runtime_id", runtime_id)?; + validate_identifier("actor_account_id", actor_account_id)?; + validate_non_empty("revoked_at", revoked_at)?; + self.with_conn_mut(|conn| { + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let existing = tx + .query_row( + r#"SELECT workspace_id, runtime_id, display_name, base_url, public_key, + public_key_fingerprint, binding_revision, created_at, updated_at, revoked_at + FROM workspace_runtime_bindings + WHERE workspace_id = ?1 AND runtime_id = ?2"#, + params![workspace_id, runtime_id], + read_workspace_runtime_binding, + ) + .optional()? + .ok_or_else(|| Error::RuntimeBindingNotFound { + runtime_id: runtime_id.to_string(), + })?; + if existing.revoked_at.is_some() { + tx.commit()?; + return Ok((WorkspaceRuntimeBindingMutation::Unchanged, existing)); + } + if expected_revision != existing.binding_revision { + return Err(Error::RuntimeBindingRevisionConflict { + expected: Some(expected_revision), + actual: Some(existing.binding_revision), + }); + } + let next_revision = existing + .binding_revision + .checked_add(1) + .ok_or_else(|| Error::Store("Runtime binding revision overflow".to_string()))?; + tx.execute( + r#"UPDATE workspace_runtime_bindings + SET revoked_at = ?3, updated_at = ?3, binding_revision = ?4 + WHERE workspace_id = ?1 AND runtime_id = ?2"#, + params![workspace_id, runtime_id, revoked_at, next_revision], + )?; + insert_workspace_runtime_binding_audit( + &tx, + workspace_id, + runtime_id, + actor_account_id, + "revoked", + Some(&existing.public_key_fingerprint), + None, + next_revision, + revoked_at, + )?; + let updated = tx.query_row( + r#"SELECT workspace_id, runtime_id, display_name, base_url, public_key, + public_key_fingerprint, binding_revision, created_at, updated_at, revoked_at + FROM workspace_runtime_bindings + WHERE workspace_id = ?1 AND runtime_id = ?2"#, + params![workspace_id, runtime_id], + read_workspace_runtime_binding, + )?; + tx.commit()?; + Ok((WorkspaceRuntimeBindingMutation::Revoked, updated)) + }) + } + + pub fn list_workspace_runtime_binding_audit( + &self, + workspace_id: &str, + runtime_id: &str, + limit: usize, + ) -> Result> { + validate_identifier("workspace_id", workspace_id)?; + validate_identifier("runtime_id", runtime_id)?; + let limit = limit.clamp(1, 50) as i64; + self.with_conn(|conn| { + let mut stmt = conn.prepare( + r#"SELECT workspace_id, runtime_id, actor_account_id, action, + old_fingerprint, new_fingerprint, binding_revision, at + FROM workspace_runtime_binding_audit + WHERE workspace_id = ?1 AND runtime_id = ?2 + ORDER BY binding_revision DESC + LIMIT ?3"#, + )?; + let rows = stmt.query_map(params![workspace_id, runtime_id, limit], |row| { + Ok(WorkspaceRuntimeBindingAuditRecord { + workspace_id: row.get(0)?, + runtime_id: row.get(1)?, + actor_account_id: row.get(2)?, + action: row.get(3)?, + old_fingerprint: row.get(4)?, + new_fingerprint: row.get(5)?, + binding_revision: row.get(6)?, + at: row.get(7)?, + }) + })?; + rows.collect::, _>>() + .map_err(Error::from) + }) + } +} + +fn insert_workspace_runtime_binding_audit( + tx: &rusqlite::Transaction<'_>, + workspace_id: &str, + runtime_id: &str, + actor_account_id: &str, + action: &str, + old_fingerprint: Option<&str>, + new_fingerprint: Option<&str>, + binding_revision: u64, + at: &str, +) -> Result<()> { + tx.execute( + r#"INSERT INTO workspace_runtime_binding_audit ( + workspace_id, runtime_id, actor_account_id, action, + old_fingerprint, new_fingerprint, binding_revision, at + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)"#, + params![ + workspace_id, + runtime_id, + actor_account_id, + action, + old_fingerprint, + new_fingerprint, + binding_revision, + at, + ], + )?; + Ok(()) } #[async_trait] @@ -1711,25 +2197,105 @@ impl ControlPlaneStore for SqliteWorkspaceStore { }) } - async fn get_trusted_runtime(&self, runtime_id: &str) -> Result> { - self.with_conn(|conn| { - conn.query_row( - r#"SELECT runtime_id, workspace_id, display_name, base_url, public_key, created_at, updated_at, revoked_at - FROM trusted_runtime_records WHERE runtime_id = ?1"#, - params![runtime_id], - read_trusted_runtime_record, - ) - .optional() - .map_err(Error::from) - }) + fn workspace_runtime_binding_matches( + &self, + expected: &WorkspaceRuntimeBinding, + ) -> Result { + Ok(SqliteWorkspaceStore::get_workspace_runtime_binding( + self, + &expected.workspace_id, + &expected.runtime_id, + )? + .is_some_and(|binding| binding == *expected && binding.revoked_at.is_none())) } - async fn upsert_trusted_runtime_record(&self, record: &TrustedRuntimeRecord) -> Result<()> { - SqliteWorkspaceStore::upsert_trusted_runtime(self, record) + async fn get_workspace_runtime_binding( + &self, + workspace_id: &str, + runtime_id: &str, + ) -> Result> { + SqliteWorkspaceStore::get_workspace_runtime_binding(self, workspace_id, runtime_id) + } + + async fn list_workspace_runtime_bindings( + &self, + workspace_id: &str, + include_revoked: bool, + ) -> Result> { + SqliteWorkspaceStore::list_workspace_runtime_bindings(self, workspace_id, include_revoked) + } + + async fn upsert_workspace_runtime_binding_record( + &self, + record: WorkspaceRuntimeBinding, + replace: bool, + ) -> Result { + SqliteWorkspaceStore::upsert_workspace_runtime_binding(self, record, replace) + } + + async fn revoke_workspace_runtime_binding_record( + &self, + workspace_id: &str, + runtime_id: &str, + revoked_at: &str, + ) -> Result { + SqliteWorkspaceStore::revoke_workspace_runtime_binding( + self, + workspace_id, + runtime_id, + revoked_at, + ) + } + + async fn put_workspace_runtime_binding_key( + &self, + record: WorkspaceRuntimeBinding, + expected_revision: Option, + actor_account_id: &str, + ) -> Result<(WorkspaceRuntimeBindingMutation, WorkspaceRuntimeBinding)> { + SqliteWorkspaceStore::put_workspace_runtime_binding_key( + self, + record, + expected_revision, + actor_account_id, + ) + } + + async fn revoke_workspace_runtime_binding_key( + &self, + workspace_id: &str, + runtime_id: &str, + expected_revision: u64, + actor_account_id: &str, + revoked_at: &str, + ) -> Result<(WorkspaceRuntimeBindingMutation, WorkspaceRuntimeBinding)> { + SqliteWorkspaceStore::revoke_workspace_runtime_binding_key( + self, + workspace_id, + runtime_id, + expected_revision, + actor_account_id, + revoked_at, + ) + } + + async fn list_workspace_runtime_binding_audit( + &self, + workspace_id: &str, + runtime_id: &str, + limit: usize, + ) -> Result> { + SqliteWorkspaceStore::list_workspace_runtime_binding_audit( + self, + workspace_id, + runtime_id, + limit, + ) } async fn consume_worker_mutation_source_jti( &self, + workspace_id: &str, runtime_id: &str, jti: &str, expires_at: u64, @@ -1744,9 +2310,9 @@ impl ControlPlaneStore for SqliteWorkspaceStore { )?; let inserted = transaction.execute( r#"INSERT OR IGNORE INTO worker_mutation_source_proof_jtis ( - runtime_id, jti, expires_at, consumed_at - ) VALUES (?1, ?2, ?3, ?4)"#, - params![runtime_id, jti, expires_at, consumed_at], + workspace_id, runtime_id, jti, expires_at, consumed_at + ) VALUES (?1, ?2, ?3, ?4, ?5)"#, + params![workspace_id, runtime_id, jti, expires_at, consumed_at], )?; transaction.commit()?; Ok(inserted == 1) @@ -3186,6 +3752,20 @@ impl ControlPlaneStore for SqliteWorkspaceStore { "DELETE FROM worker_registry WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3", params![workspace_id, worker.runtime_id, worker.worker_id], )?; + if changed != 0 { + tx.execute( + "UPDATE worker_create_reservations + SET state = 'removed', updated_at = ?4 + WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3 + AND state = 'created'", + params![ + workspace_id, + worker.runtime_id, + worker.worker_id, + chrono::Utc::now().to_rfc3339(), + ], + )?; + } tx.commit()?; Ok(changed > 0) }) @@ -5094,19 +5674,92 @@ fn account_select_sql(where_clause: &str) -> String { ) } -fn read_trusted_runtime_record(row: &rusqlite::Row<'_>) -> rusqlite::Result { - Ok(TrustedRuntimeRecord { - runtime_id: row.get(0)?, - workspace_id: row.get(1)?, +fn read_workspace_runtime_binding( + row: &rusqlite::Row<'_>, +) -> rusqlite::Result { + Ok(WorkspaceRuntimeBinding { + workspace_id: row.get(0)?, + runtime_id: row.get(1)?, display_name: row.get(2)?, base_url: row.get(3)?, public_key: row.get(4)?, - created_at: row.get(5)?, - updated_at: row.get(6)?, - revoked_at: row.get(7)?, + public_key_fingerprint: row.get(5)?, + binding_revision: row.get(6)?, + created_at: row.get(7)?, + updated_at: row.get(8)?, + revoked_at: row.get(9)?, }) } +fn validate_identifier(field: &str, value: &str) -> Result<()> { + if value.trim().is_empty() || value.trim() != value { + return Err(Error::InvalidInput(format!( + "{field} must be non-empty and must not contain surrounding whitespace" + ))); + } + Ok(()) +} + +fn validate_non_empty(field: &str, value: &str) -> Result<()> { + if value.trim().is_empty() { + return Err(Error::InvalidInput(format!("{field} must not be empty"))); + } + Ok(()) +} + +fn validate_runtime_base_url(base_url: &str) -> Result<()> { + let base_url = base_url.trim(); + if base_url.starts_with("http://") + || base_url.starts_with("https://") + || base_url == "in-process://embedded" + { + Ok(()) + } else { + Err(Error::InvalidInput( + "Runtime base_url must be an absolute http/https URL or the embedded Runtime endpoint" + .to_string(), + )) + } +} + +pub fn normalize_runtime_public_key(public_key: &str) -> Result<(String, String)> { + let bytes = worker_runtime::auth::decode_public_key(public_key) + .map_err(|err| Error::InvalidInput(format!("invalid Runtime public key: {err}")))?; + let canonical = worker_runtime::auth::encode_public_key(&bytes); + let digest = Sha256::digest(&bytes) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::(); + let fingerprint = format!("sha256:{digest}"); + Ok((canonical, fingerprint)) +} + +fn normalize_workspace_runtime_binding_key(record: &mut WorkspaceRuntimeBinding) -> Result<()> { + let (canonical, fingerprint) = normalize_runtime_public_key(&record.public_key)?; + if !record.public_key_fingerprint.is_empty() && record.public_key_fingerprint != fingerprint { + return Err(Error::InvalidInput( + "Runtime public key fingerprint does not match the public key".into(), + )); + } + record.public_key = canonical; + record.public_key_fingerprint = fingerprint; + Ok(()) +} + +fn map_runtime_binding_write_error(err: rusqlite::Error) -> Error { + if matches!( + &err, + rusqlite::Error::SqliteFailure(error, _) + if error.code == rusqlite::ErrorCode::ConstraintViolation + ) { + Error::RuntimeBindingConflict( + "the Runtime id or public key fingerprint is already bound in this Workspace".into(), + ) + } else { + Error::from(err) + } +} + fn read_account_record(row: &rusqlite::Row<'_>) -> rusqlite::Result { Ok(AccountRecord { account_id: row.get(0)?, @@ -5786,6 +6439,246 @@ CREATE TABLE IF NOT EXISTS __yoi_schema_migrations ( Ok(()) } +fn migrate_workspace_deletion_v52_to_v53(conn: &Connection) -> Result<()> { + let current = current_schema_version(conn)?; + if current != PREVIOUS_SCHEMA_VERSION { + return Err(Error::Store(format!( + "expected schema version {PREVIOUS_SCHEMA_VERSION} before {WORKSPACE_DELETION_MIGRATION_NAME} migration, found {current}" + ))); + } + + let tx = rusqlite::Transaction::new_unchecked(conn, TransactionBehavior::Exclusive)?; + tx.execute_batch( + r#" + CREATE TABLE worker_create_reservations_v53 ( + workspace_id TEXT NOT NULL, + allocation_key TEXT NOT NULL, + worker_id TEXT NOT NULL, + runtime_id TEXT NOT NULL, + create_fingerprint TEXT NOT NULL, + state TEXT NOT NULL CHECK (state IN ('reserved', 'created', 'removed')), + request_fingerprint TEXT, + memory_settings_revision INTEGER, + memory_language TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (workspace_id, allocation_key), + UNIQUE (workspace_id, worker_id), + FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE, + CHECK ( + (request_fingerprint IS NULL AND memory_settings_revision IS NULL AND memory_language IS NULL) + OR + (request_fingerprint IS NOT NULL AND memory_settings_revision IS NOT NULL AND memory_settings_revision > 0 AND memory_language IS NOT NULL AND length(trim(memory_language)) > 0) + ) + ); + INSERT INTO worker_create_reservations_v53 ( + workspace_id, allocation_key, worker_id, runtime_id, create_fingerprint, + state, request_fingerprint, memory_settings_revision, memory_language, + created_at, updated_at + ) + SELECT workspace_id, allocation_key, worker_id, runtime_id, create_fingerprint, + state, request_fingerprint, memory_settings_revision, memory_language, + created_at, updated_at + FROM worker_create_reservations; + DROP TABLE worker_create_reservations; + ALTER TABLE worker_create_reservations_v53 RENAME TO worker_create_reservations; + + CREATE TABLE workspace_deletion_operations ( + operation_id TEXT PRIMARY KEY, + request_fingerprint TEXT NOT NULL, + workspace_id TEXT NOT NULL, + workspace_display_name TEXT NOT NULL, + workspace_revision TEXT NOT NULL, + owner_account_id TEXT NOT NULL, + actor_account_id TEXT NOT NULL, + state TEXT NOT NULL CHECK(state IN ('queued', 'running', 'blocked', 'failed', 'succeeded')), + resource_counts_json TEXT NOT NULL, + child_operation_ids_json TEXT NOT NULL, + blockers_json TEXT NOT NULL, + failure_category TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + completed_at TEXT, + FOREIGN KEY(owner_account_id) REFERENCES accounts(account_id) ON DELETE RESTRICT, + FOREIGN KEY(actor_account_id) REFERENCES accounts(account_id) ON DELETE RESTRICT + ); + CREATE INDEX workspace_deletion_operations_workspace_recent + ON workspace_deletion_operations(workspace_id, created_at DESC); + "#, + )?; + verify_workspace_deletion_schema(&tx)?; + tx.execute( + "INSERT INTO __yoi_schema_migrations (version, name) VALUES (?1, ?2)", + params![LATEST_SCHEMA_VERSION, WORKSPACE_DELETION_MIGRATION_NAME], + )?; + tx.commit()?; + Ok(()) +} + +fn verify_workspace_deletion_schema(conn: &Connection) -> Result<()> { + let columns = table_columns(conn, "workspace_deletion_operations")? + .into_iter() + .collect::>(); + let expected = [ + "operation_id", + "request_fingerprint", + "workspace_id", + "workspace_display_name", + "workspace_revision", + "owner_account_id", + "actor_account_id", + "state", + "resource_counts_json", + "child_operation_ids_json", + "blockers_json", + "failure_category", + "created_at", + "updated_at", + "completed_at", + ] + .into_iter() + .map(str::to_string) + .collect::>(); + if columns != expected { + return Err(Error::Store( + "workspace_deletion_operations schema does not match schema-53".to_string(), + )); + } + let reservation_sql: String = conn.query_row( + "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'worker_create_reservations'", + [], + |row| row.get(0), + )?; + if !reservation_sql.contains("'removed'") { + return Err(Error::Store( + "worker_create_reservations schema lacks terminal removed state".to_string(), + )); + } + Ok(()) +} + +fn verify_workspace_runtime_binding_schema(conn: &Connection) -> Result<()> { + let columns = table_columns(conn, "workspace_runtime_bindings")? + .into_iter() + .collect::>(); + let expected = [ + "workspace_id", + "runtime_id", + "display_name", + "base_url", + "public_key", + "public_key_fingerprint", + "binding_revision", + "created_at", + "updated_at", + "revoked_at", + ] + .into_iter() + .map(str::to_string) + .collect::>(); + if columns != expected { + return Err(Error::Store( + "workspace_runtime_bindings schema does not match schema-52".to_string(), + )); + } + let revision_default = conn.query_row( + "SELECT dflt_value FROM pragma_table_info('workspace_runtime_bindings') WHERE name = 'binding_revision'", + [], + |row| row.get::<_, Option>(0), + )?; + if revision_default.as_deref() != Some("1") { + return Err(Error::Store( + "workspace_runtime_bindings binding_revision default does not match schema-52" + .to_string(), + )); + } + let sql = conn.query_row( + "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'workspace_runtime_bindings'", + [], + |row| row.get::<_, String>(0), + )?; + if !sql.contains("PRIMARY KEY (workspace_id, runtime_id)") + || !sql.contains("UNIQUE (workspace_id, public_key_fingerprint)") + { + return Err(Error::Store( + "workspace_runtime_bindings is missing composite identity or trust uniqueness" + .to_string(), + )); + } + let index_exists = conn.query_row( + "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'index' AND name = 'idx_workspace_runtime_bindings_workspace')", + [], + |row| row.get::<_, i64>(0), + )? != 0; + if !index_exists { + return Err(Error::Store( + "workspace_runtime_bindings is missing its Workspace lookup index".to_string(), + )); + } + let audit_columns = table_columns(conn, "workspace_runtime_binding_audit")? + .into_iter() + .collect::>(); + let expected_audit_columns = [ + "workspace_id", + "runtime_id", + "actor_account_id", + "action", + "old_fingerprint", + "new_fingerprint", + "binding_revision", + "at", + ] + .into_iter() + .map(str::to_string) + .collect::>(); + if audit_columns != expected_audit_columns { + return Err(Error::Store( + "workspace_runtime_binding_audit schema does not match schema-52".to_string(), + )); + } + let audit_index_exists = conn.query_row( + "SELECT EXISTS(SELECT 1 FROM sqlite_master WHERE type = 'index' AND name = 'idx_workspace_runtime_binding_audit_recent')", + [], + |row| row.get::<_, i64>(0), + )? != 0; + if !audit_index_exists { + return Err(Error::Store( + "workspace_runtime_binding_audit recent index is missing".to_string(), + )); + } + let jti_sql = conn.query_row( + "SELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'worker_mutation_source_proof_jtis'", + [], + |row| row.get::<_, String>(0), + )?; + if !jti_sql.contains("PRIMARY KEY (workspace_id, runtime_id, jti)") { + return Err(Error::Store( + "worker_mutation_source_proof_jtis is missing Workspace-scoped replay identity" + .to_string(), + )); + } + let mut stmt = conn.prepare( + r#"SELECT workspace_id, runtime_id, display_name, base_url, public_key, + public_key_fingerprint, binding_revision, created_at, updated_at, revoked_at + FROM workspace_runtime_bindings"#, + )?; + let rows = stmt.query_map([], read_workspace_runtime_binding)?; + for row in rows { + let binding = row?; + let mut normalized = binding.clone(); + normalize_workspace_runtime_binding_key(&mut normalized)?; + if normalized.public_key != binding.public_key + || normalized.public_key_fingerprint != binding.public_key_fingerprint + { + return Err(Error::Store(format!( + "Runtime binding `{}/{}` has non-canonical trust content", + binding.workspace_id, binding.runtime_id + ))); + } + } + Ok(()) +} + fn create_latest_workspace_schema(conn: &Connection) -> Result<()> { conn.execute_batch(include_str!("latest_schema.sql"))?; Ok(()) @@ -6233,24 +7126,51 @@ fn allocate_resource_key( Ok(resource_key) } -fn verify_baseline_history(conn: &Connection) -> Result<()> { - let rows = conn.query_row( - "SELECT COUNT(*), COALESCE(MAX(version), 0), COALESCE(MAX(name), '') \ - FROM __yoi_schema_migrations", - [], - |row| { - Ok(( - row.get::<_, i64>(0)?, - row.get::<_, i64>(1)?, - row.get::<_, String>(2)?, - )) - }, - )?; - let baseline = &MIGRATIONS[0]; - if rows != (1, baseline.version, baseline.name.to_string()) { +fn verify_current_schema_history(conn: &Connection) -> Result<()> { + let mut stmt = + conn.prepare("SELECT version, name FROM __yoi_schema_migrations ORDER BY version ASC")?; + let rows = stmt + .query_map([], |row| { + Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?)) + })? + .collect::>>()?; + let fresh = vec![( + LATEST_SCHEMA_VERSION, + "workspace schema baseline".to_string(), + )]; + let upgraded = vec![ + ( + PREVIOUS_SCHEMA_VERSION, + "workspace schema baseline".to_string(), + ), + ( + LATEST_SCHEMA_VERSION, + WORKSPACE_DELETION_MIGRATION_NAME.to_string(), + ), + ]; + if rows != fresh && rows != upgraded { return Err(Error::Store(format!( - "database migration history is not the canonical schema baseline: expected only version {} ({:?}), found {} row(s) ending at version {} ({:?})", - baseline.version, baseline.name, rows.0, rows.1, rows.2 + "database migration history is not canonical for schema version {LATEST_SCHEMA_VERSION}: found {rows:?}" + ))); + } + Ok(()) +} + +fn verify_previous_schema_history(conn: &Connection) -> Result<()> { + let rows = conn.query_row( + "SELECT COUNT(*), COALESCE(MAX(version), 0), COALESCE(MAX(name), '') FROM __yoi_schema_migrations", + [], + |row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?, row.get::<_, String>(2)?)), + )?; + if rows + != ( + 1, + PREVIOUS_SCHEMA_VERSION, + "workspace schema baseline".to_string(), + ) + { + return Err(Error::Store(format!( + "database migration history is not the canonical schema-{PREVIOUS_SCHEMA_VERSION} baseline" ))); } Ok(()) @@ -6261,23 +7181,33 @@ fn apply_migrations(conn: &Connection) -> Result<()> { let current = current_schema_version(conn)?; match current { 0 => { - let tx = conn.unchecked_transaction()?; + let tx = rusqlite::Transaction::new_unchecked(conn, TransactionBehavior::Exclusive)?; (baseline.apply)(&tx)?; tx.execute( "INSERT INTO __yoi_schema_migrations (version, name) VALUES (?1, ?2)", params![baseline.version, baseline.name], )?; tx.commit()?; - Ok(()) + verify_workspace_runtime_binding_schema(conn)?; + verify_workspace_deletion_schema(conn) } - version if version == baseline.version => verify_baseline_history(conn), - version if version > baseline.version => Err(Error::Store(format!( - "database schema version {version} is newer than this server supports ({}); refusing to serve with an older binary", - baseline.version + PREVIOUS_SCHEMA_VERSION => { + verify_previous_schema_history(conn)?; + migrate_workspace_deletion_v52_to_v53(conn)?; + verify_current_schema_history(conn)?; + verify_workspace_runtime_binding_schema(conn)?; + verify_workspace_deletion_schema(conn) + } + LATEST_SCHEMA_VERSION => { + verify_current_schema_history(conn)?; + verify_workspace_runtime_binding_schema(conn)?; + verify_workspace_deletion_schema(conn) + } + version if version > LATEST_SCHEMA_VERSION => Err(Error::Store(format!( + "database schema version {version} is newer than this server supports ({LATEST_SCHEMA_VERSION}); refusing to serve with an older binary" ))), version => Err(Error::Store(format!( - "database schema version {version} predates the canonical baseline ({}); migrate its data manually before starting this server", - baseline.version + "database schema version {version} predates the supported upgrade window ({PREVIOUS_SCHEMA_VERSION}); migrate its data manually before starting this server" ))), } } @@ -6340,12 +7270,415 @@ mod tests { .unwrap(); } + fn prepare_schema_v52(path: &Path) { + let conn = Connection::open(path).unwrap(); + configure_sqlite(&conn).unwrap(); + ticket::migrate_sqlite_ticket_schema(&conn).unwrap(); + merge_request::migrate(&conn).unwrap(); + create_latest_workspace_schema(&conn).unwrap(); + conn.execute_batch( + r#" + DROP INDEX workspace_deletion_operations_workspace_recent; + DROP TABLE workspace_deletion_operations; + CREATE TABLE worker_create_reservations_v52 ( + workspace_id TEXT NOT NULL, + allocation_key TEXT NOT NULL, + worker_id TEXT NOT NULL, + runtime_id TEXT NOT NULL, + create_fingerprint TEXT NOT NULL, + state TEXT NOT NULL CHECK (state IN ('reserved', 'created')), + request_fingerprint TEXT, + memory_settings_revision INTEGER, + memory_language TEXT, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL, + PRIMARY KEY (workspace_id, allocation_key), + UNIQUE (workspace_id, worker_id), + FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE, + CHECK ( + (request_fingerprint IS NULL AND memory_settings_revision IS NULL AND memory_language IS NULL) + OR + (request_fingerprint IS NOT NULL AND memory_settings_revision IS NOT NULL AND memory_settings_revision > 0 AND memory_language IS NOT NULL AND length(trim(memory_language)) > 0) + ) + ); + INSERT INTO worker_create_reservations_v52 ( + workspace_id, allocation_key, worker_id, runtime_id, create_fingerprint, + state, request_fingerprint, memory_settings_revision, memory_language, + created_at, updated_at + ) + SELECT workspace_id, allocation_key, worker_id, runtime_id, create_fingerprint, + state, request_fingerprint, memory_settings_revision, memory_language, + created_at, updated_at + FROM worker_create_reservations; + DROP TABLE worker_create_reservations; + ALTER TABLE worker_create_reservations_v52 RENAME TO worker_create_reservations; + DELETE FROM __yoi_schema_migrations; + INSERT INTO __yoi_schema_migrations(version, name) + VALUES (52, 'workspace schema baseline'); + "#, + ) + .unwrap(); + } + + #[test] + fn schema_v52_migrates_workspace_deletion_operations() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("server.db"); + prepare_schema_v52(&path); + let conn = Connection::open(&path).unwrap(); + conn.execute_batch( + "INSERT INTO accounts(account_id,kind,handle,display_name,created_at,updated_at) + VALUES('owner','user','owner','Owner','1','1'); + INSERT INTO workspaces(workspace_id,owner_account_id,display_name,state,created_at,updated_at) + VALUES('workspace-a','owner','Workspace A','active','1','1'); + INSERT INTO worker_create_reservations( + workspace_id,allocation_key,worker_id,runtime_id,create_fingerprint,state,created_at,updated_at + ) VALUES('workspace-a','allocation','worker-a','runtime-a','fingerprint','created','1','1');", + ) + .unwrap(); + drop(conn); + + let store = SqliteWorkspaceStore::open(&path).unwrap(); + store + .with_conn(|conn| { + assert_eq!(current_schema_version(conn)?, LATEST_SCHEMA_VERSION); + verify_workspace_deletion_schema(conn)?; + let reservation_state: String = conn.query_row( + "SELECT state FROM worker_create_reservations WHERE workspace_id='workspace-a' AND allocation_key='allocation'", + [], + |row| row.get(0), + )?; + assert_eq!(reservation_state, "created"); + let violations: i64 = + conn.query_row("SELECT COUNT(*) FROM pragma_foreign_key_check", [], |row| { + row.get(0) + })?; + assert_eq!(violations, 0); + Ok(()) + }) + .unwrap(); + } + + #[test] + fn schema_v52_workspace_deletion_migration_rolls_back_on_failure() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("server.db"); + prepare_schema_v52(&path); + let conn = Connection::open(&path).unwrap(); + conn.execute_batch( + "CREATE TABLE workspace_deletion_operations (unexpected TEXT NOT NULL);", + ) + .unwrap(); + drop(conn); + + let error = SqliteWorkspaceStore::open(&path) + .err() + .expect("migration must fail") + .to_string(); + assert!(error.contains("already exists"), "{error}"); + let conn = Connection::open(&path).unwrap(); + assert_eq!( + table_columns(&conn, "workspace_deletion_operations").unwrap(), + vec!["unexpected".to_string()] + ); + assert_eq!(current_schema_version(&conn).unwrap(), 52); + let reservation_sql: String = conn + .query_row( + "SELECT sql FROM sqlite_master WHERE type='table' AND name='worker_create_reservations'", + [], + |row| row.get(0), + ) + .unwrap(); + assert!(!reservation_sql.contains("'removed'")); + } + + #[test] + fn runtime_binding_identity_and_trust_uniqueness_are_workspace_scoped() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("server.db"); + let store = SqliteWorkspaceStore::open(&path).unwrap(); + store + .with_conn(|conn| { + conn.execute_batch( + r#" + INSERT INTO accounts(account_id, kind, handle, display_name, created_at, updated_at) + VALUES ('owner', 'user', 'owner', 'Owner', '1', '1'); + INSERT INTO workspaces(workspace_id, owner_account_id, display_name, state, created_at, updated_at) + VALUES + ('workspace-a', 'owner', 'Workspace A', 'active', '1', '1'), + ('workspace-b', 'owner', 'Workspace B', 'active', '1', '1'); + "#, + )?; + Ok(()) + }) + .unwrap(); + let identity = worker_runtime::auth::RuntimeIdentityMaterial::generate("shared").unwrap(); + let binding = |workspace_id: &str, runtime_id: &str| WorkspaceRuntimeBinding { + workspace_id: workspace_id.to_string(), + runtime_id: runtime_id.to_string(), + display_name: runtime_id.to_string(), + base_url: format!("https://{workspace_id}.runtime.test"), + public_key: identity.public_key.clone(), + public_key_fingerprint: String::new(), + binding_revision: 1, + created_at: "1".to_string(), + updated_at: "1".to_string(), + revoked_at: None, + }; + assert_eq!( + store + .upsert_workspace_runtime_binding(binding("workspace-a", "shared"), false) + .unwrap(), + WorkspaceRuntimeBindingUpsert::Created + ); + assert_eq!( + store + .upsert_workspace_runtime_binding(binding("workspace-b", "shared"), false) + .unwrap(), + WorkspaceRuntimeBindingUpsert::Created + ); + assert_eq!( + store + .upsert_workspace_runtime_binding(binding("workspace-a", "shared"), false) + .unwrap(), + WorkspaceRuntimeBindingUpsert::Unchanged + ); + let conflict = store + .upsert_workspace_runtime_binding(binding("workspace-a", "other"), false) + .unwrap_err(); + assert!(matches!(conflict, Error::RuntimeBindingConflict(_))); + assert_eq!( + store + .list_workspace_runtime_bindings("workspace-a", false) + .unwrap() + .len(), + 1 + ); + assert_eq!( + store + .list_workspace_runtime_bindings("workspace-b", false) + .unwrap() + .len(), + 1 + ); + let expected_workspace_a = store + .get_workspace_runtime_binding("workspace-a", "shared") + .unwrap() + .unwrap(); + assert!( + store + .workspace_runtime_binding_matches(&expected_workspace_a) + .unwrap() + ); + let mut replaced_workspace_a = expected_workspace_a.clone(); + replaced_workspace_a.base_url = "https://replacement.runtime.test".to_string(); + replaced_workspace_a.updated_at = "2".to_string(); + assert_eq!( + store + .upsert_workspace_runtime_binding(replaced_workspace_a, true) + .unwrap(), + WorkspaceRuntimeBindingUpsert::Replaced + ); + assert!( + !store + .workspace_runtime_binding_matches(&expected_workspace_a) + .unwrap() + ); + assert!( + store + .revoke_workspace_runtime_binding("workspace-a", "shared", "3") + .unwrap() + ); + drop(store); + + let reopened = SqliteWorkspaceStore::open(&path).unwrap(); + assert!( + reopened + .list_workspace_runtime_bindings("workspace-a", false) + .unwrap() + .is_empty() + ); + assert_eq!( + reopened + .list_workspace_runtime_bindings("workspace-b", false) + .unwrap() + .len(), + 1 + ); + assert!( + reopened + .get_workspace_runtime_binding("workspace-a", "shared") + .unwrap() + .unwrap() + .revoked_at + .is_some() + ); + } + + #[test] + fn runtime_binding_key_mutations_are_revisioned_idempotent_and_audited() { + let store = SqliteWorkspaceStore::in_memory().unwrap(); + store + .with_conn(|conn| { + conn.execute_batch( + r#" + INSERT INTO accounts(account_id, kind, handle, display_name, created_at, updated_at) + VALUES ('owner', 'user', 'owner', 'Owner', '1', '1'); + INSERT INTO workspaces(workspace_id, owner_account_id, display_name, state, created_at, updated_at) + VALUES ('workspace-a', 'owner', 'Workspace A', 'active', '1', '1'); + "#, + )?; + Ok(()) + }) + .unwrap(); + let first = worker_runtime::auth::RuntimeIdentityMaterial::generate("first").unwrap(); + let second = worker_runtime::auth::RuntimeIdentityMaterial::generate("second").unwrap(); + let binding = |public_key: String, at: &str| WorkspaceRuntimeBinding { + workspace_id: "workspace-a".to_string(), + runtime_id: "runtime-a".to_string(), + display_name: "Runtime A".to_string(), + base_url: "https://runtime.test".to_string(), + public_key, + public_key_fingerprint: String::new(), + binding_revision: 1, + created_at: at.to_string(), + updated_at: at.to_string(), + revoked_at: None, + }; + + let (created, created_binding) = store + .put_workspace_runtime_binding_key( + binding(first.public_key.clone(), "1"), + None, + "owner", + ) + .unwrap(); + assert_eq!(created, WorkspaceRuntimeBindingMutation::Created); + assert_eq!(created_binding.binding_revision, 1); + let (replayed, replayed_binding) = store + .put_workspace_runtime_binding_key(binding(first.public_key, "2"), None, "owner") + .unwrap(); + assert_eq!(replayed, WorkspaceRuntimeBindingMutation::Unchanged); + assert_eq!(replayed_binding.binding_revision, 1); + + let stale = store + .put_workspace_runtime_binding_key( + binding(second.public_key.clone(), "3"), + Some(0), + "owner", + ) + .unwrap_err(); + assert!(matches!( + stale, + Error::RuntimeBindingRevisionConflict { + expected: Some(0), + actual: Some(1) + } + )); + let (replaced, replaced_binding) = store + .put_workspace_runtime_binding_key( + binding(second.public_key.clone(), "3"), + Some(1), + "owner", + ) + .unwrap(); + assert_eq!(replaced, WorkspaceRuntimeBindingMutation::Replaced); + assert_eq!(replaced_binding.binding_revision, 2); + let (revoked, revoked_binding) = store + .revoke_workspace_runtime_binding_key("workspace-a", "runtime-a", 2, "owner", "4") + .unwrap(); + assert_eq!(revoked, WorkspaceRuntimeBindingMutation::Revoked); + assert_eq!(revoked_binding.binding_revision, 3); + assert_eq!(revoked_binding.revoked_at.as_deref(), Some("4")); + let (reactivated, reactivated_binding) = store + .put_workspace_runtime_binding_key( + binding(second.public_key.clone(), "5"), + Some(3), + "owner", + ) + .unwrap(); + assert_eq!(reactivated, WorkspaceRuntimeBindingMutation::Reactivated); + assert_eq!(reactivated_binding.binding_revision, 4); + + let mut duplicate = binding(second.public_key, "6"); + duplicate.runtime_id = "runtime-b".to_string(); + let duplicate_error = store + .put_workspace_runtime_binding_key(duplicate, None, "owner") + .unwrap_err(); + assert!(matches!( + duplicate_error, + Error::RuntimeBindingFingerprintConflict { .. } + )); + + let audit = store + .list_workspace_runtime_binding_audit("workspace-a", "runtime-a", 50) + .unwrap(); + assert_eq!(audit.len(), 4); + assert_eq!(audit[0].action, "reactivated"); + assert_eq!(audit[0].binding_revision, 4); + assert_eq!(audit[1].action, "revoked"); + assert_eq!(audit[2].action, "replaced"); + assert_eq!(audit[3].action, "created"); + } + + #[test] + fn embedded_runtime_binding_can_explicitly_rotate_restart_identity() { + let store = SqliteWorkspaceStore::in_memory().unwrap(); + store + .with_conn(|conn| { + conn.execute_batch( + r#" + INSERT INTO accounts(account_id, kind, handle, display_name, created_at, updated_at) + VALUES ('owner', 'user', 'owner', 'Owner', '1', '1'); + INSERT INTO workspaces(workspace_id, owner_account_id, display_name, state, created_at, updated_at) + VALUES ('workspace-a', 'owner', 'Workspace A', 'active', '1', '1'); + "#, + )?; + Ok(()) + }) + .unwrap(); + let first = + worker_runtime::auth::RuntimeIdentityMaterial::generate("embedded-first").unwrap(); + let second = + worker_runtime::auth::RuntimeIdentityMaterial::generate("embedded-second").unwrap(); + let binding = |public_key: String| WorkspaceRuntimeBinding { + workspace_id: "workspace-a".to_string(), + runtime_id: crate::hosts::EMBEDDED_RUNTIME_ID.to_string(), + display_name: "Embedded Runtime".to_string(), + base_url: "in-process://embedded".to_string(), + public_key, + public_key_fingerprint: String::new(), + binding_revision: 1, + created_at: "1".to_string(), + updated_at: "1".to_string(), + revoked_at: None, + }; + store + .upsert_workspace_runtime_binding(binding(first.public_key.clone()), false) + .unwrap(); + assert!(matches!( + store.upsert_workspace_runtime_binding(binding(second.public_key.clone()), false), + Err(Error::RuntimeBindingConflict(_)) + )); + assert_eq!( + store + .upsert_workspace_runtime_binding(binding(second.public_key.clone()), true) + .unwrap(), + WorkspaceRuntimeBindingUpsert::Replaced + ); + let persisted = store + .get_workspace_runtime_binding("workspace-a", crate::hosts::EMBEDDED_RUNTIME_ID) + .unwrap() + .unwrap(); + assert_eq!(persisted.public_key, second.public_key); + } + #[test] fn startup_rejects_prebaseline_workspace_history() { let conn = Connection::open_in_memory().unwrap(); configure_sqlite(&conn).unwrap(); conn.execute( - "INSERT INTO __yoi_schema_migrations (version, name) VALUES (49, 'legacy')", + "INSERT INTO __yoi_schema_migrations (version, name) VALUES (50, 'legacy')", [], ) .unwrap(); @@ -6353,7 +7686,7 @@ mod tests { assert!( error .to_string() - .contains("predates the canonical baseline") + .contains("predates the supported upgrade window") ); } @@ -6375,7 +7708,7 @@ mod tests { assert!( error .to_string() - .contains("migration history is not the canonical schema baseline") + .contains("migration history is not canonical") ); } @@ -6460,7 +7793,7 @@ mod tests { let db = dir.path().join("control-plane.sqlite"); let store = SqliteWorkspaceStore::open(&db).unwrap(); - assert_eq!(store.schema_version().await.unwrap(), 50); + assert_eq!(store.schema_version().await.unwrap(), LATEST_SCHEMA_VERSION); assert!( !store .with_conn(|conn| table_exists(conn, "worker_workspace_credentials")) @@ -6477,7 +7810,10 @@ mod tests { store.upsert_workspace(&record).await.unwrap(); let reopened = SqliteWorkspaceStore::open(&db).unwrap(); - assert_eq!(reopened.schema_version().await.unwrap(), 50); + assert_eq!( + reopened.schema_version().await.unwrap(), + LATEST_SCHEMA_VERSION + ); assert_eq!( reopened.get_workspace("local-dev").await.unwrap(), Some(record) @@ -6660,6 +7996,46 @@ mod tests { }) .unwrap(); assert_eq!(state, "created"); + store + .with_conn(|conn| { + conn.execute( + "INSERT INTO worker_registry ( + workspace_id, runtime_id, worker_id, display_name, + created_at, updated_at, retention_state + ) VALUES ('workspace-a', 'arcadia', ?1, 'Worker', '1', '1', 'normal')", + [reserved.worker_id.to_string()], + )?; + Ok(()) + }) + .unwrap(); + assert!( + store + .delete_worker_registry("workspace-a", &reserved_worker) + .unwrap() + ); + let removed_state: String = store + .with_conn(|conn| { + conn.query_row( + "SELECT state FROM worker_create_reservations + WHERE workspace_id = 'workspace-a' AND worker_id = ?1", + [reserved.worker_id.to_string()], + |row| row.get(0), + ) + .map_err(Error::from) + }) + .unwrap(); + assert_eq!(removed_state, "removed"); + assert!( + store + .reserve_worker_create( + "workspace-a", + "arcadia", + "operation-1", + "sha256:one", + &updated_memory_settings, + ) + .is_err() + ); store .with_conn(|conn| { @@ -7342,13 +8718,13 @@ INSERT INTO worker_registry ( let conn = Connection::open_in_memory().unwrap(); configure_sqlite(&conn).unwrap(); conn.execute( - "INSERT INTO __yoi_schema_migrations (version, name) VALUES (51, 'future')", + "INSERT INTO __yoi_schema_migrations (version, name) VALUES (54, 'future')", [], ) .unwrap(); let error = apply_migrations(&conn).unwrap_err().to_string(); - assert!(error.contains("schema version 51 is newer"), "{error}"); + assert!(error.contains("schema version 54 is newer"), "{error}"); assert!(error.contains("refusing to serve"), "{error}"); } @@ -7555,7 +8931,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn repository_records_round_trip() { let store = SqliteWorkspaceStore::in_memory().unwrap(); - assert_eq!(store.schema_version().await.unwrap(), 50); + assert_eq!(store.schema_version().await.unwrap(), LATEST_SCHEMA_VERSION); let workspace = WorkspaceRecord { workspace_id: "local-dev".to_string(), owner_account_id: "owner-account".to_string(), @@ -7636,7 +9012,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn memory_authority_records_round_trip_and_close_staging() { let store = SqliteWorkspaceStore::in_memory().unwrap(); - assert_eq!(store.schema_version().await.unwrap(), 50); + assert_eq!(store.schema_version().await.unwrap(), LATEST_SCHEMA_VERSION); let workspace = WorkspaceRecord { workspace_id: "local-dev".to_string(), owner_account_id: "owner-account".to_string(), @@ -8049,7 +9425,7 @@ INSERT INTO worker_registry ( #[tokio::test] async fn account_and_login_records_round_trip() { let store = SqliteWorkspaceStore::in_memory().unwrap(); - assert_eq!(store.schema_version().await.unwrap(), 50); + assert_eq!(store.schema_version().await.unwrap(), LATEST_SCHEMA_VERSION); let now = "2026-07-22T00:00:00Z".to_string(); let account = AccountRecord { account_id: "acct-user-alice".to_string(), diff --git a/crates/workspace-server/src/worker_source.rs b/crates/workspace-server/src/worker_source.rs index 1144a132..308b116a 100644 --- a/crates/workspace-server/src/worker_source.rs +++ b/crates/workspace-server/src/worker_source.rs @@ -57,16 +57,12 @@ pub async fn verify_runtime_request_source_proof_with_store( .map_err(|_| WorkerMutationSourceProofError::Invalid)?; let audience = remote_audience(config, &unverified.iss, workspace_id)?; let trusted = store - .get_trusted_runtime(&unverified.iss) + .get_workspace_runtime_binding(workspace_id, &unverified.iss) .await .map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))? .filter(|record| record.revoked_at.is_none()) .ok_or(WorkerMutationSourceProofError::RevokedRuntimeTrust)?; - let trusted_for_workspace = trusted.workspace_id.as_deref() == Some(workspace_id) - || (unverified.iss == crate::hosts::EMBEDDED_RUNTIME_ID && trusted.workspace_id.is_none()); - if !trusted_for_workspace { - return Err(WorkerMutationSourceProofError::WrongWorkspace); - } + let public_key = trusted.public_key.as_str(); let expected = RuntimeRequestSourceExpectation { identity_id: &unverified.iss, audience: audience.as_ref(), @@ -78,8 +74,8 @@ pub async fn verify_runtime_request_source_proof_with_store( body_digest, now_unix: i64::try_from(unix_now_seconds()).unwrap_or(i64::MAX), }; - let claims = verify_runtime_request_source(proof, &trusted.public_key, &expected) - .map_err(map_auth_error)?; + let claims = + verify_runtime_request_source(proof, public_key, &expected).map_err(map_auth_error)?; let now_seconds = u64::try_from(expected.now_unix).unwrap_or(u64::MAX); let expires_at = u64::try_from(claims.exp).unwrap_or(0); let consumed_at = chrono::DateTime::from_timestamp(expected.now_unix, 0) @@ -87,6 +83,7 @@ pub async fn verify_runtime_request_source_proof_with_store( .to_rfc3339(); if !store .consume_worker_mutation_source_jti( + workspace_id, &claims.iss, &claims.jti, expires_at, @@ -206,14 +203,12 @@ async fn verify_worker_remove_source_with( .map_err(|_| WorkerMutationSourceProofError::Invalid)?; let audience = remote_audience(config, &unverified.iss, &config.workspace_id)?; let trusted = store - .get_trusted_runtime(&unverified.iss) + .get_workspace_runtime_binding(&config.workspace_id, &unverified.iss) .await .map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))? .filter(|record| record.revoked_at.is_none()) .ok_or(WorkerMutationSourceProofError::RevokedRuntimeTrust)?; - if trusted.workspace_id.as_deref() != Some(config.workspace_id.as_str()) { - return Err(WorkerMutationSourceProofError::WrongWorkspace); - } + let public_key = trusted.public_key.as_str(); let expected = WorkerMutationSourceExpectation { runtime_id: &unverified.iss, audience: audience.as_ref(), @@ -225,7 +220,7 @@ async fn verify_worker_remove_source_with( target_worker_id, permission: required_permission, }; - verify_worker_mutation_source_proof(&trusted.public_key, token, &expected, now) + verify_worker_mutation_source_proof(public_key, token, &expected, now) .map_err(map_auth_error)? } PresentedWorkerMutationSourceProof::InProcess(proof) => { @@ -263,7 +258,14 @@ async fn verify_worker_remove_source_with( let consumed_at = chrono::Utc::now().to_rfc3339(); let consumed = store - .consume_worker_mutation_source_jti(&claims.iss, &claims.jti, claims.exp, now, &consumed_at) + .consume_worker_mutation_source_jti( + &config.workspace_id, + &claims.iss, + &claims.jti, + claims.exp, + now, + &consumed_at, + ) .await .map_err(|error| WorkerMutationSourceProofError::Authority(error.to_string()))?; if !consumed { diff --git a/crates/workspace-server/src/workspace_deletion.rs b/crates/workspace-server/src/workspace_deletion.rs new file mode 100644 index 00000000..1382edcf --- /dev/null +++ b/crates/workspace-server/src/workspace_deletion.rs @@ -0,0 +1,1322 @@ +use chrono::Utc; +use rusqlite::{OptionalExtension, params}; +use sha2::{Digest, Sha256}; +use workspace_api::{ + WORKSPACE_DELETION_MAX_BLOCKER_MESSAGE_BYTES, WORKSPACE_DELETION_MAX_BLOCKERS, + WORKSPACE_DELETION_MAX_CHILD_OPERATION_IDS, WORKSPACE_DELETION_MAX_OPERATION_ID_BYTES, + WORKSPACE_DELETION_MAX_RESOURCE_VALUE_BYTES, WORKSPACE_DELETION_MAX_REVISION_BYTES, + WorkspaceDeletionBlocker, WorkspaceDeletionBlockerKind, WorkspaceDeletionOperationResponse, + WorkspaceDeletionPreflightResponse, WorkspaceDeletionRequest, WorkspaceDeletionResourceCounts, + WorkspaceDeletionState, +}; + +use crate::store::{SqliteWorkspaceStore, WorkspaceRecord}; +use crate::{Error, Result}; + +/// Explicit domain-owned purge inventory. The deletion operation tombstone is intentionally +/// excluded so retries and audit remain available after the Workspace row is gone. +const WORKSPACE_DELETION_PURGE_TABLES: &[&str] = &[ + "artifacts", + "audit_events", + "flow_source_revisions", + "flow_sources", + "memory_staging_records", + "memory_staging_resolutions", + "merge_request_review_grants", + "merge_request_reviewer_child_sessions", + "merge_request_thread_events", + "merge_request_ticket_relations", + "merge_requests", + "objective_events", + "objective_resources", + "objective_ticket_links", + "objectives", + "repositories", + "repository_secret_audit_events", + "repository_secret_operations", + "repository_ssh_credential_revisions", + "repository_ssh_credentials", + "repository_ssh_host_trust_revisions", + "repository_ssh_host_trusts", + "server_secret_versions", + "ticket_assignment_operations", + "ticket_assignment_ticket_tombstones", + "ticket_assignment_worker_tombstones", + "ticket_current_worker_assignments", + "ticket_worker_assignment_events", + "ticket_worker_assignments", + "typed_ticket_artifacts", + "typed_ticket_event_attributes", + "typed_ticket_event_references", + "typed_ticket_events", + "typed_ticket_labels", + "typed_ticket_orchestration_plans", + "typed_ticket_raw_frontmatter", + "typed_ticket_relations", + "typed_ticket_risk_flags", + "typed_tickets", + "workdir_create_operations", + "workdir_registry", + "workdir_removal_operations", + "worker_control_grants", + "worker_create_reservations", + "worker_diagnostics_archives", + "worker_mutation_source_proof_jtis", + "worker_orphan_diagnostics", + "worker_registry", + "worker_removal_operations", + "worker_retention_audit_events", + "worker_session_archives", + "worker_tombstones", + "worker_workdir_attachment_reservations", + "worker_workdir_links", + "workspace_config_entries", + "workspace_config_tree_revisions", + "workspace_config_trees", + "workspace_create_operations", + "workspace_memory_documents", + "workspace_memory_settings", + "workspace_resource_key_counters", + "workspace_resource_keys", + "workspace_runtime_binding_audit", + "workspace_runtime_bindings", + "workspace_worker_retention_policies", + "workspace_worker_retention_policy_revisions", +]; + +#[derive(Debug, Clone)] +pub struct WorkspaceDeletionReservation { + pub operation: WorkspaceDeletionOperationResponse, + pub replay: bool, +} + +pub trait WorkspaceDeletionStore: Send + Sync { + fn workspace_deletion_preflight( + &self, + actor_account_id: &str, + workspace_id: &str, + ) -> Result; + + fn reserve_workspace_deletion( + &self, + actor_account_id: &str, + workspace_id: &str, + request: &WorkspaceDeletionRequest, + ) -> Result; + + fn workspace_deletion_operation( + &self, + actor_account_id: &str, + operation_id: &str, + ) -> Result>; + + fn workspace_deletion_operation_for_recovery( + &self, + operation_id: &str, + ) -> Result>; + + fn resumable_workspace_deletion_operation_ids(&self) -> Result>; + + fn append_workspace_deletion_child_operation( + &self, + operation_id: &str, + child_operation_id: &str, + ) -> Result; + + fn update_workspace_deletion_operation( + &self, + operation_id: &str, + state: WorkspaceDeletionState, + child_operation_ids: &[String], + blockers: &[WorkspaceDeletionBlocker], + failure_category: Option<&str>, + ) -> Result; + + fn finalize_workspace_deletion( + &self, + operation_id: &str, + ) -> Result; +} + +impl WorkspaceDeletionStore for SqliteWorkspaceStore { + fn workspace_deletion_preflight( + &self, + actor_account_id: &str, + workspace_id: &str, + ) -> Result { + self.with_conn(|conn| { + let workspace = owner_workspace(conn, actor_account_id, workspace_id)?; + let resources = resource_counts(conn, workspace_id)?; + let accessible: u64 = conn.query_row( + "SELECT COUNT(*) FROM workspaces WHERE owner_account_id = ?1", + params![actor_account_id], + |row| row.get(0), + )?; + let mut blockers = workspace_database_blockers(conn, workspace_id)?; + if accessible <= 1 { + blockers.push(WorkspaceDeletionBlocker { + kind: WorkspaceDeletionBlockerKind::LastAccessibleWorkspace, + resource_kind: None, + resource_key: None, + message: "You cannot delete your last accessible Workspace.".to_string(), + }); + } + if resources.workers.saturating_add(resources.workdirs) + > WORKSPACE_DELETION_MAX_CHILD_OPERATION_IDS as u64 + { + blockers.push(WorkspaceDeletionBlocker { + kind: WorkspaceDeletionBlockerKind::CleanupUnavailable, + resource_kind: None, + resource_key: None, + message: + "Workspace cleanup exceeds the supported durable child-operation bound." + .to_string(), + }); + } + bound_workspace_deletion_blockers(&mut blockers); + Ok(WorkspaceDeletionPreflightResponse { + workspace_id: workspace.workspace_id, + display_name: workspace.display_name, + expected_revision: workspace.updated_at, + can_delete: blockers.is_empty(), + resources, + blockers, + }) + }) + } + + fn reserve_workspace_deletion( + &self, + actor_account_id: &str, + workspace_id: &str, + request: &WorkspaceDeletionRequest, + ) -> Result { + validate_operation_id(&request.operation_id)?; + if request.expected_revision.len() > WORKSPACE_DELETION_MAX_REVISION_BYTES + || request.confirmation.len() > workspace_api::WORKSPACE_DELETION_MAX_CONFIRMATION_BYTES + { + return Err(Error::InvalidInput( + "Workspace deletion request exceeds bounded field limits".to_string(), + )); + } + self.with_transaction(|tx| { + if let Some(existing) = read_operation(tx, &request.operation_id)? { + if existing.actor_account_id != actor_account_id { + return Err(Error::WorkspacePermissionDenied( + "Workspace deletion operation is not owned by the current account".to_string(), + )); + } + let expected_fingerprint = request_fingerprint(actor_account_id, workspace_id, request); + if existing.request_fingerprint != expected_fingerprint { + return Err(Error::WorkspaceConfigConflict( + "Workspace deletion operation_id was reused with different intent".to_string(), + )); + } + return Ok(WorkspaceDeletionReservation { + operation: existing.response, + replay: true, + }); + } + + let workspace = owner_workspace(tx, actor_account_id, workspace_id)?; + if request.confirmation != workspace.display_name { + return Err(Error::InvalidInput( + "confirmation must exactly match the displayed Workspace name".to_string(), + )); + } + if workspace.updated_at != request.expected_revision { + return Err(Error::WorkspaceConfigConflict( + "Workspace metadata changed; reload deletion impact before confirming".to_string(), + )); + } + let accessible: u64 = tx.query_row( + "SELECT COUNT(*) FROM workspaces WHERE owner_account_id = ?1", + params![actor_account_id], + |row| row.get(0), + )?; + if accessible <= 1 { + return Err(Error::WorkspaceConfigConflict( + "last_accessible_workspace: create or retain another accessible Workspace first" + .to_string(), + )); + } + if !workspace_database_blockers(tx, workspace_id)?.is_empty() { + return Err(Error::WorkspaceConfigConflict( + "Workspace deletion preflight changed; reload current blockers".to_string(), + )); + } + + let resources = resource_counts(tx, workspace_id)?; + let now = Utc::now().to_rfc3339(); + let fingerprint = request_fingerprint(actor_account_id, workspace_id, request); + tx.execute( + "INSERT INTO workspace_deletion_operations ( + operation_id, request_fingerprint, workspace_id, workspace_display_name, + workspace_revision, owner_account_id, actor_account_id, + state, resource_counts_json, + child_operation_ids_json, blockers_json, failure_category, + created_at, updated_at, completed_at + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, 'queued', ?8, '[]', '[]', NULL, ?9, ?9, NULL)", + params![ + request.operation_id, + fingerprint, + workspace_id, + workspace.display_name, + request.expected_revision, + workspace.owner_account_id, + actor_account_id, + serde_json::to_string(&resources).map_err(|error| Error::Store(error.to_string()))?, + now, + ], + )?; + let changed = tx.execute( + "UPDATE workspaces SET state = 'deleting', updated_at = ?2 + WHERE workspace_id = ?1 AND updated_at = ?3 AND state = 'active'", + params![workspace_id, now, request.expected_revision], + )?; + if changed != 1 { + return Err(Error::WorkspaceConfigConflict( + "Workspace lifecycle changed before deletion could be reserved".to_string(), + )); + } + let operation = read_operation(tx, &request.operation_id)? + .ok_or_else(|| Error::Store("reserved Workspace deletion operation disappeared".to_string()))?; + Ok(WorkspaceDeletionReservation { + operation: operation.response, + replay: false, + }) + }) + } + + fn workspace_deletion_operation( + &self, + actor_account_id: &str, + operation_id: &str, + ) -> Result> { + self.with_conn(|conn| { + let Some(operation) = read_operation(conn, operation_id)? else { + return Ok(None); + }; + if operation.actor_account_id != actor_account_id { + return Err(Error::WorkspacePermissionDenied( + "Workspace deletion operation is not owned by the current account".to_string(), + )); + } + Ok(Some(operation.response)) + }) + } + + fn workspace_deletion_operation_for_recovery( + &self, + operation_id: &str, + ) -> Result> { + self.with_conn(|conn| Ok(read_operation(conn, operation_id)?.map(|stored| stored.response))) + } + + fn resumable_workspace_deletion_operation_ids(&self) -> Result> { + self.with_conn(|conn| { + let mut statement = conn.prepare( + "SELECT operation_id FROM workspace_deletion_operations + WHERE state IN ('queued', 'running') ORDER BY created_at, operation_id", + )?; + statement + .query_map([], |row| row.get(0))? + .collect::, _>>() + .map_err(Into::into) + }) + } + + fn append_workspace_deletion_child_operation( + &self, + operation_id: &str, + child_operation_id: &str, + ) -> Result { + validate_operation_id(child_operation_id)?; + self.with_transaction(|tx| { + let operation = read_operation(tx, operation_id)? + .ok_or_else(|| Error::InvalidInput("Workspace deletion operation".to_string()))? + .response; + if !operation + .child_operation_ids + .iter() + .any(|existing| existing == child_operation_id) + { + let mut child_operation_ids = operation.child_operation_ids; + child_operation_ids.push(child_operation_id.to_string()); + validate_operation_projection(&child_operation_ids, &operation.blockers)?; + let now = Utc::now().to_rfc3339(); + tx.execute( + "UPDATE workspace_deletion_operations + SET child_operation_ids_json = ?2, updated_at = ?3 + WHERE operation_id = ?1", + params![ + operation_id, + serde_json::to_string(&child_operation_ids) + .map_err(|error| Error::Store(error.to_string()))?, + now, + ], + )?; + } + Ok(read_operation(tx, operation_id)? + .ok_or_else(|| { + Error::Store("Workspace deletion operation disappeared".to_string()) + })? + .response) + }) + } + + fn update_workspace_deletion_operation( + &self, + operation_id: &str, + state: WorkspaceDeletionState, + child_operation_ids: &[String], + blockers: &[WorkspaceDeletionBlocker], + failure_category: Option<&str>, + ) -> Result { + validate_operation_projection(child_operation_ids, blockers)?; + self.with_transaction(|tx| { + let current = read_operation(tx, operation_id)? + .ok_or_else(|| Error::InvalidInput("Workspace deletion operation".to_string()))? + .response; + let mut merged_child_operation_ids = current.child_operation_ids; + for child_operation_id in child_operation_ids { + if !merged_child_operation_ids + .iter() + .any(|existing| existing == child_operation_id) + { + merged_child_operation_ids.push(child_operation_id.clone()); + } + } + validate_operation_projection(&merged_child_operation_ids, blockers)?; + let now = Utc::now().to_rfc3339(); + let completed_at = + matches!(state, WorkspaceDeletionState::Succeeded).then_some(now.as_str()); + let changed = tx.execute( + "UPDATE workspace_deletion_operations + SET state = ?2, child_operation_ids_json = ?3, blockers_json = ?4, + failure_category = ?5, updated_at = ?6, completed_at = ?7 + WHERE operation_id = ?1", + params![ + operation_id, + deletion_state_label(state), + serde_json::to_string(&merged_child_operation_ids) + .map_err(|error| Error::Store(error.to_string()))?, + serde_json::to_string(blockers) + .map_err(|error| Error::Store(error.to_string()))?, + failure_category, + now, + completed_at, + ], + )?; + if changed != 1 { + return Err(Error::InvalidInput( + "Workspace deletion operation".to_string(), + )); + } + Ok(read_operation(tx, operation_id)? + .ok_or_else(|| { + Error::Store("Workspace deletion operation disappeared".to_string()) + })? + .response) + }) + } + + fn finalize_workspace_deletion( + &self, + operation_id: &str, + ) -> Result { + self.with_transaction(|tx| { + tx.execute_batch("PRAGMA defer_foreign_keys = ON;")?; + let operation = read_operation(tx, operation_id)? + .ok_or_else(|| Error::InvalidInput("Workspace deletion operation".to_string()))?; + if operation.response.state == WorkspaceDeletionState::Succeeded { + return Ok(operation.response); + } + let workspace_id = operation.response.workspace_id.clone(); + + for table in WORKSPACE_DELETION_PURGE_TABLES { + tx.execute( + &format!("DELETE FROM \"{table}\" WHERE workspace_id = ?1"), + params![workspace_id], + )?; + } + let deleted = tx.execute( + "DELETE FROM workspaces WHERE workspace_id = ?1", + params![workspace_id], + )?; + if deleted != 1 { + return Err(Error::WorkspaceConfigConflict( + "Workspace disappeared before deletion finalized".to_string(), + )); + } + let now = Utc::now().to_rfc3339(); + tx.execute( + "UPDATE workspace_deletion_operations + SET state = 'succeeded', blockers_json = '[]', failure_category = NULL, + updated_at = ?2, completed_at = ?2 + WHERE operation_id = ?1", + params![operation_id, now], + )?; + let fk_failures: u64 = + tx.query_row("SELECT COUNT(*) FROM pragma_foreign_key_check", [], |row| { + row.get(0) + })?; + if fk_failures != 0 { + return Err(Error::Store( + "foreign key check failed while finalizing Workspace deletion".to_string(), + )); + } + Ok(read_operation(tx, operation_id)? + .ok_or_else(|| { + Error::Store("completed Workspace deletion operation disappeared".to_string()) + })? + .response) + }) + } +} + +#[derive(Debug)] +struct StoredOperation { + request_fingerprint: String, + actor_account_id: String, + response: WorkspaceDeletionOperationResponse, +} + +fn read_operation( + conn: &rusqlite::Connection, + operation_id: &str, +) -> Result> { + conn.query_row( + "SELECT request_fingerprint, actor_account_id, workspace_id, workspace_display_name, + state, resource_counts_json, child_operation_ids_json, + blockers_json, failure_category, created_at, updated_at, completed_at + FROM workspace_deletion_operations WHERE operation_id = ?1", + params![operation_id], + |row| { + let state: String = row.get(4)?; + let resource_counts_json: String = row.get(5)?; + let child_operation_ids_json: String = row.get(6)?; + let blockers_json: String = row.get(7)?; + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + state, + resource_counts_json, + child_operation_ids_json, + blockers_json, + row.get::<_, Option>(8)?, + row.get::<_, String>(9)?, + row.get::<_, String>(10)?, + row.get::<_, Option>(11)?, + )) + }, + ) + .optional()? + .map( + |( + fingerprint, + actor, + workspace_id, + display_name, + state, + resources, + children, + blockers, + failure, + created_at, + updated_at, + completed_at, + )| { + Ok(StoredOperation { + request_fingerprint: fingerprint, + actor_account_id: actor, + response: WorkspaceDeletionOperationResponse { + operation_id: operation_id.to_string(), + workspace_id, + display_name, + state: parse_deletion_state(&state)?, + resources: serde_json::from_str(&resources) + .map_err(|error| Error::Store(error.to_string()))?, + child_operation_ids: serde_json::from_str(&children) + .map_err(|error| Error::Store(error.to_string()))?, + blockers: serde_json::from_str(&blockers) + .map_err(|error| Error::Store(error.to_string()))?, + failure_category: failure, + created_at, + updated_at, + completed_at, + }, + }) + }, + ) + .transpose() +} + +fn owner_workspace( + conn: &rusqlite::Connection, + actor_account_id: &str, + workspace_id: &str, +) -> Result { + let workspace = conn + .query_row( + "SELECT workspace_id, owner_account_id, display_name, state, created_at, updated_at + FROM workspaces WHERE workspace_id = ?1", + params![workspace_id], + |row| { + Ok(WorkspaceRecord { + workspace_id: row.get(0)?, + owner_account_id: row.get(1)?, + display_name: row.get(2)?, + state: row.get(3)?, + created_at: row.get(4)?, + updated_at: row.get(5)?, + }) + }, + ) + .optional()? + .ok_or_else(|| Error::InvalidInput("Workspace".to_string()))?; + if workspace.owner_account_id != actor_account_id { + return Err(Error::WorkspacePermissionDenied( + "Workspace owner permission is required".to_string(), + )); + } + Ok(workspace) +} + +pub(crate) fn bound_workspace_deletion_blockers(blockers: &mut Vec) { + for blocker in blockers.iter_mut() { + blocker.resource_kind = blocker + .resource_kind + .take() + .map(|value| truncate_utf8(value, WORKSPACE_DELETION_MAX_RESOURCE_VALUE_BYTES)); + blocker.resource_key = blocker + .resource_key + .take() + .map(|value| truncate_utf8(value, WORKSPACE_DELETION_MAX_RESOURCE_VALUE_BYTES)); + blocker.message = truncate_utf8( + std::mem::take(&mut blocker.message), + WORKSPACE_DELETION_MAX_BLOCKER_MESSAGE_BYTES, + ); + } + if blockers.len() > WORKSPACE_DELETION_MAX_BLOCKERS { + blockers.truncate(WORKSPACE_DELETION_MAX_BLOCKERS - 1); + blockers.push(WorkspaceDeletionBlocker { + kind: WorkspaceDeletionBlockerKind::CleanupUnavailable, + resource_kind: None, + resource_key: None, + message: "Additional deletion blockers exist; reduce Workspace resources and run preflight again." + .to_string(), + }); + } +} + +fn truncate_utf8(mut value: String, max_bytes: usize) -> String { + if value.len() <= max_bytes { + return value; + } + let mut end = max_bytes; + while !value.is_char_boundary(end) { + end -= 1; + } + value.truncate(end); + value +} + +fn workspace_database_blockers( + conn: &rusqlite::Connection, + workspace_id: &str, +) -> Result> { + let mut blockers = Vec::new(); + for (sql, kind, resource_kind, message) in [ + ( + "SELECT workdir_id FROM worker_workdir_links WHERE workspace_id = ?1 AND unlinked_at IS NULL", + WorkspaceDeletionBlockerKind::WorkdirRemovalBlocked, + "workdir", + "Release this active Worker–Workdir attachment before deleting the Workspace.", + ), + ( + "SELECT workdir_id FROM worker_workdir_attachment_reservations WHERE workspace_id = ?1", + WorkspaceDeletionBlockerKind::WorkdirRemovalBlocked, + "workdir", + "Wait for or cancel this pending Workdir attachment reservation.", + ), + ( + "SELECT display_name FROM worker_registry WHERE workspace_id = ?1 AND retention_state = 'pinned'", + WorkspaceDeletionBlockerKind::RetentionHold, + "worker", + "Remove this Worker retention pin before deleting the Workspace.", + ), + ( + "SELECT workdir_id FROM workdir_registry WHERE workspace_id = ?1 AND COALESCE(cleanliness, '') != 'clean'", + WorkspaceDeletionBlockerKind::DirtyWorkdir, + "workdir", + "Clean this Workdir and refresh unknown cleanliness before deleting the Workspace.", + ), + ] { + for resource_key in query_resource_keys(conn, sql, workspace_id)? { + blockers.push(WorkspaceDeletionBlocker { + kind, + resource_kind: Some(resource_kind.to_string()), + resource_key: Some(resource_key), + message: message.to_string(), + }); + } + } + + for (sql, kind, resource_kind, message) in [ + ( + "SELECT COUNT(*) FROM ticket_current_worker_assignments WHERE workspace_id = ?1", + WorkspaceDeletionBlockerKind::WorkerRemovalBlocked, + "ticket", + "Remove current Ticket assignments before deleting the Workspace.", + ), + ( + "SELECT COUNT(*) + FROM worker_create_reservations reservation + WHERE reservation.workspace_id = ?1 + AND ( + reservation.state = 'reserved' + OR ( + reservation.state = 'created' + AND NOT EXISTS ( + SELECT 1 FROM worker_registry worker + WHERE worker.workspace_id = reservation.workspace_id + AND worker.runtime_id = reservation.runtime_id + AND worker.worker_id = reservation.worker_id + ) + ) + )", + WorkspaceDeletionBlockerKind::CleanupUnavailable, + "worker", + "Reconcile pending or incompletely finalized Worker creation reservations.", + ), + ( + "SELECT COUNT(*) FROM worker_removal_operations WHERE workspace_id = ?1 AND state IN ('planned', 'blocked', 'executing', 'failed', 'stale')", + WorkspaceDeletionBlockerKind::CleanupUnavailable, + "worker", + "Resolve pending or failed Worker removal operations first.", + ), + ( + "SELECT COUNT(*) FROM workdir_removal_operations WHERE workspace_id = ?1 AND state IN ('pending', 'failed')", + WorkspaceDeletionBlockerKind::CleanupUnavailable, + "workdir", + "Resolve pending or failed Workdir removal operations first.", + ), + ( + "SELECT COUNT(*) FROM workdir_create_operations WHERE workspace_id = ?1 AND state = 'pending'", + WorkspaceDeletionBlockerKind::CleanupUnavailable, + "workdir", + "Wait for pending Workdir creation operations to finish.", + ), + ] { + let count: u64 = conn.query_row(sql, params![workspace_id], |row| row.get(0))?; + if count != 0 { + blockers.push(WorkspaceDeletionBlocker { + kind, + resource_kind: Some(resource_kind.to_string()), + resource_key: None, + message: format!("{message} ({count})"), + }); + } + } + Ok(blockers) +} + +fn query_resource_keys( + conn: &rusqlite::Connection, + sql: &str, + workspace_id: &str, +) -> Result> { + let mut statement = conn.prepare(sql)?; + statement + .query_map(params![workspace_id], |row| row.get(0))? + .collect::, _>>() + .map_err(Into::into) +} + +fn resource_counts( + conn: &rusqlite::Connection, + workspace_id: &str, +) -> Result { + Ok(WorkspaceDeletionResourceCounts { + workers: table_count(conn, "worker_registry", workspace_id)?, + workdirs: table_count(conn, "workdir_registry", workspace_id)?, + repositories: table_count(conn, "repositories", workspace_id)?, + runtime_bindings: table_count(conn, "workspace_runtime_bindings", workspace_id)?, + secrets: table_count(conn, "server_secret_versions", workspace_id)?, + artifacts: table_count(conn, "artifacts", workspace_id)?, + }) +} + +fn table_count(conn: &rusqlite::Connection, table: &str, workspace_id: &str) -> Result { + conn.query_row( + &format!("SELECT COUNT(*) FROM \"{table}\" WHERE workspace_id = ?1"), + params![workspace_id], + |row| row.get(0), + ) + .map_err(Into::into) +} + +fn validate_operation_projection( + child_operation_ids: &[String], + blockers: &[WorkspaceDeletionBlocker], +) -> Result<()> { + let invalid_child_ids = child_operation_ids.len() > WORKSPACE_DELETION_MAX_CHILD_OPERATION_IDS + || child_operation_ids + .iter() + .any(|value| value.len() > WORKSPACE_DELETION_MAX_OPERATION_ID_BYTES); + let invalid_blockers = + blockers.len() > WORKSPACE_DELETION_MAX_BLOCKERS + || blockers.iter().any(|blocker| { + blocker.message.len() > WORKSPACE_DELETION_MAX_BLOCKER_MESSAGE_BYTES + || blocker.resource_kind.as_ref().is_some_and(|value| { + value.len() > WORKSPACE_DELETION_MAX_RESOURCE_VALUE_BYTES + }) + || blocker.resource_key.as_ref().is_some_and(|value| { + value.len() > WORKSPACE_DELETION_MAX_RESOURCE_VALUE_BYTES + }) + }); + if invalid_child_ids || invalid_blockers { + return Err(Error::Store( + "Workspace deletion operation projection exceeds bounded limits".to_string(), + )); + } + Ok(()) +} + +fn validate_operation_id(operation_id: &str) -> Result<()> { + if operation_id.is_empty() + || operation_id.len() > WORKSPACE_DELETION_MAX_OPERATION_ID_BYTES + || !operation_id + .bytes() + .all(|byte| byte.is_ascii_alphanumeric() || matches!(byte, b'-' | b'_')) + { + return Err(Error::InvalidInput("operation_id is invalid".to_string())); + } + Ok(()) +} + +fn request_fingerprint( + actor_account_id: &str, + workspace_id: &str, + request: &WorkspaceDeletionRequest, +) -> String { + let canonical = format!( + "workspace-delete-v1\0{actor_account_id}\0{workspace_id}\0{}\0{}\0{}", + request.operation_id, request.expected_revision, request.confirmation + ); + encode_hex(&Sha256::digest(canonical.as_bytes())) +} + +fn encode_hex(bytes: &[u8]) -> String { + const HEX: &[u8; 16] = b"0123456789abcdef"; + let mut output = String::with_capacity(bytes.len() * 2); + for &byte in bytes { + output.push(HEX[(byte >> 4) as usize] as char); + output.push(HEX[(byte & 0x0f) as usize] as char); + } + output +} + +fn deletion_state_label(state: WorkspaceDeletionState) -> &'static str { + match state { + WorkspaceDeletionState::Queued => "queued", + WorkspaceDeletionState::Running => "running", + WorkspaceDeletionState::Blocked => "blocked", + WorkspaceDeletionState::Failed => "failed", + WorkspaceDeletionState::Succeeded => "succeeded", + } +} + +fn parse_deletion_state(value: &str) -> Result { + match value { + "queued" => Ok(WorkspaceDeletionState::Queued), + "running" => Ok(WorkspaceDeletionState::Running), + "blocked" => Ok(WorkspaceDeletionState::Blocked), + "failed" => Ok(WorkspaceDeletionState::Failed), + "succeeded" => Ok(WorkspaceDeletionState::Succeeded), + other => Err(Error::Store(format!( + "invalid Workspace deletion state `{other}`" + ))), + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::store::ControlPlaneStore; + use std::collections::BTreeSet; + use tempfile::tempdir; + + fn setup() -> (SqliteWorkspaceStore, String, String) { + let dir = tempdir().expect("tempdir"); + let store = SqliteWorkspaceStore::open(dir.path().join("server.db")).expect("store"); + store.with_conn(|conn| { + let now = Utc::now().to_rfc3339(); + conn.execute( + "INSERT INTO accounts (account_id, kind, handle, display_name, created_at, updated_at) + VALUES ('owner', 'user', 'owner', 'Owner', ?1, ?1)", + params![now], + )?; + for (id, name) in [("workspace-a", "Alpha"), ("workspace-b", "Beta")] { + conn.execute( + "INSERT INTO workspaces (workspace_id, owner_account_id, display_name, state, created_at, updated_at) + VALUES (?1, 'owner', ?2, 'active', ?3, ?3)", + params![id, name, now], + )?; + } + Ok(()) + }).expect("fixtures"); + (store, "owner".to_string(), "workspace-a".to_string()) + } + + #[test] + fn explicit_purge_inventory_covers_every_workspace_scoped_table() { + let (store, _, _) = setup(); + store + .with_conn(|conn| { + let mut statement = conn.prepare( + "SELECT name FROM sqlite_master + WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name", + )?; + let tables = statement + .query_map([], |row| row.get::<_, String>(0))? + .collect::, _>>()?; + drop(statement); + let mut scoped = BTreeSet::new(); + for table in tables { + let escaped = table.replace('"', "\"\""); + let mut info = conn.prepare(&format!("PRAGMA table_info(\"{escaped}\")"))?; + let columns = info + .query_map([], |row| row.get::<_, String>(1))? + .collect::, _>>()?; + if columns.iter().any(|column| column == "workspace_id") { + scoped.insert(table); + } + } + let expected = WORKSPACE_DELETION_PURGE_TABLES + .iter() + .copied() + .chain(["workspace_deletion_operations", "workspaces"]) + .map(str::to_string) + .collect::>(); + assert_eq!(scoped, expected); + Ok(()) + }) + .expect("purge inventory"); + } + + #[test] + fn deletion_is_idempotent_and_removes_workspace_scoped_rows() { + let (store, owner, workspace_id) = setup(); + store + .with_conn(|conn| { + conn.execute( + "INSERT INTO worker_mutation_source_proof_jtis ( + workspace_id, runtime_id, jti, expires_at, consumed_at + ) VALUES (?1, 'runtime-a', 'jti-a', 1, '1')", + params![workspace_id], + )?; + Ok(()) + }) + .expect("non-FK scoped audit fixture"); + let preflight = store + .workspace_deletion_preflight(&owner, &workspace_id) + .expect("preflight"); + let request = WorkspaceDeletionRequest { + operation_id: "delete-workspace-a".to_string(), + expected_revision: preflight.expected_revision, + confirmation: "Alpha".to_string(), + }; + let first = store + .reserve_workspace_deletion(&owner, &workspace_id, &request) + .expect("reserve"); + assert!(!first.replay); + assert_eq!( + store + .resumable_workspace_deletion_operation_ids() + .expect("resumable operations"), + vec![request.operation_id.clone()] + ); + let replay = store + .reserve_workspace_deletion(&owner, &workspace_id, &request) + .expect("replay"); + assert!(replay.replay); + assert!(matches!( + store.update_workspace_deletion_operation( + &request.operation_id, + WorkspaceDeletionState::Running, + &vec!["child".to_string(); WORKSPACE_DELETION_MAX_CHILD_OPERATION_IDS + 1], + &[], + None, + ), + Err(Error::Store(_)) + )); + let with_child = store + .append_workspace_deletion_child_operation(&request.operation_id, "child-operation-1") + .expect("append child operation"); + assert_eq!( + with_child.child_operation_ids, + vec!["child-operation-1".to_string()] + ); + let duplicate = store + .append_workspace_deletion_child_operation(&request.operation_id, "child-operation-1") + .expect("append child operation replay"); + assert_eq!( + duplicate.child_operation_ids, + with_child.child_operation_ids + ); + let stale_failure_update = store + .update_workspace_deletion_operation( + &request.operation_id, + WorkspaceDeletionState::Blocked, + &[], + &[], + Some("retryable_failure"), + ) + .expect("stale failure update"); + assert_eq!( + stale_failure_update.child_operation_ids, + with_child.child_operation_ids + ); + let completed = store + .finalize_workspace_deletion(&request.operation_id) + .expect("finalize"); + assert_eq!(completed.state, WorkspaceDeletionState::Succeeded); + assert_eq!( + completed.child_operation_ids, + vec!["child-operation-1".to_string()] + ); + let replayed = store + .finalize_workspace_deletion(&request.operation_id) + .expect("finalize replay"); + assert_eq!(completed, replayed); + assert!( + store + .resumable_workspace_deletion_operation_ids() + .expect("terminal operations") + .is_empty() + ); + let workspace_count: u64 = store + .with_conn(|conn| { + conn.query_row( + "SELECT COUNT(*) FROM workspaces WHERE workspace_id = ?1", + params![workspace_id], + |row| row.get(0), + ) + .map_err(Into::into) + }) + .expect("read"); + assert_eq!(workspace_count, 0); + let proof_count: u64 = store + .with_conn(|conn| { + conn.query_row( + "SELECT COUNT(*) FROM worker_mutation_source_proof_jtis WHERE workspace_id = ?1", + params![workspace_id], + |row| row.get(0), + ) + .map_err(Into::into) + }) + .expect("scoped audit read"); + assert_eq!(proof_count, 0); + } + + #[test] + fn owner_confirmation_and_revision_are_required_before_reservation() { + let (store, owner, workspace_id) = setup(); + store.with_conn(|conn| { + conn.execute( + "INSERT INTO accounts (account_id, kind, handle, display_name, created_at, updated_at) + VALUES ('other', 'user', 'other', 'Other', '1', '1')", + [], + )?; + Ok(()) + }).expect("other account"); + assert!(matches!( + store.workspace_deletion_preflight("other", &workspace_id), + Err(Error::WorkspacePermissionDenied(_)) + )); + + let preflight = store + .workspace_deletion_preflight(&owner, &workspace_id) + .expect("preflight"); + let mut request = WorkspaceDeletionRequest { + operation_id: "delete-alpha-guarded".to_string(), + expected_revision: "stale".to_string(), + confirmation: "Alpha".to_string(), + }; + assert!(matches!( + store.reserve_workspace_deletion(&owner, &workspace_id, &request), + Err(Error::WorkspaceConfigConflict(_)) + )); + let mut other_operation = request.clone(); + other_operation.operation_id = "delete-alpha-other-operation".to_string(); + assert_ne!( + request_fingerprint(&owner, &workspace_id, &request), + request_fingerprint(&owner, &workspace_id, &other_operation) + ); + request.expected_revision = preflight.expected_revision; + request.confirmation = "delete Alpha".to_string(); + assert!(matches!( + store.reserve_workspace_deletion(&owner, &workspace_id, &request), + Err(Error::InvalidInput(_)) + )); + } + + #[test] + fn incomplete_worker_create_and_pending_workdir_create_block_without_orphans() { + let (store, owner, workspace_id) = setup(); + store + .with_conn(|conn| { + conn.execute( + "INSERT INTO worker_create_reservations ( + workspace_id, allocation_key, worker_id, runtime_id, + create_fingerprint, state, created_at, updated_at + ) VALUES (?1, 'allocation', 'worker-pending', 'runtime-a', + 'fingerprint', 'created', '1', '1')", + params![workspace_id], + )?; + conn.execute( + "INSERT INTO workdir_create_operations ( + workspace_id, operation_id, request_fingerprint, repository_id, + selector, requested_runtime_id, resolved_runtime_id, config_revision, + config_projection_digest, working_directory_id, state, created_at, updated_at + ) VALUES ( + ?1, 'workdir-create', 'fingerprint', 'repository-pending', + 'develop', 'runtime-a', 'runtime-a', 1, + 'projection', 'workdir-pending', 'pending', '1', '1' + )", + params![workspace_id], + )?; + Ok(()) + }) + .expect("pending creation fixtures"); + let preflight = store + .workspace_deletion_preflight(&owner, &workspace_id) + .expect("preflight"); + assert!(!preflight.can_delete); + assert!( + preflight + .blockers + .iter() + .any(|blocker| blocker.message.contains("Worker creation reservations")) + ); + assert!( + preflight + .blockers + .iter() + .any(|blocker| blocker.message.contains("Workdir creation operations")) + ); + let request = WorkspaceDeletionRequest { + operation_id: "delete-with-pending-creates".to_string(), + expected_revision: preflight.expected_revision, + confirmation: "Alpha".to_string(), + }; + assert!(matches!( + store.reserve_workspace_deletion(&owner, &workspace_id, &request), + Err(Error::WorkspaceConfigConflict(_)) + )); + assert!( + store + .workspace_deletion_operation_for_recovery(&request.operation_id) + .expect("operation lookup") + .is_none() + ); + + store + .with_conn(|conn| { + conn.execute( + "INSERT INTO worker_registry ( + workspace_id, runtime_id, worker_id, display_name, + created_at, updated_at, retention_state + ) VALUES (?1, 'runtime-a', 'worker-pending', 'Created worker', '1', '1', 'normal')", + params![workspace_id], + )?; + conn.execute( + "DELETE FROM workdir_create_operations WHERE operation_id = 'workdir-create'", + [], + )?; + Ok(()) + }) + .expect("finalized worker creation"); + let reconciled = store + .workspace_deletion_preflight(&owner, &workspace_id) + .expect("reconciled preflight"); + assert!( + !reconciled + .blockers + .iter() + .any(|blocker| blocker.message.contains("Worker creation reservations")) + ); + } + + #[test] + fn worker_registry_removal_terminalizes_created_reservation() { + let (store, owner, workspace_id) = setup(); + store + .with_conn(|conn| { + conn.execute( + "INSERT INTO worker_create_reservations ( + workspace_id, allocation_key, worker_id, runtime_id, + create_fingerprint, state, created_at, updated_at + ) VALUES (?1, 'allocation', 'worker-created', 'runtime-a', + 'fingerprint', 'created', '1', '1')", + params![workspace_id], + )?; + conn.execute( + "INSERT INTO worker_registry ( + workspace_id, runtime_id, worker_id, display_name, + created_at, updated_at, retention_state + ) VALUES (?1, 'runtime-a', 'worker-created', 'Created worker', '1', '1', 'normal')", + params![workspace_id], + )?; + Ok(()) + }) + .expect("created worker fixture"); + let before = store + .workspace_deletion_preflight(&owner, &workspace_id) + .expect("preflight before removal"); + assert!( + !before + .blockers + .iter() + .any(|blocker| blocker.message.contains("Worker creation reservations")) + ); + + assert!( + store + .delete_worker_registry( + &workspace_id, + &worker_runtime::identity::RuntimeWorkerRef { + runtime_id: "runtime-a".to_string(), + worker_id: "worker-created".to_string(), + }, + ) + .expect("remove worker registry") + ); + let state: String = store + .with_conn(|conn| { + conn.query_row( + "SELECT state FROM worker_create_reservations + WHERE workspace_id = ?1 AND allocation_key = 'allocation'", + params![workspace_id], + |row| row.get(0), + ) + .map_err(Into::into) + }) + .expect("reservation state"); + assert_eq!(state, "removed"); + let after = store + .workspace_deletion_preflight(&owner, &workspace_id) + .expect("preflight after removal"); + assert!( + !after + .blockers + .iter() + .any(|blocker| blocker.message.contains("Worker creation reservations")) + ); + } + + #[test] + fn pinned_worker_blocks_preflight_before_operation_reservation() { + let (store, owner, workspace_id) = setup(); + store + .with_conn(|conn| { + conn.execute( + "INSERT INTO worker_registry ( + workspace_id, runtime_id, worker_id, display_name, + created_at, updated_at, retention_state + ) VALUES (?1, 'runtime-a', 'worker-a', 'Pinned worker', '1', '1', 'pinned')", + params![workspace_id], + )?; + Ok(()) + }) + .expect("worker"); + let preflight = store + .workspace_deletion_preflight(&owner, &workspace_id) + .expect("preflight"); + assert!(!preflight.can_delete); + assert!(preflight.blockers.iter().any(|blocker| { + blocker.kind == WorkspaceDeletionBlockerKind::RetentionHold + && blocker.resource_key.as_deref() == Some("Pinned worker") + })); + let request = WorkspaceDeletionRequest { + operation_id: "delete-pinned".to_string(), + expected_revision: preflight.expected_revision, + confirmation: "Alpha".to_string(), + }; + assert!(matches!( + store.reserve_workspace_deletion(&owner, &workspace_id, &request), + Err(Error::WorkspaceConfigConflict(_)) + )); + } + + #[test] + fn preflight_counts_complete_inventory_and_bounds_blocker_projection() { + let (store, owner, workspace_id) = setup(); + store + .with_conn(|conn| { + conn.execute( + "WITH RECURSIVE seq(value) AS ( + SELECT 1 UNION ALL SELECT value + 1 FROM seq WHERE value < 10001 + ) + INSERT INTO worker_registry ( + workspace_id, runtime_id, worker_id, display_name, + created_at, updated_at, retention_state + ) + SELECT ?1, 'runtime-a', 'worker-' || value, 'Pinned ' || value, + '1', '1', 'pinned' + FROM seq", + params![workspace_id], + )?; + Ok(()) + }) + .expect("worker inventory"); + let preflight = store + .workspace_deletion_preflight(&owner, &workspace_id) + .expect("preflight"); + assert_eq!(preflight.resources.workers, 10_001); + assert_eq!(preflight.blockers.len(), WORKSPACE_DELETION_MAX_BLOCKERS); + assert!(preflight.blockers.iter().any(|blocker| { + blocker + .message + .contains("Additional deletion blockers exist") + })); + } + + #[test] + fn last_accessible_workspace_and_revision_conflicts_fail_closed() { + let (store, owner, workspace_id) = setup(); + let other = "workspace-b"; + let preflight = store + .workspace_deletion_preflight(&owner, other) + .expect("preflight"); + store + .finalize_workspace_deletion( + &store + .reserve_workspace_deletion( + &owner, + other, + &WorkspaceDeletionRequest { + operation_id: "delete-beta".to_string(), + expected_revision: preflight.expected_revision, + confirmation: "Beta".to_string(), + }, + ) + .expect("reserve") + .operation + .operation_id, + ) + .expect("delete beta"); + let blocked = store + .workspace_deletion_preflight(&owner, &workspace_id) + .expect("blocked"); + assert!(!blocked.can_delete); + assert_eq!( + blocked.blockers[0].kind, + WorkspaceDeletionBlockerKind::LastAccessibleWorkspace + ); + } +} diff --git a/crates/workspace-server/src/workspace_subscription.rs b/crates/workspace-server/src/workspace_subscription.rs index fe26f7f9..018a9622 100644 --- a/crates/workspace-server/src/workspace_subscription.rs +++ b/crates/workspace-server/src/workspace_subscription.rs @@ -11,7 +11,9 @@ use tokio::sync::mpsc; use worker_runtime::identity::RuntimeWorkerRef; use crate::runtime_subscription::{BrokerSubscriptionEvent, RuntimeSubscriptionBroker}; -use crate::server::{WorkspaceApi, connect_workspace_worker_protocol}; +use crate::server::{ + WorkspaceApi, authorize_browser_worker_method, connect_workspace_worker_protocol, +}; use crate::store::WorkspaceResourceKind; const OUTBOUND_CAPACITY: usize = 256; @@ -21,7 +23,11 @@ struct ActiveSubscription { methods: Option>, } -pub(crate) async fn serve_workspace_subscription(api: WorkspaceApi, socket: WebSocket) { +pub(crate) async fn serve_workspace_subscription( + api: WorkspaceApi, + socket: WebSocket, + input_source: protocol::AuthenticatedInputSource, +) { let broker = api.runtime_subscription_broker().clone(); let (mut socket_sender, mut socket_receiver) = socket.split(); let (control_outbound, mut control_receiver) = mpsc::channel::(OUTBOUND_CAPACITY); @@ -85,7 +91,13 @@ pub(crate) async fn serve_workspace_subscription(api: WorkspaceApi, socket: WebS runtime_id: Some(runtime_id), } => { let worker = RuntimeWorkerRef::new(&runtime_id, worker_id.as_str()); - match connect_workspace_worker_protocol(&api, &worker).await { + match connect_workspace_worker_protocol( + &api, + &worker, + Some(&input_source), + ) + .await + { Ok(connection) => { let methods = connection.methods.clone(); let task = tokio::spawn(run_worker_protocol( @@ -153,7 +165,12 @@ pub(crate) async fn serve_workspace_subscription(api: WorkspaceApi, socket: WebS else { break; }; - if methods.send(message.method).await.is_err() { + let Ok(method) = + authorize_browser_worker_method(message.method, &input_source) + else { + break; + }; + if methods.send(method).await.is_err() { break; } } diff --git a/docs/design/flow-state-graph.md b/docs/design/flow-state-graph.md index 3fc49249..bd0cea00 100644 --- a/docs/design/flow-state-graph.md +++ b/docs/design/flow-state-graph.md @@ -55,11 +55,12 @@ Workspace Server schema migration v26 removes the legacy `flow_instances`, `flow ## Worker boundary -Flow invocation uses the normal Submit/Run segment vector rather than a Worker-create field: +Flow invocation uses the normal Submit segment vector rather than a Worker-create field: ```json { - "method": "run", + "method": "submit", + "submission_request_id": "018f4f15-5c41-7d3a-8a72-2e755bc71681", "input": [ { "kind": "flow", "selector": "builtin:coder-review" }, { "kind": "text", "content": "Ticket 00001... implementation" } @@ -69,7 +70,7 @@ Flow invocation uses the normal Submit/Run segment vector rather than a Worker-c Runtime accepts exactly one Flow segment only when the resolved Profile enables `feature.flow` and a Workspace client is available. The Worker asks Workspace authority only for an immutable source snapshot, creates the instance locally, replaces the Flow segment with the entered state's instructions, and commits that runtime state atomically with the remaining Submit segments before LLM execution. A Worker with an active Flow rejects the duplicate input without changing its local state or events. -The generic model-facing `WorkerSpawn` accepts `initial_submit: Vec` and routes them unchanged through the shared Workspace spawn request into Runtime `CreateWorkerRequest.initial_input`. It does not have a parallel `initial_text` or a role-specific `SpawnCoder` wrapper. Backend derives the flat content projection from the canonical segment vector, validates Flow shape before spawn, and includes the segment vector in lifecycle idempotency fingerprints. Runtime does not commit Worker creation or report spawn success merely because the initial Run method entered the Worker's in-memory channel: Runtime assigns the Submit an opaque id, the Worker commits that id as an extension on the same `UserInput` entry as any initial `FlowRuntimeState`, and the execution backend must return a matching typed input-commit acknowledgement. Restoring the same Worker never replays spawn initial segments. +The generic model-facing `WorkerSpawn` accepts `initial_submit: Vec` and routes them unchanged through the shared Workspace spawn request into Runtime `CreateWorkerRequest.initial_input`. It does not have a parallel `initial_text` or a role-specific `SpawnCoder` wrapper. Backend derives the flat content projection from the canonical segment vector, validates Flow shape before spawn, and includes the segment vector in lifecycle idempotency fingerprints. Runtime does not commit Worker creation or report spawn success merely because the initial Submit request entered the Worker's in-memory channel: Runtime assigns the Submit an opaque id, the Worker commits that id as an extension on the same `UserInput` entry as any initial `FlowRuntimeState`, and the execution backend must return a matching typed input-commit acknowledgement. Restoring the same Worker never replays spawn initial segments. When an Orchestrator supplies `ticket_id` to generic `WorkerSpawn`, the Worker tool derives the assignment operation id from the durable tool-call id rather than accepting lifecycle authority from model input. The shared Workspace worker-create route projects that request into a Coder Ticket-role intent and atomically applies the existing queued-Ticket assignment operation only after Runtime has returned the input-commit acknowledgement. A spawn or pre-commit input failure therefore leaves the Ticket queued and unassigned. diff --git a/docs/development/server-runtime-auth.md b/docs/development/server-runtime-auth.md index fd30919c..5439de6d 100644 --- a/docs/development/server-runtime-auth.md +++ b/docs/development/server-runtime-auth.md @@ -110,18 +110,19 @@ On the Workspace Server host, register the Runtime public key copied from `yoi-r ```bash yoi-server trust-runtime add \ + --workspace-id '' \ --runtime-id runtime-main \ --base-url http://127.0.0.1:38800 \ --public-key '' \ --display-name 'Runtime main' ``` -This writes a trusted Runtime record to the Server DB. During `yoi-server serve`, active trusted Runtime records are loaded as remote Runtime sources and receive signed capability tokens. You do not need to duplicate the same Runtime in `runtimes.toml` for this trust-backed path. +This writes a Workspace-scoped Runtime binding and trust fingerprint to the Server DB. During `yoi-server serve`, active bindings are loaded as remote Runtime sources and receive signed capability tokens. Repository-external Runtime files are not registration or trust authority. Verify: ```bash -yoi-server trust-runtime list --json +yoi-server trust-runtime list --workspace-id '' --json ``` ## 5. Start Runtime and Workspace Server @@ -160,7 +161,7 @@ An empty Server DB is valid. Open the Web UI, create or authenticate the Account Check both trust stores: ```bash -yoi-server trust-runtime list --json +yoi-server trust-runtime list --workspace-id '' --json yoi-runtime trust-server list --json ``` @@ -203,6 +204,7 @@ After Runtime identity rotation, Server must be updated with the new Runtime pub ```bash yoi-server trust-runtime add \ + --workspace-id '' \ --runtime-id runtime-main \ --base-url http://127.0.0.1:38800 \ --public-key '' \ @@ -214,7 +216,9 @@ yoi-server trust-runtime add \ Revoke a trusted Runtime on Server: ```bash -yoi-server trust-runtime revoke --runtime-id runtime-main +yoi-server trust-runtime revoke \ + --workspace-id '' \ + --runtime-id runtime-main ``` Remove a trusted Server from Runtime: @@ -252,7 +256,7 @@ Confirm the `--runtime-id` registered on Server exactly matches the Runtime iden ```bash yoi-runtime identity show --json -yoi-server trust-runtime list --json +yoi-server trust-runtime list --workspace-id '' --json ``` `RUNTIME_ID` is the token audience; mismatches are rejected by Runtime. diff --git a/docs/development/work-items.md b/docs/development/work-items.md index 61c174ff..34bec74f 100644 --- a/docs/development/work-items.md +++ b/docs/development/work-items.md @@ -286,7 +286,7 @@ User triggers a Ticket action in yoi panel -> client Ticket role launcher reads .yoi/workspace.toml [ticket] settings -> launcher selects the role Profile -> launcher spawns the role Worker - -> launcher sends Method::Run with Text segments + -> launcher sends Method::Submit with Text segments -> launcher waits for run-acceptance evidence -> Dashboard reports success/failure ``` diff --git a/docs/report/test-validity-20260612/pod.md b/docs/report/test-validity-20260612/pod.md index d5c5cfc8..087ea4d2 100644 --- a/docs/report/test-validity-20260612/pod.md +++ b/docs/report/test-validity-20260612/pod.md @@ -34,7 +34,7 @@ - いくつかの async/socket tests はまだ fixed sleeps と process-wide environment mutation (`YOI_RUNTIME_DIR`, `YOI_HOME`, `XDG_RUNTIME_DIR`) に依存している。一部ファイルでは env-changing tests を guards で serialize しているが、fixed timing と global env は高負荷 CI や parallel execution 下で flakiness risk のまま。 - Real provider wire behavior は意図的に `pod` の外側だが、crate と streaming edge cases の interaction はまだ大部分が mock されている。tests は重要な `Worker` outcomes をカバーしているが、malformed/partial provider streams を real provider adapter 経由では exercise していない。 - Prompt tests は数が多く有用だが、一部は behavior-coupled というより prose-coupled である。critical safety wording に対して、その文字列を意図的に stable contract として扱う場合だけこれは許容できる。そうでなければ maintenance noise になる。 -- startup profile resolution、socket server、`Method::Run`、session persistence、shutdown、restore を跨ぐ full lifecycle integration は slice ごとにしかカバーされておらず、1 つの scenario としてはカバーされていない。 +- startup profile resolution、socket server、`Method::Submit`、session persistence、shutdown、restore を跨ぐ full lifecycle integration は slice ごとにしかカバーされておらず、1 つの scenario としてはカバーされていない。 ## 追加を提案するもの diff --git a/resources/flows/coder-review.dcdl b/resources/flows/coder-review.dcdl index b42dbd76..4de10077 100644 --- a/resources/flows/coder-review.dcdl +++ b/resources/flows/coder-review.dcdl @@ -15,7 +15,7 @@ }; review = { - instructions = "Use the current Ticket Merge Request as review authority. Call `ShowMergeRequest` and confirm its source selector resolves to exact committed implementation HEAD, then spawn one actual direct-child SubWorker with profile builtin:reviewer, write scope for Workdir inspection and command validation, and only the Ticket id in the structured review handoff. The trusted spawn layer records `ReviewRequested` with the exact source ref and injects review capability; do not place commit/ref identity, capability material, or a prewritten verdict in model input. The child must commit `ReviewMergeRequest`; prose output and Worker observation are not approval authority. After the structured result for the exact current source ref exists, request a Flow transition."; + instructions = "Use the current Ticket Merge Request as review authority. Call `ShowMergeRequest` and confirm its source selector resolves to exact committed implementation HEAD, then spawn one actual direct-child SubWorker with profile builtin:reviewer, write scope plus an explicit command grant for Workdir inspection and command validation, and only the Ticket id in the structured review handoff. The trusted spawn layer records `ReviewRequested` with the exact source ref and injects review capability; do not place commit/ref identity, capability material, or a prewritten verdict in model input. The child must commit `ReviewMergeRequest`; prose output and Worker observation are not approval authority. After the structured result for the exact current source ref exists, request a Flow transition."; transitions = { approved = { target = "complete"; diff --git a/resources/prompts/internal/sub_worker_spawn_tool_description.md b/resources/prompts/internal/sub_worker_spawn_tool_description.md index a652f551..d6d50df1 100644 --- a/resources/prompts/internal/sub_worker_spawn_tool_description.md +++ b/resources/prompts/internal/sub_worker_spawn_tool_description.md @@ -1,8 +1,8 @@ Spawn a parent-owned Internal SubWorker session to split context for a delegated task. The parent Worker's write scope is reduced by the scope passed here; the Internal SubWorker starts running `task` immediately without creating a Runtime Worker record, OS process, PID, or Unix socket. It remains available for follow-up turns until explicitly stopped or its parent exits. -Optional `cwd`: when provided, the spawned SubWorker's tool default working directory only. It must be an absolute existing directory covered by the child's delegated readable scope, and it does not change workspace/Profile/memory/Ticket roots or grant authority. `name` must be unique among this Worker's direct children. +Optional `cwd`: when provided, the spawned SubWorker's tool default working directory only. It must be a Workdir-relative existing directory covered by the child's readable scope, and it does not change workspace/Profile/memory/Ticket roots or grant authority. `name` must be unique among this Worker's direct children. -Profile selection: `profile` may be omitted or set to `default` to use the effective child default profile, set to `inherit` to derive reusable child configuration from this Worker, or set to one of the registry selectors below. Raw/path profile selectors are not accepted by SubWorkerSpawn. `scope` is always the only delegated filesystem capability; profile scope is replaced by the explicit SubWorkerSpawn scope. +Profile selection: `profile` may be omitted or set to `default` to use the effective child default profile, set to `inherit` to derive reusable child configuration from this Worker, or set to one of the registry selectors below. Raw/path profile selectors are not accepted by SubWorkerSpawn. `scope` is the child's only filesystem capability and replaces profile scope. `command` is a separate explicit grant, defaults to false, and is accepted only with a writable scope; writable scope alone does not grant command execution. Default profile: {{ default_profile }} Special selector: inherit — derive reusable model/worker/tool policy from the spawner while replacing worker.name and scope. diff --git a/web/workspace/deno.json b/web/workspace/deno.json index a8b34b18..4fda14c1 100644 --- a/web/workspace/deno.json +++ b/web/workspace/deno.json @@ -6,7 +6,7 @@ "dev": "deno run -A npm:vite@7.2.7 dev", "dev:backend": "cd ../.. && cargo run -p yoi-workspace-server --bin yoi-server -- serve --listen 127.0.0.1:8787", "check": "deno run -A npm:@sveltejs/kit@2.49.4 sync && deno run -A npm:svelte-check@4.3.4 --tsconfig ./tsconfig.json", - "test": "deno test --allow-read=src,test,tests --allow-env=LOG,VSCODE_TEXTMATE_DEBUG,NODE_ENV tests/workspace-model.test.ts tests/workspace-catalog.test.ts tests/profile-api.test.ts tests/skill-api.test.ts src/lib/workspace/auth/model.test.ts tests/auth-api.test.ts src/lib/workspace/api/http.test.ts src/lib/workspace/api/workers.test.ts src/lib/workspace/header/breadcrumb-model.test.ts src/lib/workspace/console/chat-submit.test.ts test/composer-history.test.ts tests/composer-paste.test.ts src/lib/workspace/console/composer-command.test.ts src/lib/workspace/console/composer-draft.test.ts src/lib/workspace/console/composer-completion.test.ts src/lib/workspace/console/markdown.test.ts test/console/ansi.test.ts src/lib/workspace/console/model.test.ts src/lib/workspace/companion/api.test.ts tests/workdir-api.test.ts src/lib/workspace/console/tasks.test.ts test/ticket-detail-route-reuse.test.ts test/repositories/ui.test.ts src/lib/workspace/console/worker-console.ui.test.ts src/lib/workspace/settings/model.test.ts src/lib/workspace/sidebar/override-stack.test.ts src/lib/workspace/sidebar/workers.test.ts src/lib/workspace/sidebar/workspace-switcher.test.ts src/lib/workspace/sidebar/worker-subscription.test.ts src/lib/workspace/sidebar/worker-launch.test.ts test/sidebar/worker-actions.test.ts src/lib/workspace/tickets/merge-request-resources.test.ts src/lib/workspace/tickets/ticket-panel.test.ts test/merge-request-status.test.ts test/config-source/decodal-grammar.test.ts test/config-source/editor-state.test.ts test/config-source/fixed-schema-wrapper.test.ts test/config-source/toolchain.test.ts test/config-source/wasm-parity.test.ts test/repository-access/api.test.ts test/repository-access/loader.test.ts test/repository-access/ui.test.ts", + "test": "deno test --allow-read=src,test,tests --allow-env=LOG,VSCODE_TEXTMATE_DEBUG,NODE_ENV tests/workspace-model.test.ts tests/workspace-catalog.test.ts tests/profile-api.test.ts tests/skill-api.test.ts src/lib/workspace/auth/model.test.ts tests/auth-api.test.ts src/lib/workspace/api/http.test.ts src/lib/workspace/api/workers.test.ts src/lib/workspace/header/breadcrumb-model.test.ts src/lib/workspace/console/chat-submit.test.ts src/lib/workspace/console/composer-delivery.test.ts test/composer-history.test.ts tests/composer-paste.test.ts src/lib/workspace/console/composer-command.test.ts src/lib/workspace/console/composer-draft.test.ts src/lib/workspace/console/composer-completion.test.ts src/lib/workspace/console/markdown.test.ts test/console/ansi.test.ts src/lib/workspace/console/model.test.ts src/lib/workspace/companion/api.test.ts tests/workdir-api.test.ts src/lib/workspace/console/tasks.test.ts test/ticket-detail-route-reuse.test.ts test/repositories/ui.test.ts src/lib/workspace/console/worker-console.ui.test.ts src/lib/workspace/settings/model.test.ts tests/runtime-connection.test.ts tests/runtime-management.test.ts tests/runtime-management-source.test.ts src/lib/workspace/sidebar/override-stack.test.ts src/lib/workspace/sidebar/workers.test.ts src/lib/workspace/sidebar/workspace-switcher.test.ts src/lib/workspace/sidebar/worker-subscription.test.ts src/lib/workspace/sidebar/worker-launch.test.ts test/sidebar/worker-actions.test.ts src/lib/workspace/tickets/merge-request-resources.test.ts src/lib/workspace/tickets/ticket-panel.test.ts test/merge-request-status.test.ts test/config-source/decodal-grammar.test.ts test/config-source/editor-state.test.ts test/config-source/fixed-schema-wrapper.test.ts test/config-source/toolchain.test.ts test/config-source/wasm-parity.test.ts test/repository-access/api.test.ts test/repository-access/loader.test.ts test/repository-access/ui.test.ts", "build": "deno run -A npm:vite@7.2.7 build", "preview": "deno run -A npm:vite@7.2.7 preview" }, diff --git a/web/workspace/src/lib/generated/protocol.ts b/web/workspace/src/lib/generated/protocol.ts index 8cd81901..b3482873 100644 --- a/web/workspace/src/lib/generated/protocol.ts +++ b/web/workspace/src/lib/generated/protocol.ts @@ -10,6 +10,37 @@ export type CompletionKind = "file"; export type WorkerStatus = "idle" | "running" | "paused" | "stopped"; +export type WorkerCommandEnvelope = { +/** + * Caller-owned sequence. A controller accepts command ids in strictly + * increasing order for one execution generation. + */ +command_id: number, expected_execution_generation: number, expected_worker_state_revision: number, }; + +export type WorkerCommandKind = "resume" | "cancel" | "pause" | "compact" | "shutdown"; + +export type WorkerCommandDisposition = "accepted" | "stale_execution_generation" | "stale_worker_state_revision" | "stale_command_id" | "conflict" | "invalid_state"; + +export type WorkerCommandAcknowledgement = { command_id: number, command: WorkerCommandKind, disposition: WorkerCommandDisposition, +/** + * The complete authoritative state observed after command admission. + */ +state: WorkerStateSnapshot, }; + +export type WorkerRunState = "running" | "pausing" | "paused" | "cancelling"; + +export type WorkerMaintenanceState = "compacting"; + +export type WorkerBusyState = { "kind": "run", "state": WorkerRunState } | { "kind": "maintenance", "state": WorkerMaintenanceState }; + +export type WorkerState = { "kind": "idle" } | { "kind": "busy", "state": WorkerBusyState }; + +export type WorkerStateSnapshot = { execution_generation: number, revision: number, +/** + * Highest lifecycle command id observed by this controller generation. + */ +last_command_id: number, state: WorkerState, }; + export type TurnResult = "finished" | "paused"; export type InvokeKind = "user_send" | "notify" | "worker_event" | "system_reminder" | "wakeup"; @@ -103,7 +134,13 @@ entry_id: string, */ timestamp: number, provenance: SessionEntryProvenance, derived_from?: Array, } & ({ "kind": "user_input", segments: Array, } | { "kind": "message", role: SessionMessageRole, content: Array, } | { "kind": "tool_call", call_id: string, name: string, arguments: string, } | { "kind": "tool_result", call_id: string, summary: string, content?: string | null, is_error: boolean, attachments?: Array, } | { "kind": "system_item", item_kind: string, content: string, data?: unknown, } | { "kind": "run_error", message: string, }); -export type SessionSnapshot = { entries: Array, }; +export type PendingSubmissionSummary = { submission_id: string, accepted_at_ms: number, segment_count: number, byte_len: number, }; + +export type PendingSubmissionsSnapshot = { revision: number, notification_count: number, head_id: string | null, submissions: Array, }; + +export type SubmissionDisposition = "started" | "queued"; + +export type SessionSnapshot = { pending_submissions: PendingSubmissionsSnapshot, entries: Array, }; export type InternalWorkerKind = "sub_worker" | { "service": { kind: string, } }; @@ -196,7 +233,16 @@ resource_key?: string | null, /** * Producer-owned monotonic revision for this Worker subject. */ -subject_revision: number, state: SubscriptionWorkerState, has_running_internal_workers: boolean, workspace_id?: string | null, display_name?: string | null, profile?: string | null, +subject_revision: number, +/** + * Latest revisioned foreground state observed from the Worker. This remains + * absent until an authoritative Worker snapshot/event has been applied. + */ +worker_state?: WorkerStateSnapshot | null, +/** + * Runtime catalog lifecycle compatibility projection; not foreground-state authority. + */ +state: SubscriptionWorkerState, has_running_internal_workers: boolean, workspace_id?: string | null, display_name?: string | null, profile?: string | null, /** * Workspace-facing Repository key. Runtime producers leave this unset and * Workspace Server projections replace `repository_id` with this field. @@ -225,9 +271,9 @@ export type SubscriptionFramePayload = { "frame": "request", "message": Subscrip export type SubscriptionFrame = { protocol_version: number, } & ({ "frame": "request", "message": SubscriptionRequest } | { "frame": "response", "message": SubscriptionResponse } | { "frame": "event", "message": SubscriptionEvent } | { "frame": "worker_protocol", "message": SubscriptionWorkerProtocolMethod }); -export type Method = { "method": "run", "params": { input: Array, } } | { "method": "notify", "params": { message: string, auto_run?: boolean, } } | { "method": "worker_event", "params": WorkerEvent } | { "method": "resume" } | { "method": "cancel" } | { "method": "pause" } | { "method": "compact" } | { "method": "list_rewind_targets" } | { "method": "rewind_to", "params": { target: RewindTargetId, expected_head_entries: number, } } | { "method": "shutdown" } | { "method": "list_completions", "params": { kind: CompletionKind, prefix: string, } } | { "method": "list_workers" } | { "method": "restore_worker", "params": { name: string, } } | { "method": "register_peer", "params": { name: string, } }; +export type Method = { "method": "submit", "params": { submission_request_id: string, input: Array, } } | { "method": "notify", "params": { notification_request_id: string, message: string, auto_run?: boolean, } } | { "method": "worker_event", "params": WorkerEvent } | { "method": "list_pending_submissions" } | { "method": "cancel_pending_submission", "params": { submission_id: string, expected_revision: number, } } | { "method": "clear_pending_submissions", "params": { expected_revision: number, } } | { "method": "continue_pending", "params": { expected_revision: number, expected_head_id: string, } } | { "method": "resume", "params": { command: WorkerCommandEnvelope, } } | { "method": "cancel", "params": { command: WorkerCommandEnvelope, } } | { "method": "pause", "params": { command: WorkerCommandEnvelope, } } | { "method": "compact", "params": { command: WorkerCommandEnvelope, } } | { "method": "list_rewind_targets" } | { "method": "rewind_to", "params": { target: RewindTargetId, expected_head_entries: number, } } | { "method": "shutdown", "params": { command: WorkerCommandEnvelope, } } | { "method": "list_completions", "params": { kind: CompletionKind, prefix: string, } } | { "method": "list_workers" } | { "method": "restore_worker", "params": { name: string, } } | { "method": "register_peer", "params": { name: string, } }; -export type Event = { "event": "user_message", "data": { segments: Array, } } | { "event": "system_item", "data": { item: unknown, } } | { "event": "invoke_start", "data": { kind: InvokeKind, } } | { "event": "turn_start", "data": { turn: number, } } | { "event": "turn_end", "data": { turn: number, result: TurnResult, } } | { "event": "llm_call_start", "data": { llm_call: number, } } | { "event": "llm_call_end", "data": { llm_call: number, } } | { "event": "llm_retry", "data": { llm_call: number, +export type Event = { "event": "submission_accepted", "data": { submission_request_id: string, submission_id: string, disposition: SubmissionDisposition, } } | { "event": "submission_rejected", "data": { submission_request_id: string, message: string, } } | { "event": "pending_submissions_changed", "data": { pending: PendingSubmissionsSnapshot, } } | { "event": "user_message", "data": { segments: Array, } } | { "event": "system_item", "data": { item: unknown, } } | { "event": "invoke_start", "data": { kind: InvokeKind, } } | { "event": "turn_start", "data": { turn: number, } } | { "event": "turn_end", "data": { turn: number, result: TurnResult, } } | { "event": "llm_call_start", "data": { llm_call: number, } } | { "event": "llm_call_end", "data": { llm_call: number, } } | { "event": "llm_retry", "data": { llm_call: number, /** * The attempt that just failed. 1 origin. */ @@ -241,7 +287,12 @@ summary: string, * Full tool output. Absent when the tool chose to return * summary-only, or when the result was pruned. */ -output?: string | null, disposition?: ToolResultDisposition | null, is_error: boolean, } } | { "event": "usage", "data": { input_tokens: number | null, output_tokens: number | null, cache_read_input_tokens?: number | null, } } | { "event": "run_end", "data": { result: RunResult, } } | { "event": "error", "data": { code: ErrorCode, message: string, } } | { "event": "snapshot", "data": { session: SessionSnapshot, greeting: Greeting, status: WorkerStatus, +output?: string | null, disposition?: ToolResultDisposition | null, is_error: boolean, } } | { "event": "usage", "data": { input_tokens: number | null, output_tokens: number | null, cache_read_input_tokens?: number | null, } } | { "event": "run_end", "data": { result: RunResult, } } | { "event": "error", "data": { code: ErrorCode, message: string, } } | { "event": "snapshot", "data": { session: SessionSnapshot, greeting: Greeting, +/** + * Full revisioned live execution state. `Stopped` remains Runtime + * catalog authority and is deliberately not represented here. + */ +state: WorkerStateSnapshot, /** * Unfinished model output that has already streamed in the current * run but is not yet represented by committed snapshot entries. @@ -251,4 +302,4 @@ in_flight?: InFlightSnapshot, * Parent-owned Internal Worker sessions visible to this client. * Service-private Internal Workers are deliberately excluded. */ -internal_workers?: Array, } } | { "event": "internal_worker", "data": { worker: InternalWorkerRef, revision: number, event: Event, } } | { "event": "internal_worker_removed", "data": { worker: InternalWorkerRef, revision: number, } } | { "event": "segment_rotated", "data": { session: SessionSnapshot, } } | { "event": "status", "data": { status: WorkerStatus, } } | { "event": "command", "data": { event: CommandEvent, } } | { "event": "completions", "data": { kind: CompletionKind, entries: Array, } } | { "event": "rewind_targets", "data": { head_entries: number, targets: Array, } } | { "event": "rewind_applied", "data": { session: SessionSnapshot, input: Array, summary: RewindSummary, } } | { "event": "workers_listed", "data": { workers: unknown, } } | { "event": "worker_restored", "data": { result: unknown, } } | { "event": "peer_registered", "data": { result: unknown, } } | { "event": "alert", "data": Alert } | { "event": "memory_worker", "data": MemoryWorkerEvent } | { "event": "compact_start", "data": { lifecycle: CompactionLifecycle, } } | { "event": "compact_done", "data": { lifecycle: CompactionLifecycle, } } | { "event": "compact_failed", "data": { lifecycle: CompactionLifecycle, } } | { "event": "shutdown" }; +internal_workers?: Array, } } | { "event": "internal_worker", "data": { worker: InternalWorkerRef, revision: number, event: Event, } } | { "event": "internal_worker_removed", "data": { worker: InternalWorkerRef, revision: number, } } | { "event": "segment_rotated", "data": { session: SessionSnapshot, } } | { "event": "worker_state", "data": { snapshot: WorkerStateSnapshot, } } | { "event": "command_acknowledged", "data": { acknowledgement: WorkerCommandAcknowledgement, } } | { "event": "command", "data": { event: CommandEvent, } } | { "event": "completions", "data": { kind: CompletionKind, entries: Array, } } | { "event": "rewind_targets", "data": { head_entries: number, targets: Array, } } | { "event": "rewind_applied", "data": { session: SessionSnapshot, input: Array, summary: RewindSummary, } } | { "event": "workers_listed", "data": { workers: unknown, } } | { "event": "worker_restored", "data": { result: unknown, } } | { "event": "peer_registered", "data": { result: unknown, } } | { "event": "alert", "data": Alert } | { "event": "memory_worker", "data": MemoryWorkerEvent } | { "event": "compact_start", "data": { lifecycle: CompactionLifecycle, } } | { "event": "compact_done", "data": { lifecycle: CompactionLifecycle, } } | { "event": "compact_failed", "data": { lifecycle: CompactionLifecycle, } } | { "event": "shutdown" }; diff --git a/web/workspace/src/lib/generated/workspace-api.ts b/web/workspace/src/lib/generated/workspace-api.ts index 36c6c324..06cb5f1e 100644 --- a/web/workspace/src/lib/generated/workspace-api.ts +++ b/web/workspace/src/lib/generated/workspace-api.ts @@ -47,6 +47,72 @@ export type WorkspaceAuthConfig = { export type WorkspacePermissionSummary = { manage_repositories: boolean; manage_secrets: boolean; + manage_runtimes: boolean; + delete_workspace: boolean; +}; + +export type WorkspaceDeletionState = + | "queued" + | "running" + | "blocked" + | "failed" + | "succeeded"; + +export type WorkspaceDeletionBlockerKind = + | "last_accessible_workspace" + | "revision_conflict" + | "dirty_workdir" + | "worker_removal_blocked" + | "workdir_removal_blocked" + | "retention_hold" + | "cleanup_unavailable"; + +export type WorkspaceDeletionBlocker = { + kind: WorkspaceDeletionBlockerKind; + resource_kind: string | null; + resource_key: string | null; + message: string; +}; + +export type WorkspaceDeletionResourceCounts = { + workers: number; + workdirs: number; + repositories: number; + runtime_bindings: number; + secrets: number; + artifacts: number; +}; + +export type WorkspaceDeletionPreflightResponse = { + workspace_id: string; + display_name: string; + /** + * Opaque persisted Workspace metadata revision used as a CAS fence. + */ + expected_revision: string; + can_delete: boolean; + resources: WorkspaceDeletionResourceCounts; + blockers: Array; +}; + +export type WorkspaceDeletionRequest = { + operation_id: string; + expected_revision: string; + confirmation: string; +}; + +export type WorkspaceDeletionOperationResponse = { + operation_id: string; + workspace_id: string; + display_name: string; + state: WorkspaceDeletionState; + resources: WorkspaceDeletionResourceCounts; + child_operation_ids: Array; + blockers: Array; + failure_category: string | null; + created_at: string; + updated_at: string; + completed_at: string | null; }; export type DiagnosticSeverity = "info" | "warning" | "error"; @@ -220,3 +286,129 @@ export type RepositoryLogResponse = { items: Array; diagnostics: Array; }; + +export type RuntimeSourceKind = "embedded_worker_runtime" | "remote_http"; + +export type RuntimeSourceStatus = "active" | "reserved"; + +export type RuntimeIdentityAuthority = + | "runtime_registry_projection" + | "server_runtime_configuration"; + +export type RuntimeSourceSummary = { + kind: RuntimeSourceKind; + status: RuntimeSourceStatus; + identity_authority: RuntimeIdentityAuthority; + note: string; +}; + +export type RuntimeSummary = { + runtime_id: string; + label: string; + kind: string; + status: string; + source: RuntimeSourceSummary; + host_ids: Array; + worker_creation_available: boolean; + os: string; + arch: string; + diagnostics: Array; +}; + +export type RuntimeManagementSummary = { + built_in: boolean; + config_managed: boolean; + removable: boolean; + endpoint_configured: boolean; + token_ref_configured: boolean; +}; + +export type WorkspaceRuntimeResource = { + management: RuntimeManagementSummary; + runtime_id: string; + label: string; + kind: string; + status: string; + source: RuntimeSourceSummary; + host_ids: Array; + worker_creation_available: boolean; + os: string; + arch: string; + diagnostics: Array; +}; + +export type RuntimeTrustKeyStatus = "unconfigured" | "active" | "revoked"; + +export type RuntimeTrustKeyState = { + status: RuntimeTrustKeyStatus; + fingerprint?: string | null; + revision?: number | null; + created_at?: string | null; + updated_at?: string | null; + revoked_at?: string | null; +}; + +export type RuntimeTrustAuditAction = + | "created" + | "replaced" + | "reactivated" + | "revoked"; + +export type RuntimeTrustAuditEntry = { + action: RuntimeTrustAuditAction; + actor_account_id: string; + old_fingerprint?: string | null; + new_fingerprint?: string | null; + revision: number; + at: string; +}; + +export type WorkspaceRuntimeDetail = { + workspace_id: string; + runtime: WorkspaceRuntimeResource; + endpoint?: string | null; + trust_key: RuntimeTrustKeyState; + recent_audit: Array; +}; + +export type RuntimeTrustKeyRevealResponse = { public_key: string }; + +export type PutRuntimeTrustKeyRequest = { + public_key: string; + expected_revision: number | null; +}; + +export type RevokeRuntimeTrustKeyRequest = { expected_revision: number }; + +export type RuntimeTrustConflictKind = "stale_revision" | "fingerprint_in_use"; + +export type RuntimeTrustConflictResponse = { + error: RuntimeTrustConflictKind; + message: string; + current_revision?: number; + current_fingerprint?: string | null; +}; + +export type RuntimeConnectionTestStatus = "compatible" | "failed"; + +export type RuntimeConnectionTestFailureKind = + | "authentication" + | "authorization" + | "network_unreachable" + | "timeout" + | "tls_or_transport" + | "malformed_response" + | "protocol_version_mismatch" + | "runtime_identity_mismatch" + | "configuration"; + +export type RuntimeConnectionTestResponse = { + workspace_id: string; + runtime_id: string; + checked_at: string; + status: RuntimeConnectionTestStatus; + failure_kind: RuntimeConnectionTestFailureKind | null; + expected_protocol_version: number; + actual_protocol_version: number | null; + diagnostics: Array; +}; diff --git a/web/workspace/src/lib/workspace/api/runtime-connection.ts b/web/workspace/src/lib/workspace/api/runtime-connection.ts new file mode 100644 index 00000000..6eb58965 --- /dev/null +++ b/web/workspace/src/lib/workspace/api/runtime-connection.ts @@ -0,0 +1,142 @@ +import type { + Diagnostic, + RuntimeConnectionTestFailureKind, + RuntimeConnectionTestResponse, +} from "$lib/generated/workspace-api"; + +const RESPONSE_KEYS = [ + "workspace_id", + "runtime_id", + "checked_at", + "status", + "failure_kind", + "expected_protocol_version", + "actual_protocol_version", + "diagnostics", +] as const; +const DIAGNOSTIC_KEYS = ["code", "severity", "message"] as const; +const FAILURE_KINDS = new Set([ + "authentication", + "authorization", + "network_unreachable", + "timeout", + "tls_or_transport", + "malformed_response", + "protocol_version_mismatch", + "runtime_identity_mismatch", + "configuration", +]); + +function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null && !Array.isArray(value); +} + +function hasExactKeys( + record: Record, + expected: readonly string[], +): boolean { + const actual = Object.keys(record).sort(); + const wanted = [...expected].sort(); + return actual.length === wanted.length && + actual.every((key, index) => key === wanted[index]); +} + +function isBoundedString(value: unknown, max = 1024): value is string { + return typeof value === "string" && value.length > 0 && value.length <= max; +} + +function isProtocolVersion(value: unknown): value is number { + return Number.isSafeInteger(value) && (value as number) >= 0; +} + +function parseDiagnostic(value: unknown): Diagnostic | null { + if (!isRecord(value) || !hasExactKeys(value, DIAGNOSTIC_KEYS)) return null; + if (!isBoundedString(value.code, 128) || !isBoundedString(value.message)) { + return null; + } + if ( + value.severity !== "info" && value.severity !== "warning" && + value.severity !== "error" + ) { + return null; + } + return { + code: value.code, + severity: value.severity, + message: value.message, + }; +} + +export function parseRuntimeConnectionTestResponse( + value: unknown, +): RuntimeConnectionTestResponse | null { + if (!isRecord(value) || !hasExactKeys(value, RESPONSE_KEYS)) return null; + if ( + !isBoundedString(value.workspace_id, 256) || + !isBoundedString(value.runtime_id, 256) || + !isBoundedString(value.checked_at, 128) || + Number.isNaN(Date.parse(value.checked_at)) || + (value.status !== "compatible" && value.status !== "failed") || + !isProtocolVersion(value.expected_protocol_version) || + (value.actual_protocol_version !== null && + !isProtocolVersion(value.actual_protocol_version)) || + !Array.isArray(value.diagnostics) || + value.diagnostics.length > 16 + ) { + return null; + } + const failureKind = value.failure_kind; + if ( + failureKind !== null && + !FAILURE_KINDS.has(failureKind as RuntimeConnectionTestFailureKind) + ) { + return null; + } + const diagnostics = value.diagnostics.map(parseDiagnostic); + if (diagnostics.some((diagnostic) => diagnostic === null)) return null; + if ( + (value.status === "compatible" && + (failureKind !== null || + value.actual_protocol_version !== value.expected_protocol_version || + diagnostics.length !== 0)) || + (value.status === "failed" && failureKind === null) + ) { + return null; + } + return { + workspace_id: value.workspace_id, + runtime_id: value.runtime_id, + checked_at: value.checked_at, + status: value.status, + failure_kind: failureKind as RuntimeConnectionTestFailureKind | null, + expected_protocol_version: value.expected_protocol_version, + actual_protocol_version: value.actual_protocol_version, + diagnostics: diagnostics as Diagnostic[], + }; +} + +export async function testRuntimeConnection( + workspaceId: string, + runtimeId: string, + fetchImpl: typeof fetch = fetch, +): Promise { + const response = await fetchImpl( + `/api/w/${encodeURIComponent(workspaceId)}/runtimes/${ + encodeURIComponent(runtimeId) + }/connection-tests`, + { method: "POST" }, + ); + if (!response.ok) { + throw new Error(`Connection test failed (${response.status})`); + } + const parsed = parseRuntimeConnectionTestResponse(await response.json()); + if (!parsed) { + throw new Error("Connection test returned an invalid response"); + } + if (parsed.workspace_id !== workspaceId || parsed.runtime_id !== runtimeId) { + throw new Error( + "Connection test response did not match the selected Runtime", + ); + } + return parsed; +} diff --git a/web/workspace/src/lib/workspace/api/runtime-management.ts b/web/workspace/src/lib/workspace/api/runtime-management.ts new file mode 100644 index 00000000..ac15d012 --- /dev/null +++ b/web/workspace/src/lib/workspace/api/runtime-management.ts @@ -0,0 +1,816 @@ +import type { + Diagnostic, + PutRuntimeTrustKeyRequest, + RevokeRuntimeTrustKeyRequest, + RuntimeIdentityAuthority, + RuntimeManagementSummary, + RuntimeSourceKind, + RuntimeSourceStatus, + RuntimeSourceSummary, + RuntimeTrustAuditAction, + RuntimeTrustAuditEntry, + RuntimeTrustConflictKind, + RuntimeTrustConflictResponse, + RuntimeTrustKeyRevealResponse, + RuntimeTrustKeyState, + RuntimeTrustKeyStatus, + WorkspaceRuntimeDetail, + WorkspaceRuntimeResource, +} from "$lib/generated/workspace-api.ts"; +import type { ListResponse } from "$lib/workspace/sidebar/types"; +import { workspaceApiPath } from "./http.ts"; + +export type WorkspaceRuntimeList = ListResponse; + +const LIMITS = { + runtimeItems: 200, + auditEntries: 20, + hostIds: 128, + diagnostics: 64, + idBytes: 256, + labelBytes: 512, + kindBytes: 128, + statusBytes: 128, + noteBytes: 2_048, + endpointBytes: 4_096, + publicKeyBytes: 16 * 1_024, + fingerprintBytes: 512, + timestampBytes: 128, + diagnosticCodeBytes: 128, + diagnosticMessageBytes: 2_048, + conflictMessageBytes: 1_024, + responseBytes: 512 * 1_024, +} as const; + +const SOURCE_KINDS = new Set([ + "embedded_worker_runtime", + "remote_http", +]); +const SOURCE_STATUSES = new Set(["active", "reserved"]); +const IDENTITY_AUTHORITIES = new Set([ + "runtime_registry_projection", + "server_runtime_configuration", +]); +const DIAGNOSTIC_SEVERITIES = new Set(["info", "warning", "error"]); +const TRUST_STATUSES = new Set([ + "unconfigured", + "active", + "revoked", +]); +const AUDIT_ACTIONS = new Set([ + "created", + "replaced", + "reactivated", + "revoked", +]); +const CONFLICT_KINDS = new Set([ + "stale_revision", + "fingerprint_in_use", +]); + +const encoder = new TextEncoder(); +type JsonObject = Record; + +export class RuntimeManagementValidationError extends Error { + constructor(message: string) { + super(message.slice(0, 256)); + this.name = "RuntimeManagementValidationError"; + } +} + +export class RuntimeTrustConflictError extends Error { + readonly conflict: RuntimeTrustConflictResponse; + + constructor(conflict: RuntimeTrustConflictResponse) { + super(conflict.message); + this.name = "RuntimeTrustConflictError"; + this.conflict = conflict; + } +} + +export class RuntimeTrustRequestError extends Error { + readonly field: "public_key" | null; + + constructor(message: string, field: "public_key" | null = null) { + super(message.slice(0, 256)); + this.name = "RuntimeTrustRequestError"; + this.field = field; + } +} + +export type RuntimeTrustRouteOperation = Readonly<{ + runtimeId: string; + generation: number; +}>; + +export class RuntimeTrustRouteFence { + #runtimeId: string | null = null; + #generation = 0; + + enter(runtimeId: string): number { + if (this.#runtimeId !== runtimeId) { + this.#runtimeId = runtimeId; + this.#generation += 1; + } + return this.#generation; + } + + capture(runtimeId: string): RuntimeTrustRouteOperation { + return { runtimeId, generation: this.enter(runtimeId) }; + } + + isCurrent(operation: RuntimeTrustRouteOperation, runtimeId: string): boolean { + return operation.runtimeId === runtimeId && + operation.generation === this.#generation && + this.#runtimeId === runtimeId; + } +} + +function fail(path: string, message: string): never { + throw new RuntimeManagementValidationError(`${path} ${message}`); +} + +function object(value: unknown, path: string): JsonObject { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + return fail(path, "must be an object"); + } + return value as JsonObject; +} + +function exactKeys( + value: JsonObject, + required: readonly string[], + optional: readonly string[], + path: string, +): void { + const allowed = new Set([...required, ...optional]); + for (const key of Object.keys(value)) { + if (!allowed.has(key)) { + fail(`${path}.${key}`, "is not part of the wire contract"); + } + } + for (const key of required) { + if (!Object.hasOwn(value, key)) fail(`${path}.${key}`, "is required"); + } +} + +function array(value: unknown, path: string, max: number): unknown[] { + if (!Array.isArray(value)) return fail(path, "must be an array"); + if (value.length > max) { + return fail(path, `must contain at most ${max} items`); + } + return value; +} + +function boundedString( + value: unknown, + path: string, + maxBytes: number, + allowEmpty = false, +): string { + if (typeof value !== "string") return fail(path, "must be a string"); + if (!allowEmpty && value.length === 0) return fail(path, "must not be empty"); + if (encoder.encode(value).byteLength > maxBytes) { + return fail(path, `must be at most ${maxBytes} UTF-8 bytes`); + } + return value; +} + +function boolean(value: unknown, path: string): boolean { + if (typeof value !== "boolean") return fail(path, "must be a boolean"); + return value; +} + +function safeInteger(value: unknown, path: string, minimum = 0): number { + if ( + typeof value !== "number" || !Number.isSafeInteger(value) || value < minimum + ) { + return fail(path, `must be a safe integer of at least ${minimum}`); + } + return value; +} + +function safeRevision(value: unknown, path: string): number { + return safeInteger(value, path, 1); +} + +function optionalNullableString( + value: unknown, + path: string, + maxBytes: number, + allowEmpty = false, +): string | null | undefined { + if (value === undefined || value === null) return value; + return boundedString(value, path, maxBytes, allowEmpty); +} + +function optionalRevision( + value: unknown, + path: string, +): number | undefined { + if (value === undefined || value === null) return undefined; + return safeRevision(value, path); +} + +function optionalNullableRevision( + value: unknown, + path: string, +): number | null | undefined { + if (value === undefined || value === null) return value; + return safeRevision(value, path); +} + +function timestamp(value: unknown, path: string): string { + const result = boundedString(value, path, LIMITS.timestampBytes); + if ( + !/^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:Z|[+-]\d{2}:\d{2})$/ + .test(result) + ) { + return fail(path, "must be an RFC 3339 timestamp"); + } + return result; +} + +function optionalNullableTimestamp( + value: unknown, + path: string, +): string | null | undefined { + if (value === undefined || value === null) return value; + return timestamp(value, path); +} + +function enumValue( + value: unknown, + path: string, + variants: ReadonlySet, +): T { + const result = boundedString(value, path, LIMITS.kindBytes); + if (!variants.has(result as T)) { + return fail(path, "contains an unknown enum value"); + } + return result as T; +} + +function diagnostic(value: unknown, path: string): Diagnostic { + const item = object(value, path); + exactKeys(item, ["code", "severity", "message"], [], path); + const severity = enumValue( + item.severity, + `${path}.severity`, + DIAGNOSTIC_SEVERITIES, + ) as Diagnostic["severity"]; + return { + code: boundedString(item.code, `${path}.code`, LIMITS.diagnosticCodeBytes), + severity, + message: boundedString( + item.message, + `${path}.message`, + LIMITS.diagnosticMessageBytes, + true, + ), + }; +} + +function runtimeSource(value: unknown, path: string): RuntimeSourceSummary { + const item = object(value, path); + exactKeys(item, ["kind", "status", "identity_authority", "note"], [], path); + return { + kind: enumValue(item.kind, `${path}.kind`, SOURCE_KINDS), + status: enumValue(item.status, `${path}.status`, SOURCE_STATUSES), + identity_authority: enumValue( + item.identity_authority, + `${path}.identity_authority`, + IDENTITY_AUTHORITIES, + ), + note: boundedString(item.note, `${path}.note`, LIMITS.noteBytes, true), + }; +} + +function runtimeManagement( + value: unknown, + path: string, +): RuntimeManagementSummary { + const item = object(value, path); + exactKeys( + item, + [ + "built_in", + "config_managed", + "removable", + "endpoint_configured", + "token_ref_configured", + ], + [], + path, + ); + return { + built_in: boolean(item.built_in, `${path}.built_in`), + config_managed: boolean(item.config_managed, `${path}.config_managed`), + removable: boolean(item.removable, `${path}.removable`), + endpoint_configured: boolean( + item.endpoint_configured, + `${path}.endpoint_configured`, + ), + token_ref_configured: boolean( + item.token_ref_configured, + `${path}.token_ref_configured`, + ), + }; +} + +function runtimeResource( + value: unknown, + path: string, +): WorkspaceRuntimeResource { + const item = object(value, path); + exactKeys( + item, + [ + "management", + "runtime_id", + "label", + "kind", + "status", + "source", + "host_ids", + "worker_creation_available", + "os", + "arch", + "diagnostics", + ], + [], + path, + ); + const hostIds = array(item.host_ids, `${path}.host_ids`, LIMITS.hostIds).map( + (entry, index) => + boundedString( + entry, + `${path}.host_ids[${index}]`, + LIMITS.idBytes, + ), + ); + if (new Set(hostIds).size !== hostIds.length) { + fail(`${path}.host_ids`, "must not contain duplicate IDs"); + } + return { + management: runtimeManagement(item.management, `${path}.management`), + runtime_id: boundedString( + item.runtime_id, + `${path}.runtime_id`, + LIMITS.idBytes, + ), + label: boundedString(item.label, `${path}.label`, LIMITS.labelBytes), + kind: boundedString(item.kind, `${path}.kind`, LIMITS.kindBytes), + status: boundedString(item.status, `${path}.status`, LIMITS.statusBytes), + source: runtimeSource(item.source, `${path}.source`), + host_ids: hostIds, + worker_creation_available: boolean( + item.worker_creation_available, + `${path}.worker_creation_available`, + ), + os: boundedString(item.os, `${path}.os`, LIMITS.kindBytes, true), + arch: boundedString(item.arch, `${path}.arch`, LIMITS.kindBytes, true), + diagnostics: array( + item.diagnostics, + `${path}.diagnostics`, + LIMITS.diagnostics, + ).map((entry, index) => diagnostic(entry, `${path}.diagnostics[${index}]`)), + }; +} + +function trustKey(value: unknown, path: string): RuntimeTrustKeyState { + const item = object(value, path); + exactKeys( + item, + ["status"], + ["fingerprint", "revision", "created_at", "updated_at", "revoked_at"], + path, + ); + const result: RuntimeTrustKeyState = { + status: enumValue(item.status, `${path}.status`, TRUST_STATUSES), + fingerprint: optionalNullableString( + item.fingerprint, + `${path}.fingerprint`, + LIMITS.fingerprintBytes, + ), + revision: optionalNullableRevision(item.revision, `${path}.revision`), + created_at: optionalNullableTimestamp( + item.created_at, + `${path}.created_at`, + ), + updated_at: optionalNullableTimestamp( + item.updated_at, + `${path}.updated_at`, + ), + revoked_at: optionalNullableTimestamp( + item.revoked_at, + `${path}.revoked_at`, + ), + }; + + const hasBinding = result.status !== "unconfigured"; + if ( + hasBinding && + (result.fingerprint == null || result.revision == null || + result.created_at == null || result.updated_at == null) + ) { + fail( + path, + "must include fingerprint, revision, created_at, and updated_at", + ); + } + if ( + !hasBinding && + Object.entries(result).some(([key, entry]) => + key !== "status" && entry != null + ) + ) { + fail(path, "must not include binding values while unconfigured"); + } + if (result.status === "revoked" && result.revoked_at == null) { + fail(`${path}.revoked_at`, "is required for a revoked key"); + } + if (result.status === "active" && result.revoked_at != null) { + fail(`${path}.revoked_at`, "must be absent for an active key"); + } + return result; +} + +function auditEntry(value: unknown, path: string): RuntimeTrustAuditEntry { + const item = object(value, path); + exactKeys( + item, + ["action", "actor_account_id", "revision", "at"], + ["old_fingerprint", "new_fingerprint"], + path, + ); + return { + action: enumValue(item.action, `${path}.action`, AUDIT_ACTIONS), + actor_account_id: boundedString( + item.actor_account_id, + `${path}.actor_account_id`, + LIMITS.idBytes, + ), + old_fingerprint: optionalNullableString( + item.old_fingerprint, + `${path}.old_fingerprint`, + LIMITS.fingerprintBytes, + ), + new_fingerprint: optionalNullableString( + item.new_fingerprint, + `${path}.new_fingerprint`, + LIMITS.fingerprintBytes, + ), + revision: safeRevision(item.revision, `${path}.revision`), + at: timestamp(item.at, `${path}.at`), + }; +} + +export function parseWorkspaceRuntimeList( + value: unknown, +): WorkspaceRuntimeList { + const response = object(value, "Runtime list response"); + exactKeys( + response, + ["workspace_id", "limit", "items", "source", "diagnostics"], + [], + "Runtime list response", + ); + const limit = safeInteger(response.limit, "Runtime list response.limit", 0); + if (limit > LIMITS.runtimeItems) { + fail( + "Runtime list response.limit", + `must not exceed ${LIMITS.runtimeItems}`, + ); + } + const items = array( + response.items, + "Runtime list response.items", + LIMITS.runtimeItems, + ).map((entry, index) => + runtimeResource(entry, `Runtime list response.items[${index}]`) + ); + if (items.length > limit) { + fail("Runtime list response.items", "must not exceed the declared limit"); + } + return { + workspace_id: boundedString( + response.workspace_id, + "Runtime list response.workspace_id", + LIMITS.idBytes, + ), + limit, + items, + source: boundedString( + response.source, + "Runtime list response.source", + LIMITS.kindBytes, + ), + diagnostics: array( + response.diagnostics, + "Runtime list response.diagnostics", + LIMITS.diagnostics, + ).map((entry, index) => + diagnostic(entry, `Runtime list response.diagnostics[${index}]`) + ), + }; +} + +export function parseWorkspaceRuntimeDetail( + value: unknown, +): WorkspaceRuntimeDetail { + const response = object(value, "Runtime detail response"); + exactKeys( + response, + ["workspace_id", "runtime", "trust_key", "recent_audit"], + ["endpoint"], + "Runtime detail response", + ); + return { + workspace_id: boundedString( + response.workspace_id, + "Runtime detail response.workspace_id", + LIMITS.idBytes, + ), + runtime: runtimeResource( + response.runtime, + "Runtime detail response.runtime", + ), + endpoint: optionalNullableString( + response.endpoint, + "Runtime detail response.endpoint", + LIMITS.endpointBytes, + ), + trust_key: trustKey( + response.trust_key, + "Runtime detail response.trust_key", + ), + recent_audit: array( + response.recent_audit, + "Runtime detail response.recent_audit", + LIMITS.auditEntries, + ).map((entry, index) => + auditEntry(entry, `Runtime detail response.recent_audit[${index}]`) + ), + }; +} + +export function parseRuntimeTrustKeyRevealResponse( + value: unknown, +): RuntimeTrustKeyRevealResponse { + const response = object(value, "Runtime trust key reveal response"); + exactKeys( + response, + ["public_key"], + [], + "Runtime trust key reveal response", + ); + return { + public_key: boundedString( + response.public_key, + "Runtime trust key reveal response.public_key", + LIMITS.publicKeyBytes, + ), + }; +} + +export function parseRuntimeTrustConflict( + value: unknown, +): RuntimeTrustConflictResponse { + const response = object(value, "Runtime trust conflict"); + exactKeys( + response, + ["error", "message"], + ["current_revision", "current_fingerprint"], + "Runtime trust conflict", + ); + return { + error: enumValue( + response.error, + "Runtime trust conflict.error", + CONFLICT_KINDS, + ), + message: boundedString( + response.message, + "Runtime trust conflict.message", + LIMITS.conflictMessageBytes, + ), + current_revision: optionalRevision( + response.current_revision, + "Runtime trust conflict.current_revision", + ), + current_fingerprint: optionalNullableString( + response.current_fingerprint, + "Runtime trust conflict.current_fingerprint", + LIMITS.fingerprintBytes, + ), + }; +} + +function revisionForJson(revision: number | null): number | null { + if (revision === null) return null; + if (!Number.isSafeInteger(revision) || revision < 1) { + throw new RuntimeTrustRequestError( + "Runtime trust revision is not a safe integer", + ); + } + return revision; +} + +async function readBoundedJson(response: Response): Promise { + const contentLength = response.headers.get("content-length"); + if (contentLength !== null) { + const parsed = Number(contentLength); + if (Number.isFinite(parsed) && parsed > LIMITS.responseBytes) { + throw new RuntimeTrustRequestError( + "Runtime trust response exceeds its byte limit", + ); + } + } + const text = await response.text(); + if (encoder.encode(text).byteLength > LIMITS.responseBytes) { + throw new RuntimeTrustRequestError( + "Runtime trust response exceeds its byte limit", + ); + } + try { + return JSON.parse(text) as unknown; + } catch { + throw new RuntimeTrustRequestError( + "Runtime trust response is not valid JSON", + ); + } +} + +function requestErrorFrom( + value: unknown, + status: number, +): RuntimeTrustRequestError { + try { + const response = object(value, "Runtime trust error"); + exactKeys( + response, + ["error", "message", "diagnostics"], + [], + "Runtime trust error", + ); + const diagnostics = array( + response.diagnostics, + "Runtime trust error.diagnostics", + LIMITS.diagnostics, + ).map((entry, index) => + diagnostic(entry, `Runtime trust error.diagnostics[${index}]`) + ); + const message = boundedString( + response.message, + "Runtime trust error.message", + LIMITS.conflictMessageBytes, + ); + const field = diagnostics.some((entry) => + entry.code.startsWith("runtime_public_key_") + ) + ? "public_key" + : null; + return new RuntimeTrustRequestError(message, field); + } catch { + return new RuntimeTrustRequestError( + `Runtime trust request failed (${status})`, + ); + } +} + +async function finishMutation( + response: Response, + workspaceId: string, + runtimeId: string, +): Promise { + const payload = await readBoundedJson(response); + if (response.status === 409) { + try { + throw new RuntimeTrustConflictError(parseRuntimeTrustConflict(payload)); + } catch (error) { + if (error instanceof RuntimeTrustConflictError) throw error; + throw new RuntimeTrustRequestError( + "Runtime trust conflict response was invalid", + ); + } + } + if (!response.ok) throw requestErrorFrom(payload, response.status); + let detail: WorkspaceRuntimeDetail; + try { + detail = parseWorkspaceRuntimeDetail(payload); + } catch { + throw new RuntimeTrustRequestError("Runtime trust response was invalid"); + } + if ( + detail.workspace_id !== workspaceId || + detail.runtime.runtime_id !== runtimeId + ) { + throw new RuntimeTrustRequestError( + "Runtime trust response did not match the selected Runtime", + ); + } + return detail; +} + +export async function revealRuntimeTrustKey( + workspaceId: string, + runtimeId: string, +): Promise { + const response = await fetch( + workspaceApiPath( + workspaceId, + `/runtimes/${encodeURIComponent(runtimeId)}/trust-key`, + ), + ); + const payload = await readBoundedJson(response); + if (!response.ok) throw requestErrorFrom(payload, response.status); + return parseRuntimeTrustKeyRevealResponse(payload); +} + +export async function previewRuntimePublicKeyFingerprint( + publicKey: string, +): Promise { + const normalized = publicKey.trim(); + const prefix = "yoi-ed25519-pub:v1:"; + if (!normalized.startsWith(prefix)) { + throw new RuntimeTrustRequestError( + `Public key must start with ${prefix}`, + ); + } + const encoded = normalized.slice(prefix.length); + if (!/^[A-Za-z0-9_-]+$/.test(encoded)) { + throw new RuntimeTrustRequestError("Public key encoding is invalid"); + } + const padded = encoded.replaceAll("-", "+").replaceAll("_", "/") + + "=".repeat((4 - (encoded.length % 4)) % 4); + let decoded: string; + try { + decoded = atob(padded); + } catch { + throw new RuntimeTrustRequestError("Public key encoding is invalid"); + } + if (decoded.length !== 32) { + throw new RuntimeTrustRequestError("Public key must contain 32 bytes"); + } + const bytes = Uint8Array.from( + decoded, + (character) => character.charCodeAt(0), + ); + const digest = new Uint8Array(await crypto.subtle.digest("SHA-256", bytes)); + const hex = Array.from(digest, (byte) => byte.toString(16).padStart(2, "0")) + .join(""); + return `sha256:${hex}`; +} + +export async function putRuntimeTrustKey( + workspaceId: string, + runtimeId: string, + request: PutRuntimeTrustKeyRequest, + fetchImpl: typeof fetch = fetch, +): Promise { + const response = await fetchImpl( + workspaceApiPath( + workspaceId, + `/runtimes/${encodeURIComponent(runtimeId)}/trust-key`, + ), + { + method: "PUT", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + public_key: request.public_key, + expected_revision: revisionForJson(request.expected_revision), + }), + }, + ); + return await finishMutation(response, workspaceId, runtimeId); +} + +export async function revokeRuntimeTrustKey( + workspaceId: string, + runtimeId: string, + request: RevokeRuntimeTrustKeyRequest, + currentFingerprint: string, + confirmation: string, + fetchImpl: typeof fetch = fetch, +): Promise { + if (!currentFingerprint || confirmation.trim() !== currentFingerprint) { + throw new RuntimeTrustRequestError( + "Enter the current fingerprint exactly before revoking Workspace trust.", + ); + } + const response = await fetchImpl( + workspaceApiPath( + workspaceId, + `/runtimes/${encodeURIComponent(runtimeId)}/trust-key`, + ), + { + method: "DELETE", + headers: { "content-type": "application/json" }, + body: JSON.stringify({ + expected_revision: revisionForJson(request.expected_revision), + }), + }, + ); + return await finishMutation(response, workspaceId, runtimeId); +} diff --git a/web/workspace/src/lib/workspace/api/workspace-model.ts b/web/workspace/src/lib/workspace/api/workspace-model.ts index b6701aa0..2a62b3bb 100644 --- a/web/workspace/src/lib/workspace/api/workspace-model.ts +++ b/web/workspace/src/lib/workspace/api/workspace-model.ts @@ -14,6 +14,12 @@ import type { WorkspaceAuthConfig, WorkspaceCatalogListResponse, WorkspaceCreateResponse, + WorkspaceDeletionBlocker, + WorkspaceDeletionBlockerKind, + WorkspaceDeletionOperationResponse, + WorkspaceDeletionPreflightResponse, + WorkspaceDeletionResourceCounts, + WorkspaceDeletionState, WorkspaceExtensionPoints, WorkspaceExtensionPointState, WorkspacePermissionSummary, @@ -32,6 +38,8 @@ export type { RepositorySummary, WorkspaceCatalogListResponse, WorkspaceCreateResponse, + WorkspaceDeletionOperationResponse, + WorkspaceDeletionPreflightResponse, WorkspacePermissionSummary, WorkspaceResponse, WorkspaceSummary, @@ -367,13 +375,27 @@ function authConfig(value: unknown, path: string): WorkspaceAuthConfig { function permissions(value: unknown, path: string): WorkspacePermissionSummary { const item = object(value, path); - exactKeys(item, ["manage_repositories", "manage_secrets"], path); + exactKeys( + item, + [ + "manage_repositories", + "manage_secrets", + "manage_runtimes", + "delete_workspace", + ], + path, + ); return { manage_repositories: boolean( item.manage_repositories, `${path}.manage_repositories`, ), manage_secrets: boolean(item.manage_secrets, `${path}.manage_secrets`), + manage_runtimes: boolean(item.manage_runtimes, `${path}.manage_runtimes`), + delete_workspace: boolean( + item.delete_workspace, + `${path}.delete_workspace`, + ), }; } @@ -561,6 +583,256 @@ export function parseRepositoryDetailResponse( }; } +const WORKSPACE_DELETION_MAX_OPERATION_ID_BYTES = 128; +const WORKSPACE_DELETION_MAX_REVISION_BYTES = 128; +const WORKSPACE_DELETION_MAX_BLOCKERS = 1024; +const WORKSPACE_DELETION_MAX_CHILD_OPERATION_IDS = 4096; +const WORKSPACE_DELETION_MAX_RESOURCE_VALUE_BYTES = 128; +const WORKSPACE_DELETION_MAX_BLOCKER_MESSAGE_BYTES = 512; + +function deletionBoundedString( + value: unknown, + path: string, + maxBytes: number, +): string { + const candidate = string(value, path); + if (new TextEncoder().encode(candidate).length > maxBytes) { + throw new Error(`${path} is too long`); + } + return candidate; +} + +function deletionBoundedArray( + value: unknown, + path: string, + maxItems: number, +): unknown[] { + const candidate = array(value, path); + if (candidate.length > maxItems) { + throw new Error(`${path} has too many items`); + } + return candidate; +} + +const deletionStates = new Set([ + "queued", + "running", + "blocked", + "failed", + "succeeded", +]); +const deletionBlockerKinds = new Set([ + "last_accessible_workspace", + "revision_conflict", + "dirty_workdir", + "worker_removal_blocked", + "workdir_removal_blocked", + "retention_hold", + "cleanup_unavailable", +]); + +function deletionState(value: unknown, path: string): WorkspaceDeletionState { + const candidate = string(value, path) as WorkspaceDeletionState; + if (!deletionStates.has(candidate)) throw new Error(`${path} is invalid`); + return candidate; +} + +function deletionBlocker( + value: unknown, + path: string, +): WorkspaceDeletionBlocker { + const item = object(value, path); + exactKeys(item, ["kind", "resource_kind", "resource_key", "message"], path); + const kind = string( + item.kind, + `${path}.kind`, + ) as WorkspaceDeletionBlockerKind; + if (!deletionBlockerKinds.has(kind)) { + throw new Error(`${path}.kind is invalid`); + } + const resourceKind = optionalNullableString( + item.resource_kind, + `${path}.resource_kind`, + ); + const resourceKey = optionalNullableString( + item.resource_key, + `${path}.resource_key`, + ); + return { + kind, + resource_kind: resourceKind === undefined || resourceKind === null + ? null + : deletionBoundedString( + resourceKind, + `${path}.resource_kind`, + WORKSPACE_DELETION_MAX_RESOURCE_VALUE_BYTES, + ), + resource_key: resourceKey === undefined || resourceKey === null + ? null + : deletionBoundedString( + resourceKey, + `${path}.resource_key`, + WORKSPACE_DELETION_MAX_RESOURCE_VALUE_BYTES, + ), + message: deletionBoundedString( + item.message, + `${path}.message`, + WORKSPACE_DELETION_MAX_BLOCKER_MESSAGE_BYTES, + ), + }; +} + +function deletionResourceCounts( + value: unknown, + path: string, +): WorkspaceDeletionResourceCounts { + const item = object(value, path); + exactKeys(item, [ + "workers", + "workdirs", + "repositories", + "runtime_bindings", + "secrets", + "artifacts", + ], path); + return { + workers: integer(item.workers, `${path}.workers`), + workdirs: integer(item.workdirs, `${path}.workdirs`), + repositories: integer(item.repositories, `${path}.repositories`), + runtime_bindings: integer( + item.runtime_bindings, + `${path}.runtime_bindings`, + ), + secrets: integer(item.secrets, `${path}.secrets`), + artifacts: integer(item.artifacts, `${path}.artifacts`), + }; +} + +export function parseWorkspaceDeletionPreflightResponse( + value: unknown, +): WorkspaceDeletionPreflightResponse { + const item = object(value, "Workspace deletion preflight"); + exactKeys(item, [ + "workspace_id", + "display_name", + "expected_revision", + "can_delete", + "resources", + "blockers", + ], "Workspace deletion preflight"); + return { + workspace_id: string( + item.workspace_id, + "Workspace deletion preflight.workspace_id", + ), + display_name: string( + item.display_name, + "Workspace deletion preflight.display_name", + ), + expected_revision: deletionBoundedString( + item.expected_revision, + "Workspace deletion preflight.expected_revision", + WORKSPACE_DELETION_MAX_REVISION_BYTES, + ), + can_delete: boolean( + item.can_delete, + "Workspace deletion preflight.can_delete", + ), + resources: deletionResourceCounts( + item.resources, + "Workspace deletion preflight.resources", + ), + blockers: deletionBoundedArray( + item.blockers, + "Workspace deletion preflight.blockers", + WORKSPACE_DELETION_MAX_BLOCKERS, + ).map( + (entry, index) => + deletionBlocker( + entry, + `Workspace deletion preflight.blockers[${index}]`, + ), + ), + }; +} + +export function parseWorkspaceDeletionOperationResponse( + value: unknown, +): WorkspaceDeletionOperationResponse { + const item = object(value, "Workspace deletion operation"); + exactKeys(item, [ + "operation_id", + "workspace_id", + "display_name", + "state", + "resources", + "child_operation_ids", + "blockers", + "failure_category", + "created_at", + "updated_at", + "completed_at", + ], "Workspace deletion operation"); + return { + operation_id: deletionBoundedString( + item.operation_id, + "Workspace deletion operation.operation_id", + WORKSPACE_DELETION_MAX_OPERATION_ID_BYTES, + ), + workspace_id: string( + item.workspace_id, + "Workspace deletion operation.workspace_id", + ), + display_name: string( + item.display_name, + "Workspace deletion operation.display_name", + ), + state: deletionState(item.state, "Workspace deletion operation.state"), + resources: deletionResourceCounts( + item.resources, + "Workspace deletion operation.resources", + ), + child_operation_ids: deletionBoundedArray( + item.child_operation_ids, + "Workspace deletion operation.child_operation_ids", + WORKSPACE_DELETION_MAX_CHILD_OPERATION_IDS, + ).map((entry, index) => + deletionBoundedString( + entry, + `Workspace deletion operation.child_operation_ids[${index}]`, + WORKSPACE_DELETION_MAX_OPERATION_ID_BYTES, + ) + ), + blockers: deletionBoundedArray( + item.blockers, + "Workspace deletion operation.blockers", + WORKSPACE_DELETION_MAX_BLOCKERS, + ).map( + (entry, index) => + deletionBlocker( + entry, + `Workspace deletion operation.blockers[${index}]`, + ), + ), + failure_category: optionalNullableString( + item.failure_category, + "Workspace deletion operation.failure_category", + ) ?? null, + created_at: string( + item.created_at, + "Workspace deletion operation.created_at", + ), + updated_at: string( + item.updated_at, + "Workspace deletion operation.updated_at", + ), + completed_at: optionalNullableString( + item.completed_at, + "Workspace deletion operation.completed_at", + ) ?? null, + }; +} + export function parseRepositoryLogResponse( value: unknown, ): RepositoryLogResponse { diff --git a/web/workspace/src/lib/workspace/console/composer-delivery.test.ts b/web/workspace/src/lib/workspace/console/composer-delivery.test.ts new file mode 100644 index 00000000..4d052215 --- /dev/null +++ b/web/workspace/src/lib/workspace/console/composer-delivery.test.ts @@ -0,0 +1,136 @@ +declare const Deno: { + test(name: string, fn: () => void): void; +}; + +import { + canDeliverComposerDraft, + sendComposerDelivery, +} from "./composer-delivery.ts"; + +function assertEquals(actual: unknown, expected: unknown): void { + if (actual !== expected) { + throw new Error(`Expected ${String(expected)}, got ${String(actual)}`); + } +} + +const base = { + protocolOpen: true, + sending: false, + hasText: true, + hasAttachments: false, +}; + +Deno.test("running Composer enables Queue Submit and Notify but not immediate Submit", () => { + assertEquals( + canDeliverComposerDraft({ + ...base, + delivery: "queue", + workerState: "running", + }), + true, + ); + assertEquals( + canDeliverComposerDraft({ + ...base, + delivery: "notify", + workerState: "running", + }), + true, + ); + assertEquals( + canDeliverComposerDraft({ + ...base, + delivery: "submit", + workerState: "running", + }), + false, + ); +}); + +Deno.test("running Queue Submit and Notify dispatch their protocol methods", () => { + const sent: string[] = []; + assertEquals( + sendComposerDelivery( + { ...base, delivery: "queue", workerState: "running" }, + "submit", + (method) => sent.push(method), + ), + true, + ); + assertEquals( + sendComposerDelivery( + { ...base, delivery: "notify", workerState: "running" }, + "notify", + (method) => sent.push(method), + ), + true, + ); + assertEquals(sent.join(","), "submit,notify"); +}); + +Deno.test("idle Composer enables only immediate Submit", () => { + assertEquals( + canDeliverComposerDraft({ + ...base, + delivery: "submit", + workerState: "idle", + }), + true, + ); + assertEquals( + canDeliverComposerDraft({ + ...base, + delivery: "queue", + workerState: "idle", + }), + false, + ); + assertEquals( + canDeliverComposerDraft({ + ...base, + delivery: "notify", + workerState: "idle", + }), + false, + ); +}); + +Deno.test("running delivery remains fenced by protocol, send state, and payload kind", () => { + assertEquals( + canDeliverComposerDraft({ + ...base, + delivery: "queue", + workerState: "running", + protocolOpen: false, + }), + false, + ); + assertEquals( + canDeliverComposerDraft({ + ...base, + delivery: "notify", + workerState: "running", + sending: true, + }), + false, + ); + assertEquals( + canDeliverComposerDraft({ + ...base, + delivery: "notify", + workerState: "running", + hasAttachments: true, + }), + false, + ); + assertEquals( + canDeliverComposerDraft({ + ...base, + delivery: "queue", + workerState: "running", + hasText: false, + hasAttachments: true, + }), + true, + ); +}); diff --git a/web/workspace/src/lib/workspace/console/composer-delivery.ts b/web/workspace/src/lib/workspace/console/composer-delivery.ts new file mode 100644 index 00000000..adf17883 --- /dev/null +++ b/web/workspace/src/lib/workspace/console/composer-delivery.ts @@ -0,0 +1,39 @@ +export type ComposerDelivery = "submit" | "queue" | "notify"; + +export type ComposerDeliveryState = { + delivery: ComposerDelivery; + workerState: string; + protocolOpen: boolean; + sending: boolean; + hasText: boolean; + hasAttachments: boolean; +}; + +/** + * Resolve whether the current Composer draft can use one delivery action. + * Immediate Submit is idle-only; Queue and Notify are running-only. + */ +export function canDeliverComposerDraft(state: ComposerDeliveryState): boolean { + if (!state.protocolOpen || state.sending) return false; + + const hasInput = state.hasText || state.hasAttachments; + switch (state.delivery) { + case "submit": + return state.workerState === "idle" && hasInput; + case "queue": + return state.workerState === "running" && hasInput; + case "notify": + return state.workerState === "running" && state.hasText && + !state.hasAttachments; + } +} + +export function sendComposerDelivery( + state: ComposerDeliveryState, + method: T, + send: (method: T) => void, +): boolean { + if (!canDeliverComposerDraft(state)) return false; + send(method); + return true; +} diff --git a/web/workspace/src/lib/workspace/console/model.test.ts b/web/workspace/src/lib/workspace/console/model.test.ts index 6700d70c..8be2bb77 100644 --- a/web/workspace/src/lib/workspace/console/model.test.ts +++ b/web/workspace/src/lib/workspace/console/model.test.ts @@ -1,4 +1,4 @@ -import type { Event } from "$lib/generated/protocol"; +import type { Event, WorkerStateSnapshot, WorkerStatus } from "$lib/generated/protocol"; import { type ConsoleEventInput, type ConsoleLine, @@ -19,6 +19,23 @@ declare const Deno: { test(name: string, fn: () => void): void; }; +function workerState(status: WorkerStatus): WorkerStateSnapshot { + return { + execution_generation: 1, + revision: status === "idle" ? 0 : 1, + last_command_id: 0, + state: status === "idle" + ? { kind: "idle" } + : { + kind: "busy", + state: { + kind: "run", + state: status === "paused" ? "paused" : "running", + }, + }, + }; +} + function assert(condition: unknown, message: string): asserts condition { if (!condition) { throw new Error(message); @@ -131,7 +148,7 @@ function snapshotEvent(cwd: string, entries: unknown[] = []): Event { context_window: 100, context_tokens: 20, }, - status: "idle", + state: workerState("idle"), in_flight: { blocks: [] }, }, }; @@ -201,6 +218,66 @@ Deno.test("console routing projects live errors but not completion replies", () ); }); +Deno.test("Worker state events and acknowledgements apply monotonically", () => { + const projector = createConsoleProjector(); + const running: WorkerStateSnapshot = { + execution_generation: 4, + revision: 3, + last_command_id: 2, + state: { kind: "busy", state: { kind: "run", state: "running" } }, + }; + const paused: WorkerStateSnapshot = { + ...running, + revision: 4, + last_command_id: 3, + state: { kind: "busy", state: { kind: "run", state: "paused" } }, + }; + let projection = projector.append([ + { + eventId: "running", + event: { event: "worker_state", data: { snapshot: running } }, + }, + { + eventId: "stale", + event: { + event: "worker_state", + data: { snapshot: { ...running, revision: 2, state: { kind: "idle" } } }, + }, + }, + { + eventId: "pause-ack", + event: { + event: "command_acknowledged", + data: { + acknowledgement: { + command_id: 3, + command: "pause", + disposition: "accepted", + state: paused, + }, + }, + }, + }, + ]); + assertEquals(projection.workerState, paused); + assertEquals(projection.status, "paused"); + + projection = projector.append([{ + eventId: "conflict", + event: { + event: "worker_state", + data: { snapshot: { ...paused, state: { kind: "idle" } } }, + }, + }]); + assertEquals(projection.workerState, paused); + assert( + projection.lines.some((line) => + line.eventId === "conflict:worker-state-conflict" && line.error + ), + "conflicting equal-version snapshots must fail closed", + ); +}); + Deno.test("snapshot replaces a live error with one durable run_errored row", () => { const projector = createConsoleProjector(); let projection = projector.append([ @@ -213,7 +290,7 @@ Deno.test("snapshot replaces a live error with one durable run_errored row", () }, { eventId: "idle-after-error", - event: { event: "status", data: { status: "idle" } } satisfies Event, + event: { event: "worker_state", data: { snapshot: workerState("idle") } } satisfies Event, }, ]); @@ -653,7 +730,7 @@ Deno.test("projectConsole streams distinct Bash stdout and stderr through termin Deno.test("snapshot restores bounded in-flight Bash command output", () => { const snapshot = snapshotEvent("/repo"); if (snapshot.event !== "snapshot") throw new Error("snapshot fixture expected"); - snapshot.data.status = "running"; + snapshot.data.state = workerState("running"); snapshot.data.in_flight = { blocks: [{ kind: "tool_call", @@ -1403,7 +1480,7 @@ Deno.test("projectConsole hides lifecycle events and renders system items", () = const projection = projectConsole([ { eventId: "30", - event: { event: "status", data: { status: "running" } } satisfies Event, + event: { event: "worker_state", data: { snapshot: workerState("running") } } satisfies Event, }, { eventId: "31", @@ -1527,7 +1604,7 @@ Deno.test("projectConsole renders snapshot entries and in-flight output", () => context_window: 100, context_tokens: 20, }, - status: "running", + state: workerState("running"), in_flight: { blocks: [ { kind: "text", text: "partial" }, @@ -1578,7 +1655,7 @@ Deno.test("projectConsole restores system items from snapshot entries", () => { context_window: 100, context_tokens: 20, }, - status: "idle", + state: workerState("idle"), }, } satisfies Event, }]); @@ -1922,7 +1999,7 @@ Deno.test("console Worker views expose only direct Internal Workers", () => { kind: "sub_worker", }, revision: 1, - event: { event: "status", data: { status: "running" } }, + event: { event: "worker_state", data: { snapshot: workerState("running") } }, }, }, }, @@ -1941,7 +2018,7 @@ Deno.test("console Worker views expose only direct Internal Workers", () => { kind: "sub_worker", }, revision: 1, - event: { event: "status", data: { status: "idle" } }, + event: { event: "worker_state", data: { snapshot: workerState("idle") } }, }, }, }]); @@ -2033,7 +2110,7 @@ Deno.test("parent snapshot authoritatively replaces Internal Worker projections" kind: "sub_worker", }, revision: 1, - event: { event: "status", data: { status: "running" } }, + event: { event: "worker_state", data: { snapshot: workerState("running") } }, }, }, }]); @@ -2150,6 +2227,12 @@ Deno.test("snapshot restores TaskStore state from system history", () => { const event = snapshotEvent("/repo"); if (event.event !== "snapshot") throw new Error("snapshot fixture expected"); event.data.session = { + pending_submissions: { + revision: 0, + notification_count: 0, + head_id: null, + submissions: [], + }, entries: [{ entry_id: "task-reminder-1", timestamp: 1, diff --git a/web/workspace/src/lib/workspace/console/model.ts b/web/workspace/src/lib/workspace/console/model.ts index 7bac8543..481c5313 100644 --- a/web/workspace/src/lib/workspace/console/model.ts +++ b/web/workspace/src/lib/workspace/console/model.ts @@ -10,6 +10,9 @@ import type { InternalWorkerRef, InternalWorkerSnapshot, Segment, + WorkerState, + WorkerStateSnapshot, + WorkerStatus, } from "$lib/generated/protocol"; import { stringify as stringifyYaml } from "yaml"; import { workspaceRoute } from "$lib/workspace/api/http"; @@ -169,6 +172,7 @@ export type ConsoleProjection = { tasks: ConsoleTask[]; taskNextId: number; status: string | null; + workerState: WorkerStateSnapshot | null; usage: string | null; runActivity: RunActivityStats; cwd: string | null; @@ -251,12 +255,22 @@ export function isConsoleProjectionEvent(event: ProtocolEvent): boolean { return event.event !== "completions"; } +function workerStatusFromState(snapshot: WorkerStateSnapshot): WorkerStatus { + if (snapshot.state.kind === "idle") return "idle"; + if ( + snapshot.state.state.kind === "run" && + snapshot.state.state.state === "paused" + ) return "paused"; + return "running"; +} + export function emptyConsoleProjection(): ConsoleProjection { return { lines: [], tasks: [], taskNextId: 1, status: null, + workerState: null, usage: null, runActivity: emptyRunActivityStats(), cwd: null, @@ -783,6 +797,60 @@ function refreshCompactionActivity( return changed ? { ...projection, lines } : projection; } +function workerStateEqual(left: WorkerState, right: WorkerState): boolean { + if (left.kind !== right.kind) return false; + if (left.kind === "idle" || right.kind === "idle") return true; + return left.state.kind === right.state.kind && + left.state.state === right.state.state; +} + +function workerStateSnapshotEqual( + left: WorkerStateSnapshot, + right: WorkerStateSnapshot, +): boolean { + return left.execution_generation === right.execution_generation && + left.revision === right.revision && + left.last_command_id === right.last_command_id && + workerStateEqual(left.state, right.state); +} + +function applyWorkerStateSnapshot( + projection: ConsoleProjection, + incoming: WorkerStateSnapshot, + eventId: string, +): void { + const current = projection.workerState; + if (!current) { + projection.workerState = incoming; + projection.status = workerStatusFromState(incoming); + return; + } + const generationOrder = incoming.execution_generation - + current.execution_generation; + const revisionOrder = incoming.revision - current.revision; + if (generationOrder > 0 || (generationOrder === 0 && revisionOrder > 0)) { + projection.workerState = incoming; + projection.status = workerStatusFromState(incoming); + return; + } + if (generationOrder < 0 || (generationOrder === 0 && revisionOrder < 0)) { + return; + } + if (!workerStateSnapshotEqual(current, incoming)) { + projection.lines.push( + line( + `${eventId}:worker-state-conflict`, + "error", + "error · internal", + `worker state stream rejected: conflicting snapshots at generation ${incoming.execution_generation} revision ${incoming.revision}`, + undefined, + false, + true, + ), + ); + } +} + export function applyProtocolEvent( projection: ConsoleProjection, envelope: ConsoleEventInput, @@ -793,6 +861,7 @@ export function applyProtocolEvent( tasks: [...projection.tasks], taskNextId: projection.taskNextId, status: projection.status, + workerState: projection.workerState, usage: projection.usage, runActivity: applyRunActivityEvent( projection.runActivity, @@ -903,7 +972,6 @@ export function applyProtocolEvent( ); break; case "snapshot": { - next.status = event.data.status; next.cwd = event.data.greeting.cwd; const snapshot = snapshotProjectionFromSession( envelope.eventId, @@ -953,6 +1021,7 @@ export function applyProtocolEvent( }; } } + applyWorkerStateSnapshot(next, event.data.state, envelope.eventId); break; } case "internal_worker": { @@ -1000,8 +1069,15 @@ export function applyProtocolEvent( if (existingIndex >= 0) next.internalWorkers.splice(existingIndex, 1); break; } - case "status": - next.status = event.data.status; + case "worker_state": + applyWorkerStateSnapshot(next, event.data.snapshot, envelope.eventId); + break; + case "command_acknowledged": + applyWorkerStateSnapshot( + next, + event.data.acknowledgement.state, + envelope.eventId, + ); break; case "command": applyCommandEvent(next, envelope.eventId, event.data.event); @@ -1939,6 +2015,7 @@ function snapshotProjectionFromSession( tasks: [], taskNextId: 1, status: null, + workerState: null, usage: null, runActivity: emptyRunActivityStats(), cwd, diff --git a/web/workspace/src/lib/workspace/console/run-status.test.ts b/web/workspace/src/lib/workspace/console/run-status.test.ts index 575ce733..0063a242 100644 --- a/web/workspace/src/lib/workspace/console/run-status.test.ts +++ b/web/workspace/src/lib/workspace/console/run-status.test.ts @@ -75,7 +75,12 @@ Deno.test("new invoke and running snapshot reset run activity", () => { data: { entries: [], greeting: { text: "", profile: "" }, - status: "idle", + state: { + execution_generation: 1, + revision: 0, + last_command_id: 0, + state: { kind: "idle" }, + }, in_flight: {}, internal_workers: [], }, diff --git a/web/workspace/src/lib/workspace/console/run-status.ts b/web/workspace/src/lib/workspace/console/run-status.ts index aabb45c2..37495dc1 100644 --- a/web/workspace/src/lib/workspace/console/run-status.ts +++ b/web/workspace/src/lib/workspace/console/run-status.ts @@ -25,7 +25,9 @@ export function applyRunActivityEvent( case "invoke_start": return { ...emptyRunActivityStats(), startedAtMs: observedAtMs }; case "snapshot": - return event.data.status === "running" + return event.data.state.state.kind === "busy" && + !(event.data.state.state.state.kind === "run" && + event.data.state.state.state.state === "paused") ? { ...emptyRunActivityStats(), startedAtMs: observedAtMs } : emptyRunActivityStats(); case "turn_start": diff --git a/web/workspace/src/lib/workspace/console/worker-console.ui.test.ts b/web/workspace/src/lib/workspace/console/worker-console.ui.test.ts index d9e8650c..8b45f90c 100644 --- a/web/workspace/src/lib/workspace/console/worker-console.ui.test.ts +++ b/web/workspace/src/lib/workspace/console/worker-console.ui.test.ts @@ -620,7 +620,7 @@ Deno.test("Worker Console paste chips preserve typed draft and target authority" consolePage.includes("preserveExactText: value.textPastes.length > 0") && consolePage.includes("composerDrafts.set(activeComposerTargetKey") && consolePage.includes("switchComposerTarget(target)") && - consolePage.includes('sendControl({ method: "cancel" }, "Stop")'), + consolePage.includes('sendWorkerControl("cancel")'), "Paste chips should use shared threshold classification, atomic keyboard behavior, accessible labels, typed restore, and per-Worker draft authority", ); }); @@ -655,6 +655,9 @@ Deno.test("workspace Runtime inventory lives under Settings admin routes", async import.meta.url, ), ); + const runtimeConnectionApi = await Deno.readTextFile( + new URL("../api/runtime-connection.ts", import.meta.url), + ); const workdirsPage = await Deno.readTextFile( new URL( "./../../../routes/w/[workspaceId]/settings/runtimes/[runtimeId]/workdirs/+page.svelte", @@ -678,9 +681,11 @@ Deno.test("workspace Runtime inventory lives under Settings admin routes", async runtimesPage.includes("Add remote Runtime") && runtimesPage.includes("Open workdirs") && runtimesPage.includes("settings-runtime-table") && - runtimesPage.includes( - "/runtimes/${encodeURIComponent(runtime.runtime_id)}/connection-tests", - ) && + runtimesPage.includes("testRuntimeConnection") && + runtimesPage.includes("data.workspaceId") && + runtimesPage.includes("runtime.runtime_id") && + runtimeConnectionApi.includes("/runtimes/${") && + runtimeConnectionApi.includes("}/connection-tests") && runtimesPage.includes( "/settings/runtimes/${encodeURIComponent(runtime.runtime_id)}/workdirs", ), @@ -782,7 +787,10 @@ Deno.test("Worker Console page is routed by runtime_id and worker_id through bac consolePage.includes( 'const composerEditable = $derived(protocolState === "open" && !sending);', ) && - consolePage.includes('sendControl({ method: "cancel" }, "Stop")') && + consolePage.includes('sendWorkerControl("cancel")') && + consolePage.includes("lifecycleMethod(command)") && + consolePage.includes("expected_worker_state_revision") && + consolePage.includes("expected_execution_generation") && consolePage.includes("onsubmit={handleComposerSubmit}") && consolePage.includes("disabled={!composerEditable}") && consolePage.includes("class:stop={workerRunning}") && @@ -1059,3 +1067,46 @@ Deno.test("Web Console switches main and direct SubWorker views from the Tasks r "Worker view selection should expose only direct SubWorker session identities with main fallback", ); }); + +Deno.test("Web Console uses Notify while running and exposes durable pending controls", async () => { + const consolePage = await Deno.readTextFile( + new URL( + "./../../../routes/w/[workspaceId]/runtimes/[runtimeId]/workers/[workerId]/console/+page.svelte", + import.meta.url, + ), + ); + + for ( + const token of [ + 'method: "submit"', + 'method: "notify"', + "notification_request_id: crypto.randomUUID()", + "submission_request_id: crypto.randomUUID()", + 'payload.event === "pending_submissions_changed"', + 'method: "cancel_pending_submission"', + 'method: "clear_pending_submissions"', + 'method: "continue_pending"', + "handleQueueSubmit", + "handleNotifySubmit", + 'submitDraft(composerInputElement?.snapshot() ?? draft, "queue")', + "disabled={!canQueueDraft}", + "disabled={!canNotifyDraft}", + ">Queue Submit", + ">Notify", + ] + ) { + assert( + consolePage.includes(token), + `missing durable pending control token: ${token}`, + ); + } + + const userCase = consolePage.slice( + consolePage.indexOf('case "user":'), + consolePage.indexOf('case "compact":'), + ); + assert( + !userCase.includes("workerRunning"), + "ordinary text must remain Submit instead of being implicitly converted to Notify", + ); +}); diff --git a/web/workspace/src/lib/workspace/settings/workspace-deletion-api.ts b/web/workspace/src/lib/workspace/settings/workspace-deletion-api.ts new file mode 100644 index 00000000..449bff2e --- /dev/null +++ b/web/workspace/src/lib/workspace/settings/workspace-deletion-api.ts @@ -0,0 +1,70 @@ +import type { + WorkspaceDeletionOperationResponse, + WorkspaceDeletionPreflightResponse, + WorkspaceDeletionRequest, +} from "$lib/generated/workspace-api"; +import { loadJson } from "$lib/workspace/api/http"; +import { + parseWorkspaceDeletionOperationResponse, + parseWorkspaceDeletionPreflightResponse, +} from "$lib/workspace/api/workspace-model"; + +const deletionResponsePolicy = { + maxResponseBytes: 2 * 1024 * 1024, + diagnosticLabel: "Workspace deletion", +} as const; + +async function deletionJson( + path: string, + init: RequestInit | undefined, + parse: (value: unknown) => T, +): Promise { + const result = await loadJson( + fetch, + path, + init, + parse, + deletionResponsePolicy, + ); + if (result.error !== null || result.data === null) { + throw new Error( + result.error ?? "Workspace deletion response is unavailable", + ); + } + return result.data; +} + +export async function preflightWorkspaceDeletion( + workspaceId: string, +): Promise { + return await deletionJson( + `/api/workspaces/${encodeURIComponent(workspaceId)}/deletion`, + undefined, + parseWorkspaceDeletionPreflightResponse, + ); +} + +export async function startWorkspaceDeletion( + workspaceId: string, + request: WorkspaceDeletionRequest, +): Promise { + return await deletionJson( + `/api/workspaces/${encodeURIComponent(workspaceId)}/deletion`, + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(request), + }, + parseWorkspaceDeletionOperationResponse, + ); +} + +export async function getWorkspaceDeletion( + operationId: string, +): Promise { + return await deletionJson( + `/api/workspace-deletions/${encodeURIComponent(operationId)}`, + undefined, + parseWorkspaceDeletionOperationResponse, + ); +} diff --git a/web/workspace/src/lib/workspace/sidebar/types.ts b/web/workspace/src/lib/workspace/sidebar/types.ts index 67d7aa10..856076fd 100644 --- a/web/workspace/src/lib/workspace/sidebar/types.ts +++ b/web/workspace/src/lib/workspace/sidebar/types.ts @@ -19,6 +19,7 @@ import type { Event as PodProtocolEvent, Method as PodProtocolMethod, Segment as PodProtocolSegment, + WorkerStateSnapshot, } from "$lib/generated/protocol"; import type { GitCommitSummary as SharedGitCommitSummary, @@ -99,6 +100,7 @@ export type Worker = { tags: string[]; workspace: { visibility: string; identity: string }; state: string; + worker_state?: WorkerStateSnapshot | null; pinned?: boolean; retention_state?: string; last_seen_at?: string | null; diff --git a/web/workspace/src/lib/workspace/sidebar/worker-state.ts b/web/workspace/src/lib/workspace/sidebar/worker-state.ts new file mode 100644 index 00000000..0f45e9f3 --- /dev/null +++ b/web/workspace/src/lib/workspace/sidebar/worker-state.ts @@ -0,0 +1,12 @@ +import type { WorkerStateSnapshot } from "$lib/generated/protocol"; + +export function liveWorkerState(worker: { + state: string; + worker_state?: WorkerStateSnapshot | null; +}): string { + const state = worker.worker_state?.state; + if (!state) return worker.state === "stopped" ? "stopped" : "unknown"; + if (state.kind === "idle") return "idle"; + if (state.state.kind === "maintenance") return "running"; + return state.state.state === "paused" ? "paused" : "running"; +} diff --git a/web/workspace/src/lib/workspace/sidebar/worker-subscription.test.ts b/web/workspace/src/lib/workspace/sidebar/worker-subscription.test.ts index 279afa0d..84e81598 100644 --- a/web/workspace/src/lib/workspace/sidebar/worker-subscription.test.ts +++ b/web/workspace/src/lib/workspace/sidebar/worker-subscription.test.ts @@ -5,6 +5,7 @@ function assertEquals(actual: unknown, expected: unknown): void { throw new Error(`expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`); } } +import { liveWorkerState } from './worker-state'; import { applyWorkspaceWorkersFrame, createWorkspaceWorkersProjection, @@ -33,6 +34,22 @@ function worker( }; } +Deno.test('Worker list state uses the authoritative live snapshot separately from lifecycle', () => { + const active = worker('runtime-a', 'worker-1', 1); + active.worker_state = { + execution_generation: 4, + revision: 2, + last_command_id: 1, + state: { kind: 'busy', state: { kind: 'run', state: 'paused' } }, + }; + assertEquals(liveWorkerState(active), 'paused'); + + const unavailable = worker('runtime-a', 'worker-2', 1); + assertEquals(liveWorkerState(unavailable), 'unknown'); + unavailable.state = 'stopped'; + assertEquals(liveWorkerState(unavailable), 'stopped'); +}); + Deno.test('workspace Worker snapshot keeps equal local ids from different Runtimes', () => { const projection = createWorkspaceWorkersProjection(); const frame: SubscriptionFrame = { diff --git a/web/workspace/src/lib/workspace/sidebar/worker-subscription.ts b/web/workspace/src/lib/workspace/sidebar/worker-subscription.ts index a1eed656..973e2f7d 100644 --- a/web/workspace/src/lib/workspace/sidebar/worker-subscription.ts +++ b/web/workspace/src/lib/workspace/sidebar/worker-subscription.ts @@ -5,6 +5,7 @@ import { applyWorkspaceWorkersFrame, createWorkspaceWorkersProjection, } from './worker-subscription-model'; +import { liveWorkerState } from './worker-state'; import { compareWorkersForSidebar } from './workers'; import type { Worker } from './types'; @@ -22,6 +23,10 @@ export type WorkspaceWorkersState = { const stores = new Map>(); +export function disposeWorkspaceWorkersStore(workspaceId: string): void { + stores.delete(workspaceId); +} + export function workspaceWorkersStore(workspaceId: string): Readable { const cached = stores.get(workspaceId); if (cached) return cached; @@ -86,7 +91,8 @@ function projectWorker(worker: SubscriptionWorker): SidebarWorker { profile: worker.profile ?? null, tags: [], workspace: { visibility: 'workspace', identity: 'runtime_subscription_worker' }, - state: worker.state, + state: liveWorkerState(worker), + worker_state: worker.worker_state, pinned: false, retention_state: 'transient', implementation: { diff --git a/web/workspace/src/lib/workspace/styles/settings.css b/web/workspace/src/lib/workspace/styles/settings.css index 0a1d1b82..03b4c3ad 100644 --- a/web/workspace/src/lib/workspace/styles/settings.css +++ b/web/workspace/src/lib/workspace/styles/settings.css @@ -339,6 +339,227 @@ background: rgba(255, 255, 255, 0.04); padding: 0.75rem; } + .settings-test-result.failed { + border-inline-start: 3px solid var(--danger); + } + + .runtime-detail-page { + display: grid; + gap: var(--space-5); + } + + .runtime-detail-section { + display: grid; + gap: var(--space-3); + padding-top: var(--space-4); + border-top: 1px solid var(--line); + } + + .runtime-detail-section h2, + .runtime-detail-section p { + margin: 0; + } + + .runtime-detail-grid { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(min(18rem, 100%), 1fr)); + gap: var(--space-3) var(--space-5); + margin: 0; + } + + .runtime-detail-grid div { + min-width: 0; + } + + .runtime-detail-grid dt { + margin-bottom: var(--space-1); + color: var(--text-muted); + font-size: 0.72rem; + font-weight: 700; + letter-spacing: 0.05em; + text-transform: uppercase; + } + + .runtime-detail-grid dd { + margin: 0; + color: var(--text-strong); + overflow-wrap: anywhere; + } + + .runtime-public-key-actions, + .runtime-revoke-row { + display: flex; + align-items: center; + justify-content: space-between; + gap: var(--space-3); + flex-wrap: wrap; + } + + .runtime-public-key-actions { + justify-content: flex-start; + } + + .runtime-public-key-actions button, + .runtime-revoke-row button, + .runtime-trust-form button { + border: 0; + border-radius: 0.6rem; + padding: 0.5rem 0.75rem; + background: var(--accent); + color: var(--bg); + font-weight: 700; + cursor: pointer; + } + + .runtime-public-key-actions button.secondary { + border: 1px solid var(--line); + background: transparent; + color: var(--text-strong); + } + + .runtime-public-key-actions button:disabled, + .runtime-revoke-row button:disabled, + .runtime-trust-form button:disabled { + cursor: not-allowed; + opacity: 0.55; + } + + .runtime-public-key, + .runtime-trust-form textarea, + .runtime-trust-form input, + .runtime-revoke-row input { + border: 1px solid var(--line); + border-radius: 0.5rem; + background: var(--bg-raised); + color: var(--text-strong); + font-family: var(--font-mono); + font-size: 0.78rem; + } + + .runtime-public-key { + max-height: 14rem; + margin: 0; + padding: var(--space-3); + overflow: auto; + white-space: pre-wrap; + overflow-wrap: anywhere; + } + + .runtime-trust-form { + display: grid; + gap: var(--space-2); + max-width: 56rem; + } + + .runtime-trust-form label, + .runtime-revoke-row label { + color: var(--text-muted); + font-size: 0.78rem; + font-weight: 700; + } + + .runtime-trust-form textarea, + .runtime-trust-form input, + .runtime-revoke-row input { + width: 100%; + padding: 0.65rem 0.75rem; + } + + .runtime-trust-form textarea { + resize: vertical; + } + + .runtime-trust-form small, + .runtime-revoke-row small { + color: var(--text-muted); + } + + .runtime-trust-comparison { + display: grid; + grid-template-columns: repeat(2, minmax(0, 1fr)); + gap: var(--space-3); + margin: 0; + padding: var(--space-3) 0; + border-block: 1px solid var(--line); + } + + .runtime-trust-comparison div { + min-width: 0; + } + + .runtime-trust-comparison dt { + color: var(--text-muted); + font-size: 0.72rem; + font-weight: 700; + } + + .runtime-trust-comparison dd { + margin: var(--space-1) 0 0; + overflow-wrap: anywhere; + } + + .runtime-trust-form .field-error, + .runtime-detail-page .section-state.error { + color: var(--danger); + } + + .runtime-detail-page .section-state.success { + color: var(--success); + } + + .runtime-revoke-row { + padding-top: var(--space-3); + border-top: 1px solid var(--line); + } + + .runtime-revoke-row div { + display: grid; + gap: var(--space-1); + } + + .runtime-revoke-row p { + color: var(--text-muted); + } + + .runtime-revoke-row button.danger { + background: var(--danger); + } + + .runtime-audit-table-wrap { + overflow-x: auto; + } + + .runtime-audit-table { + width: 100%; + min-width: 48rem; + border-collapse: collapse; + } + + .runtime-audit-table th, + .runtime-audit-table td { + padding: 0.7rem 0.5rem; + border-bottom: 1px solid var(--line); + text-align: left; + vertical-align: top; + } + + .runtime-audit-table th { + color: var(--text-muted); + font-size: 0.72rem; + letter-spacing: 0.05em; + text-transform: uppercase; + } + + .runtime-audit-table code { + overflow-wrap: anywhere; + } + + @media (max-width: 760px) { + .runtime-revoke-row { + align-items: stretch; + } + } + .settings-page { display: grid; gap: var(--space-5); diff --git a/web/workspace/src/routes/w/[workspaceId]/+layout.svelte b/web/workspace/src/routes/w/[workspaceId]/+layout.svelte index 6adaeb73..f5e8587f 100644 --- a/web/workspace/src/routes/w/[workspaceId]/+layout.svelte +++ b/web/workspace/src/routes/w/[workspaceId]/+layout.svelte @@ -12,6 +12,7 @@ } from '$lib/workspace/sidebar/context'; import { createOverrideStack } from '$lib/workspace/sidebar/override-stack'; import { disposeWorkspaceMultiplexer } from '$lib/workspace/multiplexer'; + import { disposeWorkspaceWorkersStore } from '$lib/workspace/sidebar/worker-subscription'; import WorkspaceSidebar from '$lib/workspace/sidebar/WorkspaceSidebar.svelte'; import '$lib/workspace/styles/workspace-pages.css'; import '$lib/workspace/styles/tickets.css'; @@ -32,7 +33,10 @@ $effect(() => { const workspaceId = data.workspace?.workspace_id; if (!workspaceId) return; - return () => disposeWorkspaceMultiplexer(workspaceId); + return () => { + disposeWorkspaceMultiplexer(workspaceId); + disposeWorkspaceWorkersStore(workspaceId); + }; }); diff --git a/web/workspace/src/routes/w/[workspaceId]/runtimes/[runtimeId]/workers/[workerId]/console/+page.svelte b/web/workspace/src/routes/w/[workspaceId]/runtimes/[runtimeId]/workers/[workerId]/console/+page.svelte index 03efcf8d..ed8d1df4 100644 --- a/web/workspace/src/routes/w/[workspaceId]/runtimes/[runtimeId]/workers/[workerId]/console/+page.svelte +++ b/web/workspace/src/routes/w/[workspaceId]/runtimes/[runtimeId]/workers/[workerId]/console/+page.svelte @@ -5,6 +5,11 @@ import ConsoleTimeline from "$lib/workspace/console/ConsoleTimeline.svelte"; import ComposerInput from "$lib/workspace/console/ComposerInput.svelte"; import type { ComposerDraftSnapshot } from "$lib/workspace/console/composer-draft"; + import { + canDeliverComposerDraft, + sendComposerDelivery, + type ComposerDelivery, + } from "$lib/workspace/console/composer-delivery"; import { buildComposerSegmentsRequest, type WorkerConsoleInputRequest, @@ -31,7 +36,13 @@ type ConsoleViewMode, type ConsoleViewScroll, } from "$lib/workspace/console/model"; - import type { Event as ProtocolEvent, Method as ProtocolMethod, RewindTarget, Segment } from "$lib/generated/protocol"; + import type { + Event as ProtocolEvent, + Method as ProtocolMethod, + PendingSubmissionsSnapshot, + RewindTarget, + Segment, + } from "$lib/generated/protocol"; import { MAX_FILES_PER_SUBMISSION, uploadAttachment, @@ -41,11 +52,7 @@ import { pushWorkspaceAlert } from "$lib/workspace/alerts/store"; import { workspaceApiPath } from "$lib/workspace/api/http"; import { workspaceMultiplexer, type WorkspaceMultiplexerSubscription } from "$lib/workspace/multiplexer"; - import type { - Diagnostic, - Worker, - PodProtocolEvent, - } from "$lib/workspace/sidebar/types"; + import type { Diagnostic, Worker } from "$lib/workspace/sidebar/types"; type Props = { data: { @@ -152,6 +159,13 @@ "connecting", ); let protocolSubscription: WorkspaceMultiplexerSubscription | null = null; + let pendingSubmissions = $state({ + revision: 0, + notification_count: 0, + head_id: null, + submissions: [], + }); + let pendingSubmissionItems = $derived(pendingSubmissions.submissions ?? []); let pendingCompletionRequest: { resolve: (entries: ComposerCompletionEntry[]) => void; reject: (error: Error) => void; @@ -189,7 +203,6 @@ ); let pendingObservationEvents: ConsoleEventInput[] = []; let protocolEventSequence = 0; - let pendingObservedStates: Array = []; let pendingStreamDiagnostics: Diagnostic[] = []; let observationFlushHandle: number | null = null; let nextReloadToken = 0; @@ -231,16 +244,47 @@ const diagnostics = $derived( mergeDiagnostics(worker?.diagnostics ?? [], streamDiagnostics), ); - const workerState = $derived(liveWorkerState ?? worker?.state ?? "loading"); + const workerState = $derived( + liveWorkerState ?? (worker?.state === "stopped" ? "stopped" : "loading"), + ); const workerRunning = $derived(workerState === "running"); const workerPaused = $derived(workerState === "paused"); - const inputReady = $derived(workerState === "idle"); const composerEditable = $derived(protocolState === "open" && !sending); - const canSubmitDraft = $derived(inputReady && composerEditable); - const canSend = $derived(canSubmitDraft && draft.content.trim().length > 0); + const draftHasText = $derived(draft.content.trim().length > 0); + const draftHasAttachments = $derived(attachments.length > 0); + const canSubmitDraft = $derived( + canDeliverComposerDraft({ + delivery: "submit", + workerState, + protocolOpen: protocolState === "open", + sending, + hasText: draftHasText, + hasAttachments: draftHasAttachments, + }), + ); + const canQueueDraft = $derived( + canDeliverComposerDraft({ + delivery: "queue", + workerState, + protocolOpen: protocolState === "open", + sending, + hasText: draftHasText, + hasAttachments: draftHasAttachments, + }), + ); + const canNotifyDraft = $derived( + canDeliverComposerDraft({ + delivery: "notify", + workerState, + protocolOpen: protocolState === "open", + sending, + hasText: draftHasText, + hasAttachments: draftHasAttachments, + }), + ); const canStopFromComposer = $derived(workerRunning && composerEditable); const composerSubmitDisabled = $derived( - workerRunning ? !canStopFromComposer : !canSend, + workerRunning ? !canStopFromComposer : !canSubmitDraft, ); async function getJson(path: string): Promise { @@ -296,7 +340,6 @@ observationFlushHandle = null; } pendingObservationEvents = []; - pendingObservedStates = []; pendingStreamDiagnostics = []; } @@ -312,18 +355,15 @@ function flushObservationBatch() { observationFlushHandle = null; const eventBatch = pendingObservationEvents; - const stateBatch = pendingObservedStates; const diagnosticBatch = pendingStreamDiagnostics; pendingObservationEvents = []; - pendingObservedStates = []; pendingStreamDiagnostics = []; if (eventBatch.length > 0) { - const latestState = stateBatch.findLast((state) => state !== null); - if (latestState) { - liveWorkerState = latestState; - } consoleProjection = consoleProjector.append(eventBatch); + liveWorkerState = consoleProjection.status === "shutdown" + ? "shutdown" + : workerStateFromSnapshot(consoleProjection.workerState); advanceEventObservedAtVersion(); } @@ -334,6 +374,13 @@ function handleIncomingProtocolEvent(payload: ProtocolEvent) { handleProtocolCommandEvent(payload); + if (payload.event === "snapshot") { + pendingSubmissions = payload.data.session.pending_submissions; + } else if (payload.event === "segment_rotated") { + pendingSubmissions = payload.data.session.pending_submissions; + } else if (payload.event === "pending_submissions_changed") { + pendingSubmissions = payload.data.pending; + } if (payload.event === "error") { queueObservationDiagnostic({ code: payload.data.code, @@ -353,7 +400,6 @@ event: payload, observedAtMs, }); - pendingObservedStates.push(workerStateFromProtocolEvent(payload)); scheduleObservationFlush(); } @@ -487,9 +533,42 @@ } } + let nextWorkerCommandId = 1; + + function lifecycleMethod( + command: "pause" | "cancel" | "resume" | "compact", + ): ProtocolMethod | null { + const state = consoleProjection.workerState; + if (!state) { + sendError = "Worker state snapshot is not available; reconnect before sending control."; + return null; + } + const commandId = Math.max( + nextWorkerCommandId, + state.last_command_id + 1, + ); + nextWorkerCommandId = commandId + 1; + const envelope = { + command_id: commandId, + expected_execution_generation: state.execution_generation, + expected_worker_state_revision: state.revision, + }; + switch (command) { + case "pause": + return { method: "pause", params: { command: envelope } }; + case "cancel": + return { method: "cancel", params: { command: envelope } }; + case "resume": + return { method: "resume", params: { command: envelope } }; + case "compact": + return { method: "compact", params: { command: envelope } }; + } + } + function sendWorkerControl(command: "pause" | "cancel" | "resume") { const label = command[0].toUpperCase() + command.slice(1); - sendControl({ method: command }, label); + const method = lifecycleMethod(command); + if (method) sendControl(method, label); } function isEditableTarget(target: EventTarget | null): boolean { @@ -556,8 +635,9 @@ switch (request.kind) { case "user": return { - method: "run", + method: "submit", params: { + submission_request_id: crypto.randomUUID(), input: request.segments ?? [ { kind: "text", content: request.content }, ], @@ -566,10 +646,17 @@ case "notify": return { method: "notify", - params: { message: request.content, auto_run: true }, + params: { + notification_request_id: crypto.randomUUID(), + message: request.content, + auto_run: true, + }, }; - case "compact": - return { method: "compact" }; + case "compact": { + const method = lifecycleMethod("compact"); + if (!method) throw new Error("Worker state snapshot is not available"); + return method; + } case "list_rewind_targets": return { method: "list_rewind_targets" }; case "register_peer": @@ -632,12 +719,20 @@ function handleComposerSubmit() { if (workerRunning) { - sendControl({ method: "cancel" }, "Stop"); + sendWorkerControl("cancel"); return; } void submitDraft(composerInputElement?.snapshot() ?? draft); } + function handleQueueSubmit() { + void submitDraft(composerInputElement?.snapshot() ?? draft, "queue"); + } + + function handleNotifySubmit() { + void submitDraft(composerInputElement?.snapshot() ?? draft, "notify"); + } + function attachmentPath(): string { return `/api/w/${encodeURIComponent(workspaceId)}/runtimes/${encodeURIComponent(runtimeId)}/workers/${encodeURIComponent(workerId)}`; } @@ -737,7 +832,15 @@ if (event.dataTransfer?.files) addAttachmentFiles(event.dataTransfer.files); } - async function submitDraft(value: ComposerDraftSnapshot) { + async function submitDraft( + value: ComposerDraftSnapshot, + delivery: ComposerDelivery = "submit", + ) { + if (delivery === "notify" && attachments.length > 0) { + composerNotice = null; + sendError = "Notify accepts text only; remove attachments or queue a Submit."; + return; + } const incompleteAttachment = attachments.find((attachment) => attachment.state !== "uploaded" || !attachment.reference ); @@ -767,19 +870,38 @@ composerInputElement?.clear(); return; } - if (sending || !inputReady) { + const deliveryState = { + delivery, + workerState, + protocolOpen: protocolState === "open", + sending, + hasText: value.content.trim().length > 0, + hasAttachments: attachments.length > 0, + }; + if (!canDeliverComposerDraft(deliveryState)) { return; } + let request: WorkerConsoleInputRequest = command.request; + if (delivery === "notify") { + if (request.kind !== "user") { + composerNotice = null; + sendError = "Notify accepts ordinary text, not a Composer command."; + return; + } + request = { kind: "notify", content: request.content }; + } sending = true; sendError = null; try { - const method = composerRequestToProtocolMethod(command.request); - sendProtocolMethod(method); + const method = composerRequestToProtocolMethod(request); + if (!sendComposerDelivery(deliveryState, method, sendProtocolMethod)) { + return; + } composerInputElement?.recordHistory(value); composerInputElement?.clear(); attachments = []; - if (method.method === "run" || method.method === "notify") { + if (method.method === "submit" || method.method === "notify") { liveWorkerState = "running"; } composerNotice = "Sent through Worker protocol."; @@ -795,18 +917,16 @@ handleComposerSubmit(); } - function workerStateFromProtocolEvent( - event: PodProtocolEvent, + function workerStateFromSnapshot( + snapshot: ConsoleProjection["workerState"], ): string | null { - switch (event.event) { - case "snapshot": - case "status": - return event.data.status; - case "shutdown": - return "shutdown"; - default: - return null; - } + if (!snapshot) return null; + return snapshot.state.kind === "idle" + ? "idle" + : snapshot.state.state.kind === "run" && + snapshot.state.state.state === "paused" + ? "paused" + : "running"; } function connectProtocolTransport( @@ -1526,7 +1646,10 @@ type="button" class="secondary-button" disabled={protocolState !== "open"} - onclick={() => sendControl({ method: "compact" }, "Compact")} + onclick={() => { + const method = lifecycleMethod("compact"); + if (method) sendControl(method, "Compact"); + }} > Compact @@ -1722,6 +1845,62 @@ {/if} + {#if pendingSubmissionItems.length > 0 || pendingSubmissions.notification_count > 0} +
+ + Pending activations ({pendingSubmissionItems.length} submissions · {pendingSubmissions.notification_count} notifications) + +
    + {#each pendingSubmissionItems as submission (submission.submission_id)} +
  1. + {submission.submission_id} + {submission.segment_count} segments · {submission.byte_len} bytes + +
    2. + {/each} +
+ + +
+ {/if} + {#if workerRunning}
+ {#if workerRunning} + + + {/if} {#if composerNotice} {composerNotice} {/if} @@ -2035,6 +2226,31 @@ display: none; } + .pending-submissions { + margin: 0 var(--space-3); + color: var(--muted); + font-size: 0.75rem; + } + + .pending-submissions ol { + display: grid; + gap: var(--space-1); + margin: var(--space-2) 0; + padding-left: var(--space-5); + } + + .pending-submissions li { + display: flex; + gap: var(--space-2); + align-items: center; + } + + .pending-submissions code { + max-width: 16rem; + overflow: hidden; + text-overflow: ellipsis; + } + .console-log { display: grid; align-content: start; diff --git a/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/+page.svelte b/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/+page.svelte index f1e3e25d..35c87b2c 100644 --- a/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/+page.svelte +++ b/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/+page.svelte @@ -1,20 +1,13 @@ + + + {data.runtimeDetail?.runtime.label ?? data.runtimeId} · Runtime Settings · Yoi Workspace + + + +
+
+
+ Runtimes +

{data.runtimeDetail?.runtime.label ?? data.runtimeId}

+

{data.runtimeId}

+
+ + Workdirs + +
+ + {#if data.runtimeDetailError} +

{data.runtimeDetailError}

+ {:else if !data.runtimeDetail} +

Loading Runtime…

+ {:else} + {@const detail = data.runtimeDetail} + {@const runtime = detail.runtime} + {@const trust = detail.trust_key} + {@const currentAction = trustAction(trust.status)} + +
+

Identity and binding

+
+
Runtime ID
{runtime.runtime_id}
+
Kind
{runtime.kind}
+
Endpoint
{detail.endpoint ?? 'Not configured'}
+
Status
{runtime.status}
+
Binding status
{trust.status}
+
Fingerprint
{trust.fingerprint ?? '—'}
+
Revision
{trust.revision?.toString() ?? '—'}
+
Created
{formatTimestamp(trust.created_at)}
+
Updated
{formatTimestamp(trust.updated_at)}
+
Revoked
{formatTimestamp(trust.revoked_at)}
+
+ {#if runtime.diagnostics.length > 0} +
    + {#each runtime.diagnostics as diagnostic} +
  • + {diagnostic.code} + {diagnostic.message} +
    • + {/each} +
+ {/if} +
+ + {#if data.workspace.permissions.manage_runtimes && !runtime.management.built_in} +
+

Workspace trust

+ + {#if trust.status !== 'unconfigured'} +
+ + +
+ {#if showPublicKey && revealedPublicKey} + 
{revealedPublicKey}
+ {/if} + {/if} + +
+ + + +
+
+
Current fingerprint
+
{trust.fingerprint ?? 'Not configured'}
+
+
+
Replacement fingerprint
+
{replacementFingerprint ?? 'Enter a valid public key'}
+
+
+ {#if replacementFingerprintError} +

{replacementFingerprintError}

+ {/if} + + {#if currentAction !== 'create'} + + + Enter {trust.fingerprint ?? 'the current fingerprint'} exactly. + {/if} + + {#if fieldError} +

{fieldError}

+ {/if} +
+ +
+
+ +
+
+ Revoke Workspace trust +

Workspace trust only; this does not delete the Runtime process, Workers, or Workdirs.

+ +
+ +
+ + {#if requestError} + + {/if} + {#if successMessage} +

{successMessage}

+ {/if} +
+ {/if} + +
+

Recent trust audit

+ {#if detail.recent_audit.length === 0} +

No trust changes are recorded.

+ {:else} +
+ + + + + + {#each detail.recent_audit as entry} + + + + + + + + {/each} + +
ActionRevisionFingerprintActorTime
{entry.action}{entry.revision.toString()}{entry.new_fingerprint ?? entry.old_fingerprint ?? '—'}{entry.actor_account_id}{formatTimestamp(entry.at)}
+
+ {/if} +
+ {/if} +
diff --git a/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/+page.ts b/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/+page.ts new file mode 100644 index 00000000..100e82f5 --- /dev/null +++ b/web/workspace/src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/+page.ts @@ -0,0 +1,31 @@ +import { loadJson, workspaceApiPath } from "$lib/workspace/api/http"; +import { parseWorkspaceRuntimeDetail } from "$lib/workspace/api/runtime-management"; +import type { PageLoad } from "./$types"; + +export const load: PageLoad = async ({ fetch, params }) => { + const detail = await loadJson( + fetch, + workspaceApiPath( + params.workspaceId, + `/runtimes/${encodeURIComponent(params.runtimeId)}`, + ), + undefined, + (value) => { + const response = parseWorkspaceRuntimeDetail(value); + if ( + response.workspace_id !== params.workspaceId || + response.runtime.runtime_id !== params.runtimeId + ) { + throw new Error("Runtime detail did not match the route"); + } + return response; + }, + ); + + return { + workspaceId: params.workspaceId, + runtimeId: params.runtimeId, + runtimeDetail: detail.data, + runtimeDetailError: detail.error, + }; +}; diff --git a/web/workspace/src/routes/w/[workspaceId]/settings/workspace/+page.svelte b/web/workspace/src/routes/w/[workspaceId]/settings/workspace/+page.svelte index 5c1fdf7d..26ec7005 100644 --- a/web/workspace/src/routes/w/[workspaceId]/settings/workspace/+page.svelte +++ b/web/workspace/src/routes/w/[workspaceId]/settings/workspace/+page.svelte @@ -1,8 +1,20 @@ @@ -113,3 +230,67 @@ {/if} + +{#if data.workspace?.permissions.delete_workspace} +
+
+

Danger zone

+

Deleting this Workspace permanently removes its Workers, Workdirs, repositories, configuration, Memory, Tickets, and audit data.

+
+ +
+{/if} + +{#if deletionOpen} + +{/if} + + diff --git a/web/workspace/src/routes/w/[workspaceId]/workers/+page.svelte b/web/workspace/src/routes/w/[workspaceId]/workers/+page.svelte index e734def2..e3a0d542 100644 --- a/web/workspace/src/routes/w/[workspaceId]/workers/+page.svelte +++ b/web/workspace/src/routes/w/[workspaceId]/workers/+page.svelte @@ -4,6 +4,7 @@ import { workerHref } from '$lib/workspace/resource-links'; import { formatCurrentWorkdirRevision } from '$lib/workspace/settings/workdir-revision'; import { canOpenWorkerConsole } from '$lib/workspace/sidebar/workers'; + import { liveWorkerState } from '$lib/workspace/sidebar/worker-state'; import type { CleanupWorkerCandidate, RuntimeCleanupExecutionResponse, RuntimeCleanupPlanResponse, Worker } from '$lib/workspace/sidebar/types'; import type { PageProps } from './$types'; @@ -136,7 +137,7 @@ } function workerStatus(worker: Worker): string { - return worker.state; + return liveWorkerState(worker); } function workerProfile(worker: Worker): string { diff --git a/web/workspace/src/routes/w/[workspaceId]/workers/[workerRef]/+page.svelte b/web/workspace/src/routes/w/[workspaceId]/workers/[workerRef]/+page.svelte index ef3d04bb..15d3ffe4 100644 --- a/web/workspace/src/routes/w/[workspaceId]/workers/[workerRef]/+page.svelte +++ b/web/workspace/src/routes/w/[workspaceId]/workers/[workerRef]/+page.svelte @@ -1,5 +1,6 @@ @@ -24,7 +25,7 @@ >Open console
-
Status
{data.worker.state}
+
Status
{liveWorkerState(data.worker)}
Profile
{data.worker.profile}
Internal ID
{data.worker.worker_id}
diff --git a/web/workspace/tests/runtime-connection.test.ts b/web/workspace/tests/runtime-connection.test.ts new file mode 100644 index 00000000..5e1a85ea --- /dev/null +++ b/web/workspace/tests/runtime-connection.test.ts @@ -0,0 +1,103 @@ +declare const Deno: { + test(name: string, fn: () => void | Promise): void; +}; + +import { + parseRuntimeConnectionTestResponse, + testRuntimeConnection, +} from "../src/lib/workspace/api/runtime-connection.ts"; + +function assertEquals(actual: unknown, expected: unknown): void { + if (JSON.stringify(actual) !== JSON.stringify(expected)) { + throw new Error( + `expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`, + ); + } +} + +function compatibleResponse(): Record { + return { + workspace_id: "workspace-a", + runtime_id: "runtime-a", + checked_at: "2026-09-01T12:00:00Z", + status: "compatible", + failure_kind: null, + expected_protocol_version: 1, + actual_protocol_version: 1, + diagnostics: [], + }; +} + +Deno.test("runtime connection response accepts the exact compatible contract", () => { + assertEquals( + parseRuntimeConnectionTestResponse(compatibleResponse()), + compatibleResponse(), + ); +}); + +Deno.test("runtime connection response rejects unknown fields and incoherent compatibility", () => { + assertEquals( + parseRuntimeConnectionTestResponse({ + ...compatibleResponse(), + capabilities: ["shell"], + }), + null, + ); + assertEquals( + parseRuntimeConnectionTestResponse({ + ...compatibleResponse(), + actual_protocol_version: 2, + }), + null, + ); + assertEquals( + parseRuntimeConnectionTestResponse({ + ...compatibleResponse(), + failure_kind: "timeout", + }), + null, + ); +}); + +Deno.test("runtime connection response rejects unknown failure kinds and unbounded diagnostics", () => { + const failed = { + ...compatibleResponse(), + status: "failed", + failure_kind: "future_failure", + actual_protocol_version: null, + diagnostics: [], + }; + assertEquals(parseRuntimeConnectionTestResponse(failed), null); + assertEquals( + parseRuntimeConnectionTestResponse({ + ...failed, + failure_kind: "timeout", + diagnostics: Array.from({ length: 17 }, () => ({ + code: "timeout", + severity: "error", + message: "Timed out", + })), + }), + null, + ); +}); + +Deno.test("runtime connection request rejects a mismatched response identity", async () => { + const fetchImpl = (() => + Promise.resolve( + new Response( + JSON.stringify({ ...compatibleResponse(), runtime_id: "runtime-b" }), + { status: 200, headers: { "content-type": "application/json" } }, + ), + )) as typeof fetch; + let message = ""; + try { + await testRuntimeConnection("workspace-a", "runtime-a", fetchImpl); + } catch (error) { + message = error instanceof Error ? error.message : String(error); + } + assertEquals( + message, + "Connection test response did not match the selected Runtime", + ); +}); diff --git a/web/workspace/tests/runtime-management-source.test.ts b/web/workspace/tests/runtime-management-source.test.ts new file mode 100644 index 00000000..1aaee2c1 --- /dev/null +++ b/web/workspace/tests/runtime-management-source.test.ts @@ -0,0 +1,157 @@ +declare const Deno: { + test(name: string, fn: () => void | Promise): void; + readTextFile(path: URL): Promise; +}; + +function assert(condition: unknown, message: string): asserts condition { + if (!condition) throw new Error(message); +} + +Deno.test("Runtime Settings routes validate unknown JSON through the shared Runtime parser", async () => { + const [listLoader, detailLoader] = await Promise.all([ + Deno.readTextFile( + new URL( + "../src/routes/w/[workspaceId]/settings/runtimes/+page.ts", + import.meta.url, + ), + ), + Deno.readTextFile( + new URL( + "../src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/+page.ts", + import.meta.url, + ), + ), + ]); + + assert( + listLoader.includes("parseWorkspaceRuntimeList(value)"), + "Runtime list loader should validate unknown JSON", + ); + assert( + detailLoader.includes("parseWorkspaceRuntimeDetail(value)"), + "Runtime detail loader should validate unknown JSON", + ); + for (const source of [listLoader, detailLoader]) { + assert( + !source.includes("loadJson<"), + "Runtime loaders must not cast response JSON to a handwritten DTO", + ); + } +}); + +Deno.test("Runtime list links to canonical detail and has no inline delete action", async () => { + const page = await Deno.readTextFile( + new URL( + "../src/routes/w/[workspaceId]/settings/runtimes/+page.svelte", + import.meta.url, + ), + ); + + assert( + page.includes( + "/settings/runtimes/${encodeURIComponent(runtime.runtime_id)}", + ), + "Runtime name should link to canonical detail", + ); + assert( + page.includes("testRuntime(runtime)"), + "connection Test should remain available", + ); + assert(page.includes("Add Runtime"), "Add Runtime should remain available"); + assert( + page.includes("data.workspace.permissions.manage_runtimes"), + "Add Runtime should be hidden from non-owners", + ); + assert( + !page.includes("deleteRuntime"), + "inline Runtime delete logic must be removed", + ); + assert( + !page.includes(">Delete"), + "inline Runtime delete control must be removed", + ); +}); + +Deno.test("Runtime detail keeps trust controls owner-only and conflict-safe", async () => { + const page = await Deno.readTextFile( + new URL( + "../src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/+page.svelte", + import.meta.url, + ), + ); + + const ownerGate = page.indexOf("data.workspace.permissions.manage_runtimes"); + const reveal = page.indexOf("Reveal public key"); + const mutation = page.indexOf('id="runtime-public-key-input"'); + assert(ownerGate >= 0, "Runtime trust controls should use manage_runtimes"); + assert( + page.includes("Current fingerprint"), + "current fingerprint must be explicit", + ); + assert( + page.includes("Replacement fingerprint"), + "replacement fingerprint must be previewed before confirmation", + ); + assert( + page.includes("!runtime.management.built_in"), + "Runtime trust controls should be hidden for the built-in Runtime", + ); + assert( + ownerGate < reveal && ownerGate < mutation, + "owner gate should wrap key controls", + ); + + for ( + const token of [ + "Create Workspace trust", + "Replace trusted key", + "Reactivate with this key", + "Confirm current fingerprint", + "Revoke Workspace trust", + "Workspace trust only; this does not delete the Runtime process, Workers, or Workdirs.", + "RuntimeTrustConflictError", + "RuntimeTrustRouteFence", + "routeFence.enter(data.runtimeId)", + "showPublicKey = false", + "revealedPublicKey = null", + "publicKey = ''", + "fingerprintConfirmation = ''", + "revokeFingerprintConfirmation = ''", + "requestError = null", + "successMessage = null", + "isCurrentRoute(operation)", + "revealRuntimeTrustKey", + "revokeFingerprintConfirmation.trim() !== trust.fingerprint", + "await reloadAuthority()", + "busyAction !== null", + "Workdirs", + "Recent trust audit", + ] + ) { + assert(page.includes(token), `Runtime detail should include ${token}`); + } +}); + +Deno.test("Runtime detail uses flat sections instead of nested cards", async () => { + const [page, css] = await Promise.all([ + Deno.readTextFile( + new URL( + "../src/routes/w/[workspaceId]/settings/runtimes/[runtimeId]/+page.svelte", + import.meta.url, + ), + ), + Deno.readTextFile( + new URL("../src/lib/workspace/styles/settings.css", import.meta.url), + ), + ]); + + assert( + !page.includes('class="card"') && !page.includes("settings-card"), + "Runtime detail should not add card nesting", + ); + assert( + css.includes(".runtime-detail-section") && + css.includes("border-top: 1px solid var(--line)"), + "Runtime detail hierarchy should use flat section separators", + ); +}); diff --git a/web/workspace/tests/runtime-management.test.ts b/web/workspace/tests/runtime-management.test.ts new file mode 100644 index 00000000..3d5c83b6 --- /dev/null +++ b/web/workspace/tests/runtime-management.test.ts @@ -0,0 +1,287 @@ +declare const Deno: { + test(name: string, fn: () => void | Promise): void; +}; + +import { + parseRuntimeTrustConflict, + parseRuntimeTrustKeyRevealResponse, + parseWorkspaceRuntimeDetail, + parseWorkspaceRuntimeList, + previewRuntimePublicKeyFingerprint, + putRuntimeTrustKey, + revokeRuntimeTrustKey, + RuntimeTrustConflictError, + RuntimeTrustRouteFence, +} from "../src/lib/workspace/api/runtime-management.ts"; + +function assert(condition: unknown, message: string): asserts condition { + if (!condition) throw new Error(message); +} + +function assertThrows(operation: () => unknown, expected: string): void { + try { + operation(); + } catch (error) { + const message = error instanceof Error ? error.message : String(error); + if (message.includes(expected)) return; + throw new Error( + `expected error containing ${expected}, received ${message}`, + ); + } + throw new Error("expected operation to throw"); +} + +function runtime() { + return { + management: { + built_in: false, + config_managed: true, + removable: false, + endpoint_configured: true, + token_ref_configured: false, + }, + runtime_id: "arcadia", + label: "Arcadia", + kind: "remote", + status: "started", + source: { + kind: "remote_http", + status: "active", + identity_authority: "server_runtime_configuration", + note: "Configured by Server authority", + }, + host_ids: ["host-a"], + worker_creation_available: true, + os: "linux", + arch: "x86_64", + diagnostics: [], + }; +} + +function detail() { + return { + workspace_id: "workspace-a", + runtime: runtime(), + endpoint: "https://runtime.example.test", + trust_key: { + status: "active", + fingerprint: "SHA256:current", + revision: 3, + created_at: "2026-09-01T12:00:00Z", + updated_at: "2026-09-01T13:00:00Z", + revoked_at: null, + }, + recent_audit: [{ + action: "created", + actor_account_id: "account-a", + old_fingerprint: null, + new_fingerprint: "SHA256:current", + revision: 3, + at: "2026-09-01T13:00:00Z", + }], + }; +} + +Deno.test("Runtime list and detail parsers return generated Runtime DTO shapes", () => { + const list = parseWorkspaceRuntimeList({ + workspace_id: "workspace-a", + limit: 200, + items: [runtime()], + source: "workspace-control-plane", + diagnostics: [], + }); + assert( + list.items[0]?.runtime_id === "arcadia", + "Runtime ID was not preserved", + ); + + const parsed = parseWorkspaceRuntimeDetail(detail()); + assert( + parsed.trust_key.revision === 3, + "revision was not preserved as a safe integer", + ); + assert( + parsed.recent_audit[0]?.revision === 3, + "audit revision was not normalized", + ); +}); + +Deno.test("Runtime validators reject unknown object keys and enum variants", () => { + assertThrows( + () => parseWorkspaceRuntimeDetail({ ...detail(), head_tree: "stale" }), + "head_tree is not part", + ); + + const futureSource = structuredClone(detail()); + futureSource.runtime.source.kind = "future_transport"; + assertThrows( + () => parseWorkspaceRuntimeDetail(futureSource), + "contains an unknown enum value", + ); + + assertThrows( + () => + parseRuntimeTrustConflict({ + error: "future_conflict", + message: "conflict", + current_revision: 4, + current_fingerprint: "SHA256:new", + }), + "contains an unknown enum value", + ); +}); + +Deno.test("Runtime validators reject unsafe revisions and bounded collection overflow", () => { + const unsafeRevision = structuredClone(detail()); + unsafeRevision.trust_key.revision = Number.MAX_SAFE_INTEGER + 1; + assertThrows( + () => parseWorkspaceRuntimeDetail(unsafeRevision), + "must be a safe integer", + ); + + const tooMuchAudit = structuredClone(detail()); + tooMuchAudit.recent_audit = Array.from( + { length: 21 }, + () => structuredClone(detail().recent_audit[0]), + ); + assertThrows( + () => parseWorkspaceRuntimeDetail(tooMuchAudit), + "must contain at most 20 items", + ); + + const tooManyItems = Array.from({ length: 201 }, () => runtime()); + assertThrows( + () => + parseWorkspaceRuntimeList({ + workspace_id: "workspace-a", + limit: 200, + items: tooManyItems, + source: "workspace-control-plane", + diagnostics: [], + }), + "must contain at most 200 items", + ); +}); + +Deno.test("Runtime detail rejects unbounded strings and incoherent trust state", () => { + assertThrows( + () => + parseRuntimeTrustKeyRevealResponse({ + public_key: "x".repeat(16 * 1024 + 1), + }), + "must be at most 16384 UTF-8 bytes", + ); + + const activeWithoutFingerprint = structuredClone(detail()) as Record< + string, + unknown + >; + (activeWithoutFingerprint.trust_key as Record).fingerprint = + null; + assertThrows( + () => parseWorkspaceRuntimeDetail(activeWithoutFingerprint), + "must include fingerprint", + ); +}); + +Deno.test("mismatched revoke fingerprint never sends a request", async () => { + let requests = 0; + const fetchImpl: typeof fetch = () => { + requests += 1; + return Promise.reject(new Error("request must not be sent")); + }; + let rejected = false; + try { + await revokeRuntimeTrustKey( + "workspace-a", + "runtime-a", + { expected_revision: 3 }, + "sha256:current", + "sha256:different", + fetchImpl, + ); + } catch (error) { + rejected = error instanceof Error && + error.message.includes("current fingerprint exactly"); + } + assert(rejected, "mismatched fingerprint should be rejected locally"); + assert(requests === 0, "mismatched fingerprint sent a revoke request"); +}); + +Deno.test("Runtime route fence rejects a delayed reveal from the prior Runtime", async () => { + const fence = new RuntimeTrustRouteFence(); + fence.enter("runtime-a"); + const operation = fence.capture("runtime-a"); + let renderedKey: string | null = null; + let resolveReveal!: (key: string) => void; + const delayedReveal = new Promise((resolve) => { + resolveReveal = resolve; + }).then((key) => { + if (fence.isCurrent(operation, "runtime-b")) renderedKey = key; + }); + + fence.enter("runtime-b"); + resolveReveal("runtime-a-public-key"); + await delayedReveal; + assert( + renderedKey === null, + "Runtime A key rendered after navigating to Runtime B", + ); +}); + +Deno.test("Runtime public key preview matches the Server fingerprint contract", async () => { + const fingerprint = await previewRuntimePublicKeyFingerprint( + "yoi-ed25519-pub:v1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + ); + assert( + fingerprint === + "sha256:66687aadf862bd776c8fc18b8e9f8e20089714856ee233b3902a591d0d5f2925", + "fingerprint preview drifted from the Server SHA-256 contract", + ); +}); + +Deno.test("typed trust conflict is validated and preserves authoritative revision", async () => { + let sentBody: unknown = null; + const fetchImpl = ((_: RequestInfo | URL, init?: RequestInit) => { + sentBody = JSON.parse(String(init?.body)) as unknown; + return Promise.resolve( + new Response( + JSON.stringify({ + error: "stale_revision", + message: "Runtime trust changed", + current_revision: 4, + current_fingerprint: "SHA256:new", + }), + { status: 409, headers: { "content-type": "application/json" } }, + ), + ); + }) as typeof fetch; + + try { + await putRuntimeTrustKey( + "workspace-a", + "arcadia", + { public_key: "ssh-ed25519 AAAA-new", expected_revision: 3 }, + fetchImpl, + ); + throw new Error("expected mutation to reject"); + } catch (error) { + assert( + error instanceof RuntimeTrustConflictError, + "expected typed conflict", + ); + assert( + error.conflict.current_revision === 4, + "authoritative revision was lost", + ); + } + + assert( + JSON.stringify(sentBody) === + JSON.stringify({ + public_key: "ssh-ed25519 AAAA-new", + expected_revision: 3, + }), + "request should serialize the generated bigint revision as a safe JSON integer", + ); +}); diff --git a/web/workspace/tests/workspace-model.test.ts b/web/workspace/tests/workspace-model.test.ts index 66efbae9..223d4b17 100644 --- a/web/workspace/tests/workspace-model.test.ts +++ b/web/workspace/tests/workspace-model.test.ts @@ -6,6 +6,8 @@ declare const Deno: { import { parseRepositoryListApiResult, parseRepositoryListResponse, + parseWorkspaceDeletionOperationResponse, + parseWorkspaceDeletionPreflightResponse, parseWorkspaceResponse, } from "../src/lib/workspace/api/workspace-model.ts"; @@ -109,6 +111,108 @@ Deno.test("workspace response requires the permission projection", () => { ); }); +Deno.test("Workspace deletion DTOs fail closed and preserve durable operation state", () => { + const preflight = parseWorkspaceDeletionPreflightResponse({ + workspace_id: "workspace-a", + display_name: "Alpha", + expected_revision: "2026-01-01T00:00:00Z", + can_delete: true, + resources: { + workers: 2, + workdirs: 1, + repositories: 1, + runtime_bindings: 1, + secrets: 0, + artifacts: 3, + }, + blockers: [], + }); + if (preflight.resources.workers !== 2) { + throw new Error("worker count was not preserved"); + } + + const operation = parseWorkspaceDeletionOperationResponse({ + operation_id: "delete-alpha", + workspace_id: "workspace-a", + display_name: "Alpha", + state: "blocked", + resources: preflight.resources, + child_operation_ids: ["worker-remove:arcadia/7"], + blockers: [{ + kind: "dirty_workdir", + resource_kind: "workdir", + resource_key: "WD-1", + message: "Workdir is dirty", + }], + failure_category: null, + created_at: "2026-01-01T00:00:00Z", + updated_at: "2026-01-01T00:01:00Z", + completed_at: null, + }); + if (operation.state !== "blocked") { + throw new Error("operation state was not preserved"); + } + + assertThrows( + () => + parseWorkspaceDeletionPreflightResponse({ + ...preflight, + unexpected: true, + }), + "unexpected is not part", + ); + assertThrows( + () => + parseWorkspaceDeletionOperationResponse({ + ...operation, + state: "unknown", + }), + ".state is invalid", + ); + assertThrows( + () => + parseWorkspaceDeletionOperationResponse({ + ...operation, + operation_id: "x".repeat(129), + }), + ".operation_id is too long", + ); + assertThrows( + () => + parseWorkspaceDeletionOperationResponse({ + ...operation, + blockers: Array.from({ length: 1025 }, () => operation.blockers[0]), + }), + ".blockers has too many items", + ); +}); + +Deno.test("Workspace settings exposes owner-gated typed destructive confirmation", async () => { + const source = await Deno.readTextFile( + new URL( + "../src/routes/w/[workspaceId]/settings/workspace/+page.svelte", + import.meta.url, + ), + ); + for ( + const token of [ + "permissions.delete_workspace", + "preflightWorkspaceDeletion", + "startWorkspaceDeletion", + "deletionConfirmation", + "disposeWorkspaceMultiplexer(workspaceId)", + "disposeWorkspaceWorkersStore(workspaceId)", + "sessionStorage.setItem(deletionStorageKey", + "storedDeletionRequest()", + "trackDeletion(request.operation_id)", + ] + ) { + if (!source.includes(token)) { + throw new Error(`Workspace deletion UI should include ${token}`); + } + } +}); + Deno.test("Repository settings consume the validated shared wire shape", async () => { const [loadSource, pageSource] = await Promise.all([ Deno.readTextFile(