diff --git a/crates/protocol/src/subscription.rs b/crates/protocol/src/subscription.rs index 8a036b05b..642a1528d 100644 --- a/crates/protocol/src/subscription.rs +++ b/crates/protocol/src/subscription.rs @@ -12,7 +12,7 @@ use serde::{Deserialize, Deserializer, Serialize}; use crate::Event as WorkerProtocolEvent; -pub const SUBSCRIPTION_PROTOCOL_VERSION: u16 = 1; +pub const SUBSCRIPTION_PROTOCOL_VERSION: u16 = 2; /// Browser/runtime wire guardrails. These bounds are generated into the Workspace /// Browser validator so aggregate limits remain part of the Rust-owned contract. pub const MAX_SUBSCRIPTION_FRAME_JSON_BYTES: usize = 16 * 1024 * 1024; @@ -426,7 +426,6 @@ pub enum SubscriptionResponse { request_id: SubscriptionRequestId, subscription_id: SubscriptionId, selector: EventSubscriptionSelector, - snapshot_revision: u64, snapshot: SubscriptionSnapshot, }, Unsubscribed { @@ -512,7 +511,6 @@ pub enum SubscriptionTerminationCode { pub enum SubscriptionEvent { Event { subscription_id: SubscriptionId, - subject_revision: u64, payload: SubscriptionEventPayload, }, SubscriptionClosed { @@ -666,9 +664,7 @@ pub struct SubscriptionWorker { /// `Unavailable` preserves catalog membership without claiming that execution stopped. #[serde(default)] pub availability: SubscriptionWorkerAvailability, - /// Producer-owned monotonic revision for this Worker subject. - pub subject_revision: u64, - /// Latest revisioned foreground state observed from the Worker. This remains + /// Latest foreground state observed from the Worker. This remains /// absent until an authoritative Worker snapshot/event has been applied. #[serde(default, skip_serializing_if = "Option::is_none")] pub worker_state: Option, @@ -970,7 +966,6 @@ mod tests { runtime_id: None, resource_key: None, availability: SubscriptionWorkerAvailability::Observed, - subject_revision: 0, worker_state: None, state: SubscriptionWorkerState::Idle, has_running_internal_workers: false, @@ -1093,7 +1088,7 @@ mod tests { assert_eq!( json, serde_json::json!({ - "protocol_version": 1, + "protocol_version": 2, "frame": "request", "message": { "method": "subscribe_events", @@ -1119,7 +1114,6 @@ mod tests { request_id: request_id(), subscription_id: subscription_id(), selector: EventSubscriptionSelector::RuntimeWorkers, - snapshot_revision: 7, snapshot: SubscriptionSnapshot::Workers { workers: vec![worker("worker-1")], }, @@ -1158,10 +1152,10 @@ mod tests { subscription_id: subscription_id(), }, )); - frame.protocol_version = 2; + frame.protocol_version = 1; assert!(matches!( frame.validate(), - Err(SubscriptionValidationError::UnsupportedProtocolVersion { actual: 2 }) + Err(SubscriptionValidationError::UnsupportedProtocolVersion { actual: 1 }) )); } @@ -1180,7 +1174,6 @@ mod tests { let event = SubscriptionEvent::Event { subscription_id: subscription_id(), - subject_revision: 8, payload: SubscriptionEventPayload::WorkerRemoved { worker_id: worker_id("worker-2"), runtime_id: None, @@ -1211,7 +1204,7 @@ mod tests { assert_eq!( serde_json::to_value(frame).unwrap(), serde_json::json!({ - "protocol_version": 1, + "protocol_version": 2, "frame": "worker_protocol", "message": { "subscription_id": "subscription-1", @@ -1253,7 +1246,7 @@ mod tests { assert_eq!( serde_json::to_value(frame).unwrap(), serde_json::json!({ - "protocol_version": 1, + "protocol_version": 2, "frame": "event", "message": { "event": "subscription_closed", diff --git a/crates/protocol/src/typescript.rs b/crates/protocol/src/typescript.rs index 281e8ece6..5535bb557 100644 --- a/crates/protocol/src/typescript.rs +++ b/crates/protocol/src/typescript.rs @@ -230,7 +230,6 @@ fn rust_serialized_subscription_frame_fixtures() -> Vec { runtime_id: Some("fixture-runtime".to_string()), resource_key: Some("fixture-worker-resource".to_string()), availability: SubscriptionWorkerAvailability::default(), - subject_revision: 1, worker_state: None, state: SubscriptionWorkerState::Idle, has_running_internal_workers: false, @@ -250,7 +249,6 @@ fn rust_serialized_subscription_frame_fixtures() -> Vec { request_id, subscription_id: subscription_id.clone(), selector: EventSubscriptionSelector::WorkspaceWorkers, - snapshot_revision: 1, snapshot: SubscriptionSnapshot::Workers { workers: vec![worker], }, @@ -259,7 +257,6 @@ fn rust_serialized_subscription_frame_fixtures() -> Vec { let pending = SubscriptionFrame::new(SubscriptionFramePayload::Event(SubscriptionEvent::Event { subscription_id, - subject_revision: 2, payload: SubscriptionEventPayload::WorkerProtocol { worker_id, event: Event::PendingSubmissionsChanged { diff --git a/crates/worker-runtime/src/http_server.rs b/crates/worker-runtime/src/http_server.rs index 9c1a4f7fa..e3b5ca2c3 100644 --- a/crates/worker-runtime/src/http_server.rs +++ b/crates/worker-runtime/src/http_server.rs @@ -1334,7 +1334,6 @@ async fn runtime_protocol_ws_session( request_id, subscription_id: subscription_id.clone(), selector: selector.clone(), - snapshot_revision: subscription.snapshot_revision(), snapshot: subscription.snapshot().clone(), }, )); @@ -1354,7 +1353,6 @@ async fn runtime_protocol_ws_session( SubscriptionFramePayload::Event( SubscriptionEvent::Event { subscription_id: event_subscription_id.clone(), - subject_revision: update.subject_revision, payload: update.payload, }, ), diff --git a/crates/worker-runtime/src/lib.rs b/crates/worker-runtime/src/lib.rs index a2b49dcb7..9c7c3df04 100644 --- a/crates/worker-runtime/src/lib.rs +++ b/crates/worker-runtime/src/lib.rs @@ -36,5 +36,8 @@ pub mod workspace_request; #[cfg(feature = "fs-store")] pub use fs_store::{FsRuntimeStore, FsRuntimeStoreOptions}; pub use management::RuntimeOptions; -pub use runtime::{Runtime, RuntimeWorkspaceScope}; +pub use runtime::{ + Runtime, RuntimeEventSelectorSubscription, RuntimeSubscriptionRecvError, + RuntimeSubscriptionUpdate, RuntimeWorkspaceScope, +}; pub use session_store::UploadedFileUploadContext; diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index f16d5f603..c80787cf3 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -151,7 +151,6 @@ impl RuntimeWorkerRestoreResult { #[derive(Clone, Debug)] pub struct RuntimeSubscriptionUpdate { - pub subject_revision: u64, pub payload: SubscriptionEventPayload, } @@ -168,7 +167,6 @@ pub enum RuntimeSubscriptionRecvError { pub struct RuntimeEventSelectorSubscription { subscription_id: u64, selector: EventSubscriptionSelector, - snapshot_revision: u64, snapshot: SubscriptionSnapshot, receiver: mpsc::Receiver, lagged: Arc, @@ -180,10 +178,6 @@ impl RuntimeEventSelectorSubscription { &self.selector } - pub fn snapshot_revision(&self) -> u64 { - self.snapshot_revision - } - pub fn snapshot(&self) -> &SubscriptionSnapshot { &self.snapshot } @@ -1280,7 +1274,6 @@ impl Runtime { state.persist_runtime_snapshot()?; } let snapshot = state.subscription_snapshot(scope, &selector)?; - let snapshot_revision = state.subscription_revision; let subscription_id = state.next_event_subscription_id; state.next_event_subscription_id = state.next_event_subscription_id.saturating_add(1); let (sender, receiver) = mpsc::channel(SUBSCRIPTION_QUEUE_CAPACITY); @@ -1298,7 +1291,6 @@ impl Runtime { Ok(RuntimeEventSelectorSubscription { subscription_id, selector, - snapshot_revision, snapshot, receiver, lagged, @@ -2217,9 +2209,9 @@ impl Runtime { if let Some(payload) = input_protocol_event(&input) { state.push_worker_observation_event(worker_ref.clone(), payload); } - state.publish_worker_upsert(worker_ref.worker_id)?; state.persist_runtime_snapshot()?; state.persist_worker(&worker_ref.worker_id)?; + state.publish_worker_upsert(worker_ref.worker_id)?; Ok(WorkerInteractionAck { worker_ref: worker_ref.clone(), @@ -2500,9 +2492,9 @@ impl Runtime { worker.workdir_attachments = workdir_attachments; worker.detail() }; - state.publish_worker_upsert(worker_ref.worker_id)?; state.persist_runtime_snapshot()?; state.persist_worker(&worker_ref.worker_id)?; + state.publish_worker_upsert(worker_ref.worker_id)?; Ok(detail) } @@ -2541,8 +2533,8 @@ impl Runtime { state .observation_events .retain(|event| event.worker_ref != *worker_ref); - state.publish_worker_removed(worker_ref.worker_id, workspace_id.as_deref())?; state.persist_runtime_snapshot()?; + state.publish_worker_removed(worker_ref.worker_id, workspace_id.as_deref())?; } Ok(()) } @@ -2564,9 +2556,9 @@ impl Runtime { let mut state = self.lock()?; let worker = state.worker_mut(worker_ref)?; worker.apply_worker_state(&snapshot); - state.publish_worker_upsert(worker_ref.worker_id)?; state.persist_runtime_snapshot()?; state.persist_worker(&worker_ref.worker_id)?; + state.publish_worker_upsert(worker_ref.worker_id)?; Ok(()) } @@ -2870,8 +2862,8 @@ impl Runtime { state .observation_events .retain(|event| event.worker_ref != *worker_ref); - state.publish_worker_removed(worker_ref.worker_id, removed_workspace_id.as_deref())?; state.persist_runtime_snapshot()?; + state.publish_worker_removed(worker_ref.worker_id, removed_workspace_id.as_deref())?; Ok(WorkerDeleteResult { worker_id: removed.worker_id, deleted: true, @@ -2959,13 +2951,13 @@ impl Runtime { let worker_state_changed = state.project_protocol_event_to_worker_state(worker_ref, &payload); let activity_changed = state.project_internal_worker_activity(worker_ref, &payload); - if worker_state_changed || activity_changed { - state.publish_worker_upsert(worker_ref.worker_id)?; - } if worker_state_changed { state.persist_runtime_snapshot()?; state.persist_worker(&worker_ref.worker_id)?; } + if worker_state_changed || activity_changed { + state.publish_worker_upsert(worker_ref.worker_id)?; + } let event = state.push_worker_observation_event(worker_ref.clone(), payload); Ok(event) } @@ -3021,13 +3013,13 @@ impl Runtime { state.project_protocol_event_to_worker_state(&worker_ref, &payload); let activity_changed = state.project_internal_worker_activity(&worker_ref, &payload); - if worker_state_changed || activity_changed { - state.publish_worker_upsert(worker_ref.worker_id)?; - } if worker_state_changed { state.persist_runtime_snapshot()?; state.persist_worker(&worker_ref.worker_id)?; } + if worker_state_changed || activity_changed { + state.publish_worker_upsert(worker_ref.worker_id)?; + } Ok(state.push_worker_observation_event(worker_ref, payload)) }), ) @@ -3054,13 +3046,13 @@ impl Runtime { state.project_protocol_event_to_worker_state(&worker_ref, &payload); let activity_changed = state.project_internal_worker_activity(&worker_ref, &payload); - if worker_state_changed || activity_changed { - state.publish_worker_upsert(worker_ref.worker_id)?; - } if worker_state_changed { state.persist_runtime_snapshot()?; state.persist_worker(&worker_ref.worker_id)?; } + if worker_state_changed || activity_changed { + state.publish_worker_upsert(worker_ref.worker_id)?; + } Ok(state.push_worker_observation_event(worker_ref, payload)) }), ) @@ -3454,8 +3446,8 @@ impl Runtime { logical_workdir_attachments_from_statuses(&workdir_attachments, claims); candidate.workdir_attachments = workdir_attachments; state.workers.insert(worker_ref.worker_id, candidate); - state.publish_worker_upsert(worker_ref.worker_id)?; state.persist_worker(&worker_ref.worker_id)?; + state.publish_worker_upsert(worker_ref.worker_id)?; Ok(()) } @@ -3702,8 +3694,6 @@ struct RuntimeState { workspace_config_latest: BTreeMap, workspace_config_fetch_gates: BTreeMap>>, diagnostics: Vec, - subscription_revision: u64, - worker_subject_revisions: BTreeMap, next_event_subscription_id: u64, subscriptions: BTreeMap, #[cfg(feature = "ws-server")] @@ -3733,8 +3723,6 @@ impl RuntimeState { workspace_config_latest: BTreeMap::new(), workspace_config_fetch_gates: BTreeMap::new(), diagnostics: Vec::new(), - subscription_revision: 0, - worker_subject_revisions: BTreeMap::new(), next_event_subscription_id: 1, subscriptions: BTreeMap::new(), #[cfg(feature = "ws-server")] @@ -3765,8 +3753,6 @@ impl RuntimeState { workspace_config_latest: BTreeMap::new(), workspace_config_fetch_gates: BTreeMap::new(), diagnostics: Vec::new(), - subscription_revision: 0, - worker_subject_revisions: BTreeMap::new(), next_event_subscription_id: 1, subscriptions: BTreeMap::new(), #[cfg(feature = "ws-server")] @@ -3895,8 +3881,6 @@ impl RuntimeState { workspace_config_fetch_gates: BTreeMap::new(), workspace_owners: persisted.workspace_owners, diagnostics, - subscription_revision: 0, - worker_subject_revisions: BTreeMap::new(), next_event_subscription_id: 1, subscriptions: BTreeMap::new(), #[cfg(feature = "ws-server")] @@ -4215,11 +4199,6 @@ impl RuntimeState { runtime_id: None, resource_key: None, availability: protocol::subscription::SubscriptionWorkerAvailability::Observed, - subject_revision: self - .worker_subject_revisions - .get(&worker.worker_id) - .copied() - .unwrap_or(0), worker_state: worker.worker_state.clone(), state: subscription_worker_state(worker.status), has_running_internal_workers: worker.internal_workers.has_running_worker(), @@ -4232,25 +4211,17 @@ impl RuntimeState { } fn publish_worker_upsert(&mut self, worker_id: WorkerId) -> Result<(), RuntimeError> { - self.subscription_revision = self.subscription_revision.saturating_add(1); - let subject_revision = { - let revision = self.worker_subject_revisions.entry(worker_id).or_insert(0); - *revision = revision.saturating_add(1); - *revision - }; let worker = self .workers .get(&worker_id) .ok_or(RuntimeError::WorkerNotFound { worker_id })?; let workspace_id = worker.workspace_id.clone(); - let mut projected = self.subscription_worker(worker)?; - projected.subject_revision = subject_revision; + let projected = self.subscription_worker(worker)?; let projected_worker_id = projected.worker_id.clone(); self.deliver_worker_subscription_update( &projected_worker_id, workspace_id.as_deref(), RuntimeSubscriptionUpdate { - subject_revision, payload: SubscriptionEventPayload::WorkerUpserted { worker: projected }, }, ); @@ -4262,19 +4233,12 @@ impl RuntimeState { worker_id: WorkerId, workspace_id: Option<&str>, ) -> Result<(), RuntimeError> { - self.subscription_revision = self.subscription_revision.saturating_add(1); - let subject_revision = { - let revision = self.worker_subject_revisions.entry(worker_id).or_insert(0); - *revision = revision.saturating_add(1); - *revision - }; let worker_id = SubscriptionWorkerId::new(worker_id.to_string()) .map_err(subscription_validation_error)?; self.deliver_worker_subscription_update( &worker_id, workspace_id, RuntimeSubscriptionUpdate { - subject_revision, payload: SubscriptionEventPayload::WorkerRemoved { worker_id: worker_id.clone(), runtime_id: None, @@ -4417,8 +4381,8 @@ impl RuntimeState { worker_ref: Some(worker_ref.clone()), }); self.workers.insert(worker_ref.worker_id, candidate); - self.publish_worker_upsert(worker_ref.worker_id)?; self.persist_runtime_snapshot()?; + self.publish_worker_upsert(worker_ref.worker_id)?; Ok(()) } @@ -6637,7 +6601,6 @@ mod tests { let mut subscription = runtime .subscribe_event_selector(EventSubscriptionSelector::RuntimeWorkers) .unwrap(); - assert_eq!(subscription.snapshot_revision(), 0); let SubscriptionSnapshot::Workers { workers } = subscription.snapshot() else { panic!("runtime_workers must return a Worker snapshot"); }; @@ -6645,11 +6608,9 @@ mod tests { let created = runtime.create_worker(task_request("live")).unwrap(); let update = receive_subscription_update(&mut subscription).unwrap(); - assert_eq!(update.subject_revision, 1); match update.payload { SubscriptionEventPayload::WorkerUpserted { worker } => { assert_eq!(worker.worker_id.as_str(), created.worker_id.to_string()); - assert_eq!(worker.subject_revision, 1); assert_eq!(worker.state, SubscriptionWorkerState::Idle); } payload => panic!("unexpected subscription payload: {payload:?}"), @@ -6666,7 +6627,6 @@ mod tests { ) .unwrap(); let update = receive_subscription_update(&mut subscription).unwrap(); - assert_eq!(update.subject_revision, 2); match update.payload { SubscriptionEventPayload::WorkerUpserted { worker } => { assert_eq!(worker.state, SubscriptionWorkerState::Idle); @@ -6685,7 +6645,6 @@ mod tests { ) .unwrap(); let update = receive_subscription_update(&mut subscription).unwrap(); - assert_eq!(update.subject_revision, 3); match update.payload { SubscriptionEventPayload::WorkerUpserted { worker } => { assert_eq!(worker.state, SubscriptionWorkerState::Idle); @@ -6727,7 +6686,6 @@ mod tests { ) .unwrap(); let update = receive_subscription_update(&mut subscription).unwrap(); - assert_eq!(update.subject_revision, 4); match update.payload { SubscriptionEventPayload::WorkerUpserted { worker } => { assert_eq!(worker.state, SubscriptionWorkerState::Idle); @@ -6738,7 +6696,6 @@ mod tests { runtime.stop_worker(&created.worker_ref, None).unwrap(); let update = receive_subscription_update(&mut subscription).unwrap(); - assert_eq!(update.subject_revision, 5); match update.payload { SubscriptionEventPayload::WorkerUpserted { worker } => { assert_eq!(worker.worker_id.as_str(), created.worker_id.to_string()); @@ -6750,6 +6707,87 @@ mod tests { runtime.stop_runtime().unwrap(); } + #[test] + fn concurrent_snapshot_and_worker_creation_cross_the_boundary_exactly_once() { + let runtime = runtime_with_backend(); + let barrier = Arc::new(Barrier::new(2)); + let subscribing_runtime = runtime.clone(); + let subscribing_barrier = barrier.clone(); + let subscriber = std::thread::spawn(move || { + subscribing_barrier.wait(); + subscribing_runtime + .subscribe_event_selector(EventSubscriptionSelector::RuntimeWorkers) + .unwrap() + }); + let creating_runtime = runtime.clone(); + let creating_barrier = barrier.clone(); + let creator = std::thread::spawn(move || { + creating_barrier.wait(); + creating_runtime + .create_worker(task_request("concurrent-boundary")) + .unwrap() + }); + + let mut subscription = subscriber.join().unwrap(); + let created = creator.join().unwrap(); + let SubscriptionSnapshot::Workers { workers } = subscription.snapshot() else { + panic!("runtime_workers must return a Worker snapshot"); + }; + let in_snapshot = workers + .iter() + .any(|worker| worker.worker_id.as_str() == created.worker_id.to_string()); + let in_event = match subscription.receiver.try_recv() { + Ok(RuntimeSubscriptionUpdate { + payload: SubscriptionEventPayload::WorkerUpserted { worker }, + }) => worker.worker_id.as_str() == created.worker_id.to_string(), + Ok(update) => panic!("unexpected subscription update: {update:?}"), + Err(mpsc::error::TryRecvError::Empty) => false, + Err(error) => panic!("unexpected subscription receive error: {error}"), + }; + assert_ne!( + in_snapshot, in_event, + "the concurrent creation must appear in the snapshot or its ordered suffix, never both" + ); + } + + #[cfg(feature = "fs-store")] + #[test] + fn restarted_runtime_subscription_starts_from_persisted_authoritative_snapshot() { + let root = tempfile::tempdir().unwrap(); + let options = crate::fs_store::FsRuntimeStoreOptions { + root: root.path().join("runtime"), + runtime_id: "subscription-restart".to_string(), + display_name: None, + }; + let runtime = Runtime::with_fs_store_and_execution_backend( + options.clone(), + Arc::new(TestExecutionBackend::default()), + ) + .unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("persisted-subscription-worker")) + .unwrap(); + drop(runtime); + + let restarted = Runtime::with_fs_store_and_execution_backend( + options, + Arc::new(TestExecutionBackend::default()), + ) + .unwrap(); + let subscription = restarted + .subscribe_event_selector(EventSubscriptionSelector::RuntimeWorkers) + .unwrap(); + let SubscriptionSnapshot::Workers { workers } = subscription.snapshot() else { + panic!("runtime_workers must return a Worker snapshot"); + }; + assert!(workers.iter().any(|candidate| { + candidate.worker_id.as_str() == worker.worker_id.to_string() + && candidate.availability + == protocol::subscription::SubscriptionWorkerAvailability::Observed + })); + } + #[test] fn worker_lifecycle_subscription_delivers_only_selected_workers() { let runtime = runtime_with_backend(); diff --git a/crates/workspace-server/src/latest_schema.sql b/crates/workspace-server/src/latest_schema.sql index f5f021c53..1ba828b75 100644 --- a/crates/workspace-server/src/latest_schema.sql +++ b/crates/workspace-server/src/latest_schema.sql @@ -914,31 +914,15 @@ CREATE TABLE worker_registry_observations ( worker_id TEXT NOT NULL, availability TEXT NOT NULL CHECK (availability IN ('observed', 'unavailable')), worker_json TEXT, - connection_generation INTEGER NOT NULL, - subject_revision INTEGER NOT NULL, - snapshot_revision INTEGER NOT NULL, - projection_revision INTEGER NOT NULL, observed_at TEXT NOT NULL, PRIMARY KEY (workspace_id, runtime_id, worker_id), FOREIGN KEY (workspace_id, worker_id) REFERENCES worker_registry(workspace_id, worker_id) ON DELETE CASCADE ); -CREATE TABLE worker_registry_projection_cursors ( - workspace_id TEXT NOT NULL, - runtime_id TEXT NOT NULL, - connection_generation INTEGER NOT NULL, - snapshot_revision INTEGER NOT NULL, - PRIMARY KEY (workspace_id, runtime_id) -); -CREATE TABLE worker_registry_projection_revisions ( - workspace_id TEXT PRIMARY KEY, - revision INTEGER NOT NULL -); CREATE TABLE worker_registry_projection_removals ( workspace_id TEXT NOT NULL, runtime_id TEXT NOT NULL, worker_id TEXT NOT NULL, - projection_revision INTEGER NOT NULL, PRIMARY KEY (workspace_id, runtime_id, worker_id) ); CREATE TABLE worker_registry_projection_diagnostics ( diff --git a/crates/workspace-server/src/runtime_subscription.rs b/crates/workspace-server/src/runtime_subscription.rs index 5b5eb6bcf..3a7fdd0bd 100644 --- a/crates/workspace-server/src/runtime_subscription.rs +++ b/crates/workspace-server/src/runtime_subscription.rs @@ -33,26 +33,19 @@ pub enum RuntimeSubscriptionBrokerError { #[derive(Clone, Debug)] pub enum BrokerSubscriptionEvent { Snapshot { - connection_generation: u64, - snapshot_revision: u64, snapshot: SubscriptionSnapshot, }, Event { - connection_generation: u64, - subject_revision: u64, payload: SubscriptionEventPayload, }, Disconnected { - connection_generation: u64, message: String, }, Rejected { - connection_generation: u64, code: SubscriptionRejectionCode, message: String, }, Closed { - connection_generation: u64, code: SubscriptionTerminationCode, message: String, }, @@ -60,7 +53,6 @@ pub enum BrokerSubscriptionEvent { #[derive(Clone, Debug, Default, PartialEq, Eq)] pub struct RuntimeSubscriptionBrokerStatus { - pub connection_generation: u64, pub connected: bool, pub desired_selectors: usize, pub upstream_subscriptions: usize, @@ -82,7 +74,30 @@ impl BrokerSubscription { &self.selector } pub async fn recv(&mut self) -> Option { - self.receiver.recv().await + loop { + if let Some(event) = self.receiver.recv().await { + return Some(event); + } + if self.restart().is_err() { + return None; + } + } + } + + pub(crate) fn restart(&mut self) -> Result<(), RuntimeSubscriptionBrokerError> { + self.commands + .send(Command::Unsubscribe(self.downstream_id)) + .map_err(|_| RuntimeSubscriptionBrokerError::Closed)?; + let (events, receiver) = mpsc::channel(DOWNSTREAM_QUEUE_CAPACITY); + self.commands + .send(Command::Subscribe { + downstream_id: self.downstream_id, + selector: self.selector.clone(), + events, + }) + .map_err(|_| RuntimeSubscriptionBrokerError::Closed)?; + self.receiver = receiver; + Ok(()) } } @@ -120,10 +135,7 @@ impl RuntimeSubscriptionBroker { pub fn register_remote_runtime(&self, config: RemoteRuntimeConfig) -> u64 { let generation = self.next_generation.fetch_add(1, Ordering::Relaxed); let (commands, receiver) = mpsc::unbounded_channel(); - let status = Arc::new(RwLock::new(RuntimeSubscriptionBrokerStatus { - connection_generation: generation, - ..Default::default() - })); + let status = Arc::new(RwLock::new(RuntimeSubscriptionBrokerStatus::default())); let registration = Registration { generation, commands: commands.clone(), @@ -156,7 +168,6 @@ impl RuntimeSubscriptionBroker { let generation = self.next_generation.fetch_add(1, Ordering::Relaxed); let (commands, receiver) = mpsc::unbounded_channel(); let status = Arc::new(RwLock::new(RuntimeSubscriptionBrokerStatus { - connection_generation: generation, connected: true, ..Default::default() })); @@ -257,7 +268,6 @@ impl RuntimeSubscriptionBroker { .clone(); if !initial_status.connected { let _ = initial_events.try_send(BrokerSubscriptionEvent::Disconnected { - connection_generation: initial_status.connection_generation, message: "Runtime subscription connection is not currently available".to_string(), }); } @@ -286,8 +296,7 @@ struct SelectorState { downstreams: HashMap>, upstream_id: Option, pending: bool, - snapshot: Option<(u64, SubscriptionSnapshot)>, - revisions: HashMap, + snapshot: Option, } impl SelectorState { fn new() -> Self { @@ -296,7 +305,6 @@ impl SelectorState { upstream_id: None, pending: false, snapshot: None, - revisions: HashMap::new(), } } } @@ -334,11 +342,9 @@ impl State { selector.upstream_id = None; selector.pending = false; selector.snapshot = None; - selector.revisions.clear(); broadcast( &mut selector.downstreams, BrokerSubscriptionEvent::Disconnected { - connection_generation: self.generation, message: message.clone(), }, ); @@ -347,12 +353,77 @@ impl State { } struct EmbeddedEntry { + lifetime: u64, downstreams: HashMap>, - snapshot_revision: u64, snapshot: SubscriptionSnapshot, task: tokio::task::JoinHandle<()>, } +enum EmbeddedUpdate { + Event { + selector: EventSubscriptionSelector, + lifetime: u64, + payload: SubscriptionEventPayload, + }, + Closed { + selector: EventSubscriptionSelector, + lifetime: u64, + code: SubscriptionTerminationCode, + message: String, + }, +} + +fn start_embedded_subscription( + runtime: &worker_runtime::Runtime, + runtime_id: &str, + selector: EventSubscriptionSelector, + lifetime: u64, + updates: mpsc::UnboundedSender, +) -> Result<(SubscriptionSnapshot, tokio::task::JoinHandle<()>), worker_runtime::error::RuntimeError> +{ + let mut subscription = runtime.subscribe_event_selector(selector.clone())?; + let snapshot = project_snapshot_runtime(subscription.snapshot().clone(), runtime_id); + let task_runtime_id = runtime_id.to_string(); + let task = tokio::spawn(async move { + loop { + match subscription.recv().await { + Ok(update) => { + let payload = project_payload_runtime(update.payload, &task_runtime_id); + if updates + .send(EmbeddedUpdate::Event { + selector: selector.clone(), + lifetime, + payload, + }) + .is_err() + { + return; + } + } + Err(worker_runtime::RuntimeSubscriptionRecvError::Lagged) => { + let _ = updates.send(EmbeddedUpdate::Closed { + selector, + lifetime, + code: SubscriptionTerminationCode::Lagged, + message: "embedded Runtime subscription lagged; resubscribing from a fresh snapshot".to_string(), + }); + return; + } + Err(worker_runtime::RuntimeSubscriptionRecvError::Closed) => { + let _ = updates.send(EmbeddedUpdate::Closed { + selector, + lifetime, + code: SubscriptionTerminationCode::ServerShutdown, + message: "embedded Runtime subscription closed".to_string(), + }); + return; + } + } + } + }); + Ok((snapshot, task)) +} + async fn run_embedded_connection( runtime_id: String, runtime: worker_runtime::Runtime, @@ -364,6 +435,7 @@ async fn run_embedded_connection( let (updates, mut update_receiver) = mpsc::unbounded_channel(); let mut entries = HashMap::::new(); let mut downstream_index = HashMap::::new(); + let mut next_lifetime = 1_u64; loop { tokio::select! { command = commands.recv() => match command { @@ -371,30 +443,38 @@ async fn run_embedded_connection( downstream_index.insert(downstream_id, selector.clone()); if let Some(entry) = entries.get_mut(&selector) { let _ = events.try_send(BrokerSubscriptionEvent::Snapshot { - connection_generation: generation, - snapshot_revision: entry.snapshot_revision, snapshot: entry.snapshot.clone(), }); entry.downstreams.insert(downstream_id, events); } else { - match runtime.subscribe_event_selector(selector.clone()) { - Ok(mut subscription) => { - let snapshot_revision = subscription.snapshot_revision(); - let snapshot = project_snapshot_runtime(subscription.snapshot().clone(), &runtime_id); - let _ = events.try_send(BrokerSubscriptionEvent::Snapshot { connection_generation: generation, snapshot_revision, snapshot: snapshot.clone() }); - let sender = updates.clone(); - let task_selector = selector.clone(); - let task_runtime_id = runtime_id.clone(); - let task = tokio::spawn(async move { - while let Ok(update) = subscription.recv().await { - let payload = project_payload_runtime(update.payload, &task_runtime_id); - if sender.send((task_selector.clone(), update.subject_revision, payload)).is_err() { break; } - } + let lifetime = next_lifetime; + next_lifetime = next_lifetime.saturating_add(1); + match start_embedded_subscription( + &runtime, + &runtime_id, + selector.clone(), + lifetime, + updates.clone(), + ) { + Ok((snapshot, task)) => { + let _ = events.try_send(BrokerSubscriptionEvent::Snapshot { + snapshot: snapshot.clone(), }); - entries.insert(selector, EmbeddedEntry { downstreams: HashMap::from([(downstream_id, events)]), snapshot_revision, snapshot, task }); + entries.insert( + selector, + EmbeddedEntry { + lifetime, + downstreams: HashMap::from([(downstream_id, events)]), + snapshot, + task, + }, + ); } Err(error) => { - let _ = events.try_send(BrokerSubscriptionEvent::Rejected { connection_generation: generation, code: SubscriptionRejectionCode::UnsupportedSelector, message: error.to_string() }); + let _ = events.try_send(BrokerSubscriptionEvent::Rejected { + code: SubscriptionRejectionCode::UnsupportedSelector, + message: error.to_string(), + }); } } } @@ -407,7 +487,7 @@ async fn run_embedded_connection( } Some(Command::Shutdown(replacement)) => { for entry in entries.values_mut() { - broadcast(&mut entry.downstreams, BrokerSubscriptionEvent::Closed { connection_generation: generation, code: SubscriptionTerminationCode::ServerShutdown, message: format!("embedded Runtime generation {generation} was fenced by {replacement}") }); + broadcast(&mut entry.downstreams, BrokerSubscriptionEvent::Closed { code: SubscriptionTerminationCode::ServerShutdown, message: format!("embedded Runtime registration {generation} was replaced by {replacement}") }); entry.task.abort(); } return; @@ -415,16 +495,68 @@ async fn run_embedded_connection( None => return, }, update = update_receiver.recv() => { - let Some((selector, subject_revision, payload)) = update else { return; }; - if let Some(entry) = entries.get_mut(&selector) { - entry.snapshot_revision = entry.snapshot_revision.saturating_add(1); - apply_event_to_cached_snapshot(&mut entry.snapshot, &payload); - broadcast(&mut entry.downstreams, BrokerSubscriptionEvent::Event { connection_generation: generation, subject_revision, payload }); + let Some(update) = update else { return; }; + match update { + EmbeddedUpdate::Event { selector, lifetime, payload } => { + if let Some(entry) = entries + .get_mut(&selector) + .filter(|entry| entry.lifetime == lifetime) + { + apply_event_to_cached_snapshot(&mut entry.snapshot, &payload); + broadcast(&mut entry.downstreams, BrokerSubscriptionEvent::Event { payload }); + } + } + EmbeddedUpdate::Closed { selector, lifetime, code, message } => { + if !entries + .get(&selector) + .is_some_and(|entry| entry.lifetime == lifetime) + { + continue; + } + let Some(mut entry) = entries.remove(&selector) else { continue; }; + broadcast( + &mut entry.downstreams, + BrokerSubscriptionEvent::Closed { code, message }, + ); + if entry.downstreams.is_empty() { + continue; + } + let new_lifetime = next_lifetime; + next_lifetime = next_lifetime.saturating_add(1); + match start_embedded_subscription( + &runtime, + &runtime_id, + selector.clone(), + new_lifetime, + updates.clone(), + ) { + Ok((snapshot, task)) => { + broadcast( + &mut entry.downstreams, + BrokerSubscriptionEvent::Snapshot { + snapshot: snapshot.clone(), + }, + ); + entry.snapshot = snapshot; + entry.task = task; + entry.lifetime = new_lifetime; + entries.insert(selector, entry); + } + Err(error) => { + broadcast( + &mut entry.downstreams, + BrokerSubscriptionEvent::Rejected { + code: SubscriptionRejectionCode::Internal, + message: error.to_string(), + }, + ); + } + } + } } } } *status.write().expect("broker status poisoned") = RuntimeSubscriptionBrokerStatus { - connection_generation: generation, connected: true, desired_selectors: entries.len(), upstream_subscriptions: entries.len(), @@ -597,10 +729,8 @@ async fn apply_online( .selectors .entry(selector.clone()) .or_insert_with(SelectorState::new); - if let Some((revision, snapshot)) = &entry.snapshot { + if let Some(snapshot) = &entry.snapshot { let _ = events.try_send(BrokerSubscriptionEvent::Snapshot { - connection_generation: state.generation, - snapshot_revision: *revision, snapshot: snapshot.clone(), }); } @@ -696,7 +826,6 @@ async fn handle_frame( request_id, subscription_id, selector, - snapshot_revision, snapshot, }) => { if state.pending.remove(&request_id) != Some(selector.clone()) { @@ -706,18 +835,13 @@ async fn handle_frame( let entry = state.selectors.get_mut(&selector).ok_or(())?; entry.pending = false; entry.upstream_id = Some(subscription_id.clone()); - entry.snapshot = Some((snapshot_revision, snapshot.clone())); - entry.revisions = snapshot_revisions(&snapshot); + entry.snapshot = Some(snapshot.clone()); state .upstream_index .insert(subscription_id, selector.clone()); broadcast( &mut entry.downstreams, - BrokerSubscriptionEvent::Snapshot { - connection_generation: state.generation, - snapshot_revision, - snapshot, - }, + BrokerSubscriptionEvent::Snapshot { snapshot }, ); if entry.downstreams.is_empty() { maybe_unsubscribe(socket, state, selector).await?; @@ -734,18 +858,13 @@ async fn handle_frame( if let Some(mut entry) = state.selectors.remove(&selector) { broadcast( &mut entry.downstreams, - BrokerSubscriptionEvent::Rejected { - connection_generation: state.generation, - code, - message, - }, + BrokerSubscriptionEvent::Rejected { code, message }, ); } } } SubscriptionFramePayload::Event(SubscriptionEvent::Event { subscription_id, - subject_revision, payload, }) => { let selector = state @@ -756,24 +875,12 @@ async fn handle_frame( payload.validate_for_selector(&selector).map_err(|_| ())?; let payload = project_payload_runtime(payload, &state.runtime_id); let entry = state.selectors.get_mut(&selector).ok_or(())?; - if let Some(subject) = event_subject(&payload) { - let revision = entry.revisions.entry(subject).or_insert(0); - if subject_revision <= *revision { - return Ok(()); - } - *revision = subject_revision; - } - if let Some((snapshot_revision, snapshot)) = entry.snapshot.as_mut() { - *snapshot_revision = snapshot_revision.saturating_add(1); + if let Some(snapshot) = entry.snapshot.as_mut() { apply_event_to_cached_snapshot(snapshot, &payload); } broadcast( &mut entry.downstreams, - BrokerSubscriptionEvent::Event { - connection_generation: state.generation, - subject_revision, - payload, - }, + BrokerSubscriptionEvent::Event { payload }, ); } SubscriptionFramePayload::Event(SubscriptionEvent::SubscriptionClosed { @@ -785,14 +892,9 @@ async fn handle_frame( let should_resubscribe = if let Some(entry) = state.selectors.get_mut(&selector) { entry.upstream_id = None; entry.snapshot = None; - entry.revisions.clear(); broadcast( &mut entry.downstreams, - BrokerSubscriptionEvent::Closed { - connection_generation: state.generation, - code, - message, - }, + BrokerSubscriptionEvent::Closed { code, message }, ); !entry.downstreams.is_empty() } else { @@ -822,10 +924,9 @@ fn close_all(state: &mut State, replacement: u64) { broadcast( &mut entry.downstreams, BrokerSubscriptionEvent::Closed { - connection_generation: state.generation, code: SubscriptionTerminationCode::ServerShutdown, message: format!( - "Runtime connection generation {} was fenced by generation {replacement}", + "Runtime registration {} was replaced by {replacement}", state.generation ), }, @@ -846,51 +947,6 @@ fn broadcast( downstreams.retain(|id, _| !closed.contains(id)); } -fn snapshot_revisions(snapshot: &SubscriptionSnapshot) -> HashMap { - match snapshot { - SubscriptionSnapshot::Workers { workers } => workers - .iter() - .map(|worker| { - ( - format!( - "{}:{}", - worker.runtime_id.as_deref().unwrap_or_default(), - worker.worker_id - ), - worker.subject_revision, - ) - }) - .collect(), - SubscriptionSnapshot::WorkerProtocol { worker_id, .. } => { - HashMap::from([(worker_id.to_string(), 0)]) - } - SubscriptionSnapshot::WorkspaceWorkdirs { .. } => HashMap::new(), - } -} -fn event_subject(payload: &SubscriptionEventPayload) -> Option { - Some(match payload { - SubscriptionEventPayload::WorkerUpserted { worker } => format!( - "{}:{}", - worker.runtime_id.as_deref().unwrap_or_default(), - worker.worker_id - ), - SubscriptionEventPayload::WorkerRemoved { - worker_id, - runtime_id, - } => format!( - "{}:{}", - runtime_id.as_deref().unwrap_or_default(), - worker_id - ), - SubscriptionEventPayload::WorkerProtocol { worker_id, .. } => worker_id.to_string(), - SubscriptionEventPayload::WorkdirUpserted { workdir } => { - workdir.working_directory_id.to_string() - } - SubscriptionEventPayload::WorkdirRemoved { - working_directory_id, - } => working_directory_id.to_string(), - }) -} async fn send_frame(socket: &mut RuntimeSocket, frame: SubscriptionFrame) -> Result<(), ()> { frame.validate().map_err(|_| ())?; socket @@ -979,7 +1035,6 @@ fn runtime_token( } fn update_status(status: &RwLock, state: &State, connected: bool) { *status.write().expect("broker status poisoned") = RuntimeSubscriptionBrokerStatus { - connection_generation: state.generation, connected, desired_selectors: state .selectors diff --git a/crates/workspace-server/src/runtime_subscription_tests.rs b/crates/workspace-server/src/runtime_subscription_tests.rs index 07529ec24..302c0db3e 100644 --- a/crates/workspace-server/src/runtime_subscription_tests.rs +++ b/crates/workspace-server/src/runtime_subscription_tests.rs @@ -264,16 +264,9 @@ async fn replacing_runtime_registration_fences_the_old_generation() { assert!(second_generation > first_generation); assert!(matches!( next_event(&mut old).await, - BrokerSubscriptionEvent::Closed { - connection_generation, - .. - } if connection_generation == first_generation + BrokerSubscriptionEvent::Closed { .. } )); - let status = wait_for_status(&broker, "runtime-test", |status| { - status.connection_generation == second_generation && status.connected - }) - .await; - assert_eq!(status.connection_generation, second_generation); + wait_for_status(&broker, "runtime-test", |status| status.connected).await; server.abort(); } @@ -430,6 +423,75 @@ async fn embedded_runtime_uses_in_process_subscription_source() { server.abort(); } +#[tokio::test] +async fn embedded_downstream_overflow_recovers_through_a_fresh_snapshot() { + let (runtime, _config, server) = fixture().await; + let worker = runtime.list_workers().unwrap().remove(0); + let broker = RuntimeSubscriptionBroker::new("local"); + broker.register_embedded_runtime("embedded-worker-runtime", runtime.clone()); + let mut subscription = broker + .subscribe( + "embedded-worker-runtime", + EventSubscriptionSelector::RuntimeWorkers, + ) + .unwrap(); + assert!(matches!( + next_snapshot(&mut subscription).await, + BrokerSubscriptionEvent::Snapshot { .. } + )); + + let final_command_id = (DOWNSTREAM_QUEUE_CAPACITY as u64) * 3; + for command_id in 1..=final_command_id { + let mut snapshot = worker.worker_state.clone().unwrap(); + snapshot.last_command_id = command_id; + runtime + .observe_worker_event( + &worker.worker_ref, + protocol::Event::WorkerState { snapshot }, + ) + .unwrap(); + } + + let mut saw_recovery_snapshot = false; + let mut converged = false; + tokio::time::timeout(Duration::from_secs(5), async { + while !converged { + match subscription + .recv() + .await + .expect("broker subscription closed") + { + BrokerSubscriptionEvent::Snapshot { + snapshot: SubscriptionSnapshot::Workers { workers }, + } => { + saw_recovery_snapshot = true; + converged = workers.iter().any(|candidate| { + candidate.worker_id.as_str() == worker.worker_id.to_string() + && candidate + .worker_state + .as_ref() + .is_some_and(|state| state.last_command_id == final_command_id) + }); + } + BrokerSubscriptionEvent::Event { + payload: SubscriptionEventPayload::WorkerUpserted { worker: candidate }, + } if saw_recovery_snapshot => { + converged = candidate.worker_id.as_str() == worker.worker_id.to_string() + && candidate + .worker_state + .as_ref() + .is_some_and(|state| state.last_command_id == final_command_id); + } + _ => {} + } + } + }) + .await + .expect("overflow recovery did not converge"); + assert!(saw_recovery_snapshot); + server.abort(); +} + #[tokio::test] async fn strict_runtime_subscription_rejects_private_endpoint_before_websocket_connect() { let config = RemoteRuntimeConfig::new( diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index ba610e53c..a5504d863 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -1501,10 +1501,11 @@ impl WorkerRemovalService { /// and WorkerRemove use the retention commit below; compensation preserves /// its pre-create semantics while sharing the atomic catalog contract. fn commit_catalog_removal(&self, target: &RuntimeWorkerRef) -> crate::Result<()> { - let commit = self - .store - .delete_worker_registry(&self.workspace_id, target)?; - self.worker_projection.publish_commit(commit) + self.worker_projection.publish_ordered(|| { + self.store + .delete_worker_registry(&self.workspace_id, target) + .map(|commit| ((), commit)) + }) } async fn execute_cleanup_removal(&self, candidate: &CleanupWorkerCandidate) -> ApiResult<()> { @@ -1631,18 +1632,17 @@ impl WorkerRemovalService { )); } }; - match self.store.commit_worker_removal( - &self.workspace_id, - &prepared.plan.operation_id, - &prepared.plan.input_fingerprint, - &result, - ) { - Ok(commit) => { - self.worker_projection - .publish_commit(commit.catalog) - .map_err(|error| error.to_string())?; - Ok(worker_remove_success_response(target)) - } + match self.worker_projection.publish_ordered(|| { + self.store + .commit_worker_removal( + &self.workspace_id, + &prepared.plan.operation_id, + &prepared.plan.input_fingerprint, + &result, + ) + .map(|commit| ((), commit.catalog)) + }) { + Ok(()) => Ok(worker_remove_success_response(target)), Err(error) => Ok(worker_retention_error_response(error)), } } @@ -1764,12 +1764,12 @@ impl WorkerRemovalService { .map_err(|_| "Worker removal recovery authority is unavailable".to_string())?; if let Some(prepared) = prepared { if prepared.plan.state == crate::retention::WorkerRemovalPlanState::Succeeded { - let commit = self - .store - .recover_succeeded_worker_removal_catalog(&self.workspace_id, target) - .map_err(|error| error.to_string())?; self.worker_projection - .publish_commit(commit) + .publish_ordered(|| { + self.store + .recover_succeeded_worker_removal_catalog(&self.workspace_id, target) + .map(|commit| ((), commit)) + }) .map_err(|error| error.to_string())?; return Ok(worker_remove_success_response(target)); } @@ -1947,17 +1947,17 @@ impl WorkerRemovalService { )); } }; - match self.store.commit_worker_removal( - &self.workspace_id, - &plan.operation_id, - &plan.input_fingerprint, - &retention_result, - ) { - Ok(commit) => { - self.worker_projection - .publish_commit(commit.catalog) - .map_err(|error| error.to_string())?; - } + match self.worker_projection.publish_ordered(|| { + self.store + .commit_worker_removal( + &self.workspace_id, + &plan.operation_id, + &plan.input_fingerprint, + &retention_result, + ) + .map(|commit| ((), commit.catalog)) + }) { + Ok(()) => {} Err(error) => { let _ = self.store.fail_worker_removal( &self.workspace_id, @@ -3454,17 +3454,19 @@ impl WorkspaceApi { }; let bound_at = Utc::now().to_rfc3339_opts(SecondsFormat::Nanos, true); let runtime_run_id = crate::backend_job::runtime_run_id(&reservation.attempt.attempt_id); - let (commit, newly_bound) = self.store.bind_backend_job_attempt_worker( - &self.config.workspace_id, - &request.job_id, - &reservation.attempt.attempt_id, - &worker.worker, - Some(&runtime_run_id), - &bound_at, - )?; - if newly_bound { - self.worker_projection.publish_commit(commit)?; - } else { + let newly_bound = self.worker_projection.publish_ordered(|| { + self.store + .bind_backend_job_attempt_worker( + &self.config.workspace_id, + &request.job_id, + &reservation.attempt.attempt_id, + &worker.worker, + Some(&runtime_run_id), + &bound_at, + ) + .map(|(commit, newly_bound)| (newly_bound, commit)) + })?; + if !newly_bound { let job = self .store .get_backend_job(&self.config.workspace_id, &request.job_id)? @@ -19435,10 +19437,11 @@ async fn scoped_start_workspace_orchestrator( deleted.diagnostics, )); } - let removal = api - .store - .delete_worker_registry(&api.config.workspace_id, &existing.worker)?; - api.worker_projection.publish_commit(removal)?; + api.worker_projection.publish_ordered(|| { + api.store + .delete_worker_registry(&api.config.workspace_id, &existing.worker) + .map(|commit| ((), commit)) + })?; disposition = "recreated"; } @@ -21768,22 +21771,23 @@ async fn set_worker_retention( } } let retention_state = if pinned { "pinned" } else { "normal" }; - let commit = api.store.update_worker_retention( - &api.config.workspace_id, - &worker_ref, - retention_state, - now_registry_timestamp().as_str(), - )?; - if commit.changes.is_empty() { + let changed = api.worker_projection.publish_ordered(|| { + api.store + .update_worker_retention( + &api.config.workspace_id, + &worker_ref, + retention_state, + now_registry_timestamp().as_str(), + ) + .map(|commit| (!commit.changes.is_empty(), commit)) + })?; + if !changed { return Err(cleanup_api_error( runtime_id.as_str(), "workspace_worker_retention_unknown_worker", "Worker is not known to the Backend registry", )); } - api.worker_projection - .publish_commit(commit) - .map_err(ApiError::from)?; Ok(Json(WorkerRetentionResponse { workspace_id: api.config.workspace_id, runtime_id: worker_ref.runtime_id, @@ -26680,7 +26684,7 @@ fn project_observed_workspace_workers( fn workers_response(api: WorkspaceApi) -> ApiResult { let limit = api.config.max_records.min(200); - let (_, projections) = api + let projections = api .store .worker_registry_projection_snapshot(&api.config.workspace_id, limit)?; let workdir_records = api @@ -27478,9 +27482,12 @@ fn record_worker_summary( created_at: timestamp.clone(), updated_at: timestamp, }; - let catalog_commit = api.store.upsert_worker_registry(&record)?; api.worker_projection - .publish_commit(catalog_commit) + .publish_ordered(|| { + api.store + .upsert_worker_registry(&record) + .map(|commit| ((), commit)) + }) .map_err(ApiError::from)?; if let Ok(worker_id) = protocol::subscription::SubscriptionWorkerId::new(worker.worker.worker_id.clone()) @@ -27496,7 +27503,6 @@ fn record_worker_summary( runtime_id: Some(worker.worker.runtime_id.clone()), resource_key: None, availability: protocol::subscription::SubscriptionWorkerAvailability::Observed, - subject_revision: 0, worker_state: worker.worker_state.clone(), state, has_running_internal_workers: false, @@ -29427,7 +29433,6 @@ mod tests { runtime_id: Some("runtime-a".to_string()), resource_key: Some("W-1".to_string()), availability, - subject_revision: 4, worker_state: Some(protocol::WorkerStateSnapshot::initial()), state: protocol::subscription::SubscriptionWorkerState::Idle, has_running_internal_workers: false, @@ -29438,10 +29443,6 @@ mod tests { workdir_attachments: Vec::new(), }, availability, - connection_generation: 2, - subject_revision: 4, - snapshot_revision: 7, - projection_revision: 9, observed_at: "2026-09-24T00:00:01Z".to_string(), }), job: None, @@ -33018,7 +33019,6 @@ mod tests { runtime_id: Some(EMBEDDED_WORKER_RUNTIME_ID.to_string()), resource_key: None, availability: protocol::subscription::SubscriptionWorkerAvailability::Observed, - subject_revision: 1, worker_state: None, state: protocol::subscription::SubscriptionWorkerState::Paused, has_running_internal_workers: false, @@ -44053,7 +44053,8 @@ mod tests { let summary = api.runtime.worker(&target).unwrap(); sync_worker_observation(&api, &summary).unwrap(); seed_worker_control_grant(&api, &source, &target, "embedded-valid-proof"); - let mut subscriber = api.worker_projection.subscribe(); + let mut subscriber = api.worker_projection.subscribe_ordered(10).unwrap(); + let _ = subscriber.take_snapshot(); let response = WorkerRemovalService::new(&api) .execute_async( @@ -44086,7 +44087,7 @@ mod tests { .unwrap() .is_none() ); - let removal = tokio::time::timeout(std::time::Duration::from_secs(1), async { + tokio::time::timeout(std::time::Duration::from_secs(1), async { loop { let event = subscriber.recv().await.expect("projection event"); if event.changes.iter().any(|change| { @@ -44098,7 +44099,6 @@ mod tests { }) .await .expect("WorkerRemove catalog removal broadcast"); - assert!(removal.revision > 0); } #[tokio::test] @@ -44135,7 +44135,8 @@ mod tests { let worker = spawned.worker.unwrap().worker; let summary = api.runtime.worker(&worker).unwrap(); sync_worker_observation(&api, &summary).unwrap(); - let mut subscriber = api.worker_projection.subscribe(); + let mut subscriber = api.worker_projection.subscribe_ordered(10).unwrap(); + let _ = subscriber.take_snapshot(); let diagnostics = compensate_failed_worker_spawn( &api, &summary, @@ -44151,7 +44152,7 @@ mod tests { .unwrap() .is_none() ); - let removal = tokio::time::timeout(std::time::Duration::from_secs(1), async { + tokio::time::timeout(std::time::Duration::from_secs(1), async { loop { let event = subscriber.recv().await.expect("projection event"); if event.changes.iter().any(|change| { @@ -44163,7 +44164,6 @@ mod tests { }) .await .expect("spawn compensation catalog removal broadcast"); - assert!(removal.revision > 0); sync_worker_observation(&api, &summary).unwrap(); assert!( api.store @@ -46005,7 +46005,8 @@ mod tests { .expect("singleton Worker cleanup candidate") .clone(); assert_eq!(candidate.blocking_reason, None); - let mut subscriber = api.worker_projection.subscribe(); + let mut subscriber = api.worker_projection.subscribe_ordered(10).unwrap(); + let _ = subscriber.take_snapshot(); WorkerRemovalService::new(&api) .execute_cleanup_removal(&candidate) @@ -46129,7 +46130,8 @@ mod tests { .iter() .any(|item| item.worker_id.as_str() == worker_id) ); - let mut subscriber = api.worker_projection.subscribe(); + let mut subscriber = api.worker_projection.subscribe_ordered(10).unwrap(); + let _ = subscriber.take_snapshot(); let plan = build_runtime_cleanup_plan(&api, EMBEDDED_WORKER_RUNTIME_ID) .unwrap_or_else(|error| panic!("cleanup plan: {}", error.error)); let candidate = plan @@ -46173,20 +46175,18 @@ mod tests { event.changes.as_slice(), [crate::store::WorkerCatalogChange::Removed(removed)] if removed == &worker )); - let acting_revision = tokio::time::timeout( + tokio::time::timeout( std::time::Duration::from_secs(1), next_workspace_worker_removal(&mut acting_socket, worker_id.as_str()), ) .await .expect("acting Workspace Worker subscription removal"); - let observing_revision = tokio::time::timeout( + tokio::time::timeout( std::time::Duration::from_secs(1), next_workspace_worker_removal(&mut observing_socket, worker_id.as_str()), ) .await .expect("independent Workspace Worker subscription removal"); - assert_eq!(acting_revision, event.revision); - assert_eq!(observing_revision, event.revision); assert!( api.store .get_worker_registry(&api.config.workspace_id, &worker) @@ -46202,11 +46202,10 @@ mod tests { recovered.plan.state, crate::retention::WorkerRemovalPlanState::Succeeded ); - let (snapshot_revision, snapshot) = api + let snapshot = api .store .worker_registry_projection_snapshot(&api.config.workspace_id, 100) .unwrap(); - assert_eq!(snapshot_revision, event.revision); assert!( snapshot .iter() @@ -46228,8 +46227,8 @@ mod tests { .all(|item| item.worker_id.as_str() != worker_id) ); - // A retry replays the durable fence at the same revision, healing an - // in-process broadcast omitted after the first DB commit. + // A retry replays the durable fence, healing an in-process broadcast + // omitted after the first DB commit. WorkerRemovalService::new(&api) .commit_catalog_removal(&worker) .unwrap(); @@ -46237,7 +46236,6 @@ mod tests { .await .expect("catalog removal replay") .expect("projection replay"); - assert_eq!(replay.revision, event.revision); assert!(matches!( replay.changes.as_slice(), [crate::store::WorkerCatalogChange::Removed(removed)] if removed == &worker @@ -49669,10 +49667,7 @@ mod tests { } } - async fn next_workspace_worker_removal( - socket: &mut TestWebSocket, - expected_worker_id: &str, - ) -> u64 { + async fn next_workspace_worker_removal(socket: &mut TestWebSocket, expected_worker_id: &str) { loop { let Message::Text(text) = socket.next().await.unwrap().unwrap() else { continue; @@ -49681,7 +49676,6 @@ mod tests { serde_json::from_str(text.as_str()).unwrap(); if let protocol::subscription::SubscriptionFramePayload::Event( protocol::subscription::SubscriptionEvent::Event { - subject_revision, payload: protocol::subscription::SubscriptionEventPayload::WorkerRemoved { worker_id, @@ -49692,7 +49686,7 @@ mod tests { ) = frame.payload && worker_id.as_str() == expected_worker_id { - return subject_revision; + return; } } } @@ -49812,7 +49806,7 @@ mod tests { .unwrap(); assert_eq!(spawned.state, InternalWorkerOperationState::Accepted); let worker_id = spawned.worker.unwrap().worker.worker_id; - let (_, projections) = api + let projections = api .store .worker_registry_projection_snapshot(TEST_WORKSPACE_ID, 10) .unwrap(); diff --git a/crates/workspace-server/src/store.rs b/crates/workspace-server/src/store.rs index 88c87cd6c..4dfe1198f 100644 --- a/crates/workspace-server/src/store.rs +++ b/crates/workspace-server/src/store.rs @@ -27,7 +27,7 @@ use crate::workspace_deletion::WorkspaceDeletionStore; use crate::{Error, Result}; const OLDEST_SCHEMA_VERSION: i64 = 50; -const LATEST_SCHEMA_VERSION: i64 = 73; +const LATEST_SCHEMA_VERSION: i64 = 74; const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline"; const WORKSPACE_RUNTIME_BINDINGS_MIGRATION_NAME: &str = "workspace runtime bindings"; const RUNTIME_BINDING_AUDIT_MIGRATION_NAME: &str = "workspace Runtime binding revision and audit"; @@ -68,6 +68,8 @@ const BACKEND_JOB_DISPATCH_QUEUE_MIGRATION_NAME: &str = "durable bounded Backend Job dispatch queue"; const WORKER_SINGLETON_OWNERSHIP_MIGRATION_NAME: &str = "durable Workspace Worker singleton ownership"; +const ORDERED_WORKER_PROJECTION_MIGRATION_NAME: &str = + "ordered Worker projection subscriptions without revision reconciliation"; const TICKET_SCHEMA_VERSION_WITH_TARGETS: i64 = 7; const TICKET_SCHEMA_BASELINE_NAME: &str = "ticket schema baseline"; const WORKER_REGISTRY_PROJECTION_SCHEMA: &str = r#" @@ -230,6 +232,11 @@ const MIGRATIONS: &[Migration] = &[ name: WORKER_SINGLETON_OWNERSHIP_MIGRATION_NAME, apply: migrate_worker_singleton_ownership_v72_to_v73, }, + Migration { + version: 74, + name: ORDERED_WORKER_PROJECTION_MIGRATION_NAME, + apply: migrate_ordered_worker_projection_v73_to_v74, + }, ]; #[derive(Clone, Copy)] @@ -638,17 +645,10 @@ pub struct WorkerRegistryRecord { } /// Runtime-backed observation joined to the durable Worker catalog. -/// -/// The Runtime payload remains intact so protocol projection is lossless. Ordering -/// metadata is persisted separately to reject stale reconnect snapshots and events. #[derive(Debug, Clone, PartialEq, Eq)] pub struct WorkerRegistryObservationRecord { pub worker: SubscriptionWorker, pub availability: SubscriptionWorkerAvailability, - pub connection_generation: u64, - pub subject_revision: u64, - pub snapshot_revision: u64, - pub projection_revision: u64, pub observed_at: String, } @@ -666,14 +666,9 @@ pub enum WorkerCatalogChange { Removed(RuntimeWorkerRef), } -/// A catalog mutation and the exact projection payload committed at its revision. -/// -/// Callers may broadcast this value, but must never re-read catalog rows to infer -/// what happened: a later catalog mutation could otherwise be paired with an -/// older revision. +/// A catalog mutation and the exact projection payload captured in its transaction. #[derive(Debug, Clone, PartialEq, Eq)] pub struct WorkerRegistryProjectionCommit { - pub revision: u64, pub changes: Vec, } @@ -1623,7 +1618,7 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore { &self, workspace_id: &str, limit: usize, - ) -> Result<(u64, Vec)>; + ) -> Result>; fn worker_registry_projection( &self, workspace_id: &str, @@ -1633,8 +1628,6 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore { &self, workspace_id: &str, runtime_id: &str, - connection_generation: u64, - snapshot_revision: u64, workers: &[SubscriptionWorker], observed_at: &str, ) -> Result; @@ -1642,7 +1635,6 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore { &self, workspace_id: &str, runtime_id: &str, - connection_generation: u64, worker: &SubscriptionWorker, observed_at: &str, ) -> Result; @@ -1650,9 +1642,7 @@ pub trait ControlPlaneStore: Send + Sync + WorkspaceDeletionStore { &self, workspace_id: &str, runtime_id: &str, - connection_generation: u64, worker_id: Option<&str>, - subject_revision: Option, observed_at: &str, ) -> Result; fn commit_worker_registry_projection_refresh( @@ -6383,7 +6373,6 @@ impl ControlPlaneStore for SqliteWorkspaceStore { )); } let commit = WorkerRegistryProjectionCommit { - revision: current_worker_projection_revision(&tx, workspace_id)?, changes: Vec::new(), }; tx.commit()?; @@ -6404,10 +6393,9 @@ impl ControlPlaneStore for SqliteWorkspaceStore { )); } let changed_workers = vec![worker.clone()]; - let revision = commit_worker_projection_changes(&tx, workspace_id, &changed_workers)?; let changes = worker_catalog_upsert_changes(&tx, workspace_id, &changed_workers)?; tx.commit()?; - Ok((WorkerRegistryProjectionCommit { revision, changes }, true)) + Ok((WorkerRegistryProjectionCommit { changes }, true)) }) } @@ -6775,7 +6763,6 @@ impl ControlPlaneStore for SqliteWorkspaceStore { )?; if removal_blocks_upsert { return Ok(WorkerRegistryProjectionCommit { - revision: current_worker_projection_revision(&tx, &record.workspace_id)?, changes: Vec::new(), }); } @@ -6833,15 +6820,13 @@ impl ControlPlaneStore for SqliteWorkspaceStore { } else { vec![record.worker.clone()] }; - let revision = - commit_worker_projection_changes(&tx, &record.workspace_id, &changed_workers)?; let changes = worker_catalog_upsert_changes( &tx, &record.workspace_id, changed_workers.as_slice(), )?; tx.commit()?; - Ok(WorkerRegistryProjectionCommit { revision, changes }) + Ok(WorkerRegistryProjectionCommit { changes }) }) } @@ -6930,20 +6915,24 @@ impl ControlPlaneStore for SqliteWorkspaceStore { &self, workspace_id: &str, limit: usize, - ) -> Result<(u64, Vec)> { + ) -> Result> { self.with_conn(|conn| { - let revision = current_worker_projection_revision(conn, workspace_id)?; let mut statement = conn.prepare( "SELECT runtime_id, worker_id FROM worker_registry WHERE workspace_id = ?1 ORDER BY updated_at DESC LIMIT ?2", )?; - let identities = statement.query_map(params![workspace_id, limit as i64], |row| { - Ok(RuntimeWorkerRef { runtime_id: row.get(0)?, worker_id: row.get(1)? }) - })?.collect::, _>>()?; - let records = identities.iter() + let identities = statement + .query_map(params![workspace_id, limit as i64], |row| { + Ok(RuntimeWorkerRef { + runtime_id: row.get(0)?, + worker_id: row.get(1)?, + }) + })? + .collect::, _>>()?; + identities + .iter() .map(|worker| read_worker_registry_projection(conn, workspace_id, worker)) - .collect::>>()? - .into_iter().flatten().collect(); - Ok((revision, records)) + .collect::>>() + .map(|records| records.into_iter().flatten().collect()) }) } @@ -6959,33 +6948,11 @@ impl ControlPlaneStore for SqliteWorkspaceStore { &self, workspace_id: &str, runtime_id: &str, - connection_generation: u64, - snapshot_revision: u64, workers: &[SubscriptionWorker], observed_at: &str, ) -> Result { self.with_conn_mut(|conn| { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; - let (current_generation, current_snapshot) = - worker_projection_cursor(&tx, workspace_id, runtime_id)?; - if connection_generation < current_generation - || (connection_generation == current_generation - && snapshot_revision < current_snapshot) - { - let revision = current_worker_projection_revision(&tx, workspace_id)?; - tx.commit()?; - return Ok(WorkerRegistryProjectionCommit { - revision, - changes: Vec::new(), - }); - } - upsert_worker_projection_cursor( - &tx, - workspace_id, - runtime_id, - connection_generation, - snapshot_revision, - )?; let catalog = worker_registry_identities_for_runtime(&tx, workspace_id, runtime_id)?; let runtime_workers = workers .iter() @@ -7014,32 +6981,20 @@ impl ControlPlaneStore for SqliteWorkspaceStore { &tx, workspace_id, runtime_id, - connection_generation, - snapshot_revision, worker, SubscriptionWorkerAvailability::Observed, observed_at, )? } else { - set_worker_observation_unavailable( - &tx, - workspace_id, - &identity, - connection_generation, - None, - snapshot_revision, - observed_at, - )? + set_worker_observation_unavailable(&tx, workspace_id, &identity, observed_at)? }; if changed { changed_workers.push(identity); } } - let revision = commit_worker_projection_changes(&tx, workspace_id, &changed_workers)?; - let changes = - worker_catalog_upsert_changes(&tx, workspace_id, changed_workers.as_slice())?; + let changes = worker_catalog_upsert_changes(&tx, workspace_id, &changed_workers)?; tx.commit()?; - Ok(WorkerRegistryProjectionCommit { revision, changes }) + Ok(WorkerRegistryProjectionCommit { changes }) }) } @@ -7047,16 +7002,12 @@ impl ControlPlaneStore for SqliteWorkspaceStore { &self, workspace_id: &str, runtime_id: &str, - connection_generation: u64, worker: &SubscriptionWorker, observed_at: &str, ) -> Result { self.with_conn_mut(|conn| { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; - let identity = RuntimeWorkerRef { - runtime_id: runtime_id.to_string(), - worker_id: worker.worker_id.to_string(), - }; + let identity = RuntimeWorkerRef::new(runtime_id, worker.worker_id.to_string()); if !worker_registry_identity_exists(&tx, workspace_id, &identity)? { insert_worker_projection_orphan_diagnostic( &tx, @@ -7066,10 +7017,8 @@ impl ControlPlaneStore for SqliteWorkspaceStore { "runtime_worker_without_backend_catalog", observed_at, )?; - let revision = current_worker_projection_revision(&tx, workspace_id)?; tx.commit()?; return Ok(WorkerRegistryProjectionCommit { - revision, changes: Vec::new(), }); } @@ -7077,18 +7026,14 @@ impl ControlPlaneStore for SqliteWorkspaceStore { &tx, workspace_id, runtime_id, - connection_generation, - 0, worker, SubscriptionWorkerAvailability::Observed, observed_at, )?; let changed_workers = if changed { vec![identity] } else { Vec::new() }; - let revision = commit_worker_projection_changes(&tx, workspace_id, &changed_workers)?; - let changes = - worker_catalog_upsert_changes(&tx, workspace_id, changed_workers.as_slice())?; + let changes = worker_catalog_upsert_changes(&tx, workspace_id, &changed_workers)?; tx.commit()?; - Ok(WorkerRegistryProjectionCommit { revision, changes }) + Ok(WorkerRegistryProjectionCommit { changes }) }) } @@ -7096,18 +7041,13 @@ impl ControlPlaneStore for SqliteWorkspaceStore { &self, workspace_id: &str, runtime_id: &str, - connection_generation: u64, worker_id: Option<&str>, - subject_revision: Option, observed_at: &str, ) -> Result { self.with_conn_mut(|conn| { let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; let identities = if let Some(worker_id) = worker_id { - vec![RuntimeWorkerRef { - runtime_id: runtime_id.to_string(), - worker_id: worker_id.to_string(), - }] + vec![RuntimeWorkerRef::new(runtime_id, worker_id)] } else { worker_registry_identities_for_runtime(&tx, workspace_id, runtime_id)? }; @@ -7124,23 +7064,13 @@ impl ControlPlaneStore for SqliteWorkspaceStore { )?; continue; } - if set_worker_observation_unavailable( - &tx, - workspace_id, - &identity, - connection_generation, - subject_revision, - 0, - observed_at, - )? { + if set_worker_observation_unavailable(&tx, workspace_id, &identity, observed_at)? { changed_workers.push(identity); } } - let revision = commit_worker_projection_changes(&tx, workspace_id, &changed_workers)?; - let changes = - worker_catalog_upsert_changes(&tx, workspace_id, changed_workers.as_slice())?; + let changes = worker_catalog_upsert_changes(&tx, workspace_id, &changed_workers)?; tx.commit()?; - Ok(WorkerRegistryProjectionCommit { revision, changes }) + Ok(WorkerRegistryProjectionCommit { changes }) }) } @@ -7156,11 +7086,10 @@ impl ControlPlaneStore for SqliteWorkspaceStore { } else { Vec::new() }; - let revision = commit_worker_projection_changes(&tx, workspace_id, &changed_workers)?; let changes = worker_catalog_upsert_changes(&tx, workspace_id, changed_workers.as_slice())?; tx.commit()?; - Ok(WorkerRegistryProjectionCommit { revision, changes }) + Ok(WorkerRegistryProjectionCommit { changes }) }) } @@ -7196,11 +7125,10 @@ impl ControlPlaneStore for SqliteWorkspaceStore { } else { vec![worker.clone()] }; - let revision = commit_worker_projection_changes(&tx, workspace_id, &changed_workers)?; let changes = worker_catalog_upsert_changes(&tx, workspace_id, changed_workers.as_slice())?; tx.commit()?; - Ok(WorkerRegistryProjectionCommit { revision, changes }) + Ok(WorkerRegistryProjectionCommit { changes }) }) } @@ -7235,21 +7163,20 @@ impl ControlPlaneStore for SqliteWorkspaceStore { ], )?; commit_worker_catalog_removal(&tx, workspace_id, worker)? - } else if let Some(revision) = tx + } else if tx .query_row( - "SELECT projection_revision FROM worker_registry_projection_removals WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3", + "SELECT 1 FROM worker_registry_projection_removals WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3", params![workspace_id, worker.runtime_id, worker.worker_id], - |row| row.get::<_, i64>(0), + |_| Ok(()), ) .optional()? + .is_some() { WorkerRegistryProjectionCommit { - revision: revision.max(0) as u64, changes: vec![WorkerCatalogChange::Removed(worker.clone())], } } else { WorkerRegistryProjectionCommit { - revision: current_worker_projection_revision(&tx, workspace_id)?, changes: Vec::new(), } }; @@ -10539,72 +10466,57 @@ fn read_worker_registry_projection( .optional()?; let observation = conn .query_row( - "SELECT availability, worker_json, connection_generation, subject_revision, snapshot_revision, projection_revision, observed_at FROM worker_registry_observations WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3", + "SELECT availability, worker_json, observed_at FROM worker_registry_observations WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3", params![workspace_id, worker.runtime_id, worker.worker_id], |row| { - let availability: String = row.get(0)?; - let worker_json: Option = row.get(1)?; - Ok((availability, worker_json, row.get::<_, i64>(2)?, row.get::<_, i64>(3)?, row.get::<_, i64>(4)?, row.get::<_, i64>(5)?, row.get::<_, String>(6)?)) + Ok(( + row.get::<_, String>(0)?, + row.get::<_, Option>(1)?, + row.get::<_, String>(2)?, + )) }, ) .optional()?; let observation = observation - .map( - |( - availability, - worker_json, - generation, - subject_revision, - snapshot_revision, - projection_revision, - observed_at, - )| - -> Result<_> { - let availability = match availability.as_str() { - "observed" => SubscriptionWorkerAvailability::Observed, - "unavailable" => SubscriptionWorkerAvailability::Unavailable, - other => { - return Err(Error::InvalidInput(format!( - "invalid Worker observation availability `{other}`" - ))); - } - }; - let mut observed_worker: SubscriptionWorker = if let Some(worker_json) = worker_json - { - serde_json::from_str(&worker_json).map_err(|error| { - Error::InvalidInput(format!("invalid stored Worker observation: {error}")) - })? - } else { - SubscriptionWorker { - worker_id: SubscriptionWorkerId::new(worker.worker_id.to_string()) - .map_err(|error| Error::InvalidInput(error.to_string()))?, - runtime_id: Some(worker.runtime_id.to_string()), - resource_key: resource_key.clone(), - availability, - subject_revision: subject_revision.max(0) as u64, - state: protocol::subscription::SubscriptionWorkerState::Stopped, - worker_state: None, - has_running_internal_workers: false, - workspace_id: Some(workspace_id.to_string()), - display_name: Some(registry.display_name.clone()), - profile: registry.profile.clone(), - job: None, - workdir_attachments: Vec::new(), - } - }; - observed_worker.availability = availability; - observed_worker.resource_key = resource_key.clone(); - Ok(WorkerRegistryObservationRecord { - worker: observed_worker, + .map(|(availability, worker_json, observed_at)| -> Result<_> { + let availability = match availability.as_str() { + "observed" => SubscriptionWorkerAvailability::Observed, + "unavailable" => SubscriptionWorkerAvailability::Unavailable, + other => { + return Err(Error::InvalidInput(format!( + "invalid Worker observation availability `{other}`" + ))); + } + }; + let mut observed_worker: SubscriptionWorker = if let Some(worker_json) = worker_json { + serde_json::from_str(&worker_json).map_err(|error| { + Error::InvalidInput(format!("invalid stored Worker observation: {error}")) + })? + } else { + SubscriptionWorker { + worker_id: SubscriptionWorkerId::new(worker.worker_id.to_string()) + .map_err(|error| Error::InvalidInput(error.to_string()))?, + runtime_id: Some(worker.runtime_id.to_string()), + resource_key: resource_key.clone(), availability, - connection_generation: generation.max(0) as u64, - subject_revision: subject_revision.max(0) as u64, - snapshot_revision: snapshot_revision.max(0) as u64, - projection_revision: projection_revision.max(0) as u64, - observed_at, - }) - }, - ) + state: protocol::subscription::SubscriptionWorkerState::Stopped, + worker_state: None, + has_running_internal_workers: false, + workspace_id: Some(workspace_id.to_string()), + display_name: Some(registry.display_name.clone()), + profile: registry.profile.clone(), + job: None, + workdir_attachments: Vec::new(), + } + }; + observed_worker.availability = availability; + observed_worker.resource_key = resource_key.clone(); + Ok(WorkerRegistryObservationRecord { + worker: observed_worker, + availability, + observed_at, + }) + }) .transpose()?; let job = conn .query_row( @@ -10630,44 +10542,6 @@ fn read_worker_registry_projection( })) } -fn current_worker_projection_revision(conn: &Connection, workspace_id: &str) -> Result { - Ok(conn - .query_row( - "SELECT revision FROM worker_registry_projection_revisions WHERE workspace_id = ?1", - params![workspace_id], - |row| row.get::<_, i64>(0), - ) - .optional()? - .unwrap_or(0) - .max(0) as u64) -} - -fn worker_projection_cursor( - conn: &Connection, - workspace_id: &str, - runtime_id: &str, -) -> Result<(u64, u64)> { - Ok(conn.query_row( - "SELECT connection_generation, snapshot_revision FROM worker_registry_projection_cursors WHERE workspace_id = ?1 AND runtime_id = ?2", - params![workspace_id, runtime_id], - |row| Ok((row.get::<_, i64>(0)?, row.get::<_, i64>(1)?)), - ).optional()?.map(|(generation, revision)| (generation.max(0) as u64, revision.max(0) as u64)).unwrap_or((0, 0))) -} - -fn upsert_worker_projection_cursor( - conn: &Connection, - workspace_id: &str, - runtime_id: &str, - generation: u64, - snapshot_revision: u64, -) -> Result<()> { - conn.execute( - "INSERT INTO worker_registry_projection_cursors (workspace_id, runtime_id, connection_generation, snapshot_revision) VALUES (?1, ?2, ?3, ?4) ON CONFLICT(workspace_id, runtime_id) DO UPDATE SET connection_generation = excluded.connection_generation, snapshot_revision = excluded.snapshot_revision", - params![workspace_id, runtime_id, generation as i64, snapshot_revision as i64], - )?; - Ok(()) -} - fn worker_registry_identities_for_runtime( conn: &Connection, workspace_id: &str, @@ -10715,30 +10589,17 @@ fn upsert_worker_observation( conn: &Connection, workspace_id: &str, runtime_id: &str, - connection_generation: u64, - snapshot_revision: u64, worker: &SubscriptionWorker, availability: SubscriptionWorkerAvailability, observed_at: &str, ) -> Result { - let existing = conn.query_row( - "SELECT availability, worker_json, connection_generation, subject_revision FROM worker_registry_observations WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3", - params![workspace_id, runtime_id, worker.worker_id.as_str()], - |row| Ok((row.get::<_, String>(0)?, row.get::<_, Option>(1)?, row.get::<_, i64>(2)?, row.get::<_, i64>(3)?)), - ).optional()?; - if let Some((old_availability, _, generation, revision)) = existing.as_ref() { - let generation = (*generation).max(0) as u64; - let revision = (*revision).max(0) as u64; - let restores_unavailable = old_availability == "unavailable" - && availability == SubscriptionWorkerAvailability::Observed; - if connection_generation < generation - || (connection_generation == generation - && (worker.subject_revision < revision - || (worker.subject_revision == revision && !restores_unavailable))) - { - return Ok(false); - } - } + let existing = conn + .query_row( + "SELECT availability, worker_json FROM worker_registry_observations WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3", + params![workspace_id, runtime_id, worker.worker_id.as_str()], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, Option>(1)?)), + ) + .optional()?; let mut persisted_worker = worker.clone(); persisted_worker.runtime_id = Some(runtime_id.to_string()); persisted_worker.availability = availability; @@ -10751,13 +10612,13 @@ fn upsert_worker_observation( }; let changed = existing .as_ref() - .is_none_or(|(old_availability, old_json, _, _)| { + .is_none_or(|(old_availability, old_json)| { old_availability != availability_text || old_json.as_deref() != Some(worker_json.as_str()) }); conn.execute( - "INSERT INTO worker_registry_observations (workspace_id, runtime_id, worker_id, availability, worker_json, connection_generation, subject_revision, snapshot_revision, projection_revision, observed_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, 0, ?9) ON CONFLICT(workspace_id, runtime_id, worker_id) DO UPDATE SET availability = excluded.availability, worker_json = excluded.worker_json, connection_generation = excluded.connection_generation, subject_revision = excluded.subject_revision, snapshot_revision = MAX(worker_registry_observations.snapshot_revision, excluded.snapshot_revision), observed_at = excluded.observed_at", - params![workspace_id, runtime_id, worker.worker_id.as_str(), availability_text, worker_json, connection_generation as i64, worker.subject_revision as i64, snapshot_revision as i64, observed_at], + "INSERT INTO worker_registry_observations (workspace_id, runtime_id, worker_id, availability, worker_json, observed_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6) ON CONFLICT(workspace_id, runtime_id, worker_id) DO UPDATE SET availability = excluded.availability, worker_json = excluded.worker_json, observed_at = excluded.observed_at", + params![workspace_id, runtime_id, worker.worker_id.as_str(), availability_text, worker_json, observed_at], )?; Ok(changed) } @@ -10766,39 +10627,25 @@ fn set_worker_observation_unavailable( conn: &Connection, workspace_id: &str, worker: &RuntimeWorkerRef, - connection_generation: u64, - subject_revision: Option, - snapshot_revision: u64, observed_at: &str, ) -> Result { - let existing = conn.query_row( - "SELECT availability, connection_generation, subject_revision, worker_json FROM worker_registry_observations WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3", - params![workspace_id, worker.runtime_id, worker.worker_id], - |row| Ok((row.get::<_, String>(0)?, row.get::<_, i64>(1)?, row.get::<_, i64>(2)?, row.get::<_, Option>(3)?)), - ).optional()?; - if let Some((availability, generation, revision, _)) = existing.as_ref() { - let generation = (*generation).max(0) as u64; - let revision = (*revision).max(0) as u64; - if connection_generation < generation - || (connection_generation == generation - && subject_revision.is_some_and(|incoming| incoming <= revision)) - { - return Ok(false); - } - if availability == "unavailable" - && subject_revision.is_none() - && connection_generation == generation - { - return Ok(false); - } + let existing = conn + .query_row( + "SELECT availability, worker_json FROM worker_registry_observations WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3", + params![workspace_id, worker.runtime_id, worker.worker_id], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, Option>(1)?)), + ) + .optional()?; + if existing + .as_ref() + .is_some_and(|(availability, _)| availability == "unavailable") + { + return Ok(false); } - let revision = subject_revision - .or_else(|| existing.as_ref().map(|item| item.2.max(0) as u64)) - .unwrap_or(0); - let worker_json = existing.and_then(|item| item.3); + let worker_json = existing.and_then(|item| item.1); conn.execute( - "INSERT INTO worker_registry_observations (workspace_id, runtime_id, worker_id, availability, worker_json, connection_generation, subject_revision, snapshot_revision, projection_revision, observed_at) VALUES (?1, ?2, ?3, 'unavailable', ?4, ?5, ?6, ?7, 0, ?8) ON CONFLICT(workspace_id, runtime_id, worker_id) DO UPDATE SET availability = 'unavailable', connection_generation = excluded.connection_generation, subject_revision = excluded.subject_revision, snapshot_revision = MAX(worker_registry_observations.snapshot_revision, excluded.snapshot_revision), observed_at = excluded.observed_at", - params![workspace_id, worker.runtime_id, worker.worker_id, worker_json, connection_generation as i64, revision as i64, snapshot_revision as i64, observed_at], + "INSERT INTO worker_registry_observations (workspace_id, runtime_id, worker_id, availability, worker_json, observed_at) VALUES (?1, ?2, ?3, 'unavailable', ?4, ?5) ON CONFLICT(workspace_id, runtime_id, worker_id) DO UPDATE SET availability = 'unavailable', observed_at = excluded.observed_at", + params![workspace_id, worker.runtime_id, worker.worker_id, worker_json, observed_at], )?; Ok(true) } @@ -10832,52 +10679,23 @@ pub(crate) fn commit_worker_catalog_removal( workspace_id: &str, worker: &RuntimeWorkerRef, ) -> Result { - if let Some(revision) = conn + let existing = conn .query_row( - "SELECT projection_revision FROM worker_registry_projection_removals WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3", + "SELECT 1 FROM worker_registry_projection_removals WHERE workspace_id = ?1 AND runtime_id = ?2 AND worker_id = ?3", params![workspace_id, worker.runtime_id, worker.worker_id], - |row| row.get::<_, i64>(0), + |_| Ok(()), ) .optional()? - { - return Ok(WorkerRegistryProjectionCommit { - revision: revision.max(0) as u64, - changes: vec![WorkerCatalogChange::Removed(worker.clone())], - }); - } - let revision = - commit_worker_projection_changes(conn, workspace_id, std::slice::from_ref(worker))?; - conn.execute( - "INSERT INTO worker_registry_projection_removals (workspace_id, runtime_id, worker_id, projection_revision) VALUES (?1, ?2, ?3, ?4)", - params![workspace_id, worker.runtime_id, worker.worker_id, revision as i64], - )?; - Ok(WorkerRegistryProjectionCommit { - revision, - changes: vec![WorkerCatalogChange::Removed(worker.clone())], - }) -} - -fn commit_worker_projection_changes( - conn: &Connection, - workspace_id: &str, - changed_workers: &[RuntimeWorkerRef], -) -> Result { - let current = current_worker_projection_revision(conn, workspace_id)?; - if changed_workers.is_empty() { - return Ok(current); - } - let revision = current.saturating_add(1); - conn.execute( - "INSERT INTO worker_registry_projection_revisions (workspace_id, revision) VALUES (?1, ?2) ON CONFLICT(workspace_id) DO UPDATE SET revision = excluded.revision", - params![workspace_id, revision as i64], - )?; - for worker in changed_workers { + .is_some(); + if !existing { conn.execute( - "UPDATE worker_registry_observations SET projection_revision = ?1 WHERE workspace_id = ?2 AND runtime_id = ?3 AND worker_id = ?4", - params![revision as i64, workspace_id, worker.runtime_id, worker.worker_id], + "INSERT INTO worker_registry_projection_removals (workspace_id, runtime_id, worker_id) VALUES (?1, ?2, ?3)", + params![workspace_id, worker.runtime_id, worker.worker_id], )?; } - Ok(revision) + Ok(WorkerRegistryProjectionCommit { + changes: vec![WorkerCatalogChange::Removed(worker.clone())], + }) } fn read_worker_control_grant_record( @@ -13911,6 +13729,60 @@ fn migrate_worker_singleton_ownership_v72_to_v73(conn: &Connection) -> Result<() Ok(()) } +fn migrate_ordered_worker_projection_v73_to_v74(conn: &Connection) -> Result<()> { + let current = current_schema_version(conn)?; + if current != 73 { + return Err(Error::Store(format!( + "expected schema version 73 before {ORDERED_WORKER_PROJECTION_MIGRATION_NAME} migration, found {current}" + ))); + } + let tx = rusqlite::Transaction::new_unchecked(conn, TransactionBehavior::Exclusive)?; + tx.execute_batch( + r#" + ALTER TABLE worker_registry_observations RENAME TO worker_registry_observations_v73; + CREATE TABLE worker_registry_observations ( + workspace_id TEXT NOT NULL, + runtime_id TEXT NOT NULL, + worker_id TEXT NOT NULL, + availability TEXT NOT NULL CHECK (availability IN ('observed', 'unavailable')), + worker_json TEXT, + observed_at TEXT NOT NULL, + PRIMARY KEY (workspace_id, runtime_id, worker_id), + FOREIGN KEY (workspace_id, worker_id) + REFERENCES worker_registry(workspace_id, worker_id) ON DELETE CASCADE + ); + INSERT INTO worker_registry_observations ( + workspace_id, runtime_id, worker_id, availability, worker_json, observed_at + ) + SELECT workspace_id, runtime_id, worker_id, availability, worker_json, observed_at + FROM worker_registry_observations_v73; + DROP TABLE worker_registry_observations_v73; + + DROP TABLE IF EXISTS worker_registry_projection_cursors; + DROP TABLE IF EXISTS worker_registry_projection_revisions; + + ALTER TABLE worker_registry_projection_removals + RENAME TO worker_registry_projection_removals_v73; + CREATE TABLE worker_registry_projection_removals ( + workspace_id TEXT NOT NULL, + runtime_id TEXT NOT NULL, + worker_id TEXT NOT NULL, + PRIMARY KEY (workspace_id, runtime_id, worker_id) + ); + INSERT INTO worker_registry_projection_removals (workspace_id, runtime_id, worker_id) + SELECT workspace_id, runtime_id, worker_id + FROM worker_registry_projection_removals_v73; + DROP TABLE worker_registry_projection_removals_v73; + "#, + )?; + tx.execute( + "INSERT INTO __yoi_schema_migrations (version, name) VALUES (?1, ?2)", + params![74_i64, ORDERED_WORKER_PROJECTION_MIGRATION_NAME], + )?; + tx.commit()?; + Ok(()) +} + fn create_latest_workspace_schema(conn: &Connection) -> Result<()> { conn.execute_batch("DROP TABLE IF EXISTS typed_ticket_targets;")?; conn.execute_batch(include_str!("latest_schema.sql"))?; @@ -14742,6 +14614,130 @@ fn migrate_workdir_credential_candidate_snapshots_v58_to_v59(conn: &Connection) mod tests { use super::*; + #[test] + fn schema_v74_migrates_historical_worker_projection_rows_without_revision_state() { + let temp = tempfile::tempdir().unwrap(); + let path = temp.path().join("workspace.db"); + drop(SqliteWorkspaceStore::open(&path).unwrap()); + let conn = Connection::open(&path).unwrap(); + configure_sqlite(&conn).unwrap(); + conn.execute_batch( + r#" + INSERT INTO accounts ( + account_id, kind, handle, display_name, created_at, updated_at + ) VALUES ('owner', 'user', 'owner', 'Owner', '1', '1'); + INSERT INTO workspaces ( + workspace_id, owner_account_id, display_name, state, created_at, updated_at + ) VALUES ('workspace-a', 'owner', 'Workspace A', 'active', '1', '1'); + INSERT INTO worker_registry ( + workspace_id, worker_id, runtime_id, display_name, profile, + retention_state, created_at, updated_at + ) VALUES ('workspace-a', 'worker-a', 'runtime-a', 'Worker A', NULL, + 'normal', '1', '1'); + + ALTER TABLE worker_registry_observations + RENAME TO worker_registry_observations_v74; + CREATE TABLE worker_registry_observations ( + workspace_id TEXT NOT NULL, + runtime_id TEXT NOT NULL, + worker_id TEXT NOT NULL, + availability TEXT NOT NULL CHECK (availability IN ('observed', 'unavailable')), + worker_json TEXT, + connection_generation INTEGER NOT NULL, + subject_revision INTEGER NOT NULL, + snapshot_revision INTEGER NOT NULL, + projection_revision INTEGER NOT NULL, + observed_at TEXT NOT NULL, + PRIMARY KEY (workspace_id, runtime_id, worker_id), + FOREIGN KEY (workspace_id, worker_id) + REFERENCES worker_registry(workspace_id, worker_id) ON DELETE CASCADE + ); + INSERT INTO worker_registry_observations VALUES ( + 'workspace-a', 'runtime-a', 'worker-a', 'observed', '{}', + 8, 13, 21, 34, '2' + ); + DROP TABLE worker_registry_observations_v74; + + CREATE TABLE worker_registry_projection_cursors ( + workspace_id TEXT NOT NULL, + runtime_id TEXT NOT NULL, + connection_generation INTEGER NOT NULL, + snapshot_revision INTEGER NOT NULL, + PRIMARY KEY (workspace_id, runtime_id) + ); + INSERT INTO worker_registry_projection_cursors + VALUES ('workspace-a', 'runtime-a', 8, 21); + CREATE TABLE worker_registry_projection_revisions ( + workspace_id TEXT PRIMARY KEY, + revision INTEGER NOT NULL + ); + INSERT INTO worker_registry_projection_revisions VALUES ('workspace-a', 34); + + ALTER TABLE worker_registry_projection_removals + RENAME TO worker_registry_projection_removals_v74; + CREATE TABLE worker_registry_projection_removals ( + workspace_id TEXT NOT NULL, + runtime_id TEXT NOT NULL, + worker_id TEXT NOT NULL, + projection_revision INTEGER NOT NULL, + PRIMARY KEY (workspace_id, runtime_id, worker_id) + ); + INSERT INTO worker_registry_projection_removals + VALUES ('workspace-a', 'runtime-a', 'removed-worker', 35); + DROP TABLE worker_registry_projection_removals_v74; + + UPDATE __yoi_schema_migrations SET version = 73; + "#, + ) + .unwrap(); + + migrate_ordered_worker_projection_v73_to_v74(&conn).unwrap(); + + assert_eq!(current_schema_version(&conn).unwrap(), 74); + assert_eq!( + table_columns(&conn, "worker_registry_observations").unwrap(), + vec![ + "workspace_id", + "runtime_id", + "worker_id", + "availability", + "worker_json", + "observed_at", + ] + ); + assert_eq!( + conn.query_row( + "SELECT availability || ':' || observed_at FROM worker_registry_observations", + [], + |row| row.get::<_, String>(0), + ) + .unwrap(), + "observed:2" + ); + assert!(!table_exists(&conn, "worker_registry_projection_cursors").unwrap()); + assert!(!table_exists(&conn, "worker_registry_projection_revisions").unwrap()); + assert_eq!( + conn.query_row( + "SELECT runtime_id || ':' || worker_id FROM worker_registry_projection_removals", + [], + |row| row.get::<_, String>(0), + ) + .unwrap(), + "runtime-a:removed-worker" + ); + assert_eq!( + conn.query_row("PRAGMA quick_check", [], |row| row.get::<_, String>(0)) + .unwrap(), + "ok" + ); + let foreign_key_failures: i64 = conn + .query_row("SELECT COUNT(*) FROM pragma_foreign_key_check", [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!(foreign_key_failures, 0); + } + fn downgrade_schema_66_ticket_and_workdir_authority(conn: &Connection) -> Result<()> { conn.execute_batch( "DROP TABLE backend_job_deliveries; @@ -16182,6 +16178,10 @@ mod tests { version: 73, name: WORKER_SINGLETON_OWNERSHIP_MIGRATION_NAME.to_string(), }, + WorkspaceSchemaMigrationStep { + version: 74, + name: ORDERED_WORKER_PROJECTION_MIGRATION_NAME.to_string(), + }, ] ); @@ -16259,6 +16259,7 @@ mod tests { ), (72, BACKEND_JOB_DISPATCH_QUEUE_MIGRATION_NAME.to_string()), (73, WORKER_SINGLETON_OWNERSHIP_MIGRATION_NAME.to_string()), + (74, ORDERED_WORKER_PROJECTION_MIGRATION_NAME.to_string()), ] ); assert!(!table_exists(conn, "trusted_runtime_records")?); @@ -16545,7 +16546,7 @@ mod tests { .collect::>(), vec![ 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, - 73 + 73, 74 ] ); SqliteWorkspaceStore::migrate_database(&path).unwrap(); @@ -16554,7 +16555,7 @@ mod tests { current_schema_version(&conn).unwrap(), LATEST_SCHEMA_VERSION ); - assert_eq!(workspace_schema_migration_history(&conn).unwrap().len(), 24); + assert_eq!(workspace_schema_migration_history(&conn).unwrap().len(), 25); assert!(column_exists(&conn, "worker_workdir_links", "alias").unwrap()); assert!(column_exists(&conn, "worker_workdir_links", "capabilities").unwrap()); assert!(!column_exists(&conn, "worker_workdir_links", "role").unwrap()); @@ -19975,7 +19976,6 @@ INSERT INTO worker_registry ( runtime_id: Some("embedded".to_string()), resource_key: None, availability: SubscriptionWorkerAvailability::Observed, - subject_revision: 4, worker_state: None, state: protocol::subscription::SubscriptionWorkerState::Running, has_running_internal_workers: true, @@ -19985,6 +19985,21 @@ INSERT INTO worker_registry ( job: None, workdir_attachments: Vec::new(), }; + let other_catalog = WorkerRegistryRecord { + worker: RuntimeWorkerRef::new("runtime-other", "other"), + display_name: "Other Runtime Worker".to_string(), + ..catalog.clone() + }; + store.upsert_worker_registry(&other_catalog).unwrap(); + let other_observed = SubscriptionWorker { + worker_id: SubscriptionWorkerId::new("other").unwrap(), + runtime_id: Some("runtime-other".to_string()), + display_name: Some("Other Runtime Worker".to_string()), + ..observed.clone() + }; + store + .apply_worker_registry_observation("local-dev", "runtime-other", &other_observed, "2") + .unwrap(); let orphan = SubscriptionWorker { worker_id: SubscriptionWorkerId::new("orphan").unwrap(), ..observed.clone() @@ -19993,8 +20008,6 @@ INSERT INTO worker_registry ( .reconcile_worker_registry_snapshot( "local-dev", "embedded", - 1, - 10, &[observed.clone(), orphan], "2", ) @@ -20003,15 +20016,19 @@ INSERT INTO worker_registry ( first.changes.as_slice(), [WorkerCatalogChange::Upsert(record)] if record.registry.worker == catalog.worker )); - let (_, records) = store + let records = store .worker_registry_projection_snapshot("local-dev", 10) .unwrap(); assert_eq!( records.len(), - 1, + 2, "orphan Runtime rows must not create catalog rows" ); - let observation = records[0].observation.as_ref().unwrap(); + let observation = records + .iter() + .find(|record| record.registry.worker == catalog.worker) + .and_then(|record| record.observation.as_ref()) + .unwrap(); assert_eq!( observation.availability, SubscriptionWorkerAvailability::Observed @@ -20022,7 +20039,7 @@ INSERT INTO worker_registry ( ); let unavailable = store - .reconcile_worker_registry_snapshot("local-dev", "embedded", 1, 11, &[], "3") + .reconcile_worker_registry_snapshot("local-dev", "embedded", &[], "3") .unwrap(); assert!(matches!( unavailable.changes.as_slice(), @@ -20042,21 +20059,25 @@ INSERT INTO worker_registry ( protocol::subscription::SubscriptionWorkerState::Running, "unavailable must retain the last lifecycle observation rather than becoming stopped" ); + assert_eq!( + store + .worker_registry_projection("local-dev", &other_catalog.worker) + .unwrap() + .unwrap() + .observation + .unwrap() + .availability, + SubscriptionWorkerAvailability::Observed, + "reconciling one Runtime must not alter another Runtime's observations" + ); let duplicate = store - .reconcile_worker_registry_snapshot("local-dev", "embedded", 1, 11, &[], "4") + .reconcile_worker_registry_snapshot("local-dev", "embedded", &[], "4") .unwrap(); assert!(duplicate.changes.is_empty()); let reconnected = store - .reconcile_worker_registry_snapshot( - "local-dev", - "embedded", - 1, - 11, - &[observed.clone()], - "5", - ) + .reconcile_worker_registry_snapshot("local-dev", "embedded", &[observed.clone()], "5") .unwrap(); assert!(matches!( reconnected.changes.as_slice(), @@ -20069,10 +20090,10 @@ INSERT INTO worker_registry ( assert_eq!( projection.observation.unwrap().availability, SubscriptionWorkerAvailability::Observed, - "an equal-revision reconnect snapshot must restore availability" + "a reconnect snapshot must restore availability" ); let duplicate_reconnect = store - .reconcile_worker_registry_snapshot("local-dev", "embedded", 1, 11, &[observed], "6") + .reconcile_worker_registry_snapshot("local-dev", "embedded", &[observed], "6") .unwrap(); assert!(duplicate_reconnect.changes.is_empty()); } @@ -20122,7 +20143,6 @@ INSERT INTO worker_registry ( duplicate_removal.changes.as_slice(), [WorkerCatalogChange::Removed(worker)] if worker == &worker_ref )); - assert_eq!(duplicate_removal.revision, first_removal.revision); let stale_upsert = store .upsert_worker_registry(&WorkerRegistryRecord { workspace_id: "local-dev".to_string(), @@ -20139,7 +20159,6 @@ INSERT INTO worker_registry ( }) .unwrap(); assert!(stale_upsert.changes.is_empty()); - assert_eq!(stale_upsert.revision, first_removal.revision); assert!( store .get_worker_registry("local-dev", &worker_ref) @@ -20151,7 +20170,6 @@ INSERT INTO worker_registry ( runtime_id: Some("embedded".to_string()), resource_key: None, availability: SubscriptionWorkerAvailability::Observed, - subject_revision: 99, worker_state: None, state: protocol::subscription::SubscriptionWorkerState::Running, has_running_internal_workers: false, @@ -20162,7 +20180,7 @@ INSERT INTO worker_registry ( workdir_attachments: Vec::new(), }; let commit = store - .apply_worker_registry_observation("local-dev", "embedded", 2, &event, "2") + .apply_worker_registry_observation("local-dev", "embedded", &event, "2") .unwrap(); assert!(commit.changes.is_empty()); assert!( diff --git a/crates/workspace-server/src/worker_projection.rs b/crates/workspace-server/src/worker_projection.rs index 22095f8c0..ed910de04 100644 --- a/crates/workspace-server/src/worker_projection.rs +++ b/crates/workspace-server/src/worker_projection.rs @@ -11,7 +11,10 @@ use tokio::task::JoinHandle; use crate::runtime_subscription::{ BrokerSubscriptionEvent, RuntimeSubscriptionBroker, RuntimeSubscriptionBrokerError, }; -use crate::store::{ControlPlaneStore, WorkerCatalogChange, WorkerRegistryProjectionCommit}; +use crate::store::{ + ControlPlaneStore, WorkerCatalogChange, WorkerRegistryProjectionCommit, + WorkerRegistryProjectionRecord, +}; use worker_runtime::identity::RuntimeWorkerRef; const PROJECTION_EVENT_CAPACITY: usize = 256; @@ -20,21 +23,42 @@ pub type WorkerProjectionChange = WorkerCatalogChange; #[derive(Debug, Clone)] pub struct WorkerProjectionEvent { - pub revision: u64, pub changes: Vec, } +/// An authoritative durable snapshot followed by changes committed after that snapshot. +pub struct WorkerProjectionSubscription { + snapshot: Vec, + receiver: broadcast::Receiver, +} + +impl WorkerProjectionSubscription { + pub fn take_snapshot(&mut self) -> Vec { + std::mem::take(&mut self.snapshot) + } + + pub async fn recv(&mut self) -> Result { + self.receiver.recv().await + } + + #[cfg(test)] + pub fn try_recv(&mut self) -> Result { + self.receiver.try_recv() + } +} + /// Owns the durable Runtime-to-Workspace Worker projection and its independent /// low-frequency publication stream. /// -/// One Runtime subscription is kept alive per registered Runtime even when no -/// browser is connected. Browser subscriptions therefore never become the -/// authority for populating `worker_registry` observations. +/// The ordering gate covers both durable commits and subscription snapshot creation. +/// Every subscription therefore starts with one authoritative snapshot and receives +/// only commits that completed after it, in the same order in which they were stored. pub struct WorkerProjectionService { workspace_id: String, store: Arc, events: broadcast::Sender, - connection_generation_epoch: u64, + ordering: Mutex<()>, + attach_gate: tokio::sync::Mutex<()>, tasks: Mutex>>, } @@ -45,8 +69,8 @@ impl WorkerProjectionService { workspace_id: workspace_id.into(), store, events, - connection_generation_epoch: Utc::now().timestamp_nanos_opt().unwrap_or_default().max(0) - as u64, + ordering: Mutex::new(()), + attach_gate: tokio::sync::Mutex::new(()), tasks: Mutex::new(HashMap::new()), }) } @@ -62,8 +86,38 @@ impl WorkerProjectionService { } } - pub fn subscribe(&self) -> broadcast::Receiver { - self.events.subscribe() + pub fn subscribe_ordered(&self, limit: usize) -> crate::Result { + let _ordering = self + .ordering + .lock() + .expect("Worker projection ordering lock poisoned"); + let receiver = self.events.subscribe(); + let snapshot_limit = limit.saturating_add(1); + let mut snapshot = self + .store + .worker_registry_projection_snapshot(self.workspace_id.as_str(), snapshot_limit)?; + if snapshot.len() > limit { + return Err(crate::Error::Store(format!( + "authoritative Worker projection exceeds the configured snapshot limit of {limit} records" + ))); + } + snapshot.shrink_to_fit(); + Ok(WorkerProjectionSubscription { snapshot, receiver }) + } + + /// Serialize one durable projection mutation with its publication. The closure + /// must return the exact projection payload captured by the same transaction. + pub(crate) fn publish_ordered( + &self, + mutation: impl FnOnce() -> std::result::Result<(T, WorkerRegistryProjectionCommit), E>, + ) -> std::result::Result { + let _ordering = self + .ordering + .lock() + .expect("Worker projection ordering lock poisoned"); + let (result, commit) = mutation()?; + self.publish_commit(commit); + Ok(result) } pub async fn attach_runtime( @@ -71,6 +125,20 @@ impl WorkerProjectionService { broker: &RuntimeSubscriptionBroker, runtime_id: &str, ) -> Result<(), RuntimeSubscriptionBrokerError> { + // Serialize replacement and wait for the old consumer to stop before the new + // lifetime can apply its snapshot. Late notifications can therefore never + // cross a Runtime ownership boundary. + let _attach = self.attach_gate.lock().await; + let previous = self + .tasks + .lock() + .expect("Worker projection task lock poisoned") + .remove(runtime_id); + if let Some(previous) = previous { + previous.abort(); + let _ = previous.await; + } + let mut subscription = broker.subscribe(runtime_id, EventSubscriptionSelector::RuntimeWorkers)?; let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(2); @@ -80,12 +148,14 @@ impl WorkerProjectionService { break; }; let initialized = matches!(event, BrokerSubscriptionEvent::Snapshot { .. }); - if let Err(error) = self.apply_broker_event(runtime_id, event).await { + if let Err(error) = self.apply_broker_event(runtime_id, event) { tracing::warn!( runtime_id, error = %error, - "failed to initialize durable Worker projection" + "failed to initialize durable Worker projection; restarting from a fresh snapshot" ); + subscription.restart()?; + continue; } if initialized { break; @@ -99,26 +169,22 @@ impl WorkerProjectionService { let Some(service) = service.upgrade() else { return; }; - if let Err(error) = service - .apply_broker_event(task_runtime_id.as_str(), event) - .await - { + if let Err(error) = service.apply_broker_event(task_runtime_id.as_str(), event) { tracing::warn!( runtime_id = %task_runtime_id, error = %error, - "failed to update durable Worker projection" + "failed to update durable Worker projection; restarting from a fresh snapshot" ); + if subscription.restart().is_err() { + return; + } } } }); - if let Some(previous) = self - .tasks + self.tasks .lock() .expect("Worker projection task lock poisoned") - .insert(runtime_id, task) - { - previous.abort(); - } + .insert(runtime_id, task); Ok(()) } @@ -128,110 +194,96 @@ impl WorkerProjectionService { worker: &protocol::subscription::SubscriptionWorker, observed_at: &str, ) -> crate::Result<()> { - let commit = self.store.apply_worker_registry_observation( - self.workspace_id.as_str(), - runtime_id, - 0, - worker, - observed_at, - )?; - self.publish_commit(commit) + self.publish_ordered(|| { + self.store + .apply_worker_registry_observation( + self.workspace_id.as_str(), + runtime_id, + worker, + observed_at, + ) + .map(|commit| ((), commit)) + }) } pub fn refresh(&self, worker: &RuntimeWorkerRef) -> crate::Result<()> { - let commit = self - .store - .commit_worker_registry_projection_refresh(self.workspace_id.as_str(), worker)?; - self.publish_commit(commit) + self.publish_ordered(|| { + self.store + .commit_worker_registry_projection_refresh(self.workspace_id.as_str(), worker) + .map(|commit| ((), commit)) + }) } - async fn apply_broker_event( + fn apply_broker_event( &self, runtime_id: &str, event: BrokerSubscriptionEvent, ) -> crate::Result<()> { let observed_at = Utc::now().to_rfc3339(); - let projection_generation = |connection_generation: u64| { - self.connection_generation_epoch - .saturating_add(connection_generation) - }; - let commit = match event { - BrokerSubscriptionEvent::Snapshot { - connection_generation, - snapshot_revision, - snapshot: SubscriptionSnapshot::Workers { workers }, - } => self.store.reconcile_worker_registry_snapshot( - self.workspace_id.as_str(), - runtime_id, - projection_generation(connection_generation), - snapshot_revision, - workers.as_slice(), - observed_at.as_str(), - )?, - BrokerSubscriptionEvent::Snapshot { .. } => return Ok(()), - BrokerSubscriptionEvent::Event { - connection_generation, - payload: SubscriptionEventPayload::WorkerUpserted { worker }, - .. - } => self.store.apply_worker_registry_observation( - self.workspace_id.as_str(), - runtime_id, - projection_generation(connection_generation), - &worker, - observed_at.as_str(), - )?, - BrokerSubscriptionEvent::Event { - connection_generation, - subject_revision, - payload: SubscriptionEventPayload::WorkerRemoved { worker_id, .. }, - } => self.store.mark_worker_registry_observation_unavailable( - self.workspace_id.as_str(), - runtime_id, - projection_generation(connection_generation), - Some(worker_id.as_str()), - Some(subject_revision), - observed_at.as_str(), - )?, - BrokerSubscriptionEvent::Event { .. } => return Ok(()), - BrokerSubscriptionEvent::Disconnected { - connection_generation, - .. - } - | BrokerSubscriptionEvent::Rejected { - connection_generation, - .. - } - | BrokerSubscriptionEvent::Closed { - connection_generation, - .. - } => self.store.mark_worker_registry_observation_unavailable( - self.workspace_id.as_str(), - runtime_id, - projection_generation(connection_generation), - None, - None, - observed_at.as_str(), - )?, - }; - self.publish_commit(commit) + self.publish_ordered(|| { + let commit = match event { + BrokerSubscriptionEvent::Snapshot { + snapshot: SubscriptionSnapshot::Workers { workers }, + } => self.store.reconcile_worker_registry_snapshot( + self.workspace_id.as_str(), + runtime_id, + workers.as_slice(), + observed_at.as_str(), + )?, + BrokerSubscriptionEvent::Snapshot { .. } => { + return Ok(( + (), + WorkerRegistryProjectionCommit { + changes: Vec::new(), + }, + )); + } + BrokerSubscriptionEvent::Event { + payload: SubscriptionEventPayload::WorkerUpserted { worker }, + } => self.store.apply_worker_registry_observation( + self.workspace_id.as_str(), + runtime_id, + &worker, + observed_at.as_str(), + )?, + BrokerSubscriptionEvent::Event { + payload: SubscriptionEventPayload::WorkerRemoved { worker_id, .. }, + } => self.store.mark_worker_registry_observation_unavailable( + self.workspace_id.as_str(), + runtime_id, + Some(worker_id.as_str()), + observed_at.as_str(), + )?, + BrokerSubscriptionEvent::Event { .. } => { + return Ok(( + (), + WorkerRegistryProjectionCommit { + changes: Vec::new(), + }, + )); + } + BrokerSubscriptionEvent::Disconnected { .. } + | BrokerSubscriptionEvent::Rejected { .. } + | BrokerSubscriptionEvent::Closed { .. } => { + self.store.mark_worker_registry_observation_unavailable( + self.workspace_id.as_str(), + runtime_id, + None, + observed_at.as_str(), + )? + } + }; + Ok(((), commit)) + }) } - /// Broadcast an already committed catalog payload. Persistence always happens - /// before this call; a process crash therefore closes subscribers, whose next - /// subscription starts from the durable snapshot. Retrying a removal returns - /// the same durable change and can heal an in-process missed broadcast. - pub(crate) fn publish_commit( - &self, - commit: WorkerRegistryProjectionCommit, - ) -> crate::Result<()> { + fn publish_commit(&self, commit: WorkerRegistryProjectionCommit) { if commit.changes.is_empty() { - return Ok(()); + return; } let _ = self.events.send(WorkerProjectionEvent { - revision: commit.revision, changes: commit.changes, }); - Ok(()) } } @@ -240,3 +292,159 @@ impl Drop for WorkerProjectionService { self.shutdown(); } } + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Barrier, Mutex}; + + use crate::store::{ + AccountRecord, ControlPlaneStore, SqliteWorkspaceStore, WorkerCatalogChange, + WorkerRegistryRecord, WorkspaceRecord, + }; + use worker_runtime::identity::RuntimeWorkerRef; + + use super::WorkerProjectionService; + + const WORKSPACE_ID: &str = "projection-ordering-test"; + + async fn fixture() -> ( + Arc, + Arc, + tempfile::TempDir, + ) { + let temp = tempfile::tempdir().unwrap(); + let store = Arc::new(SqliteWorkspaceStore::open(temp.path().join("workspace.db")).unwrap()); + store + .upsert_account(&AccountRecord { + account_id: "owner".to_string(), + kind: "user".to_string(), + handle: "owner".to_string(), + display_name: "Owner".to_string(), + created_at: "1".to_string(), + updated_at: "1".to_string(), + }) + .unwrap(); + store + .upsert_workspace(&WorkspaceRecord { + workspace_id: WORKSPACE_ID.to_string(), + owner_account_id: "owner".to_string(), + display_name: "Projection ordering".to_string(), + state: "active".to_string(), + created_at: "1".to_string(), + updated_at: "1".to_string(), + }) + .await + .unwrap(); + let service = WorkerProjectionService::new(WORKSPACE_ID, store.clone()); + (store, service, temp) + } + + fn worker(runtime_id: &str, worker_id: &str) -> WorkerRegistryRecord { + WorkerRegistryRecord { + workspace_id: WORKSPACE_ID.to_string(), + worker: RuntimeWorkerRef::new(runtime_id, worker_id), + display_name: format!("{runtime_id}-{worker_id}"), + profile: Some("builtin:coder".to_string()), + retention_state: "normal".to_string(), + transcript_ref: None, + session_ref: None, + summary_ref: None, + diagnostics_ref: None, + created_at: "1".to_string(), + updated_at: "1".to_string(), + } + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn snapshot_registration_is_atomic_with_concurrent_publication() { + let (store, service, _temp) = fixture().await; + let barrier = Arc::new(Barrier::new(2)); + let subscriber_service = service.clone(); + let subscriber_barrier = barrier.clone(); + let subscriber = std::thread::spawn(move || { + subscriber_barrier.wait(); + subscriber_service.subscribe_ordered(10).unwrap() + }); + let publisher_service = service.clone(); + let publisher_barrier = barrier.clone(); + let publisher_store = store.clone(); + let record = worker("runtime-a", "worker-a"); + let publisher = std::thread::spawn(move || { + publisher_barrier.wait(); + publisher_service + .publish_ordered(|| { + publisher_store + .upsert_worker_registry(&record) + .map(|commit| ((), commit)) + }) + .unwrap(); + }); + + let mut subscription = subscriber.join().unwrap(); + publisher.join().unwrap(); + let in_snapshot = subscription + .take_snapshot() + .iter() + .any(|record| record.registry.worker.worker_id == "worker-a"); + let in_event = match subscription.try_recv() { + Ok(event) => matches!( + event.changes.as_slice(), + [WorkerCatalogChange::Upsert(record)] + if record.registry.worker.worker_id == "worker-a" + ), + Err(tokio::sync::broadcast::error::TryRecvError::Empty) => false, + Err(error) => panic!("unexpected projection receive error: {error}"), + }; + assert_ne!( + in_snapshot, in_event, + "the commit must cross the boundary exactly once" + ); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] + async fn concurrent_commits_are_published_in_durable_execution_order() { + let (store, service, _temp) = fixture().await; + let mut subscription = service.subscribe_ordered(10).unwrap(); + assert!(subscription.take_snapshot().is_empty()); + let barrier = Arc::new(Barrier::new(3)); + let durable_order = Arc::new(Mutex::new(Vec::new())); + let mut publishers = Vec::new(); + for worker_id in ["worker-a", "worker-b"] { + let service = service.clone(); + let store = store.clone(); + let barrier = barrier.clone(); + let durable_order = durable_order.clone(); + let record = worker("runtime-a", worker_id); + publishers.push(std::thread::spawn(move || { + barrier.wait(); + service + .publish_ordered(|| { + durable_order + .lock() + .unwrap() + .push(record.worker.worker_id.clone()); + store + .upsert_worker_registry(&record) + .map(|commit| ((), commit)) + }) + .unwrap(); + })); + } + barrier.wait(); + for publisher in publishers { + publisher.join().unwrap(); + } + + let published_order = [ + subscription.try_recv().unwrap(), + subscription.try_recv().unwrap(), + ] + .into_iter() + .map(|event| match event.changes.as_slice() { + [WorkerCatalogChange::Upsert(record)] => record.registry.worker.worker_id.clone(), + changes => panic!("unexpected projection changes: {changes:?}"), + }) + .collect::>(); + assert_eq!(published_order, *durable_order.lock().unwrap()); + } +} diff --git a/crates/workspace-server/src/workspace_deletion.rs b/crates/workspace-server/src/workspace_deletion.rs index fb5d1a6a0..cfe89abdd 100644 --- a/crates/workspace-server/src/workspace_deletion.rs +++ b/crates/workspace-server/src/workspace_deletion.rs @@ -73,10 +73,8 @@ const WORKSPACE_DELETION_PURGE_TABLES: &[&str] = &[ "worker_orphan_diagnostics", "worker_registry", "worker_registry_observations", - "worker_registry_projection_cursors", "worker_registry_projection_diagnostics", "worker_registry_projection_removals", - "worker_registry_projection_revisions", "worker_removal_operations", "worker_retention_audit_events", "worker_session_archives", diff --git a/crates/workspace-server/src/workspace_subscription.rs b/crates/workspace-server/src/workspace_subscription.rs index afd4fbfbe..54f3182ae 100644 --- a/crates/workspace-server/src/workspace_subscription.rs +++ b/crates/workspace-server/src/workspace_subscription.rs @@ -271,7 +271,6 @@ async fn run_worker_protocol( worker_id: worker_id.clone(), runtime_id: Some(runtime_id), }, - snapshot_revision: 0, snapshot: SubscriptionSnapshot::WorkerProtocol { worker_id: worker_id.clone(), events: vec![initial_event], @@ -284,13 +283,10 @@ async fn run_worker_protocol( { return; } - let mut subject_revision = 0_u64; while let Some(event) = events.recv().await { - subject_revision = subject_revision.saturating_add(1); let frame = SubscriptionFrame::new(SubscriptionFramePayload::Event(SubscriptionEvent::Event { subscription_id: subscription_id.clone(), - subject_revision, payload: SubscriptionEventPayload::WorkerProtocol { worker_id: worker_id.clone(), event, @@ -333,12 +329,10 @@ async fn run_workspace_workers( subscription_id: SubscriptionId, outbound: mpsc::Sender, ) { - // Subscribe before reading so commits racing with the snapshot are replayed. - let mut events = api.worker_projection.subscribe(); - let Ok((snapshot_revision, records)) = api.store.worker_registry_projection_snapshot( - &api.config.workspace_id, - api.config.max_records.max(1), - ) else { + let Ok(mut projection) = api + .worker_projection + .subscribe_ordered(api.config.max_records.max(1)) + else { let _ = send_rejected( &outbound, request_id, @@ -348,7 +342,8 @@ async fn run_workspace_workers( .await; return; }; - let mut workers = records + let mut workers = projection + .take_snapshot() .into_iter() .filter_map(|record| project_registry_worker(&api, record)) .collect::>(); @@ -360,7 +355,6 @@ async fn run_workspace_workers( request_id, subscription_id: subscription_id.clone(), selector: EventSubscriptionSelector::WorkspaceWorkers, - snapshot_revision, snapshot: SubscriptionSnapshot::Workers { workers }, }, )), @@ -372,16 +366,14 @@ async fn run_workspace_workers( } loop { - match events.recv().await { - Ok(event) if event.revision <= snapshot_revision => continue, + match projection.recv().await { Ok(event) => { for change in event.changes { let payload = match change { crate::worker_projection::WorkerProjectionChange::Upsert(record) => { - let Some(mut worker) = project_registry_worker(&api, record) else { + let Some(worker) = project_registry_worker(&api, record) else { continue; }; - worker.subject_revision = event.revision; SubscriptionEventPayload::WorkerUpserted { worker } } crate::worker_projection::WorkerProjectionChange::Removed(worker) => { @@ -396,7 +388,7 @@ async fn run_workspace_workers( } } }; - if send_event(&outbound, &subscription_id, event.revision, payload) + if send_event(&outbound, &subscription_id, payload) .await .is_err() { @@ -431,7 +423,6 @@ fn project_registry_worker( let mut worker = if let Some(observation) = record.observation { let mut worker = observation.worker; worker.availability = observation.availability; - worker.subject_revision = observation.projection_revision; worker } else { SubscriptionWorker { @@ -442,7 +433,6 @@ fn project_registry_worker( runtime_id: Some(runtime_id.clone()), resource_key: record.resource_key.clone(), availability: protocol::subscription::SubscriptionWorkerAvailability::Unavailable, - subject_revision: 0, worker_state: None, state: protocol::subscription::SubscriptionWorkerState::Stopped, has_running_internal_workers: false, @@ -559,14 +549,12 @@ fn sort_workers(workers: &mut [SubscriptionWorker]) { async fn send_event( outbound: &mpsc::Sender, subscription_id: &SubscriptionId, - subject_revision: u64, payload: SubscriptionEventPayload, ) -> Result<(), ()> { send_frame( outbound, SubscriptionFrame::new(SubscriptionFramePayload::Event(SubscriptionEvent::Event { subscription_id: subscription_id.clone(), - subject_revision, payload, })), ) @@ -617,7 +605,6 @@ mod tests { runtime_id: Some("runtime-1".to_string()), resource_key: Some("W-1".to_string()), availability: protocol::subscription::SubscriptionWorkerAvailability::Observed, - subject_revision: 1, worker_state: None, state: protocol::subscription::SubscriptionWorkerState::Idle, has_running_internal_workers: false, diff --git a/tools/web-ux/browser-tests/console_history_fixture_server.ts b/tools/web-ux/browser-tests/console_history_fixture_server.ts index 627def943..f4c79d17a 100644 --- a/tools/web-ux/browser-tests/console_history_fixture_server.ts +++ b/tools/web-ux/browser-tests/console_history_fixture_server.ts @@ -227,7 +227,7 @@ Deno.serve({ hostname: "127.0.0.1", port }, async (request) => { if (frame?.message?.method !== "subscribe_events") return; const requestId = frame.message.params.request_id; socket.send(JSON.stringify({ - protocol_version: 1, + protocol_version: 2, frame: "response", message: { result: "subscribed", @@ -235,7 +235,6 @@ Deno.serve({ hostname: "127.0.0.1", port }, async (request) => { request_id: requestId, subscription_id: "fixture-workers", selector: frame.message.params.selector, - snapshot_revision: 1, snapshot: { topic: "workers", data: { workers: [] } }, }, }, diff --git a/tools/web-ux/browser-tests/memory_document_fixture_server.ts b/tools/web-ux/browser-tests/memory_document_fixture_server.ts index ba1c24614..89fa3d098 100644 --- a/tools/web-ux/browser-tests/memory_document_fixture_server.ts +++ b/tools/web-ux/browser-tests/memory_document_fixture_server.ts @@ -129,7 +129,7 @@ Deno.serve({ hostname: "127.0.0.1", port }, async (request) => { if (frame?.message?.method !== "subscribe_events") return; const requestId = frame.message.params.request_id; socket.send(JSON.stringify({ - protocol_version: 1, + protocol_version: 2, frame: "response", message: { result: "subscribed", @@ -137,7 +137,6 @@ Deno.serve({ hostname: "127.0.0.1", port }, async (request) => { request_id: requestId, subscription_id: "fixture-workers", selector: { topic: "workspace_workers" }, - snapshot_revision: 1, snapshot: { topic: "workers", data: { workers: [] } }, }, }, diff --git a/tools/web-ux/browser-tests/workspace_home_fixture_server.ts b/tools/web-ux/browser-tests/workspace_home_fixture_server.ts index b1bf91a9b..27a3d3667 100644 --- a/tools/web-ux/browser-tests/workspace_home_fixture_server.ts +++ b/tools/web-ux/browser-tests/workspace_home_fixture_server.ts @@ -138,7 +138,7 @@ Deno.serve({ hostname: "127.0.0.1", port }, async (request) => { const frame = JSON.parse(String(event.data)); if (frame?.message?.method !== "subscribe_events") return; socket.send(JSON.stringify({ - protocol_version: 1, + protocol_version: 2, frame: "response", message: { result: "subscribed", @@ -146,7 +146,6 @@ Deno.serve({ hostname: "127.0.0.1", port }, async (request) => { request_id: frame.message.params.request_id, subscription_id: "fixture-workers", selector: { topic: "workspace_workers" }, - snapshot_revision: 1, snapshot: { topic: "workers", data: { workers: [] } }, }, }, diff --git a/web/workspace/src/lib/generated/protocol-validator.ts b/web/workspace/src/lib/generated/protocol-validator.ts index e36d8205b..e2ab28acc 100644 --- a/web/workspace/src/lib/generated/protocol-validator.ts +++ b/web/workspace/src/lib/generated/protocol-validator.ts @@ -10,7 +10,7 @@ import type { WorkspaceSubscriptionWorkdir, } from './protocol'; -export const SUBSCRIPTION_PROTOCOL_VERSION = 1; +export const SUBSCRIPTION_PROTOCOL_VERSION = 2; export const MAX_SUBSCRIPTION_FRAME_JSON_BYTES = 16777216; export const MAX_SUBSCRIPTION_STRING_BYTES = 4194304; export const MAX_SUBSCRIPTION_COLLECTION_ITEMS = 100000; @@ -21,9 +21,9 @@ const MAX_CORRELATION_ID_BYTES = 128; const MAX_RESOURCE_ID_BYTES = 256; const MAX_WORKER_IDS_PER_SELECTOR = 256; const MAX_REJECTION_MESSAGE_BYTES = 1024; -const subscriptionFrameSchema: unknown = {"$defs":{"Alert":{"additionalProperties":false,"description":"User-facing alert emitted from the Worker layer.\n\nThis is a separate channel from `tracing` (developer logs): entries\nhere are assembled explicitly by the Worker when a condition should be\nsurfaced to the person driving the client. Keep messages short and\nhuman-readable.","properties":{"level":{"$ref":"#/$defs/AlertLevel"},"message":{"type":"string"},"source":{"$ref":"#/$defs/AlertSource"},"timestamp_ms":{"description":"Milliseconds since the Unix epoch.","format":"int64","maximum":9007199254740991.0,"minimum":-9007199254740991.0,"type":"integer"}},"required":["level","source","message","timestamp_ms"],"type":"object"},"AlertLevel":{"enum":["warn","error"],"type":"string"},"AlertSource":{"enum":["worker","engine","compactor","agents_md"],"type":"string"},"CommandEvent":{"oneOf":[{"additionalProperties":false,"properties":{"command_id":{"type":"string"},"kind":{"const":"started","type":"string"},"observed_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"tool_call_id":{"type":["string","null"]}},"required":["kind","command_id","observed_at_ms"],"type":"object"},{"additionalProperties":false,"properties":{"command_id":{"type":"string"},"content":{"type":"string"},"end_offset":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"kind":{"const":"output","type":"string"},"observed_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"start_offset":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"stream":{"$ref":"#/$defs/CommandStream"}},"required":["kind","command_id","stream","start_offset","end_offset","content","observed_at_ms"],"type":"object"},{"additionalProperties":false,"properties":{"command_id":{"type":"string"},"exit_code":{"format":"int32","maximum":2147483647.0,"minimum":-2147483648.0,"type":["integer","null"]},"kind":{"const":"terminal","type":"string"},"observed_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"status":{"$ref":"#/$defs/CommandStatus"},"stderr_end_offset":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"stdout_end_offset":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","command_id","status","stdout_end_offset","stderr_end_offset","observed_at_ms"],"type":"object"}]},"CommandSnapshot":{"additionalProperties":false,"properties":{"command_id":{"type":"string"},"exit_code":{"format":"int32","maximum":2147483647.0,"minimum":-2147483648.0,"type":["integer","null"]},"last_output_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":["integer","null"]},"observed_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"started_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"status":{"$ref":"#/$defs/CommandStatus"},"stderr":{"$ref":"#/$defs/CommandStreamSlice"},"stdout":{"$ref":"#/$defs/CommandStreamSlice"},"tool_call_id":{"type":["string","null"]}},"required":["command_id","status","started_at_ms","observed_at_ms","stdout","stderr"],"type":"object"},"CommandStatus":{"description":"Live provider-owned command status. These values are operational Console\nstate only and are never appended to Worker history.","enum":["running","completed","failed","timed_out","cancelled"],"type":"string"},"CommandStream":{"enum":["stdout","stderr"],"type":"string"},"CommandStreamSlice":{"additionalProperties":false,"properties":{"content":{"type":"string"},"end_offset":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"start_offset":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"truncated":{"type":"boolean"}},"required":["start_offset","end_offset","content","truncated"],"type":"object"},"CompactionLifecycle":{"additionalProperties":false,"description":"Stable parent-owned lifecycle for one compaction run.","properties":{"compaction_id":{"type":"string"},"ended_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":["integer","null"]},"error":{"type":["string","null"]},"internal_worker":{"anyOf":[{"$ref":"#/$defs/InternalWorkerRef"},{"type":"null"}]},"new_segment_id":{"type":["string","null"]},"revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"schema_version":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"started_at_ms":{"description":"Milliseconds since the Unix epoch.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"state":{"$ref":"#/$defs/CompactionLifecycleState"},"summary":{"type":["string","null"]}},"required":["schema_version","compaction_id","revision","state","started_at_ms"],"type":"object"},"CompactionLifecycleState":{"enum":["running","done","failed","interrupted"],"type":"string"},"CompactionPhase":{"enum":["preparing","summarizing","committing"],"type":"string"},"CompactionTrigger":{"enum":["manual","pre_run","request_threshold"],"type":"string"},"CompletionEntry":{"additionalProperties":false,"description":"One completion candidate for a prefix query.\n\n`value` is a path (file kind).","properties":{"is_dir":{"default":false,"type":"boolean"},"value":{"type":"string"}},"required":["value","is_dir"],"type":"object"},"CompletionKind":{"description":"Kind of completion requested by `Method::ListCompletions`.\n\nMirrors the completion prefix sigil: `@` → `File`.","enum":["file"],"type":"string"},"ErrorCode":{"enum":["already_running","not_running","not_paused","provider_error","tool_error","invalid_request","internal"],"type":"string"},"Event":{"oneOf":[{"additionalProperties":false,"description":"Durable Submit acceptance. A `Queued` receipt follows the durable FIFO\ncheckpoint. A `Started` receipt follows the pending activation checkpoint;\nthe corresponding [`Event::UserMessage`] marks the later history commit.\nRepeating the same request id and exact payload returns the same receipt\nwithout appending or activating twice.","properties":{"data":{"additionalProperties":false,"properties":{"disposition":{"$ref":"#/$defs/SubmissionDisposition"},"submission_id":{"type":"string"},"submission_request_id":{"type":"string"}},"required":["submission_request_id","submission_id","disposition"],"type":"object"},"event":{"const":"submission_accepted","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Correlated rejection before durable acceptance.","properties":{"data":{"additionalProperties":false,"properties":{"message":{"type":"string"},"submission_request_id":{"type":"string"}},"required":["submission_request_id","message"],"type":"object"},"event":{"const":"submission_rejected","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Durable notification acceptance. The request identity is persisted with\nthe notification receipt before this event is emitted. Repeating the\nsame request id and exact payload returns the same acknowledgement\nwithout appending the notification twice.","properties":{"data":{"additionalProperties":false,"properties":{"notification_request_id":{"type":"string"}},"required":["notification_request_id"],"type":"object"},"event":{"const":"notification_accepted","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Correlated notification rejection before durable acceptance.","properties":{"data":{"additionalProperties":false,"properties":{"message":{"type":"string"},"notification_request_id":{"type":"string"}},"required":["notification_request_id","message"],"type":"object"},"event":{"const":"notification_rejected","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Revisioned FIFO replacement following enqueue, activation, cancel, or clear.","properties":{"data":{"additionalProperties":false,"properties":{"pending":{"$ref":"#/$defs/PendingSubmissionsSnapshot"}},"required":["pending"],"type":"object"},"event":{"const":"pending_submissions_changed","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A user input message persisted as `LogEntry::AnnotatedUserInput` and\nactivated for a turn. Broadcast to every subscribed client so TUI / GUI\ninstances show the same user line that reconnect snapshots replay.","properties":{"data":{"additionalProperties":false,"properties":{"entry_id":{"description":"Stable durable history identity. Older mixed-version producers omit it.","type":["string","null"]},"segments":{"items":{"$ref":"#/$defs/Segment"},"type":"array"}},"required":["segments"],"type":"object"},"event":{"const":"user_message","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A canonical assistant/tool entry has committed to durable history.\nStreaming events remain the in-flight authority; this event supplies the\nstable identity used to reconcile that live block with paged history.","properties":{"data":{"additionalProperties":false,"properties":{"entry":{"$ref":"#/$defs/SessionSnapshotEntry"}},"required":["entry"],"type":"object"},"event":{"const":"session_entry_committed","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"One agent-injected system item committed to history.\n\nCarries the JSON form of `session_store::SystemItem`. Covers\n`Method::Notify` echoes, child-Worker lifecycle events from\n`Method::WorkerEvent`, `@` resolution payloads, and any future\nClients dispatch on the `kind` tag for typed rendering instead\nof parsing free-text prefixes like `[Notification] …` or\n`[File: …]`.\n\nOne event per `LogEntry::AnnotatedSystemItem` commit. Disk-side and\nwire-side are 1:1.","properties":{"data":{"additionalProperties":false,"properties":{"item":true},"required":["item"],"type":"object"},"event":{"const":"system_item","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A new self-driving cycle has begun (IDLE → active transition).\n\nMarker event for the start of an Invoke range; the range extends\nimplicitly until the next `InvokeStart`. Fires for every accepted\n`Method::Submit` (kind=`UserSend`), `Method::Notify` (kind=`Notify`),\n`Method::WorkerEvent` re-injection (kind=`WorkerEvent`), and any other\nIDLE-breaking trigger. Mid-run interrupts (e.g. hook output,\ntyped system reminder insertion that doesn't break IDLE) do not\nemit `InvokeStart` — they appear as `SystemItem` only.\n\nCarries `kind` only; the payload (user text / notify message /\nworker event body) is delivered separately via the immediately\nfollowing `UserMessage` / `SystemItem` event.","properties":{"data":{"additionalProperties":false,"properties":{"kind":{"$ref":"#/$defs/InvokeKind"}},"required":["kind"],"type":"object"},"event":{"const":"invoke_start","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"One AgentTurn boundary opened. An AgentTurn is a maximal run of\nLLM generation calls whose input messages are identical (i.e.\nretries from network errors / 5xx / stream disconnects collapse\ninto the same AgentTurn). When the input changes (a new tool\nresult lands, a user interrupts, etc.), the next LLM call belongs\nto a new AgentTurn.\n\n`turn` is the AgentTurn index from `Engine::turn_count`.\n\nCurrently retry is not yet implemented (`agen-stream-continuation`)\nso AgentTurn and `LlmCall` fire 1:1, but the contract here is\nthe AgentTurn boundary; consumers that want per-LLM-call signals\nmust subscribe to `LlmCallStart` / `LlmCallEnd` instead.","properties":{"data":{"additionalProperties":false,"properties":{"turn":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["turn"],"type":"object"},"event":{"const":"turn_start","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"AgentTurn closed.","properties":{"data":{"additionalProperties":false,"properties":{"result":{"$ref":"#/$defs/TurnResult"},"turn":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["turn","result"],"type":"object"},"event":{"const":"turn_end","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"One LLM generation call started (1 request → 1 generation, retry\nincluded). Multiple `LlmCall*` pairs may fire inside a single\n`TurnStart` / `TurnEnd` pair when a request is retried; today\nthey fire 1:1 because retry is not implemented.\n\n`llm_call` is the worker-wide running counter from\n`Engine::llm_call_count`.","properties":{"data":{"additionalProperties":false,"properties":{"llm_call":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["llm_call"],"type":"object"},"event":{"const":"llm_call_start","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"LLM generation call ended.","properties":{"data":{"additionalProperties":false,"properties":{"llm_call":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["llm_call"],"type":"object"},"event":{"const":"llm_call_end","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A transport-level LLM request retry has been scheduled.\n\nThis is operational state for clients to render while the worker is\nwaiting in backoff. It is not part of conversation history.","properties":{"data":{"additionalProperties":false,"properties":{"elapsed_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"error":{"type":"string"},"failed_attempt":{"description":"The attempt that just failed. 1 origin.","format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"llm_call":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"max_attempts":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"status":{"format":"uint16","maximum":65535.0,"minimum":0.0,"type":["integer","null"]},"wait_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["llm_call","failed_attempt","max_attempts","wait_ms","elapsed_ms","error"],"type":"object"},"event":{"const":"llm_retry","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Stream generation was interrupted after events had begun and the worker\nis continuing with a follow-up LLM request.","properties":{"data":{"additionalProperties":false,"properties":{"attempt":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"llm_call":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"max_attempts":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"reason":{"type":"string"}},"required":["llm_call","attempt","max_attempts","reason"],"type":"object"},"event":{"const":"llm_continuation","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"text":{"type":"string"}},"required":["text"],"type":"object"},"event":{"const":"text_delta","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"text":{"type":"string"}},"required":["text"],"type":"object"},"event":{"const":"text_done","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A reasoning / thinking block has started.\n\nAlways paired with a `ThinkingDone`. `ThinkingDelta` is optional:\nsome providers (or some configurations) emit thinking metadata\nwithout plaintext, in which case Start → Done arrive with no\ndeltas in between. Multiple thinking blocks per turn are allowed.","properties":{"event":{"const":"thinking_start","type":"string"}},"required":["event"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"text":{"type":"string"}},"required":["text"],"type":"object"},"event":{"const":"thinking_delta","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Thinking block completed. `text` is the full accumulated body\n(empty string when the provider didn't emit plaintext).","properties":{"data":{"additionalProperties":false,"properties":{"text":{"type":"string"}},"required":["text"],"type":"object"},"event":{"const":"thinking_done","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"id":{"type":"string"},"name":{"type":"string"}},"required":["id","name"],"type":"object"},"event":{"const":"tool_call_start","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"id":{"type":"string"},"json":{"type":"string"}},"required":["id","json"],"type":"object"},"event":{"const":"tool_call_args_delta","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"arguments":{"type":"string"},"id":{"type":"string"},"name":{"type":"string"}},"required":["id","name","arguments"],"type":"object"},"event":{"const":"tool_call_done","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"disposition":{"anyOf":[{"$ref":"#/$defs/ToolResultDisposition"},{"type":"null"}]},"id":{"type":"string"},"is_error":{"default":false,"type":"boolean"},"output":{"description":"Full tool output. Absent when the tool chose to return\nsummary-only, or when the result was pruned.","type":["string","null"]},"summary":{"description":"Short human-readable summary. Always present; used by clients\nthat only want a 1-line rendering (e.g. collapsed views).","type":"string"}},"required":["id","summary","is_error"],"type":"object"},"event":{"const":"tool_result","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Token accounting for one LLM request.\n\n`input_tokens` is the prompt prefix occupancy (cache reads /\ncache writes included), as normalised by the worker layer.\n`cache_read_input_tokens` is the cache-hit subset of that\noccupancy; subtracting it yields the \"net upload\" the client\nactually paid full price to send on this request, which is what\nthe TUI status line accumulates per turn.","properties":{"data":{"additionalProperties":false,"properties":{"cache_read_input_tokens":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":["integer","null"]},"input_tokens":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":["integer","null"]},"output_tokens":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":["integer","null"]}},"type":"object"},"event":{"const":"usage","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"result":{"$ref":"#/$defs/RunResult"}},"required":["result"],"type":"object"},"event":{"const":"run_end","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"code":{"$ref":"#/$defs/ErrorCode"},"message":{"type":"string"}},"required":["code","message"],"type":"object"},"event":{"const":"error","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Sent exactly once at the start of every client connection.\n\n`entries` is the session-log mirror at subscribe time, serialised\nas the JSON form of `session_store::LogEntry`. This is the\nbulk-reconstruction lane: clients walk the entries to seed their\nderived view.\n\n`greeting` and `status` accompany the snapshot so clients render\nworker identity and current controller state without an extra round\ntrip.\n\nLive updates after the snapshot arrive through the streaming\nevents (`TextDelta` / `ToolCall*` / `ToolResult` / etc.) plus\nrole-specific entry events (`SegmentRotated` / `SystemItem`) —\nthere is no generic \"every committed entry\" broadcast.","properties":{"data":{"additionalProperties":false,"properties":{"greeting":{"$ref":"#/$defs/Greeting"},"in_flight":{"$ref":"#/$defs/InFlightSnapshot","description":"Unfinished model output that has already streamed in the current\nrun but is not yet represented by committed snapshot entries."},"internal_workers":{"description":"Parent-owned Internal Worker sessions visible to this client.\nService-private Internal Workers are deliberately excluded.","items":{"$ref":"#/$defs/InternalWorkerSnapshot"},"type":"array"},"session":{"$ref":"#/$defs/SessionSnapshot"},"state":{"$ref":"#/$defs/WorkerStateSnapshot","description":"Full revisioned live execution state. `Stopped` remains Runtime\ncatalog authority and is deliberately not represented here."}},"required":["session","greeting","state"],"type":"object"},"event":{"const":"snapshot","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A live event from a parent-owned Internal Worker. The payload reuses the\nnormal Worker event vocabulary while the wrapper carries stable origin\nidentity and a per-child revision fence.","properties":{"data":{"additionalProperties":false,"properties":{"event":{"$ref":"#/$defs/Event"},"revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"worker":{"$ref":"#/$defs/InternalWorkerRef"}},"required":["worker","revision","event"],"type":"object"},"event":{"const":"internal_worker","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Terminal removal fence for one parent-owned Internal Worker session.\n\nClients discard the matching child and descendants, then ignore later\nnested events for this identity until an authoritative snapshot replaces\nthe projection.","properties":{"data":{"additionalProperties":false,"properties":{"revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"worker":{"$ref":"#/$defs/InternalWorkerRef"}},"required":["worker","revision"],"type":"object"},"event":{"const":"internal_worker_removed","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Server-side segment log rotated to a fresh `SegmentStart`.\n\nFires on compaction and on auto-fork when the store head drifts\nA compaction/fork has replaced the authoritative segment. Clients drop\ntheir derived view and reseed from the canonical committed snapshot.","properties":{"data":{"additionalProperties":false,"properties":{"session":{"$ref":"#/$defs/SessionSnapshot"}},"required":["session"],"type":"object"},"event":{"const":"segment_rotated","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Current Worker controller status. Broadcast on every controller-level\ntransition and included in `History` snapshots for late attach.","properties":{"data":{"additionalProperties":false,"properties":{"snapshot":{"$ref":"#/$defs/WorkerStateSnapshot"}},"required":["snapshot"],"type":"object"},"event":{"const":"worker_state","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"acknowledgement":{"$ref":"#/$defs/WorkerCommandAcknowledgement"}},"required":["acknowledgement"],"type":"object"},"event":{"const":"command_acknowledged","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Bounded, provider-owned command telemetry for the live Console. This is\nintentionally not a history entry and is reconstructed from\n`Snapshot.in_flight.commands` after reconnect.","properties":{"data":{"additionalProperties":false,"properties":{"event":{"$ref":"#/$defs/CommandEvent"}},"required":["event"],"type":"object"},"event":{"const":"command","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Reply to `Method::ListCompletions`. Delivered only to the\nrequesting socket (not broadcast). `entries` is empty when no\ncandidates match or when the requested kind has no resolver\nwired up.","properties":{"data":{"additionalProperties":false,"properties":{"entries":{"items":{"$ref":"#/$defs/CompletionEntry"},"type":"array"},"kind":{"$ref":"#/$defs/CompletionKind"}},"required":["kind","entries"],"type":"object"},"event":{"const":"completions","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Reply to `Method::ListRewindTargets`. Clients should only open a picker\nin response to their own pending request; the event may be broadcast.","properties":{"data":{"additionalProperties":false,"properties":{"head_entries":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"targets":{"items":{"$ref":"#/$defs/RewindTarget"},"type":"array"}},"required":["head_entries","targets"],"type":"object"},"event":{"const":"rewind_targets","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A rewind has truncated the authoritative session. `session` is the\nretained canonical snapshot clients should use to reseed display state.","properties":{"data":{"additionalProperties":false,"properties":{"input":{"items":{"$ref":"#/$defs/Segment"},"type":"array"},"session":{"$ref":"#/$defs/SessionSnapshot"},"summary":{"$ref":"#/$defs/RewindSummary"}},"required":["session","input","summary"],"type":"object"},"event":{"const":"rewind_applied","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Reply to `Method::ListWorkers`. Payload is a stable JSON value so the Worker\ncrate can evolve discovery fields without introducing a protocol\ndependency on session-store.","properties":{"data":{"additionalProperties":false,"properties":{"workers":true},"required":["workers"],"type":"object"},"event":{"const":"workers_listed","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Reply to `Method::RestoreWorker`.","properties":{"data":{"additionalProperties":false,"properties":{"result":true},"required":["result"],"type":"object"},"event":{"const":"worker_restored","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Reply to `Method::RegisterPeer`.","properties":{"data":{"additionalProperties":false,"properties":{"result":true},"required":["result"],"type":"object"},"event":{"const":"peer_registered","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"$ref":"#/$defs/Alert"},"event":{"const":"alert","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Latest memory extract/consolidation lifecycle event for UI observability.\n\nThis is not part of LLM history or prompt context; clients may display it\nbriefly as operational status.","properties":{"data":{"$ref":"#/$defs/MemoryWorkerEvent"},"event":{"const":"memory_worker","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Runtime-only compaction progress. `None` clears the current status.\nThis never enters Session history and carries no operation or Segment identity.","properties":{"data":{"additionalProperties":false,"properties":{"compaction":{"anyOf":[{"$ref":"#/$defs/InFlightCompaction"},{"type":"null"}]}},"type":"object"},"event":{"const":"compaction_progress","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Legacy compaction lifecycle event retained for wire read compatibility.","properties":{"data":{"additionalProperties":false,"properties":{"lifecycle":{"$ref":"#/$defs/CompactionLifecycle"}},"required":["lifecycle"],"type":"object"},"event":{"const":"compact_start","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Compaction completed and the session was rotated.","properties":{"data":{"additionalProperties":false,"properties":{"lifecycle":{"$ref":"#/$defs/CompactionLifecycle"}},"required":["lifecycle"],"type":"object"},"event":{"const":"compact_done","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Compaction failed or was cancelled. The session is unchanged.","properties":{"data":{"additionalProperties":false,"properties":{"lifecycle":{"$ref":"#/$defs/CompactionLifecycle"}},"required":["lifecycle"],"type":"object"},"event":{"const":"compact_failed","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"event":{"const":"shutdown","type":"string"}},"required":["event"],"type":"object"}]},"EventSubscriptionSelector":{"oneOf":[{"additionalProperties":false,"properties":{"topic":{"const":"runtime_workers","type":"string"}},"required":["topic"],"type":"object"},{"additionalProperties":false,"properties":{"topic":{"const":"worker_lifecycle","type":"string"},"worker_ids":{"$ref":"#/$defs/SubscriptionWorkerIds"}},"required":["topic","worker_ids"],"type":"object"},{"additionalProperties":false,"properties":{"runtime_id":{"type":["string","null"]},"topic":{"const":"worker_protocol","type":"string"},"worker_id":{"$ref":"#/$defs/SubscriptionWorkerId"}},"required":["topic","worker_id"],"type":"object"},{"additionalProperties":false,"description":"Server-derived Workspace projection. Workspace identity comes from the\nauthenticated connection and is deliberately absent from this selector.","properties":{"topic":{"const":"workspace_workers","type":"string"}},"required":["topic"],"type":"object"},{"additionalProperties":false,"description":"Typed extension point for the later Workdir subscription slice.","properties":{"topic":{"const":"workspace_workdirs","type":"string"}},"required":["topic"],"type":"object"}]},"Greeting":{"additionalProperties":false,"description":"Worker self-description rendered by the TUI when a session starts empty.\n\nBuilt once in the Worker controller from the resolved manifest and\ntransmitted alongside `Event::Snapshot` so clients don't need\ntheir own view of the manifest.","properties":{"context_tokens":{"default":0,"description":"Estimated current session context tokens at connect time.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"context_window":{"default":0,"description":"Model context window in tokens. Always filled by the Worker greeting.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"cwd":{"type":"string"},"model":{"type":"string"},"provider":{"type":"string"},"scope_summary":{"type":"string"},"tools":{"items":{"type":"string"},"type":"array"},"worker_name":{"type":"string"}},"required":["worker_name","cwd","provider","model","scope_summary","tools","context_tokens","context_window"],"type":"object"},"InFlightBlock":{"oneOf":[{"additionalProperties":false,"properties":{"finished":{"type":"boolean"},"kind":{"const":"text","type":"string"},"text":{"type":"string"}},"required":["kind","text"],"type":"object"},{"additionalProperties":false,"properties":{"finished":{"type":"boolean"},"kind":{"const":"thinking","type":"string"},"text":{"type":"string"}},"required":["kind","text"],"type":"object"},{"additionalProperties":false,"properties":{"args":{"type":"string"},"id":{"type":"string"},"kind":{"const":"tool_call","type":"string"},"name":{"type":"string"},"state":{"$ref":"#/$defs/InFlightToolCallState"}},"required":["kind","id","name","args"],"type":"object"}]},"InFlightCompaction":{"additionalProperties":false,"properties":{"phase":{"$ref":"#/$defs/CompactionPhase"},"started_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"trigger":{"$ref":"#/$defs/CompactionTrigger"}},"required":["phase","started_at_ms","trigger"],"type":"object"},"InFlightSnapshot":{"additionalProperties":false,"description":"Unfinished model output and active command state included in\n`Event::Snapshot` for clients that attach while work is still streaming.\n\nThese blocks are presentation state only: they are reconstructed from the\nactive Worker controller and must not be treated as committed assistant\nhistory. Finalized assistant items continue to come from ordinary snapshot\nentries.","properties":{"blocks":{"items":{"$ref":"#/$defs/InFlightBlock"},"type":"array"},"commands":{"items":{"$ref":"#/$defs/CommandSnapshot"},"type":"array"},"compaction":{"anyOf":[{"$ref":"#/$defs/InFlightCompaction"},{"type":"null"}],"description":"The currently running compaction, if any.\n\nThis is lifecycle progress only. Candidate history and the staged\nSegment remain private until the Segment is activated atomically."}},"type":"object"},"InFlightToolCallState":{"enum":["pending","streaming_args","done"],"type":"string"},"InternalWorkerKind":{"description":"Presentation category for an Internal Worker exposed through its parent's\nprotocol stream. Internal Workers never become independently addressable\nprotocol subjects.","oneOf":[{"enum":["sub_worker"],"type":"string"},{"additionalProperties":false,"properties":{"service":{"additionalProperties":false,"properties":{"kind":{"type":"string"}},"required":["kind"],"type":"object"}},"required":["service"],"type":"object"}]},"InternalWorkerRef":{"additionalProperties":false,"description":"Stable presentation identity for one parent-owned Internal Worker session.\n`name` is display-only; `session_id` is the identity used by clients.","properties":{"kind":{"$ref":"#/$defs/InternalWorkerKind"},"name":{"type":"string"},"parent_session_id":{"type":["string","null"]},"session_id":{"type":"string"}},"required":["session_id","name","kind"],"type":"object"},"InternalWorkerSnapshot":{"additionalProperties":false,"description":"Reconnect state for one visible Internal Worker. The revision fences live\n`Event::InternalWorker` updates that raced with parent snapshot assembly.","properties":{"error":{"type":["string","null"]},"in_flight":{"$ref":"#/$defs/InFlightSnapshot"},"internal_workers":{"items":{"$ref":"#/$defs/InternalWorkerSnapshot"},"type":"array"},"revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"session":{"$ref":"#/$defs/SessionSnapshot"},"status":{"$ref":"#/$defs/WorkerStatus","default":"idle"},"worker":{"$ref":"#/$defs/InternalWorkerRef"}},"required":["worker","revision","session","status"],"type":"object"},"InvokeKind":{"description":"Kind of trigger that opened a new Invoke (IDLE → active) range.\n\nOne Invoke groups all entries from this trigger up to the next\n`Invoke` marker. The kind is the only payload — content (user text,\nnotify message, worker event body) is delivered by the immediately\nfollowing Turn entry, not by the marker itself.","oneOf":[{"const":"user_send","description":"`Method::Submit` — a user submission.","type":"string"},{"const":"notify","description":"`Method::Notify` — free-text notification injected into history.","type":"string"},{"const":"worker_event","description":"`Method::WorkerEvent` — typed lifecycle report from a child Worker.","type":"string"},{"const":"system_reminder","description":"A typed system reminder that crosses an IDLE boundary (mid-run\nreminders that don't break IDLE are SystemItem-only and do not\nopen a new Invoke).","type":"string"},{"const":"wakeup","description":"Cron / RemoteTrigger style scheduled wake-up. Reserved; no\nproducer is wired today but the variant is part of the initial\nset so future schedulers don't have to amend the wire enum.","type":"string"}]},"MemoryWorkerEvent":{"additionalProperties":false,"properties":{"message":{"description":"Human-readable compact form for actionbar rendering.","type":"string"},"reason":{"type":"string"},"run_id":{"type":"string"},"status":{"type":"string"},"timestamp_ms":{"description":"Milliseconds since the Unix epoch.","format":"int64","maximum":9007199254740991.0,"minimum":-9007199254740991.0,"type":"integer"},"trigger":{"type":"string"},"worker":{"type":"string"}},"required":["worker","status","run_id","trigger","reason","message","timestamp_ms"],"type":"object"},"Method":{"oneOf":[{"additionalProperties":false,"description":"Durably accept typed input for immediate activation or the session FIFO.\nRunning or Paused Workers queue Submit for a later turn; use Notify for\nadvisory information to incorporate into work already in progress.\n\n`submission_request_id` is generated by the authenticated caller and is\nused only for idempotent retry. Worker allocates the durable\n`submission_id` returned by [`Event::SubmissionAccepted`].","properties":{"method":{"const":"submit","type":"string"},"params":{"additionalProperties":false,"properties":{"input":{"items":{"$ref":"#/$defs/Segment"},"type":"array"},"submission_request_id":{"type":"string"}},"required":["submission_request_id","input"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Human-readable text injected into the target Worker's LLM context as a\nnon-blocking system message. Accepted notifications are delivered at the\nnext between-turn boundary, or start a coalesced notification run while\nthe Worker is idle. No side effects beyond LLM context; use\n`WorkerEvent` for typed lifecycle reports.","properties":{"method":{"const":"notify","type":"string"},"params":{"additionalProperties":false,"properties":{"message":{"type":"string"},"notification_request_id":{"type":"string"}},"required":["notification_request_id","message"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Typed lifecycle report from a child Worker to its direct parent.","properties":{"method":{"const":"worker_event","type":"string"},"params":{"$ref":"#/$defs/WorkerEvent"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Return the authoritative FIFO summary without exposing queued payloads.","properties":{"method":{"const":"list_pending_submissions","type":"string"}},"required":["method"],"type":"object"},{"additionalProperties":false,"description":"Remove one queued submission. Running or already activated submissions\nare immutable and therefore cannot be cancelled here.","properties":{"method":{"const":"cancel_pending_submission","type":"string"},"params":{"additionalProperties":false,"properties":{"expected_revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"submission_id":{"type":"string"}},"required":["submission_id","expected_revision"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Remove every queued submission while preserving the active run.","properties":{"method":{"const":"clear_pending_submissions","type":"string"},"params":{"additionalProperties":false,"properties":{"expected_revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["expected_revision"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Activate the next queued submission while the Worker is idle. This is an\nexplicit recovery operation and never resumes a paused run implicitly.","properties":{"method":{"const":"continue_pending","type":"string"},"params":{"additionalProperties":false,"properties":{"expected_head_id":{"type":"string"},"expected_revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["expected_revision","expected_head_id"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"properties":{"method":{"const":"resume","type":"string"},"params":{"additionalProperties":false,"properties":{"command":{"$ref":"#/$defs/WorkerCommandEnvelope"}},"required":["command"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"properties":{"method":{"const":"cancel","type":"string"},"params":{"additionalProperties":false,"properties":{"command":{"$ref":"#/$defs/WorkerCommandEnvelope"}},"required":["command"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Stop the in-flight turn and transition to `Paused`.\n\nUnlike `Cancel` (which discards and returns to `Idle`), a paused\nWorker can resume the interrupted work via `Resume`, or accept a\nfresh `Submit` (orphan `tool_use` items are closed with a\nsynthetic tool result before the new user message is appended).","properties":{"method":{"const":"pause","type":"string"},"params":{"additionalProperties":false,"properties":{"command":{"$ref":"#/$defs/WorkerCommandEnvelope"}},"required":["command"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Request an explicit compaction while the Worker is otherwise idle.\n\nThis is a typed control method: clients must not send `compact` as a\n`Method::Submit` user message.","properties":{"method":{"const":"compact","type":"string"},"params":{"additionalProperties":false,"properties":{"command":{"$ref":"#/$defs/WorkerCommandEnvelope"}},"required":["command"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Ask the Worker to list valid rewind targets from its authoritative session log.","properties":{"method":{"const":"list_rewind_targets","type":"string"}},"required":["method"],"type":"object"},{"additionalProperties":false,"description":"Truncate the current session back to the selected rewind target and\nreturn the selected user input to the client composer.","properties":{"method":{"const":"rewind_to","type":"string"},"params":{"additionalProperties":false,"properties":{"expected_head_entries":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"target":{"$ref":"#/$defs/RewindTargetId"}},"required":["target","expected_head_entries"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"properties":{"method":{"const":"shutdown","type":"string"},"params":{"additionalProperties":false,"properties":{"command":{"$ref":"#/$defs/WorkerCommandEnvelope"}},"required":["command"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Request a list of completion candidates from the Worker.\n\nReply is sent on the same socket as `Event::Completions` (not\nbroadcast). The IPC server handles this directly and writes\nthe response straight back to the requesting socket.","properties":{"method":{"const":"list_completions","type":"string"},"params":{"additionalProperties":false,"properties":{"kind":{"$ref":"#/$defs/CompletionKind"},"prefix":{"type":"string"}},"required":["kind","prefix"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"List Workers visible to this Worker from durable Worker state and the spawned-child\nregistry. This is not a host-wide Worker universe query.","properties":{"method":{"const":"list_workers","type":"string"}},"required":["method"],"type":"object"},{"additionalProperties":false,"description":"Restore a visible stopped/restorable Worker, or report that it is already\nlive. Missing state and not-visible state are distinct errors.","properties":{"method":{"const":"restore_worker","type":"string"},"params":{"additionalProperties":false,"properties":{"name":{"type":"string"}},"required":["name"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Register another existing Worker as a reciprocal peer of this Worker.\n\nThis is metadata/control state only: it does not ask the target's live\ncontroller for consent, and it must not grant delegated scope,\nspawned-child ownership, output cursors, or child lifecycle authority.","properties":{"method":{"const":"register_peer","type":"string"},"params":{"additionalProperties":false,"properties":{"name":{"type":"string"}},"required":["name"],"type":"object"}},"required":["method","params"],"type":"object"}]},"PasteArtifactAvailability":{"enum":["available","unavailable","integrity_failed"],"type":"string"},"PasteArtifactMediaType":{"description":"One typed piece of a user submission.\n\n`Method::Submit` and `Event::UserMessage` carry `Vec`. Dumb\nclients (CLI piping, scripts) only need to produce a single\n`Segment::Text`; richer clients (TUI / GUI) construct typed atoms\n(paste chips, file refs) and\nsend them through directly so the Worker side never has to re-parse a\nflattened string.\n\nForward compat: payloads with unknown `kind` deserialize to\n`Segment::Unknown`. Worker treats this the same as known-but-unresolved\nvariants — emits an alert and inserts a `[unknown input segment]`\nplaceholder into the LLM context so neither user nor LLM is blind to\nthe dropped intent.","enum":["text_plain_utf8"],"type":"string"},"PasteArtifactRef":{"additionalProperties":false,"description":"Session-owned reference to a large pasted-input artifact.\n\nThe reference contains only bounded integrity and provenance metadata. The\nartifact body remains in session storage and is available to the model only\nthrough the scoped paste-artifact tools installed by Worker.","properties":{"artifact_id":{"type":"string"},"availability":{"$ref":"#/$defs/PasteArtifactAvailability","description":"Availability observed when this immutable reference was committed.\nReads revalidate storage and integrity rather than trusting this field."},"byte_len":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"char_count":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"created_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"line_count":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"media_type":{"$ref":"#/$defs/PasteArtifactMediaType"},"sha256":{"type":"string"},"source_entry_id":{"type":"string"}},"required":["artifact_id","created_at_ms","media_type","availability","byte_len","char_count","line_count","sha256","source_entry_id"],"type":"object"},"PendingSubmissionSummary":{"additionalProperties":false,"description":"Bounded public projection of one pending submission. Only a short display\npreview is exposed; full payload segments and provenance remain in the log.","properties":{"accepted_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"byte_len":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"preview":{"description":"Single-line display text, at most 240 Unicode scalars plus an ellipsis.\nAbsent on older runtimes. Never used to reconstruct the submitted input.","type":["string","null"]},"segment_count":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"submission_id":{"type":"string"}},"required":["submission_id","accepted_at_ms","segment_count","byte_len"],"type":"object"},"PendingSubmissionsSnapshot":{"additionalProperties":false,"description":"Revisioned session-owned FIFO projection used by snapshots and live events.","properties":{"head_id":{"default":null,"type":["string","null"]},"notification_count":{"default":0,"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"notification_previews":{"description":"Ordered display-only previews of waiting notifications, each bounded like\nPendingSubmissionSummary::preview. Older runtimes omit this field.","items":{"type":"string"},"type":"array"},"revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"submissions":{"default":[],"items":{"$ref":"#/$defs/PendingSubmissionSummary"},"type":"array"}},"required":["revision","head_id","notification_count","submissions"],"type":"object"},"Permission":{"description":"Permission lattice used by [`ScopeRule`].\n\nThe derived `Ord` instance follows declaration order, so\n`Read < Write`. Allow rules grant the stated level (and by extension\neverything below); deny rules cap the effective level **strictly\nbelow** the stated level.","enum":["read","write"],"type":"string"},"RewindSummary":{"additionalProperties":false,"properties":{"discarded_entries":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"tool_side_effect_warning":{"type":"boolean"},"truncated_to_entries":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["truncated_to_entries","discarded_entries","tool_side_effect_warning"],"type":"object"},"RewindTarget":{"additionalProperties":false,"properties":{"disabled_reason":{"type":["string","null"]},"eligible":{"type":"boolean"},"expected_head_entries":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"id":{"$ref":"#/$defs/RewindTargetId"},"preview":{"type":"string"},"timestamp_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":["integer","null"]},"truncate_entries":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"turn_index":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"warning":{"type":["string","null"]}},"required":["id","expected_head_entries","truncate_entries","turn_index","preview","eligible"],"type":"object"},"RewindTargetId":{"additionalProperties":false,"properties":{"segment_id":{"type":"string"},"user_input_entry_index":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["segment_id","user_input_entry_index"],"type":"object"},"RunFailureKind":{"enum":["engine","compaction"],"type":"string"},"RunResult":{"oneOf":[{"enum":["finished","paused","limit_reached"],"type":"string"},{"const":"cancelled","description":"The logical Run ended because the user intentionally cancelled it after\nobservable work. Unlike `RolledBack`, committed history is retained.","type":"string"},{"const":"rolled_back","description":"The accepted Method::Submit produced no assistant/tool output before\nuser interruption, so the Worker rolled the submit-time turn state back\nto its pre-submit snapshot. Clients should treat the Worker as Idle and\nrestore the just-submitted input into the editable composer if desired.","type":"string"}]},"RunResumeSource":{"enum":["compaction","pause"],"type":"string"},"RunYieldReason":{"enum":["compaction"],"type":"string"},"ScopeRule":{"additionalProperties":false,"description":"A single allow or deny rule inside a scope configuration.","properties":{"permission":{"$ref":"#/$defs/Permission","description":"Permission level this rule grants (allow) or caps strictly below\n(deny)."},"recursive":{"default":true,"description":"When `false`, the rule only matches the target itself and its\ndirect children. Defaults to `true`.","type":"boolean"},"symlink_policy":{"$ref":"#/$defs/SymlinkPolicy","default":"resolved","description":"Which path identity an allow rule uses when symbolic links are\nencountered. Deny rules always inspect both identities."},"target":{"description":"Target path. Must be absolute by the time a `Scope` is built from\nthis rule — relative paths are resolved per-layer against the\nmanifest file's directory (cwd for overlay layers) before cascade\nmerge.","type":"string"}},"required":["target","permission","recursive","symlink_policy"],"type":"object"},"Segment":{"oneOf":[{"additionalProperties":false,"description":"Free-form text. The fallback every client can produce.","properties":{"content":{"type":"string"},"kind":{"const":"text","type":"string"}},"required":["kind","content"],"type":"object"},{"additionalProperties":false,"description":"Bracketed-paste capture from a TUI-style client. `id`, `chars`\nand `lines` carry the metadata needed to re-render a\n`[Clipboard #N | X chars, Y lines]` chip in `Event::UserMessage`\nre-broadcast.","properties":{"chars":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"content":{"type":"string"},"id":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"kind":{"const":"paste","type":"string"},"lines":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"}},"required":["kind","id","chars","lines","content"],"type":"object"},{"additionalProperties":false,"description":"Internal reference produced when Worker stores a large `Paste` before\ncommitting input. Clients may receive this in history/event projections;\nthe body is intentionally absent.","properties":{"artifact":{"$ref":"#/$defs/PasteArtifactRef"},"kind":{"const":"paste_artifact","type":"string"}},"required":["kind","artifact"],"type":"object"},{"additionalProperties":false,"description":"Client-local file uploaded into the owning Worker session before submit.\nThe Session Log stores only this immutable reference, never file bytes or\nthe client's local path.","properties":{"file":{"$ref":"#/$defs/UploadedFileRef"},"kind":{"const":"uploaded_file","type":"string"}},"required":["kind","file"],"type":"object"},{"additionalProperties":false,"description":"`@` file-system reference. Worker resolves readable files to\n`[File: ]` attachments and readable normal directories to shallow\n`[Dir: ]` listings; the flattened user text keeps the literal\n`@` placeholder either way.","properties":{"kind":{"const":"file_ref","type":"string"},"path":{"type":"string"}},"required":["kind","path"],"type":"object"},{"additionalProperties":false,"description":"Source-qualified request for the host to start a Flow before committing\nthis user input. Runtime resolves it through Workspace authority and\nreplaces it with the entered state's instructions before Worker-side\ninput resolution.","properties":{"kind":{"const":"flow","type":"string"},"selector":{"type":"string"}},"required":["kind","selector"],"type":"object"},{"additionalProperties":false,"description":"Unknown variant from a newer client. Worker treats this as an\nunresolved input — surfaces an alert and inserts a placeholder.\nRound-trip is lossy: re-serializing yields `{\"kind\":\"unknown\"}`.","properties":{"kind":{"const":"unknown","type":"string"}},"required":["kind"],"type":"object"}]},"SessionContentPart":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"text","type":"string"},"text":{"type":"string"}},"required":["kind","text"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"refusal","type":"string"},"refusal":{"type":"string"}},"required":["kind","refusal"],"type":"object"}]},"SessionEntryProvenance":{"enum":["human_input","worker_input","flow_instruction","backend_instruction","model_output","tool_output","derived_summary","legacy_unknown"],"type":"string"},"SessionMessageRole":{"enum":["user","assistant"],"type":"string"},"SessionSnapshot":{"additionalProperties":false,"description":"Canonical, storage-independent projection of committed session history.\n\nWorker protocols expose this DTO instead of append-log records. New\nstorage variants can therefore be added without teaching every client how\nto replay the durable log format.","properties":{"entries":{"items":{"$ref":"#/$defs/SessionSnapshotEntry"},"type":"array"},"pending_submissions":{"$ref":"#/$defs/PendingSubmissionsSnapshot","default":{"head_id":null,"notification_count":0,"revision":0,"submissions":[]}}},"required":["entries","pending_submissions"],"type":"object"},"SessionSnapshotEntry":{"oneOf":[{"additionalProperties":false,"properties":{"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"kind":{"const":"user_input","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"segments":{"items":{"$ref":"#/$defs/Segment"},"type":"array"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","segments","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"properties":{"content":{"items":{"$ref":"#/$defs/SessionContentPart"},"type":"array"},"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"kind":{"const":"message","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"role":{"$ref":"#/$defs/SessionMessageRole"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","role","content","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"properties":{"arguments":{"type":"string"},"call_id":{"type":"string"},"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"kind":{"const":"tool_call","type":"string"},"name":{"type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","call_id","name","arguments","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"properties":{"attachments":{"items":{"$ref":"#/$defs/SessionToolAttachment"},"type":"array"},"call_id":{"type":"string"},"content":{"type":["string","null"]},"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"is_error":{"type":"boolean"},"kind":{"const":"tool_result","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"summary":{"type":"string"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","call_id","summary","is_error","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"properties":{"content":{"type":"string"},"data":true,"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"item_kind":{"type":"string"},"kind":{"const":"system_item","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","item_kind","content","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"description":"Durable, non-terminal control return from one logical Run. This is not a\ncompleted run and must not be used to infer the current Worker state.","properties":{"active_run_turn_count":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"kind":{"const":"run_yielded","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"reason":{"$ref":"#/$defs/RunYieldReason"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","reason","active_run_turn_count","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"description":"Durable boundary immediately before Engine execution resumes. The source\ndistinguishes automatic compaction continuation from an intentional\nuser-pause resume; it does not prove that later model output occurred.","properties":{"active_run_turn_count":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"kind":{"const":"run_resumed","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"source":{"$ref":"#/$defs/RunResumeSource"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","source","active_run_turn_count","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"description":"Durable terminal for an intentional cancellation. Consumers must not\npresent this lifecycle result as an execution failure.","properties":{"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"kind":{"const":"run_cancelled","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"properties":{"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"failure":{"anyOf":[{"$ref":"#/$defs/RunFailureKind"},{"type":"null"}]},"kind":{"const":"run_error","type":"string"},"message":{"type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","message","entry_id","timestamp","provenance"],"type":"object"}]},"SessionToolAttachment":{"additionalProperties":false,"properties":{"data_base64":{"description":"Base64-encoded durable attachment body. Public snapshots preserve the\ncommitted multimodal value instead of replacing it with placeholder text.","type":"string"},"media_type":{"type":"string"}},"required":["media_type","data_base64"],"type":"object"},"SubmissionDisposition":{"description":"Durable acceptance result for one idempotent Submit request.","oneOf":[{"enum":["started"],"type":"string"},{"const":"queued","description":"Input is durably held in the session FIFO until eligible for activation.","type":"string"}]},"SubscriptionEvent":{"oneOf":[{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"payload":{"$ref":"#/$defs/SubscriptionEventPayload"},"subject_revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"subscription_id":{"$ref":"#/$defs/SubscriptionId"}},"required":["subscription_id","subject_revision","payload"],"type":"object"},"event":{"const":"event","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"code":{"$ref":"#/$defs/SubscriptionTerminationCode"},"message":{"type":"string"},"subscription_id":{"$ref":"#/$defs/SubscriptionId"}},"required":["subscription_id","code","message"],"type":"object"},"event":{"const":"subscription_closed","type":"string"}},"required":["event","data"],"type":"object"}]},"SubscriptionEventPayload":{"oneOf":[{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"worker":{"$ref":"#/$defs/SubscriptionWorker"}},"required":["worker"],"type":"object"},"event":{"const":"worker_upserted","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"runtime_id":{"type":["string","null"]},"worker_id":{"$ref":"#/$defs/SubscriptionWorkerId"}},"required":["worker_id"],"type":"object"},"event":{"const":"worker_removed","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"event":{"$ref":"#/$defs/Event"},"worker_id":{"$ref":"#/$defs/SubscriptionWorkerId"}},"required":["worker_id","event"],"type":"object"},"event":{"const":"worker_protocol","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"workdir":{"$ref":"#/$defs/WorkspaceSubscriptionWorkdir"}},"required":["workdir"],"type":"object"},"event":{"const":"workdir_upserted","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"working_directory_id":{"$ref":"#/$defs/SubscriptionWorkdirId"}},"required":["working_directory_id"],"type":"object"},"event":{"const":"workdir_removed","type":"string"}},"required":["event","data"],"type":"object"}]},"SubscriptionId":{"type":"string"},"SubscriptionRejectionCode":{"enum":["invalid_request","unsupported_protocol_version","unsupported_selector","unauthorized","resource_not_found","capacity_exceeded","internal"],"type":"string"},"SubscriptionRequest":{"oneOf":[{"additionalProperties":false,"properties":{"method":{"const":"subscribe_events","type":"string"},"params":{"additionalProperties":false,"properties":{"request_id":{"$ref":"#/$defs/SubscriptionRequestId"},"selector":{"$ref":"#/$defs/EventSubscriptionSelector"}},"required":["request_id","selector"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"properties":{"method":{"const":"unsubscribe_events","type":"string"},"params":{"additionalProperties":false,"properties":{"request_id":{"$ref":"#/$defs/SubscriptionRequestId"},"subscription_id":{"$ref":"#/$defs/SubscriptionId"}},"required":["request_id","subscription_id"],"type":"object"}},"required":["method","params"],"type":"object"}]},"SubscriptionRequestId":{"type":"string"},"SubscriptionResponse":{"oneOf":[{"additionalProperties":false,"properties":{"payload":{"additionalProperties":false,"properties":{"request_id":{"$ref":"#/$defs/SubscriptionRequestId"},"selector":{"$ref":"#/$defs/EventSubscriptionSelector"},"snapshot":{"$ref":"#/$defs/SubscriptionSnapshot"},"snapshot_revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"subscription_id":{"$ref":"#/$defs/SubscriptionId"}},"required":["request_id","subscription_id","selector","snapshot_revision","snapshot"],"type":"object"},"result":{"const":"subscribed","type":"string"}},"required":["result","payload"],"type":"object"},{"additionalProperties":false,"properties":{"payload":{"additionalProperties":false,"properties":{"request_id":{"$ref":"#/$defs/SubscriptionRequestId"},"subscription_id":{"$ref":"#/$defs/SubscriptionId"}},"required":["request_id","subscription_id"],"type":"object"},"result":{"const":"unsubscribed","type":"string"}},"required":["result","payload"],"type":"object"},{"additionalProperties":false,"properties":{"payload":{"additionalProperties":false,"properties":{"code":{"$ref":"#/$defs/SubscriptionRejectionCode"},"message":{"type":"string"},"request_id":{"$ref":"#/$defs/SubscriptionRequestId"},"subscription_id":{"anyOf":[{"$ref":"#/$defs/SubscriptionId"},{"type":"null"}]}},"required":["request_id","code","message"],"type":"object"},"result":{"const":"subscription_rejected","type":"string"}},"required":["result","payload"],"type":"object"}]},"SubscriptionSnapshot":{"oneOf":[{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"workers":{"items":{"$ref":"#/$defs/SubscriptionWorker"},"type":"array"}},"required":["workers"],"type":"object"},"topic":{"const":"workers","type":"string"}},"required":["topic","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"events":{"items":{"$ref":"#/$defs/Event"},"type":"array"},"worker_id":{"$ref":"#/$defs/SubscriptionWorkerId"}},"required":["worker_id","events"],"type":"object"},"topic":{"const":"worker_protocol","type":"string"}},"required":["topic","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"workdirs":{"items":{"$ref":"#/$defs/WorkspaceSubscriptionWorkdir"},"type":"array"}},"required":["workdirs"],"type":"object"},"topic":{"const":"workspace_workdirs","type":"string"}},"required":["topic","data"],"type":"object"}]},"SubscriptionTerminationCode":{"enum":["lagged","resource_gone","unauthorized","server_shutdown"],"type":"string"},"SubscriptionWorkdirId":{"type":"string"},"SubscriptionWorker":{"additionalProperties":false,"properties":{"availability":{"$ref":"#/$defs/SubscriptionWorkerAvailability","default":"observed","description":"Freshness of the Runtime-backed observation carried by this projection.\n`Unavailable` preserves catalog membership without claiming that execution stopped."},"display_name":{"type":["string","null"]},"has_running_internal_workers":{"default":false,"type":"boolean"},"job":{"anyOf":[{"$ref":"#/$defs/SubscriptionWorkerJob"},{"type":"null"}],"description":"Backend-owned Job binding. Runtime producers leave this unset; the\nWorkspace projection enriches both snapshots and live updates."},"profile":{"type":["string","null"]},"resource_key":{"description":"Workspace-scoped canonical resource key. Runtime producers leave this unset;\nWorkspace-facing projections must populate it before publishing the Worker.","type":["string","null"]},"runtime_id":{"description":"Set by the Workspace Server when projecting a Runtime-owned Worker to clients.\nRuntime producers leave this unset because the connection identifies the Runtime.","type":["string","null"]},"state":{"$ref":"#/$defs/SubscriptionWorkerState","description":"Runtime catalog lifecycle compatibility projection; not foreground-state authority."},"subject_revision":{"description":"Producer-owned monotonic revision for this Worker subject.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"workdir_attachments":{"items":{"$ref":"#/$defs/SubscriptionWorkerWorkdirAttachment"},"type":"array"},"worker_id":{"$ref":"#/$defs/SubscriptionWorkerId"},"worker_state":{"anyOf":[{"$ref":"#/$defs/WorkerStateSnapshot"},{"type":"null"}],"description":"Latest revisioned foreground state observed from the Worker. This remains\nabsent until an authoritative Worker snapshot/event has been applied."},"workspace_id":{"type":["string","null"]}},"required":["worker_id","subject_revision","state","availability","has_running_internal_workers"],"type":"object"},"SubscriptionWorkerAvailability":{"description":"Whether the Workspace Server has a current Runtime observation for a Worker.\n\n`Unavailable` deliberately does not imply `Stopped`: it means the last observed\nlifecycle state is retained while the Runtime observation source is unavailable.","enum":["observed","unavailable"],"type":"string"},"SubscriptionWorkerId":{"type":"string"},"SubscriptionWorkerIds":{"description":"A bounded, canonical worker-id set. Ordering on the wire and in equality/hash\nsemantics is lexical, so equivalent selectors aggregate to one upstream key.","items":{"$ref":"#/$defs/SubscriptionWorkerId"},"type":"array"},"SubscriptionWorkerJob":{"additionalProperties":false,"properties":{"attempt_id":{"type":"string"},"job_id":{"type":"string"},"purpose":{"type":"string"}},"required":["job_id","attempt_id","purpose"],"type":"object"},"SubscriptionWorkerProtocolMethod":{"additionalProperties":false,"properties":{"method":{"$ref":"#/$defs/Method"},"subscription_id":{"$ref":"#/$defs/SubscriptionId"}},"required":["subscription_id","method"],"type":"object"},"SubscriptionWorkerState":{"enum":["idle","running","paused","stopped"],"type":"string"},"SubscriptionWorkerWorkdirAttachment":{"additionalProperties":false,"properties":{"alias":{"description":"Stable Worker-local routing alias.","type":"string"},"repository_key":{"type":["string","null"]},"working_directory_id":{"$ref":"#/$defs/SubscriptionWorkdirId"}},"required":["alias","working_directory_id"],"type":"object"},"SymlinkPolicy":{"description":"Symbolic-link identity used by one filesystem allow rule.\n\n`Resolved` is the least authority and the default: access is matched\nagainst the provider-resolved target. `Logical` intentionally grants the\npath as presented through the Workdir, even when it aliases another target.","enum":["resolved","logical"],"type":"string"},"ToolResultDisposition":{"enum":["success","error","interrupted","cancelled","outcome_unknown"],"type":"string"},"TurnResult":{"enum":["finished","paused"],"type":"string"},"UploadedFileAvailability":{"description":"Availability recorded for an uploaded client-local file.","enum":["available","unavailable","integrity_failed"],"type":"string"},"UploadedFileRef":{"additionalProperties":false,"description":"Session-owned immutable reference to a client-local uploaded file.\n\nUpload transports return an unbound reference. Worker fills\n`source_entry_id` immediately before the containing user input is committed;\ncommitted Session Log and public snapshot records therefore always retain\nthe durable source-entry identity without storing the file body.","properties":{"artifact_id":{"type":"string"},"availability":{"$ref":"#/$defs/UploadedFileAvailability"},"byte_len":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"created_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"file_name":{"type":"string"},"media_type":{"type":"string"},"sha256":{"type":"string"},"source_entry_id":{"type":["string","null"]}},"required":["artifact_id","file_name","media_type","created_at_ms","availability","byte_len","sha256"],"type":"object"},"WorkerBusyState":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"run","type":"string"},"state":{"$ref":"#/$defs/WorkerRunState"}},"required":["kind","state"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"maintenance","type":"string"},"state":{"$ref":"#/$defs/WorkerMaintenanceState"}},"required":["kind","state"],"type":"object"}]},"WorkerCommandAcknowledgement":{"additionalProperties":false,"properties":{"command":{"$ref":"#/$defs/WorkerCommandKind"},"command_id":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"disposition":{"$ref":"#/$defs/WorkerCommandDisposition"},"state":{"$ref":"#/$defs/WorkerStateSnapshot","description":"The complete authoritative state observed after command admission."}},"required":["command_id","command","disposition","state"],"type":"object"},"WorkerCommandDisposition":{"enum":["accepted","stale_command_id","conflict","invalid_state"],"type":"string"},"WorkerCommandEnvelope":{"additionalProperties":false,"description":"Caller-owned identity for one state-changing Worker command.\n\nA controller accepts command ids in strictly increasing order. Exact retries\nof an accepted id must retain the same command kind.","properties":{"command_id":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["command_id"],"type":"object"},"WorkerCommandKind":{"enum":["resume","cancel","pause","compact","shutdown"],"type":"string"},"WorkerEvent":{"description":"Typed lifecycle events sent from a child Worker to its parent.\n\nDelivered as `Method::WorkerEvent` over the parent's Unix socket. The\nparent Controller always applies variant-specific side effects\n(registry / worker-allocation updates). Agent-visible variants are also\nqueued into the notification buffer; control-plane-only variants are\nnot injected into the parent's LLM context.\n\nTransport is fire-and-forget; receivers must tolerate out-of-order\ndelivery (e.g. `TurnEnded` arriving after `ShutDown` for the same\nchild Worker).","oneOf":[{"additionalProperties":false,"description":"Child finished one turn and is back to IDLE.","properties":{"kind":{"const":"turn_ended","type":"string"},"worker_name":{"type":"string"}},"required":["kind","worker_name"],"type":"object"},{"additionalProperties":false,"description":"Engine execution error occurred inside the child's turn.\n\nLimited to worker runtime failures (provider / tool errors) —\ndoes not include transient method-rejection responses such as\n`AlreadyRunning`.","properties":{"kind":{"const":"errored","type":"string"},"message":{"type":"string"},"worker_name":{"type":"string"}},"required":["kind","worker_name","message"],"type":"object"},{"additionalProperties":false,"description":"Child has stopped (controller loop is exiting).","properties":{"kind":{"const":"shut_down","type":"string"},"worker_name":{"type":"string"}},"required":["kind","worker_name"],"type":"object"},{"additionalProperties":false,"description":"Child SubWorker sub-delegated scope to a grandchild SubWorker via `SubWorkerSpawn`.\n\nControl-plane only: receivers apply registry side effects and\npropagate upward, but do not expose this as an agent notification.\n\nThe parent uses this to add the grandchild to its own\n`spawned_workers.json` so it can manage the grandchild directly\neven if the intermediate child dies. The parent then re-fires\nthis event upward (if it has a parent of its own) to maintain\nthe chain to root.","properties":{"kind":{"const":"scope_sub_delegated","type":"string"},"parent_worker":{"description":"Sub-delegating Worker (= the sender itself).","type":"string"},"scope":{"description":"Scope delegated to the grandchild.","items":{"$ref":"#/$defs/ScopeRule"},"type":"array"},"sub_socket":{"description":"Unix-socket path where the grandchild is reachable.","type":"string"},"sub_worker":{"description":"Name of the grandchild Worker.","type":"string"}},"required":["kind","parent_worker","sub_worker","sub_socket","scope"],"type":"object"}]},"WorkerMaintenanceState":{"enum":["compacting"],"type":"string"},"WorkerRunState":{"enum":["running","pausing","paused","cancelling"],"type":"string"},"WorkerState":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"idle","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"busy","type":"string"},"state":{"$ref":"#/$defs/WorkerBusyState"}},"required":["kind","state"],"type":"object"}]},"WorkerStateSnapshot":{"additionalProperties":false,"properties":{"last_command_id":{"description":"Highest lifecycle command id observed by this controller instance.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"state":{"$ref":"#/$defs/WorkerState"}},"required":["last_command_id","state"],"type":"object"},"WorkerStatus":{"enum":["idle","running","paused","stopped"],"type":"string"},"WorkspaceSubscriptionWorkdir":{"additionalProperties":false,"description":"Workspace-facing Workdir summary. Backend-generated Repository UUIDs never\nenter this DTO; Workspace Server must resolve the required Repository key.","properties":{"repository_key":{"type":"string"},"state":{"type":"string"},"working_directory_id":{"$ref":"#/$defs/SubscriptionWorkdirId"}},"required":["working_directory_id","repository_key","state"],"type":"object"}},"$schema":"https://json-schema.org/draft/2020-12/schema","description":"Direction is explicit at the outer envelope, while concrete message kinds\nremain typed within each lane.","oneOf":[{"additionalProperties":false,"properties":{"frame":{"const":"request","type":"string"},"message":{"$ref":"#/$defs/SubscriptionRequest"},"protocol_version":{"format":"uint16","maximum":65535.0,"minimum":0.0,"type":"integer"}},"required":["frame","message","protocol_version"],"type":"object"},{"additionalProperties":false,"properties":{"frame":{"const":"response","type":"string"},"message":{"$ref":"#/$defs/SubscriptionResponse"},"protocol_version":{"format":"uint16","maximum":65535.0,"minimum":0.0,"type":"integer"}},"required":["frame","message","protocol_version"],"type":"object"},{"additionalProperties":false,"properties":{"frame":{"const":"event","type":"string"},"message":{"$ref":"#/$defs/SubscriptionEvent"},"protocol_version":{"format":"uint16","maximum":65535.0,"minimum":0.0,"type":"integer"}},"required":["frame","message","protocol_version"],"type":"object"},{"additionalProperties":false,"properties":{"frame":{"const":"worker_protocol","type":"string"},"message":{"$ref":"#/$defs/SubscriptionWorkerProtocolMethod"},"protocol_version":{"format":"uint16","maximum":65535.0,"minimum":0.0,"type":"integer"}},"required":["frame","message","protocol_version"],"type":"object"}],"title":"SubscriptionFrame"}; +const subscriptionFrameSchema: unknown = {"$defs":{"Alert":{"additionalProperties":false,"description":"User-facing alert emitted from the Worker layer.\n\nThis is a separate channel from `tracing` (developer logs): entries\nhere are assembled explicitly by the Worker when a condition should be\nsurfaced to the person driving the client. Keep messages short and\nhuman-readable.","properties":{"level":{"$ref":"#/$defs/AlertLevel"},"message":{"type":"string"},"source":{"$ref":"#/$defs/AlertSource"},"timestamp_ms":{"description":"Milliseconds since the Unix epoch.","format":"int64","maximum":9007199254740991.0,"minimum":-9007199254740991.0,"type":"integer"}},"required":["level","source","message","timestamp_ms"],"type":"object"},"AlertLevel":{"enum":["warn","error"],"type":"string"},"AlertSource":{"enum":["worker","engine","compactor","agents_md"],"type":"string"},"CommandEvent":{"oneOf":[{"additionalProperties":false,"properties":{"command_id":{"type":"string"},"kind":{"const":"started","type":"string"},"observed_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"tool_call_id":{"type":["string","null"]}},"required":["kind","command_id","observed_at_ms"],"type":"object"},{"additionalProperties":false,"properties":{"command_id":{"type":"string"},"content":{"type":"string"},"end_offset":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"kind":{"const":"output","type":"string"},"observed_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"start_offset":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"stream":{"$ref":"#/$defs/CommandStream"}},"required":["kind","command_id","stream","start_offset","end_offset","content","observed_at_ms"],"type":"object"},{"additionalProperties":false,"properties":{"command_id":{"type":"string"},"exit_code":{"format":"int32","maximum":2147483647.0,"minimum":-2147483648.0,"type":["integer","null"]},"kind":{"const":"terminal","type":"string"},"observed_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"status":{"$ref":"#/$defs/CommandStatus"},"stderr_end_offset":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"stdout_end_offset":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","command_id","status","stdout_end_offset","stderr_end_offset","observed_at_ms"],"type":"object"}]},"CommandSnapshot":{"additionalProperties":false,"properties":{"command_id":{"type":"string"},"exit_code":{"format":"int32","maximum":2147483647.0,"minimum":-2147483648.0,"type":["integer","null"]},"last_output_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":["integer","null"]},"observed_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"started_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"status":{"$ref":"#/$defs/CommandStatus"},"stderr":{"$ref":"#/$defs/CommandStreamSlice"},"stdout":{"$ref":"#/$defs/CommandStreamSlice"},"tool_call_id":{"type":["string","null"]}},"required":["command_id","status","started_at_ms","observed_at_ms","stdout","stderr"],"type":"object"},"CommandStatus":{"description":"Live provider-owned command status. These values are operational Console\nstate only and are never appended to Worker history.","enum":["running","completed","failed","timed_out","cancelled"],"type":"string"},"CommandStream":{"enum":["stdout","stderr"],"type":"string"},"CommandStreamSlice":{"additionalProperties":false,"properties":{"content":{"type":"string"},"end_offset":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"start_offset":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"truncated":{"type":"boolean"}},"required":["start_offset","end_offset","content","truncated"],"type":"object"},"CompactionLifecycle":{"additionalProperties":false,"description":"Stable parent-owned lifecycle for one compaction run.","properties":{"compaction_id":{"type":"string"},"ended_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":["integer","null"]},"error":{"type":["string","null"]},"internal_worker":{"anyOf":[{"$ref":"#/$defs/InternalWorkerRef"},{"type":"null"}]},"new_segment_id":{"type":["string","null"]},"revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"schema_version":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"started_at_ms":{"description":"Milliseconds since the Unix epoch.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"state":{"$ref":"#/$defs/CompactionLifecycleState"},"summary":{"type":["string","null"]}},"required":["schema_version","compaction_id","revision","state","started_at_ms"],"type":"object"},"CompactionLifecycleState":{"enum":["running","done","failed","interrupted"],"type":"string"},"CompactionPhase":{"enum":["preparing","summarizing","committing"],"type":"string"},"CompactionTrigger":{"enum":["manual","pre_run","request_threshold"],"type":"string"},"CompletionEntry":{"additionalProperties":false,"description":"One completion candidate for a prefix query.\n\n`value` is a path (file kind).","properties":{"is_dir":{"default":false,"type":"boolean"},"value":{"type":"string"}},"required":["value","is_dir"],"type":"object"},"CompletionKind":{"description":"Kind of completion requested by `Method::ListCompletions`.\n\nMirrors the completion prefix sigil: `@` → `File`.","enum":["file"],"type":"string"},"ErrorCode":{"enum":["already_running","not_running","not_paused","provider_error","tool_error","invalid_request","internal"],"type":"string"},"Event":{"oneOf":[{"additionalProperties":false,"description":"Durable Submit acceptance. A `Queued` receipt follows the durable FIFO\ncheckpoint. A `Started` receipt follows the pending activation checkpoint;\nthe corresponding [`Event::UserMessage`] marks the later history commit.\nRepeating the same request id and exact payload returns the same receipt\nwithout appending or activating twice.","properties":{"data":{"additionalProperties":false,"properties":{"disposition":{"$ref":"#/$defs/SubmissionDisposition"},"submission_id":{"type":"string"},"submission_request_id":{"type":"string"}},"required":["submission_request_id","submission_id","disposition"],"type":"object"},"event":{"const":"submission_accepted","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Correlated rejection before durable acceptance.","properties":{"data":{"additionalProperties":false,"properties":{"message":{"type":"string"},"submission_request_id":{"type":"string"}},"required":["submission_request_id","message"],"type":"object"},"event":{"const":"submission_rejected","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Durable notification acceptance. The request identity is persisted with\nthe notification receipt before this event is emitted. Repeating the\nsame request id and exact payload returns the same acknowledgement\nwithout appending the notification twice.","properties":{"data":{"additionalProperties":false,"properties":{"notification_request_id":{"type":"string"}},"required":["notification_request_id"],"type":"object"},"event":{"const":"notification_accepted","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Correlated notification rejection before durable acceptance.","properties":{"data":{"additionalProperties":false,"properties":{"message":{"type":"string"},"notification_request_id":{"type":"string"}},"required":["notification_request_id","message"],"type":"object"},"event":{"const":"notification_rejected","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Revisioned FIFO replacement following enqueue, activation, cancel, or clear.","properties":{"data":{"additionalProperties":false,"properties":{"pending":{"$ref":"#/$defs/PendingSubmissionsSnapshot"}},"required":["pending"],"type":"object"},"event":{"const":"pending_submissions_changed","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A user input message persisted as `LogEntry::AnnotatedUserInput` and\nactivated for a turn. Broadcast to every subscribed client so TUI / GUI\ninstances show the same user line that reconnect snapshots replay.","properties":{"data":{"additionalProperties":false,"properties":{"entry_id":{"description":"Stable durable history identity. Older mixed-version producers omit it.","type":["string","null"]},"segments":{"items":{"$ref":"#/$defs/Segment"},"type":"array"}},"required":["segments"],"type":"object"},"event":{"const":"user_message","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A canonical assistant/tool entry has committed to durable history.\nStreaming events remain the in-flight authority; this event supplies the\nstable identity used to reconcile that live block with paged history.","properties":{"data":{"additionalProperties":false,"properties":{"entry":{"$ref":"#/$defs/SessionSnapshotEntry"}},"required":["entry"],"type":"object"},"event":{"const":"session_entry_committed","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"One agent-injected system item committed to history.\n\nCarries the JSON form of `session_store::SystemItem`. Covers\n`Method::Notify` echoes, child-Worker lifecycle events from\n`Method::WorkerEvent`, `@` resolution payloads, and any future\nClients dispatch on the `kind` tag for typed rendering instead\nof parsing free-text prefixes like `[Notification] …` or\n`[File: …]`.\n\nOne event per `LogEntry::AnnotatedSystemItem` commit. Disk-side and\nwire-side are 1:1.","properties":{"data":{"additionalProperties":false,"properties":{"item":true},"required":["item"],"type":"object"},"event":{"const":"system_item","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A new self-driving cycle has begun (IDLE → active transition).\n\nMarker event for the start of an Invoke range; the range extends\nimplicitly until the next `InvokeStart`. Fires for every accepted\n`Method::Submit` (kind=`UserSend`), `Method::Notify` (kind=`Notify`),\n`Method::WorkerEvent` re-injection (kind=`WorkerEvent`), and any other\nIDLE-breaking trigger. Mid-run interrupts (e.g. hook output,\ntyped system reminder insertion that doesn't break IDLE) do not\nemit `InvokeStart` — they appear as `SystemItem` only.\n\nCarries `kind` only; the payload (user text / notify message /\nworker event body) is delivered separately via the immediately\nfollowing `UserMessage` / `SystemItem` event.","properties":{"data":{"additionalProperties":false,"properties":{"kind":{"$ref":"#/$defs/InvokeKind"}},"required":["kind"],"type":"object"},"event":{"const":"invoke_start","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"One AgentTurn boundary opened. An AgentTurn is a maximal run of\nLLM generation calls whose input messages are identical (i.e.\nretries from network errors / 5xx / stream disconnects collapse\ninto the same AgentTurn). When the input changes (a new tool\nresult lands, a user interrupts, etc.), the next LLM call belongs\nto a new AgentTurn.\n\n`turn` is the AgentTurn index from `Engine::turn_count`.\n\nCurrently retry is not yet implemented (`agen-stream-continuation`)\nso AgentTurn and `LlmCall` fire 1:1, but the contract here is\nthe AgentTurn boundary; consumers that want per-LLM-call signals\nmust subscribe to `LlmCallStart` / `LlmCallEnd` instead.","properties":{"data":{"additionalProperties":false,"properties":{"turn":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["turn"],"type":"object"},"event":{"const":"turn_start","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"AgentTurn closed.","properties":{"data":{"additionalProperties":false,"properties":{"result":{"$ref":"#/$defs/TurnResult"},"turn":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["turn","result"],"type":"object"},"event":{"const":"turn_end","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"One LLM generation call started (1 request → 1 generation, retry\nincluded). Multiple `LlmCall*` pairs may fire inside a single\n`TurnStart` / `TurnEnd` pair when a request is retried; today\nthey fire 1:1 because retry is not implemented.\n\n`llm_call` is the worker-wide running counter from\n`Engine::llm_call_count`.","properties":{"data":{"additionalProperties":false,"properties":{"llm_call":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["llm_call"],"type":"object"},"event":{"const":"llm_call_start","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"LLM generation call ended.","properties":{"data":{"additionalProperties":false,"properties":{"llm_call":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["llm_call"],"type":"object"},"event":{"const":"llm_call_end","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A transport-level LLM request retry has been scheduled.\n\nThis is operational state for clients to render while the worker is\nwaiting in backoff. It is not part of conversation history.","properties":{"data":{"additionalProperties":false,"properties":{"elapsed_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"error":{"type":"string"},"failed_attempt":{"description":"The attempt that just failed. 1 origin.","format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"llm_call":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"max_attempts":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"status":{"format":"uint16","maximum":65535.0,"minimum":0.0,"type":["integer","null"]},"wait_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["llm_call","failed_attempt","max_attempts","wait_ms","elapsed_ms","error"],"type":"object"},"event":{"const":"llm_retry","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Stream generation was interrupted after events had begun and the worker\nis continuing with a follow-up LLM request.","properties":{"data":{"additionalProperties":false,"properties":{"attempt":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"llm_call":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"max_attempts":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"reason":{"type":"string"}},"required":["llm_call","attempt","max_attempts","reason"],"type":"object"},"event":{"const":"llm_continuation","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"text":{"type":"string"}},"required":["text"],"type":"object"},"event":{"const":"text_delta","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"text":{"type":"string"}},"required":["text"],"type":"object"},"event":{"const":"text_done","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A reasoning / thinking block has started.\n\nAlways paired with a `ThinkingDone`. `ThinkingDelta` is optional:\nsome providers (or some configurations) emit thinking metadata\nwithout plaintext, in which case Start → Done arrive with no\ndeltas in between. Multiple thinking blocks per turn are allowed.","properties":{"event":{"const":"thinking_start","type":"string"}},"required":["event"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"text":{"type":"string"}},"required":["text"],"type":"object"},"event":{"const":"thinking_delta","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Thinking block completed. `text` is the full accumulated body\n(empty string when the provider didn't emit plaintext).","properties":{"data":{"additionalProperties":false,"properties":{"text":{"type":"string"}},"required":["text"],"type":"object"},"event":{"const":"thinking_done","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"id":{"type":"string"},"name":{"type":"string"}},"required":["id","name"],"type":"object"},"event":{"const":"tool_call_start","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"id":{"type":"string"},"json":{"type":"string"}},"required":["id","json"],"type":"object"},"event":{"const":"tool_call_args_delta","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"arguments":{"type":"string"},"id":{"type":"string"},"name":{"type":"string"}},"required":["id","name","arguments"],"type":"object"},"event":{"const":"tool_call_done","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"disposition":{"anyOf":[{"$ref":"#/$defs/ToolResultDisposition"},{"type":"null"}]},"id":{"type":"string"},"is_error":{"default":false,"type":"boolean"},"output":{"description":"Full tool output. Absent when the tool chose to return\nsummary-only, or when the result was pruned.","type":["string","null"]},"summary":{"description":"Short human-readable summary. Always present; used by clients\nthat only want a 1-line rendering (e.g. collapsed views).","type":"string"}},"required":["id","summary","is_error"],"type":"object"},"event":{"const":"tool_result","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Token accounting for one LLM request.\n\n`input_tokens` is the prompt prefix occupancy (cache reads /\ncache writes included), as normalised by the worker layer.\n`cache_read_input_tokens` is the cache-hit subset of that\noccupancy; subtracting it yields the \"net upload\" the client\nactually paid full price to send on this request, which is what\nthe TUI status line accumulates per turn.","properties":{"data":{"additionalProperties":false,"properties":{"cache_read_input_tokens":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":["integer","null"]},"input_tokens":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":["integer","null"]},"output_tokens":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":["integer","null"]}},"type":"object"},"event":{"const":"usage","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"result":{"$ref":"#/$defs/RunResult"}},"required":["result"],"type":"object"},"event":{"const":"run_end","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"code":{"$ref":"#/$defs/ErrorCode"},"message":{"type":"string"}},"required":["code","message"],"type":"object"},"event":{"const":"error","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Sent exactly once at the start of every client connection.\n\n`entries` is the session-log mirror at subscribe time, serialised\nas the JSON form of `session_store::LogEntry`. This is the\nbulk-reconstruction lane: clients walk the entries to seed their\nderived view.\n\n`greeting` and `status` accompany the snapshot so clients render\nworker identity and current controller state without an extra round\ntrip.\n\nLive updates after the snapshot arrive through the streaming\nevents (`TextDelta` / `ToolCall*` / `ToolResult` / etc.) plus\nrole-specific entry events (`SegmentRotated` / `SystemItem`) —\nthere is no generic \"every committed entry\" broadcast.","properties":{"data":{"additionalProperties":false,"properties":{"greeting":{"$ref":"#/$defs/Greeting"},"in_flight":{"$ref":"#/$defs/InFlightSnapshot","description":"Unfinished model output that has already streamed in the current\nrun but is not yet represented by committed snapshot entries."},"internal_workers":{"description":"Parent-owned Internal Worker sessions visible to this client.\nService-private Internal Workers are deliberately excluded.","items":{"$ref":"#/$defs/InternalWorkerSnapshot"},"type":"array"},"session":{"$ref":"#/$defs/SessionSnapshot"},"state":{"$ref":"#/$defs/WorkerStateSnapshot","description":"Full revisioned live execution state. `Stopped` remains Runtime\ncatalog authority and is deliberately not represented here."}},"required":["session","greeting","state"],"type":"object"},"event":{"const":"snapshot","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A live event from a parent-owned Internal Worker. The payload reuses the\nnormal Worker event vocabulary while the wrapper carries stable origin\nidentity and a per-child revision fence.","properties":{"data":{"additionalProperties":false,"properties":{"event":{"$ref":"#/$defs/Event"},"revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"worker":{"$ref":"#/$defs/InternalWorkerRef"}},"required":["worker","revision","event"],"type":"object"},"event":{"const":"internal_worker","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Terminal removal fence for one parent-owned Internal Worker session.\n\nClients discard the matching child and descendants, then ignore later\nnested events for this identity until an authoritative snapshot replaces\nthe projection.","properties":{"data":{"additionalProperties":false,"properties":{"revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"worker":{"$ref":"#/$defs/InternalWorkerRef"}},"required":["worker","revision"],"type":"object"},"event":{"const":"internal_worker_removed","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Server-side segment log rotated to a fresh `SegmentStart`.\n\nFires on compaction and on auto-fork when the store head drifts\nA compaction/fork has replaced the authoritative segment. Clients drop\ntheir derived view and reseed from the canonical committed snapshot.","properties":{"data":{"additionalProperties":false,"properties":{"session":{"$ref":"#/$defs/SessionSnapshot"}},"required":["session"],"type":"object"},"event":{"const":"segment_rotated","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Current Worker controller status. Broadcast on every controller-level\ntransition and included in `History` snapshots for late attach.","properties":{"data":{"additionalProperties":false,"properties":{"snapshot":{"$ref":"#/$defs/WorkerStateSnapshot"}},"required":["snapshot"],"type":"object"},"event":{"const":"worker_state","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"acknowledgement":{"$ref":"#/$defs/WorkerCommandAcknowledgement"}},"required":["acknowledgement"],"type":"object"},"event":{"const":"command_acknowledged","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Bounded, provider-owned command telemetry for the live Console. This is\nintentionally not a history entry and is reconstructed from\n`Snapshot.in_flight.commands` after reconnect.","properties":{"data":{"additionalProperties":false,"properties":{"event":{"$ref":"#/$defs/CommandEvent"}},"required":["event"],"type":"object"},"event":{"const":"command","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Reply to `Method::ListCompletions`. Delivered only to the\nrequesting socket (not broadcast). `entries` is empty when no\ncandidates match or when the requested kind has no resolver\nwired up.","properties":{"data":{"additionalProperties":false,"properties":{"entries":{"items":{"$ref":"#/$defs/CompletionEntry"},"type":"array"},"kind":{"$ref":"#/$defs/CompletionKind"}},"required":["kind","entries"],"type":"object"},"event":{"const":"completions","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Reply to `Method::ListRewindTargets`. Clients should only open a picker\nin response to their own pending request; the event may be broadcast.","properties":{"data":{"additionalProperties":false,"properties":{"head_entries":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"targets":{"items":{"$ref":"#/$defs/RewindTarget"},"type":"array"}},"required":["head_entries","targets"],"type":"object"},"event":{"const":"rewind_targets","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A rewind has truncated the authoritative session. `session` is the\nretained canonical snapshot clients should use to reseed display state.","properties":{"data":{"additionalProperties":false,"properties":{"input":{"items":{"$ref":"#/$defs/Segment"},"type":"array"},"session":{"$ref":"#/$defs/SessionSnapshot"},"summary":{"$ref":"#/$defs/RewindSummary"}},"required":["session","input","summary"],"type":"object"},"event":{"const":"rewind_applied","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Reply to `Method::ListWorkers`. Payload is a stable JSON value so the Worker\ncrate can evolve discovery fields without introducing a protocol\ndependency on session-store.","properties":{"data":{"additionalProperties":false,"properties":{"workers":true},"required":["workers"],"type":"object"},"event":{"const":"workers_listed","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Reply to `Method::RestoreWorker`.","properties":{"data":{"additionalProperties":false,"properties":{"result":true},"required":["result"],"type":"object"},"event":{"const":"worker_restored","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Reply to `Method::RegisterPeer`.","properties":{"data":{"additionalProperties":false,"properties":{"result":true},"required":["result"],"type":"object"},"event":{"const":"peer_registered","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"$ref":"#/$defs/Alert"},"event":{"const":"alert","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Latest memory extract/consolidation lifecycle event for UI observability.\n\nThis is not part of LLM history or prompt context; clients may display it\nbriefly as operational status.","properties":{"data":{"$ref":"#/$defs/MemoryWorkerEvent"},"event":{"const":"memory_worker","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Runtime-only compaction progress. `None` clears the current status.\nThis never enters Session history and carries no operation or Segment identity.","properties":{"data":{"additionalProperties":false,"properties":{"compaction":{"anyOf":[{"$ref":"#/$defs/InFlightCompaction"},{"type":"null"}]}},"type":"object"},"event":{"const":"compaction_progress","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Legacy compaction lifecycle event retained for wire read compatibility.","properties":{"data":{"additionalProperties":false,"properties":{"lifecycle":{"$ref":"#/$defs/CompactionLifecycle"}},"required":["lifecycle"],"type":"object"},"event":{"const":"compact_start","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Compaction completed and the session was rotated.","properties":{"data":{"additionalProperties":false,"properties":{"lifecycle":{"$ref":"#/$defs/CompactionLifecycle"}},"required":["lifecycle"],"type":"object"},"event":{"const":"compact_done","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Compaction failed or was cancelled. The session is unchanged.","properties":{"data":{"additionalProperties":false,"properties":{"lifecycle":{"$ref":"#/$defs/CompactionLifecycle"}},"required":["lifecycle"],"type":"object"},"event":{"const":"compact_failed","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"event":{"const":"shutdown","type":"string"}},"required":["event"],"type":"object"}]},"EventSubscriptionSelector":{"oneOf":[{"additionalProperties":false,"properties":{"topic":{"const":"runtime_workers","type":"string"}},"required":["topic"],"type":"object"},{"additionalProperties":false,"properties":{"topic":{"const":"worker_lifecycle","type":"string"},"worker_ids":{"$ref":"#/$defs/SubscriptionWorkerIds"}},"required":["topic","worker_ids"],"type":"object"},{"additionalProperties":false,"properties":{"runtime_id":{"type":["string","null"]},"topic":{"const":"worker_protocol","type":"string"},"worker_id":{"$ref":"#/$defs/SubscriptionWorkerId"}},"required":["topic","worker_id"],"type":"object"},{"additionalProperties":false,"description":"Server-derived Workspace projection. Workspace identity comes from the\nauthenticated connection and is deliberately absent from this selector.","properties":{"topic":{"const":"workspace_workers","type":"string"}},"required":["topic"],"type":"object"},{"additionalProperties":false,"description":"Typed extension point for the later Workdir subscription slice.","properties":{"topic":{"const":"workspace_workdirs","type":"string"}},"required":["topic"],"type":"object"}]},"Greeting":{"additionalProperties":false,"description":"Worker self-description rendered by the TUI when a session starts empty.\n\nBuilt once in the Worker controller from the resolved manifest and\ntransmitted alongside `Event::Snapshot` so clients don't need\ntheir own view of the manifest.","properties":{"context_tokens":{"default":0,"description":"Estimated current session context tokens at connect time.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"context_window":{"default":0,"description":"Model context window in tokens. Always filled by the Worker greeting.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"cwd":{"type":"string"},"model":{"type":"string"},"provider":{"type":"string"},"scope_summary":{"type":"string"},"tools":{"items":{"type":"string"},"type":"array"},"worker_name":{"type":"string"}},"required":["worker_name","cwd","provider","model","scope_summary","tools","context_tokens","context_window"],"type":"object"},"InFlightBlock":{"oneOf":[{"additionalProperties":false,"properties":{"finished":{"type":"boolean"},"kind":{"const":"text","type":"string"},"text":{"type":"string"}},"required":["kind","text"],"type":"object"},{"additionalProperties":false,"properties":{"finished":{"type":"boolean"},"kind":{"const":"thinking","type":"string"},"text":{"type":"string"}},"required":["kind","text"],"type":"object"},{"additionalProperties":false,"properties":{"args":{"type":"string"},"id":{"type":"string"},"kind":{"const":"tool_call","type":"string"},"name":{"type":"string"},"state":{"$ref":"#/$defs/InFlightToolCallState"}},"required":["kind","id","name","args"],"type":"object"}]},"InFlightCompaction":{"additionalProperties":false,"properties":{"phase":{"$ref":"#/$defs/CompactionPhase"},"started_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"trigger":{"$ref":"#/$defs/CompactionTrigger"}},"required":["phase","started_at_ms","trigger"],"type":"object"},"InFlightSnapshot":{"additionalProperties":false,"description":"Unfinished model output and active command state included in\n`Event::Snapshot` for clients that attach while work is still streaming.\n\nThese blocks are presentation state only: they are reconstructed from the\nactive Worker controller and must not be treated as committed assistant\nhistory. Finalized assistant items continue to come from ordinary snapshot\nentries.","properties":{"blocks":{"items":{"$ref":"#/$defs/InFlightBlock"},"type":"array"},"commands":{"items":{"$ref":"#/$defs/CommandSnapshot"},"type":"array"},"compaction":{"anyOf":[{"$ref":"#/$defs/InFlightCompaction"},{"type":"null"}],"description":"The currently running compaction, if any.\n\nThis is lifecycle progress only. Candidate history and the staged\nSegment remain private until the Segment is activated atomically."}},"type":"object"},"InFlightToolCallState":{"enum":["pending","streaming_args","done"],"type":"string"},"InternalWorkerKind":{"description":"Presentation category for an Internal Worker exposed through its parent's\nprotocol stream. Internal Workers never become independently addressable\nprotocol subjects.","oneOf":[{"enum":["sub_worker"],"type":"string"},{"additionalProperties":false,"properties":{"service":{"additionalProperties":false,"properties":{"kind":{"type":"string"}},"required":["kind"],"type":"object"}},"required":["service"],"type":"object"}]},"InternalWorkerRef":{"additionalProperties":false,"description":"Stable presentation identity for one parent-owned Internal Worker session.\n`name` is display-only; `session_id` is the identity used by clients.","properties":{"kind":{"$ref":"#/$defs/InternalWorkerKind"},"name":{"type":"string"},"parent_session_id":{"type":["string","null"]},"session_id":{"type":"string"}},"required":["session_id","name","kind"],"type":"object"},"InternalWorkerSnapshot":{"additionalProperties":false,"description":"Reconnect state for one visible Internal Worker. The revision fences live\n`Event::InternalWorker` updates that raced with parent snapshot assembly.","properties":{"error":{"type":["string","null"]},"in_flight":{"$ref":"#/$defs/InFlightSnapshot"},"internal_workers":{"items":{"$ref":"#/$defs/InternalWorkerSnapshot"},"type":"array"},"revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"session":{"$ref":"#/$defs/SessionSnapshot"},"status":{"$ref":"#/$defs/WorkerStatus","default":"idle"},"worker":{"$ref":"#/$defs/InternalWorkerRef"}},"required":["worker","revision","session","status"],"type":"object"},"InvokeKind":{"description":"Kind of trigger that opened a new Invoke (IDLE → active) range.\n\nOne Invoke groups all entries from this trigger up to the next\n`Invoke` marker. The kind is the only payload — content (user text,\nnotify message, worker event body) is delivered by the immediately\nfollowing Turn entry, not by the marker itself.","oneOf":[{"const":"user_send","description":"`Method::Submit` — a user submission.","type":"string"},{"const":"notify","description":"`Method::Notify` — free-text notification injected into history.","type":"string"},{"const":"worker_event","description":"`Method::WorkerEvent` — typed lifecycle report from a child Worker.","type":"string"},{"const":"system_reminder","description":"A typed system reminder that crosses an IDLE boundary (mid-run\nreminders that don't break IDLE are SystemItem-only and do not\nopen a new Invoke).","type":"string"},{"const":"wakeup","description":"Cron / RemoteTrigger style scheduled wake-up. Reserved; no\nproducer is wired today but the variant is part of the initial\nset so future schedulers don't have to amend the wire enum.","type":"string"}]},"MemoryWorkerEvent":{"additionalProperties":false,"properties":{"message":{"description":"Human-readable compact form for actionbar rendering.","type":"string"},"reason":{"type":"string"},"run_id":{"type":"string"},"status":{"type":"string"},"timestamp_ms":{"description":"Milliseconds since the Unix epoch.","format":"int64","maximum":9007199254740991.0,"minimum":-9007199254740991.0,"type":"integer"},"trigger":{"type":"string"},"worker":{"type":"string"}},"required":["worker","status","run_id","trigger","reason","message","timestamp_ms"],"type":"object"},"Method":{"oneOf":[{"additionalProperties":false,"description":"Durably accept typed input for immediate activation or the session FIFO.\nRunning or Paused Workers queue Submit for a later turn; use Notify for\nadvisory information to incorporate into work already in progress.\n\n`submission_request_id` is generated by the authenticated caller and is\nused only for idempotent retry. Worker allocates the durable\n`submission_id` returned by [`Event::SubmissionAccepted`].","properties":{"method":{"const":"submit","type":"string"},"params":{"additionalProperties":false,"properties":{"input":{"items":{"$ref":"#/$defs/Segment"},"type":"array"},"submission_request_id":{"type":"string"}},"required":["submission_request_id","input"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Human-readable text injected into the target Worker's LLM context as a\nnon-blocking system message. Accepted notifications are delivered at the\nnext between-turn boundary, or start a coalesced notification run while\nthe Worker is idle. No side effects beyond LLM context; use\n`WorkerEvent` for typed lifecycle reports.","properties":{"method":{"const":"notify","type":"string"},"params":{"additionalProperties":false,"properties":{"message":{"type":"string"},"notification_request_id":{"type":"string"}},"required":["notification_request_id","message"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Typed lifecycle report from a child Worker to its direct parent.","properties":{"method":{"const":"worker_event","type":"string"},"params":{"$ref":"#/$defs/WorkerEvent"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Return the authoritative FIFO summary without exposing queued payloads.","properties":{"method":{"const":"list_pending_submissions","type":"string"}},"required":["method"],"type":"object"},{"additionalProperties":false,"description":"Remove one queued submission. Running or already activated submissions\nare immutable and therefore cannot be cancelled here.","properties":{"method":{"const":"cancel_pending_submission","type":"string"},"params":{"additionalProperties":false,"properties":{"expected_revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"submission_id":{"type":"string"}},"required":["submission_id","expected_revision"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Remove every queued submission while preserving the active run.","properties":{"method":{"const":"clear_pending_submissions","type":"string"},"params":{"additionalProperties":false,"properties":{"expected_revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["expected_revision"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Activate the next queued submission while the Worker is idle. This is an\nexplicit recovery operation and never resumes a paused run implicitly.","properties":{"method":{"const":"continue_pending","type":"string"},"params":{"additionalProperties":false,"properties":{"expected_head_id":{"type":"string"},"expected_revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["expected_revision","expected_head_id"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"properties":{"method":{"const":"resume","type":"string"},"params":{"additionalProperties":false,"properties":{"command":{"$ref":"#/$defs/WorkerCommandEnvelope"}},"required":["command"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"properties":{"method":{"const":"cancel","type":"string"},"params":{"additionalProperties":false,"properties":{"command":{"$ref":"#/$defs/WorkerCommandEnvelope"}},"required":["command"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Stop the in-flight turn and transition to `Paused`.\n\nUnlike `Cancel` (which discards and returns to `Idle`), a paused\nWorker can resume the interrupted work via `Resume`, or accept a\nfresh `Submit` (orphan `tool_use` items are closed with a\nsynthetic tool result before the new user message is appended).","properties":{"method":{"const":"pause","type":"string"},"params":{"additionalProperties":false,"properties":{"command":{"$ref":"#/$defs/WorkerCommandEnvelope"}},"required":["command"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Request an explicit compaction while the Worker is otherwise idle.\n\nThis is a typed control method: clients must not send `compact` as a\n`Method::Submit` user message.","properties":{"method":{"const":"compact","type":"string"},"params":{"additionalProperties":false,"properties":{"command":{"$ref":"#/$defs/WorkerCommandEnvelope"}},"required":["command"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Ask the Worker to list valid rewind targets from its authoritative session log.","properties":{"method":{"const":"list_rewind_targets","type":"string"}},"required":["method"],"type":"object"},{"additionalProperties":false,"description":"Truncate the current session back to the selected rewind target and\nreturn the selected user input to the client composer.","properties":{"method":{"const":"rewind_to","type":"string"},"params":{"additionalProperties":false,"properties":{"expected_head_entries":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"target":{"$ref":"#/$defs/RewindTargetId"}},"required":["target","expected_head_entries"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"properties":{"method":{"const":"shutdown","type":"string"},"params":{"additionalProperties":false,"properties":{"command":{"$ref":"#/$defs/WorkerCommandEnvelope"}},"required":["command"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Request a list of completion candidates from the Worker.\n\nReply is sent on the same socket as `Event::Completions` (not\nbroadcast). The IPC server handles this directly and writes\nthe response straight back to the requesting socket.","properties":{"method":{"const":"list_completions","type":"string"},"params":{"additionalProperties":false,"properties":{"kind":{"$ref":"#/$defs/CompletionKind"},"prefix":{"type":"string"}},"required":["kind","prefix"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"List Workers visible to this Worker from durable Worker state and the spawned-child\nregistry. This is not a host-wide Worker universe query.","properties":{"method":{"const":"list_workers","type":"string"}},"required":["method"],"type":"object"},{"additionalProperties":false,"description":"Restore a visible stopped/restorable Worker, or report that it is already\nlive. Missing state and not-visible state are distinct errors.","properties":{"method":{"const":"restore_worker","type":"string"},"params":{"additionalProperties":false,"properties":{"name":{"type":"string"}},"required":["name"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Register another existing Worker as a reciprocal peer of this Worker.\n\nThis is metadata/control state only: it does not ask the target's live\ncontroller for consent, and it must not grant delegated scope,\nspawned-child ownership, output cursors, or child lifecycle authority.","properties":{"method":{"const":"register_peer","type":"string"},"params":{"additionalProperties":false,"properties":{"name":{"type":"string"}},"required":["name"],"type":"object"}},"required":["method","params"],"type":"object"}]},"PasteArtifactAvailability":{"enum":["available","unavailable","integrity_failed"],"type":"string"},"PasteArtifactMediaType":{"description":"One typed piece of a user submission.\n\n`Method::Submit` and `Event::UserMessage` carry `Vec`. Dumb\nclients (CLI piping, scripts) only need to produce a single\n`Segment::Text`; richer clients (TUI / GUI) construct typed atoms\n(paste chips, file refs) and\nsend them through directly so the Worker side never has to re-parse a\nflattened string.\n\nForward compat: payloads with unknown `kind` deserialize to\n`Segment::Unknown`. Worker treats this the same as known-but-unresolved\nvariants — emits an alert and inserts a `[unknown input segment]`\nplaceholder into the LLM context so neither user nor LLM is blind to\nthe dropped intent.","enum":["text_plain_utf8"],"type":"string"},"PasteArtifactRef":{"additionalProperties":false,"description":"Session-owned reference to a large pasted-input artifact.\n\nThe reference contains only bounded integrity and provenance metadata. The\nartifact body remains in session storage and is available to the model only\nthrough the scoped paste-artifact tools installed by Worker.","properties":{"artifact_id":{"type":"string"},"availability":{"$ref":"#/$defs/PasteArtifactAvailability","description":"Availability observed when this immutable reference was committed.\nReads revalidate storage and integrity rather than trusting this field."},"byte_len":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"char_count":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"created_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"line_count":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"media_type":{"$ref":"#/$defs/PasteArtifactMediaType"},"sha256":{"type":"string"},"source_entry_id":{"type":"string"}},"required":["artifact_id","created_at_ms","media_type","availability","byte_len","char_count","line_count","sha256","source_entry_id"],"type":"object"},"PendingSubmissionSummary":{"additionalProperties":false,"description":"Bounded public projection of one pending submission. Only a short display\npreview is exposed; full payload segments and provenance remain in the log.","properties":{"accepted_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"byte_len":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"preview":{"description":"Single-line display text, at most 240 Unicode scalars plus an ellipsis.\nAbsent on older runtimes. Never used to reconstruct the submitted input.","type":["string","null"]},"segment_count":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"submission_id":{"type":"string"}},"required":["submission_id","accepted_at_ms","segment_count","byte_len"],"type":"object"},"PendingSubmissionsSnapshot":{"additionalProperties":false,"description":"Revisioned session-owned FIFO projection used by snapshots and live events.","properties":{"head_id":{"default":null,"type":["string","null"]},"notification_count":{"default":0,"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"notification_previews":{"description":"Ordered display-only previews of waiting notifications, each bounded like\nPendingSubmissionSummary::preview. Older runtimes omit this field.","items":{"type":"string"},"type":"array"},"revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"submissions":{"default":[],"items":{"$ref":"#/$defs/PendingSubmissionSummary"},"type":"array"}},"required":["revision","head_id","notification_count","submissions"],"type":"object"},"Permission":{"description":"Permission lattice used by [`ScopeRule`].\n\nThe derived `Ord` instance follows declaration order, so\n`Read < Write`. Allow rules grant the stated level (and by extension\neverything below); deny rules cap the effective level **strictly\nbelow** the stated level.","enum":["read","write"],"type":"string"},"RewindSummary":{"additionalProperties":false,"properties":{"discarded_entries":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"tool_side_effect_warning":{"type":"boolean"},"truncated_to_entries":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["truncated_to_entries","discarded_entries","tool_side_effect_warning"],"type":"object"},"RewindTarget":{"additionalProperties":false,"properties":{"disabled_reason":{"type":["string","null"]},"eligible":{"type":"boolean"},"expected_head_entries":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"id":{"$ref":"#/$defs/RewindTargetId"},"preview":{"type":"string"},"timestamp_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":["integer","null"]},"truncate_entries":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"turn_index":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"warning":{"type":["string","null"]}},"required":["id","expected_head_entries","truncate_entries","turn_index","preview","eligible"],"type":"object"},"RewindTargetId":{"additionalProperties":false,"properties":{"segment_id":{"type":"string"},"user_input_entry_index":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["segment_id","user_input_entry_index"],"type":"object"},"RunFailureKind":{"enum":["engine","compaction"],"type":"string"},"RunResult":{"oneOf":[{"enum":["finished","paused","limit_reached"],"type":"string"},{"const":"cancelled","description":"The logical Run ended because the user intentionally cancelled it after\nobservable work. Unlike `RolledBack`, committed history is retained.","type":"string"},{"const":"rolled_back","description":"The accepted Method::Submit produced no assistant/tool output before\nuser interruption, so the Worker rolled the submit-time turn state back\nto its pre-submit snapshot. Clients should treat the Worker as Idle and\nrestore the just-submitted input into the editable composer if desired.","type":"string"}]},"RunResumeSource":{"enum":["compaction","pause"],"type":"string"},"RunYieldReason":{"enum":["compaction"],"type":"string"},"ScopeRule":{"additionalProperties":false,"description":"A single allow or deny rule inside a scope configuration.","properties":{"permission":{"$ref":"#/$defs/Permission","description":"Permission level this rule grants (allow) or caps strictly below\n(deny)."},"recursive":{"default":true,"description":"When `false`, the rule only matches the target itself and its\ndirect children. Defaults to `true`.","type":"boolean"},"symlink_policy":{"$ref":"#/$defs/SymlinkPolicy","default":"resolved","description":"Which path identity an allow rule uses when symbolic links are\nencountered. Deny rules always inspect both identities."},"target":{"description":"Target path. Must be absolute by the time a `Scope` is built from\nthis rule — relative paths are resolved per-layer against the\nmanifest file's directory (cwd for overlay layers) before cascade\nmerge.","type":"string"}},"required":["target","permission","recursive","symlink_policy"],"type":"object"},"Segment":{"oneOf":[{"additionalProperties":false,"description":"Free-form text. The fallback every client can produce.","properties":{"content":{"type":"string"},"kind":{"const":"text","type":"string"}},"required":["kind","content"],"type":"object"},{"additionalProperties":false,"description":"Bracketed-paste capture from a TUI-style client. `id`, `chars`\nand `lines` carry the metadata needed to re-render a\n`[Clipboard #N | X chars, Y lines]` chip in `Event::UserMessage`\nre-broadcast.","properties":{"chars":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"content":{"type":"string"},"id":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"kind":{"const":"paste","type":"string"},"lines":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"}},"required":["kind","id","chars","lines","content"],"type":"object"},{"additionalProperties":false,"description":"Internal reference produced when Worker stores a large `Paste` before\ncommitting input. Clients may receive this in history/event projections;\nthe body is intentionally absent.","properties":{"artifact":{"$ref":"#/$defs/PasteArtifactRef"},"kind":{"const":"paste_artifact","type":"string"}},"required":["kind","artifact"],"type":"object"},{"additionalProperties":false,"description":"Client-local file uploaded into the owning Worker session before submit.\nThe Session Log stores only this immutable reference, never file bytes or\nthe client's local path.","properties":{"file":{"$ref":"#/$defs/UploadedFileRef"},"kind":{"const":"uploaded_file","type":"string"}},"required":["kind","file"],"type":"object"},{"additionalProperties":false,"description":"`@` file-system reference. Worker resolves readable files to\n`[File: ]` attachments and readable normal directories to shallow\n`[Dir: ]` listings; the flattened user text keeps the literal\n`@` placeholder either way.","properties":{"kind":{"const":"file_ref","type":"string"},"path":{"type":"string"}},"required":["kind","path"],"type":"object"},{"additionalProperties":false,"description":"Source-qualified request for the host to start a Flow before committing\nthis user input. Runtime resolves it through Workspace authority and\nreplaces it with the entered state's instructions before Worker-side\ninput resolution.","properties":{"kind":{"const":"flow","type":"string"},"selector":{"type":"string"}},"required":["kind","selector"],"type":"object"},{"additionalProperties":false,"description":"Unknown variant from a newer client. Worker treats this as an\nunresolved input — surfaces an alert and inserts a placeholder.\nRound-trip is lossy: re-serializing yields `{\"kind\":\"unknown\"}`.","properties":{"kind":{"const":"unknown","type":"string"}},"required":["kind"],"type":"object"}]},"SessionContentPart":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"text","type":"string"},"text":{"type":"string"}},"required":["kind","text"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"refusal","type":"string"},"refusal":{"type":"string"}},"required":["kind","refusal"],"type":"object"}]},"SessionEntryProvenance":{"enum":["human_input","worker_input","flow_instruction","backend_instruction","model_output","tool_output","derived_summary","legacy_unknown"],"type":"string"},"SessionMessageRole":{"enum":["user","assistant"],"type":"string"},"SessionSnapshot":{"additionalProperties":false,"description":"Canonical, storage-independent projection of committed session history.\n\nWorker protocols expose this DTO instead of append-log records. New\nstorage variants can therefore be added without teaching every client how\nto replay the durable log format.","properties":{"entries":{"items":{"$ref":"#/$defs/SessionSnapshotEntry"},"type":"array"},"pending_submissions":{"$ref":"#/$defs/PendingSubmissionsSnapshot","default":{"head_id":null,"notification_count":0,"revision":0,"submissions":[]}}},"required":["entries","pending_submissions"],"type":"object"},"SessionSnapshotEntry":{"oneOf":[{"additionalProperties":false,"properties":{"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"kind":{"const":"user_input","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"segments":{"items":{"$ref":"#/$defs/Segment"},"type":"array"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","segments","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"properties":{"content":{"items":{"$ref":"#/$defs/SessionContentPart"},"type":"array"},"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"kind":{"const":"message","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"role":{"$ref":"#/$defs/SessionMessageRole"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","role","content","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"properties":{"arguments":{"type":"string"},"call_id":{"type":"string"},"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"kind":{"const":"tool_call","type":"string"},"name":{"type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","call_id","name","arguments","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"properties":{"attachments":{"items":{"$ref":"#/$defs/SessionToolAttachment"},"type":"array"},"call_id":{"type":"string"},"content":{"type":["string","null"]},"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"is_error":{"type":"boolean"},"kind":{"const":"tool_result","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"summary":{"type":"string"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","call_id","summary","is_error","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"properties":{"content":{"type":"string"},"data":true,"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"item_kind":{"type":"string"},"kind":{"const":"system_item","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","item_kind","content","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"description":"Durable, non-terminal control return from one logical Run. This is not a\ncompleted run and must not be used to infer the current Worker state.","properties":{"active_run_turn_count":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"kind":{"const":"run_yielded","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"reason":{"$ref":"#/$defs/RunYieldReason"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","reason","active_run_turn_count","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"description":"Durable boundary immediately before Engine execution resumes. The source\ndistinguishes automatic compaction continuation from an intentional\nuser-pause resume; it does not prove that later model output occurred.","properties":{"active_run_turn_count":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"kind":{"const":"run_resumed","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"source":{"$ref":"#/$defs/RunResumeSource"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","source","active_run_turn_count","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"description":"Durable terminal for an intentional cancellation. Consumers must not\npresent this lifecycle result as an execution failure.","properties":{"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"kind":{"const":"run_cancelled","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"properties":{"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"failure":{"anyOf":[{"$ref":"#/$defs/RunFailureKind"},{"type":"null"}]},"kind":{"const":"run_error","type":"string"},"message":{"type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","message","entry_id","timestamp","provenance"],"type":"object"}]},"SessionToolAttachment":{"additionalProperties":false,"properties":{"data_base64":{"description":"Base64-encoded durable attachment body. Public snapshots preserve the\ncommitted multimodal value instead of replacing it with placeholder text.","type":"string"},"media_type":{"type":"string"}},"required":["media_type","data_base64"],"type":"object"},"SubmissionDisposition":{"description":"Durable acceptance result for one idempotent Submit request.","oneOf":[{"enum":["started"],"type":"string"},{"const":"queued","description":"Input is durably held in the session FIFO until eligible for activation.","type":"string"}]},"SubscriptionEvent":{"oneOf":[{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"payload":{"$ref":"#/$defs/SubscriptionEventPayload"},"subscription_id":{"$ref":"#/$defs/SubscriptionId"}},"required":["subscription_id","payload"],"type":"object"},"event":{"const":"event","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"code":{"$ref":"#/$defs/SubscriptionTerminationCode"},"message":{"type":"string"},"subscription_id":{"$ref":"#/$defs/SubscriptionId"}},"required":["subscription_id","code","message"],"type":"object"},"event":{"const":"subscription_closed","type":"string"}},"required":["event","data"],"type":"object"}]},"SubscriptionEventPayload":{"oneOf":[{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"worker":{"$ref":"#/$defs/SubscriptionWorker"}},"required":["worker"],"type":"object"},"event":{"const":"worker_upserted","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"runtime_id":{"type":["string","null"]},"worker_id":{"$ref":"#/$defs/SubscriptionWorkerId"}},"required":["worker_id"],"type":"object"},"event":{"const":"worker_removed","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"event":{"$ref":"#/$defs/Event"},"worker_id":{"$ref":"#/$defs/SubscriptionWorkerId"}},"required":["worker_id","event"],"type":"object"},"event":{"const":"worker_protocol","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"workdir":{"$ref":"#/$defs/WorkspaceSubscriptionWorkdir"}},"required":["workdir"],"type":"object"},"event":{"const":"workdir_upserted","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"working_directory_id":{"$ref":"#/$defs/SubscriptionWorkdirId"}},"required":["working_directory_id"],"type":"object"},"event":{"const":"workdir_removed","type":"string"}},"required":["event","data"],"type":"object"}]},"SubscriptionId":{"type":"string"},"SubscriptionRejectionCode":{"enum":["invalid_request","unsupported_protocol_version","unsupported_selector","unauthorized","resource_not_found","capacity_exceeded","internal"],"type":"string"},"SubscriptionRequest":{"oneOf":[{"additionalProperties":false,"properties":{"method":{"const":"subscribe_events","type":"string"},"params":{"additionalProperties":false,"properties":{"request_id":{"$ref":"#/$defs/SubscriptionRequestId"},"selector":{"$ref":"#/$defs/EventSubscriptionSelector"}},"required":["request_id","selector"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"properties":{"method":{"const":"unsubscribe_events","type":"string"},"params":{"additionalProperties":false,"properties":{"request_id":{"$ref":"#/$defs/SubscriptionRequestId"},"subscription_id":{"$ref":"#/$defs/SubscriptionId"}},"required":["request_id","subscription_id"],"type":"object"}},"required":["method","params"],"type":"object"}]},"SubscriptionRequestId":{"type":"string"},"SubscriptionResponse":{"oneOf":[{"additionalProperties":false,"properties":{"payload":{"additionalProperties":false,"properties":{"request_id":{"$ref":"#/$defs/SubscriptionRequestId"},"selector":{"$ref":"#/$defs/EventSubscriptionSelector"},"snapshot":{"$ref":"#/$defs/SubscriptionSnapshot"},"subscription_id":{"$ref":"#/$defs/SubscriptionId"}},"required":["request_id","subscription_id","selector","snapshot"],"type":"object"},"result":{"const":"subscribed","type":"string"}},"required":["result","payload"],"type":"object"},{"additionalProperties":false,"properties":{"payload":{"additionalProperties":false,"properties":{"request_id":{"$ref":"#/$defs/SubscriptionRequestId"},"subscription_id":{"$ref":"#/$defs/SubscriptionId"}},"required":["request_id","subscription_id"],"type":"object"},"result":{"const":"unsubscribed","type":"string"}},"required":["result","payload"],"type":"object"},{"additionalProperties":false,"properties":{"payload":{"additionalProperties":false,"properties":{"code":{"$ref":"#/$defs/SubscriptionRejectionCode"},"message":{"type":"string"},"request_id":{"$ref":"#/$defs/SubscriptionRequestId"},"subscription_id":{"anyOf":[{"$ref":"#/$defs/SubscriptionId"},{"type":"null"}]}},"required":["request_id","code","message"],"type":"object"},"result":{"const":"subscription_rejected","type":"string"}},"required":["result","payload"],"type":"object"}]},"SubscriptionSnapshot":{"oneOf":[{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"workers":{"items":{"$ref":"#/$defs/SubscriptionWorker"},"type":"array"}},"required":["workers"],"type":"object"},"topic":{"const":"workers","type":"string"}},"required":["topic","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"events":{"items":{"$ref":"#/$defs/Event"},"type":"array"},"worker_id":{"$ref":"#/$defs/SubscriptionWorkerId"}},"required":["worker_id","events"],"type":"object"},"topic":{"const":"worker_protocol","type":"string"}},"required":["topic","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"workdirs":{"items":{"$ref":"#/$defs/WorkspaceSubscriptionWorkdir"},"type":"array"}},"required":["workdirs"],"type":"object"},"topic":{"const":"workspace_workdirs","type":"string"}},"required":["topic","data"],"type":"object"}]},"SubscriptionTerminationCode":{"enum":["lagged","resource_gone","unauthorized","server_shutdown"],"type":"string"},"SubscriptionWorkdirId":{"type":"string"},"SubscriptionWorker":{"additionalProperties":false,"properties":{"availability":{"$ref":"#/$defs/SubscriptionWorkerAvailability","default":"observed","description":"Freshness of the Runtime-backed observation carried by this projection.\n`Unavailable` preserves catalog membership without claiming that execution stopped."},"display_name":{"type":["string","null"]},"has_running_internal_workers":{"default":false,"type":"boolean"},"job":{"anyOf":[{"$ref":"#/$defs/SubscriptionWorkerJob"},{"type":"null"}],"description":"Backend-owned Job binding. Runtime producers leave this unset; the\nWorkspace projection enriches both snapshots and live updates."},"profile":{"type":["string","null"]},"resource_key":{"description":"Workspace-scoped canonical resource key. Runtime producers leave this unset;\nWorkspace-facing projections must populate it before publishing the Worker.","type":["string","null"]},"runtime_id":{"description":"Set by the Workspace Server when projecting a Runtime-owned Worker to clients.\nRuntime producers leave this unset because the connection identifies the Runtime.","type":["string","null"]},"state":{"$ref":"#/$defs/SubscriptionWorkerState","description":"Runtime catalog lifecycle compatibility projection; not foreground-state authority."},"workdir_attachments":{"items":{"$ref":"#/$defs/SubscriptionWorkerWorkdirAttachment"},"type":"array"},"worker_id":{"$ref":"#/$defs/SubscriptionWorkerId"},"worker_state":{"anyOf":[{"$ref":"#/$defs/WorkerStateSnapshot"},{"type":"null"}],"description":"Latest foreground state observed from the Worker. This remains\nabsent until an authoritative Worker snapshot/event has been applied."},"workspace_id":{"type":["string","null"]}},"required":["worker_id","state","availability","has_running_internal_workers"],"type":"object"},"SubscriptionWorkerAvailability":{"description":"Whether the Workspace Server has a current Runtime observation for a Worker.\n\n`Unavailable` deliberately does not imply `Stopped`: it means the last observed\nlifecycle state is retained while the Runtime observation source is unavailable.","enum":["observed","unavailable"],"type":"string"},"SubscriptionWorkerId":{"type":"string"},"SubscriptionWorkerIds":{"description":"A bounded, canonical worker-id set. Ordering on the wire and in equality/hash\nsemantics is lexical, so equivalent selectors aggregate to one upstream key.","items":{"$ref":"#/$defs/SubscriptionWorkerId"},"type":"array"},"SubscriptionWorkerJob":{"additionalProperties":false,"properties":{"attempt_id":{"type":"string"},"job_id":{"type":"string"},"purpose":{"type":"string"}},"required":["job_id","attempt_id","purpose"],"type":"object"},"SubscriptionWorkerProtocolMethod":{"additionalProperties":false,"properties":{"method":{"$ref":"#/$defs/Method"},"subscription_id":{"$ref":"#/$defs/SubscriptionId"}},"required":["subscription_id","method"],"type":"object"},"SubscriptionWorkerState":{"enum":["idle","running","paused","stopped"],"type":"string"},"SubscriptionWorkerWorkdirAttachment":{"additionalProperties":false,"properties":{"alias":{"description":"Stable Worker-local routing alias.","type":"string"},"repository_key":{"type":["string","null"]},"working_directory_id":{"$ref":"#/$defs/SubscriptionWorkdirId"}},"required":["alias","working_directory_id"],"type":"object"},"SymlinkPolicy":{"description":"Symbolic-link identity used by one filesystem allow rule.\n\n`Resolved` is the least authority and the default: access is matched\nagainst the provider-resolved target. `Logical` intentionally grants the\npath as presented through the Workdir, even when it aliases another target.","enum":["resolved","logical"],"type":"string"},"ToolResultDisposition":{"enum":["success","error","interrupted","cancelled","outcome_unknown"],"type":"string"},"TurnResult":{"enum":["finished","paused"],"type":"string"},"UploadedFileAvailability":{"description":"Availability recorded for an uploaded client-local file.","enum":["available","unavailable","integrity_failed"],"type":"string"},"UploadedFileRef":{"additionalProperties":false,"description":"Session-owned immutable reference to a client-local uploaded file.\n\nUpload transports return an unbound reference. Worker fills\n`source_entry_id` immediately before the containing user input is committed;\ncommitted Session Log and public snapshot records therefore always retain\nthe durable source-entry identity without storing the file body.","properties":{"artifact_id":{"type":"string"},"availability":{"$ref":"#/$defs/UploadedFileAvailability"},"byte_len":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"created_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"file_name":{"type":"string"},"media_type":{"type":"string"},"sha256":{"type":"string"},"source_entry_id":{"type":["string","null"]}},"required":["artifact_id","file_name","media_type","created_at_ms","availability","byte_len","sha256"],"type":"object"},"WorkerBusyState":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"run","type":"string"},"state":{"$ref":"#/$defs/WorkerRunState"}},"required":["kind","state"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"maintenance","type":"string"},"state":{"$ref":"#/$defs/WorkerMaintenanceState"}},"required":["kind","state"],"type":"object"}]},"WorkerCommandAcknowledgement":{"additionalProperties":false,"properties":{"command":{"$ref":"#/$defs/WorkerCommandKind"},"command_id":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"disposition":{"$ref":"#/$defs/WorkerCommandDisposition"},"state":{"$ref":"#/$defs/WorkerStateSnapshot","description":"The complete authoritative state observed after command admission."}},"required":["command_id","command","disposition","state"],"type":"object"},"WorkerCommandDisposition":{"enum":["accepted","stale_command_id","conflict","invalid_state"],"type":"string"},"WorkerCommandEnvelope":{"additionalProperties":false,"description":"Caller-owned identity for one state-changing Worker command.\n\nA controller accepts command ids in strictly increasing order. Exact retries\nof an accepted id must retain the same command kind.","properties":{"command_id":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["command_id"],"type":"object"},"WorkerCommandKind":{"enum":["resume","cancel","pause","compact","shutdown"],"type":"string"},"WorkerEvent":{"description":"Typed lifecycle events sent from a child Worker to its parent.\n\nDelivered as `Method::WorkerEvent` over the parent's Unix socket. The\nparent Controller always applies variant-specific side effects\n(registry / worker-allocation updates). Agent-visible variants are also\nqueued into the notification buffer; control-plane-only variants are\nnot injected into the parent's LLM context.\n\nTransport is fire-and-forget; receivers must tolerate out-of-order\ndelivery (e.g. `TurnEnded` arriving after `ShutDown` for the same\nchild Worker).","oneOf":[{"additionalProperties":false,"description":"Child finished one turn and is back to IDLE.","properties":{"kind":{"const":"turn_ended","type":"string"},"worker_name":{"type":"string"}},"required":["kind","worker_name"],"type":"object"},{"additionalProperties":false,"description":"Engine execution error occurred inside the child's turn.\n\nLimited to worker runtime failures (provider / tool errors) —\ndoes not include transient method-rejection responses such as\n`AlreadyRunning`.","properties":{"kind":{"const":"errored","type":"string"},"message":{"type":"string"},"worker_name":{"type":"string"}},"required":["kind","worker_name","message"],"type":"object"},{"additionalProperties":false,"description":"Child has stopped (controller loop is exiting).","properties":{"kind":{"const":"shut_down","type":"string"},"worker_name":{"type":"string"}},"required":["kind","worker_name"],"type":"object"},{"additionalProperties":false,"description":"Child SubWorker sub-delegated scope to a grandchild SubWorker via `SubWorkerSpawn`.\n\nControl-plane only: receivers apply registry side effects and\npropagate upward, but do not expose this as an agent notification.\n\nThe parent uses this to add the grandchild to its own\n`spawned_workers.json` so it can manage the grandchild directly\neven if the intermediate child dies. The parent then re-fires\nthis event upward (if it has a parent of its own) to maintain\nthe chain to root.","properties":{"kind":{"const":"scope_sub_delegated","type":"string"},"parent_worker":{"description":"Sub-delegating Worker (= the sender itself).","type":"string"},"scope":{"description":"Scope delegated to the grandchild.","items":{"$ref":"#/$defs/ScopeRule"},"type":"array"},"sub_socket":{"description":"Unix-socket path where the grandchild is reachable.","type":"string"},"sub_worker":{"description":"Name of the grandchild Worker.","type":"string"}},"required":["kind","parent_worker","sub_worker","sub_socket","scope"],"type":"object"}]},"WorkerMaintenanceState":{"enum":["compacting"],"type":"string"},"WorkerRunState":{"enum":["running","pausing","paused","cancelling"],"type":"string"},"WorkerState":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"idle","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"busy","type":"string"},"state":{"$ref":"#/$defs/WorkerBusyState"}},"required":["kind","state"],"type":"object"}]},"WorkerStateSnapshot":{"additionalProperties":false,"properties":{"last_command_id":{"description":"Highest lifecycle command id observed by this controller instance.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"state":{"$ref":"#/$defs/WorkerState"}},"required":["last_command_id","state"],"type":"object"},"WorkerStatus":{"enum":["idle","running","paused","stopped"],"type":"string"},"WorkspaceSubscriptionWorkdir":{"additionalProperties":false,"description":"Workspace-facing Workdir summary. Backend-generated Repository UUIDs never\nenter this DTO; Workspace Server must resolve the required Repository key.","properties":{"repository_key":{"type":"string"},"state":{"type":"string"},"working_directory_id":{"$ref":"#/$defs/SubscriptionWorkdirId"}},"required":["working_directory_id","repository_key","state"],"type":"object"}},"$schema":"https://json-schema.org/draft/2020-12/schema","description":"Direction is explicit at the outer envelope, while concrete message kinds\nremain typed within each lane.","oneOf":[{"additionalProperties":false,"properties":{"frame":{"const":"request","type":"string"},"message":{"$ref":"#/$defs/SubscriptionRequest"},"protocol_version":{"format":"uint16","maximum":65535.0,"minimum":0.0,"type":"integer"}},"required":["frame","message","protocol_version"],"type":"object"},{"additionalProperties":false,"properties":{"frame":{"const":"response","type":"string"},"message":{"$ref":"#/$defs/SubscriptionResponse"},"protocol_version":{"format":"uint16","maximum":65535.0,"minimum":0.0,"type":"integer"}},"required":["frame","message","protocol_version"],"type":"object"},{"additionalProperties":false,"properties":{"frame":{"const":"event","type":"string"},"message":{"$ref":"#/$defs/SubscriptionEvent"},"protocol_version":{"format":"uint16","maximum":65535.0,"minimum":0.0,"type":"integer"}},"required":["frame","message","protocol_version"],"type":"object"},{"additionalProperties":false,"properties":{"frame":{"const":"worker_protocol","type":"string"},"message":{"$ref":"#/$defs/SubscriptionWorkerProtocolMethod"},"protocol_version":{"format":"uint16","maximum":65535.0,"minimum":0.0,"type":"integer"}},"required":["frame","message","protocol_version"],"type":"object"}],"title":"SubscriptionFrame"}; export const RUST_SERIALIZED_SUBSCRIPTION_FRAME_FIXTURES: readonly unknown[] = - [{"protocol_version":1,"frame":"response","message":{"result":"subscribed","payload":{"request_id":"fixture-request","subscription_id":"fixture-subscription","selector":{"topic":"workspace_workers"},"snapshot_revision":1,"snapshot":{"topic":"workers","data":{"workers":[{"worker_id":"fixture-worker","runtime_id":"fixture-runtime","resource_key":"fixture-worker-resource","availability":"observed","subject_revision":1,"state":"idle","has_running_internal_workers":false,"workspace_id":"fixture-workspace","workdir_attachments":[{"alias":"external","working_directory_id":"fixture-external-workdir"}]}]}}}}},{"protocol_version":1,"frame":"event","message":{"event":"event","data":{"subscription_id":"fixture-subscription","subject_revision":2,"payload":{"event":"worker_protocol","data":{"worker_id":"fixture-worker","event":{"event":"pending_submissions_changed","data":{"pending":{"revision":0,"notification_count":0,"head_id":null,"submissions":[]}}}}}}}}]; + [{"protocol_version":2,"frame":"response","message":{"result":"subscribed","payload":{"request_id":"fixture-request","subscription_id":"fixture-subscription","selector":{"topic":"workspace_workers"},"snapshot":{"topic":"workers","data":{"workers":[{"worker_id":"fixture-worker","runtime_id":"fixture-runtime","resource_key":"fixture-worker-resource","availability":"observed","state":"idle","has_running_internal_workers":false,"workspace_id":"fixture-workspace","workdir_attachments":[{"alias":"external","working_directory_id":"fixture-external-workdir"}]}]}}}}},{"protocol_version":2,"frame":"event","message":{"event":"event","data":{"subscription_id":"fixture-subscription","payload":{"event":"worker_protocol","data":{"worker_id":"fixture-worker","event":{"event":"pending_submissions_changed","data":{"pending":{"revision":0,"notification_count":0,"head_id":null,"submissions":[]}}}}}}}}]; const utf8 = new TextEncoder(); type JsonObject = Record; diff --git a/web/workspace/src/lib/generated/protocol.ts b/web/workspace/src/lib/generated/protocol.ts index 8bae7b59a..7063b7214 100644 --- a/web/workspace/src/lib/generated/protocol.ts +++ b/web/workspace/src/lib/generated/protocol.ts @@ -304,11 +304,7 @@ resource_key?: string | null, */ availability: SubscriptionWorkerAvailability, /** - * Producer-owned monotonic revision for this Worker subject. - */ -subject_revision: number, -/** - * Latest revisioned foreground state observed from the Worker. This remains + * Latest foreground state observed from the Worker. This remains * absent until an authoritative Worker snapshot/event has been applied. */ worker_state?: WorkerStateSnapshot | null, @@ -336,9 +332,9 @@ export type SubscriptionRequest = { "method": "subscribe_events", "params": { re export type SubscriptionWorkerProtocolMethod = { subscription_id: SubscriptionId, method: Method, }; -export type SubscriptionResponse = { "result": "subscribed", "payload": { request_id: SubscriptionRequestId, subscription_id: SubscriptionId, selector: EventSubscriptionSelector, snapshot_revision: number, snapshot: SubscriptionSnapshot, } } | { "result": "unsubscribed", "payload": { request_id: SubscriptionRequestId, subscription_id: SubscriptionId, } } | { "result": "subscription_rejected", "payload": { request_id: SubscriptionRequestId, subscription_id?: SubscriptionId | null, code: SubscriptionRejectionCode, message: string, } }; +export type SubscriptionResponse = { "result": "subscribed", "payload": { request_id: SubscriptionRequestId, subscription_id: SubscriptionId, selector: EventSubscriptionSelector, snapshot: SubscriptionSnapshot, } } | { "result": "unsubscribed", "payload": { request_id: SubscriptionRequestId, subscription_id: SubscriptionId, } } | { "result": "subscription_rejected", "payload": { request_id: SubscriptionRequestId, subscription_id?: SubscriptionId | null, code: SubscriptionRejectionCode, message: string, } }; -export type SubscriptionEvent = { "event": "event", "data": { subscription_id: SubscriptionId, subject_revision: number, payload: SubscriptionEventPayload, } } | { "event": "subscription_closed", "data": { subscription_id: SubscriptionId, code: SubscriptionTerminationCode, message: string, } }; +export type SubscriptionEvent = { "event": "event", "data": { subscription_id: SubscriptionId, payload: SubscriptionEventPayload, } } | { "event": "subscription_closed", "data": { subscription_id: SubscriptionId, code: SubscriptionTerminationCode, message: string, } }; export type SubscriptionFramePayload = { "frame": "request", "message": SubscriptionRequest } | { "frame": "response", "message": SubscriptionResponse } | { "frame": "event", "message": SubscriptionEvent } | { "frame": "worker_protocol", "message": SubscriptionWorkerProtocolMethod }; diff --git a/web/workspace/src/lib/workspace/multiplexer.browser.spec.ts b/web/workspace/src/lib/workspace/multiplexer.browser.spec.ts index 77ab8f56d..251ebe0fc 100644 --- a/web/workspace/src/lib/workspace/multiplexer.browser.spec.ts +++ b/web/workspace/src/lib/workspace/multiplexer.browser.spec.ts @@ -83,7 +83,7 @@ test('invalid inbound frame closes before dispatch and reconnects for a fresh sn first.message( JSON.stringify({ - protocol_version: 1, + protocol_version: 2, frame: 'event', message: { event: 'subscription_closed', @@ -141,7 +141,7 @@ test('valid subscription closure remains isolated and resubscribes only its sele : { topic: 'workspace_workdirs', data: { workdirs: [] } }; socket.message( JSON.stringify({ - protocol_version: 1, + protocol_version: 2, frame: 'response', message: { result: 'subscribed', @@ -149,7 +149,6 @@ test('valid subscription closure remains isolated and resubscribes only its sele request_id: request.message.params.request_id, subscription_id: `subscription-${index + 1}`, selector, - snapshot_revision: 1, snapshot, }, }, @@ -160,7 +159,7 @@ test('valid subscription closure remains isolated and resubscribes only its sele const sentBeforeClosure = socket.sent.length; socket.message( JSON.stringify({ - protocol_version: 1, + protocol_version: 2, frame: 'event', message: { event: 'subscription_closed', diff --git a/web/workspace/src/lib/workspace/multiplexer.ts b/web/workspace/src/lib/workspace/multiplexer.ts index 453aaf9d5..f88cf9462 100644 --- a/web/workspace/src/lib/workspace/multiplexer.ts +++ b/web/workspace/src/lib/workspace/multiplexer.ts @@ -138,7 +138,7 @@ export class WorkspaceMultiplexer { subscription.requestId = requestId; this.#requests.set(requestId, subscription.clientId); this.#send({ - protocol_version: 1, + protocol_version: 2, frame: 'request', message: { method: 'subscribe_events', @@ -213,7 +213,7 @@ export class WorkspaceMultiplexer { const subscription = this.#subscriptions.get(clientId); if (!subscription?.subscriptionId) throw new Error('Worker protocol subscription is not open'); this.#send({ - protocol_version: 1, + protocol_version: 2, frame: 'worker_protocol', message: { subscription_id: subscription.subscriptionId, method }, }); @@ -225,7 +225,7 @@ export class WorkspaceMultiplexer { this.#subscriptions.delete(clientId); if (subscription.subscriptionId && this.#socket?.readyState === WebSocket.OPEN) { this.#send({ - protocol_version: 1, + protocol_version: 2, frame: 'request', message: { method: 'unsubscribe_events', diff --git a/web/workspace/src/lib/workspace/protocol-validator.test.ts b/web/workspace/src/lib/workspace/protocol-validator.test.ts index bffe6ced3..8683b6052 100644 --- a/web/workspace/src/lib/workspace/protocol-validator.test.ts +++ b/web/workspace/src/lib/workspace/protocol-validator.test.ts @@ -16,7 +16,7 @@ function assert(condition: unknown, message: string): asserts condition { function subscribedFrame(overrides: Record = {}): Record { return { - protocol_version: 1, + protocol_version: 2, frame: 'response', message: { result: 'subscribed', @@ -24,7 +24,6 @@ function subscribedFrame(overrides: Record = {}): Record { const oversizedKey = 'k'.repeat(MAX_SUBSCRIPTION_STRING_BYTES + 1); const result = decode({ - protocol_version: 1, + protocol_version: 2, frame: 'event', message: { event: 'event', data: { subscription_id: 'subscription-1', - subject_revision: 2, payload: { event: 'worker_protocol', data: { @@ -306,13 +299,12 @@ Deno.test('aggregate limits include keys in unconstrained Worker JSON values', ( Deno.test('stateful selector validation fences a routed event before projection', () => { const frame: SubscriptionFrame = { - protocol_version: 1, + protocol_version: 2, frame: 'event', message: { event: 'event', data: { subscription_id: 'subscription-1', - subject_revision: 2, payload: { event: 'worker_protocol', data: { worker_id: 'worker-2', event: { event: 'thinking_start' } }, diff --git a/web/workspace/src/lib/workspace/sidebar/worker-stop.browser.spec.ts b/web/workspace/src/lib/workspace/sidebar/worker-stop.browser.spec.ts index 1ff7485aa..ce03bfb8a 100644 --- a/web/workspace/src/lib/workspace/sidebar/worker-stop.browser.spec.ts +++ b/web/workspace/src/lib/workspace/sidebar/worker-stop.browser.spec.ts @@ -43,7 +43,7 @@ function snapshot( availability: SubscriptionWorker["availability"], ): SubscriptionFrame { return { - protocol_version: 1, + protocol_version: 2, frame: "response", message: { result: "subscribed", @@ -51,7 +51,6 @@ function snapshot( request_id: "request-1", subscription_id: "subscription-1", selector: { topic: "workspace_workers" }, - snapshot_revision: 1, snapshot: { topic: "workers", data: { @@ -60,7 +59,6 @@ function snapshot( runtime_id: "runtime-a", resource_key: "W-1", availability, - subject_revision: 1, state, // An absent foreground snapshot must not prevent stopping an active catalog Worker. worker_state: null, diff --git a/web/workspace/src/lib/workspace/sidebar/worker-subscription-model.ts b/web/workspace/src/lib/workspace/sidebar/worker-subscription-model.ts index e2df7b0d4..6ec7107ce 100644 --- a/web/workspace/src/lib/workspace/sidebar/worker-subscription-model.ts +++ b/web/workspace/src/lib/workspace/sidebar/worker-subscription-model.ts @@ -6,55 +6,55 @@ import type { export type WorkspaceWorkersProjection = { workers: Map; - revisions: Map; + subscriptionId: string | null; }; export function createWorkspaceWorkersProjection(): WorkspaceWorkersProjection { - return { workers: new Map(), revisions: new Map() }; + return { workers: new Map(), subscriptionId: null }; } export function applyWorkspaceWorkersFrame( projection: WorkspaceWorkersProjection, frame: SubscriptionFrame, ): void { - if (frame.protocol_version !== 1) throw new Error('unsupported Worker subscription protocol'); + if (frame.protocol_version !== 2) throw new Error('unsupported Worker subscription protocol'); if (frame.frame === 'response' && frame.message.result === 'subscribed') { if (frame.message.payload.selector.topic !== 'workspace_workers') return; const snapshot = frame.message.payload.snapshot; if (snapshot.topic !== 'workers') throw new Error('workspace_workers returned a non-Worker snapshot'); - projection.workers.clear(); - projection.revisions.clear(); + + // Build the replacement before mutating the live projection so one malformed + // Worker cannot partially clear the previous subscription lifetime. + const workers = new Map(); for (const worker of snapshot.data.workers) { - const key = workerKey(worker.runtime_id, worker.worker_id); - projection.workers.set(key, worker); - projection.revisions.set(key, worker.subject_revision); + workers.set(workerKey(worker.runtime_id, worker.worker_id), worker); } + projection.workers = workers; + projection.subscriptionId = frame.message.payload.subscription_id; return; } if (frame.frame !== 'event' || frame.message.event !== 'event') return; - applyPayload(projection, frame.message.data.subject_revision, frame.message.data.payload); + const subscriptionId = frame.message.data.subscription_id; + if (!projection.subscriptionId) { + throw new Error('workspace_workers event arrived before its snapshot'); + } + if (subscriptionId !== projection.subscriptionId) return; + applyPayload(projection, frame.message.data.payload); } function applyPayload( projection: WorkspaceWorkersProjection, - subjectRevision: number, payload: SubscriptionEventPayload, ): void { if (payload.event === 'worker_upserted') { const worker = payload.data.worker; - const key = workerKey(worker.runtime_id, worker.worker_id); - if (subjectRevision <= (projection.revisions.get(key) ?? 0)) return; - projection.revisions.set(key, subjectRevision); - projection.workers.set(key, worker); + projection.workers.set(workerKey(worker.runtime_id, worker.worker_id), worker); } else if (payload.event === 'worker_removed') { - const key = workerKey(payload.data.runtime_id, payload.data.worker_id); - if (subjectRevision <= (projection.revisions.get(key) ?? 0)) return; - projection.revisions.set(key, subjectRevision); - projection.workers.delete(key); + projection.workers.delete(workerKey(payload.data.runtime_id, payload.data.worker_id)); } } function workerKey(runtimeId: string | null | undefined, workerId: string): string { if (!runtimeId) throw new Error('Workspace Worker projection is missing runtime_id'); - return `${runtimeId}:${workerId}`; + return JSON.stringify([runtimeId, workerId]); } diff --git a/web/workspace/src/lib/workspace/sidebar/worker-subscription.test.ts b/web/workspace/src/lib/workspace/sidebar/worker-subscription.test.ts index 9ed0e52b4..cf23b0bbc 100644 --- a/web/workspace/src/lib/workspace/sidebar/worker-subscription.test.ts +++ b/web/workspace/src/lib/workspace/sidebar/worker-subscription.test.ts @@ -1,15 +1,29 @@ -import type { SubscriptionFrame, SubscriptionWorker } from '$lib/generated/protocol'; +import type { + SubscriptionEventPayload, + SubscriptionFrame, + SubscriptionWorker, +} from '$lib/generated/protocol'; + +import { liveWorkerState } from './worker-state'; +import { + applyWorkspaceWorkersFrame, + createWorkspaceWorkersProjection, +} from './worker-subscription-model'; function assertEquals(actual: unknown, expected: unknown): void { if (JSON.stringify(actual) !== JSON.stringify(expected)) { throw new Error(`expected ${JSON.stringify(expected)}, got ${JSON.stringify(actual)}`); } } -import { liveWorkerState } from './worker-state'; -import { - applyWorkspaceWorkersFrame, - createWorkspaceWorkersProjection, -} from './worker-subscription-model'; + +function assertThrows(fn: () => void, message: string): void { + try { + fn(); + } catch { + return; + } + throw new Error(message); +} declare const Deno: { test(name: string, fn: () => void | Promise): void; @@ -18,7 +32,6 @@ declare const Deno: { function worker( runtimeId: string, workerId: string, - revision: number, hasRunningInternalWorkers = false, availability: 'observed' | 'unavailable' = 'observed', ): SubscriptionWorker { @@ -26,7 +39,6 @@ function worker( worker_id: workerId, runtime_id: runtimeId, availability, - subject_revision: revision, state: 'idle', has_running_internal_workers: hasRunningInternalWorkers, workspace_id: 'workspace-test', @@ -36,15 +48,52 @@ function worker( }; } +function key(runtimeId: string, workerId: string): string { + return JSON.stringify([runtimeId, workerId]); +} + +function snapshot( + subscriptionId: string, + workers: SubscriptionWorker[], +): SubscriptionFrame { + return { + protocol_version: 2, + frame: 'response', + message: { + result: 'subscribed', + payload: { + request_id: `request-${subscriptionId}`, + subscription_id: subscriptionId, + selector: { topic: 'workspace_workers' }, + snapshot: { topic: 'workers', data: { workers } }, + }, + }, + }; +} + +function event( + subscriptionId: string, + payload: SubscriptionEventPayload, +): SubscriptionFrame { + return { + protocol_version: 2, + frame: 'event', + message: { + event: 'event', + data: { subscription_id: subscriptionId, payload }, + }, + }; +} + Deno.test('Worker list state uses the authoritative live snapshot separately from lifecycle', () => { - const active = worker('runtime-a', 'worker-1', 1); + const active = worker('runtime-a', 'worker-1'); active.worker_state = { last_command_id: 1, state: { kind: 'busy', state: { kind: 'run', state: 'paused' } }, }; assertEquals(liveWorkerState(active), 'paused'); - const unavailable = worker('runtime-a', 'worker-2', 1, false, 'unavailable'); + const unavailable = worker('runtime-a', 'worker-2', false, 'unavailable'); unavailable.worker_state = { last_command_id: 2, state: { kind: 'busy', state: { kind: 'run', state: 'running' } }, @@ -67,127 +116,118 @@ Deno.test('Worker list state uses the authoritative live snapshot separately fro Deno.test('workspace Worker snapshot keeps equal local ids from different Runtimes', () => { const projection = createWorkspaceWorkersProjection(); - const frame: SubscriptionFrame = { - protocol_version: 1, - frame: 'response', - message: { - result: 'subscribed', - payload: { - request_id: 'request-1', - subscription_id: 'subscription-1', - selector: { topic: 'workspace_workers' }, - snapshot_revision: 1, - snapshot: { - topic: 'workers', - data: { workers: [worker('runtime-a', '1', 1), worker('runtime-b', '1', 1)] }, - }, - }, - }, - }; - applyWorkspaceWorkersFrame(projection, frame); - assertEquals([...projection.workers.keys()].sort(), ['runtime-a:1', 'runtime-b:1']); + applyWorkspaceWorkersFrame( + projection, + snapshot('subscription-1', [worker('runtime-a', '1'), worker('runtime-b', '1')]), + ); + assertEquals([...projection.workers.keys()].sort(), [key('runtime-a', '1'), key('runtime-b', '1')]); }); -Deno.test('workspace Worker snapshot and live update preserve canonical Job metadata', () => { +Deno.test('workspace Worker snapshot and ordered live update preserve canonical Job metadata', () => { const projection = createWorkspaceWorkersProjection(); - const snapshotWorker = worker('runtime-a', 'job-worker', 1); + const snapshotWorker = worker('runtime-a', 'job-worker'); snapshotWorker.job = { job_id: 'check:T-1:r1', attempt_id: 'check:T-1:r1:attempt:1', purpose: 'ticket_item_check', }; - applyWorkspaceWorkersFrame(projection, { - protocol_version: 1, - frame: 'response', - message: { - result: 'subscribed', - payload: { - request_id: 'request-1', - subscription_id: 'subscription-1', - selector: { topic: 'workspace_workers' }, - snapshot_revision: 1, - snapshot: { topic: 'workers', data: { workers: [snapshotWorker] } }, - }, - }, - }); - assertEquals(projection.workers.get('runtime-a:job-worker')?.job, snapshotWorker.job); + applyWorkspaceWorkersFrame(projection, snapshot('subscription-1', [snapshotWorker])); + assertEquals(projection.workers.get(key('runtime-a', 'job-worker'))?.job, snapshotWorker.job); - const updated = { ...snapshotWorker, subject_revision: 2 }; + const updated = { ...snapshotWorker }; updated.job = { ...snapshotWorker.job, attempt_id: 'check:T-1:r1:attempt:2' }; - applyWorkspaceWorkersFrame(projection, { - protocol_version: 1, - frame: 'event', - message: { - event: 'event', - data: { - subscription_id: 'subscription-1', - subject_revision: 2, - payload: { event: 'worker_upserted', data: { worker: updated } }, - }, - }, - }); - assertEquals(projection.workers.get('runtime-a:job-worker')?.job, updated.job); + applyWorkspaceWorkersFrame( + projection, + event('subscription-1', { event: 'worker_upserted', data: { worker: updated } }), + ); + assertEquals(projection.workers.get(key('runtime-a', 'job-worker'))?.job, updated.job); }); -Deno.test('workspace Worker reducer ignores stale events and removes composite subject', () => { +Deno.test('fresh subscription snapshot replaces old state and fences delayed old events', () => { const projection = createWorkspaceWorkersProjection(); - projection.workers.set('runtime-a:1', worker('runtime-a', '1', 3)); - projection.revisions.set('runtime-a:1', 3); + applyWorkspaceWorkersFrame( + projection, + snapshot('subscription-old', [worker('runtime-a', 'old'), worker('runtime-b', 'same')]), + ); + applyWorkspaceWorkersFrame( + projection, + event('subscription-old', { + event: 'worker_upserted', + data: { worker: worker('runtime-a', 'before-reconnect') }, + }), + ); - applyWorkspaceWorkersFrame(projection, { - protocol_version: 1, - frame: 'event', - message: { - event: 'event', - data: { - subscription_id: 'subscription-1', - subject_revision: 2, - payload: { event: 'worker_upserted', data: { worker: worker('runtime-a', '1', 2) } }, - }, - }, - }); - assertEquals(projection.revisions.get('runtime-a:1'), 3); + applyWorkspaceWorkersFrame( + projection, + snapshot('subscription-new', [worker('runtime-a', 'current')]), + ); + applyWorkspaceWorkersFrame( + projection, + event('subscription-old', { + event: 'worker_upserted', + data: { worker: worker('runtime-a', 'late-old-event') }, + }), + ); - applyWorkspaceWorkersFrame(projection, { - protocol_version: 1, - frame: 'event', - message: { - event: 'event', - data: { - subscription_id: 'subscription-1', - subject_revision: 4, - payload: { - event: 'worker_removed', - data: { worker_id: '1', runtime_id: 'runtime-a' }, - }, - }, - }, - }); - assertEquals(projection.workers.size, 0); - assertEquals(projection.revisions.get('runtime-a:1'), 4); + assertEquals(projection.subscriptionId, 'subscription-new'); + assertEquals([...projection.workers.keys()], [key('runtime-a', 'current')]); +}); + +Deno.test('workspace Worker reducer applies ordered updates and composite removal', () => { + const projection = createWorkspaceWorkersProjection(); + applyWorkspaceWorkersFrame( + projection, + snapshot('subscription-1', [worker('runtime-a', '1'), worker('runtime-b', '1')]), + ); + + const updated = worker('runtime-a', '1'); + updated.state = 'running'; + applyWorkspaceWorkersFrame( + projection, + event('subscription-1', { event: 'worker_upserted', data: { worker: updated } }), + ); + applyWorkspaceWorkersFrame( + projection, + event('subscription-1', { + event: 'worker_removed', + data: { worker_id: '1', runtime_id: 'runtime-a' }, + }), + ); + + assertEquals([...projection.workers.keys()], [key('runtime-b', '1')]); +}); + +Deno.test('workspace Worker event before a snapshot is rejected', () => { + const projection = createWorkspaceWorkersProjection(); + assertThrows( + () => + applyWorkspaceWorkersFrame( + projection, + event('subscription-1', { + event: 'worker_upserted', + data: { worker: worker('runtime-a', '1') }, + }), + ), + 'event-before-snapshot must be rejected', + ); }); Deno.test('fatal child stop replaces the running-child sidebar projection', () => { const projection = createWorkspaceWorkersProjection(); - projection.workers.set('runtime-a:1', worker('runtime-a', '1', 1, true)); - projection.revisions.set('runtime-a:1', 1); + applyWorkspaceWorkersFrame( + projection, + snapshot('subscription-1', [worker('runtime-a', '1', true)]), + ); + applyWorkspaceWorkersFrame( + projection, + event('subscription-1', { + event: 'worker_upserted', + data: { worker: worker('runtime-a', '1', false) }, + }), + ); - applyWorkspaceWorkersFrame(projection, { - protocol_version: 1, - frame: 'event', - message: { - event: 'event', - data: { - subscription_id: 'subscription-1', - subject_revision: 2, - payload: { - event: 'worker_upserted', - data: { worker: worker('runtime-a', '1', 2, false) }, - }, - }, - }, - }); - - assertEquals(projection.workers.get('runtime-a:1')?.has_running_internal_workers, false); - assertEquals(projection.revisions.get('runtime-a:1'), 2); + assertEquals( + projection.workers.get(key('runtime-a', '1'))?.has_running_internal_workers, + false, + ); });