From fdce12a47bdd408e6e1a0275dfb218951f1a2f1f Mon Sep 17 00:00:00 2001 From: Hare Date: Tue, 6 Oct 2026 17:45:54 +0900 Subject: [PATCH] feat: guarded Worker Restore from shared Web actions (T-701) --- crates/client/src/backend_runtime.rs | 12 + crates/client/src/lib.rs | 12 +- crates/protocol/src/subscription.rs | 8 + crates/protocol/src/typescript.rs | 1 + .../src/typescript_validator_template.ts | 1 + crates/runtime-api/src/lib.rs | 120 +- crates/server-api/src/lib.rs | 40 + .../src/repository_openapi_typescript.rs | 4 +- crates/tui/src/backend_worker_picker.rs | 81 +- crates/worker-runtime/src/catalog.rs | 4 + crates/worker-runtime/src/error.rs | 3 + crates/worker-runtime/src/fs_store.rs | 7 + crates/worker-runtime/src/http_server.rs | 143 +- .../src/http_server/runtime_management_api.rs | 73 +- crates/worker-runtime/src/runtime.rs | 2251 ++++++++++++++++- crates/worker-runtime/src/worker_backend.rs | 23 +- crates/worker-runtime/src/worker_source.rs | 8 +- .../src/feature/builtin/manage_worker.rs | 154 +- crates/worker/src/worker.rs | 15 +- crates/workspace-server/src/hosts.rs | 208 +- crates/workspace-server/src/lib.rs | 4 + .../workspace-server/src/resource_broker.rs | 10 + crates/workspace-server/src/server.rs | 827 ++++-- .../server/tests/worker_operations_tests.rs | 723 +++++- .../src/server/worker_operations.rs | 14 +- crates/workspace-server/src/store.rs | 94 +- .../src/workspace_deletion.rs | 1 + .../src/workspace_subscription.rs | 2 + .../worker-restore-observations.md | 70 + docs/report/2026-10-06-t701-validation.md | 49 + openapi/server-api.json | 53 +- web/workspace/deno.json | 2 +- .../src/lib/generated/companion-api.ts | 2 +- .../src/lib/generated/protocol-validator.ts | 5 +- web/workspace/src/lib/generated/protocol.ts | 6 +- .../src/lib/generated/repository-api.ts | 4 +- .../src/lib/generated/runtime-api.ts | 8 +- web/workspace/src/lib/generated/ticket-api.ts | 2 +- .../src/lib/generated/workdir-api.ts | 2 +- .../src/lib/generated/worker-launch-api.ts | 4 +- .../src/lib/workspace/api/runtime-workers.ts | 227 +- .../src/lib/workspace/api/workers.ts | 110 +- .../sidebar/WorkersNavSection.svelte | 104 +- .../src/lib/workspace/sidebar/types.ts | 16 +- .../lib/workspace/sidebar/worker-actions.ts | 134 +- .../worker-restore-store.browser.spec.ts | 233 ++ .../sidebar/worker-stop.browser.spec.ts | 264 +- .../worker-subscription.browser.spec.ts | 15 +- .../workspace/sidebar/worker-subscription.ts | 141 +- .../w/[workspaceId]/workers/+page.svelte | 167 +- .../workers/workers-page.browser.spec.ts | 600 +++++ ...orkers-restore-integration.browser.spec.ts | 314 +++ .../test/sidebar/worker-actions.test.ts | 178 +- web/workspace/tests/runtime-workers.test.ts | 188 +- 54 files changed, 7214 insertions(+), 527 deletions(-) create mode 100644 docs/architecture/worker-restore-observations.md create mode 100644 docs/report/2026-10-06-t701-validation.md create mode 100644 web/workspace/src/lib/workspace/sidebar/worker-restore-store.browser.spec.ts create mode 100644 web/workspace/src/routes/w/[workspaceId]/workers/workers-page.browser.spec.ts create mode 100644 web/workspace/src/routes/w/[workspaceId]/workers/workers-restore-integration.browser.spec.ts diff --git a/crates/client/src/backend_runtime.rs b/crates/client/src/backend_runtime.rs index 7b9fbd42a..00feb7b74 100644 --- a/crates/client/src/backend_runtime.rs +++ b/crates/client/src/backend_runtime.rs @@ -10,6 +10,7 @@ pub use server_api::{ WorkerLaunchProfileCandidate as BackendWorkerLaunchProfileCandidate, WorkerLaunchRuntimeOption as BackendWorkerLaunchRuntimeOption, WorkerOperationState as BackendWorkerOperationState, + WorkerRestoreRequest as BackendWorkerRestoreRequest, WorkerRestoreResponse as BackendWorkerRestoreResponse, WorkerRestoreResult as BackendWorkerRestoreResult, WorkerRestoreState as BackendWorkerRestoreState, WorkerSummary as BackendWorkerSummary, @@ -259,6 +260,15 @@ impl fmt::Display for BackendRuntimeClientError { } } +impl BackendRuntimeClientError { + /// A stale intent is not a rejected Restore operation and must not be retried + /// with a silently refreshed token. + pub fn is_restore_observation_conflict(&self) -> bool { + matches!(self, Self::ContractApi(server_api::client_support::ClientError::Public { status, error }) + if *status == reqwest::StatusCode::CONFLICT && error.error == "restore_observation_conflict") + } +} + impl std::error::Error for BackendRuntimeClientError {} impl From for BackendRuntimeClientError { @@ -502,6 +512,7 @@ pub async fn list_backend_stopped_workers( pub async fn restore_backend_worker( target: &BackendRuntimeTarget, + request: BackendWorkerRestoreRequest, ) -> Result { validate_target(target)?; let api = BackendApiClient::from_stored_token(&target.base_url)?; @@ -514,6 +525,7 @@ pub async fn restore_backend_worker( ticket_id: None, assignment_operation_id: None, }, + request, ) .await .map_err(Into::into) diff --git a/crates/client/src/lib.rs b/crates/client/src/lib.rs index 32e1c1a10..53e0392dc 100644 --- a/crates/client/src/lib.rs +++ b/crates/client/src/lib.rs @@ -26,12 +26,12 @@ pub use backend_runtime::{ BackendRuntimeListTarget, BackendRuntimeSummary, BackendRuntimeTarget, BackendWorkerImplementationSummary, BackendWorkerLaunchOptions, BackendWorkerLaunchProfileCandidate, BackendWorkerLaunchRuntimeOption, - BackendWorkerLaunchTarget, BackendWorkerOperationState, BackendWorkerRestoreResponse, - BackendWorkerRestoreResult, BackendWorkerRestoreState, BackendWorkerSummary, - BackendWorkerWorkspaceSummary, BackendWorkingDirectorySource, BackendWorkingDirectorySummary, - connect_backend_runtime, create_backend_worker, get_backend_worker_launch_options, - list_backend_stopped_workers, list_backend_workers, observe_backend_worker_session, - restore_backend_worker, + BackendWorkerLaunchTarget, BackendWorkerOperationState, BackendWorkerRestoreRequest, + BackendWorkerRestoreResponse, BackendWorkerRestoreResult, BackendWorkerRestoreState, + BackendWorkerSummary, BackendWorkerWorkspaceSummary, BackendWorkingDirectorySource, + BackendWorkingDirectorySummary, connect_backend_runtime, create_backend_worker, + get_backend_worker_launch_options, list_backend_stopped_workers, list_backend_workers, + observe_backend_worker_session, restore_backend_worker, }; pub use backend_workspace::{ BackendWorkspace, BackendWorkspaceCatalogTarget, BackendWorkspaceClientError, diff --git a/crates/protocol/src/subscription.rs b/crates/protocol/src/subscription.rs index a27bca31f..5a59175ca 100644 --- a/crates/protocol/src/subscription.rs +++ b/crates/protocol/src/subscription.rs @@ -651,6 +651,10 @@ impl SubscriptionWorkerJob { #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[cfg_attr(feature = "json-schema", derive(schemars::JsonSchema))] pub struct SubscriptionWorker { + /// Opaque Runtime-owned Restore observation identity, never a display-state hash. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[cfg_attr(feature = "typescript", ts(optional = nullable))] + pub restore_observation_token: Option, pub worker_id: SubscriptionWorkerId, /// Set by the Workspace Server when projecting a Runtime-owned Worker to clients. /// Runtime producers leave this unset because the connection identifies the Runtime. @@ -689,6 +693,9 @@ pub struct SubscriptionWorker { impl SubscriptionWorker { pub fn validate(&self) -> Result<(), SubscriptionValidationError> { self.worker_id.validate()?; + if let Some(token) = &self.restore_observation_token { + validate_identifier("restore_observation_token", token, MAX_RESOURCE_ID_BYTES)?; + } if let Some(runtime_id) = &self.runtime_id { validate_identifier("runtime_id", runtime_id, MAX_RESOURCE_ID_BYTES)?; } @@ -962,6 +969,7 @@ mod tests { fn worker(value: &str) -> SubscriptionWorker { SubscriptionWorker { + restore_observation_token: None, worker_id: worker_id(value), runtime_id: None, resource_key: None, diff --git a/crates/protocol/src/typescript.rs b/crates/protocol/src/typescript.rs index c1f2d9b00..201534567 100644 --- a/crates/protocol/src/typescript.rs +++ b/crates/protocol/src/typescript.rs @@ -246,6 +246,7 @@ fn rust_serialized_subscription_frame_fixtures() -> Vec { let external_workdir_id = SubscriptionWorkdirId::new("fixture-external-workdir") .expect("fixture external Workdir id must be valid"); let worker = SubscriptionWorker { + restore_observation_token: Some("fixture-restore-observation".into()), worker_id: worker_id.clone(), runtime_id: Some("fixture-runtime".to_string()), resource_key: Some("fixture-worker-resource".to_string()), diff --git a/crates/protocol/src/typescript_validator_template.ts b/crates/protocol/src/typescript_validator_template.ts index 976f9b4bd..fc0da94d0 100644 --- a/crates/protocol/src/typescript_validator_template.ts +++ b/crates/protocol/src/typescript_validator_template.ts @@ -378,6 +378,7 @@ function validateWorkers(workers: SubscriptionWorker[]): boolean { } function validateWorker(worker: SubscriptionWorker): boolean { + if (worker.restore_observation_token != null && !validateIdentifier(worker.restore_observation_token, MAX_RESOURCE_ID_BYTES)) return false; if (!validateIdentifier(worker.worker_id, MAX_RESOURCE_ID_BYTES)) return false; if (worker.runtime_id != null && !validateIdentifier(worker.runtime_id, MAX_RESOURCE_ID_BYTES)) { return false; diff --git a/crates/runtime-api/src/lib.rs b/crates/runtime-api/src/lib.rs index 618089bb7..0b34c83c7 100644 --- a/crates/runtime-api/src/lib.rs +++ b/crates/runtime-api/src/lib.rs @@ -452,6 +452,8 @@ pub struct CreateWorkerRequest { #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] pub struct WorkerSummary { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub restore_observation_token: Option, pub worker_ref: WorkerRef, pub worker_id: WorkerId, pub status: WorkerStatus, @@ -484,6 +486,62 @@ pub struct WorkerResponse { pub worker: WorkerDetail, } +/// Explicit observed Restore intent. Preparation is authored by the Workspace +/// Backend, not browser/Tool JSON, and applied only after Runtime admission. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct WorkerRestoreRequest { + pub expected_observation_token: String, + pub request_id: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub preparation: Option, +} + +#[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct WorkerRestorePreparation { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workspace_api: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub workdir_attachments: Option>, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub repository_access: Vec, + /// Read-only references resolved by Workspace only after this operation owns admission. + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub repository_access_workdirs: Vec, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct RepositoryAccessWorkdirReference { + pub runtime_id: String, + pub working_directory_id: String, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct WorkerRestoreCoordinationRequest { + pub expected_observation_token: String, + pub request_id: String, + /// None means owner/result recovery only; it can never admit a new intent. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub preparation: Option, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +pub struct WorkerRestoreCoordinationResponse { + pub result: Option, + /// Original read-only preparation, only when the existing operation still needs it. + pub preparation: Option, +} + +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct WorkingDirectoryRepositoryAccessRequest { + pub working_directory_id: String, + pub materialization: RepositoryMaterializationContext, +} + /// Explicit restore outcome returned for every reachable Runtime restore attempt, /// including preflight rejection and uncertain post-side-effect reconciliation. #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] @@ -1000,11 +1058,18 @@ pub trait RuntimeApi { #[body] request: WorkerLifecycleRequest, ) -> Result; - #[post("/v1/workers/{worker_id}/restore", status = 200, error_status = 400)] + #[post("/v1/workers/{worker_id}/restore/coordinate", status = 200, error_status = 400, additional_error_statuses = [409])] + async fn coordinate_worker_restore( + &self, + #[path] worker_id: String, + #[body] request: WorkerRestoreCoordinationRequest, + ) -> Result; + + #[post("/v1/workers/{worker_id}/restore", status = 200, error_status = 400, additional_error_statuses = [409])] async fn restore_worker( &self, #[path] worker_id: String, - #[body] request: EmptyObjectRequest, + #[body] request: WorkerRestoreRequest, ) -> Result; #[post( @@ -1210,6 +1275,46 @@ pub const REMAINING_RUNTIME_ROUTES: &[RemainingRuntimeRoute] = &[ mod tests { use super::*; + #[test] + fn restore_request_is_mandatory_and_conflict_is_not_an_operation_state() { + for invalid in [ + serde_json::json!({}), + serde_json::json!({"request_id":"r"}), + serde_json::json!({"expected_observation_token":"t"}), + ] { + assert!(serde_json::from_value::(invalid).is_err()); + } + let request: WorkerRestoreRequest = serde_json::from_value( + serde_json::json!({"expected_observation_token":"t", "request_id":"r"}), + ) + .unwrap(); + assert!(request.preparation.is_none()); + for state in [ + "accepted", + "rejected", + "rolled_back", + "reconciliation_required", + ] { + assert!(serde_json::from_value::(serde_json::json!(state)).is_ok()); + } + assert!( + serde_json::from_value::(serde_json::json!( + "restore_observation_conflict" + )) + .is_err() + ); + let conflict = RuntimeApiError::new( + 409, + "restore_observation_conflict", + "refresh before a new intent", + ); + assert_eq!(conflict.status(), 409); + assert_eq!( + serde_json::to_value(conflict).unwrap()["error"]["code"], + "restore_observation_conflict" + ); + } + #[test] fn feature_argument_completion_wire_preserves_source_and_argument() { let json = serde_json::json!({"kind":"feature_argument", "prefix":"資料/", "context": {"invocation":"builtin:test/prepare", "argument":"path"}}); @@ -1388,10 +1493,17 @@ mod tests { ) -> Result { Err(test_error(501)) } + async fn coordinate_worker_restore( + &self, + _worker_id: String, + _request: WorkerRestoreCoordinationRequest, + ) -> Result { + Err(test_error(501)) + } async fn restore_worker( &self, _worker_id: String, - _request: EmptyObjectRequest, + _request: WorkerRestoreRequest, ) -> Result { Err(test_error(501)) } @@ -1491,7 +1603,7 @@ mod tests { #[test] fn contract_inventory_is_complete_and_unique() { let operations = RuntimeApiMetadata::OPERATIONS; - assert_eq!(operations.len(), 19); + assert_eq!(operations.len(), 20); let mut routes = operations .iter() .map(|operation| (format!("{:?}", operation.method), operation.path)) diff --git a/crates/server-api/src/lib.rs b/crates/server-api/src/lib.rs index 471f2ff06..8d99e78b9 100644 --- a/crates/server-api/src/lib.rs +++ b/crates/server-api/src/lib.rs @@ -389,6 +389,7 @@ impl_openapi_schema!( WorkspaceRuntimeListResponse, RuntimeWorkerListResponse, WorkerRestoreResponse, + WorkerRestoreRequest, WorkerControlSubject, WorkerControlRecord, WorkerControlListResponse, @@ -1552,6 +1553,7 @@ pub trait ServerApi { #[path] workspace_id: String, #[path] runtime_id: String, #[path] worker_id: String, + #[body] request: WorkerRestoreRequest, ) -> Result; #[get( "/api/w/{workspace_id}/worker-observation/sessions", @@ -2080,6 +2082,7 @@ pub trait ServerApi { #[extension] context: ServerRequestContext, #[path] runtime_id: String, #[path] worker_id: String, + #[body] request: WorkerRestoreRequest, ) -> Result; #[post( @@ -2097,6 +2100,7 @@ pub trait ServerApi { #[path] runtime_id: String, #[path] worker_id: String, #[query] query: RestoreTicketAssignmentQuery, + #[body] request: WorkerRestoreRequest, ) -> Result; #[put( @@ -9594,6 +9598,10 @@ pub struct RuntimeWorkerWorkdirAttachmentSummary { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] pub struct WorkerSummary { + /// Opaque Runtime-owned observation fence used by explicit Restore intent. + #[serde(default, skip_serializing_if = "Option::is_none")] + #[cfg_attr(feature = "typescript", ts(optional = nullable))] + pub restore_observation_token: Option, pub runtime_id: String, pub worker_id: String, pub resource_key: String, @@ -9830,6 +9838,16 @@ pub enum WorkerRestoreState { ReconciliationRequired, } +/// One explicit Restore intent. Retries retain both fields; conflicts require a +/// fresh observation and a deliberate new intent, not automatic token replacement. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] +pub struct WorkerRestoreRequest { + pub expected_observation_token: String, + pub request_id: String, +} + #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] #[cfg_attr(feature = "typescript", derive(ts_rs::TS))] #[serde(deny_unknown_fields)] @@ -11694,6 +11712,28 @@ mod tests { ); } + #[test] + fn worker_restore_request_requires_observation_and_request_identity() { + let valid = serde_json::json!({"expected_observation_token":"opaque-generation", "request_id":"restore-request"}); + let request: WorkerRestoreRequest = serde_json::from_value(valid.clone()).unwrap(); + assert_eq!(serde_json::to_value(request).unwrap(), valid); + for invalid in [ + serde_json::json!({}), + serde_json::json!({"request_id":"restore-request"}), + serde_json::json!({"expected_observation_token":"opaque-generation"}), + serde_json::json!({"expected_observation_token":"opaque-generation", "request_id":"restore-request", "command_id":701}), + ] { + assert!(serde_json::from_value::(invalid).is_err()); + } + let contract = canonical_openapi_document().unwrap().to_json().unwrap(); + let contract: serde_json::Value = serde_json::from_str(&contract).unwrap(); + let restore = &contract["paths"]["/api/w/{workspace_id}/runtimes/{runtime_id}/workers/{worker_id}/restore"] + ["post"]; + assert_eq!(restore["requestBody"]["required"], true); + assert!(restore["responses"]["409"].is_object()); + assert!(contract["components"]["schemas"]["WorkerRestoreRequest"].is_object()); + } + #[test] fn worker_restore_result_rejects_unknown_fields() { let result = serde_json::from_value::(serde_json::json!({ diff --git a/crates/server-api/src/repository_openapi_typescript.rs b/crates/server-api/src/repository_openapi_typescript.rs index 235c95458..658133616 100644 --- a/crates/server-api/src/repository_openapi_typescript.rs +++ b/crates/server-api/src/repository_openapi_typescript.rs @@ -872,14 +872,14 @@ fn validate_runtime_operations( "/api/runtimes/{runtime_id}/workers/{worker_id}/restore", "post", "runtime_worker_restore_alias", - None, + Some("WorkerRestoreRequest"), &[("200", "WorkerRestoreResponse")], ), runtime_operation( "/api/w/{workspace_id}/runtimes/{runtime_id}/workers/{worker_id}/restore", "post", "runtime_worker_restore", - None, + Some("WorkerRestoreRequest"), &[("200", "WorkerRestoreResponse")], ), runtime_operation( diff --git a/crates/tui/src/backend_worker_picker.rs b/crates/tui/src/backend_worker_picker.rs index 30e168e14..4e97a8943 100644 --- a/crates/tui/src/backend_worker_picker.rs +++ b/crates/tui/src/backend_worker_picker.rs @@ -29,6 +29,7 @@ trait BackendWorkerLifecycle { async fn restore( &self, target: &client::BackendRuntimeTarget, + request: client::BackendWorkerRestoreRequest, ) -> Result; async fn observe( @@ -44,13 +45,19 @@ impl BackendWorkerLifecycle for LiveBackendWorkerLifecycle { async fn restore( &self, target: &client::BackendRuntimeTarget, + request: client::BackendWorkerRestoreRequest, ) -> Result { - restore_backend_worker(target).await.map_err(|error| { - io::Error::other(format!( - "failed to restore Backend Worker {}: {error}", - target.display_label() - )) - }) + restore_backend_worker(target, request) + .await + .map_err(|error| { + if error.is_restore_observation_conflict() { + return io::Error::other("Restore observation changed; refresh the Worker list and select again. No new Restore was started."); + } + io::Error::other(format!( + "failed to restore Backend Worker {}: {error}", + target.display_label() + )) + }) } async fn observe( @@ -133,8 +140,20 @@ pub(crate) async fn run( let attach_target = target .runtime_target(selected.runtime_id.clone(), selected.worker_id.clone()) .map_err(|error| io::Error::other(error.to_string()))?; - let attach_target = - prepare_selected_worker(&LiveBackendWorkerLifecycle, attach_target, intent).await?; + let restore_request = + selected + .restore_observation_token + .map(|token| client::BackendWorkerRestoreRequest { + expected_observation_token: token, + request_id: uuid::Uuid::now_v7().to_string(), + }); + let attach_target = prepare_selected_worker( + &LiveBackendWorkerLifecycle, + attach_target, + intent, + restore_request, + ) + .await?; return console::run_backend_runtime(attach_target).await; } } @@ -143,9 +162,13 @@ async fn prepare_selected_worker( lifecycle: &impl BackendWorkerLifecycle, mut target: client::BackendRuntimeTarget, intent: BackendWorkerPickerIntent, + restore_request: Option, ) -> Result { if intent == BackendWorkerPickerIntent::Resume { - let response = lifecycle.restore(&target).await?; + let request = restore_request.ok_or_else(|| { + io::Error::other("Worker has no current Restore observation; refresh and select again") + })?; + let response = lifecycle.restore(&target, request).await?; ensure_restore_accepted(&response)?; let observation = lifecycle.observe(&target).await?; require_live_resume_observation(observation.observation)?; @@ -158,7 +181,10 @@ async fn prepare_selected_worker( } fn is_resume_candidate(worker: &BackendWorkerSummary) -> bool { - worker.worker_state.is_none() + worker.availability == protocol::subscription::SubscriptionWorkerAvailability::Observed + && worker.state == "stopped" + && worker.worker_state.is_none() + && worker.restore_observation_token.is_some() } fn ensure_restore_accepted(response: &BackendWorkerRestoreResponse) -> Result<(), io::Error> { @@ -570,6 +596,7 @@ mod tests { fn worker(runtime_id: &str, worker_id: &str, profile: Option<&str>) -> BackendWorkerSummary { BackendWorkerSummary { + restore_observation_token: Some("observed-worker-generation".into()), runtime_id: runtime_id.to_string(), worker_id: worker_id.to_string(), resource_key: "W-1".to_string(), @@ -642,6 +669,7 @@ mod tests { async fn restore( &self, _target: &client::BackendRuntimeTarget, + _request: client::BackendWorkerRestoreRequest, ) -> Result { self.calls.lock().unwrap().push("restore"); Ok(self.restore_response.clone()) @@ -689,10 +717,17 @@ mod tests { "worker-a", ); - let prepared = - prepare_selected_worker(&lifecycle, target, BackendWorkerPickerIntent::Resume) - .await - .unwrap(); + let prepared = prepare_selected_worker( + &lifecycle, + target, + BackendWorkerPickerIntent::Resume, + Some(client::BackendWorkerRestoreRequest { + expected_observation_token: "observed-worker-generation".into(), + request_id: "resume-operation".into(), + }), + ) + .await + .unwrap(); assert_eq!(*lifecycle.calls.lock().unwrap(), ["restore", "observe"]); assert!(prepared.initial_snapshot.is_none()); @@ -716,10 +751,18 @@ mod tests { "worker-a", ); - let error = prepare_selected_worker(&lifecycle, target, BackendWorkerPickerIntent::Resume) - .await - .unwrap_err() - .to_string(); + let error = prepare_selected_worker( + &lifecycle, + target, + BackendWorkerPickerIntent::Resume, + Some(client::BackendWorkerRestoreRequest { + expected_observation_token: "observed-worker-generation".into(), + request_id: "resume-operation".into(), + }), + ) + .await + .unwrap_err() + .to_string(); assert!(error.contains("Rejected")); assert_eq!(*lifecycle.calls.lock().unwrap(), ["restore"]); @@ -736,7 +779,7 @@ mod tests { assert!(!is_resume_candidate(&live)); assert!(is_resume_candidate(&stopped)); - assert!(is_resume_candidate(&unknown)); + assert!(!is_resume_candidate(&unknown)); } #[test] diff --git a/crates/worker-runtime/src/catalog.rs b/crates/worker-runtime/src/catalog.rs index ed393fca9..d5d0405bc 100644 --- a/crates/worker-runtime/src/catalog.rs +++ b/crates/worker-runtime/src/catalog.rs @@ -334,6 +334,8 @@ pub(crate) enum WorkerRestoreIntent { /// Lightweight persisted Worker identity projection. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct WorkerSummary { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub restore_observation_token: Option, pub worker_ref: WorkerRef, pub worker_id: WorkerId, pub status: WorkerStatus, @@ -359,6 +361,8 @@ pub struct WorkerSummary { /// Full persisted Worker identity and lifecycle detail. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct WorkerDetail { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub restore_observation_token: Option, pub worker_ref: WorkerRef, pub worker_id: WorkerId, pub status: WorkerStatus, diff --git a/crates/worker-runtime/src/error.rs b/crates/worker-runtime/src/error.rs index bbbea4435..ef9609cc0 100644 --- a/crates/worker-runtime/src/error.rs +++ b/crates/worker-runtime/src/error.rs @@ -12,6 +12,9 @@ pub enum RuntimeError { #[error("initial worker input must be user input, got {kind}")] InvalidInitialInputKind { kind: String }, + #[error("worker {worker_id} Restore observation conflicts with the current operation")] + RestoreObservationConflict { worker_id: WorkerId }, + #[error("worker {worker_id} was not found")] WorkerNotFound { worker_id: WorkerId }, diff --git a/crates/worker-runtime/src/fs_store.rs b/crates/worker-runtime/src/fs_store.rs index b5a9e54fe..0e5c8b5c5 100644 --- a/crates/worker-runtime/src/fs_store.rs +++ b/crates/worker-runtime/src/fs_store.rs @@ -520,6 +520,7 @@ pub(crate) enum PersistedWorkerExecutionState { #[derive(Clone, Debug)] pub(crate) struct PersistedWorkerRecord { + pub(crate) restore_guard: crate::runtime::RestoreGuard, pub(crate) worker_ref: WorkerRef, pub(crate) worker_id: WorkerId, pub(crate) profile: ProfileSelector, @@ -1819,6 +1820,8 @@ impl RuntimeSnapshot { #[derive(Clone, Debug, Serialize, Deserialize)] #[serde(deny_unknown_fields)] struct WorkerAggregateRecord { + #[serde(default)] + restore_guard: crate::runtime::RestoreGuard, schema_version: u32, worker_ref: WorkerRef, worker_id: WorkerId, @@ -1874,6 +1877,7 @@ impl WorkerAggregateRecord { }; Self { schema_version: SCHEMA_VERSION, + restore_guard: worker.restore_guard.clone(), worker_ref: worker.worker_ref.clone(), worker_id: worker.worker_id, profile: worker.profile.clone(), @@ -1902,6 +1906,7 @@ impl WorkerAggregateRecord { } fn validate(self, path: &Path) -> Result { + let restore_guard = self.restore_guard; let last_finished_submission_request_id = self.last_finished_submission_request_id; let identity = WorkerIdentityRecord { schema_version: self.schema_version, @@ -2007,6 +2012,7 @@ impl WorkerAggregateRecord { WorkerExecutionStateRecord::Unavailable => PersistedWorkerExecutionState::Unavailable, }; let mut persisted = identity.into_persisted(execution_state); + persisted.restore_guard = restore_guard; persisted.last_finished_submission_request_id = last_finished_submission_request_id; Ok(persisted) } @@ -2180,6 +2186,7 @@ impl WorkerIdentityRecord { self.workdir_attachments }; PersistedWorkerRecord { + restore_guard: crate::runtime::RestoreGuard::default(), worker_ref: self.worker_ref, worker_id: self.worker_id, profile: self.profile, diff --git a/crates/worker-runtime/src/http_server.rs b/crates/worker-runtime/src/http_server.rs index a2ae8d744..623e5b524 100644 --- a/crates/worker-runtime/src/http_server.rs +++ b/crates/worker-runtime/src/http_server.rs @@ -2302,6 +2302,7 @@ fn required_runtime_permission(method: &Method, path: &str) -> Option<&'static s } if path.ends_with("/input") || path.ends_with("/restore") + || path.ends_with("/restore/coordinate") || (path.contains("/attachments") && *method != Method::GET) || path.contains("/workdir-attachments") { @@ -2514,6 +2515,7 @@ fn status_for_runtime_error(error: &RuntimeError) -> StatusCode { StatusCode::CONFLICT } RuntimeError::RuntimeStopped + | RuntimeError::RestoreObservationConflict { .. } | RuntimeError::RuntimeStoreAlreadyOpen { .. } | RuntimeError::WorkerExecutionUnavailable { .. } | RuntimeError::ExecutionBackendUnavailable { .. } @@ -2538,6 +2540,9 @@ fn status_for_runtime_error(error: &RuntimeError) -> StatusCode { fn code_for_runtime_error(error: &RuntimeError) -> String { match error { RuntimeError::RuntimeStopped => "runtime_stopped".to_string(), + RuntimeError::RestoreObservationConflict { .. } => { + "restore_observation_conflict".to_string() + } RuntimeError::RuntimeStoreAlreadyOpen { .. } => "runtime_store_already_open".to_string(), RuntimeError::WorkerNotFound { .. } => "worker_not_found".to_string(), RuntimeError::WorkerExecutionUnavailable { .. } => { @@ -3642,7 +3647,17 @@ mod tests { client .restore_worker( generated_worker_id.clone(), - runtime_api::EmptyObjectRequest::default(), + runtime_api::WorkerRestoreRequest { + expected_observation_token: client + .get_worker(generated_worker_id.clone()) + .await + .unwrap() + .worker + .restore_observation_token + .unwrap(), + request_id: "generated-restore".into(), + preparation: Some(runtime_api::WorkerRestorePreparation::default()), + }, ) .await .unwrap(); @@ -3786,18 +3801,86 @@ mod tests { let stop: RuntimeHttpWorkerLifecycleResponse = read_json(response).await; assert_eq!(stop.ack.worker_ref, created.worker.worker_ref); - let response = authed_empty_request( + let restore_uri = format!("/v1/workers/{}/restore", created.worker.worker_id); + let missing = authed_empty_request(app.clone(), Method::POST, &restore_uri, token).await; + assert_eq!(missing.status(), StatusCode::BAD_REQUEST); + let stale = runtime_api::WorkerRestoreRequest { + expected_observation_token: created.worker.restore_observation_token.clone().unwrap(), + request_id: "stale-http-restore".into(), + preparation: None, + }; + let response = + authed_json_request(app.clone(), Method::POST, &restore_uri, token, &stale).await; + assert_eq!(response.status(), StatusCode::CONFLICT); + let conflict: RuntimeHttpErrorResponse = read_json(response).await; + assert_eq!(conflict.error.code, "restore_observation_conflict"); + + let mut restore_request = runtime.test_restore_request(&created.worker.worker_ref); + let coordinate_uri = format!("{restore_uri}/coordinate"); + let mut coordination = runtime_api::WorkerRestoreCoordinationRequest { + expected_observation_token: restore_request.expected_observation_token.clone(), + request_id: restore_request.request_id.clone(), + preparation: None, + }; + let before_lookup = runtime.worker_detail(&created.worker.worker_ref).unwrap(); + let response = authed_json_request( app.clone(), Method::POST, - &format!("/v1/workers/{}/restore", created.worker.worker_id), + &coordinate_uri, token, + &coordination, + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + let missing_owner: runtime_api::WorkerRestoreCoordinationResponse = + read_json(response).await; + assert!(missing_owner.result.is_none() && missing_owner.preparation.is_none()); + assert_eq!( + runtime.worker_detail(&created.worker.worker_ref).unwrap(), + before_lookup + ); + let response = authed_json_request( + app.clone(), + Method::POST, + &coordinate_uri, + "wrong-token", + &coordination, + ) + .await; + assert_eq!(response.status(), StatusCode::UNAUTHORIZED); + coordination.preparation = Some(runtime_api::WorkerRestorePreparation::default()); + let response = authed_json_request( + app.clone(), + Method::POST, + &coordinate_uri, + token, + &coordination, + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + let admitted: runtime_api::WorkerRestoreCoordinationResponse = read_json(response).await; + assert!(admitted.result.is_none() && admitted.preparation.is_some()); + assert_eq!( + runtime + .worker_detail(&created.worker.worker_ref) + .unwrap() + .status, + crate::catalog::WorkerStatus::Stopped + ); + restore_request.preparation = admitted.preparation; + let response = authed_json_request( + app.clone(), + Method::POST, + &restore_uri, + token, + &restore_request, ) .await; assert_eq!(response.status(), StatusCode::OK); let restored: runtime_api::WorkerRestoreResponse = read_json(response).await; assert_eq!(restored.state, WorkerRestoreState::Accepted); assert_eq!( - restored.worker.unwrap().status, + restored.worker.as_ref().unwrap().status, runtime_api::WorkerStatus::Idle ); @@ -3810,6 +3893,47 @@ mod tests { .await; assert_eq!(response.status(), StatusCode::OK); + coordination.preparation = None; + let response = authed_json_request( + app.clone(), + Method::POST, + &coordinate_uri, + token, + &coordination, + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + let recovered: runtime_api::WorkerRestoreCoordinationResponse = read_json(response).await; + assert!(recovered.preparation.is_none()); + assert_eq!(recovered.result, Some(restored.clone())); + + restore_request.preparation = Some(serde_json::from_value(serde_json::json!({ + "workspace_api": { "workspace_id": "must-not-bind", "base_url": "https://fresh.invalid.example" }, + "workdir_attachments": [], + "repository_access": [] + })).unwrap()); + let response = authed_json_request( + app.clone(), + Method::POST, + &restore_uri, + token, + &restore_request, + ) + .await; + assert_eq!(response.status(), StatusCode::OK); + let replayed: runtime_api::WorkerRestoreResponse = read_json(response).await; + assert_eq!( + serde_json::to_value(replayed).unwrap(), + serde_json::to_value(restored).unwrap() + ); + assert_eq!( + runtime + .worker_detail(&created.worker.worker_ref) + .unwrap() + .status, + crate::catalog::WorkerStatus::Stopped + ); + let response = authed_empty_request( app.clone(), Method::POST, @@ -3892,11 +4016,12 @@ mod tests { assert_eq!(listed.workers[0].status, crate::catalog::WorkerStatus::Idle); assert!(listed.workers[0].worker_state.is_none()); - let response = authed_empty_request( + let response = authed_json_request( app.clone(), Method::POST, &format!("{worker_uri}/restore"), token, + &runtime.test_restore_request(&created.worker_ref), ) .await; assert_eq!(response.status(), StatusCode::OK); @@ -4864,7 +4989,9 @@ mod ws_tests { let (mut old_stream, _) = connect_async(authed_ws_request(&url)).await.unwrap(); let _ = next_frame(&mut old_stream).await; runtime.stop_worker(&worker_ref, None).unwrap(); - runtime.restore_worker(&worker_ref).unwrap(); + runtime + .restore_worker(&worker_ref, runtime.test_restore_request(&worker_ref)) + .unwrap(); backend.publish( &worker_ref, protocol::Event::TextDelta { @@ -4946,7 +5073,9 @@ mod ws_tests { ); backend.publish(&worker_ref, protocol::Event::Shutdown); runtime.stop_worker(&worker_ref, None).unwrap(); - runtime.restore_worker(&worker_ref).unwrap(); + runtime + .restore_worker(&worker_ref, runtime.test_restore_request(&worker_ref)) + .unwrap(); let resume_url = format!("{url}?cursor={}", old.cursor); let (mut stream, _) = connect_async(authed_ws_request(&resume_url)).await.unwrap(); assert!(matches!( diff --git a/crates/worker-runtime/src/http_server/runtime_management_api.rs b/crates/worker-runtime/src/http_server/runtime_management_api.rs index eaf3ea5bf..b92e22731 100644 --- a/crates/worker-runtime/src/http_server/runtime_management_api.rs +++ b/crates/worker-runtime/src/http_server/runtime_management_api.rs @@ -462,21 +462,74 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi { Ok(runtime_api::WorkerLifecycleResponse { ack }) } + async fn coordinate_worker_restore( + &self, + worker_id: String, + request: runtime_api::WorkerRestoreCoordinationRequest, + ) -> Result { + let worker_ref = worker_ref_for(&self.state.runtime, worker_id).map_err(api_error)?; + let scope = + auth_workspace_scope(&self.state, auth_extension().as_ref()).map_err(api_error)?; + let runtime = self.state.runtime.clone(); + let (result, preparation) = tokio::task::spawn_blocking(move || { + runtime.coordinate_worker_restore_operation(&worker_ref, request, scope.as_ref()) + }) + .await + .map_err(|error| { + runtime_api::RuntimeApiError::new(500, "runtime_restore_task_failed", error.to_string()) + })? + .map_err(RuntimeHttpRestError::runtime) + .map_err(api_error)?; + let result = result + .map(|result| -> Result<_, runtime_api::RuntimeApiError> { + Ok(runtime_api::WorkerRestoreResponse { + state: match result.state { + server_api::WorkerRestoreState::Accepted => { + runtime_api::WorkerRestoreState::Accepted + } + server_api::WorkerRestoreState::Rejected => { + runtime_api::WorkerRestoreState::Rejected + } + server_api::WorkerRestoreState::RolledBack => { + runtime_api::WorkerRestoreState::RolledBack + } + server_api::WorkerRestoreState::ReconciliationRequired => { + runtime_api::WorkerRestoreState::ReconciliationRequired + } + }, + worker: result.worker.map(response).transpose()?, + reason_code: result.reason_code, + message: result.message, + }) + }) + .transpose()?; + Ok(runtime_api::WorkerRestoreCoordinationResponse { + result, + preparation, + }) + } + async fn restore_worker( &self, worker_id: String, - _request: runtime_api::EmptyObjectRequest, + request: runtime_api::WorkerRestoreRequest, ) -> Result { let worker_ref = worker_ref_for(&self.state.runtime, worker_id).map_err(api_error)?; - let result = match auth_workspace_scope(&self.state, auth_extension().as_ref()) - .map_err(api_error)? - { - Some(scope) => self - .state - .runtime - .restore_worker_operation_scoped(&scope, &worker_ref), - None => self.state.runtime.restore_worker_operation(&worker_ref), - } + let scope = + auth_workspace_scope(&self.state, auth_extension().as_ref()).map_err(api_error)?; + let runtime = self.state.runtime.clone(); + let result = tokio::task::spawn_blocking(move || match scope { + Some(scope) => runtime.restore_worker_operation_scoped(&scope, &worker_ref, request), + None => runtime.restore_worker_operation(&worker_ref, request), + }) + .await + .map_err(|error| { + runtime_api::RuntimeApiError::new( + StatusCode::INTERNAL_SERVER_ERROR.as_u16(), + "runtime_restore_task_failed", + error.to_string(), + ) + })? .map_err(RuntimeHttpRestError::runtime) .map_err(api_error)?; Ok(runtime_api::WorkerRestoreResponse { diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index 3d37c59d6..119a7e4d4 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -145,7 +145,7 @@ impl From for WorkerRestoreMode { /// Runtime-internal restore result consumed by both embedded and HTTP /// providers. Diagnostics are stable and path-free; detailed backend errors /// remain in Runtime logs rather than crossing the API boundary. -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq, Eq, serde::Serialize, serde::Deserialize)] pub struct RuntimeWorkerRestoreResult { pub state: WorkerRestoreState, pub worker: Option, @@ -378,6 +378,8 @@ impl Runtime { state }; state.execution_backend = execution_backend; + // Initialize legacy generations durably before exposing an observation. + state.persist_workers()?; let runtime = Self { inner: Arc::new(Mutex::new(state)), worker_operations: Arc::new(Mutex::new(BTreeMap::new())), @@ -1094,6 +1096,7 @@ impl Runtime { let durable_request = durable_create_worker_request(&request); let record = WorkerRecord { + restore_guard: RestoreGuard::default(), worker_ref: worker_ref.clone(), worker_id: worker_id.clone(), status: WorkerStatus::Stopped, @@ -1529,7 +1532,7 @@ impl Runtime { fn replace_worker_workdir_attachments_with_scope( &self, worker_ref: &WorkerRef, - mut attachments: Vec, + attachments: Vec, scope: Option<&RuntimeWorkspaceScope>, ) -> Result { let operation_lock = self.worker_operation_lock(worker_ref.worker_id)?; @@ -1539,6 +1542,14 @@ impl Runtime { if let Some(scope) = scope { self.ensure_worker_in_workspace(scope, worker_ref)?; } + self.replace_worker_workdir_attachments_under_lock(worker_ref, attachments) + } + + fn replace_worker_workdir_attachments_under_lock( + &self, + worker_ref: &WorkerRef, + mut attachments: Vec, + ) -> Result { validate_logical_workdir_attachments(&attachments)?; attachments.sort_by(|left, right| left.alias.cmp(&right.alias)); @@ -1712,13 +1723,14 @@ impl Runtime { }) } - /// Attach a live execution through a workspace-scoped Runtime authorization context. + /// Restore admission is fenced by the same lock as every lifecycle mutation. pub fn restore_worker_scoped( &self, scope: &RuntimeWorkspaceScope, worker_ref: &WorkerRef, + request: runtime_api::WorkerRestoreRequest, ) -> Result { - self.restore_worker_operation_scoped(scope, worker_ref)? + self.restore_worker_operation_scoped(scope, worker_ref, request)? .into_legacy_result() } @@ -1726,41 +1738,681 @@ impl Runtime { &self, scope: &RuntimeWorkspaceScope, worker_ref: &WorkerRef, + request: runtime_api::WorkerRestoreRequest, ) -> Result { - self.restore_worker_operation_with_scope(worker_ref, Some(scope)) + self.restore_worker_operation_with_scope(worker_ref, request, Some(scope)) } - /// Attach a live execution to a persisted Worker definition. - /// - /// Every lifecycle mutation for a Worker is serialized by the same operation - /// lock. A concurrent exact restore waits for the first caller and then - /// converges on its already-installed execution rather than spawning another - /// controller. - pub fn restore_worker(&self, worker_ref: &WorkerRef) -> Result { - self.restore_worker_operation(worker_ref)? + pub fn restore_worker( + &self, + worker_ref: &WorkerRef, + request: runtime_api::WorkerRestoreRequest, + ) -> Result { + self.restore_worker_operation(worker_ref, request)? .into_legacy_result() } pub fn restore_worker_operation( &self, worker_ref: &WorkerRef, + request: runtime_api::WorkerRestoreRequest, ) -> Result { - self.restore_worker_operation_with_scope(worker_ref, None) + self.restore_worker_operation_with_scope(worker_ref, request, None) + } + + pub fn coordinate_worker_restore_operation( + &self, + worker_ref: &WorkerRef, + request: runtime_api::WorkerRestoreCoordinationRequest, + scope: Option<&RuntimeWorkspaceScope>, + ) -> Result< + ( + Option, + Option, + ), + RuntimeError, + > { + self.dispatch_restore_worker_with_scope( + worker_ref, + runtime_api::WorkerRestoreRequest { + expected_observation_token: request.expected_observation_token, + request_id: request.request_id, + preparation: request.preparation, + }, + scope, + true, + ) } fn restore_worker_operation_with_scope( &self, worker_ref: &WorkerRef, + request: runtime_api::WorkerRestoreRequest, scope: Option<&RuntimeWorkspaceScope>, ) -> Result { + self.dispatch_restore_worker_with_scope(worker_ref, request, scope, false)? + .0 + .ok_or_else(|| RuntimeError::InvalidRequest("Restore preparation is required".into())) + } + + fn dispatch_restore_worker_with_scope( + &self, + worker_ref: &WorkerRef, + mut request: runtime_api::WorkerRestoreRequest, + scope: Option<&RuntimeWorkspaceScope>, + coordinate: bool, + ) -> Result< + ( + Option, + Option, + ), + RuntimeError, + > { + let mut preparation_authorized = !coordinate && request.preparation.is_some(); let operation_lock = self.worker_operation_lock(worker_ref.worker_id)?; let _operation_guard = operation_lock .lock() .map_err(|_| RuntimeError::StatePoisoned)?; if let Some(scope) = scope { - self.ensure_worker_in_workspace(scope, worker_ref)?; + let mut state = self.lock()?; + if !state + .worker(worker_ref)? + .belongs_to_workspace(&scope.workspace_id) + { + return Err(RuntimeError::WorkerNotFound { + worker_id: worker_ref.worker_id, + }); + } + // Authorization is read-only until token admission succeeds. + state.ensure_workspace_owner(scope, false)?; } - self.restore_worker_under_lock(worker_ref, WorkerRestoreMode::Explicit) + if request.request_id.trim().is_empty() + || request.request_id.len() > 512 + || request.expected_observation_token.is_empty() + || request.expected_observation_token.len() > 256 + { + return Err(RuntimeError::InvalidRequest( + "Restore requires a nonempty request_id and expected_observation_token".to_string(), + )); + } + use sha2::Digest; + // Preparation contains ephemeral Workspace/resource handles, not public + // intent. Retries must use the first admitted owner's durable payload. + // The owner map is scoped to this Worker, so bind only the external fence. + let bytes = serde_json::to_vec(&(&request.expected_observation_token, &request.request_id)) + .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?; + let fingerprint = sha2::Sha256::digest(bytes) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::(); + { + let mut state = self.lock()?; + let mut candidate = state.worker(worker_ref)?.clone(); + let conflict = || RuntimeError::RestoreObservationConflict { + worker_id: worker_ref.worker_id, + }; + if let Some(mut owner) = candidate + .restore_guard + .owners + .get(&request.request_id) + .cloned() + { + if owner.fingerprint != fingerprint { + return Err(conflict()); + } + // Replay is historical: it must not attach a controller after a later stop. + if let Some(receipt) = &owner.receipt { + return Ok((Some(receipt.clone()), None)); + } + if candidate.restore_guard.active_request_id.as_deref() != Some(&request.request_id) + || candidate + .pending_restore + .is_none_or(|pending| pending.operation_id != owner.operation_id) + { + return Err(conflict()); + } + if coordinate && !owner.preparation_applied { + if let Some(preparation) = owner.request.preparation.as_mut() { + if preparation.repository_access_workdirs.is_empty() + && !preparation.repository_access.is_empty() + { + preparation.repository_access_workdirs = preparation + .repository_access + .iter() + .map(|access| runtime_api::RepositoryAccessWorkdirReference { + runtime_id: access.materialization.runtime_id.clone(), + working_directory_id: access.working_directory_id.clone(), + }) + .collect(); + candidate + .restore_guard + .owners + .insert(request.request_id.clone(), owner.clone()); + match state.persist_worker_record(&candidate) { + Ok(()) => { + state + .workers + .insert(worker_ref.worker_id, candidate.clone()); + } + Err(error @ RuntimeError::StoreCommitOutcomeUnknown { .. }) => { + state.workers.insert(worker_ref.worker_id, candidate); + return Ok(( + Some(RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::ReconciliationRequired, + "worker_restore_preparation_commit_unknown", + sanitize_worker_create_failure_message( + &error.to_string(), + ), + )), + None, + )); + } + Err(error) => { + return Ok(( + Some(RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::ReconciliationRequired, + "worker_restore_preparation_commit_failed", + sanitize_worker_create_failure_message( + &error.to_string(), + ), + )), + None, + )); + } + } + } + } + } + if !owner.preparation_applied { + if let (Some(original), Some(incoming)) = + (&owner.request.preparation, &request.preparation) + { + // Server-regenerated bindings are not part of the public fingerprint. + // Reuse admitted bindings and accept only resource completion for those references. + let references_match = incoming.repository_access.len() + == original.repository_access_workdirs.len() + && original.repository_access_workdirs.iter().all(|reference| { + incoming + .repository_access + .iter() + .filter(|access| { + reference.working_directory_id + == access.working_directory_id + && reference.runtime_id + == access.materialization.runtime_id + }) + .count() + == 1 + }); + let snapshot_matches = original.repository_access.is_empty() + || (original.repository_access.len() + == incoming.repository_access.len() + && original + .repository_access + .iter() + .zip(&incoming.repository_access) + .all(|(original, incoming)| { + let mut normalized = incoming.clone(); + if let (Some(old), Some(new)) = ( + &original.materialization.ssh, + &mut normalized.materialization.ssh, + ) { + new.secret_resource = old.secret_resource.clone(); + new.expires_at_epoch_seconds = + old.expires_at_epoch_seconds; + } + &normalized == original + })); + if !original.repository_access_workdirs.is_empty() { + preparation_authorized = + !coordinate && references_match && snapshot_matches; + } + if references_match + && snapshot_matches + && original.repository_access != incoming.repository_access + { + candidate + .restore_guard + .owners + .get_mut(&request.request_id) + .unwrap() + .request + .preparation + .as_mut() + .unwrap() + .repository_access = incoming.repository_access.clone(); + match state.persist_worker_record(&candidate) { + Ok(()) => { + state + .workers + .insert(worker_ref.worker_id, candidate.clone()); + } + Err(error @ RuntimeError::StoreCommitOutcomeUnknown { .. }) => { + state.workers.insert(worker_ref.worker_id, candidate); + return Ok(( + Some(RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::ReconciliationRequired, + "worker_restore_preparation_commit_unknown", + sanitize_worker_create_failure_message( + &error.to_string(), + ), + )), + None, + )); + } + Err(error) => { + return Ok(( + Some(RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::ReconciliationRequired, + "worker_restore_preparation_commit_failed", + sanitize_worker_create_failure_message( + &error.to_string(), + ), + )), + None, + )); + } + } + } + } + } + } else { + // Startup/older persisted pending operations have a lifecycle + // owner but no public request receipt. Bind recovery to that + // exact operation; never allocate replacement execution or apply + // candidate preparation. The current observation still fences + // stale callers, and a second request cannot steal this receipt. + let recovering_pending = candidate.pending_restore.is_some() + && candidate.pending_stop.is_none() + && candidate.restore_guard.active_request_id.is_none(); + if coordinate && request.preparation.is_none() && !recovering_pending { + if candidate.restore_guard.generation != request.expected_observation_token + || candidate.restore_guard.active_request_id.is_some() + || candidate.has_pending_lifecycle_operation() + { + return Err(conflict()); + } + return Ok((None, None)); + } + if coordinate + && request + .preparation + .as_ref() + .is_some_and(|prep| !prep.repository_access.is_empty()) + { + return Err(RuntimeError::InvalidRequest( + "Admission accepts read-only preparation references only".into(), + )); + } + state.ensure_running()?; + if candidate.restore_guard.generation != request.expected_observation_token + || candidate.restore_guard.active_request_id.is_some() + || (candidate.has_pending_lifecycle_operation() && !recovering_pending) + { + return Err(conflict()); + } + if (scope.is_some() || candidate.workspace_id.is_some()) + && request.preparation.is_none() + && !recovering_pending + { + return Err(RuntimeError::InvalidRequest( + "New Workspace Restore requires preparation".into(), + )); + } + if let Some(scope) = scope { + state.ensure_workspace_owner(scope, true)?; + state.persist_runtime_snapshot()?; + } + let pending = candidate + .pending_restore + .unwrap_or_else(|| PendingWorkerRestore { + operation_id: WorkerLifecycleOperationId::new(), + mode: WorkerRestoreMode::Explicit, + last_settled_status: candidate.status, + }); + let operation_id = pending.operation_id; + candidate.pending_restore = Some(pending); + if recovering_pending { + request.preparation = None; + } + candidate.restore_guard.rotate(); + candidate.restore_guard.active_request_id = Some(request.request_id.clone()); + candidate.restore_guard.owners.insert( + request.request_id.clone(), + RestoreRequestOwner { + fingerprint, + operation_id, + request, + preparation_applied: recovering_pending, + execution_started: recovering_pending, + receipt: None, + }, + ); + match state.persist_worker_record(&candidate) { + Ok(()) => {} + Err(error @ RuntimeError::StoreCommitOutcomeUnknown { .. }) => { + state.workers.insert(worker_ref.worker_id, candidate); + return Ok(( + Some(RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::ReconciliationRequired, + "worker_restore_admission_commit_unknown", + sanitize_worker_create_failure_message(&error.to_string()), + )), + None, + )); + } + Err(error) => return Err(error), + } + state.workers.insert(worker_ref.worker_id, candidate); + state.publish_worker_upsert(worker_ref.worker_id)?; + } + } + if coordinate { + let state = self.lock()?; + let candidate = state.worker(worker_ref)?; + let id = candidate + .restore_guard + .active_request_id + .as_ref() + .expect("admitted request"); + let owner = &candidate.restore_guard.owners[id]; + if !owner.preparation_applied + && !owner.execution_started + && owner.request.preparation.is_some() + { + return Ok((None, owner.request.preparation.clone())); + } + } + self.restore_admitted_worker_under_lock_with_preparation( + worker_ref, + WorkerRestoreMode::Explicit, + preparation_authorized, + ) + .map(|result| (Some(result), None)) + } + + #[cfg(test)] + pub(crate) fn test_restore_request( + &self, + worker_ref: &WorkerRef, + ) -> runtime_api::WorkerRestoreRequest { + let state = self.lock().unwrap(); + let worker = state.worker(worker_ref).unwrap(); + if let Some(id) = &worker.restore_guard.active_request_id { + return worker.restore_guard.owners[id].request.clone(); + } + runtime_api::WorkerRestoreRequest { + expected_observation_token: worker.restore_guard.generation.clone(), + request_id: uuid::Uuid::now_v7().to_string(), + preparation: worker + .workspace_id + .as_ref() + .map(|_| runtime_api::WorkerRestorePreparation::default()), + } + } + + /// Private startup and exact pending-owner reconciliation use the durable payload. + fn restore_admitted_worker_under_lock( + &self, + worker_ref: &WorkerRef, + mode: WorkerRestoreMode, + ) -> Result { + self.restore_admitted_worker_under_lock_with_preparation(worker_ref, mode, false) + } + + fn restore_admitted_worker_under_lock_with_preparation( + &self, + worker_ref: &WorkerRef, + mode: WorkerRestoreMode, + preparation_authorized: bool, + ) -> Result { + let owner = { + let state = self.lock()?; + let worker = state.worker(worker_ref)?; + match worker.restore_guard.active_request_id.as_ref() { + Some(id) => { + let owner = worker.restore_guard.owners.get(id).ok_or_else(|| { + RuntimeError::InvalidRequest( + "Active Restore request has no persisted owner".into(), + ) + })?; + if worker + .pending_restore + .is_some_and(|pending| pending.operation_id != owner.operation_id) + { + return Err(RuntimeError::RestoreObservationConflict { + worker_id: worker_ref.worker_id, + }); + } + Some(owner.clone()) + } + None => None, + } + }; + if let Some(owner) = owner { + // A settled owner must never turn into a fresh execution, including + // when startup encounters an old active-request marker. + if let Some(receipt) = owner.receipt { + return Ok(receipt); + } + if owner.execution_started { + let committed_detail = { + let state = self.lock()?; + let worker = state.worker(worker_ref)?; + if worker.pending_restore.is_none() && worker.pending_stop.is_none() { + if !worker.execution_bound || !worker.status.is_active() { + return Ok(RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::ReconciliationRequired, + "worker_restore_terminal_receipt_missing", + "Restore execution was started but its terminal receipt is unavailable", + )); + } + Some(worker.detail()) + } else { + None + } + }; + if let Some(detail) = committed_detail { + // Recover a receipt gap without launching execution again. + let result = RuntimeWorkerRestoreResult::accepted(detail); + self.finish_restore_receipt(worker_ref, &result)?; + return Ok(result); + } + } + if !owner.preparation_applied { + let uncertain = |message: String| { + RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::ReconciliationRequired, + "worker_restore_preparation_reconciliation_required", + message, + ) + }; + if owner.request.preparation.as_ref().is_some_and(|prep| { + prep.repository_access.len() < prep.repository_access_workdirs.len() + || (!prep.repository_access_workdirs.is_empty() && !preparation_authorized) + }) { + return Ok(uncertain( + "Admitted Restore is awaiting Workspace resource completion".into(), + )); + } + // Complete idempotent binding/access effects under the original pending_restore ID. + let mut effects_started = false; + if let Err(error) = self.apply_restore_preparation_under_lock( + worker_ref, + owner.request.preparation, + owner.operation_id, + &mut effects_started, + ) { + if effects_started + || matches!(error, RuntimeError::StoreCommitOutcomeUnknown { .. }) + { + return Ok(uncertain(sanitize_worker_create_failure_message( + &error.to_string(), + ))); + } + let result = RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::Rejected, + "worker_restore_preparation_rejected", + sanitize_worker_create_failure_message(&error.to_string()), + ); + self.finish_restore_receipt(worker_ref, &result)?; + return Ok(result); + } + let mut state = self.lock()?; + let mut candidate = state.worker(worker_ref)?.clone(); + candidate + .restore_guard + .owners + .get_mut(&owner.request.request_id) + .expect("admitted owner") + .preparation_applied = true; + if let Err(error) = state.persist_worker_record(&candidate) { + // Keep the started (not applied) record in memory. Even a + // known failed completion write follows possible effects. + return Ok(uncertain(sanitize_worker_create_failure_message( + &error.to_string(), + ))); + } + state.workers.insert(worker_ref.worker_id, candidate); + } + } + let result = self.restore_worker_under_lock(worker_ref, mode)?; + self.finish_restore_receipt(worker_ref, &result)?; + Ok(result) + } + + fn finish_restore_receipt( + &self, + worker_ref: &WorkerRef, + result: &RuntimeWorkerRestoreResult, + ) -> Result<(), RuntimeError> { + if result.state == WorkerRestoreState::ReconciliationRequired { + return Ok(()); + } + let mut state = self.lock()?; + let mut candidate = state.worker(worker_ref)?.clone(); + if candidate.restore_guard.active_request_id.is_some() { + candidate.restore_guard.finish(result.clone()); + candidate.pending_restore = None; + state.persist_worker_record(&candidate)?; + state.workers.insert(worker_ref.worker_id, candidate); + } + Ok(()) + } + + fn apply_restore_preparation_under_lock( + &self, + worker_ref: &WorkerRef, + preparation: Option, + operation_id: WorkerLifecycleOperationId, + effects_started: &mut bool, + ) -> Result<(), RuntimeError> { + let Some(preparation) = preparation else { + return Ok(()); + }; + // Validate all read-only structure before any binding effect. + if let Some(attachments) = &preparation.workdir_attachments { + let attachments: Vec = serde_json::from_value( + serde_json::to_value(attachments) + .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?, + ) + .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?; + validate_logical_workdir_attachments(&attachments)?; + } + // DTO conversion and every Runtime preparation effect occur AFTER admission. + if let Some(api) = preparation.workspace_api { + let api: WorkspaceApiRef = serde_json::from_value( + serde_json::to_value(api) + .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?, + ) + .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?; + if self.lock()?.worker(worker_ref)?.workspace_id.as_deref() + != Some(api.workspace_id.as_str()) + { + return Err(RuntimeError::InvalidRequest( + "Prepared Workspace API does not match Worker Workspace identity".into(), + )); + } + let mut state = self.lock()?; + let mut candidate = state.worker(worker_ref)?.clone(); + if candidate.pending_stop.is_some() + || candidate + .pending_restore + .is_none_or(|pending| pending.operation_id != operation_id) + { + return Err(RuntimeError::RestoreObservationConflict { + worker_id: worker_ref.worker_id, + }); + } + let persisted = candidate.request.as_mut().ok_or_else(|| { + RuntimeError::InvalidRequest("Restore specification is missing".into()) + })?; + if persisted.workspace_api.as_ref().is_some_and(|existing| { + existing.workspace_id != api.workspace_id + || existing.base_url.trim_end_matches('/') != api.base_url.trim_end_matches('/') + }) { + return Err(RuntimeError::InvalidRequest( + "Workspace API preparation cannot change Worker identity or base URL".into(), + )); + } + persisted.workspace_api = Some(api); + *effects_started = true; + match state.persist_worker_record(&candidate) { + Ok(()) => { + state.workers.insert(worker_ref.worker_id, candidate); + } + Err(error @ RuntimeError::StoreCommitOutcomeUnknown { .. }) => { + state.workers.insert(worker_ref.worker_id, candidate); + return Err(error); + } + Err(error) => return Err(error), + } + } + if let Some(attachments) = preparation.workdir_attachments { + let attachments: Vec = serde_json::from_value( + serde_json::to_value(attachments) + .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?, + ) + .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?; + validate_logical_workdir_attachments(&attachments)?; + *effects_started = true; + self.replace_worker_workdir_attachments_under_lock(worker_ref, attachments)?; + } + for access in preparation.repository_access { + if preparation + .repository_access_workdirs + .iter() + .any(|reference| { + reference.working_directory_id == access.working_directory_id + && reference.runtime_id == access.materialization.runtime_id + }) + { + // Workspace completed these at the Workdir owner Runtime after snapshot persistence. + // Never send cross-Runtime credentials to this Worker's Runtime. + continue; + } + let access: WorkingDirectoryRepositoryAccessRequest = serde_json::from_value( + serde_json::to_value(access) + .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?, + ) + .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?; + *effects_started = true; + let runtime = self.clone(); + // The public embedded interface is synchronous. A separate IO executor + // avoids nested Tokio block_on while retaining the lifecycle lock. + std::thread::spawn(move || { + let executor = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?; + executor.block_on( + runtime.authorize_working_directory_repository_access_from_resource(access), + ) + }) + .join() + .map_err(|_| RuntimeError::StatePoisoned)??; + } + Ok(()) } fn restore_worker_under_lock( @@ -1768,7 +2420,17 @@ impl Runtime { worker_ref: &WorkerRef, mode: WorkerRestoreMode, ) -> Result { - let fresh_operation_id = WorkerLifecycleOperationId::new(); + let fresh_operation_id = { + let state = self.lock()?; + let worker = state.worker(worker_ref)?; + worker + .restore_guard + .active_request_id + .as_ref() + .and_then(|id| worker.restore_guard.owners.get(id)) + .map(|owner| owner.operation_id) + .unwrap_or_else(WorkerLifecycleOperationId::new) + }; let (backend, request, operation_id, operation_mode, reconcile, candidate_context) = { let state = self.lock()?; state.ensure_running()?; @@ -1781,7 +2443,13 @@ impl Runtime { )); } let pending_restore = worker.pending_restore; - let reconcile = pending_restore.is_some(); + let reconcile = pending_restore.is_some() + && worker + .restore_guard + .active_request_id + .as_ref() + .and_then(|id| worker.restore_guard.owners.get(id)) + .is_none_or(|owner| owner.execution_started); let operation_id = pending_restore .map(|pending| pending.operation_id) .unwrap_or(fresh_operation_id); @@ -1967,7 +2635,14 @@ impl Runtime { error = %error, "Worker restore commit failed and was rolled back" ); - if let Err(settle_error) = self.settle_restore_rollback(worker_ref) { + let receipt = RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::RolledBack, + "worker_restore_commit_rolled_back", + "Worker restore commit failed and cleanup completed", + ); + if let Err(settle_error) = + self.settle_restore_rollback(worker_ref, &receipt) + { tracing::error!( worker_id = %worker_ref.worker_id, error = %settle_error, @@ -1979,11 +2654,7 @@ impl Runtime { "Worker restore cleanup completed but durable reconciliation is still required", )); } - Ok(RuntimeWorkerRestoreResult::failed( - WorkerRestoreState::RolledBack, - "worker_restore_commit_rolled_back", - "Worker restore commit failed and cleanup completed", - )) + Ok(receipt) } Err(cleanup_error) => { tracing::error!( @@ -2022,7 +2693,12 @@ impl Runtime { )); } self.discard_restore_candidate_context(worker_ref, &candidate_context)?; - if let Err(error) = self.settle_restore_rollback(worker_ref) { + let receipt = RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::Rejected, + "worker_restore_rejected", + sanitize_worker_create_failure_message(&result.message_or_default()), + ); + if let Err(error) = self.settle_restore_rollback(worker_ref, &receipt) { tracing::error!( worker_id = %worker_ref.worker_id, error = %error, @@ -2034,16 +2710,20 @@ impl Runtime { "Worker restore was rejected but durable reconciliation is still required", )); } - Ok(RuntimeWorkerRestoreResult::failed( - WorkerRestoreState::Rejected, - "worker_restore_rejected", - sanitize_worker_create_failure_message(&result.message_or_default()), - )) + Ok(receipt) } WorkerExecutionSpawnResult::RolledBack(_result) => { + let receipt = RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::RolledBack, + "worker_restore_rolled_back", + sanitize_worker_create_failure_message(&_result.message_or_default()), + ); self.discard_restore_candidate_context(worker_ref, &candidate_context)?; #[cfg(feature = "fs-store")] - if let Err(error) = self.lock()?.record_restore_failure(worker_ref, &_result) { + if let Err(error) = self + .lock()? + .record_restore_failure(worker_ref, &_result, &receipt) + { tracing::error!( worker_id = %worker_ref.worker_id, error = %error, @@ -2056,12 +2736,8 @@ impl Runtime { )); } #[cfg(not(feature = "fs-store"))] - self.settle_restore_rollback(worker_ref)?; - Ok(RuntimeWorkerRestoreResult::failed( - WorkerRestoreState::RolledBack, - "worker_restore_rolled_back", - sanitize_worker_create_failure_message(&_result.message_or_default()), - )) + self.settle_restore_rollback(worker_ref, &receipt)?; + Ok(receipt) } WorkerExecutionSpawnResult::ReconciliationRequired { result: _result, @@ -2945,13 +3621,25 @@ impl Runtime { ) }; if status == WorkerStatus::Stopped && pending_stop.is_none() && pending_restore.is_none() { - let state = self.lock()?; - let worker = state.worker(worker_ref)?; - return Ok(WorkerLifecycleAck { + let mut state = self.lock()?; + let mut worker = state.worker(worker_ref)?.clone(); + worker.restore_guard.rotate(); + worker + .restore_guard + .finish(RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::RolledBack, + "worker_restore_superseded_by_stop", + "Worker Restore was superseded by Stop", + )); + state.persist_worker_record(&worker)?; + let ack = WorkerLifecycleAck { worker_ref: worker_ref.clone(), status: worker.status, worker_state: worker.worker_state.clone(), - }); + }; + state.workers.insert(worker_ref.worker_id, worker); + state.publish_worker_upsert(worker_ref.worker_id)?; + return Ok(ack); } let backend = backend.ok_or_else(|| RuntimeError::WorkerExecutionUnavailable { worker_id: worker_ref.worker_id, @@ -3357,7 +4045,8 @@ impl Runtime { .workers .values() .filter(|worker| { - worker.pending_restore.is_none() + worker.restore_guard.active_request_id.is_none() + && worker.pending_restore.is_none() && worker.pending_stop.is_none() && worker.execution_bound && worker.status.is_active() @@ -3393,25 +4082,31 @@ impl Runtime { } } - let pending_restores = { - let state = self.lock()?; - state - .workers - .values() - .filter(|worker| worker.pending_stop.is_none()) - .filter_map(|worker| { - worker - .pending_restore - .map(|pending| (worker.worker_ref.clone(), pending.mode)) - }) - .collect::>() - }; + let pending_restores = + { + let state = self.lock()?; + state + .workers + .values() + .filter(|worker| worker.pending_stop.is_none()) + .filter_map(|worker| { + worker + .pending_restore + .map(|pending| (worker.worker_ref.clone(), pending.mode)) + .or_else(|| { + worker.restore_guard.active_request_id.as_ref().map(|_| { + (worker.worker_ref.clone(), WorkerRestoreMode::Explicit) + }) + }) + }) + .collect::>() + }; for (worker_ref, mode) in pending_restores { let operation_lock = self.worker_operation_lock(worker_ref.worker_id)?; let _operation_guard = operation_lock .lock() .map_err(|_| RuntimeError::StatePoisoned)?; - let result = self.restore_worker_under_lock(&worker_ref, mode)?; + let result = self.restore_admitted_worker_under_lock(&worker_ref, mode)?; self.record_startup_restore_result(&worker_ref, &result)?; } @@ -3421,7 +4116,7 @@ impl Runtime { .lock() .map_err(|_| RuntimeError::StatePoisoned)?; let result = - self.restore_worker_under_lock(&worker_ref, WorkerRestoreMode::Automatic)?; + self.restore_admitted_worker_under_lock(&worker_ref, WorkerRestoreMode::Automatic)?; self.record_startup_restore_result(&worker_ref, &result)?; } Ok(()) @@ -3516,6 +4211,7 @@ impl Runtime { worker_ref.worker_id ))); } + candidate.restore_guard.rotate(); candidate.pending_stop = Some(PendingWorkerStop { operation_id, last_settled_status: candidate.status, @@ -3554,10 +4250,18 @@ impl Runtime { if let Some(context) = candidate.restore_candidate_context.take() { context.discard_candidate(); } + candidate.restore_guard.rotate(); candidate.status = WorkerStatus::Stopped; candidate.worker_state = None; candidate.restore_intent = WorkerRestoreIntent::Explicit; + candidate + .restore_guard + .finish(RuntimeWorkerRestoreResult::failed( + WorkerRestoreState::RolledBack, + "worker_restore_superseded_by_stop", + "Worker Restore was superseded by Stop", + )); candidate.internal_workers.clear(); state.persist_worker_record(&candidate)?; let ack = WorkerLifecycleAck { @@ -3587,7 +4291,11 @@ impl Runtime { let mut state = self.lock()?; state.ensure_worker_ref(worker_ref)?; let mut candidate = state.worker(worker_ref)?.clone(); - if candidate.has_pending_lifecycle_operation() { + if candidate.pending_stop.is_some() + || candidate + .pending_restore + .is_some_and(|pending| pending.operation_id != operation_id) + { return Err(RuntimeError::InvalidRequest(format!( "worker {} already has pending lifecycle reconciliation", worker_ref.worker_id @@ -3599,6 +4307,18 @@ impl Runtime { worker_ref.worker_id ))); } + if let Some(id) = &candidate.restore_guard.active_request_id { + let owner = candidate.restore_guard.owners.get_mut(id).ok_or_else(|| { + RuntimeError::InvalidRequest("Active Restore request has no persisted owner".into()) + })?; + if owner.operation_id != operation_id || owner.receipt.is_some() { + return Err(RuntimeError::RestoreObservationConflict { + worker_id: worker_ref.worker_id, + }); + } + owner.execution_started = true; + } + candidate.restore_guard.rotate(); candidate.pending_restore = Some(PendingWorkerRestore { operation_id, mode, @@ -3618,9 +4338,15 @@ impl Runtime { } } - fn settle_restore_rollback(&self, worker_ref: &WorkerRef) -> Result<(), RuntimeError> { + fn settle_restore_rollback( + &self, + worker_ref: &WorkerRef, + receipt: &RuntimeWorkerRestoreResult, + ) -> Result<(), RuntimeError> { let mut state = self.lock()?; let mut candidate = state.worker(worker_ref)?.clone(); + candidate.restore_guard.rotate(); + candidate.restore_guard.finish(receipt.clone()); candidate.pending_restore = None; candidate.restore_candidate_context = None; state.persist_worker_record(&candidate)?; @@ -3639,6 +4365,7 @@ impl Runtime { let mut state = self.lock()?; state.ensure_worker_ref(worker_ref)?; let mut candidate = state.worker(worker_ref)?.clone(); + candidate.restore_guard.rotate(); candidate.execution_metadata_available = true; candidate.execution_bound = true; candidate.pending_restore = None; @@ -3662,6 +4389,9 @@ impl Runtime { // Validate the exact subscription projection before persistence. The // subsequent publish uses the same pure projection while the Worker // operation lock keeps this candidate stable. + candidate.restore_guard.rotate(); + let accepted = RuntimeWorkerRestoreResult::accepted(candidate.detail()); + candidate.restore_guard.finish(accepted); let _subscription_projection = state.subscription_worker(&candidate)?; // Persist the complete candidate before making it observable. A failed @@ -3691,6 +4421,7 @@ impl Runtime { let mut state = self.lock()?; let mut candidate = state.worker(worker_ref)?.clone(); + candidate.restore_guard.rotate(); candidate.execution_metadata_available = true; candidate.execution_bound = true; candidate.status = WorkerStatus::Idle; @@ -3720,6 +4451,7 @@ impl Runtime { ))); } + candidate.restore_guard.rotate(); candidate.execution_metadata_available = true; candidate.execution_bound = true; candidate.restore_candidate_context = Some(candidate_context); @@ -3737,6 +4469,7 @@ impl Runtime { let mut state = self.lock()?; let mut candidate = state.worker(worker_ref)?.clone(); + candidate.restore_guard.rotate(); candidate.execution_metadata_available = false; candidate.execution_bound = true; candidate.status = WorkerStatus::Idle; @@ -4266,6 +4999,15 @@ impl RuntimeState { workers.insert( worker_id, WorkerRecord { + restore_guard: { + let mut guard = worker.restore_guard; + // A prior active controller is not live in this process. + // Stopped observations remain stable across restart. + if worker.status.is_active() && execution_bound { + guard.rotate(); + } + guard + }, worker_ref: worker.worker_ref, worker_id: worker.worker_id, status: worker.status, @@ -4622,6 +5364,7 @@ impl RuntimeState { ProfileSelector::Builtin(name) | ProfileSelector::Named(name) => Some(name.clone()), }; Ok(SubscriptionWorker { + restore_observation_token: Some(worker.restore_guard.generation.clone()), worker_id, runtime_id: None, resource_key: None, @@ -4767,6 +5510,7 @@ impl RuntimeState { &mut self, worker_ref: &WorkerRef, result: &WorkerExecutionResult, + receipt: &RuntimeWorkerRestoreResult, ) -> Result<(), RuntimeError> { let message = sanitize_worker_create_failure_message(&result.message_or_default()); let diagnostic_id = self.next_diagnostic_id; @@ -4778,9 +5522,11 @@ impl RuntimeState { let mut candidate = self.worker(worker_ref)?.clone(); candidate.execution_bound = false; + candidate.restore_guard.finish(receipt.clone()); candidate.pending_restore = None; candidate.pending_stop = None; candidate.restore_candidate_context = None; + candidate.restore_guard.rotate(); candidate.status = WorkerStatus::Stopped; candidate.restore_intent = WorkerRestoreIntent::Explicit; candidate.internal_workers.clear(); @@ -5159,8 +5905,52 @@ struct PendingWorkerStop { last_settled_status: WorkerStatus, } +/// Durable Restore admission/replay metadata. Never discard receipts on stop. +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +pub(crate) struct RestoreRequestOwner { + fingerprint: String, + operation_id: WorkerLifecycleOperationId, + request: runtime_api::WorkerRestoreRequest, + preparation_applied: bool, + #[serde(default)] + execution_started: bool, + receipt: Option, +} + +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +pub(crate) struct RestoreGuard { + generation: String, + active_request_id: Option, + owners: BTreeMap, +} + +impl Default for RestoreGuard { + fn default() -> Self { + Self { + generation: uuid::Uuid::now_v7().to_string(), + active_request_id: None, + owners: BTreeMap::new(), + } + } +} + +impl RestoreGuard { + fn rotate(&mut self) { + self.generation = uuid::Uuid::now_v7().to_string(); + } + + fn finish(&mut self, result: RuntimeWorkerRestoreResult) { + if let Some(id) = self.active_request_id.take() + && let Some(owner) = self.owners.get_mut(&id) + { + owner.receipt = Some(result); + } + } +} + #[derive(Debug, Clone)] struct WorkerRecord { + restore_guard: RestoreGuard, worker_ref: WorkerRef, worker_id: WorkerId, status: WorkerStatus, @@ -5187,6 +5977,9 @@ struct WorkerRecord { impl WorkerRecord { fn apply_worker_state(&mut self, incoming: &protocol::WorkerStateSnapshot) { + if self.worker_state.as_ref().map(|state| &state.state) != Some(&incoming.state) { + self.restore_guard.rotate(); + } if let Some(request_id) = &incoming.last_finished_submission_request_id { self.last_finished_submission_request_id = Some(request_id.clone()); } @@ -5213,6 +6006,7 @@ impl WorkerRecord { fn summary(&self) -> WorkerSummary { WorkerSummary { + restore_observation_token: Some(self.restore_guard.generation.clone()), worker_ref: self.worker_ref.clone(), worker_id: self.worker_id, status: self.status, @@ -5230,6 +6024,7 @@ impl WorkerRecord { fn detail(&self) -> WorkerDetail { WorkerDetail { + restore_observation_token: Some(self.restore_guard.generation.clone()), worker_ref: self.worker_ref.clone(), worker_id: self.worker_id, status: self.status, @@ -5301,6 +6096,7 @@ impl WorkerRecord { } }; PersistedWorkerRecord { + restore_guard: self.restore_guard.clone(), worker_ref: self.worker_ref.clone(), worker_id: self.worker_id, profile: self.profile.clone(), @@ -5543,6 +6339,9 @@ fn runtime_worker_create_failure_fields( error: &RuntimeError, ) -> (&'static str, Option, Option) { match error { + RuntimeError::RestoreObservationConflict { .. } => { + ("restore_observation_conflict", None, None) + } RuntimeError::RuntimeStopped => ("runtime_stopped", None, None), RuntimeError::RuntimeStoreAlreadyOpen { .. } => ("runtime_store_already_open", None, None), RuntimeError::InvalidInitialInputKind { .. } => ("invalid_initial_input_kind", None, None), @@ -7682,7 +8481,12 @@ mod tests { vec![attachment.clone()], ) .unwrap(); - runtime.restore_worker(&worker.worker_ref).unwrap(); + runtime + .restore_worker( + &worker.worker_ref, + runtime.test_restore_request(&worker.worker_ref), + ) + .unwrap(); let state = runtime.lock().unwrap(); assert_eq!( state @@ -7880,7 +8684,12 @@ mod tests { ); runtime.stop_worker(&detail.worker_ref, None).unwrap(); - runtime.restore_worker(&detail.worker_ref).unwrap(); + runtime + .restore_worker( + &detail.worker_ref, + runtime.test_restore_request(&detail.worker_ref), + ) + .unwrap(); assert_eq!( backend.config_bundles.lock().unwrap().as_slice(), &[Some(bundle), None] @@ -7940,7 +8749,12 @@ mod tests { .unwrap(); runtime.stop_worker(&detail.worker_ref, None).unwrap(); runtime.lock().unwrap().config_bundles.clear(); - runtime.restore_worker(&detail.worker_ref).unwrap(); + runtime + .restore_worker( + &detail.worker_ref, + runtime.test_restore_request(&detail.worker_ref), + ) + .unwrap(); assert_eq!( backend.config_bundles.lock().unwrap().as_slice(), &[Some(bundle), None] @@ -7960,7 +8774,12 @@ mod tests { .unwrap() .config_bundles .insert(replacement.metadata.id.clone(), replacement); - runtime.restore_worker(&detail.worker_ref).unwrap(); + runtime + .restore_worker( + &detail.worker_ref, + runtime.test_restore_request(&detail.worker_ref), + ) + .unwrap(); assert_eq!( backend.config_bundles.lock().unwrap().as_slice(), &[Some(bundle), None] @@ -8077,7 +8896,12 @@ mod tests { let restore_runtime = runtime.clone(); let restore_ref = created.worker_ref.clone(); - let restoring = std::thread::spawn(move || restore_runtime.restore_worker(&restore_ref)); + let restoring = std::thread::spawn(move || { + restore_runtime.restore_worker( + &restore_ref, + restore_runtime.test_restore_request(&restore_ref), + ) + }); assert!(gate.wait_for_entered(1, std::time::Duration::from_secs(2))); let remove_runtime = runtime.clone(); let remove_ref = created.worker_ref.clone(); @@ -8355,7 +9179,9 @@ mod tests { let restore_runtime = runtime.clone(); let restore_ref = created.worker_ref.clone(); - let restoring = std::thread::spawn(move || restore_runtime.restore_worker(&restore_ref)); + let request = runtime.test_restore_request(&restore_ref); + let restoring = + std::thread::spawn(move || restore_runtime.restore_worker(&restore_ref, request)); std::thread::sleep(std::time::Duration::from_millis(100)); assert_eq!( *backend.restore_count.lock().unwrap(), @@ -8365,8 +9191,18 @@ mod tests { gate.release(); stopping.join().unwrap().unwrap(); - let restored = restoring.join().unwrap().unwrap(); + assert!(matches!( + restoring.join().unwrap(), + Err(RuntimeError::RestoreObservationConflict { .. }) + )); assert_eq!(*backend.stop_count.lock().unwrap(), 1); + assert_eq!(*backend.restore_count.lock().unwrap(), 0); + let restored = runtime + .restore_worker( + &created.worker_ref, + runtime.test_restore_request(&created.worker_ref), + ) + .unwrap(); assert_eq!(*backend.restore_count.lock().unwrap(), 1); assert_eq!(restored.status, WorkerStatus::Idle); assert_eq!( @@ -8478,15 +9314,17 @@ mod tests { let gate = Arc::new(RestoreGate::default()); *backend.restore_gate.lock().unwrap() = Some(gate.clone()); + let request = runtime.test_restore_request(&created.worker_ref); let start = Arc::new(Barrier::new(3)); let mut threads = Vec::new(); for _ in 0..2 { let runtime = runtime.clone(); let worker_ref = created.worker_ref.clone(); let start = start.clone(); + let request = request.clone(); threads.push(std::thread::spawn(move || { start.wait(); - runtime.restore_worker(&worker_ref) + runtime.restore_worker(&worker_ref, request) })); } start.wait(); @@ -8508,6 +9346,1089 @@ mod tests { assert!(runtime.worker_operations.lock().unwrap().is_empty()); } + #[test] + fn restore_guard_coordination_is_authorized_and_none_cannot_admit_new_intent() { + let (runtime, backend) = runtime_and_backend(); + let scope = scope("workspace-a", "server-a"); + let worker = runtime + .create_worker_scoped(&scope, scoped_task_request("coordinate", "workspace-a")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let observed = runtime.worker_detail(&worker.worker_ref).unwrap(); + let mut request = runtime.test_restore_request(&worker.worker_ref); + request.preparation = None; + let coordination = runtime_api::WorkerRestoreCoordinationRequest { + expected_observation_token: request.expected_observation_token.clone(), + request_id: request.request_id.clone(), + preparation: None, + }; + assert_eq!( + runtime + .coordinate_worker_restore_operation( + &worker.worker_ref, + coordination.clone(), + Some(&scope) + ) + .unwrap(), + (None, None) + ); + assert_eq!(runtime.worker_detail(&worker.worker_ref).unwrap(), observed); + assert!( + runtime + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .restore_guard + .owners + .is_empty() + ); + assert!(matches!( + runtime.restore_worker_operation_scoped(&scope, &worker.worker_ref, request.clone()), + Err(RuntimeError::InvalidRequest(_)) + )); + let wrong = RuntimeWorkspaceScope::new("workspace-a", "different-server"); + assert!( + runtime + .coordinate_worker_restore_operation( + &worker.worker_ref, + coordination.clone(), + Some(&wrong) + ) + .is_err() + ); + let mut admitted = coordination.clone(); + admitted.preparation = Some(runtime_api::WorkerRestorePreparation::default()); + assert!( + runtime + .coordinate_worker_restore_operation(&worker.worker_ref, admitted, Some(&scope)) + .unwrap() + .1 + .is_some() + ); + let operation_id = runtime + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .pending_restore + .unwrap() + .operation_id; + assert!( + runtime + .coordinate_worker_restore_operation( + &worker.worker_ref, + coordination.clone(), + Some(&scope) + ) + .unwrap() + .1 + .is_some() + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 0); + request.preparation = Some(runtime_api::WorkerRestorePreparation::default()); + let receipt = runtime + .restore_worker_operation_scoped(&scope, &worker.worker_ref, request) + .unwrap(); + assert_eq!(receipt.state, WorkerRestoreState::Accepted); + assert_eq!( + backend.restore_operation_ids.lock().unwrap().as_slice(), + &[operation_id] + ); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + assert!( + runtime + .coordinate_worker_restore_operation( + &worker.worker_ref, + coordination.clone(), + Some(&wrong) + ) + .is_err() + ); + assert_eq!( + runtime + .coordinate_worker_restore_operation(&worker.worker_ref, coordination, Some(&scope)) + .unwrap() + .0, + Some(receipt) + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 1); + assert_eq!( + runtime.worker_detail(&worker.worker_ref).unwrap().status, + WorkerStatus::Stopped + ); + } + + #[test] + fn restore_guard_resource_snapshot_requires_workspace_completion_before_execution() { + let (runtime, backend) = runtime_and_backend(); + let scope = scope("workspace-a", "server-a"); + let worker = runtime + .create_worker_scoped( + &scope, + scoped_task_request("resource coordination", "workspace-a"), + ) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let mut request = runtime.test_restore_request(&worker.worker_ref); + let mut preparation = runtime_api::WorkerRestorePreparation::default(); + preparation.repository_access_workdirs.push( + runtime_api::RepositoryAccessWorkdirReference { + runtime_id: "peer-runtime".into(), + working_directory_id: "peer-workdir".into(), + }, + ); + let mut coordination = runtime_api::WorkerRestoreCoordinationRequest { + expected_observation_token: request.expected_observation_token.clone(), + request_id: request.request_id.clone(), + preparation: Some(preparation.clone()), + }; + runtime + .coordinate_worker_restore_operation( + &worker.worker_ref, + coordination.clone(), + Some(&scope), + ) + .unwrap(); + let operation_id = runtime + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .pending_restore + .unwrap() + .operation_id; + // Model a retained resource snapshot before the peer authorization callback completes. + preparation + .repository_access + .push(runtime_api::WorkingDirectoryRepositoryAccessRequest { + working_directory_id: "peer-workdir".into(), + materialization: runtime_api::RepositoryMaterializationContext { + workspace_id: "workspace-a".into(), + runtime_id: "peer-runtime".into(), + operation_id: format!("worker-restore:{}", request.request_id), + config_revision: 1, + config_projection_digest: "snapshot".into(), + ssh: None, + }, + }); + coordination.preparation = Some(preparation.clone()); + assert!( + runtime + .coordinate_worker_restore_operation(&worker.worker_ref, coordination, Some(&scope)) + .unwrap() + .1 + .is_some() + ); + request.preparation = None; + assert_eq!( + runtime + .restore_worker_operation_scoped(&scope, &worker.worker_ref, request.clone()) + .unwrap() + .state, + WorkerRestoreState::ReconciliationRequired + ); + assert_eq!( + runtime + .restore_admitted_worker_under_lock(&worker.worker_ref, WorkerRestoreMode::Explicit) + .unwrap() + .state, + WorkerRestoreState::ReconciliationRequired + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 0); + request.preparation = Some(runtime_api::WorkerRestorePreparation::default()); + assert_eq!( + runtime + .restore_worker_operation_scoped(&scope, &worker.worker_ref, request.clone()) + .unwrap() + .state, + WorkerRestoreState::ReconciliationRequired + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 0); + request.preparation = Some(preparation); + assert_eq!( + runtime + .restore_worker_operation_scoped(&scope, &worker.worker_ref, request) + .unwrap() + .state, + WorkerRestoreState::Accepted + ); + assert_eq!( + backend.restore_operation_ids.lock().unwrap().as_slice(), + &[operation_id] + ); + } + + #[test] + fn restore_guard_stale_preparation_has_no_effects() { + let (runtime, backend) = runtime_and_backend(); + let worker = runtime + .create_worker(task_request("stale preparation")) + .unwrap(); + let mut stale = runtime.test_restore_request(&worker.worker_ref); + stale.preparation = Some(serde_json::from_value(serde_json::json!({ + "workspace_api": { "workspace_id": "must-not-bind", "base_url": "https://invalid.example" }, + "workdir_attachments": [], + "repository_access": [] + })).unwrap()); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let before = runtime.worker_detail(&worker.worker_ref).unwrap(); + assert!(matches!( + runtime.restore_worker_operation(&worker.worker_ref, stale), + Err(RuntimeError::RestoreObservationConflict { .. }) + )); + assert_eq!(runtime.worker_detail(&worker.worker_ref).unwrap(), before); + let state = runtime.lock().unwrap(); + let record = state.worker(&worker.worker_ref).unwrap(); + assert!(record.request.as_ref().unwrap().workspace_api.is_none()); + assert!(record.restore_guard.owners.is_empty()); + assert_eq!(*backend.restore_count.lock().unwrap(), 0); + assert!(backend.repository_accesses.lock().unwrap().is_empty()); + } + + #[test] + fn restore_guard_lifecycle_aba_and_historical_replay_after_stop() { + let (runtime, backend) = runtime_and_backend(); + let worker = runtime.create_worker(task_request("restore ABA")).unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let request = runtime.test_restore_request(&worker.worker_ref); + let mut other_intent = request.clone(); + other_intent.request_id = "other-intent".into(); + let accepted = runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!(accepted.state, WorkerRestoreState::Accepted); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + assert!(matches!( + runtime.restore_worker_operation(&worker.worker_ref, other_intent), + Err(RuntimeError::RestoreObservationConflict { .. }) + )); + let stopped = runtime.worker_detail(&worker.worker_ref).unwrap(); + assert_ne!( + stopped.restore_observation_token.as_deref(), + Some(request.expected_observation_token.as_str()) + ); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap(), + accepted + ); + assert_eq!(runtime.worker_detail(&worker.worker_ref).unwrap(), stopped); + assert_eq!(*backend.restore_count.lock().unwrap(), 1); + let mut altered = request; + altered.expected_observation_token = stopped.restore_observation_token.unwrap(); + assert!(matches!( + runtime.restore_worker_operation(&worker.worker_ref, altered), + Err(RuntimeError::RestoreObservationConflict { .. }) + )); + } + + #[test] + fn restore_guard_protocol_state_aba_rotates_generation() { + let (runtime, backend) = runtime_and_backend(); + let worker = runtime.create_worker(task_request("state ABA")).unwrap(); + let request = runtime.test_restore_request(&worker.worker_ref); + { + let mut state = runtime.lock().unwrap(); + for status in [ + protocol::WorkerStatus::Running, + protocol::WorkerStatus::Idle, + ] { + assert!(state.project_protocol_event_to_worker_state( + &worker.worker_ref, + &protocol::Event::WorkerState { + snapshot: status.into() + } + )); + state.persist_worker(&worker.worker_id).unwrap(); + } + } + assert_eq!( + runtime + .worker_detail(&worker.worker_ref) + .unwrap() + .worker_state + .unwrap() + .state, + protocol::WorkerState::Idle + ); + assert!(matches!( + runtime.restore_worker_operation(&worker.worker_ref, request), + Err(RuntimeError::RestoreObservationConflict { .. }) + )); + assert_eq!(*backend.restore_count.lock().unwrap(), 0); + } + + #[test] + fn restore_guard_distinct_contenders_are_fenced_before_preparation() { + let (runtime, backend) = runtime_and_backend(); + let worker = runtime + .create_worker(task_request("distinct contenders")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let request = runtime.test_restore_request(&worker.worker_ref); + let mut contender = request.clone(); + contender.request_id = "losing-request".into(); + contender.preparation = Some(serde_json::from_value(serde_json::json!({ + "workspace_api": { "workspace_id": "must-not-bind", "base_url": "https://invalid.example" }, + "repository_access": [] + })).unwrap()); + let gate = Arc::new(RestoreGate::default()); + backend.restore_gate.lock().unwrap().replace(gate.clone()); + let first_runtime = runtime.clone(); + let first_ref = worker.worker_ref.clone(); + let first = + std::thread::spawn(move || first_runtime.restore_worker_operation(&first_ref, request)); + assert!(gate.wait_for_entered(1, std::time::Duration::from_secs(2))); + let second_runtime = runtime.clone(); + let second_ref = worker.worker_ref.clone(); + let second = std::thread::spawn(move || { + second_runtime.restore_worker_operation(&second_ref, contender) + }); + gate.release(); + assert_eq!( + first.join().unwrap().unwrap().state, + WorkerRestoreState::Accepted + ); + assert!(matches!( + second.join().unwrap(), + Err(RuntimeError::RestoreObservationConflict { .. }) + )); + assert_eq!(*backend.restore_count.lock().unwrap(), 1); + assert!( + runtime + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .request + .as_ref() + .unwrap() + .workspace_api + .is_none() + ); + } + + #[test] + fn restore_guard_applies_preparation_once_and_replays_changed_candidate() { + let (runtime, backend) = runtime_and_backend(); + let scope = scope("workspace-a", "server-a"); + let worker = runtime + .create_worker_scoped( + &scope, + scoped_task_request("prepared restore", "workspace-a"), + ) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let mut request = runtime.test_restore_request(&worker.worker_ref); + request.preparation = Some(serde_json::from_value(serde_json::json!({ + "workspace_api": { "workspace_id": "workspace-a", "base_url": "https://workspace.example/workspace-a" }, + "workdir_attachments": [], + "repository_access": [] + })).unwrap()); + let receipt = runtime + .restore_worker_operation_scoped(&scope, &worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!(receipt.state, WorkerRestoreState::Accepted); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let logical = vec![LogicalWorkdirAttachment { + alias: workdir::WorkdirAttachmentAlias::new("changed-after-stop").unwrap(), + working_directory_id: "workdir-later".into(), + capabilities: workdir::WorkdirSessionCapabilities::READ_ONLY, + }]; + runtime + .replace_worker_workdir_attachments(&worker.worker_ref, logical.clone()) + .unwrap(); + assert_eq!( + runtime + .restore_worker_operation_scoped(&scope, &worker.worker_ref, request.clone()) + .unwrap(), + receipt + ); + assert_eq!( + runtime + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .logical_workdir_attachments, + logical + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 1); + request.preparation = Some(serde_json::from_value(serde_json::json!({ + "workspace_api": { "workspace_id": "must-not-bind", "base_url": "https://fresh.invalid.example" }, + "workdir_attachments": [], + "repository_access": [] + })).unwrap()); + assert_eq!( + runtime + .restore_worker_operation_scoped(&scope, &worker.worker_ref, request.clone()) + .unwrap(), + receipt + ); + let state = runtime.lock().unwrap(); + let record = state.worker(&worker.worker_ref).unwrap(); + assert_eq!(record.logical_workdir_attachments, logical); + let owner = &record.restore_guard.owners[&request.request_id]; + assert_eq!( + owner + .request + .preparation + .as_ref() + .unwrap() + .workspace_api + .as_ref() + .unwrap() + .base_url, + "https://workspace.example/workspace-a" + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 1); + } + + #[cfg(feature = "fs-store")] + #[test] + fn restore_guard_admission_retry_uses_first_persisted_preparation() { + let dir = tempfile::tempdir().unwrap(); + let options = FsRuntimeStoreOptions::new(dir.path().join("runtime")) + .with_runtime_id("guard-first-preparation"); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = + Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()).unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let scope = scope("workspace-a", "server-a"); + let mut create = scoped_task_request("first preparation", "workspace-a"); + create.workspace_api.as_mut().unwrap().base_url = "https://first.example".into(); + let worker = runtime.create_worker_scoped(&scope, create).unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let mut request = runtime.test_restore_request(&worker.worker_ref); + request.preparation = Some(serde_json::from_value(serde_json::json!({ + "workspace_api": { "workspace_id": "workspace-a", "base_url": "https://first.example" }, + "repository_access": [] + })).unwrap()); + runtime_store(&runtime) + .fail_next_worker_write(crate::fs_store::AtomicWriteFault::AfterRename); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap() + .state, + WorkerRestoreState::ReconciliationRequired + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 0); + drop(runtime); + // No backend on restart: admission survives, preparation has not run. + let resumed = Runtime::with_fs_store(options).unwrap(); + resumed.lock().unwrap().execution_backend = + Some(crate::execution::WorkerExecutionBackendRef::new(backend.clone()).unwrap()); + request.preparation = Some(serde_json::from_value(serde_json::json!({ + "workspace_api": { "workspace_id": "must-not-bind", "base_url": "https://fresh.invalid.example" }, + "repository_access": [] + })).unwrap()); + assert!( + resumed.worker_detail(&worker.worker_ref).is_ok(), + "{:?}", + resumed.lock().unwrap().diagnostics + ); + let receipt = resumed + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!(receipt.state, WorkerRestoreState::Accepted); + assert_eq!(*backend.restore_count.lock().unwrap(), 1); + let state = resumed.lock().unwrap(); + let record = state.worker(&worker.worker_ref).unwrap(); + assert_eq!( + record + .request + .as_ref() + .unwrap() + .workspace_api + .as_ref() + .unwrap() + .base_url, + "https://first.example" + ); + let owner = &record.restore_guard.owners[&request.request_id]; + assert_eq!( + owner + .request + .preparation + .as_ref() + .unwrap() + .workspace_api + .as_ref() + .unwrap() + .base_url, + "https://first.example" + ); + assert_eq!(owner.receipt.as_ref(), Some(&receipt)); + } + + #[cfg(feature = "fs-store")] + #[test] + fn restore_guard_interrupted_preparation_replays_original_pending_operation() { + let dir = tempfile::tempdir().unwrap(); + let options = FsRuntimeStoreOptions::new(dir.path().join("runtime")) + .with_runtime_id("guard-partial-preparation"); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = + Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()).unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let scope = scope("workspace-a", "server-a"); + let mut create = scoped_task_request("interrupted preparation", "workspace-a"); + create.workspace_api = None; + let worker = runtime.create_worker_scoped(&scope, create).unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let mut request = runtime.test_restore_request(&worker.worker_ref); + request.preparation = Some(serde_json::from_value(serde_json::json!({ + "workspace_api": { "workspace_id": "workspace-a", "base_url": "https://partially-applied.example" }, + "workdir_attachments": [], "repository_access": [] + })).unwrap()); + let (_, preparation) = runtime + .coordinate_worker_restore_operation( + &worker.worker_ref, + runtime_api::WorkerRestoreCoordinationRequest { + expected_observation_token: request.expected_observation_token.clone(), + request_id: request.request_id.clone(), + preparation: request.preparation.clone(), + }, + Some(&scope), + ) + .unwrap(); + assert!(preparation.is_some()); + let operation_id = runtime + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .pending_restore + .unwrap() + .operation_id; + runtime_store(&runtime) + .fail_next_worker_write(crate::fs_store::AtomicWriteFault::AfterRename); + request.preparation = None; + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap() + .state, + WorkerRestoreState::ReconciliationRequired + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 0); + assert_eq!( + runtime + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .pending_restore + .unwrap() + .operation_id, + operation_id + ); + drop(runtime); + let resumed = Runtime::with_fs_store(options).unwrap(); + resumed.lock().unwrap().execution_backend = + Some(crate::execution::WorkerExecutionBackendRef::new(backend.clone()).unwrap()); + let receipt = resumed + .restore_worker_operation(&worker.worker_ref, request) + .unwrap(); + assert_eq!(receipt.state, WorkerRestoreState::Accepted); + assert_eq!(*backend.restore_count.lock().unwrap(), 1); + assert_eq!( + backend.restore_operation_ids.lock().unwrap().as_slice(), + &[operation_id] + ); + assert!( + resumed + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .pending_restore + .is_none() + ); + } + + #[cfg(feature = "fs-store")] + #[test] + fn restore_guard_unowned_pending_operation_recovers_exact_lifecycle_owner_after_restart() { + let dir = tempfile::tempdir().unwrap(); + let options = FsRuntimeStoreOptions::new(dir.path().join("runtime")) + .with_runtime_id("guard-unowned-pending"); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = + Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()).unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("recover existing pending operation")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + *backend.restore_result.lock().unwrap() = + Some(WorkerExecutionSpawnResult::ReconciliationRequired { + result: WorkerExecutionResult::errored( + WorkerExecutionOperation::Restore, + "unknown acknowledgement", + ), + worker_state: None, + workdir_attachments: Vec::new(), + }); + // Existing startup/old-contract pending operation: no public receipt owner. + assert_eq!( + runtime + .restore_worker_under_lock(&worker.worker_ref, WorkerRestoreMode::Explicit) + .unwrap() + .state, + WorkerRestoreState::ReconciliationRequired + ); + let operation_id = runtime + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .pending_restore + .unwrap() + .operation_id; + drop(runtime); + let path = dir + .path() + .join("runtime/workers") + .join(worker.worker_id.to_string()) + .join("worker.json"); + let mut aggregate: serde_json::Value = + serde_json::from_slice(&std::fs::read(&path).unwrap()).unwrap(); + aggregate.as_object_mut().unwrap().remove("restore_guard"); + std::fs::write(&path, serde_json::to_vec_pretty(&aggregate).unwrap()).unwrap(); + let resumed = Runtime::with_fs_store(options).unwrap(); + resumed.lock().unwrap().execution_backend = + Some(crate::execution::WorkerExecutionBackendRef::new(backend.clone()).unwrap()); + let request = runtime_api::WorkerRestoreCoordinationRequest { + expected_observation_token: resumed + .worker_detail(&worker.worker_ref) + .unwrap() + .restore_observation_token + .unwrap(), + request_id: "explicit-pending-recovery".into(), + preparation: None, + }; + assert_eq!( + resumed + .coordinate_worker_restore_operation(&worker.worker_ref, request.clone(), None) + .unwrap() + .0 + .unwrap() + .state, + WorkerRestoreState::ReconciliationRequired + ); + let count = *backend.restore_count.lock().unwrap(); + let mut competitor = request.clone(); + competitor.request_id = "different-intent".into(); + assert!(matches!( + resumed.coordinate_worker_restore_operation(&worker.worker_ref, competitor, None), + Err(RuntimeError::RestoreObservationConflict { .. }) + )); + assert_eq!(*backend.restore_count.lock().unwrap(), count); + *backend.restore_result.lock().unwrap() = None; + assert_eq!( + resumed + .coordinate_worker_restore_operation(&worker.worker_ref, request.clone(), None) + .unwrap() + .0 + .unwrap() + .state, + WorkerRestoreState::Accepted + ); + assert!( + backend + .restore_operation_ids + .lock() + .unwrap() + .iter() + .all(|id| *id == operation_id) + ); + assert!( + resumed + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .pending_restore + .is_none() + ); + resumed.stop_worker(&worker.worker_ref, None).unwrap(); + let count = *backend.restore_count.lock().unwrap(); + assert_eq!( + resumed + .coordinate_worker_restore_operation(&worker.worker_ref, request, None) + .unwrap() + .0 + .unwrap() + .state, + WorkerRestoreState::Accepted + ); + assert_eq!(*backend.restore_count.lock().unwrap(), count); + assert_eq!( + resumed.worker_detail(&worker.worker_ref).unwrap().status, + WorkerStatus::Stopped + ); + } + + #[cfg(feature = "fs-store")] + #[test] + fn restore_guard_unknown_preparation_write_retries_same_pending_operation() { + let dir = tempfile::tempdir().unwrap(); + let options = FsRuntimeStoreOptions::new(dir.path().join("runtime")) + .with_runtime_id("guard-unknown-preparation"); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = + Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()).unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("unknown preparation commit")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let mut request = runtime.test_restore_request(&worker.worker_ref); + request.preparation = Some( + serde_json::from_value(serde_json::json!({ + "workdir_attachments": [], "repository_access": [] + })) + .unwrap(), + ); + runtime_store(&runtime) + .fail_next_worker_write(crate::fs_store::AtomicWriteFault::AfterRename); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap() + .state, + WorkerRestoreState::ReconciliationRequired + ); + // This write interrupts idempotent attachment completion after admission. + runtime_store(&runtime) + .fail_next_worker_write(crate::fs_store::AtomicWriteFault::AfterRename); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap() + .state, + WorkerRestoreState::ReconciliationRequired + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 0); + drop(runtime); + let resumed_backend = Arc::new(TestExecutionBackend::default()); + let resumed = + Runtime::with_fs_store_and_execution_backend(options, resumed_backend.clone()).unwrap(); + request.preparation = None; + assert_eq!( + resumed + .restore_worker_operation(&worker.worker_ref, request) + .unwrap() + .state, + WorkerRestoreState::Accepted + ); + assert_eq!(*resumed_backend.restore_count.lock().unwrap(), 1); + assert_eq!( + resumed.worker_detail(&worker.worker_ref).unwrap().status, + WorkerStatus::Idle + ); + } + + #[cfg(feature = "fs-store")] + #[test] + fn restore_guard_terminal_commit_is_atomic_and_settled_owner_never_resumes() { + let dir = tempfile::tempdir().unwrap(); + let options = FsRuntimeStoreOptions::new(dir.path().join("runtime")) + .with_runtime_id("guard-terminal-commit"); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = + Runtime::with_fs_store_and_execution_backend(options.clone(), backend).unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("atomic terminal receipt")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let request = runtime.test_restore_request(&worker.worker_ref); + let receipt = runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap(); + let aggregate: serde_json::Value = serde_json::from_slice( + &std::fs::read( + dir.path() + .join("runtime/workers") + .join(worker.worker_id.to_string()) + .join("worker.json"), + ) + .unwrap(), + ) + .unwrap(); + assert_eq!(aggregate["status"], "idle"); + assert!(aggregate["restore_guard"]["active_request_id"].is_null()); + assert_eq!( + aggregate["restore_guard"]["owners"][&request.request_id]["receipt"], + serde_json::to_value(&receipt).unwrap() + ); + // Simulate a legacy/inconsistent active marker on an already-settled + // owner. Startup must not silently create a new execution for it. + { + let mut state = runtime.lock().unwrap(); + let mut candidate = state.worker(&worker.worker_ref).unwrap().clone(); + candidate.restore_guard.active_request_id = Some(request.request_id.clone()); + state.persist_worker_record(&candidate).unwrap(); + state.workers.insert(worker.worker_id, candidate); + } + drop(runtime); + let resumed_backend = Arc::new(TestExecutionBackend::default()); + let resumed = + Runtime::with_fs_store_and_execution_backend(options, resumed_backend.clone()).unwrap(); + assert_eq!(*resumed_backend.restore_count.lock().unwrap(), 0); + assert_eq!( + resumed + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap(), + receipt + ); + // Also exercise the accepted-aggregate/receipt-pending crash gap. + { + let mut state = resumed.lock().unwrap(); + let mut candidate = state.worker(&worker.worker_ref).unwrap().clone(); + let owner = candidate + .restore_guard + .owners + .get_mut(&request.request_id) + .unwrap(); + assert!(owner.execution_started); + owner.receipt = None; + candidate.restore_guard.active_request_id = Some(request.request_id.clone()); + state.persist_worker_record(&candidate).unwrap(); + state.workers.insert(worker.worker_id, candidate); + } + drop(resumed); + let gap_backend = Arc::new(TestExecutionBackend::default()); + let gap_resumed = Runtime::with_fs_store_and_execution_backend( + FsRuntimeStoreOptions::new(dir.path().join("runtime")) + .with_runtime_id("guard-terminal-commit"), + gap_backend.clone(), + ) + .unwrap(); + assert_eq!(*gap_backend.restore_count.lock().unwrap(), 0); + assert!(gap_backend.restore_operation_ids.lock().unwrap().is_empty()); + assert_eq!( + gap_resumed + .restore_worker_operation(&worker.worker_ref, request) + .unwrap() + .state, + WorkerRestoreState::Accepted + ); + } + + #[test] + fn restore_guard_terminal_rejection_replays_without_repreparation() { + let (runtime, backend) = runtime_and_backend(); + let worker = runtime + .create_worker(task_request("rejected owner")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let request = runtime.test_restore_request(&worker.worker_ref); + backend + .restore_result + .lock() + .unwrap() + .replace(WorkerExecutionSpawnResult::Rejected( + WorkerExecutionResult::rejected( + WorkerExecutionOperation::Restore, + "injected rejection", + ), + )); + let receipt = runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap(); + assert_eq!(receipt.state, WorkerRestoreState::Rejected); + backend.restore_result.lock().unwrap().take(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request) + .unwrap(), + receipt + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 1); + assert_eq!( + runtime.worker_detail(&worker.worker_ref).unwrap().status, + WorkerStatus::Stopped + ); + } + + #[cfg(feature = "fs-store")] + #[test] + fn restore_guard_pending_owner_survives_restart_and_reuses_operation_id() { + let dir = tempfile::tempdir().unwrap(); + let options = + FsRuntimeStoreOptions::new(dir.path().join("runtime")).with_runtime_id("guard-pending"); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = + Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()).unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("durable pending owner")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let request = runtime.test_restore_request(&worker.worker_ref); + backend + .restore_result + .lock() + .unwrap() + .replace(WorkerExecutionSpawnResult::Errored( + WorkerExecutionResult::rejected(WorkerExecutionOperation::Restore, "uncertain"), + )); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap() + .state, + WorkerRestoreState::ReconciliationRequired + ); + let operation_id = backend.restore_operation_ids.lock().unwrap()[0]; + drop(runtime); + let restarted = Runtime::with_fs_store(options.clone()).unwrap(); + let mut other = request.clone(); + other.request_id = "not-persisted-owner".into(); + other.expected_observation_token = restarted + .worker_detail(&worker.worker_ref) + .unwrap() + .restore_observation_token + .unwrap(); + assert!(matches!( + restarted.restore_worker_operation(&worker.worker_ref, other), + Err(RuntimeError::RestoreObservationConflict { .. }) + )); + drop(restarted); + let resumed_backend = Arc::new(TestExecutionBackend::default()); + let resumed = + Runtime::with_fs_store_and_execution_backend(options, resumed_backend.clone()).unwrap(); + assert_eq!( + resumed_backend + .restore_operation_ids + .lock() + .unwrap() + .as_slice(), + &[operation_id] + ); + let receipt = resumed + .restore_worker_operation(&worker.worker_ref, request) + .unwrap(); + assert_eq!(receipt.state, WorkerRestoreState::Accepted); + assert_eq!(*resumed_backend.restore_count.lock().unwrap(), 1); + } + + #[test] + fn restore_guard_pending_owner_is_only_reconciliation_intent() { + let (runtime, backend) = runtime_and_backend(); + let worker = runtime + .create_worker(task_request("pending owner")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let mut request = runtime.test_restore_request(&worker.worker_ref); + request.preparation = Some( + serde_json::from_value(serde_json::json!({ + "workdir_attachments": [], "repository_access": [] + })) + .unwrap(), + ); + backend + .restore_result + .lock() + .unwrap() + .replace(WorkerExecutionSpawnResult::Errored( + WorkerExecutionResult::rejected(WorkerExecutionOperation::Restore, "uncertain"), + )); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap() + .state, + WorkerRestoreState::ReconciliationRequired + ); + // A retry may carry newly minted preparation. It must reconcile the + // first persisted owner, not apply this invalid replacement payload. + request.preparation = Some(serde_json::from_value(serde_json::json!({ + "workspace_api": { "workspace_id": "must-not-bind", "base_url": "https://fresh.invalid.example" }, + "repository_access": [] + })).unwrap()); + let mut fresh = runtime.test_restore_request(&worker.worker_ref); + fresh.request_id = "new-owner".into(); + fresh.expected_observation_token = runtime + .worker_detail(&worker.worker_ref) + .unwrap() + .restore_observation_token + .unwrap(); + assert!(matches!( + runtime.restore_worker_operation(&worker.worker_ref, fresh), + Err(RuntimeError::RestoreObservationConflict { .. }) + )); + backend.restore_result.lock().unwrap().take(); + assert_eq!( + runtime + .restore_worker_operation(&worker.worker_ref, request) + .unwrap() + .state, + WorkerRestoreState::Accepted + ); + let ids = backend.restore_operation_ids.lock().unwrap(); + assert_eq!(ids.len(), 2); + assert_eq!(ids[0], ids[1]); + } + + #[cfg(feature = "fs-store")] + #[test] + fn restore_guard_generation_and_terminal_receipt_survive_restart() { + let dir = tempfile::tempdir().unwrap(); + let options = + FsRuntimeStoreOptions::new(dir.path().join("runtime")).with_runtime_id("guard-test"); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = + Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()).unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("durable replay")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let request = runtime.test_restore_request(&worker.worker_ref); + let receipt = runtime + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let generation = runtime + .worker_detail(&worker.worker_ref) + .unwrap() + .restore_observation_token; + drop(runtime); + let restarted = Runtime::with_fs_store(options).unwrap(); + assert_eq!( + restarted + .worker_detail(&worker.worker_ref) + .unwrap() + .restore_observation_token, + generation + ); + assert_eq!( + restarted + .restore_worker_operation(&worker.worker_ref, request.clone()) + .unwrap(), + receipt + ); + let mut stale = request; + stale.request_id = "stale-after-restart".into(); + assert!(matches!( + restarted.restore_worker_operation(&worker.worker_ref, stale), + Err(RuntimeError::RestoreObservationConflict { .. }) + )); + assert_eq!( + restarted.worker_detail(&worker.worker_ref).unwrap().status, + WorkerStatus::Stopped + ); + assert_eq!(*backend.restore_count.lock().unwrap(), 1); + } + #[test] fn operation_lock_registry_reclaims_idle_worker_entries() { let runtime = runtime_with_backend(); @@ -8660,8 +10581,12 @@ mod tests { backend.restore_gate.lock().unwrap().replace(gate.clone()); let restoring_runtime = runtime.clone(); let worker_ref = worker.worker_ref.clone(); - let restoring = - std::thread::spawn(move || restoring_runtime.restore_worker_operation(&worker_ref)); + let restoring = std::thread::spawn(move || { + restoring_runtime.restore_worker_operation( + &worker_ref, + restoring_runtime.test_restore_request(&worker_ref), + ) + }); assert!(gate.wait_for_entered(1, std::time::Duration::from_secs(2))); let operation_id = runtime .lock() @@ -8698,7 +10623,10 @@ mod tests { .unwrap(); assert_eq!( runtime - .restore_worker_operation(&worker.worker_ref) + .restore_worker_operation( + &worker.worker_ref, + runtime.test_restore_request(&worker.worker_ref) + ) .unwrap() .state, WorkerRestoreState::Accepted @@ -8760,7 +10688,7 @@ mod tests { text: "first".into(), }); let first = runtime - .restore_worker_operation(&worker.worker_ref) + .restore_admitted_worker_under_lock(&worker.worker_ref, WorkerRestoreMode::Explicit) .unwrap(); assert_eq!(first.state, WorkerRestoreState::ReconciliationRequired); assert!( @@ -8782,7 +10710,10 @@ mod tests { .publish_text_delta(&worker.worker_ref, "second") .unwrap(); let retry = runtime - .restore_worker_operation(&worker.worker_ref) + .restore_worker_operation( + &worker.worker_ref, + runtime.test_restore_request(&worker.worker_ref), + ) .unwrap(); assert_eq!(retry.state, WorkerRestoreState::Accepted); assert_eq!( @@ -8841,8 +10772,12 @@ mod tests { let restoring_runtime = runtime.clone(); let restoring_ref = worker.worker_ref.clone(); - let restoring = - std::thread::spawn(move || restoring_runtime.restore_worker_operation(&restoring_ref)); + let restoring = std::thread::spawn(move || { + restoring_runtime.restore_worker_operation( + &restoring_ref, + restoring_runtime.test_restore_request(&restoring_ref), + ) + }); assert!(gate.wait_for_entered(1, std::time::Duration::from_secs(2))); let pending = runtime.worker_detail(&worker.worker_ref).unwrap(); @@ -8883,7 +10818,12 @@ mod tests { .unwrap(); runtime.stop_worker(&created.worker_ref, None).unwrap(); - let restored = runtime.restore_worker(&created.worker_ref).unwrap(); + let restored = runtime + .restore_worker( + &created.worker_ref, + runtime.test_restore_request(&created.worker_ref), + ) + .unwrap(); let worker_state = restored .worker_state @@ -9362,7 +11302,12 @@ mod tests { }), ); runtime.stop_worker(&worker.worker_ref, None).unwrap(); - runtime.restore_worker(&worker.worker_ref).unwrap(); + runtime + .restore_worker( + &worker.worker_ref, + runtime.test_restore_request(&worker.worker_ref), + ) + .unwrap(); let stop_count = *backend.stop_count.lock().unwrap(); let detail = runtime.worker_detail(&worker.worker_ref).unwrap(); for method in [ @@ -9479,7 +11424,12 @@ mod tests { )); assert_eq!(*backend.restore_count.lock().unwrap(), 0); - runtime.restore_worker(&detail.worker_ref).unwrap(); + runtime + .restore_worker( + &detail.worker_ref, + runtime.test_restore_request(&detail.worker_ref), + ) + .unwrap(); runtime .send_input(&detail.worker_ref, WorkerInput::user("wake up")) .unwrap(); @@ -9525,7 +11475,12 @@ mod tests { runtime .stop_worker(&detail.worker_ref, Some("restore test".to_string())) .unwrap(); - runtime.restore_worker(&detail.worker_ref).unwrap(); + runtime + .restore_worker( + &detail.worker_ref, + runtime.test_restore_request(&detail.worker_ref), + ) + .unwrap(); assert_eq!( backend.dispatched_inputs.lock().unwrap().len(), @@ -10308,21 +12263,49 @@ mod tests { .unwrap(); assert_eq!( runtime - .restore_worker_operation(&worker.worker_ref) + .restore_worker_operation( + &worker.worker_ref, + runtime.test_restore_request(&worker.worker_ref) + ) .unwrap() .state, WorkerRestoreState::Rejected ); - assert_eq!( - std::fs::read( + let mut before: serde_json::Value = serde_json::from_slice(&before).unwrap(); + let mut after: serde_json::Value = serde_json::from_slice( + &std::fs::read( dir.path() .join("runtime/workers") .join(worker.worker_id.to_string()) - .join("worker.json") + .join("worker.json"), ) .unwrap(), - before, - "explicit preflight rejection is read-only" + ) + .unwrap(); + let old_guard = before + .as_object_mut() + .unwrap() + .remove("restore_guard") + .unwrap(); + let guard = after + .as_object_mut() + .unwrap() + .remove("restore_guard") + .unwrap(); + assert_ne!(guard["generation"], old_guard["generation"]); + assert!(guard["active_request_id"].is_null()); + assert_eq!( + guard["owners"] + .as_object() + .unwrap() + .values() + .next() + .unwrap()["receipt"]["state"], + "rejected" + ); + assert_eq!( + after, before, + "preflight changes only durable intent/receipt metadata, never execution" ); assert!(matches!( runtime.send_input( @@ -10389,7 +12372,10 @@ mod tests { restoring_backend.restore_result.lock().unwrap().take(); let retry = restored - .restore_worker_operation(&worker.worker_ref) + .restore_worker_operation( + &worker.worker_ref, + restored.test_restore_request(&worker.worker_ref), + ) .unwrap(); assert_eq!(retry.state, WorkerRestoreState::Accepted); @@ -10424,7 +12410,10 @@ mod tests { )); let result = runtime - .restore_worker_operation(&worker.worker_ref) + .restore_worker_operation( + &worker.worker_ref, + runtime.test_restore_request(&worker.worker_ref), + ) .unwrap(); assert_eq!(result.state, WorkerRestoreState::RolledBack); @@ -10441,7 +12430,10 @@ mod tests { backend.restore_result.lock().unwrap().take(); let retry = runtime - .restore_worker_operation(&worker.worker_ref) + .restore_worker_operation( + &worker.worker_ref, + runtime.test_restore_request(&worker.worker_ref), + ) .unwrap(); assert_eq!(retry.state, WorkerRestoreState::Accepted); assert_eq!(*backend.restore_count.lock().unwrap(), 2); @@ -10466,13 +12458,41 @@ mod tests { )); let result = runtime - .restore_worker_operation(&created.worker_ref) + .restore_worker_operation( + &created.worker_ref, + runtime.test_restore_request(&created.worker_ref), + ) .unwrap(); assert_eq!(result.state, WorkerRestoreState::Rejected); assert!(result.worker.is_none()); assert_eq!(*backend.restore_count.lock().unwrap(), 0); - assert_eq!(runtime.worker_detail(&created.worker_ref).unwrap(), before); + let mut after = runtime.worker_detail(&created.worker_ref).unwrap(); + assert_ne!( + after.restore_observation_token, + before.restore_observation_token + ); + after.restore_observation_token = before.restore_observation_token.clone(); + assert_eq!( + after, before, + "admission may rotate the fence but must not change execution state" + ); + let state = runtime.lock().unwrap(); + let record = state.worker(&created.worker_ref).unwrap(); + assert!(record.pending_restore.is_none()); + assert!(record.restore_guard.active_request_id.is_none()); + assert_eq!(record.restore_guard.owners.len(), 1); + assert_eq!( + record + .restore_guard + .owners + .values() + .next() + .unwrap() + .receipt + .as_ref(), + Some(&result) + ); } #[test] @@ -10495,7 +12515,10 @@ mod tests { ); let result = runtime - .restore_worker_operation(&created.worker_ref) + .restore_worker_operation( + &created.worker_ref, + runtime.test_restore_request(&created.worker_ref), + ) .unwrap(); assert_eq!(result.state, WorkerRestoreState::ReconciliationRequired); @@ -10525,7 +12548,10 @@ mod tests { ); backend.restore_result.lock().unwrap().take(); let retry = runtime - .restore_worker_operation(&created.worker_ref) + .restore_worker_operation( + &created.worker_ref, + runtime.test_restore_request(&created.worker_ref), + ) .unwrap(); assert_eq!(retry.state, WorkerRestoreState::Accepted); assert_eq!(*backend.restore_count.lock().unwrap(), 2); @@ -10655,7 +12681,10 @@ mod tests { ); let restored_detail = restored - .restore_worker(&restorable.worker_ref) + .restore_worker( + &restorable.worker_ref, + restored.test_restore_request(&restorable.worker_ref), + ) .expect("explicit restore reconstructs execution metadata"); assert!(restored_detail.execution_metadata_available); restored @@ -10764,7 +12793,12 @@ mod tests { )); assert_eq!(*backend.restore_count.lock().unwrap(), 0); - restored.restore_worker(&worker.worker_ref).unwrap(); + restored + .restore_worker( + &worker.worker_ref, + restored.test_restore_request(&worker.worker_ref), + ) + .unwrap(); assert_eq!(*backend.restore_count.lock().unwrap(), 1); assert_eq!( restored.worker_detail(&worker.worker_ref).unwrap().status, @@ -10811,6 +12845,10 @@ mod tests { serde_json::from_slice(&std::fs::read(&worker_path).expect("worker aggregate")) .expect("worker aggregate json"); worker_identity["schema_version"] = serde_json::json!(7); + worker_identity + .as_object_mut() + .unwrap() + .remove("restore_guard"); let mut worker_execution = worker_identity .as_object_mut() .expect("worker aggregate object") @@ -10898,11 +12936,17 @@ mod tests { runtime_store(&runtime) .fail_next_worker_write(crate::fs_store::AtomicWriteFault::AfterRename); - let result = runtime - .restore_worker_operation(&worker.worker_ref) - .unwrap(); + let result = runtime.begin_restore_operation( + &worker.worker_ref, + WorkerLifecycleOperationId::new(), + WorkerRestoreMode::Explicit, + runtime.restore_candidate_execution_context(worker.worker_ref.clone()), + ); - assert_eq!(result.state, WorkerRestoreState::ReconciliationRequired); + assert!(matches!( + result, + Err(RuntimeError::StoreCommitOutcomeUnknown { .. }) + )); assert_eq!(*backend.restore_count.lock().unwrap(), 0); assert_eq!(*backend.stop_count.lock().unwrap(), 1); assert_eq!( @@ -10956,7 +13000,10 @@ mod tests { ) .unwrap(); let retry = restarted - .restore_worker_operation(&worker.worker_ref) + .restore_worker_operation( + &worker.worker_ref, + restarted.test_restore_request(&worker.worker_ref), + ) .unwrap(); assert_eq!(retry.state, WorkerRestoreState::Accepted); assert_eq!(*restarted_backend.restore_count.lock().unwrap(), 1); @@ -11031,8 +13078,14 @@ mod tests { .unwrap() .operation_id; restoring_backend.restore_result.lock().unwrap().take(); + // An explicit observed recovery may bind a receipt to an unowned + // startup pending operation, but must preserve its lifecycle ID. It + // cannot allocate a new execution operation or overwrite preparation. let retry = restored - .restore_worker_operation(&worker.worker_ref) + .restore_worker_operation( + &worker.worker_ref, + restored.test_restore_request(&worker.worker_ref), + ) .unwrap(); assert_eq!(retry.state, WorkerRestoreState::Accepted); assert_eq!(*restoring_backend.restore_count.lock().unwrap(), 2); diff --git a/crates/worker-runtime/src/worker_backend.rs b/crates/worker-runtime/src/worker_backend.rs index e47eadb8f..cb5830148 100644 --- a/crates/worker-runtime/src/worker_backend.rs +++ b/crates/worker-runtime/src/worker_backend.rs @@ -7394,7 +7394,14 @@ mod tests { .create_worker(create_request("restart cleanup")) .unwrap(); runtime.stop_worker(&worker.worker_ref, None).unwrap(); - assert!(runtime.restore_worker(&worker.worker_ref).is_err()); + assert!( + runtime + .restore_worker( + &worker.worker_ref, + runtime.test_restore_request(&worker.worker_ref) + ) + .is_err() + ); assert!(runtime.stop_worker(&worker.worker_ref, None).is_err()); let prior_restore_id = restore_id.lock().unwrap().unwrap(); let aggregate = runtime_store @@ -8904,7 +8911,12 @@ mod tests { .unwrap(); assert!(detached.workdir_attachments.is_empty()); runtime.stop_worker(&detail.worker_ref, None).unwrap(); - let restored_without_workdir = runtime.restore_worker(&detail.worker_ref).unwrap(); + let restored_without_workdir = runtime + .restore_worker( + &detail.worker_ref, + runtime.test_restore_request(&detail.worker_ref), + ) + .unwrap(); assert!(restored_without_workdir.workdir_attachments.is_empty()); assert!( !repo.path().join(".yoi").exists(), @@ -9191,7 +9203,12 @@ mod tests { .contains_key(&detail.worker_ref) ); - runtime.restore_worker(&detail.worker_ref).unwrap(); + runtime + .restore_worker( + &detail.worker_ref, + runtime.test_restore_request(&detail.worker_ref), + ) + .unwrap(); assert_eq!( runtime.worker_detail(&detail.worker_ref).unwrap().status, crate::catalog::WorkerStatus::Idle diff --git a/crates/worker-runtime/src/worker_source.rs b/crates/worker-runtime/src/worker_source.rs index 089ed9f8f..22f81aa79 100644 --- a/crates/worker-runtime/src/worker_source.rs +++ b/crates/worker-runtime/src/worker_source.rs @@ -555,9 +555,15 @@ impl WorkspaceClient for RuntimeOwnedWorkspaceClient { WorkspaceServerOperation::WorkerControlRestore { runtime_id, worker_id, + request, } => generated_workspace_response( client - .worker_control_restore(workspace_id, runtime_id, worker_id) + .worker_control_restore( + workspace_id, + runtime_id, + worker_id, + request, + ) .await, ), WorkspaceServerOperation::WorkerObservationSessions => { diff --git a/crates/worker/src/feature/builtin/manage_worker.rs b/crates/worker/src/feature/builtin/manage_worker.rs index 5c746f9b6..789229581 100644 --- a/crates/worker/src/feature/builtin/manage_worker.rs +++ b/crates/worker/src/feature/builtin/manage_worker.rs @@ -77,8 +77,9 @@ pub trait WorkerControlService: Send + Sync { &self, runtime_id: String, worker_id: String, + request: server_api::WorkerRestoreRequest, ) -> Result { - let _ = (runtime_id, worker_id); + let _ = (runtime_id, worker_id, request); Err(WorkspaceClientError::Unavailable( "Runtime Worker restore is unavailable".to_string(), )) @@ -347,6 +348,7 @@ impl WorkerControlService for WorkspaceWorkerControlService { &self, runtime_id: String, worker_id: String, + request: server_api::WorkerRestoreRequest, ) -> Result { if !self.runtime_worker_control { return Err(WorkspaceClientError::Unavailable( @@ -357,6 +359,7 @@ impl WorkerControlService for WorkspaceWorkerControlService { crate::worker::WorkspaceServerOperation::WorkerControlRestore { runtime_id, worker_id, + request, }, ) } @@ -750,8 +753,13 @@ enum WorkerSubjectInput { #[derive(Debug, Deserialize, JsonSchema)] #[serde(deny_unknown_fields)] -struct WorkerTargetInput { +struct WorkerRestoreInput { subject: WorkerSubjectInput, + expected_observation_token: String, + /// Reuse the reported ID for explicit unknown-outcome recovery. A new intent + /// defaults to the durable Tool call identity. + #[serde(default)] + request_id: Option, } #[derive(Debug, Deserialize, JsonSchema)] @@ -864,7 +872,7 @@ impl WorkerOperation { Self::Cancel => "Cancel the current turn of a known Runtime Worker when allowed.", Self::Stop => "Stop a known Runtime Worker when allowed.", Self::Restore => { - "Restore a stopped Backend/Runtime Worker session in the current Workspace." + "Restore a stopped Backend/Runtime Worker session in the current Workspace. Supply expected_observation_token from WorkerList; a conflict requires refreshing and deliberate new intent. New requests default to the durable Tool call ID. For explicit recovery of an unknown/pending outcome, supply the reported request_id and the original token unchanged; never create a new intent or automatically retry." } Self::Remove => { "Remove an eligible stopped, unassigned, non-internal Worker. Supply a bounded reason; Backend validation and retention are authoritative." @@ -981,12 +989,41 @@ impl Tool for WorkspaceWorkerTool { .map_err(control_tool_error)? } WorkerOperation::Restore => { - let input = parse::(input_json, "WorkerRestore")?; + let input = parse::(input_json, "WorkerRestore")?; let (runtime_id, worker_id) = runtime_subject_ids(&input.subject, self.operation)?; - self.control - .restore_worker(runtime_id, worker_id) + let request_id = match input.request_id { + Some(id) => non_empty(id, "request_id")?, + None => format!( + "worker-restore:{}", + non_empty(ctx.call_id.clone(), "tool call_id")? + ), + }; + if request_id.len() > 512 { + return Err(ToolError::InvalidArgument( + "request_id must contain at most 512 bytes".into(), + )); + } + let recoverable_error = |error| { + ToolError::ExecutionFailed(format!( + "WorkerRestore request_id={request_id}; reuse this ID with the original observation token only for explicit unknown/pending outcome recovery, not after a stale-intent conflict: {error}" + )) + }; + let response = self + .control + .restore_worker( + runtime_id, + worker_id, + server_api::WorkerRestoreRequest { + expected_observation_token: non_empty( + input.expected_observation_token, + "expected_observation_token", + )?, + request_id: request_id.clone(), + }, + ) .await - .map_err(control_tool_error)? + .map_err(|error| recoverable_error(control_tool_error(error)))?; + return tool_output(self.operation, response).map_err(recoverable_error); } WorkerOperation::Remove => { let input = parse::(input_json, "WorkerRemove")?; @@ -1058,6 +1095,18 @@ fn tool_output( response.status, response.body ))); } + if operation == WorkerOperation::Restore { + let restored: server_api::WorkerRestoreResponse = serde_json::from_str(&response.body) + .map_err(|error| { + ToolError::ExecutionFailed(format!("invalid WorkerRestore response: {error}")) + })?; + if restored.result.state != server_api::WorkerRestoreState::Accepted { + return Err(ToolError::ExecutionFailed(format!( + "WorkerRestore returned {:?}: {}", + restored.result.state, response.body + ))); + } + } if operation == WorkerOperation::Stop && let Ok(summary) = serde_json::from_str::(&response.body) { @@ -1150,7 +1199,7 @@ fn worker_tool_contribution( } } WorkerOperation::Restore => { - definition::(operation, control, runtime_worker_control) + definition::(operation, control, runtime_worker_control) } WorkerOperation::Remove => { definition::(operation, control, runtime_worker_control) @@ -1410,6 +1459,95 @@ mod tests { }) } + #[tokio::test] + async fn worker_restore_forwards_exact_observation_and_request_identity() { + let client = Arc::new(RecordingWorkspaceClient::default()); + let control = test_control(client.clone()); + for _ in 0..2 { + control + .restore_worker( + "runtime/a".into(), + "worker/b".into(), + server_api::WorkerRestoreRequest { + expected_observation_token: "same-observation".into(), + request_id: "same-request".into(), + }, + ) + .await + .unwrap(); + } + let requests = client.requests.lock().unwrap(); + assert_eq!(requests.len(), 2); + assert_eq!( + requests[0].path, + "/api/w/workspace%2Ftest/worker-control/workers/runtime%2Fa/worker%2Fb/restore" + ); + assert_eq!(requests[0].body, requests[1].body); + let body: serde_json::Value = + serde_json::from_str(requests[0].body.as_deref().unwrap()).unwrap(); + assert_eq!( + body, + serde_json::json!({"expected_observation_token":"same-observation","request_id":"same-request"}) + ); + assert!(serde_json::from_str::(r#"{"subject":{"kind":"runtime_worker","runtime_id":"runtime-a","worker_id":"worker-a"}}"#).is_err()); + } + + #[tokio::test] + async fn worker_restore_reports_retry_identity_and_reuses_it_across_explicit_tool_calls() { + let client = Arc::new(RecordingWorkspaceClient::default()); + let tool = WorkspaceWorkerTool { + operation: WorkerOperation::Restore, + control: test_control(client.clone()), + runtime_worker_control: true, + }; + let mut input = serde_json::json!({ + "subject": {"kind":"runtime_worker", "runtime_id":"runtime-a", "worker_id":"worker-b"}, + "expected_observation_token":"original-observation" + }); + // The fixture deliberately returns malformed JSON shape: delivery may + // have succeeded, so retain the exact tuple rather than issuing new intent. + let first = tool + .execute( + &input.to_string(), + ToolExecutionContext::new("original-call", "batch", 0), + ) + .await + .unwrap_err(); + assert!( + first + .to_string() + .contains("request_id=worker-restore:original-call") + ); + input["request_id"] = "worker-restore:original-call".into(); + let second = tool + .execute( + &input.to_string(), + ToolExecutionContext::new("explicit-recovery-call", "batch", 1), + ) + .await + .unwrap_err(); + assert!( + second + .to_string() + .contains("request_id=worker-restore:original-call") + ); + { + let requests = client.requests.lock().unwrap(); + assert_eq!(requests.len(), 2); + assert_eq!(requests[0].body, requests[1].body); + } + input["request_id"] = "x".repeat(513).into(); + assert!( + tool.execute( + &input.to_string(), + ToolExecutionContext::new("invalid", "batch", 2) + ) + .await + .is_err() + ); + assert_eq!(client.requests.lock().unwrap().len(), 2); + } + #[tokio::test] async fn worker_spawn_forwards_typed_initial_submit_to_workspace_api() { let client = Arc::new(RecordingWorkspaceClient::default()); diff --git a/crates/worker/src/worker.rs b/crates/worker/src/worker.rs index 70b633367..417886ae1 100644 --- a/crates/worker/src/worker.rs +++ b/crates/worker/src/worker.rs @@ -728,6 +728,7 @@ pub enum WorkspaceServerOperation { WorkerControlRestore { runtime_id: String, worker_id: String, + request: server_api::WorkerRestoreRequest, }, WorkerObservationSessions, WorkerObservationCapture(server_api::WorkerObservationSubjectRef), @@ -802,11 +803,15 @@ fn workspace_server_operation_request( WorkspaceServerOperation::WorkerControlRestore { runtime_id, worker_id, - } => Ok(WorkspaceRequest::json( - WorkspaceRequestMethod::Post, - format!("{base}/worker-control/workers/{runtime_id}/{worker_id}/restore"), - "".to_string(), - )), + request, + } => workspace_server_json_request( + format!( + "{base}/worker-control/workers/{}/{}/restore", + encode_workspace_path_segment(&runtime_id), + encode_workspace_path_segment(&worker_id) + ), + &request, + ), WorkspaceServerOperation::WorkerObservationSessions => Ok(WorkspaceRequest::get(format!( "{base}/worker-observation/sessions" ))), diff --git a/crates/workspace-server/src/hosts.rs b/crates/workspace-server/src/hosts.rs index fde6c2427..c03a04f9b 100644 --- a/crates/workspace-server/src/hosts.rs +++ b/crates/workspace-server/src/hosts.rs @@ -242,6 +242,8 @@ pub struct InternalWorkerImplementationSummary { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct InternalWorkerSummary { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub restore_observation_token: Option, #[serde(flatten)] pub worker: RuntimeWorkerRef, pub host_id: String, @@ -356,6 +358,7 @@ pub(crate) fn workspace_worker_summary( workdir_attachments: Vec, ) -> server_api::WorkerSummary { server_api::WorkerSummary { + restore_observation_token: summary.restore_observation_token, runtime_id: summary.worker.runtime_id, worker_id: summary.worker.worker_id, resource_key, @@ -834,11 +837,17 @@ impl RuntimeRegistryError { runtime_id, code, message, - } => Error::RuntimeOperationFailed { - runtime_id, - code, - message, - }, + } => { + if code == "restore_observation_conflict" { + Error::RestoreObservationConflict + } else { + Error::RuntimeOperationFailed { + runtime_id, + code, + message, + } + } + } } } } @@ -938,7 +947,29 @@ pub trait WorkspaceWorkerRuntime: Send + Sync { fn worker(&self, worker_id: &str) -> WorkerLookupResult; - fn restore_worker(&self, worker_id: &str) -> InternalWorkerRestoreResult { + fn coordinate_worker_restore( + &self, + _worker_id: &str, + _request: runtime_api::WorkerRestoreCoordinationRequest, + ) -> Result< + ( + Option, + Option, + ), + RuntimeDiagnostic, + > { + Err(diagnostic( + "worker_restore_coordination_unsupported", + HostDiagnosticSeverity::Error, + "Runtime does not support guarded Restore coordination", + )) + } + + fn restore_worker( + &self, + worker_id: &str, + _request: runtime_api::WorkerRestoreRequest, + ) -> InternalWorkerRestoreResult { InternalWorkerRestoreResult { state: server_api::WorkerRestoreState::Rejected, worker: None, @@ -1602,16 +1633,39 @@ impl RuntimeRegistry { Ok(worker) } + pub fn coordinate_worker_restore( + &self, + worker: &RuntimeWorkerRef, + request: runtime_api::WorkerRestoreCoordinationRequest, + ) -> Result< + ( + Option, + Option, + ), + RuntimeRegistryError, + > { + validate_backend_identifier("runtime_id", &worker.runtime_id)?; + validate_backend_identifier("worker_id", &worker.worker_id)?; + self.runtime(&worker.runtime_id)? + .coordinate_worker_restore(&worker.worker_id, request) + .map_err(|error| RuntimeRegistryError::RuntimeOperationFailed { + runtime_id: worker.runtime_id.clone(), + code: error.code, + message: error.message, + }) + } + pub fn restore_worker( &self, worker: &RuntimeWorkerRef, + request: runtime_api::WorkerRestoreRequest, ) -> Result { let runtime_id = worker.runtime_id.as_str(); let worker_id = worker.worker_id.as_str(); validate_backend_identifier("runtime_id", runtime_id)?; validate_backend_identifier("worker_id", worker_id)?; let runtime = self.runtime(runtime_id)?; - Ok(runtime.restore_worker(worker_id)) + Ok(runtime.restore_worker(worker_id, request)) } pub fn replace_worker_workspace_api( @@ -2372,6 +2426,7 @@ impl EmbeddedWorkerRuntime { true, ); InternalWorkerSummary { + restore_observation_token: summary.restore_observation_token.clone(), worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id.clone()), host_id: self.host_id.clone(), display_name: display.display_name.clone(), @@ -2416,6 +2471,7 @@ impl EmbeddedWorkerRuntime { true, ); InternalWorkerSummary { + restore_observation_token: detail.restore_observation_token.clone(), worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id.clone()), host_id: self.host_id.clone(), display_name: display.display_name.clone(), @@ -2582,7 +2638,48 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime { } } - fn restore_worker(&self, worker_id: &str) -> InternalWorkerRestoreResult { + fn coordinate_worker_restore( + &self, + worker_id: &str, + request: runtime_api::WorkerRestoreCoordinationRequest, + ) -> Result< + ( + Option, + Option, + ), + RuntimeDiagnostic, + > { + let worker_ref = self.worker_ref(worker_id).ok_or_else(|| { + diagnostic( + "worker_not_found", + HostDiagnosticSeverity::Warning, + "Unknown Worker", + ) + })?; + let (result, preparation) = self + .runtime + .coordinate_worker_restore_operation(&worker_ref, request, None) + .map_err(|error| embedded_runtime_diagnostic(&error))?; + Ok(( + result.map(|result| InternalWorkerRestoreResult { + state: result.state, + worker: result.worker.map(|detail| self.map_worker_detail(detail)), + diagnostics: match (result.reason_code, result.message) { + (Some(code), Some(message)) => { + vec![diagnostic(code, HostDiagnosticSeverity::Warning, message)] + } + _ => Vec::new(), + }, + }), + preparation, + )) + } + + fn restore_worker( + &self, + worker_id: &str, + request: runtime_api::WorkerRestoreRequest, + ) -> InternalWorkerRestoreResult { let Some(worker_ref) = self.worker_ref(worker_id) else { return InternalWorkerRestoreResult { state: server_api::WorkerRestoreState::Rejected, @@ -2594,7 +2691,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime { )], }; }; - match self.runtime.restore_worker_operation(&worker_ref) { + match self.runtime.restore_worker_operation(&worker_ref, request) { Ok(result) => { let diagnostics = match (result.reason_code, result.message) { (Some(code), Some(message)) => { @@ -2608,6 +2705,17 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime { diagnostics, } } + Err(worker_runtime::error::RuntimeError::RestoreObservationConflict { .. }) => { + InternalWorkerRestoreResult { + state: server_api::WorkerRestoreState::Rejected, + worker: None, + diagnostics: vec![diagnostic( + "restore_observation_conflict", + HostDiagnosticSeverity::Warning, + "Restore observation changed; refresh before a new intent".to_string(), + )], + } + } Err(err) => InternalWorkerRestoreResult { state: server_api::WorkerRestoreState::ReconciliationRequired, worker: None, @@ -4375,6 +4483,7 @@ impl RemoteWorkerRuntime { false, ); InternalWorkerSummary { + restore_observation_token: summary.restore_observation_token.clone(), worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id.clone()), host_id: self.host_id.clone(), display_name: display.display_name.clone(), @@ -4417,6 +4526,7 @@ impl RemoteWorkerRuntime { false, ); InternalWorkerSummary { + restore_observation_token: detail.restore_observation_token.clone(), worker: RuntimeWorkerRef::new(&self.runtime_id, worker_id.clone()), host_id: self.host_id.clone(), display_name: display.display_name.clone(), @@ -4708,17 +4818,58 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime { } } - fn restore_worker(&self, worker_id: &str) -> InternalWorkerRestoreResult { + fn coordinate_worker_restore( + &self, + worker_id: &str, + request: runtime_api::WorkerRestoreCoordinationRequest, + ) -> Result< + ( + Option, + Option, + ), + RuntimeDiagnostic, + > { + let worker_id = worker_id.to_string(); + let response = self.run_runtime_api( + self.request_timeout, + MAX_REMOTE_RUNTIME_RESPONSE_BYTES, + move |client| async move { client.coordinate_worker_restore(worker_id, request).await }, + )?; + let result = response + .result + .map(|result| -> Result<_, RuntimeDiagnostic> { + let worker = result + .worker + .map(runtime_contract_convert::<_, worker_runtime::catalog::WorkerDetail>) + .transpose()?; + let state = + runtime_contract_convert::<_, server_api::WorkerRestoreState>(result.state)?; + Ok(InternalWorkerRestoreResult { + state, + worker: worker.map(|detail| self.map_worker_detail(detail)), + diagnostics: match (result.reason_code, result.message) { + (Some(code), Some(message)) => { + vec![diagnostic(code, HostDiagnosticSeverity::Warning, message)] + } + _ => Vec::new(), + }, + }) + }) + .transpose()?; + Ok((result, response.preparation)) + } + + fn restore_worker( + &self, + worker_id: &str, + request: runtime_api::WorkerRestoreRequest, + ) -> InternalWorkerRestoreResult { let worker_id_owned = worker_id.to_string(); match self .run_runtime_api( self.request_timeout, MAX_REMOTE_RUNTIME_RESPONSE_BYTES, - move |client| async move { - client - .restore_worker(worker_id_owned, runtime_api::EmptyObjectRequest::default()) - .await - }, + move |client| async move { client.restore_worker(worker_id_owned, request).await }, ) .and_then(|value| { let worker = value @@ -5886,6 +6037,11 @@ fn sanitize_embedded_execution_detail(summary: &str, message: &str) -> String { fn embedded_runtime_diagnostic(error: &EmbeddedRuntimeError) -> RuntimeDiagnostic { match error { + EmbeddedRuntimeError::RestoreObservationConflict { .. } => diagnostic( + "restore_observation_conflict", + HostDiagnosticSeverity::Warning, + "Restore observation changed; refresh before a new intent".to_string(), + ), EmbeddedRuntimeError::RuntimeStopped => diagnostic( "embedded_runtime_stopped", HostDiagnosticSeverity::Warning, @@ -6292,6 +6448,7 @@ fn worker_spawn_intent_label(intent: &WorkerSpawnIntent) -> &'static str { pub fn placeholder_worker(host_id: impl Into) -> InternalWorkerSummary { let host_id = host_id.into(); InternalWorkerSummary { + restore_observation_token: None, worker: RuntimeWorkerRef::new("placeholder", "worker-placeholder"), host_id, display_name: "Worker runtime actions are not implemented".to_string(), @@ -6369,7 +6526,14 @@ mod tests { ) .unwrap(); - let result = provider.restore_worker("worker-transport-uncertain"); + let result = provider.restore_worker( + "worker-transport-uncertain", + runtime_api::WorkerRestoreRequest { + expected_observation_token: "observation".into(), + request_id: "transport-uncertain".into(), + preparation: None, + }, + ); assert_eq!( result.state, @@ -7052,6 +7216,7 @@ mod tests { runtime_id: runtime_id.to_string(), host_id: host_id.to_string(), workers: vec![InternalWorkerSummary { + restore_observation_token: None, worker: RuntimeWorkerRef::new(runtime_id, worker_id), host_id: host_id.to_string(), display_name: label.to_string(), @@ -7698,7 +7863,16 @@ mod tests { let detail = registry.worker(&worker.worker).unwrap(); - let json = serde_json::to_string(&(embedded_summary, worker, input, detail)).unwrap(); + // The public observation generation is intentionally exposed, unlike + // private authentication tokens. Whitelist its key, not arbitrary token + // values or other token-bearing fields. + for summary in [&worker, &detail] { + let token = summary.restore_observation_token.as_ref().unwrap(); + assert_eq!(uuid::Uuid::parse_str(token).unwrap().get_version_num(), 7); + } + let json = serde_json::to_string(&(embedded_summary, worker, input, detail)) + .unwrap() + .replace("\"restore_observation_token\":", "\"restore_observation\":"); for forbidden in [ "/workspace/project", "metadata.json", diff --git a/crates/workspace-server/src/lib.rs b/crates/workspace-server/src/lib.rs index c23670ced..652c6ad65 100644 --- a/crates/workspace-server/src/lib.rs +++ b/crates/workspace-server/src/lib.rs @@ -139,6 +139,10 @@ pub enum Error { RuntimeBindingFingerprintConflict { fingerprint: String }, #[error("Runtime binding was not found for {runtime_id}")] RuntimeBindingNotFound { runtime_id: String }, + #[error( + "restore_observation_conflict: Restore observation changed; refresh before a new intent" + )] + RestoreObservationConflict, #[error("Repository conflict: {0}")] RepositoryConflict(String), #[error("Registry inconsistency: {0}")] diff --git a/crates/workspace-server/src/resource_broker.rs b/crates/workspace-server/src/resource_broker.rs index 90bbc3efe..9bb68fdb2 100644 --- a/crates/workspace-server/src/resource_broker.rs +++ b/crates/workspace-server/src/resource_broker.rs @@ -61,6 +61,16 @@ impl Drop for StoredResource { } impl BackendResourceBroker { + #[cfg(test)] + pub(crate) fn repository_ssh_access_handle_count(&self) -> usize { + self.resources + .lock() + .unwrap() + .values() + .filter(|resource| resource.handle.kind == BackendResourceKind::RepositorySshAccess) + .count() + } + pub fn issue_profile_source_archive_handle( &self, workspace_id: impl Into, diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index fac133171..a418d7e65 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -4960,101 +4960,235 @@ impl WorkspaceApi { Ok(result) } + fn current_restore_summary( + &self, + worker: &RuntimeWorkerRef, + has_summary: bool, + ) -> Option { + if !has_summary { + return None; + } + let mut summary = self.runtime.worker(worker).ok()?; + // Historical receipts are not observations. Read current metadata without + // requiring current singleton eligibility and never write it back to catalog. + summary.singleton_key = self.config_store.with_conn(|conn| { + conn.query_row("SELECT singleton_key FROM worker_singleton_owners WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3", + rusqlite::params![self.workspace_id(), worker.runtime_id, worker.worker_id], |row| row.get::<_, String>(0)) + .map(Some).or_else(|error| if matches!(error, rusqlite::Error::QueryReturnedNoRows) { Ok(None) } else { Err(error) }).map_err(Error::from) + }).ok().flatten(); + Some(summary) + } + fn restore_workspace_worker( &self, worker: &RuntimeWorkerRef, + request: server_api::WorkerRestoreRequest, ) -> ApiResult { - // Fence displaced singleton generations before renewing credentials, replacing bindings, - // or asking any Runtime to launch execution. - let singleton_lease = self - .store - .require_current_worker_singleton_owner(&self.config.workspace_id, worker)?; - for link in self - .store - .list_worker_workdir_links(&self.config.workspace_id, worker)? - .into_iter() - .filter(|link| link.unlinked_at.is_none()) + if request.request_id.trim().is_empty() + || request.request_id.len() > 512 + || request.expected_observation_token.is_empty() + || request.expected_observation_token.len() > 256 { - let record = self + return Err(Error::InvalidInput( + "Restore requires a bounded request_id and expected_observation_token".into(), + ) + .into()); + } + // Auth is checked by WorkspaceWorker before this read-only owner dispatch. + // Existing owners reach receipts/reconciliation before NEW launch eligibility. + let (recovered, owned_preparation) = self + .runtime + .coordinate_worker_restore( + worker, + runtime_api::WorkerRestoreCoordinationRequest { + expected_observation_token: request.expected_observation_token.clone(), + request_id: request.request_id.clone(), + preparation: None, + }, + ) + .map_err(RuntimeRegistryError::into_error)?; + if let Some(mut result) = recovered { + result.worker = self.current_restore_summary(worker, result.worker.is_some()); + return Ok(result); + } + let mut preparation = if let Some(preparation) = owned_preparation { + preparation + } else { + self.store + .require_current_worker_singleton_owner(&self.config.workspace_id, worker)?; + let mut repository_access_workdirs = Vec::new(); + for link in self .store - .get_workdir_registry(&self.config.workspace_id, &link.workdir_id)? - .ok_or_else(|| Error::RuntimeOperationFailed { - runtime_id: "workspace-backend".to_string(), - code: "working_directory_not_found".to_string(), - message: format!("Unknown Workdir `{}`", link.workdir_id), - })?; - match record.source { - WorkdirRegistrySource::Repository { runtime_id, .. } => { - if let Some(access) = repository_access_request_for_workdir( - self, - &runtime_id, - &link.workdir_id, - &format!("worker-restore:{}", WorkerId::now_v7()), - )? { - self.runtime - .authorize_working_directory_repository_access(&runtime_id, access) - .map_err(RuntimeRegistryError::into_error)?; + .list_worker_workdir_links(&self.config.workspace_id, worker)? + .into_iter() + .filter(|link| link.unlinked_at.is_none()) + { + let record = self + .store + .get_workdir_registry(&self.config.workspace_id, &link.workdir_id)? + .ok_or_else(|| Error::RuntimeOperationFailed { + runtime_id: "workspace-backend".to_string(), + code: "working_directory_not_found".to_string(), + message: format!("Unknown Workdir `{}`", link.workdir_id), + })?; + match record.source { + WorkdirRegistrySource::Repository { + runtime_id, + repository_id, + } => { + if self + .require_configured_workspace_repository(&repository_id)? + .source + .kind + == server_api::RepositorySourceKind::Ssh + { + repository_access_workdirs.push( + runtime_api::RepositoryAccessWorkdirReference { + runtime_id, + working_directory_id: link.workdir_id.clone(), + }, + ); + } } - } - WorkdirRegistrySource::WorkspaceConfig { .. } => continue, - WorkdirRegistrySource::ExternalGrant { grant_id } => { - let grant = self - .store - .get_external_workdir_grant(&self.config.workspace_id, &grant_id)? - .ok_or_else(|| Error::RuntimeOperationFailed { - runtime_id: "external-workdir-provider".to_string(), - code: "external_workdir_unavailable".to_string(), - message: "External Workdir grant is unavailable".to_string(), - })?; - let grant_expired = external_workdir_grant_is_expired(&grant)?; - let connection = self - .external_workdir_providers - .lock() - .expect("External Workdir provider registry poisoned") - .connection(&grant_id); - if grant.status != "online" - || grant_expired - || !connection.is_some_and(|connection| { - connection.generation == grant.generation - && connection.provider_instance_id == grant.provider_instance_id - && connection.workdir_id == link.workdir_id - && connection - .expires_at - .is_none_or(|expires_at| expires_at > Utc::now()) - }) - { - return Err(Error::RuntimeOperationFailed { - runtime_id: "external-workdir-provider".to_string(), - code: "external_workdir_unavailable".to_string(), - message: "External Workdir provider is offline".to_string(), + WorkdirRegistrySource::WorkspaceConfig { .. } => continue, + WorkdirRegistrySource::ExternalGrant { grant_id } => { + let grant = self + .store + .get_external_workdir_grant(&self.config.workspace_id, &grant_id)? + .ok_or_else(|| Error::RuntimeOperationFailed { + runtime_id: "external-workdir-provider".to_string(), + code: "external_workdir_unavailable".to_string(), + message: "External Workdir grant is unavailable".to_string(), + })?; + let grant_expired = external_workdir_grant_is_expired(&grant)?; + let connection = self + .external_workdir_providers + .lock() + .expect("External Workdir provider registry poisoned") + .connection(&grant_id); + if grant.status != "online" + || grant_expired + || !connection.is_some_and(|connection| { + connection.generation == grant.generation + && connection.provider_instance_id == grant.provider_instance_id + && connection.workdir_id == link.workdir_id + && connection + .expires_at + .is_none_or(|expires_at| expires_at > Utc::now()) + }) + { + return Err(Error::RuntimeOperationFailed { + runtime_id: "external-workdir-provider".to_string(), + code: "external_workdir_unavailable".to_string(), + message: "External Workdir provider is offline".to_string(), + } + .into()); } - .into()); } } } - } - let mut binding = self - .runtime - .replace_worker_workspace_api(worker, self.workspace_api_ref(&worker.runtime_id)) - .map_err(|error| error.into_error())?; - if let Some(summary) = binding.worker.as_mut() { - summary.singleton_key = singleton_lease.as_ref().map(|lease| lease.key.clone()); - } - if binding.state != InternalWorkerOperationState::Accepted { + let preparation = runtime_api::WorkerRestorePreparation { + workspace_api: Some(runtime_contract_restore( + self.workspace_api_ref(&worker.runtime_id), + )?), + workdir_attachments: Some(runtime_contract_restore( + restore_logical_workdir_attachments(self, worker)?, + )?), + repository_access: Vec::new(), + repository_access_workdirs, + }; + let (result, preparation) = self + .runtime + .coordinate_worker_restore( + worker, + runtime_api::WorkerRestoreCoordinationRequest { + expected_observation_token: request.expected_observation_token.clone(), + request_id: request.request_id.clone(), + preparation: Some(preparation), + }, + ) + .map_err(RuntimeRegistryError::into_error)?; + if let Some(mut result) = result { + result.worker = self.current_restore_summary(worker, result.worker.is_some()); + return Ok(result); + } + preparation.ok_or_else(|| { + Error::Config("Runtime did not retain admitted Restore preparation".into()) + })? + }; + // Only the admitted operation may issue SSH handles. Reissue from the pinned + // credential snapshot after interruption; never renew candidate bindings. + let completion = + complete_restore_repository_access(self, &request.request_id, &mut preparation); + if let Err(error) = completion { return Ok(InternalWorkerRestoreResult { - state: server_api::WorkerRestoreState::Rejected, - worker: binding.worker, - diagnostics: binding.diagnostics, + state: server_api::WorkerRestoreState::ReconciliationRequired, + worker: None, + diagnostics: vec![RuntimeDiagnostic::new( + "worker_restore_preparation_pending", + "warning", + sanitize_backend_error(&error.error.to_string()), + )], }); } - sync_runtime_worker_workdir_attachments(self, worker)?; + // Persist the first resource snapshot at the admitted owner BEFORE any + // Workdir Runtime authorization effect. Replays renew only its opaque handles. + let (result, retained) = self + .runtime + .coordinate_worker_restore( + worker, + runtime_api::WorkerRestoreCoordinationRequest { + expected_observation_token: request.expected_observation_token.clone(), + request_id: request.request_id.clone(), + preparation: Some(preparation), + }, + ) + .map_err(RuntimeRegistryError::into_error)?; + if let Some(mut result) = result { + result.worker = self.current_restore_summary(worker, result.worker.is_some()); + return Ok(result); + } + let preparation = retained.ok_or_else(|| { + Error::Config("Runtime did not retain Restore resource snapshot".into()) + })?; + for access in &preparation.repository_access { + let access: worker_runtime::catalog::WorkingDirectoryRepositoryAccessRequest = + runtime_contract_restore(access.clone())?; + let runtime_id = access.materialization.runtime_id.clone(); + if let Err(error) = self + .runtime + .authorize_working_directory_repository_access(&runtime_id, access) + { + return Ok(InternalWorkerRestoreResult { + state: server_api::WorkerRestoreState::ReconciliationRequired, + worker: None, + diagnostics: vec![RuntimeDiagnostic::new( + "worker_restore_repository_access_pending", + "warning", + sanitize_backend_error(&error.message()), + )], + }); + } + } let mut restored = self .runtime - .restore_worker(worker) - .map_err(|error| error.into_error())?; - if let Some(summary) = restored.worker.as_mut() { - summary.singleton_key = singleton_lease.as_ref().map(|lease| lease.key.clone()); + .restore_worker( + worker, + runtime_api::WorkerRestoreRequest { + expected_observation_token: request.expected_observation_token, + request_id: request.request_id, + preparation: Some(preparation), + }, + ) + .map_err(RuntimeRegistryError::into_error)?; + if restored + .diagnostics + .iter() + .any(|d| d.code == "restore_observation_conflict") + { + return Err(Error::RestoreObservationConflict.into()); } + restored.worker = self.current_restore_summary(worker, restored.worker.is_some()); Ok(restored) } @@ -8216,6 +8350,7 @@ impl server_api::ServerApi for ServerApiContractService { workspace_id: String, runtime_id: String, worker_id: String, + request: server_api::WorkerRestoreRequest, ) -> std::result::Result { restore_known_worker( @@ -8225,6 +8360,7 @@ impl server_api::ServerApi for ServerApiContractService { worker: RuntimeWorkerRef::new(runtime_id, worker_id), }), current_worker_contract_headers(&context)?, + request, ) .await .map(|Json(response)| response) @@ -8939,11 +9075,12 @@ impl server_api::ServerApi for ServerApiContractService { context: server_api::ServerRequestContext, runtime_id: String, worker_id: String, + request: server_api::WorkerRestoreRequest, ) -> std::result::Result { let api = self.workspace_api()?.clone(); let operation_context = WorkerOperationContext::from_request(&api, &context)?; - restore_runtime_worker_with_context(api, runtime_id, worker_id, operation_context) + restore_runtime_worker_with_context(api, runtime_id, worker_id, operation_context, request) .await .map(|Json(response)| response) .map_err(ApiError::into_repository_api_error) @@ -8956,6 +9093,7 @@ impl server_api::ServerApi for ServerApiContractService { runtime_id: String, worker_id: String, query: server_api::RestoreTicketAssignmentQuery, + request: server_api::WorkerRestoreRequest, ) -> std::result::Result { let api = self.workspace_api()?.clone(); @@ -8970,6 +9108,7 @@ impl server_api::ServerApi for ServerApiContractService { }, query, operation_context, + request, ) .await .map(|Json(response)| response) @@ -12597,6 +12736,7 @@ async fn compensate_manual_ticket_coder_binding( api: &WorkspaceApi, worker: &RuntimeWorkerRef, binding: &ManualTicketCoderWorkdirBinding, + operation_id: &str, worker_may_be_live: bool, ) -> Vec { let mut diagnostics = Vec::new(); @@ -12680,7 +12820,11 @@ async fn compensate_manual_ticket_coder_binding( return diagnostics; } } - match api.restore_workspace_worker(worker) { + match restore_internal_worker_intent( + api, + worker, + format!("restore-compensation:{operation_id}"), + ) { Ok(result) if result.state == server_api::WorkerRestoreState::Accepted => {} Ok(result) => diagnostics.push(spawn_compensation_diagnostic( "manual_coder_worker_restore_failed", @@ -12758,7 +12902,11 @@ async fn start_manual_ticket_coder_assignment( .into()); } if let Err(error) = close_current_worker_session_locked(api, &worker).await { - let diagnostics = match api.restore_workspace_worker(&worker) { + let diagnostics = match restore_internal_worker_intent( + &api, + &worker, + format!("manual-coder-recover:{operation_id}"), + ) { Ok(result) if result.state == server_api::WorkerRestoreState::Accepted => Vec::new(), Ok(result) => vec![spawn_compensation_diagnostic( "manual_coder_worker_restore_failed", @@ -12780,7 +12928,8 @@ async fn start_manual_ticket_coder_assignment( Ok(result) if result.state == InternalWorkerOperationState::Accepted => {} Ok(result) => { let diagnostics = - compensate_manual_ticket_coder_binding(api, &worker, &binding, false).await; + compensate_manual_ticket_coder_binding(api, &worker, &binding, operation_id, false) + .await; return Err(ApiError::with_diagnostics( Error::RuntimeOperationFailed { runtime_id: worker.runtime_id.clone(), @@ -12792,7 +12941,8 @@ async fn start_manual_ticket_coder_assignment( } Err(error) => { let diagnostics = - compensate_manual_ticket_coder_binding(api, &worker, &binding, false).await; + compensate_manual_ticket_coder_binding(api, &worker, &binding, operation_id, false) + .await; return Err(ApiError::with_diagnostics(error, diagnostics)); } } @@ -12803,15 +12953,21 @@ async fn start_manual_ticket_coder_assignment( &binding.effective_links, ) { let diagnostics = - compensate_manual_ticket_coder_binding(api, &worker, &binding, false).await; + compensate_manual_ticket_coder_binding(api, &worker, &binding, operation_id, false) + .await; return Err(ApiError::with_diagnostics(error, diagnostics)); } - match api.restore_workspace_worker(&worker) { + match restore_internal_worker_intent( + &api, + &worker, + format!("manual-coder-restore:{operation_id}"), + ) { Ok(result) if result.state == server_api::WorkerRestoreState::Accepted => {} Ok(result) => { let diagnostics = - compensate_manual_ticket_coder_binding(api, &worker, &binding, false).await; + compensate_manual_ticket_coder_binding(api, &worker, &binding, operation_id, false) + .await; return Err(ApiError::with_diagnostics( Error::RuntimeOperationFailed { runtime_id: worker.runtime_id.clone(), @@ -12823,7 +12979,8 @@ async fn start_manual_ticket_coder_assignment( } Err(error) => { let diagnostics = - compensate_manual_ticket_coder_binding(api, &worker, &binding, false).await; + compensate_manual_ticket_coder_binding(api, &worker, &binding, operation_id, false) + .await; return Err(ApiError::with_diagnostics(error.error, diagnostics)); } } @@ -12835,7 +12992,8 @@ async fn start_manual_ticket_coder_assignment( Ok(assignment) => Ok(assignment), Err(error) => { let diagnostics = - compensate_manual_ticket_coder_binding(api, &worker, &binding, true).await; + compensate_manual_ticket_coder_binding(api, &worker, &binding, operation_id, true) + .await; Err(ApiError::with_diagnostics(error, diagnostics)) } } @@ -16620,6 +16778,50 @@ async fn close_current_worker_session_locked( }) } +fn restore_logical_workdir_attachments( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, +) -> ApiResult> { + api.store + .list_worker_workdir_links(&api.config.workspace_id, worker)? + .into_iter() + .filter(|link| link.unlinked_at.is_none()) + .filter(|link| { + api.store + .get_workdir_registry(&api.config.workspace_id, &link.workdir_id) + .ok() + .flatten() + .is_none_or(|record| { + !matches!(record.source, WorkdirRegistrySource::WorkspaceConfig { .. }) + }) + }) + .map(|link| { + let workdir = api + .store + .get_workdir_registry(&api.config.workspace_id, &link.workdir_id)? + .ok_or_else(|| { + Error::RegistryInconsistency(format!( + "Worker attachment references unknown Workdir {}", + link.workdir_id + )) + })?; + let capabilities = link + .capabilities + .intersection(workdir_source_capabilities(api, &workdir)?) + .intersection(ticket_target_capabilities_for_workdir( + api, worker, &workdir, + )?); + Ok(LogicalWorkdirAttachment { + alias: workdir::WorkdirAttachmentAlias::new(link.alias) + .map_err(|error| Error::InvalidInput(error.to_string()))?, + working_directory_id: link.workdir_id, + capabilities, + }) + }) + .collect::>>() + .map_err(ApiError::from) +} + fn sync_runtime_worker_workdir_attachments( api: &WorkspaceApi, worker: &RuntimeWorkerRef, @@ -22838,14 +23040,21 @@ async fn restore_known_worker( State(api): State, AxumPath(path): AxumPath, headers: HeaderMap, + request: server_api::WorkerRestoreRequest, ) -> ApiResult> { validate_workspace_scope(&api, &path.workspace_id)?; let source = authenticate_worker_mutation_source(&api, &path.workspace_id, &headers)?; let context = WorkerOperationContext::WorkerControl { controller: RuntimeWorkerRef::new(&source.runtime_id, &source.worker_id), }; - restore_runtime_worker_with_context(api, path.worker.runtime_id, path.worker.worker_id, context) - .await + restore_runtime_worker_with_context( + api, + path.worker.runtime_id, + path.worker.worker_id, + context, + request, + ) + .await } async fn scoped_list_worker_observation_sessions( @@ -23002,7 +23211,8 @@ async fn scoped_start_workspace_orchestrator( .accepted .as_ref() .map(|delivery| delivery.request_id.clone()); - let restored = api.restore_workspace_worker(&existing.worker)?; + let restored = + restore_internal_observed_worker(&api, &existing, "orchestrator-restore".to_owned())?; if restored.state == server_api::WorkerRestoreState::Accepted { { let mut attention = attention.lock().unwrap_or_else(|p| p.into_inner()); @@ -26046,6 +26256,7 @@ async fn scoped_restore_runtime_worker_with_context( path: ScopedRuntimeWorkerPath, query: RestoreTicketAssignmentQuery, context: WorkerOperationContext, + request: server_api::WorkerRestoreRequest, ) -> ApiResult> { validate_workspace_scope(&api, &path.workspace_id)?; let resolved = WorkspaceWorker::resolve(&api, &path.worker.runtime_id, &path.worker.worker_id)?; @@ -26100,28 +26311,6 @@ async fn scoped_restore_runtime_worker_with_context( &fingerprint, &Utc::now().to_rfc3339_opts(SecondsFormat::Secs, true), )?; - if let Some(worker) = existing_lifecycle_assignment_worker(&api, assignment, &runtime_id)? { - if worker.worker.worker_id != worker_id { - return Err(Error::TicketAssignmentConflict(format!( - "assignment operation {} belongs to worker {}, not {}", - assignment.operation_id, worker.worker.worker_id, worker_id - )) - .into()); - } - assign_ticket_worker_from_lifecycle(&api, assignment, &runtime_id, &worker_id)?; - accept_queued_ticket_after_worker_spawn(&api, assignment)?; - let worker = project_workspace_worker(&api, worker)?; - return Ok(Json(server_api::WorkerRestoreResponse { - workspace_id, - runtime_id: runtime_id.clone(), - worker_id: worker_id.clone(), - result: server_api::WorkerRestoreResult { - state: server_api::WorkerRestoreState::Accepted, - worker: Some(worker), - diagnostics: Vec::new(), - }, - })); - } } drop(_authorization); let response = restore_runtime_worker_with_context( @@ -26129,6 +26318,7 @@ async fn scoped_restore_runtime_worker_with_context( runtime_id.clone(), worker_id.clone(), context, + request, ) .await?; if let Some(assignment) = assignment_request.as_ref() { @@ -28439,7 +28629,7 @@ fn restore_reused_subject_worker( } let worker_ref = worker.worker.clone(); - let restored = api.restore_workspace_worker(&worker_ref)?; + let restored = restore_internal_observed_worker(api, worker, "subject-restore".to_owned())?; if restored.state != server_api::WorkerRestoreState::Accepted { let message = runtime_diagnostics_message(&restored.diagnostics); return Err(ApiError::with_diagnostics( @@ -28883,6 +29073,58 @@ async fn post_companion_cancel( Ok(Json(api.companion.cancel(request))) } +fn runtime_contract_restore( + value: T, +) -> ApiResult { + serde_json::from_value( + serde_json::to_value(value).map_err(|e| Error::InvalidInput(e.to_string()))?, + ) + .map_err(|e| Error::InvalidInput(e.to_string()).into()) +} + +fn restore_internal_worker_intent( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + request_id: String, +) -> ApiResult { + let observed = api + .runtime + .worker(worker) + .map_err(RuntimeRegistryError::into_error)?; + restore_internal_observed_worker(api, &observed, request_id) +} + +fn restore_internal_observed_worker( + api: &WorkspaceApi, + observed: &InternalWorkerSummary, + request_id: String, +) -> ApiResult { + let token = observed.restore_observation_token.as_deref(); + let explicit = !matches!( + request_id.as_str(), + "orchestrator-restore" | "subject-restore" + ); + let request = api.config_store.pin_internal_worker_restore_intent( + api.workspace_id(), + &observed.worker, + &request_id, + token, + explicit.then_some(request_id.as_str()), + )?; + let request_id = request.request_id.clone(); + let result = + WorkspaceWorker::resolve(api, &observed.worker.runtime_id, &observed.worker.worker_id)? + .restore_internal(request)?; + if result.state != server_api::WorkerRestoreState::ReconciliationRequired { + api.config_store.settle_internal_worker_restore_intent( + api.workspace_id(), + &observed.worker, + &request_id, + )?; + } + Ok(result) +} + fn resolve_workspace_worker_reference( api: &WorkspaceApi, runtime_id: &str, @@ -28923,24 +29165,35 @@ async fn restore_runtime_worker_with_context( runtime_id: String, worker_id: String, context: WorkerOperationContext, + request: server_api::WorkerRestoreRequest, ) -> ApiResult> { let worker = WorkspaceWorker::resolve(&api, &runtime_id, &worker_id)?; - let result = worker.restore(&context).await?; + let mut result = worker.restore(&context, request).await?; let projected_worker = if let Some(worker) = result.worker.as_ref() { - let record = sync_worker_observation(&api, worker)?; + // A replayed terminal receipt is historical evidence, not a new Runtime observation. + let record = api + .store + .get_worker_registry(api.workspace_id(), &worker.worker)? + .ok_or_else(|| Error::UnknownWorker { + worker: worker.worker.clone(), + })?; let links = api .store .list_worker_workdir_links(&api.config.workspace_id, &record.worker)?; let workdirs = api .store .list_workdir_registry(&api.config.workspace_id, 500)?; - Some(project_worker_registry_projection( - &api, - Some(worker), - &record, - links, - &workdirs, - )?) + match project_worker_registry_projection(&api, Some(worker), &record, links, &workdirs) { + Ok(worker) => Some(worker), + Err(error) => { + result.diagnostics.push(RuntimeDiagnostic::new( + "worker_restore_current_projection_unavailable", + "warning", + sanitize_backend_error(&error.error.to_string()), + )); + None + } + } } else { None }; @@ -30284,52 +30537,6 @@ async fn list_host_workers( })) } -fn project_workspace_worker( - api: &WorkspaceApi, - summary: InternalWorkerSummary, -) -> ApiResult { - let links = api - .store - .list_worker_workdir_links(&api.config.workspace_id, &summary.worker)?; - let workdir_attachments = summary - .workdir_attachments - .iter() - .map(|attachment| { - let record = workdir_record_from_summary( - api, - &summary.worker.runtime_id, - &attachment.working_directory.summary, - ); - let link = links - .iter() - .find(|link| { - link.unlinked_at.is_none() - && link.alias == attachment.alias.as_str() - && link.workdir_id == record.workdir_id - }) - .ok_or_else(|| { - Error::RegistryInconsistency(format!( - "Worker {}:{} attachment `{}` has no active durable Workdir link", - summary.worker.runtime_id, summary.worker.worker_id, attachment.alias - )) - })?; - Ok(server_api::WorkerWorkdirAttachmentSummary { - alias: attachment.alias.to_string(), - effective_permissions: workdir_permission_summary( - effective_worker_workdir_capabilities( - api, - &summary.worker, - &record, - link.capabilities, - )?, - ), - working_directory: projected_workdir_summary_from_record(api, &record)?.into(), - }) - }) - .collect::>>()?; - project_workspace_worker_with_attachments(api, summary, workdir_attachments) -} - fn project_workspace_worker_with_attachments( api: &WorkspaceApi, mut summary: InternalWorkerSummary, @@ -31239,6 +31446,7 @@ fn record_worker_summary( _ => protocol::subscription::SubscriptionWorkerState::Idle, }; let observation = protocol::subscription::SubscriptionWorker { + restore_observation_token: worker.restore_observation_token.clone(), worker_id, runtime_id: Some(worker.worker.runtime_id.clone()), resource_key: None, @@ -31271,6 +31479,7 @@ fn record_worker_summary( fn worker_summary_from_registry(record: &WorkerRegistryRecord) -> InternalWorkerSummary { InternalWorkerSummary { + restore_observation_token: None, worker: record.worker.clone(), host_id: "backend-registry".to_string(), display_name: record.display_name.clone(), @@ -31313,6 +31522,9 @@ fn worker_summary_from_projection( let observed = observation.availability == protocol::subscription::SubscriptionWorkerAvailability::Observed; summary.availability = observation.availability; + summary.restore_observation_token = observed + .then(|| observation.worker.restore_observation_token.clone()) + .flatten(); summary.state = if observed { match observation.worker.state { protocol::subscription::SubscriptionWorkerState::Idle => "idle", @@ -32439,6 +32651,82 @@ fn authorize_repository_materialization( Ok(()) } +fn complete_restore_repository_access( + api: &WorkspaceApi, + request_id: &str, + preparation: &mut runtime_api::WorkerRestorePreparation, +) -> ApiResult<()> { + if preparation.repository_access.is_empty() { + for reference in &preparation.repository_access_workdirs { + if let Some(access) = repository_access_request_for_workdir( + api, + &reference.runtime_id, + &reference.working_directory_id, + &format!("worker-restore:{request_id}"), + )? { + preparation + .repository_access + .push(runtime_contract_restore(access)?); + } + } + return Ok(()); + } + for prepared in &mut preparation.repository_access { + let mut access: worker_runtime::catalog::WorkingDirectoryRepositoryAccessRequest = + runtime_contract_restore(prepared.clone())?; + let Some(ssh) = access.materialization.ssh.as_mut() else { + continue; + }; + let mut leases = Vec::new(); + for candidate in &ssh.credential_candidates { + leases.push( + api.repository_secrets + .lease_ssh_materialization_access_revision( + api.workspace_id(), + &candidate.credential_id, + candidate.credential_revision, + &ssh.host_trust_id, + ssh.host_trust_revision, + )?, + ); + } + let known_hosts_entry = leases + .first() + .ok_or_else(|| Error::Config("Admitted SSH snapshot has no candidates".into()))? + .known_hosts_entry + .clone(); + ssh.expires_at_epoch_seconds = repository_access_expiry(); + ssh.secret_resource = api + .resource_broker + .issue_repository_ssh_access_handle( + api.workspace_id(), + &access.materialization.runtime_id, + format!( + "repository-ssh-access:{}", + access.materialization.operation_id + ), + ssh.secret_resource.revision.clone(), + i64::try_from(ssh.expires_at_epoch_seconds).unwrap_or(i64::MAX), + worker_runtime::resource::RepositorySshAccessSecret { + credential_candidates: leases + .into_iter() + .map( + |lease| worker_runtime::resource::RepositorySshAccessSecretCandidate { + credential_id: lease.credential_id, + credential_revision: lease.credential_revision, + private_key: lease.private_key.as_str().to_owned(), + }, + ) + .collect(), + known_hosts_entry, + }, + ) + .map_err(|error| Error::Config(error.to_string()))?; + *prepared = runtime_contract_restore(access)?; + } + Ok(()) +} + fn repository_access_request_for_workdir( api: &WorkspaceApi, runtime_id: &str, @@ -32926,7 +33214,11 @@ impl ApiError { let message = api_error_response_message(&self.error); server_api::RepositoryApiError::new( status.as_u16(), - status.canonical_reason().unwrap_or("error"), + if matches!(self.error, Error::RestoreObservationConflict) { + "restore_observation_conflict" + } else { + status.canonical_reason().unwrap_or("error") + }, message, working_directory_diagnostics(self.diagnostics), ) @@ -32944,7 +33236,8 @@ fn api_error_status(error: &Error) -> StatusCode { | Error::RuntimeBindingConflict(_) | Error::RuntimeBindingRevisionConflict { .. } | Error::RuntimeBindingFingerprintConflict { .. } - | Error::RepositoryConflict(_) => StatusCode::CONFLICT, + | Error::RepositoryConflict(_) + | Error::RestoreObservationConflict => StatusCode::CONFLICT, Error::WorkerSourceIdentity(_) => StatusCode::BAD_REQUEST, Error::InvalidInput(message) if message.starts_with("memory_not_found:") @@ -33262,6 +33555,7 @@ mod tests { resource_key: Some("W-1".to_string()), observation: Some(WorkerRegistryObservationRecord { worker: protocol::subscription::SubscriptionWorker { + restore_observation_token: None, worker_id: protocol::subscription::SubscriptionWorkerId::new("worker-a") .unwrap(), runtime_id: Some("runtime-a".to_string()), @@ -35356,6 +35650,9 @@ mod tests { reject_next_spawn: Arc>, workdir_repositories: Arc>>, logical_attachments: Arc>>>, + repository_access_requests: + Arc>>, + reject_repository_access: Arc>, } impl WorkdirlessFixtureRuntime { @@ -35438,6 +35735,10 @@ mod tests { request: &WorkerSpawnRequest, ) -> InternalWorkerSummary { InternalWorkerSummary { + restore_observation_token: Some(format!( + "fixture-generation:{}", + binding.worker_id + )), worker: RuntimeWorkerRef::new(Self::RUNTIME_ID, binding.worker_id.to_string()), host_id: "workdirless-host".to_string(), display_name: request @@ -35666,7 +35967,51 @@ mod tests { } } - fn restore_worker(&self, worker_id: &str) -> InternalWorkerRestoreResult { + fn authorize_working_directory_repository_access( + &self, + request: worker_runtime::catalog::WorkingDirectoryRepositoryAccessRequest, + ) -> std::result::Result<(), Error> { + assert_eq!(request.materialization.runtime_id, Self::RUNTIME_ID); + self.repository_access_requests + .lock() + .unwrap() + .push(request); + if *self.reject_repository_access.lock().unwrap() { + Err(Error::InvalidInput("repository access unavailable".into())) + } else { + Ok(()) + } + } + + fn coordinate_worker_restore( + &self, + worker_id: &str, + request: runtime_api::WorkerRestoreCoordinationRequest, + ) -> std::result::Result< + ( + Option, + Option, + ), + RuntimeDiagnostic, + > { + if self.worker(worker_id).worker.is_none() { + return Ok(( + Some(InternalWorkerRestoreResult { + state: server_api::WorkerRestoreState::Rejected, + worker: None, + diagnostics: Vec::new(), + }), + None, + )); + } + Ok((None, request.preparation)) + } + + fn restore_worker( + &self, + worker_id: &str, + _request: runtime_api::WorkerRestoreRequest, + ) -> InternalWorkerRestoreResult { let worker = self.worker(worker_id).worker; InternalWorkerRestoreResult { state: if worker.is_some() { @@ -37021,6 +37366,7 @@ mod tests { .seed_observation( EMBEDDED_WORKER_RUNTIME_ID, &protocol::subscription::SubscriptionWorker { + restore_observation_token: None, worker_id: protocol::subscription::SubscriptionWorkerId::new("paused-worker") .unwrap(), runtime_id: Some(EMBEDDED_WORKER_RUNTIME_ID.to_string()), @@ -39184,7 +39530,12 @@ mod tests { Error::RepositoryConflict(_) )); - let restored = api.restore_workspace_worker(&worker).unwrap(); + let restored = restore_internal_worker_intent( + &api, + &worker, + format!("restore-internal:{}", WorkerId::now_v7()), + ) + .unwrap(); assert_eq!(restored.state, server_api::WorkerRestoreState::Rejected); assert_eq!( api.store @@ -42013,7 +42364,12 @@ mod tests { Error::RepositoryConflict(message) if message.contains("worker_singleton_owned") )); assert_eq!(runtime.spawn_requests().len(), spawn_count); - let restored = api.restore_workspace_worker(&worker.worker).unwrap(); + let restored = restore_internal_observed_worker( + &api, + &worker, + format!("restore-test:{}", WorkerId::now_v7()), + ) + .unwrap(); assert_eq!(restored.state, server_api::WorkerRestoreState::Accepted); assert_eq!( restored.worker.unwrap().singleton_key.as_deref(), @@ -42025,7 +42381,12 @@ mod tests { let disconnected_runtime = WorkdirlessFixtureRuntime::default(); api.runtime .register_or_replace(disconnected_runtime.clone()); - let disconnected_restore = api.restore_workspace_worker(&worker.worker).unwrap(); + let disconnected_restore = restore_internal_observed_worker( + &api, + &worker, + format!("restore-test:{}", WorkerId::now_v7()), + ) + .unwrap(); assert_eq!( disconnected_restore.state, server_api::WorkerRestoreState::Rejected @@ -42054,7 +42415,13 @@ mod tests { // any Runtime restore side effect. api.runtime.register_or_replace(runtime.clone()); assert_eq!( - api.restore_workspace_worker(&worker.worker).unwrap().state, + restore_internal_observed_worker( + &api, + &worker, + format!("restore-test:{}", WorkerId::now_v7()) + ) + .unwrap() + .state, server_api::WorkerRestoreState::Accepted ); let stale_worker = runtime @@ -42076,8 +42443,14 @@ mod tests { .unwrap_or_else(std::sync::PoisonError::into_inner) .push(stale_worker); assert!(matches!( - api.restore_workspace_worker(&worker.worker).unwrap_err().error, - Error::RepositoryConflict(message) if message.contains("worker_singleton_fenced") + restore_internal_observed_worker( + &api, + &worker, + format!("restore-test:{}", WorkerId::now_v7()) + ) + .unwrap_err() + .error, + Error::UnknownWorker { .. } )); let replacement = api .spawn_workspace_worker( @@ -42089,8 +42462,14 @@ mod tests { .unwrap(); assert_ne!(replacement.worker.worker_id, worker.worker.worker_id); assert!(matches!( - api.restore_workspace_worker(&worker.worker).unwrap_err().error, - Error::RepositoryConflict(message) if message.contains("worker_singleton_fenced") + restore_internal_observed_worker( + &api, + &worker, + format!("restore-test:{}", WorkerId::now_v7()) + ) + .unwrap_err() + .error, + Error::UnknownWorker { .. } )); assert_eq!( api.store @@ -43104,6 +43483,12 @@ mod tests { >, input_request_ids: std::sync::Mutex>>, accept_restores: std::sync::atomic::AtomicBool, + restore_failure: + std::sync::Mutex>, + restore_operations: + std::sync::Mutex>, + reconcile_operations: + std::sync::Mutex>, inputs: std::sync::Mutex>, protocol_methods: Arc>>, @@ -43130,6 +43515,9 @@ mod tests { input_hook: std::sync::Mutex::new(None), input_request_ids: std::sync::Mutex::new(Vec::new()), accept_restores: std::sync::atomic::AtomicBool::new(false), + restore_failure: std::sync::Mutex::new(None), + restore_operations: std::sync::Mutex::new(Vec::new()), + reconcile_operations: std::sync::Mutex::new(Vec::new()), inputs: std::sync::Mutex::new(Vec::new()), protocol_methods: Arc::new(std::sync::Mutex::new(Vec::new())), stops: std::sync::Mutex::new(Vec::new()), @@ -43307,6 +43695,13 @@ mod tests { &self, request: worker_runtime::execution::WorkerExecutionRestoreRequest, ) -> worker_runtime::execution::WorkerExecutionSpawnResult { + self.restore_operations + .lock() + .unwrap() + .push(request.operation_id); + if let Some(result) = self.restore_failure.lock().unwrap().take() { + return result; + } if !self .accept_restores .load(std::sync::atomic::Ordering::SeqCst) @@ -43339,6 +43734,17 @@ mod tests { } } + fn reconcile_restore( + &self, + request: worker_runtime::execution::WorkerExecutionRestoreRequest, + ) -> worker_runtime::execution::WorkerExecutionSpawnResult { + self.reconcile_operations + .lock() + .unwrap() + .push(request.operation_id); + self.restore_worker(request) + } + fn dispatch_method( &self, worker_ref: &worker_runtime::identity::WorkerRef, @@ -55050,6 +55456,7 @@ mod tests { visibility: &str, ) -> server_api::WorkerSummary { server_api::WorkerSummary { + restore_observation_token: None, runtime_id: "runtime-test".to_string(), worker_id: worker_id.to_string(), resource_key: key.to_string(), @@ -55144,6 +55551,7 @@ mod tests { let discovered = workspace_worker_discovery_page( vec![server_api::WorkerSummary { + restore_observation_token: None, runtime_id: target.runtime_id.clone(), worker_id: target.worker_id.clone(), resource_key: "W-2".to_string(), @@ -57945,7 +58353,10 @@ mod tests { let restored = post_json( app.clone(), &format!("/api/runtimes/embedded-worker-runtime/workers/{worker_id}/restore"), - json!({}), + json!({ + "expected_observation_token": worker["restore_observation_token"], + "request_id": "embedded-route-fixture-restore" + }), ) .await; assert_eq!(restored["result"]["state"], "accepted"); @@ -58002,10 +58413,17 @@ mod tests { .unwrap(); assert_eq!(wrong_runtime.status(), StatusCode::NOT_FOUND); + // Only the public observation generation key is exempt from this + // credential non-leak check. Its values must be Runtime UUIDv7 tokens. + for summary in [&worker, &restored["result"]["worker"]] { + let token = summary["restore_observation_token"].as_str().unwrap(); + assert_eq!(uuid::Uuid::parse_str(token).unwrap().get_version_num(), 7); + } let projected = format!( "{}{}{}{}{}{}", embedded_summary, spawned, worker, stopped_workers, restored, accepted - ); + ) + .replace("\"restore_observation_token\":", "\"restore_observation\":"); for forbidden in [ dir.path().to_string_lossy().as_ref(), "metadata.json", @@ -58255,7 +58673,20 @@ mod tests { // B is installed and a delayed method enters its existing queue. runtime.stop_worker(&worker_ref, None).unwrap(); execution.accept_restores(); - runtime.restore_worker(&worker_ref).unwrap(); + runtime + .restore_worker( + &worker_ref, + runtime_api::WorkerRestoreRequest { + expected_observation_token: runtime + .worker_detail(&worker_ref) + .unwrap() + .restore_observation_token + .unwrap(), + request_id: WorkerId::now_v7().to_string(), + preparation: Some(runtime_api::WorkerRestorePreparation::default()), + }, + ) + .unwrap(); old.methods.try_send(delayed).unwrap(); execution.emit_worker_event( &worker_ref, @@ -58353,7 +58784,20 @@ mod tests { // Stop and restore happen before this socket loop processes closure. runtime.stop_worker(&worker_ref, None).unwrap(); execution.accept_restores(); - runtime.restore_worker(&worker_ref).unwrap(); + runtime + .restore_worker( + &worker_ref, + runtime_api::WorkerRestoreRequest { + expected_observation_token: runtime + .worker_detail(&worker_ref) + .unwrap() + .restore_observation_token + .unwrap(), + request_id: WorkerId::now_v7().to_string(), + preparation: Some(runtime_api::WorkerRestorePreparation::default()), + }, + ) + .unwrap(); // A peer close may win this race; otherwise the old connected // dispatcher must reject the frame without touching B. let _ = old @@ -58624,7 +59068,20 @@ mod tests { ); execution.accept_restores(); - runtime.restore_worker(&worker_ref).unwrap(); + runtime + .restore_worker( + &worker_ref, + runtime_api::WorkerRestoreRequest { + expected_observation_token: runtime + .worker_detail(&worker_ref) + .unwrap() + .restore_observation_token + .unwrap(), + request_id: WorkerId::now_v7().to_string(), + preparation: Some(runtime_api::WorkerRestorePreparation::default()), + }, + ) + .unwrap(); let mut current = execution .clone() .attach_worker_protocol(&worker_ref) diff --git a/crates/workspace-server/src/server/tests/worker_operations_tests.rs b/crates/workspace-server/src/server/tests/worker_operations_tests.rs index d6083877f..e6cf86e84 100644 --- a/crates/workspace-server/src/server/tests/worker_operations_tests.rs +++ b/crates/workspace-server/src/server/tests/worker_operations_tests.rs @@ -250,7 +250,13 @@ async fn offline_handle_can_pin_but_deleted_handle_cannot_mutate_or_connect() { ); assert!(matches!( handle - .restore(&WorkerOperationContext::Backend) + .restore( + &WorkerOperationContext::Backend, + server_api::WorkerRestoreRequest { + expected_observation_token: "stale".into(), + request_id: "removed-handle".into(), + } + ) .await .unwrap_err() .error, @@ -374,7 +380,17 @@ async fn worker_control_handle_rechecks_revoked_grants_for_each_operation() { ); } assert!(matches!( - handle.restore(&context).await.unwrap_err().error, + handle + .restore( + &context, + server_api::WorkerRestoreRequest { + expected_observation_token: "stale".into(), + request_id: "revoked-grant".into(), + } + ) + .await + .unwrap_err() + .error, Error::UnknownWorker { .. } )); assert!(matches!( @@ -682,7 +698,15 @@ async fn generic_management_aliases_cannot_bypass_worker_grants_or_target_bound_ ); assert!( service - .runtime_worker_restore_alias(context(), "offline-runtime".into(), "subject".into()) + .runtime_worker_restore_alias( + context(), + "offline-runtime".into(), + "subject".into(), + server_api::WorkerRestoreRequest { + expected_observation_token: "old".into(), + request_id: "unauthorized".into(), + } + ) .await .is_err() ); @@ -862,3 +886,696 @@ async fn browser_protocol_connection_and_cleanup_reject_cookie_origin_mismatch() assert!(matches!(result.error, Error::WorkspacePermissionDenied(_))); assert!(execution.protocol_methods().is_empty()); } + +fn restore_request_for( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + request_id: &str, +) -> server_api::WorkerRestoreRequest { + server_api::WorkerRestoreRequest { + expected_observation_token: api + .runtime + .worker(worker) + .unwrap() + .restore_observation_token + .unwrap(), + request_id: request_id.into(), + } +} + +#[tokio::test] +async fn restore_guard_rejects_stale_list_and_stopped_restored_stopped_aba() { + let workspace = tempfile::tempdir().unwrap(); + let (api, execution) = test_api_with_recording_backend(workspace.path()).await; + execution.accept_restores(); + let identity = spawn_ticket_check_source(&api, "restore-guard-aba"); + let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap(); + handle + .stop(&WorkerOperationContext::Backend, operation_lifecycle()) + .await + .unwrap(); + let original = restore_request_for(&api, &identity, "restore-original"); + let stale_list = server_api::WorkerRestoreRequest { + request_id: "restore-stale-list".into(), + ..original.clone() + }; + assert_eq!( + handle + .restore(&WorkerOperationContext::Backend, original.clone()) + .await + .unwrap() + .state, + server_api::WorkerRestoreState::Accepted + ); + let conflict = handle + .restore(&WorkerOperationContext::Backend, stale_list.clone()) + .await + .unwrap_err(); + assert!(matches!(conflict.error, Error::RestoreObservationConflict)); + let typed = conflict.into_repository_api_error(); + assert_eq!( + api_error_status(&Error::RestoreObservationConflict), + StatusCode::CONFLICT + ); + assert_eq!(typed.error, "restore_observation_conflict"); + handle + .stop(&WorkerOperationContext::Backend, operation_lifecycle()) + .await + .unwrap(); + assert_ne!( + original.expected_observation_token, + api.runtime + .worker(&identity) + .unwrap() + .restore_observation_token + .unwrap() + ); + assert!(matches!( + handle + .restore(&WorkerOperationContext::Backend, stale_list) + .await + .unwrap_err() + .error, + Error::RestoreObservationConflict + )); + assert_eq!(api.runtime.worker(&identity).unwrap().state, "stopped"); + assert!( + execution.contexts.lock().unwrap().is_empty(), + "conflict must not install execution" + ); + // The original request is result replay, not a new intent; it stays accepted + // without resurrecting the Worker after a subsequent stop. + assert_eq!( + handle + .restore(&WorkerOperationContext::Backend, original) + .await + .unwrap() + .state, + server_api::WorkerRestoreState::Accepted + ); + assert_eq!(api.runtime.worker(&identity).unwrap().state, "stopped"); + assert!(execution.contexts.lock().unwrap().is_empty()); +} + +fn restore_test_workdir( + api: &WorkspaceApi, + id: &str, + source: WorkdirRegistrySource, +) -> WorkdirRegistryRecord { + WorkdirRegistryRecord { + workspace_id: api.workspace_id().into(), + workdir_id: id.into(), + display_name: None, + source, + creation_selector: None, + creation_ref: None, + creation_tree: None, + current_selector: None, + current_ref: None, + current_tree: None, + observed_at_epoch_seconds: None, + materialization_status: "present".into(), + cleanliness: "clean".into(), + created_at: TEST_CREATED_AT.into(), + updated_at: TEST_CREATED_AT.into(), + } +} + +fn attach_restore_test_workdir( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + workdir: &WorkdirRegistryRecord, +) { + api.store + .attach_worker_workdir(&WorkerWorkdirLinkRecord { + connection_id: String::new(), + workspace_id: api.workspace_id().into(), + worker: worker.clone(), + workdir_id: workdir.workdir_id.clone(), + alias: workdir.workdir_id.clone(), + capabilities: workdir::WorkdirSessionCapabilities::READ_ONLY, + linked_at: TEST_CREATED_AT.into(), + unlinked_at: None, + }) + .unwrap(); +} + +fn displace_restore_singleton( + api: &WorkspaceApi, + worker: &RuntimeWorkerRef, + replacement: &RuntimeWorkerRef, +) { + api.config_store.with_conn(|conn| { + conn.execute("UPDATE worker_create_reservations SET singleton_key='restore-test-singleton', singleton_generation=1 WHERE workspace_id=?1 AND worker_id=?2", + rusqlite::params![api.workspace_id(), worker.worker_id])?; + conn.execute("INSERT INTO worker_singleton_owners(workspace_id,singleton_key,runtime_id,worker_id,generation,created_at,updated_at) VALUES(?1,'restore-test-singleton',?2,?3,2,?4,?4)", + rusqlite::params![api.workspace_id(), replacement.runtime_id, replacement.worker_id, TEST_CREATED_AT])?; + Ok(()) + }).unwrap(); + assert!( + api.store + .require_current_worker_singleton_owner(api.workspace_id(), worker) + .is_err() + ); +} + +fn add_offline_restore_workdir(api: &WorkspaceApi, worker: &RuntimeWorkerRef) { + let record = restore_test_workdir( + api, + "restore-offline", + WorkdirRegistrySource::ExternalGrant { + grant_id: "restore-offline-grant".into(), + }, + ); + api.store + .create_external_workdir_grant( + &crate::store::ExternalWorkdirGrantRecord { + grant_id: "restore-offline-grant".into(), + workspace_id: api.workspace_id().into(), + workdir_id: record.workdir_id.clone(), + provider_instance_id: "offline-provider".into(), + display_name: "Offline grant".into(), + permissions: "read_only".into(), + created_by: "owner-account".into(), + created_at: TEST_CREATED_AT.into(), + expires_at: None, + generation: 1, + status: "offline".into(), + updated_at: TEST_CREATED_AT.into(), + }, + &record, + ) + .unwrap(); + attach_restore_test_workdir(api, worker, &record); +} + +#[tokio::test] +async fn restore_guard_stale_ssh_intent_does_not_issue_workspace_resource() { + let workspace = tempfile::tempdir().unwrap(); + let api = test_api_with_remote_repository(workspace.path()).await; + let identity = spawn_ticket_check_source(&api, "restore-stale-ssh"); + let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap(); + let stale = restore_request_for(&api, &identity, "stale-ssh-intent"); + handle + .stop(&WorkerOperationContext::Backend, operation_lifecycle()) + .await + .unwrap(); + api.repository_secrets + .generate_credential( + api.workspace_id(), + GenerateRepositorySshCredentialRequest { + operation_id: "restore-default-key".into(), + credential_id: + crate::repository_access::WORKSPACE_DEFAULT_REPOSITORY_SSH_CREDENTIAL_ID.into(), + name: "Restore test default".into(), + }, + "owner-account", + ) + .unwrap(); + let public_key = api + .repository_secrets + .credential_public_key( + api.workspace_id(), + crate::repository_access::WORKSPACE_DEFAULT_REPOSITORY_SSH_CREDENTIAL_ID, + ) + .unwrap() + .unwrap() + .public_key; + api.repository_secrets + .put_host_trust( + api.workspace_id(), + PutRepositorySshHostTrustRequest { + operation_id: "restore-ssh-trust".into(), + host_trust_id: "restore-ssh-host".into(), + hostname: "example.invalid".into(), + port: 22, + host_key: public_key, + expected_revision: None, + }, + "owner-account", + ) + .unwrap(); + let record = restore_test_workdir( + &api, + "restore-ssh-workdir", + WorkdirRegistrySource::Repository { + runtime_id: identity.runtime_id.clone(), + repository_id: test_repository_id(&api), + }, + ); + api.store.upsert_workdir_registry(&record).unwrap(); + attach_restore_test_workdir(&api, &identity, &record); + // Prove this is a fully issuable SSH fixture; the old pre-admission path would mint another handle. + assert!( + repository_access_request_for_workdir( + &api, + &identity.runtime_id, + &record.workdir_id, + "restore-ssh-positive-control" + ) + .unwrap() + .is_some() + ); + let before = api.resource_broker.repository_ssh_access_handle_count(); + assert!(before > 0); + let conflict = handle + .restore(&WorkerOperationContext::Backend, stale) + .await + .unwrap_err(); + assert!(matches!(conflict.error, Error::RestoreObservationConflict)); + assert_eq!( + api.resource_broker.repository_ssh_access_handle_count(), + before + ); + assert_eq!(api.runtime.worker(&identity).unwrap().state, "stopped"); +} + +#[tokio::test] +async fn restore_guard_historical_replay_ignores_new_eligibility_but_rechecks_auth() { + let workspace = tempfile::tempdir().unwrap(); + let (api, execution) = test_api_with_recording_backend(workspace.path()).await; + execution.accept_restores(); + let identity = spawn_ticket_check_source(&api, "restore-historical"); + let replacement = spawn_ticket_check_source(&api, "restore-replacement"); + let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap(); + handle + .stop(&WorkerOperationContext::Backend, operation_lifecycle()) + .await + .unwrap(); + let request = restore_request_for(&api, &identity, "restore-historical-request"); + assert_eq!( + handle + .restore(&WorkerOperationContext::Backend, request.clone()) + .await + .unwrap() + .state, + server_api::WorkerRestoreState::Accepted + ); + handle + .stop(&WorkerOperationContext::Backend, operation_lifecycle()) + .await + .unwrap(); + displace_restore_singleton(&api, &identity, &replacement); + add_offline_restore_workdir(&api, &identity); + let before = execution.restore_operations.lock().unwrap().len(); + let replay = handle + .restore(&WorkerOperationContext::Backend, request.clone()) + .await + .unwrap(); + assert_eq!(replay.state, server_api::WorkerRestoreState::Accepted); + assert_eq!( + replay.worker.unwrap().state, + "stopped", + "never return the old idle receipt summary" + ); + assert_eq!(execution.restore_operations.lock().unwrap().len(), before); + let projected = restore_runtime_worker_with_context( + api.clone(), + identity.runtime_id.clone(), + identity.worker_id.clone(), + WorkerOperationContext::Backend, + request.clone(), + ) + .await + .unwrap() + .0; + assert_eq!( + projected.result.worker.unwrap().state, + "unavailable", + "current fenced projection, never old idle receipt" + ); + let (mut browser, _) = browser_operation_context(&api).await; + let WorkerOperationContext::Browser { headers, .. } = &mut browser else { + unreachable!() + }; + headers.insert(ORIGIN, "https://untrusted.example".parse().unwrap()); + assert!(matches!( + handle.restore(&browser, request).await.unwrap_err().error, + Error::WorkspacePermissionDenied(_) + )); + assert_eq!(execution.restore_operations.lock().unwrap().len(), before); + assert!( + handle + .restore( + &WorkerOperationContext::Backend, + restore_request_for(&api, &identity, "new-ineligible-intent") + ) + .await + .is_err() + ); +} + +#[tokio::test] +async fn restore_guard_pending_recovery_ignores_new_eligibility_and_reuses_operation() { + let workspace = tempfile::tempdir().unwrap(); + let (api, execution) = test_api_with_recording_backend(workspace.path()).await; + execution.accept_restores(); + let identity = spawn_ticket_check_source(&api, "restore-pending"); + let replacement = spawn_ticket_check_source(&api, "restore-pending-replacement"); + let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap(); + handle + .stop(&WorkerOperationContext::Backend, operation_lifecycle()) + .await + .unwrap(); + *execution.restore_failure.lock().unwrap() = Some( + worker_runtime::execution::WorkerExecutionSpawnResult::Errored( + worker_runtime::execution::WorkerExecutionResult::rejected( + worker_runtime::execution::WorkerExecutionOperation::Restore, + "uncertain launch", + ), + ), + ); + let request = restore_request_for(&api, &identity, "restore-pending-request"); + assert_eq!( + handle + .restore(&WorkerOperationContext::Backend, request.clone()) + .await + .unwrap() + .state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + let original = execution.restore_operations.lock().unwrap()[0]; + displace_restore_singleton(&api, &identity, &replacement); + add_offline_restore_workdir(&api, &identity); + assert_eq!( + handle + .restore(&WorkerOperationContext::Backend, request) + .await + .unwrap() + .state, + server_api::WorkerRestoreState::Accepted + ); + assert_eq!( + execution.reconcile_operations.lock().unwrap().as_slice(), + &[original] + ); + assert!( + execution + .restore_operations + .lock() + .unwrap() + .iter() + .all(|id| id == &original) + ); +} + +#[tokio::test] +async fn restore_guard_internal_intent_survives_client_store_restart_without_replacing_token() { + let workspace = tempfile::tempdir().unwrap(); + let (mut api, execution) = test_api_with_recording_backend(workspace.path()).await; + execution.accept_restores(); + let identity = spawn_ticket_check_source(&api, "restore-internal-restart"); + WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id) + .unwrap() + .stop(&WorkerOperationContext::Backend, operation_lifecycle()) + .await + .unwrap(); + let observed = api.runtime.worker(&identity).unwrap(); + *execution.restore_failure.lock().unwrap() = Some( + worker_runtime::execution::WorkerExecutionSpawnResult::Errored( + worker_runtime::execution::WorkerExecutionResult::rejected( + worker_runtime::execution::WorkerExecutionOperation::Restore, + "uncertain launch", + ), + ), + ); + assert_eq!( + restore_internal_observed_worker(&api, &observed, "subject-restore".into()) + .unwrap() + .state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + let original = execution.restore_operations.lock().unwrap()[0]; + let current = api.runtime.worker(&identity).unwrap(); + assert_ne!( + observed.restore_observation_token, + current.restore_observation_token + ); + // Reopen the durable caller journal, discarding all client-side memory of request identity. + api.config_store = + Arc::new(SqliteWorkspaceStore::open(api.config.database_path.clone()).unwrap()); + let pinned = api + .config_store + .pin_internal_worker_restore_intent( + api.workspace_id(), + &identity, + "subject-restore", + current.restore_observation_token.as_deref(), + None, + ) + .unwrap(); + assert_eq!( + pinned.expected_observation_token, + observed.restore_observation_token.unwrap() + ); + assert_eq!( + restore_internal_observed_worker(&api, ¤t, "subject-restore".into()) + .unwrap() + .state, + server_api::WorkerRestoreState::Accepted + ); + assert_eq!( + execution.reconcile_operations.lock().unwrap().as_slice(), + &[original] + ); + assert!( + execution + .restore_operations + .lock() + .unwrap() + .iter() + .all(|id| id == &original) + ); +} + +#[tokio::test] +async fn restore_guard_workspace_interrupted_preparation_recovers_admitted_snapshot() { + let workspace = tempfile::tempdir().unwrap(); + let (api, execution) = test_api_with_recording_backend(workspace.path()).await; + execution.accept_restores(); + let identity = spawn_ticket_check_source(&api, "restore-prepare-interrupted"); + let replacement = spawn_ticket_check_source(&api, "restore-prepare-replacement"); + let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap(); + handle + .stop(&WorkerOperationContext::Backend, operation_lifecycle()) + .await + .unwrap(); + let request = restore_request_for(&api, &identity, "restore-prepare-interrupted-request"); + let preparation = runtime_api::WorkerRestorePreparation { + workspace_api: Some( + runtime_contract_restore(api.workspace_api_ref(&identity.runtime_id)).unwrap(), + ), + workdir_attachments: Some(Vec::new()), + ..Default::default() + }; + let (result, retained) = api + .runtime + .coordinate_worker_restore( + &identity, + runtime_api::WorkerRestoreCoordinationRequest { + expected_observation_token: request.expected_observation_token.clone(), + request_id: request.request_id.clone(), + preparation: Some(preparation.clone()), + }, + ) + .unwrap(); + assert!(result.is_none()); + assert_eq!(retained, Some(preparation)); + assert!(execution.restore_operations.lock().unwrap().is_empty()); + // Model interruption between admission and completion, then changed current eligibility. + displace_restore_singleton(&api, &identity, &replacement); + add_offline_restore_workdir(&api, &identity); + let restored = handle + .restore(&WorkerOperationContext::Backend, request.clone()) + .await + .unwrap(); + assert_eq!(restored.state, server_api::WorkerRestoreState::Accepted); + assert_eq!(execution.restore_operations.lock().unwrap().len(), 1); + let original = execution.restore_operations.lock().unwrap()[0]; + assert!( + execution.reconcile_operations.lock().unwrap().is_empty(), + "no execution existed to reconcile yet" + ); + assert_eq!( + handle + .restore(&WorkerOperationContext::Backend, request) + .await + .unwrap() + .state, + server_api::WorkerRestoreState::Accepted + ); + assert_eq!( + execution.restore_operations.lock().unwrap().as_slice(), + &[original] + ); +} + +#[tokio::test] +async fn restore_guard_cross_runtime_ssh_completion_retries_pinned_snapshot_at_workdir_owner() { + let workspace = tempfile::tempdir().unwrap(); + let mut config = test_server_config(workspace.path()); + let source = server_api::RepositorySource { + kind: server_api::RepositorySourceKind::Ssh, + uri: "git@example.invalid:owner/repository.git".into(), + }; + config.repositories[0].source_fingerprint = + crate::repository_source::repository_source_fingerprint(&source); + config.repositories[0].source = source; + config.repositories[0].path = None; + let store = SqliteWorkspaceStore::open(config.database_path.clone()).unwrap(); + let execution = Arc::new(DeterministicExecutionBackend::default()); + execution.accept_restores(); + let api = WorkspaceApi::new_with_execution_backend(config, Arc::new(store), execution.clone()) + .await + .unwrap(); + let peer = WorkdirlessFixtureRuntime::default(); + *peer.reject_repository_access.lock().unwrap() = true; + api.runtime.register_or_replace(peer.clone()); + let identity = spawn_ticket_check_source(&api, "restore-cross-runtime-ssh"); + let handle = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap(); + handle + .stop(&WorkerOperationContext::Backend, operation_lifecycle()) + .await + .unwrap(); + let original_key = api + .repository_secrets + .generate_credential( + api.workspace_id(), + GenerateRepositorySshCredentialRequest { + operation_id: "cross-runtime-default-key".into(), + credential_id: + crate::repository_access::WORKSPACE_DEFAULT_REPOSITORY_SSH_CREDENTIAL_ID.into(), + name: "Restore default".into(), + }, + "owner-account", + ) + .unwrap(); + let public_key = api + .repository_secrets + .credential_public_key(api.workspace_id(), &original_key.credential_id) + .unwrap() + .unwrap() + .public_key; + api.repository_secrets + .put_host_trust( + api.workspace_id(), + PutRepositorySshHostTrustRequest { + operation_id: "cross-runtime-trust".into(), + host_trust_id: "cross-runtime-host".into(), + hostname: "example.invalid".into(), + port: 22, + host_key: public_key, + expected_revision: None, + }, + "owner-account", + ) + .unwrap(); + let record = restore_test_workdir( + &api, + "cross-runtime-restore-workdir", + WorkdirRegistrySource::Repository { + runtime_id: WorkdirlessFixtureRuntime::RUNTIME_ID.into(), + repository_id: test_repository_id(&api), + }, + ); + peer.set_workdir_repository(&record.workdir_id, &test_repository_id(&api)); + api.store.upsert_workdir_registry(&record).unwrap(); + attach_restore_test_workdir(&api, &identity, &record); + let request = restore_request_for(&api, &identity, "cross-runtime-restore-request"); + assert_eq!( + handle + .restore(&WorkerOperationContext::Backend, request.clone()) + .await + .unwrap() + .state, + server_api::WorkerRestoreState::ReconciliationRequired + ); + assert!( + execution.restore_operations.lock().unwrap().is_empty(), + "credential completion must precede execution" + ); + let first = peer.repository_access_requests.lock().unwrap()[0].clone(); + assert_eq!( + first.materialization.runtime_id, + WorkdirlessFixtureRuntime::RUNTIME_ID + ); + assert_ne!(first.materialization.runtime_id, identity.runtime_id); + // Rotate current host trust while the accepted operation is uncertain. Its + // original credential/trust revisions, source and binding must remain pinned. + let alternate = api + .repository_secrets + .generate_credential( + api.workspace_id(), + GenerateRepositorySshCredentialRequest { + operation_id: "cross-runtime-alternate-key".into(), + credential_id: "cross-runtime-alternate".into(), + name: "Alternate".into(), + }, + "owner-account", + ) + .unwrap(); + let alternate_public = api + .repository_secrets + .credential_public_key(api.workspace_id(), &alternate.credential_id) + .unwrap() + .unwrap() + .public_key; + let original_trust_revision = first + .materialization + .ssh + .as_ref() + .unwrap() + .host_trust_revision; + api.repository_secrets + .put_host_trust( + api.workspace_id(), + PutRepositorySshHostTrustRequest { + operation_id: "cross-runtime-trust-rotate".into(), + host_trust_id: "cross-runtime-host".into(), + hostname: "example.invalid".into(), + port: 22, + host_key: alternate_public, + expected_revision: Some(original_trust_revision), + }, + "owner-account", + ) + .unwrap(); + *peer.reject_repository_access.lock().unwrap() = false; + assert_eq!( + handle + .restore(&WorkerOperationContext::Backend, request.clone()) + .await + .unwrap() + .state, + server_api::WorkerRestoreState::Accepted + ); + let attempts = peer.repository_access_requests.lock().unwrap(); + assert_eq!(attempts.len(), 2); + let second = &attempts[1]; + assert_eq!( + second.materialization.operation_id, + first.materialization.operation_id + ); + let mut normalized = second.clone(); + let old_ssh = first.materialization.ssh.as_ref().unwrap(); + let new_ssh = normalized.materialization.ssh.as_mut().unwrap(); + assert_ne!(new_ssh.secret_resource.nonce, old_ssh.secret_resource.nonce); + new_ssh.secret_resource = old_ssh.secret_resource.clone(); + new_ssh.expires_at_epoch_seconds = old_ssh.expires_at_epoch_seconds; + assert_eq!(normalized, first, "only opaque handle and expiry may renew"); + drop(attempts); + assert_eq!(execution.restore_operations.lock().unwrap().len(), 1); + assert_eq!( + handle + .restore(&WorkerOperationContext::Backend, request) + .await + .unwrap() + .state, + server_api::WorkerRestoreState::Accepted + ); + assert_eq!( + peer.repository_access_requests.lock().unwrap().len(), + 2, + "terminal receipt must not renew access" + ); +} diff --git a/crates/workspace-server/src/server/worker_operations.rs b/crates/workspace-server/src/server/worker_operations.rs index d8ada5fec..754787c3c 100644 --- a/crates/workspace-server/src/server/worker_operations.rs +++ b/crates/workspace-server/src/server/worker_operations.rs @@ -291,10 +291,20 @@ impl WorkspaceWorker { pub(super) async fn restore( &self, context: &WorkerOperationContext, + request: server_api::WorkerRestoreRequest, ) -> ApiResult { let _guard = self.authorize_operation(context, "restore").await?; - // Preserve singleton, Workdir, credentials, bindings and pending operation reconciliation. - self.api.restore_workspace_worker(&self.identity) + self.restore_internal(request) + } + + /// Trusted Backend callers still supply an explicit observed intent. Runtime + /// admission is shared with browser/Tool callers; this is not an unguarded launch. + pub(super) fn restore_internal( + &self, + request: server_api::WorkerRestoreRequest, + ) -> ApiResult { + self.current_record()?; + self.api.restore_workspace_worker(&self.identity, request) } pub(super) async fn stop( diff --git a/crates/workspace-server/src/store.rs b/crates/workspace-server/src/store.rs index 69bb563d3..4e31dbc8f 100644 --- a/crates/workspace-server/src/store.rs +++ b/crates/workspace-server/src/store.rs @@ -28,7 +28,7 @@ use crate::workspace_deletion::WorkspaceDeletionStore; use crate::{Error, Result}; const OLDEST_SCHEMA_VERSION: i64 = 50; -const LATEST_SCHEMA_VERSION: i64 = 79; +const LATEST_SCHEMA_VERSION: i64 = 80; const WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME: &str = "Workspace config grants and logical Workdirs"; const SCHEMA_BASELINE_NAME: &str = "workspace schema baseline"; const WORKSPACE_RUNTIME_BINDINGS_MIGRATION_NAME: &str = "workspace runtime bindings"; @@ -79,6 +79,8 @@ const ARCHIVE_OBSERVE_GRANTS_MIGRATION_NAME: &str = const BACKEND_JOB_WORKER_CLEANUP_MIGRATION_NAME: &str = "durable terminal Backend Job Worker cleanup"; const WORKDIR_CONNECTION_ID_MIGRATION_NAME: &str = "durable Workdir attachment connection identity"; +const WORKER_RESTORE_INTENTS_MIGRATION_NAME: &str = + "durable internal Worker Restore request identity"; const TICKET_SCHEMA_VERSION_WITH_TARGETS: i64 = 7; const TICKET_SCHEMA_BASELINE_NAME: &str = "ticket schema baseline"; const WORKER_REGISTRY_PROJECTION_SCHEMA: &str = r#" @@ -271,6 +273,11 @@ const MIGRATIONS: &[Migration] = &[ name: WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME, apply: migrate_workspace_config_v78_to_v79, }, + Migration { + version: 80, + name: WORKER_RESTORE_INTENTS_MIGRATION_NAME, + apply: migrate_worker_restore_intents_v79_to_v80, + }, ]; #[derive(Clone, Copy)] @@ -2167,6 +2174,48 @@ impl SqliteWorkspaceStore { }) } + /// Client intent journal, not a lifecycle state machine: pin the public tuple + /// before dispatch and keep it unchanged until a terminal receipt is obtained. + pub(crate) fn pin_internal_worker_restore_intent( + &self, + workspace_id: &str, + worker: &RuntimeWorkerRef, + domain_key: &str, + observed_token: Option<&str>, + explicit_request_id: Option<&str>, + ) -> Result { + self.with_transaction(|tx| { + let existing: Option<(String, String)> = if let Some(id) = explicit_request_id { + tx.query_row("SELECT request_id, expected_token FROM worker_restore_intents WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND request_id=?4", + params![workspace_id, worker.runtime_id, worker.worker_id, id], |row| Ok((row.get(0)?, row.get(1)?))).optional()? + } else { + tx.query_row("SELECT request_id, expected_token FROM worker_restore_intents WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND domain_key=?4 AND settled=0", + params![workspace_id, worker.runtime_id, worker.worker_id, domain_key], |row| Ok((row.get(0)?, row.get(1)?))).optional()? + }; + if let Some((request_id, expected_observation_token)) = existing { + return Ok(server_api::WorkerRestoreRequest { request_id, expected_observation_token }); + } + let observed_token = observed_token.ok_or(Error::RestoreObservationConflict)?; + let request_id = explicit_request_id.map(str::to_owned).unwrap_or_else(|| format!("{domain_key}:{observed_token}")); + tx.execute("INSERT INTO worker_restore_intents(workspace_id,runtime_id,worker_id,domain_key,request_id,expected_token,settled) VALUES(?1,?2,?3,?4,?5,?6,0)", + params![workspace_id, worker.runtime_id, worker.worker_id, domain_key, request_id, observed_token])?; + Ok(server_api::WorkerRestoreRequest { request_id, expected_observation_token: observed_token.to_owned() }) + }) + } + + pub(crate) fn settle_internal_worker_restore_intent( + &self, + workspace_id: &str, + worker: &RuntimeWorkerRef, + request_id: &str, + ) -> Result<()> { + self.with_conn(|conn| { + conn.execute("UPDATE worker_restore_intents SET settled=1 WHERE workspace_id=?1 AND runtime_id=?2 AND worker_id=?3 AND request_id=?4", + params![workspace_id, worker.runtime_id, worker.worker_id, request_id])?; + Ok(()) + }) + } + pub(crate) fn get_workspace_memory_settings( &self, workspace_id: &str, @@ -11248,6 +11297,7 @@ fn read_worker_registry_projection( })? } else { SubscriptionWorker { + restore_observation_token: None, worker_id: SubscriptionWorkerId::new(worker.worker_id.to_string()) .map_err(|error| Error::InvalidInput(error.to_string()))?, runtime_id: Some(worker.runtime_id.to_string()), @@ -12013,6 +12063,24 @@ fn read_workdir_registry_record( }) } +fn migrate_worker_restore_intents_v79_to_v80(conn: &Connection) -> Result<()> { + let tx = rusqlite::Transaction::new_unchecked(conn, TransactionBehavior::Exclusive)?; + tx.execute_batch("CREATE TABLE worker_restore_intents ( + workspace_id TEXT NOT NULL, runtime_id TEXT NOT NULL, worker_id TEXT NOT NULL, + domain_key TEXT NOT NULL, request_id TEXT NOT NULL, expected_token TEXT NOT NULL, + settled INTEGER NOT NULL CHECK(settled IN (0,1)), + PRIMARY KEY(workspace_id,runtime_id,worker_id,request_id), + FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE + ); + CREATE UNIQUE INDEX worker_restore_intent_pending_domain ON worker_restore_intents(workspace_id,runtime_id,worker_id,domain_key) WHERE settled=0;")?; + tx.execute( + "INSERT INTO __yoi_schema_migrations(version,name) VALUES(?1,?2)", + params![80_i64, WORKER_RESTORE_INTENTS_MIGRATION_NAME], + )?; + tx.commit()?; + Ok(()) +} + fn prepare_connection(conn: &Connection) -> Result<()> { configure_sqlite(conn)?; let workspace_schema_version = current_schema_version(conn)?; @@ -15094,6 +15162,15 @@ fn migrate_workdir_connection_id_v77_to_v78(conn: &Connection) -> Result<()> { } fn create_latest_workspace_schema(conn: &Connection) -> Result<()> { + conn.execute_batch("CREATE TABLE worker_restore_intents ( + workspace_id TEXT NOT NULL, runtime_id TEXT NOT NULL, worker_id TEXT NOT NULL, + domain_key TEXT NOT NULL, request_id TEXT NOT NULL, expected_token TEXT NOT NULL, + settled INTEGER NOT NULL CHECK(settled IN (0,1)), + PRIMARY KEY(workspace_id,runtime_id,worker_id,request_id), + FOREIGN KEY(workspace_id) REFERENCES workspaces(workspace_id) ON DELETE CASCADE + ); + CREATE UNIQUE INDEX worker_restore_intent_pending_domain ON worker_restore_intents(workspace_id,runtime_id,worker_id,domain_key) WHERE settled=0;")?; + conn.execute_batch("DROP TABLE IF EXISTS typed_ticket_targets;")?; conn.execute_batch(include_str!("latest_schema.sql"))?; conn.execute_batch( @@ -16202,6 +16279,7 @@ mod tests { } fn downgrade_schema_66_ticket_and_workdir_authority(conn: &Connection) -> Result<()> { + conn.execute_batch("DROP TABLE IF EXISTS worker_restore_intents;")?; conn.execute_batch( "DROP TABLE backend_job_deliveries; DROP TABLE backend_job_attempts; @@ -17709,6 +17787,10 @@ mod tests { version: 79, name: WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME.to_string() }, + WorkspaceSchemaMigrationStep { + version: 80, + name: WORKER_RESTORE_INTENTS_MIGRATION_NAME.to_string() + }, ] ); @@ -17798,6 +17880,7 @@ mod tests { ), (78, WORKDIR_CONNECTION_ID_MIGRATION_NAME.to_string()), (79, WORKSPACE_CONFIG_GRANTS_MIGRATION_NAME.to_string()), + (80, WORKER_RESTORE_INTENTS_MIGRATION_NAME.to_string()), ] ); assert!(!table_exists(conn, "trusted_runtime_records")?); @@ -18149,7 +18232,7 @@ mod tests { .collect::>(), vec![ 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, - 73, 74, 75, 76, 77, 78, 79 + 73, 74, 75, 76, 77, 78, 79, 80 ] ); SqliteWorkspaceStore::migrate_database(&path).unwrap(); @@ -18158,7 +18241,7 @@ mod tests { current_schema_version(&conn).unwrap(), LATEST_SCHEMA_VERSION ); - assert_eq!(workspace_schema_migration_history(&conn).unwrap().len(), 30); + assert_eq!(workspace_schema_migration_history(&conn).unwrap().len(), 31); assert!(column_exists(&conn, "worker_workdir_links", "alias").unwrap()); assert!(column_exists(&conn, "worker_workdir_links", "capabilities").unwrap()); assert!(!column_exists(&conn, "worker_workdir_links", "role").unwrap()); @@ -21860,6 +21943,7 @@ INSERT INTO worker_registry ( for record in [&runtime_a, &runtime_b] { let observed = SubscriptionWorker { + restore_observation_token: None, worker_id: SubscriptionWorkerId::new("shared-local-id").unwrap(), runtime_id: Some(record.worker.runtime_id.clone()), resource_key: None, @@ -21996,6 +22080,7 @@ INSERT INTO worker_registry ( Some("W-1") ); let observed = SubscriptionWorker { + restore_observation_token: None, worker_id: SubscriptionWorkerId::new("known").unwrap(), runtime_id: Some("embedded".to_string()), resource_key: None, @@ -22016,6 +22101,7 @@ INSERT INTO worker_registry ( }; store.upsert_worker_registry(&other_catalog).unwrap(); let other_observed = SubscriptionWorker { + restore_observation_token: None, worker_id: SubscriptionWorkerId::new("other").unwrap(), runtime_id: Some("runtime-other".to_string()), display_name: Some("Other Runtime Worker".to_string()), @@ -22025,6 +22111,7 @@ INSERT INTO worker_registry ( .apply_worker_registry_observation("local-dev", "runtime-other", &other_observed, "2") .unwrap(); let orphan = SubscriptionWorker { + restore_observation_token: None, worker_id: SubscriptionWorkerId::new("orphan").unwrap(), ..observed.clone() }; @@ -22190,6 +22277,7 @@ INSERT INTO worker_registry ( .is_none() ); let event = SubscriptionWorker { + restore_observation_token: None, worker_id: SubscriptionWorkerId::new("removed").unwrap(), runtime_id: Some("embedded".to_string()), resource_key: None, diff --git a/crates/workspace-server/src/workspace_deletion.rs b/crates/workspace-server/src/workspace_deletion.rs index 484a7f048..b5d997dcf 100644 --- a/crates/workspace-server/src/workspace_deletion.rs +++ b/crates/workspace-server/src/workspace_deletion.rs @@ -76,6 +76,7 @@ const WORKSPACE_DELETION_PURGE_TABLES: &[&str] = &[ "worker_registry_projection_diagnostics", "worker_registry_projection_removals", "worker_removal_operations", + "worker_restore_intents", "worker_retention_audit_events", "worker_session_archive_observe_grants", "worker_session_archives", diff --git a/crates/workspace-server/src/workspace_subscription.rs b/crates/workspace-server/src/workspace_subscription.rs index 058d8bc99..9e7698a30 100644 --- a/crates/workspace-server/src/workspace_subscription.rs +++ b/crates/workspace-server/src/workspace_subscription.rs @@ -425,6 +425,7 @@ fn project_registry_worker( worker } else { SubscriptionWorker { + restore_observation_token: None, worker_id: protocol::subscription::SubscriptionWorkerId::new( record.registry.worker.worker_id.clone(), ) @@ -600,6 +601,7 @@ mod tests { #[test] fn external_workdir_attachment_does_not_remove_workspace_worker() { let mut worker = SubscriptionWorker { + restore_observation_token: None, worker_id: worker_id(), runtime_id: Some("runtime-1".to_string()), resource_key: Some("W-1".to_string()), diff --git a/docs/architecture/worker-restore-observations.md b/docs/architecture/worker-restore-observations.md new file mode 100644 index 000000000..eab92478a --- /dev/null +++ b/docs/architecture/worker-restore-observations.md @@ -0,0 +1,70 @@ +# Guarded Worker Restore and Web observations + +T-701 extends the [registry-bound operation reception](workspace-worker-operations.md), not the Worker creation, Resume or Restore state machines. + +## Different tokens protect different things + +| Operation | Observed identity / expected input | Atomic validation and execution owner | Conflict / uncertain outcome | Shared client behavior / deliberate difference | +| --- | --- | --- | --- | --- | +| Restore | Workspace + Runtime + Worker identity; Runtime-owned `restore_observation_token`; immutable `expected_observation_token` + `request_id` | Existing Runtime per-Worker lifecycle operation lock, durable admission and lifecycle commit. Preparation and execution follow admission, not a frontend GET comparison | Stale new intent is HTTP 409 (`restore_observation_conflict`); operation results remain accepted / rejected / rolled_back / reconciliation_required. A lost reply is unknown, not definite rollback | Both entrances use the same scoped POST/parser/notice classifiers and observation refresh. Never replace a rejected intent's token and automatically retry | +| Stop | Identity and reason, **not** a catalog revision CAS | Runtime per-Worker operation lock and existing backend Stop settlement; Workspace session finalization only after accepted | Existing flat accepted / unsupported / rejected result with diagnostics | Same encoded target paths and bounded HTTP-error messages; refresh observations instead of writing a guessed local stopped state. Stop does not gain Restore's generation guarantee | +| Pause / Resume / Cancel / Compact | Connected execution identity and `WorkerCommandEnvelope.command_id` | Existing bound execution/controller command admission | Command acknowledgement: Accepted / StaleCommandId / Conflict / InvalidState, with authoritative state | Command IDs order commands; they are not catalog observation generations. Resume is not Restore; connections do not silently move to a replacement execution | +| Delete / cleanup | Runtime + selected target IDs; `expected_plan_revision` **and** `expected_plan_digest` | Existing cleanup validation, per-target live/retention revalidation, removal service and catalog commits | Existing cleanup-plan conflicts, per-target status and diagnostics | Preserve plan protection for candidate sets and dependencies. Never substitute a Worker generation. Page plans invalidate at the beginning of every catalog refresh and are unavailable until fresh catalog and plan observations settle | +| Pin / Unpin | Registry Worker identity and explicit boolean | Registry retention update and ordered projection publication | Existing retention result/errors | Preserve retention semantics and full catalog metadata; refresh shared observations and cleanup plan | +| Submit / Notify | Authenticated source + durable submission / notification request ID | Existing controller/session durable receipt admission | Existing accepted/rejected receipts and unknown delivery diagnostics | These are not Restore requests; Restore never manufactures input, Resume or a new Worker | + +There is no universal Worker command token or shared frontend action lock. Workspace reception resolves a stable target and rechecks caller authorization; Runtime/controller/store owners retain operation-specific atomic boundaries. + +## Browser request and result + +Normal explicit Restore is enabled only for `availability = observed`, catalog `stopped`, and an available backend observation token. Sidebar display state and foreground `worker_state` are not eligibility evidence: sidebar uses `lifecycleState`; the page uses the full catalog's `state`. Neither UI guesses snapshot, singleton, Workdir provider or authorization eligibility. + +Both entrances send the generated two-field `WorkerRestoreRequest` to: + +```text +POST /api/w/{workspace_id}/runtimes/{runtime_id}/workers/{worker_id}/restore +``` + +Each ID is independently encoded. No Ticket-assignment query, Submit, Resume or Worker-create call is used. The bounded parser validates wrapper identity, the four states, diagnostics and optional **Workspace WorkerSummary**. A Stop lifecycle result or launch summary is not a Restore response. Unknown states/fields/invalid identities cannot produce a success notice. + +Accepted means the existing Restore operation committed, not that the Worker is necessarily Running or a later run succeeded. A returned summary validates result identity; display is updated from fresh shared observations, not an old POST snapshot. Rejected and rolled_back display diagnostics without success. Reconciliation_required and transport/malformed-response uncertainty retain the original immutable request for an explicit **Retry Restore** action. That action sends the same request; it does not create an operation, exchange its observation token, or automatically loop. A conflict clears this local recovery intent and asks the user to review fresh observations before a new explicit operation. A GET alone does not settle Runtime's pending lifecycle operation. + +## Backend admission, preparation and recovery + +The Workspace `WorkspaceWorker` reception still resolves target identity and rechecks caller authority. Its Restore coordinator first dispatches the public tuple to Runtime without candidate preparation. A terminal owner returns its historical receipt; an admitted pending owner supplies its stored preparation before renewed singleton/Workdir/External eligibility is evaluated. Historical result summaries are read from the current Runtime and are not written back as catalog observations. + +For a new intent, Workspace builds read-only preparation references after the normal eligibility checks. Runtime atomically checks the observation token again under the existing per-Worker lock and persists the owner together with `pending_restore` and its existing lifecycle operation ID. Only this admitted operation can proceed to SSH resource issuance, binding/access preparation and execution. A race between the first observation check and admission therefore cannot issue credentials or launch a new Restore from the old intent. + +Preparation completion is attached to that same pending operation, not a replacement operation. Retained repository-access snapshots and exact request identity allow interrupted preparation to resume without exchanging candidate bindings. Runtime retries use the original owner's payload; a freshly generated preparation cannot rewrite the admitted target or expected observation. Stop and startup reconciliation continue to use the existing pending lifecycle ownership. When startup/legacy persistence has a pending lifecycle operation but no public receipt owner, an explicit request with the current observation may bind recovery to that exact operation ID, without applying candidate preparation. Once bound, a different request cannot take over; terminal replay does not resurrect a later stopped Worker. + +| Caller | Intent identity retained / validation boundary | +| --- | --- | +| Web sidebar and page | Immutable two-field request retained locally for explicit uncertain-result recovery; no shared in-flight lock | +| Workspace alias, scoped and Worker-control HTTP | Mandatory public request; common target/auth reception and the same Runtime admission | +| Embedded / remote Runtime providers and management HTTP | Typed Runtime request and coordination transport, including original observation and lifecycle operation owner | +| `WorkerRestore` Tool | Observed token in Tool arguments and durable Tool-call-derived request identity | +| Existing TUI Restore/resume path | Selected row's observation token and retained request; no new TUI entrance | +| Internal Coder, Orchestrator, subject reuse and compensation | Domain intent journal pins the request/token before dispatch and retains it through unknown result and restart; it does not own lifecycle status or bypass Runtime validation | +| Runtime startup / pending reconciliation | Private recovery of the existing persisted Restore owner and operation; not an external unguarded new-intent endpoint | + +The Workspace internal intent journal stores client retry identity only. Runtime remains the lifecycle authority; it is not a second Restore state machine, Worker ledger or cross-screen lock. + +## Observation and cleanup-plan ordering + +`workspaceWorkersStore` retains the existing sidebar stream/projection and also fetches the full Workspace Worker catalog for the page. Full catalog metadata preserves retention, attachments, implementation and targets that are not sidebar entries. Stream updates overlay lifecycle/availability/foreground snapshot/token without interpreting absence as a new page exclusion. + +Every stream update or explicit refresh increments a local **observation scheduling version** and publishes `catalogRefreshing` before fetching. This local version fences asynchronous display work; it is **not** Restore admission authority and is never sent to Runtime. A newer event/GET supersedes older GET completion. Old POST summaries and page-loader data cannot overwrite the active catalog. On disposal, requests and subscription callbacks are fenced. Catalog fetch errors have their own stable alert episode rather than clearing transport/Workdir alerts. + +The page invalidates cleanup plans immediately when observations refresh. It does not fetch or install plans while `catalogRefreshing` is true. Successful refresh completion triggers plan reads even when the resulting catalog contents did not change. Workspace/lifetime and cleanup epochs reject late previous plan responses. A failed refresh leaves old candidates unavailable instead of treating old page data as fresh. + +Both components retain local busy state only. Requests pin their starting Workspace/target, and late replies after Workspace switch/disposal cannot update another Workspace's notices, list or busy state. Sidebar and page may issue competing requests: atomic Runtime admission—not a cross-component client lock—decides their outcomes. Console links and existing sidebar filtering remain unchanged. + +## Validation surfaces + +- Runtime admission tests prove stale/ABA and serialization, durable same-request replay and pending reconciliation. +- Runtime management, embedded/remote providers, Workspace aliases/scoped/control routes, Tool and existing TUI callers exercise the guarded API rather than a legacy unguarded Restore. +- `tests/runtime-workers.test.ts` and `test/sidebar/worker-actions.test.ts` prove response/request and notice contracts; the Runtime parser tests are included in the normal Web test task. +- Rendered `worker-stop.browser.spec.ts` and `workers-page.browser.spec.ts` cover menu/action eligibility, busy behavior, four outcomes and errors, explicit retry, Workspace/disposal fences, ordinary Console/Pin/Delete, sidebar-origin updates and cleanup epochs. +- `worker-restore-store.browser.spec.ts` tests the real store's full metadata, GET/event ordering, dependent-plan invalidation, no POST recovery loop, catalog-only target preservation and disposal. + +Exact executed validation counts and immutable source/review evidence belong to the repository Merge Request and final approved Ticket handoff. diff --git a/docs/report/2026-10-06-t701-validation.md b/docs/report/2026-10-06-t701-validation.md new file mode 100644 index 000000000..462af3db3 --- /dev/null +++ b/docs/report/2026-10-06-t701-validation.md @@ -0,0 +1,49 @@ +# T-701 validation ledger + +This records concrete intermediate failures rather than treating later passes as erasing them. Final source and review evidence are on the repository Merge Request. + +## Validation approach + +Start with Restore/parser/UI/serialization boundary filters, then the eight changed crates and the normal root `cargo check` compile closure. Cargo jobs are bounded to three; full crate test concurrency is bounded to four. No all-workspace test, `--all-targets`, Nix/Docker build, deployment, process restart or live Worker repair is part of this validation. + +The Web production bundle, `svelte-check`, normal Deno tests and actual rendered component suites have passed. Those results alone do not prove Backend admission, replay or preparation recovery. Full Rust suites exercise callers, schema migration and dependent inventories that narrow guard filters cannot prove. + +## Intermediate failures and disposition + +1. Early root checks failed because the Runtime request/preparation DTO integration was not yet present. This was an incomplete implementation, not a baseline defect. A subsequent root check passed after the integration. +2. Early Server API full tests reported three stale generated artifacts after the OpenAPI contract changed. Regenerating the affected closure artifacts resolved them; the full default suite passed 70 tests. TypeScript-feature tests were also executed after regeneration. +3. Adding full catalog GETs exposed existing browser fixtures that returned Workdir-only shapes to every endpoint. The fixtures now return the exact `/workers` contract, and the Workdir-error fixture fails only its intended endpoint. Catalog and transport/Workdir failures have separate alert identities. +4. Parent full `cargo test -p yoi-workspace-server -- --test-threads=4` produced **614 passed / 4 failed** (618 library tests): + - `hosts::tests::embedded_runtime_registers_routes_input_without_internal_leaks`: a broad forbidden-word assertion rejected the newly public observation field's word `token`. The private credential/token non-leak contract must be preserved while allowing the explicitly public observation token. + - `server::tests::embedded_runtime_api_routes_by_runtime_and_worker_ids_without_leaking_internals`: an old `{}` Restore fixture received 422 because guarded Restore now requires the observed token and request ID. Update the real route fixture, do not restore an unguarded compatibility path. + - `workspace_deletion::tests::explicit_purge_inventory_covers_every_workspace_scoped_table`: the new internal Restore intent journal was absent from the explicit purge inventory. Integrate its Workspace deletion lifecycle; do not weaken the exhaustive inventory check. + - `server::tests::ticket_checker_suppresses_stale_and_clean_results_and_diagnoses_invalid_output`: the inherited stale-checker `execution.take_inputs().is_empty()` assertion failed. Earlier T-702 evidence reproduced this named instability on an isolated original baseline; see [T-702 validation report](2026-10-06-t702-stale-checker-validation.md). T-701 has not fixed that checker, and this failure is not conflated with the three new integration regressions. + +## Follow-up disposition and final pre-publication results + +The projection fixtures now permit only the explicit `restore_observation_token` key and validate its UUIDv7 value; private token/credential fields remain forbidden. The HTTP fixture sends the real observed token and request ID. The internal intent table is in the exhaustive Workspace deletion inventory. + +The final restart recovery test exposed one obsolete Runtime expectation: an unowned startup/legacy pending operation must be explicitly recoverable, not permanently conflict. Public recovery now binds a receipt to the exact existing lifecycle operation ID, without candidate preparation or a replacement execution. Tests assert a competing request conflicts, recovery preserves the ID, and terminal replay after Stop never launches again. A Tool recovery fixture also used path-shaped IDs rejected by the existing authority-ID validator; it now uses valid authority IDs and proves the two explicit calls send an identical request body. + +After those fixes, a full Server run reported **618 passed / 1 failed**, solely the inherited stale-checker assertion above. Its isolated test subsequently passed. An additional whole-crate run with the same four-thread configuration and **no skips** passed **619 library + 11 binary tests**. This does not erase the earlier failure or claim to fix the checker race. + +| Executed command | Final pre-publication evidence | +| --- | --- | +| root `cargo check` | Passed normal default-member compile closure, including TUI and Server | +| `cargo test -p worker-runtime -- --test-threads=4` | 332 library + 34 binary tests passed | +| `cargo test -p yoi-workspace-server -- --test-threads=4` | 619 library + 11 binary tests passed in the additional no-skip run; previous flaky failure retained above | +| `cargo test -p worker -- --test-threads=4` | 804 library + 109 integration tests passed | +| `cargo test -p runtime-api` | 9 tests passed | +| `cargo test -p server-api` / `--features typescript` | 70 / 77 tests passed, including generated artifact freshness | +| `cargo test -p protocol` / `--features typescript` | 78 / 83 tests passed | +| `cargo test -p client` | 52 library + 2 integration tests passed | +| `cargo test -p tui` | 331 tests passed | +| Web `deno task check` | 0 errors, 0 warnings | +| Web `deno task test` | 504 tests passed | +| Web `deno task test:component` | 249 tests in 25 files passed, including the real paired sidebar/page/store suite | +| Web `deno task build` | Production build passed; existing chunk-size warning remains | +| `cargo fmt --all -- --check`, `git diff --check HEAD` | Passed | + +Additional exploratory Runtime feature checks did **not** pass: `--no-default-features` test/check cannot resolve existing unconditional `workspace_request` / `reqwest` imports; the test target additionally references disabled `fs_store`. HTTP-only check also lacks the existing retention and futures dependencies. Inspection of original HEAD confirms those imports/gates already exist; no isolated baseline build was performed for these configurations. These are recorded as failed exploratory checks, not semantic passes or newly supported configurations. The default Runtime configuration, which contains fs-store and ws-server, is exercised in full above. This Ticket does not redesign unrelated Runtime feature gating. + +Raw session validation logs are retained under ignored `target/t701-validation/`, not shipped as product source. Immutable published source, independent verdict and any subsequent fix/review evidence belong to the explicit repository Merge Request. diff --git a/openapi/server-api.json b/openapi/server-api.json index ed025eea0..c217d3c0f 100644 --- a/openapi/server-api.json +++ b/openapi/server-api.json @@ -8062,6 +8062,13 @@ "resource_key": { "type": "string" }, + "restore_observation_token": { + "description": "Opaque Runtime-owned observation fence used by explicit Restore intent.", + "type": [ + "string", + "null" + ] + }, "retention_state": { "default": "", "type": "string" @@ -12953,6 +12960,23 @@ ], "type": "string" }, + "WorkerRestoreRequest": { + "additionalProperties": false, + "description": "One explicit Restore intent. Retries retain both fields; conflicts require a\nfresh observation and a deliberate new intent, not automatic token replacement.", + "properties": { + "expected_observation_token": { + "type": "string" + }, + "request_id": { + "type": "string" + } + }, + "required": [ + "expected_observation_token", + "request_id" + ], + "type": "object" + }, "WorkerRestoreResponse": { "additionalProperties": false, "properties": { @@ -13157,6 +13181,13 @@ "resource_key": { "type": "string" }, + "restore_observation_token": { + "description": "Opaque Runtime-owned observation fence used by explicit Restore intent.", + "type": [ + "string", + "null" + ] + }, "retention_state": { "default": "", "type": "string" @@ -15443,7 +15474,7 @@ "info": { "title": "Yoi Server API", "version": "0.1.0", - "x-yoi-source-digest": "sha256:6001e9f7273288195fb5a9859e4a5932a521d2f2e7c3a9ecd8ec660fd6a37779" + "x-yoi-source-digest": "sha256:fd06f382c6d33a5c3cc98ef9e264545923c53fa95a9be555a85680f98c6065da" }, "openapi": "3.1.0", "paths": { @@ -18016,6 +18047,16 @@ "style": "simple" } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkerRestoreRequest" + } + } + }, + "required": true + }, "responses": { "200": { "content": { @@ -26433,6 +26474,16 @@ "style": "form" } ], + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/WorkerRestoreRequest" + } + } + }, + "required": true + }, "responses": { "200": { "content": { diff --git a/web/workspace/deno.json b/web/workspace/deno.json index a2c791543..9479ef44f 100644 --- a/web/workspace/deno.json +++ b/web/workspace/deno.json @@ -6,7 +6,7 @@ "dev": "deno run -A npm:vite@8.3.2 dev", "dev:backend": "cd ../.. && cargo run -p yoi-workspace-server --bin yoi-server -- serve --listen 127.0.0.1:8787", "check": "deno run -A npm:@sveltejs/kit@3.0.0 sync && deno run -A npm:svelte-check@4.7.6 --tsconfig ./tsconfig.json", - "test": "deno test --allow-read=src,test,tests,../../docs --allow-env=LOG,VSCODE_TEXTMATE_DEBUG,NODE_ENV tests/workspace-model.test.ts tests/repository-api.test.ts tests/workspace-catalog.test.ts tests/profile-api.test.ts tests/skill-api.test.ts tests/tooltip-contract.test.ts tests/bevel-contract.test.ts tests/bevel-line-contract.test.ts tests/typography-contract.test.ts src/lib/workspace/auth/model.test.ts src/lib/workspace/protocol-validator.test.ts tests/auth-api.test.ts src/lib/workspace/api/http.test.ts src/lib/workspace/api/workers.test.ts src/lib/workspace/api/ticket-browser.test.ts src/lib/workspace/header/breadcrumb-model.test.ts src/lib/workspace/console/chat-submit.test.ts src/lib/workspace/console/composer-delivery.test.ts test/composer-history.test.ts tests/composer-paste.test.ts src/lib/workspace/console/composer-command.test.ts src/lib/workspace/console/composer-draft.test.ts src/lib/workspace/console/composer-completion.test.ts src/lib/workspace/console/composer-invocation.test.ts src/lib/workspace/console/file-completions.test.ts test/console/ansi.test.ts src/lib/workspace/console/history.test.ts src/lib/workspace/console/model.test.ts src/lib/workspace/console/turn-navigation.test.ts src/lib/workspace/console/run-status.test.ts src/lib/workspace/companion/api.test.ts tests/workdir-api.test.ts tests/workdir-permissions.test.ts src/lib/workspace/console/tasks.test.ts test/ticket-detail-route-reuse.test.ts test/spa-shell.test.ts test/objective-pages.test.ts test/repositories/ui.test.ts src/lib/workspace/console/worker-console.ui.test.ts src/lib/workspace/settings/model.test.ts tests/runtime-connection.test.ts tests/runtime-management.test.ts tests/runtime-management-source.test.ts src/lib/workspace/sidebar/override-stack.test.ts src/lib/workspace/sidebar/route-ownership.test.ts src/lib/workspace/sidebar/workers.test.ts src/lib/workspace/sidebar/workspace-switcher.test.ts src/lib/workspace/sidebar/worker-subscription.test.ts src/lib/workspace/sidebar/worker-workdir-meta.test.ts src/lib/workspace/sidebar/worker-launch.test.ts test/sidebar/worker-actions.test.ts src/lib/workspace/tickets/merge-request-resources.test.ts src/lib/workspace/tickets/ticket-panel.test.ts test/merge-request-status.test.ts test/config-source/decodal-grammar.test.ts test/config-source/editor-state.test.ts test/config-source/fixed-schema-wrapper.test.ts test/config-source/toolchain.test.ts test/config-source/wasm-parity.test.ts test/config-source/api.test.ts tests/memory-api.test.ts test/repository-access/api.test.ts test/repository-access/loader.test.ts test/repository-access/model.test.ts test/repository-access/structure.test.ts test/repository-ssh-connection-ui.test.ts", + "test": "deno test --allow-read=src,test,tests,../../docs --allow-env=LOG,VSCODE_TEXTMATE_DEBUG,NODE_ENV tests/workspace-model.test.ts tests/repository-api.test.ts tests/workspace-catalog.test.ts tests/profile-api.test.ts tests/skill-api.test.ts tests/tooltip-contract.test.ts tests/bevel-contract.test.ts tests/bevel-line-contract.test.ts tests/typography-contract.test.ts src/lib/workspace/auth/model.test.ts src/lib/workspace/protocol-validator.test.ts tests/auth-api.test.ts src/lib/workspace/api/http.test.ts src/lib/workspace/api/workers.test.ts src/lib/workspace/api/ticket-browser.test.ts src/lib/workspace/header/breadcrumb-model.test.ts src/lib/workspace/console/chat-submit.test.ts src/lib/workspace/console/composer-delivery.test.ts test/composer-history.test.ts tests/composer-paste.test.ts src/lib/workspace/console/composer-command.test.ts src/lib/workspace/console/composer-draft.test.ts src/lib/workspace/console/composer-completion.test.ts src/lib/workspace/console/composer-invocation.test.ts src/lib/workspace/console/file-completions.test.ts test/console/ansi.test.ts src/lib/workspace/console/history.test.ts src/lib/workspace/console/model.test.ts src/lib/workspace/console/turn-navigation.test.ts src/lib/workspace/console/run-status.test.ts src/lib/workspace/companion/api.test.ts tests/workdir-api.test.ts tests/workdir-permissions.test.ts src/lib/workspace/console/tasks.test.ts test/ticket-detail-route-reuse.test.ts test/spa-shell.test.ts test/objective-pages.test.ts test/repositories/ui.test.ts src/lib/workspace/console/worker-console.ui.test.ts src/lib/workspace/settings/model.test.ts tests/runtime-connection.test.ts tests/runtime-management.test.ts tests/runtime-management-source.test.ts src/lib/workspace/sidebar/override-stack.test.ts src/lib/workspace/sidebar/route-ownership.test.ts src/lib/workspace/sidebar/workers.test.ts src/lib/workspace/sidebar/workspace-switcher.test.ts src/lib/workspace/sidebar/worker-subscription.test.ts src/lib/workspace/sidebar/worker-workdir-meta.test.ts src/lib/workspace/sidebar/worker-launch.test.ts tests/runtime-workers.test.ts test/sidebar/worker-actions.test.ts src/lib/workspace/tickets/merge-request-resources.test.ts src/lib/workspace/tickets/ticket-panel.test.ts test/merge-request-status.test.ts test/config-source/decodal-grammar.test.ts test/config-source/editor-state.test.ts test/config-source/fixed-schema-wrapper.test.ts test/config-source/toolchain.test.ts test/config-source/wasm-parity.test.ts test/config-source/api.test.ts tests/memory-api.test.ts test/repository-access/api.test.ts test/repository-access/loader.test.ts test/repository-access/model.test.ts test/repository-access/structure.test.ts test/repository-ssh-connection-ui.test.ts", "test:component": "deno run -A npm:vitest@5.0.3 run --config vitest.config.ts", "build": "deno run -A npm:vite@8.3.2 build", "preview": "deno run -A npm:vite@8.3.2 preview", diff --git a/web/workspace/src/lib/generated/companion-api.ts b/web/workspace/src/lib/generated/companion-api.ts index 9b9093be6..8a530c53d 100644 --- a/web/workspace/src/lib/generated/companion-api.ts +++ b/web/workspace/src/lib/generated/companion-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-companion-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:6001e9f7273288195fb5a9859e4a5932a521d2f2e7c3a9ecd8ec660fd6a37779) +// Input contract: 0.1.0 (sha256:fd06f382c6d33a5c3cc98ef9e264545923c53fa95a9be555a85680f98c6065da) // Options: companion-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Output: web/workspace/src/lib/generated/companion-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_companion_api_types -- --check diff --git a/web/workspace/src/lib/generated/protocol-validator.ts b/web/workspace/src/lib/generated/protocol-validator.ts index 824f113ec..1f467d468 100644 --- a/web/workspace/src/lib/generated/protocol-validator.ts +++ b/web/workspace/src/lib/generated/protocol-validator.ts @@ -21,9 +21,9 @@ const MAX_CORRELATION_ID_BYTES = 128; const MAX_RESOURCE_ID_BYTES = 256; const MAX_WORKER_IDS_PER_SELECTOR = 256; const MAX_REJECTION_MESSAGE_BYTES = 1024; -const subscriptionFrameSchema: unknown = {"$defs":{"Alert":{"additionalProperties":false,"description":"User-facing alert emitted from the Worker layer.\n\nThis is a separate channel from `tracing` (developer logs): entries\nhere are assembled explicitly by the Worker when a condition should be\nsurfaced to the person driving the client. Keep messages short and\nhuman-readable.","properties":{"level":{"$ref":"#/$defs/AlertLevel"},"message":{"type":"string"},"source":{"$ref":"#/$defs/AlertSource"},"timestamp_ms":{"description":"Milliseconds since the Unix epoch.","format":"int64","maximum":9007199254740991.0,"minimum":-9007199254740991.0,"type":"integer"}},"required":["level","source","message","timestamp_ms"],"type":"object"},"AlertLevel":{"enum":["warn","error"],"type":"string"},"AlertSource":{"enum":["worker","engine","compactor","agents_md"],"type":"string"},"CommandEvent":{"oneOf":[{"additionalProperties":false,"properties":{"command_id":{"type":"string"},"kind":{"const":"started","type":"string"},"observed_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"tool_call_id":{"type":["string","null"]}},"required":["kind","command_id","observed_at_ms"],"type":"object"},{"additionalProperties":false,"properties":{"command_id":{"type":"string"},"content":{"type":"string"},"end_offset":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"kind":{"const":"output","type":"string"},"observed_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"start_offset":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"stream":{"$ref":"#/$defs/CommandStream"}},"required":["kind","command_id","stream","start_offset","end_offset","content","observed_at_ms"],"type":"object"},{"additionalProperties":false,"properties":{"command_id":{"type":"string"},"exit_code":{"format":"int32","maximum":2147483647.0,"minimum":-2147483648.0,"type":["integer","null"]},"kind":{"const":"terminal","type":"string"},"observed_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"status":{"$ref":"#/$defs/CommandStatus"},"stderr_end_offset":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"stdout_end_offset":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","command_id","status","stdout_end_offset","stderr_end_offset","observed_at_ms"],"type":"object"}]},"CommandSnapshot":{"additionalProperties":false,"properties":{"command_id":{"type":"string"},"exit_code":{"format":"int32","maximum":2147483647.0,"minimum":-2147483648.0,"type":["integer","null"]},"last_output_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":["integer","null"]},"observed_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"started_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"status":{"$ref":"#/$defs/CommandStatus"},"stderr":{"$ref":"#/$defs/CommandStreamSlice"},"stdout":{"$ref":"#/$defs/CommandStreamSlice"},"tool_call_id":{"type":["string","null"]}},"required":["command_id","status","started_at_ms","observed_at_ms","stdout","stderr"],"type":"object"},"CommandStatus":{"description":"Live provider-owned command status. These values are operational Console\nstate only and are never appended to Worker history.","enum":["running","completed","failed","timed_out","cancelled"],"type":"string"},"CommandStream":{"enum":["stdout","stderr"],"type":"string"},"CommandStreamSlice":{"additionalProperties":false,"properties":{"content":{"type":"string"},"end_offset":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"start_offset":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"truncated":{"type":"boolean"}},"required":["start_offset","end_offset","content","truncated"],"type":"object"},"CompactionLifecycle":{"additionalProperties":false,"description":"Stable parent-owned lifecycle for one compaction run.","properties":{"compaction_id":{"type":"string"},"ended_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":["integer","null"]},"error":{"type":["string","null"]},"internal_worker":{"anyOf":[{"$ref":"#/$defs/InternalWorkerRef"},{"type":"null"}]},"new_segment_id":{"type":["string","null"]},"revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"schema_version":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"started_at_ms":{"description":"Milliseconds since the Unix epoch.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"state":{"$ref":"#/$defs/CompactionLifecycleState"},"summary":{"type":["string","null"]}},"required":["schema_version","compaction_id","revision","state","started_at_ms"],"type":"object"},"CompactionLifecycleState":{"enum":["running","done","failed","interrupted"],"type":"string"},"CompactionPhase":{"enum":["preparing","summarizing","committing"],"type":"string"},"CompactionTrigger":{"enum":["manual","pre_run","request_threshold"],"type":"string"},"CompletionContext":{"additionalProperties":false,"properties":{"argument":{"type":["string","null"]},"invocation":{"$ref":"#/$defs/FeatureInvocationIdentity"}},"required":["invocation"],"type":"object"},"CompletionEntry":{"additionalProperties":false,"description":"One completion candidate for a prefix query.","properties":{"description":{"type":["string","null"]},"invocation":{"anyOf":[{"$ref":"#/$defs/FeatureInvocationDescriptor"},{"type":"null"}]},"is_dir":{"default":false,"type":"boolean"},"usage":{"type":["string","null"]},"value":{"type":"string"}},"required":["value","is_dir"],"type":"object"},"CompletionKind":{"description":"Kind of completion requested by `Method::ListCompletions`.\n\n`File` is the existing `@` path lane. `Feature` lists installed chat\ninvocations. `FeatureArgument` resolves a declared argument using the\nsource-qualified [`CompletionContext`].","enum":["file","feature","feature_argument"],"type":"string"},"ContextTokenSource":{"description":"Provenance of the current-context token count in a [`Greeting`].","oneOf":[{"const":"measured","description":"Provider-reported input occupancy from the most recent LLM request.","type":"string"},{"const":"estimated","description":"Current history projected from the Worker's token accounting records.","type":"string"}]},"ContextUsage":{"additionalProperties":false,"description":"Current prompt-context occupancy, distinct from cumulative run traffic.","properties":{"source":{"$ref":"#/$defs/ContextTokenSource"},"tokens":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["tokens","source"],"type":"object"},"ErrorCode":{"enum":["already_running","not_running","not_paused","provider_error","tool_error","invalid_request","internal"],"type":"string"},"Event":{"oneOf":[{"additionalProperties":false,"description":"Durable Submit acceptance. A `Queued` receipt follows the durable FIFO\ncheckpoint. A `Started` receipt follows the pending activation checkpoint;\nthe corresponding [`Event::UserMessage`] marks the later history commit.\nRepeating the same request id and exact payload returns the same receipt\nwithout appending or activating twice.","properties":{"data":{"additionalProperties":false,"properties":{"disposition":{"$ref":"#/$defs/SubmissionDisposition"},"submission_id":{"type":"string"},"submission_request_id":{"type":"string"}},"required":["submission_request_id","submission_id","disposition"],"type":"object"},"event":{"const":"submission_accepted","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Correlated rejection before durable acceptance.","properties":{"data":{"additionalProperties":false,"properties":{"message":{"type":"string"},"submission_request_id":{"type":"string"}},"required":["submission_request_id","message"],"type":"object"},"event":{"const":"submission_rejected","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Durable notification acceptance. The request identity is persisted with\nthe notification receipt before this event is emitted. Repeating the\nsame request id and exact payload returns the same acknowledgement\nwithout appending the notification twice.","properties":{"data":{"additionalProperties":false,"properties":{"notification_request_id":{"type":"string"}},"required":["notification_request_id"],"type":"object"},"event":{"const":"notification_accepted","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Correlated notification rejection before durable acceptance.","properties":{"data":{"additionalProperties":false,"properties":{"message":{"type":"string"},"notification_request_id":{"type":"string"}},"required":["notification_request_id","message"],"type":"object"},"event":{"const":"notification_rejected","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Revisioned FIFO replacement following enqueue, activation, cancel, or clear.","properties":{"data":{"additionalProperties":false,"properties":{"pending":{"$ref":"#/$defs/PendingSubmissionsSnapshot"}},"required":["pending"],"type":"object"},"event":{"const":"pending_submissions_changed","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A user input message persisted as `LogEntry::AnnotatedUserInput` and\nactivated for a turn. Broadcast to every subscribed client so TUI / GUI\ninstances show the same user line that reconnect snapshots replay.","properties":{"data":{"additionalProperties":false,"properties":{"entry_id":{"description":"Stable durable history identity. Older mixed-version producers omit it.","type":["string","null"]},"segments":{"items":{"$ref":"#/$defs/Segment"},"type":"array"}},"required":["segments"],"type":"object"},"event":{"const":"user_message","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A canonical assistant/tool entry has committed to durable history.\nStreaming events remain the in-flight authority; this event supplies the\nstable identity used to reconcile that live block with paged history.","properties":{"data":{"additionalProperties":false,"properties":{"entry":{"$ref":"#/$defs/SessionSnapshotEntry"}},"required":["entry"],"type":"object"},"event":{"const":"session_entry_committed","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"One agent-injected system item committed to history.\n\nCarries the JSON form of `session_store::SystemItem`. Covers\n`Method::Notify` echoes, child-Worker lifecycle events from\n`Method::WorkerEvent`, `@` resolution payloads, and any future\nClients dispatch on the `kind` tag for typed rendering instead\nof parsing free-text prefixes like `[Notification] …` or\n`[File: …]`.\n\nOne event per `LogEntry::AnnotatedSystemItem` commit. Disk-side and\nwire-side are 1:1.","properties":{"data":{"additionalProperties":false,"properties":{"entry_id":{"description":"Stable durable history identity. Synthetic compatibility events omit it.","type":["string","null"]},"item":true},"required":["item"],"type":"object"},"event":{"const":"system_item","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A new self-driving cycle has begun (IDLE → active transition).\n\nMarker event for the start of an Invoke range; the range extends\nimplicitly until the next `InvokeStart`. Fires for every accepted\n`Method::Submit` (kind=`UserSend`), `Method::Notify` (kind=`Notify`),\n`Method::WorkerEvent` re-injection (kind=`WorkerEvent`), and any other\nIDLE-breaking trigger. Mid-run interrupts (e.g. hook output,\ntyped system reminder insertion that doesn't break IDLE) do not\nemit `InvokeStart` — they appear as `SystemItem` only.\n\nCarries `kind` only; the payload (user text / notify message /\nworker event body) is delivered separately via the immediately\nfollowing `UserMessage` / `SystemItem` event.","properties":{"data":{"additionalProperties":false,"properties":{"kind":{"$ref":"#/$defs/InvokeKind"}},"required":["kind"],"type":"object"},"event":{"const":"invoke_start","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"One AgentTurn boundary opened. An AgentTurn is a maximal run of\nLLM generation calls whose input messages are identical (i.e.\nretries from network errors / 5xx / stream disconnects collapse\ninto the same AgentTurn). When the input changes (a new tool\nresult lands, a user interrupts, etc.), the next LLM call belongs\nto a new AgentTurn.\n\n`turn` is the AgentTurn index from `Engine::turn_count`.\n\nCurrently retry is not yet implemented (`agen-stream-continuation`)\nso AgentTurn and `LlmCall` fire 1:1, but the contract here is\nthe AgentTurn boundary; consumers that want per-LLM-call signals\nmust subscribe to `LlmCallStart` / `LlmCallEnd` instead.","properties":{"data":{"additionalProperties":false,"properties":{"turn":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["turn"],"type":"object"},"event":{"const":"turn_start","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"AgentTurn closed.","properties":{"data":{"additionalProperties":false,"properties":{"result":{"$ref":"#/$defs/TurnResult"},"turn":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["turn","result"],"type":"object"},"event":{"const":"turn_end","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"One LLM generation call started (1 request → 1 generation, retry\nincluded). Multiple `LlmCall*` pairs may fire inside a single\n`TurnStart` / `TurnEnd` pair when a request is retried; today\nthey fire 1:1 because retry is not implemented.\n\n`llm_call` is the worker-wide running counter from\n`Engine::llm_call_count`.","properties":{"data":{"additionalProperties":false,"properties":{"llm_call":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["llm_call"],"type":"object"},"event":{"const":"llm_call_start","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"LLM generation call ended.","properties":{"data":{"additionalProperties":false,"properties":{"llm_call":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["llm_call"],"type":"object"},"event":{"const":"llm_call_end","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A transport-level LLM request retry has been scheduled.\n\nThis is operational state for clients to render while the worker is\nwaiting in backoff. It is not part of conversation history.","properties":{"data":{"additionalProperties":false,"properties":{"elapsed_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"error":{"type":"string"},"failed_attempt":{"description":"The attempt that just failed. 1 origin.","format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"llm_call":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"max_attempts":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"status":{"format":"uint16","maximum":65535.0,"minimum":0.0,"type":["integer","null"]},"wait_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["llm_call","failed_attempt","max_attempts","wait_ms","elapsed_ms","error"],"type":"object"},"event":{"const":"llm_retry","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Stream generation was interrupted after events had begun and the worker\nis continuing with a follow-up LLM request.","properties":{"data":{"additionalProperties":false,"properties":{"attempt":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"llm_call":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"max_attempts":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"reason":{"type":"string"}},"required":["llm_call","attempt","max_attempts","reason"],"type":"object"},"event":{"const":"llm_continuation","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"text":{"type":"string"}},"required":["text"],"type":"object"},"event":{"const":"text_delta","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"text":{"type":"string"}},"required":["text"],"type":"object"},"event":{"const":"text_done","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A reasoning / thinking block has started.\n\nAlways paired with a `ThinkingDone`. `ThinkingDelta` is optional:\nsome providers (or some configurations) emit thinking metadata\nwithout plaintext, in which case Start → Done arrive with no\ndeltas in between. Multiple thinking blocks per turn are allowed.","properties":{"event":{"const":"thinking_start","type":"string"}},"required":["event"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"text":{"type":"string"}},"required":["text"],"type":"object"},"event":{"const":"thinking_delta","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Thinking block completed. `text` is the full accumulated body\n(empty string when the provider didn't emit plaintext).","properties":{"data":{"additionalProperties":false,"properties":{"text":{"type":"string"}},"required":["text"],"type":"object"},"event":{"const":"thinking_done","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"id":{"type":"string"},"name":{"type":"string"}},"required":["id","name"],"type":"object"},"event":{"const":"tool_call_start","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"id":{"type":"string"},"json":{"type":"string"}},"required":["id","json"],"type":"object"},"event":{"const":"tool_call_args_delta","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"arguments":{"type":"string"},"id":{"type":"string"},"name":{"type":"string"}},"required":["id","name","arguments"],"type":"object"},"event":{"const":"tool_call_done","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"disposition":{"anyOf":[{"$ref":"#/$defs/ToolResultDisposition"},{"type":"null"}]},"id":{"type":"string"},"is_error":{"default":false,"type":"boolean"},"output":{"description":"Full tool output. Absent when the tool chose to return\nsummary-only, or when the result was pruned.","type":["string","null"]},"summary":{"description":"Short human-readable summary. Always present; used by clients\nthat only want a 1-line rendering (e.g. collapsed views).","type":"string"}},"required":["id","summary","is_error"],"type":"object"},"event":{"const":"tool_result","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Token accounting for one LLM request.\n\n`input_tokens` is the prompt prefix occupancy (cache reads /\ncache writes included), as normalised by the worker layer.\n`cache_read_input_tokens` is the cache-hit subset of that\noccupancy; subtracting it yields the \"net upload\" the client\nactually paid full price to send on this request, which is what\nthe TUI status line accumulates per turn.","properties":{"data":{"additionalProperties":false,"properties":{"cache_read_input_tokens":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":["integer","null"]},"input_tokens":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":["integer","null"]},"output_tokens":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":["integer","null"]}},"type":"object"},"event":{"const":"usage","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Authoritative current-context occupancy after a non-request boundary.\n\nUnlike [`Event::Usage`], this is not cumulative request traffic. It lets\nconnected clients converge with reconnect snapshots after run completion,\ncompaction, rewind, and other history rewrites. `None` explicitly means\nthe current value is unavailable and stale occupancy must be cleared.","properties":{"data":{"additionalProperties":false,"properties":{"usage":{"anyOf":[{"$ref":"#/$defs/ContextUsage"},{"type":"null"}]}},"type":"object"},"event":{"const":"context_usage","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"result":{"$ref":"#/$defs/RunResult"}},"required":["result"],"type":"object"},"event":{"const":"run_end","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"code":{"$ref":"#/$defs/ErrorCode"},"message":{"type":"string"}},"required":["code","message"],"type":"object"},"event":{"const":"error","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Sent exactly once at the start of every client connection.\n\n`entries` is the session-log mirror at subscribe time, serialised\nas the JSON form of `session_store::LogEntry`. This is the\nbulk-reconstruction lane: clients walk the entries to seed their\nderived view.\n\n`greeting` and `status` accompany the snapshot so clients render\nworker identity and current controller state without an extra round\ntrip.\n\nLive updates after the snapshot arrive through the streaming\nevents (`TextDelta` / `ToolCall*` / `ToolResult` / etc.) plus\nrole-specific entry events (`SegmentRotated` / `SystemItem`) —\nthere is no generic \"every committed entry\" broadcast.","properties":{"data":{"additionalProperties":false,"properties":{"greeting":{"$ref":"#/$defs/Greeting"},"in_flight":{"$ref":"#/$defs/InFlightSnapshot","description":"Unfinished model output that has already streamed in the current\nrun but is not yet represented by committed snapshot entries."},"internal_workers":{"description":"Parent-owned Internal Worker sessions visible to this client.\nService-private Internal Workers are deliberately excluded.","items":{"$ref":"#/$defs/InternalWorkerSnapshot"},"type":"array"},"session":{"$ref":"#/$defs/SessionSnapshot"},"state":{"$ref":"#/$defs/WorkerStateSnapshot","description":"Full revisioned live execution state. `Stopped` remains Runtime\ncatalog authority and is deliberately not represented here."}},"required":["session","greeting","state"],"type":"object"},"event":{"const":"snapshot","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A live event from a parent-owned Internal Worker. The payload reuses the\nnormal Worker event vocabulary while the wrapper carries stable origin\nidentity and a per-child revision fence.","properties":{"data":{"additionalProperties":false,"properties":{"event":{"$ref":"#/$defs/Event"},"revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"worker":{"$ref":"#/$defs/InternalWorkerRef"}},"required":["worker","revision","event"],"type":"object"},"event":{"const":"internal_worker","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Terminal removal fence for one parent-owned Internal Worker session.\n\nClients discard the matching child and descendants, then ignore later\nnested events for this identity until an authoritative snapshot replaces\nthe projection.","properties":{"data":{"additionalProperties":false,"properties":{"revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"worker":{"$ref":"#/$defs/InternalWorkerRef"}},"required":["worker","revision"],"type":"object"},"event":{"const":"internal_worker_removed","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Server-side segment log rotated to a fresh `SegmentStart`.\n\nFires on compaction and on auto-fork when the store head drifts\nA compaction/fork has replaced the authoritative segment. Clients drop\ntheir derived view and reseed from the canonical committed snapshot.","properties":{"data":{"additionalProperties":false,"properties":{"session":{"$ref":"#/$defs/SessionSnapshot"}},"required":["session"],"type":"object"},"event":{"const":"segment_rotated","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Current Worker controller status. Broadcast on every controller-level\ntransition and included in `History` snapshots for late attach.","properties":{"data":{"additionalProperties":false,"properties":{"snapshot":{"$ref":"#/$defs/WorkerStateSnapshot"}},"required":["snapshot"],"type":"object"},"event":{"const":"worker_state","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"acknowledgement":{"$ref":"#/$defs/WorkerCommandAcknowledgement"}},"required":["acknowledgement"],"type":"object"},"event":{"const":"command_acknowledged","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Bounded, provider-owned command telemetry for the live Console. This is\nintentionally not a history entry and is reconstructed from\n`Snapshot.in_flight.commands` after reconnect.","properties":{"data":{"additionalProperties":false,"properties":{"event":{"$ref":"#/$defs/CommandEvent"}},"required":["event"],"type":"object"},"event":{"const":"command","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Reply to `Method::ListCompletions`. Delivered only to the\nrequesting socket (not broadcast). `entries` is empty when no\ncandidates match or when the requested kind has no resolver\nwired up.","properties":{"data":{"additionalProperties":false,"properties":{"context":{"anyOf":[{"$ref":"#/$defs/CompletionContext"},{"type":"null"}]},"entries":{"items":{"$ref":"#/$defs/CompletionEntry"},"type":"array"},"kind":{"$ref":"#/$defs/CompletionKind"},"prefix":{"default":"","type":"string"},"request_id":{"type":["string","null"]}},"required":["kind","entries","prefix"],"type":"object"},"event":{"const":"completions","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Reply to `Method::ListRewindTargets`. Clients should only open a picker\nin response to their own pending request; the event may be broadcast.","properties":{"data":{"additionalProperties":false,"properties":{"head_entries":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"targets":{"items":{"$ref":"#/$defs/RewindTarget"},"type":"array"}},"required":["head_entries","targets"],"type":"object"},"event":{"const":"rewind_targets","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A rewind has truncated the authoritative session. `session` is the\nretained canonical snapshot clients should use to reseed display state.","properties":{"data":{"additionalProperties":false,"properties":{"input":{"items":{"$ref":"#/$defs/Segment"},"type":"array"},"session":{"$ref":"#/$defs/SessionSnapshot"},"summary":{"$ref":"#/$defs/RewindSummary"}},"required":["session","input","summary"],"type":"object"},"event":{"const":"rewind_applied","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Reply to `Method::ListWorkers`. Payload is a stable JSON value so the Worker\ncrate can evolve discovery fields without introducing a protocol\ndependency on session-store.","properties":{"data":{"additionalProperties":false,"properties":{"workers":true},"required":["workers"],"type":"object"},"event":{"const":"workers_listed","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Reply to `Method::RestoreWorker`.","properties":{"data":{"additionalProperties":false,"properties":{"result":true},"required":["result"],"type":"object"},"event":{"const":"worker_restored","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Reply to `Method::RegisterPeer`.","properties":{"data":{"additionalProperties":false,"properties":{"result":true},"required":["result"],"type":"object"},"event":{"const":"peer_registered","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"$ref":"#/$defs/Alert"},"event":{"const":"alert","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Latest memory extract/consolidation lifecycle event for UI observability.\n\nThis is not part of LLM history or prompt context; clients may display it\nbriefly as operational status.","properties":{"data":{"$ref":"#/$defs/MemoryWorkerEvent"},"event":{"const":"memory_worker","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Runtime-only compaction progress. `None` clears the current status.\nThis never enters Session history and carries no operation or Segment identity.","properties":{"data":{"additionalProperties":false,"properties":{"compaction":{"anyOf":[{"$ref":"#/$defs/InFlightCompaction"},{"type":"null"}]}},"type":"object"},"event":{"const":"compaction_progress","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Legacy compaction lifecycle event retained for wire read compatibility.","properties":{"data":{"additionalProperties":false,"properties":{"lifecycle":{"$ref":"#/$defs/CompactionLifecycle"}},"required":["lifecycle"],"type":"object"},"event":{"const":"compact_start","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Compaction completed and the session was rotated.","properties":{"data":{"additionalProperties":false,"properties":{"lifecycle":{"$ref":"#/$defs/CompactionLifecycle"}},"required":["lifecycle"],"type":"object"},"event":{"const":"compact_done","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Compaction failed or was cancelled. The session is unchanged.","properties":{"data":{"additionalProperties":false,"properties":{"lifecycle":{"$ref":"#/$defs/CompactionLifecycle"}},"required":["lifecycle"],"type":"object"},"event":{"const":"compact_failed","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"event":{"const":"shutdown","type":"string"}},"required":["event"],"type":"object"}]},"EventSubscriptionSelector":{"oneOf":[{"additionalProperties":false,"properties":{"topic":{"const":"runtime_workers","type":"string"}},"required":["topic"],"type":"object"},{"additionalProperties":false,"properties":{"topic":{"const":"worker_lifecycle","type":"string"},"worker_ids":{"$ref":"#/$defs/SubscriptionWorkerIds"}},"required":["topic","worker_ids"],"type":"object"},{"additionalProperties":false,"properties":{"runtime_id":{"type":["string","null"]},"topic":{"const":"worker_protocol","type":"string"},"worker_id":{"$ref":"#/$defs/SubscriptionWorkerId"}},"required":["topic","worker_id"],"type":"object"},{"additionalProperties":false,"description":"Server-derived Workspace projection. Workspace identity comes from the\nauthenticated connection and is deliberately absent from this selector.","properties":{"topic":{"const":"workspace_workers","type":"string"}},"required":["topic"],"type":"object"},{"additionalProperties":false,"description":"Typed extension point for the later Workdir subscription slice.","properties":{"topic":{"const":"workspace_workdirs","type":"string"}},"required":["topic"],"type":"object"}]},"FeatureInvocation":{"additionalProperties":false,"properties":{"arguments":{"items":{"$ref":"#/$defs/InvocationArgumentValue"},"type":"array"},"identity":{"$ref":"#/$defs/FeatureInvocationIdentity"},"invocation_id":{"description":"Stable client-generated id retained across queueing and transport retry.","type":"string"},"name":{"description":"Selected public name, retained for exact draft/history restoration. Host\nresolution is always by `identity`.","type":"string"}},"required":["invocation_id","identity","name","arguments"],"type":"object"},"FeatureInvocationDescriptor":{"additionalProperties":false,"properties":{"aliases":{"default":[],"items":{"type":"string"},"type":"array"},"arguments":{"default":[],"items":{"$ref":"#/$defs/InvocationArgumentDescriptor"},"type":"array"},"client_adapter":{"anyOf":[{"$ref":"#/$defs/InvocationClientAdapter"},{"type":"null"}]},"description":{"type":"string"},"display_name":{"type":"string"},"identity":{"$ref":"#/$defs/FeatureInvocationIdentity"},"name":{"type":"string"},"syntax":{"$ref":"#/$defs/FeatureInvocationSyntax"}},"required":["identity","name","display_name","description","syntax","aliases","arguments"],"type":"object"},"FeatureInvocationIdentity":{"description":"Source-qualified identity of one invocation contribution, for example\n`builtin:attachments/attach`. Display names and aliases are never authority.","type":"string"},"FeatureInvocationSyntax":{"oneOf":[{"const":"parenthesized","description":"`/name(value, key=value)` with JSON-style quoted strings and `)` as the\nexplicit boundary before following natural language.","type":"string"}]},"Greeting":{"additionalProperties":false,"description":"Worker self-description rendered by clients when a session starts empty.\n\nSnapshot metadata contains only resolved public execution information so\nclients do not need their own copy of the manifest.","properties":{"context_tokens":{"default":0,"description":"Legacy numeric mirror retained for mixed-version clients.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"context_usage":{"anyOf":[{"$ref":"#/$defs/ContextUsage"},{"type":"null"}],"description":"Typed current-context value. Older snapshots omit it; clients may fall\nback to `context_tokens` only when `context_window` is known."},"context_window":{"default":0,"description":"Resolved effective model context window in tokens. Legacy producers use\nzero when this is unavailable; clients must treat that as unknown.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"cwd":{"type":"string"},"model":{"type":"string"},"provider":{"type":"string"},"reasoning":{"anyOf":[{"$ref":"#/$defs/ReasoningConfig"},{"type":"null"}],"description":"Effective reasoning control. Absent means unspecified or unavailable."},"scope_summary":{"type":"string"},"tools":{"items":{"type":"string"},"type":"array"},"worker_name":{"type":"string"}},"required":["worker_name","cwd","provider","model","scope_summary","tools","context_tokens","context_window"],"type":"object"},"InFlightBlock":{"oneOf":[{"additionalProperties":false,"properties":{"finished":{"type":"boolean"},"kind":{"const":"text","type":"string"},"text":{"type":"string"}},"required":["kind","text"],"type":"object"},{"additionalProperties":false,"properties":{"finished":{"type":"boolean"},"kind":{"const":"thinking","type":"string"},"text":{"type":"string"}},"required":["kind","text"],"type":"object"},{"additionalProperties":false,"properties":{"args":{"type":"string"},"id":{"type":"string"},"kind":{"const":"tool_call","type":"string"},"name":{"type":"string"},"state":{"$ref":"#/$defs/InFlightToolCallState"}},"required":["kind","id","name","args"],"type":"object"}]},"InFlightCompaction":{"additionalProperties":false,"properties":{"phase":{"$ref":"#/$defs/CompactionPhase"},"started_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"trigger":{"$ref":"#/$defs/CompactionTrigger"}},"required":["phase","started_at_ms","trigger"],"type":"object"},"InFlightSnapshot":{"additionalProperties":false,"description":"Unfinished model output and active command state included in\n`Event::Snapshot` for clients that attach while work is still streaming.\n\nThese blocks are presentation state only: they are reconstructed from the\nactive Worker controller and must not be treated as committed assistant\nhistory. Finalized assistant items continue to come from ordinary snapshot\nentries.","properties":{"blocks":{"items":{"$ref":"#/$defs/InFlightBlock"},"type":"array"},"commands":{"items":{"$ref":"#/$defs/CommandSnapshot"},"type":"array"},"compaction":{"anyOf":[{"$ref":"#/$defs/InFlightCompaction"},{"type":"null"}],"description":"The currently running compaction, if any.\n\nThis is lifecycle progress only. Candidate history and the staged\nSegment remain private until the Segment is activated atomically."}},"type":"object"},"InFlightToolCallState":{"enum":["pending","streaming_args","done"],"type":"string"},"InternalWorkerKind":{"description":"Presentation category for an Internal Worker exposed through its parent's\nprotocol stream. Internal Workers never become independently addressable\nprotocol subjects.","oneOf":[{"enum":["sub_worker"],"type":"string"},{"additionalProperties":false,"properties":{"service":{"additionalProperties":false,"properties":{"kind":{"type":"string"}},"required":["kind"],"type":"object"}},"required":["service"],"type":"object"}]},"InternalWorkerRef":{"additionalProperties":false,"description":"Stable presentation identity for one parent-owned Internal Worker session.\n`name` is display-only; `session_id` is the identity used by clients.","properties":{"kind":{"$ref":"#/$defs/InternalWorkerKind"},"name":{"type":"string"},"parent_session_id":{"type":["string","null"]},"session_id":{"type":"string"}},"required":["session_id","name","kind"],"type":"object"},"InternalWorkerSnapshot":{"additionalProperties":false,"description":"Reconnect state for one visible Internal Worker. The revision fences live\n`Event::InternalWorker` updates that raced with parent snapshot assembly.","properties":{"error":{"type":["string","null"]},"greeting":{"anyOf":[{"$ref":"#/$defs/Greeting"},{"type":"null"}],"description":"Public execution metadata for this child. Older snapshots and service\nworkers may omit it; clients must never substitute the parent's values."},"in_flight":{"$ref":"#/$defs/InFlightSnapshot"},"internal_workers":{"items":{"$ref":"#/$defs/InternalWorkerSnapshot"},"type":"array"},"revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"session":{"$ref":"#/$defs/SessionSnapshot"},"status":{"$ref":"#/$defs/WorkerStatus","default":"idle"},"worker":{"$ref":"#/$defs/InternalWorkerRef"}},"required":["worker","revision","session","status"],"type":"object"},"InvocationArgumentDescriptor":{"additionalProperties":false,"properties":{"completion":{"$ref":"#/$defs/InvocationCompletion","default":{"kind":"none"}},"description":{"type":["string","null"]},"name":{"type":"string"},"position":{"description":"Zero-based positional index. `None` means named-only. A positional\nargument may also be supplied by name; this makes chip editing stable.","format":"uint16","maximum":65535.0,"minimum":0.0,"type":["integer","null"]},"required":{"default":false,"type":"boolean"},"value_type":{"$ref":"#/$defs/InvocationArgumentType"}},"required":["name","value_type","completion","required"],"type":"object"},"InvocationArgumentType":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"string","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"integer","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"boolean","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"enum","type":"string"},"values":{"items":{"type":"string"},"type":"array"}},"required":["kind","values"],"type":"object"},{"additionalProperties":false,"description":"A Worker-readable path. This does not grant filesystem authority.","properties":{"kind":{"const":"worker_file","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"description":"A resource selected and staged by the client. It must never be resolved\nas a path on the Worker host.","properties":{"kind":{"const":"client_file","type":"string"}},"required":["kind"],"type":"object"}]},"InvocationArgumentValue":{"additionalProperties":false,"properties":{"name":{"type":"string"},"value":{"$ref":"#/$defs/InvocationValue"}},"required":["name","value"],"type":"object"},"InvocationClientAdapter":{"oneOf":[{"const":"attachment","description":"The client stages a local file and replaces the draft invocation with an\n`UploadedFile` chip. Local paths are not sent to or persisted by Worker.","type":"string"}]},"InvocationCompletion":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"none","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"static","type":"string"},"values":{"items":{"type":"string"},"type":"array"}},"required":["kind","values"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"worker_file","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"client_file","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"description":"Host-owned provider selected by source-qualified id. Metadata lookup is\nside-effect free; implementations never cross this wire boundary.","properties":{"kind":{"const":"provider","type":"string"},"provider":{"type":"string"}},"required":["kind","provider"],"type":"object"}]},"InvocationValue":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"string","type":"string"},"value":{"type":"string"}},"required":["kind","value"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"integer","type":"string"},"value":{"format":"int32","maximum":2147483647.0,"minimum":-2147483648.0,"type":"integer"}},"required":["kind","value"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"boolean","type":"string"},"value":{"type":"boolean"}},"required":["kind","value"],"type":"object"}]},"InvokeKind":{"description":"Kind of trigger that opened a new Invoke (IDLE → active) range.\n\nOne Invoke groups all entries from this trigger up to the next\n`Invoke` marker. The kind is the only payload — content (user text,\nnotify message, worker event body) is delivered by the immediately\nfollowing Turn entry, not by the marker itself.","oneOf":[{"const":"user_send","description":"`Method::Submit` — a user submission.","type":"string"},{"const":"notify","description":"`Method::Notify` — free-text notification injected into history.","type":"string"},{"const":"worker_event","description":"`Method::WorkerEvent` — typed lifecycle report from a child Worker.","type":"string"},{"const":"system_reminder","description":"A typed system reminder that crosses an IDLE boundary (mid-run\nreminders that don't break IDLE are SystemItem-only and do not\nopen a new Invoke).","type":"string"},{"const":"wakeup","description":"Cron / RemoteTrigger style scheduled wake-up. Reserved; no\nproducer is wired today but the variant is part of the initial\nset so future schedulers don't have to amend the wire enum.","type":"string"}]},"MemoryWorkerEvent":{"additionalProperties":false,"properties":{"message":{"description":"Human-readable compact form for actionbar rendering.","type":"string"},"reason":{"type":"string"},"run_id":{"type":"string"},"status":{"type":"string"},"timestamp_ms":{"description":"Milliseconds since the Unix epoch.","format":"int64","maximum":9007199254740991.0,"minimum":-9007199254740991.0,"type":"integer"},"trigger":{"type":"string"},"worker":{"type":"string"}},"required":["worker","status","run_id","trigger","reason","message","timestamp_ms"],"type":"object"},"Method":{"oneOf":[{"additionalProperties":false,"description":"Durably accept typed input for immediate activation or the session FIFO.\nRunning or Paused Workers queue Submit for a later turn; use Notify for\nadvisory information to incorporate into work already in progress.\n\n`submission_request_id` is generated by the authenticated caller and is\nused only for idempotent retry. Worker allocates the durable\n`submission_id` returned by [`Event::SubmissionAccepted`].","properties":{"method":{"const":"submit","type":"string"},"params":{"additionalProperties":false,"properties":{"input":{"items":{"$ref":"#/$defs/Segment"},"type":"array"},"submission_request_id":{"type":"string"}},"required":["submission_request_id","input"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Human-readable text injected into the target Worker's LLM context as a\nnon-blocking system message. Accepted notifications are delivered at the\nnext between-turn boundary, or start a coalesced notification run while\nthe Worker is idle. No side effects beyond LLM context; use\n`WorkerEvent` for typed lifecycle reports.","properties":{"method":{"const":"notify","type":"string"},"params":{"additionalProperties":false,"properties":{"message":{"type":"string"},"notification_request_id":{"type":"string"}},"required":["notification_request_id","message"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Typed lifecycle report from a child Worker to its direct parent.","properties":{"method":{"const":"worker_event","type":"string"},"params":{"$ref":"#/$defs/WorkerEvent"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Return the authoritative FIFO summary without exposing queued payloads.","properties":{"method":{"const":"list_pending_submissions","type":"string"}},"required":["method"],"type":"object"},{"additionalProperties":false,"description":"Remove one queued submission. Running or already activated submissions\nare immutable and therefore cannot be cancelled here.","properties":{"method":{"const":"cancel_pending_submission","type":"string"},"params":{"additionalProperties":false,"properties":{"expected_revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"submission_id":{"type":"string"}},"required":["submission_id","expected_revision"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Remove every queued submission while preserving the active run.","properties":{"method":{"const":"clear_pending_submissions","type":"string"},"params":{"additionalProperties":false,"properties":{"expected_revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["expected_revision"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Activate the next queued submission while the Worker is idle. This is an\nexplicit recovery operation and never resumes a paused run implicitly.","properties":{"method":{"const":"continue_pending","type":"string"},"params":{"additionalProperties":false,"properties":{"expected_head_id":{"type":"string"},"expected_revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["expected_revision","expected_head_id"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"properties":{"method":{"const":"resume","type":"string"},"params":{"additionalProperties":false,"properties":{"command":{"$ref":"#/$defs/WorkerCommandEnvelope"}},"required":["command"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"properties":{"method":{"const":"cancel","type":"string"},"params":{"additionalProperties":false,"properties":{"command":{"$ref":"#/$defs/WorkerCommandEnvelope"}},"required":["command"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Stop the in-flight turn and transition to `Paused`.\n\nUnlike `Cancel` (which discards and returns to `Idle`), a paused\nWorker can resume the interrupted work via `Resume`, or accept a\nfresh `Submit` (orphan `tool_use` items are closed with a\nsynthetic tool result before the new user message is appended).","properties":{"method":{"const":"pause","type":"string"},"params":{"additionalProperties":false,"properties":{"command":{"$ref":"#/$defs/WorkerCommandEnvelope"}},"required":["command"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Request an explicit compaction while the Worker is otherwise idle.\n\nThis is a typed control method: clients must not send `compact` as a\n`Method::Submit` user message.","properties":{"method":{"const":"compact","type":"string"},"params":{"additionalProperties":false,"properties":{"command":{"$ref":"#/$defs/WorkerCommandEnvelope"}},"required":["command"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Ask the Worker to list valid rewind targets from its authoritative session log.","properties":{"method":{"const":"list_rewind_targets","type":"string"}},"required":["method"],"type":"object"},{"additionalProperties":false,"description":"Truncate the current session back to the selected rewind target and\nreturn the selected user input to the client composer.","properties":{"method":{"const":"rewind_to","type":"string"},"params":{"additionalProperties":false,"properties":{"expected_head_entries":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"target":{"$ref":"#/$defs/RewindTargetId"}},"required":["target","expected_head_entries"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"properties":{"method":{"const":"shutdown","type":"string"},"params":{"additionalProperties":false,"properties":{"command":{"$ref":"#/$defs/WorkerCommandEnvelope"}},"required":["command"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Request a list of completion candidates from the Worker.\n\nReply is sent on the same socket as `Event::Completions` (not\nbroadcast). The IPC server handles this directly and writes\nthe response straight back to the requesting socket.","properties":{"method":{"const":"list_completions","type":"string"},"params":{"additionalProperties":false,"properties":{"context":{"anyOf":[{"$ref":"#/$defs/CompletionContext"},{"type":"null"}]},"kind":{"$ref":"#/$defs/CompletionKind"},"prefix":{"type":"string"},"request_id":{"description":"Client query generation. Echoed verbatim so identical-prefix ABA\nresponses cannot cross edits, target switches, or permission changes.","type":["string","null"]}},"required":["kind","prefix"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"List Workers visible to this Worker from durable Worker state and the spawned-child\nregistry. This is not a host-wide Worker universe query.","properties":{"method":{"const":"list_workers","type":"string"}},"required":["method"],"type":"object"},{"additionalProperties":false,"description":"Restore a visible stopped/restorable Worker, or report that it is already\nlive. Missing state and not-visible state are distinct errors.","properties":{"method":{"const":"restore_worker","type":"string"},"params":{"additionalProperties":false,"properties":{"name":{"type":"string"}},"required":["name"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Register another existing Worker as a reciprocal peer of this Worker.\n\nThis is metadata/control state only: it does not ask the target's live\ncontroller for consent, and it must not grant delegated scope,\nspawned-child ownership, output cursors, or child lifecycle authority.","properties":{"method":{"const":"register_peer","type":"string"},"params":{"additionalProperties":false,"properties":{"name":{"type":"string"}},"required":["name"],"type":"object"}},"required":["method","params"],"type":"object"}]},"PasteArtifactAvailability":{"enum":["available","unavailable","integrity_failed"],"type":"string"},"PasteArtifactMediaType":{"description":"One typed piece of a user submission.\n\n`Method::Submit` and `Event::UserMessage` carry `Vec`. Dumb\nclients (CLI piping, scripts) only need to produce a single\n`Segment::Text`; richer clients (TUI / GUI) construct typed atoms\n(paste chips, file refs) and\nsend them through directly so the Worker side never has to re-parse a\nflattened string.\n\nForward compat: payloads with unknown `kind` deserialize to\n`Segment::Unknown`. Worker treats this the same as known-but-unresolved\nvariants — emits an alert and inserts a `[unknown input segment]`\nplaceholder into the LLM context so neither user nor LLM is blind to\nthe dropped intent.","enum":["text_plain_utf8"],"type":"string"},"PasteArtifactRef":{"additionalProperties":false,"description":"Session-owned reference to a large pasted-input artifact.\n\nThe reference contains only bounded integrity and provenance metadata. The\nartifact body remains in session storage and is available to the model only\nthrough the scoped paste-artifact tools installed by Worker.","properties":{"artifact_id":{"type":"string"},"availability":{"$ref":"#/$defs/PasteArtifactAvailability","description":"Availability observed when this immutable reference was committed.\nReads revalidate storage and integrity rather than trusting this field."},"byte_len":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"char_count":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"created_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"line_count":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"media_type":{"$ref":"#/$defs/PasteArtifactMediaType"},"sha256":{"type":"string"},"source_entry_id":{"type":"string"}},"required":["artifact_id","created_at_ms","media_type","availability","byte_len","char_count","line_count","sha256","source_entry_id"],"type":"object"},"PendingSubmissionSummary":{"additionalProperties":false,"description":"Bounded public projection of one pending submission. Only a short display\npreview is exposed; full payload segments and provenance remain in the log.","properties":{"accepted_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"byte_len":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"preview":{"description":"Single-line display text, at most 240 Unicode scalars plus an ellipsis.\nAbsent on older runtimes. Never used to reconstruct the submitted input.","type":["string","null"]},"segment_count":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"submission_id":{"type":"string"}},"required":["submission_id","accepted_at_ms","segment_count","byte_len"],"type":"object"},"PendingSubmissionsSnapshot":{"additionalProperties":false,"description":"Revisioned session-owned FIFO projection used by snapshots and live events.","properties":{"head_id":{"default":null,"type":["string","null"]},"notification_count":{"default":0,"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"notification_previews":{"description":"Ordered display-only previews of waiting notifications, each bounded like\nPendingSubmissionSummary::preview. Older runtimes omit this field.","items":{"type":"string"},"type":"array"},"revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"submissions":{"default":[],"items":{"$ref":"#/$defs/PendingSubmissionSummary"},"type":"array"}},"required":["revision","head_id","notification_count","submissions"],"type":"object"},"Permission":{"description":"Permission lattice used by [`ScopeRule`].\n\nThe derived `Ord` instance follows declaration order, so\n`Read < Write`. Allow rules grant the stated level (and by extension\neverything below); deny rules cap the effective level **strictly\nbelow** the stated level.","enum":["read","write"],"type":"string"},"ReasoningConfig":{"description":"Public reasoning configuration resolved for the model client.\n\nThis deliberately exposes only the effective provider control, not the\nWorker manifest or provider credentials. The tagged shape prevents token\nbudgets from being presented as effort labels.","oneOf":[{"additionalProperties":false,"properties":{"effort":{"type":"string"},"kind":{"const":"effort","type":"string"}},"required":["kind","effort"],"type":"object"},{"additionalProperties":false,"properties":{"budget_tokens":{"format":"int32","maximum":2147483647.0,"minimum":-2147483648.0,"type":"integer"},"kind":{"const":"budget_tokens","type":"string"}},"required":["kind","budget_tokens"],"type":"object"}]},"RewindSummary":{"additionalProperties":false,"properties":{"discarded_entries":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"tool_side_effect_warning":{"type":"boolean"},"truncated_to_entries":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["truncated_to_entries","discarded_entries","tool_side_effect_warning"],"type":"object"},"RewindTarget":{"additionalProperties":false,"properties":{"disabled_reason":{"type":["string","null"]},"eligible":{"type":"boolean"},"expected_head_entries":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"id":{"$ref":"#/$defs/RewindTargetId"},"preview":{"type":"string"},"timestamp_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":["integer","null"]},"truncate_entries":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"turn_index":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"warning":{"type":["string","null"]}},"required":["id","expected_head_entries","truncate_entries","turn_index","preview","eligible"],"type":"object"},"RewindTargetId":{"additionalProperties":false,"properties":{"segment_id":{"type":"string"},"user_input_entry_index":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["segment_id","user_input_entry_index"],"type":"object"},"RunFailureKind":{"enum":["engine","compaction"],"type":"string"},"RunResult":{"oneOf":[{"enum":["finished","paused","limit_reached"],"type":"string"},{"const":"cancelled","description":"The logical Run ended because the user intentionally cancelled it after\nobservable work. Unlike `RolledBack`, committed history is retained.","type":"string"},{"const":"rolled_back","description":"The accepted Method::Submit produced no assistant/tool output before\nuser interruption, so the Worker rolled the submit-time turn state back\nto its pre-submit snapshot. Clients should treat the Worker as Idle and\nrestore the just-submitted input into the editable composer if desired.","type":"string"}]},"RunResumeSource":{"enum":["compaction","pause"],"type":"string"},"RunYieldReason":{"enum":["compaction"],"type":"string"},"ScopeRule":{"additionalProperties":false,"description":"A single allow or deny rule inside a scope configuration.","properties":{"permission":{"$ref":"#/$defs/Permission","description":"Permission level this rule grants (allow) or caps strictly below\n(deny)."},"recursive":{"default":true,"description":"When `false`, the rule only matches the target itself and its\ndirect children. Defaults to `true`.","type":"boolean"},"symlink_policy":{"$ref":"#/$defs/SymlinkPolicy","default":"resolved","description":"Which path identity an allow rule uses when symbolic links are\nencountered. Deny rules always inspect both identities."},"target":{"description":"Target path. Must be absolute by the time a `Scope` is built from\nthis rule — relative paths are resolved per-layer against the\nmanifest file's directory (cwd for overlay layers) before cascade\nmerge.","type":"string"}},"required":["target","permission","recursive","symlink_policy"],"type":"object"},"Segment":{"oneOf":[{"additionalProperties":false,"description":"Free-form text. The fallback every client can produce.","properties":{"content":{"type":"string"},"kind":{"const":"text","type":"string"}},"required":["kind","content"],"type":"object"},{"additionalProperties":false,"description":"Bracketed-paste capture from a TUI-style client. `id`, `chars`\nand `lines` carry the metadata needed to re-render a\n`[Clipboard #N | X chars, Y lines]` chip in `Event::UserMessage`\nre-broadcast.","properties":{"chars":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"content":{"type":"string"},"id":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"kind":{"const":"paste","type":"string"},"lines":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"}},"required":["kind","id","chars","lines","content"],"type":"object"},{"additionalProperties":false,"description":"Internal reference produced when Worker stores a large `Paste` before\ncommitting input. Clients may receive this in history/event projections;\nthe body is intentionally absent.","properties":{"artifact":{"$ref":"#/$defs/PasteArtifactRef"},"kind":{"const":"paste_artifact","type":"string"}},"required":["kind","artifact"],"type":"object"},{"additionalProperties":false,"description":"Client-local file uploaded into the owning Worker session before submit.\nThe Session Log stores only this immutable reference, never file bytes or\nthe client's local path.","properties":{"file":{"$ref":"#/$defs/UploadedFileRef"},"kind":{"const":"uploaded_file","type":"string"}},"required":["kind","file"],"type":"object"},{"additionalProperties":false,"description":"`@` file-system reference. Worker resolves readable files to\n`[File: ]` attachments and readable normal directories to shallow\n`[Dir: ]` listings; the flattened user text keeps the literal\n`@` placeholder either way.","properties":{"kind":{"const":"file_ref","type":"string"},"path":{"type":"string"}},"required":["kind","path"],"type":"object"},{"additionalProperties":false,"description":"Explicitly selected, source-qualified Feature invocation. The host never\npromotes free text to this variant and validates it against the installed\ndeclaration before execution.","properties":{"invocation":{"$ref":"#/$defs/FeatureInvocation"},"kind":{"const":"feature_invoke","type":"string"}},"required":["kind","invocation"],"type":"object"},{"additionalProperties":false,"description":"Source-qualified request for the host to start a Flow before committing\nthis user input. Runtime resolves it through Workspace authority and\nreplaces it with the entered state's instructions before Worker-side\ninput resolution.","properties":{"kind":{"const":"flow","type":"string"},"selector":{"type":"string"}},"required":["kind","selector"],"type":"object"},{"additionalProperties":false,"description":"Unknown variant from a newer client. Worker treats this as an\nunresolved input — surfaces an alert and inserts a placeholder.\nRound-trip is lossy: re-serializing yields `{\"kind\":\"unknown\"}`.","properties":{"kind":{"const":"unknown","type":"string"}},"required":["kind"],"type":"object"}]},"SessionContentPart":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"text","type":"string"},"text":{"type":"string"}},"required":["kind","text"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"refusal","type":"string"},"refusal":{"type":"string"}},"required":["kind","refusal"],"type":"object"}]},"SessionEntryProvenance":{"enum":["human_input","worker_input","flow_instruction","backend_instruction","model_output","tool_output","derived_summary","legacy_unknown"],"type":"string"},"SessionMessageRole":{"enum":["user","assistant"],"type":"string"},"SessionSnapshot":{"additionalProperties":false,"description":"Canonical, storage-independent projection of committed session history.\n\nWorker protocols expose this DTO instead of append-log records. New\nstorage variants can therefore be added without teaching every client how\nto replay the durable log format.","properties":{"entries":{"items":{"$ref":"#/$defs/SessionSnapshotEntry"},"type":"array"},"pending_submissions":{"$ref":"#/$defs/PendingSubmissionsSnapshot","default":{"head_id":null,"notification_count":0,"revision":0,"submissions":[]}}},"required":["entries","pending_submissions"],"type":"object"},"SessionSnapshotEntry":{"oneOf":[{"additionalProperties":false,"properties":{"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"kind":{"const":"user_input","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"segments":{"items":{"$ref":"#/$defs/Segment"},"type":"array"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","segments","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"properties":{"content":{"items":{"$ref":"#/$defs/SessionContentPart"},"type":"array"},"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"kind":{"const":"message","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"role":{"$ref":"#/$defs/SessionMessageRole"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","role","content","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"properties":{"arguments":{"type":"string"},"call_id":{"type":"string"},"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"kind":{"const":"tool_call","type":"string"},"name":{"type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","call_id","name","arguments","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"properties":{"attachments":{"items":{"$ref":"#/$defs/SessionToolAttachment"},"type":"array"},"call_id":{"type":"string"},"content":{"type":["string","null"]},"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"is_error":{"type":"boolean"},"kind":{"const":"tool_result","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"summary":{"type":"string"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","call_id","summary","is_error","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"properties":{"content":{"type":"string"},"data":true,"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"item_kind":{"type":"string"},"kind":{"const":"system_item","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","item_kind","content","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"description":"Durable, non-terminal control return from one logical Run. This is not a\ncompleted run and must not be used to infer the current Worker state.","properties":{"active_run_turn_count":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"kind":{"const":"run_yielded","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"reason":{"$ref":"#/$defs/RunYieldReason"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","reason","active_run_turn_count","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"description":"Durable boundary immediately before Engine execution resumes. The source\ndistinguishes automatic compaction continuation from an intentional\nuser-pause resume; it does not prove that later model output occurred.","properties":{"active_run_turn_count":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"kind":{"const":"run_resumed","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"source":{"$ref":"#/$defs/RunResumeSource"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","source","active_run_turn_count","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"description":"Durable terminal for an intentional cancellation. Consumers must not\npresent this lifecycle result as an execution failure.","properties":{"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"kind":{"const":"run_cancelled","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"properties":{"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"failure":{"anyOf":[{"$ref":"#/$defs/RunFailureKind"},{"type":"null"}]},"kind":{"const":"run_error","type":"string"},"message":{"type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","message","entry_id","timestamp","provenance"],"type":"object"}]},"SessionToolAttachment":{"additionalProperties":false,"properties":{"attachment_id":{"description":"Stable, bounded identity derived from the owning Session entry, attachment\nposition, and immutable image bytes. It is a locator, not a bearer secret.","type":"string"},"byte_len":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"media_type":{"type":"string"}},"required":["attachment_id","media_type","byte_len"],"type":"object"},"SubmissionDisposition":{"description":"Durable acceptance result for one idempotent Submit request.","oneOf":[{"enum":["started"],"type":"string"},{"const":"queued","description":"Input is durably held in the session FIFO until eligible for activation.","type":"string"}]},"SubscriptionEvent":{"oneOf":[{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"payload":{"$ref":"#/$defs/SubscriptionEventPayload"},"subscription_id":{"$ref":"#/$defs/SubscriptionId"}},"required":["subscription_id","payload"],"type":"object"},"event":{"const":"event","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"code":{"$ref":"#/$defs/SubscriptionTerminationCode"},"message":{"type":"string"},"subscription_id":{"$ref":"#/$defs/SubscriptionId"}},"required":["subscription_id","code","message"],"type":"object"},"event":{"const":"subscription_closed","type":"string"}},"required":["event","data"],"type":"object"}]},"SubscriptionEventPayload":{"oneOf":[{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"worker":{"$ref":"#/$defs/SubscriptionWorker"}},"required":["worker"],"type":"object"},"event":{"const":"worker_upserted","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"runtime_id":{"type":["string","null"]},"worker_id":{"$ref":"#/$defs/SubscriptionWorkerId"}},"required":["worker_id"],"type":"object"},"event":{"const":"worker_removed","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"event":{"$ref":"#/$defs/Event"},"worker_id":{"$ref":"#/$defs/SubscriptionWorkerId"}},"required":["worker_id","event"],"type":"object"},"event":{"const":"worker_protocol","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"workdir":{"$ref":"#/$defs/WorkspaceSubscriptionWorkdir"}},"required":["workdir"],"type":"object"},"event":{"const":"workdir_upserted","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"working_directory_id":{"$ref":"#/$defs/SubscriptionWorkdirId"}},"required":["working_directory_id"],"type":"object"},"event":{"const":"workdir_removed","type":"string"}},"required":["event","data"],"type":"object"}]},"SubscriptionId":{"type":"string"},"SubscriptionRejectionCode":{"enum":["invalid_request","unsupported_protocol_version","unsupported_selector","unauthorized","resource_not_found","capacity_exceeded","internal"],"type":"string"},"SubscriptionRequest":{"oneOf":[{"additionalProperties":false,"properties":{"method":{"const":"subscribe_events","type":"string"},"params":{"additionalProperties":false,"properties":{"request_id":{"$ref":"#/$defs/SubscriptionRequestId"},"selector":{"$ref":"#/$defs/EventSubscriptionSelector"}},"required":["request_id","selector"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"properties":{"method":{"const":"unsubscribe_events","type":"string"},"params":{"additionalProperties":false,"properties":{"request_id":{"$ref":"#/$defs/SubscriptionRequestId"},"subscription_id":{"$ref":"#/$defs/SubscriptionId"}},"required":["request_id","subscription_id"],"type":"object"}},"required":["method","params"],"type":"object"}]},"SubscriptionRequestId":{"type":"string"},"SubscriptionResponse":{"oneOf":[{"additionalProperties":false,"properties":{"payload":{"additionalProperties":false,"properties":{"request_id":{"$ref":"#/$defs/SubscriptionRequestId"},"selector":{"$ref":"#/$defs/EventSubscriptionSelector"},"snapshot":{"$ref":"#/$defs/SubscriptionSnapshot"},"subscription_id":{"$ref":"#/$defs/SubscriptionId"}},"required":["request_id","subscription_id","selector","snapshot"],"type":"object"},"result":{"const":"subscribed","type":"string"}},"required":["result","payload"],"type":"object"},{"additionalProperties":false,"properties":{"payload":{"additionalProperties":false,"properties":{"request_id":{"$ref":"#/$defs/SubscriptionRequestId"},"subscription_id":{"$ref":"#/$defs/SubscriptionId"}},"required":["request_id","subscription_id"],"type":"object"},"result":{"const":"unsubscribed","type":"string"}},"required":["result","payload"],"type":"object"},{"additionalProperties":false,"properties":{"payload":{"additionalProperties":false,"properties":{"code":{"$ref":"#/$defs/SubscriptionRejectionCode"},"message":{"type":"string"},"request_id":{"$ref":"#/$defs/SubscriptionRequestId"},"subscription_id":{"anyOf":[{"$ref":"#/$defs/SubscriptionId"},{"type":"null"}]}},"required":["request_id","code","message"],"type":"object"},"result":{"const":"subscription_rejected","type":"string"}},"required":["result","payload"],"type":"object"}]},"SubscriptionSnapshot":{"oneOf":[{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"workers":{"items":{"$ref":"#/$defs/SubscriptionWorker"},"type":"array"}},"required":["workers"],"type":"object"},"topic":{"const":"workers","type":"string"}},"required":["topic","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"events":{"items":{"$ref":"#/$defs/Event"},"type":"array"},"worker_id":{"$ref":"#/$defs/SubscriptionWorkerId"}},"required":["worker_id","events"],"type":"object"},"topic":{"const":"worker_protocol","type":"string"}},"required":["topic","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"workdirs":{"items":{"$ref":"#/$defs/WorkspaceSubscriptionWorkdir"},"type":"array"}},"required":["workdirs"],"type":"object"},"topic":{"const":"workspace_workdirs","type":"string"}},"required":["topic","data"],"type":"object"}]},"SubscriptionTerminationCode":{"enum":["lagged","resource_gone","unauthorized","server_shutdown"],"type":"string"},"SubscriptionWorkdirId":{"type":"string"},"SubscriptionWorker":{"additionalProperties":false,"properties":{"availability":{"$ref":"#/$defs/SubscriptionWorkerAvailability","default":"observed","description":"Freshness of the Runtime-backed observation carried by this projection.\n`Unavailable` preserves catalog membership without claiming that execution stopped."},"display_name":{"type":["string","null"]},"has_running_internal_workers":{"default":false,"type":"boolean"},"job":{"anyOf":[{"$ref":"#/$defs/SubscriptionWorkerJob"},{"type":"null"}],"description":"Backend-owned Job binding. Runtime producers leave this unset; the\nWorkspace projection enriches both snapshots and live updates."},"profile":{"type":["string","null"]},"resource_key":{"description":"Workspace-scoped canonical resource key. Runtime producers leave this unset;\nWorkspace-facing projections must populate it before publishing the Worker.","type":["string","null"]},"runtime_id":{"description":"Set by the Workspace Server when projecting a Runtime-owned Worker to clients.\nRuntime producers leave this unset because the connection identifies the Runtime.","type":["string","null"]},"state":{"$ref":"#/$defs/SubscriptionWorkerState","description":"Runtime catalog lifecycle compatibility projection; not foreground-state authority."},"workdir_attachments":{"items":{"$ref":"#/$defs/SubscriptionWorkerWorkdirAttachment"},"type":"array"},"worker_id":{"$ref":"#/$defs/SubscriptionWorkerId"},"worker_state":{"anyOf":[{"$ref":"#/$defs/WorkerStateSnapshot"},{"type":"null"}],"description":"Latest foreground state observed from the Worker. This remains\nabsent until an authoritative Worker snapshot/event has been applied."},"workspace_id":{"type":["string","null"]}},"required":["worker_id","state","availability","has_running_internal_workers"],"type":"object"},"SubscriptionWorkerAvailability":{"description":"Whether the Workspace Server has a current Runtime observation for a Worker.\n\n`Unavailable` deliberately does not imply `Stopped`: it means the last observed\nlifecycle state is retained while the Runtime observation source is unavailable.","enum":["observed","unavailable"],"type":"string"},"SubscriptionWorkerId":{"type":"string"},"SubscriptionWorkerIds":{"description":"A bounded, canonical worker-id set. Ordering on the wire and in equality/hash\nsemantics is lexical, so equivalent selectors aggregate to one upstream key.","items":{"$ref":"#/$defs/SubscriptionWorkerId"},"type":"array"},"SubscriptionWorkerJob":{"additionalProperties":false,"properties":{"attempt_id":{"type":"string"},"job_id":{"type":"string"},"purpose":{"type":"string"}},"required":["job_id","attempt_id","purpose"],"type":"object"},"SubscriptionWorkerProtocolMethod":{"additionalProperties":false,"properties":{"method":{"$ref":"#/$defs/Method"},"subscription_id":{"$ref":"#/$defs/SubscriptionId"}},"required":["subscription_id","method"],"type":"object"},"SubscriptionWorkerState":{"enum":["idle","running","paused","stopped"],"type":"string"},"SubscriptionWorkerWorkdirAttachment":{"additionalProperties":false,"properties":{"alias":{"description":"Stable Worker-local routing alias.","type":"string"},"repository_key":{"type":["string","null"]},"working_directory_id":{"$ref":"#/$defs/SubscriptionWorkdirId"}},"required":["alias","working_directory_id"],"type":"object"},"SymlinkPolicy":{"description":"Symbolic-link identity used by one filesystem allow rule.\n\n`Resolved` is the least authority and the default: access is matched\nagainst the provider-resolved target. `Logical` intentionally grants the\npath as presented through the Workdir, even when it aliases another target.","enum":["resolved","logical"],"type":"string"},"ToolResultDisposition":{"enum":["success","error","interrupted","cancelled","outcome_unknown"],"type":"string"},"TurnResult":{"enum":["finished","paused"],"type":"string"},"UploadedFileAvailability":{"description":"Availability recorded for an uploaded client-local file.","enum":["available","unavailable","integrity_failed"],"type":"string"},"UploadedFileRef":{"additionalProperties":false,"description":"Session-owned immutable reference to a client-local uploaded file.\n\nUpload transports return an unbound reference. Worker fills\n`source_entry_id` immediately before the containing user input is committed;\ncommitted Session Log and public snapshot records therefore always retain\nthe durable source-entry identity without storing the file body.","properties":{"artifact_id":{"type":"string"},"availability":{"$ref":"#/$defs/UploadedFileAvailability"},"byte_len":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"created_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"file_name":{"type":"string"},"media_type":{"type":"string"},"sha256":{"type":"string"},"source_entry_id":{"type":["string","null"]}},"required":["artifact_id","file_name","media_type","created_at_ms","availability","byte_len","sha256"],"type":"object"},"WorkerBusyState":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"run","type":"string"},"state":{"$ref":"#/$defs/WorkerRunState"}},"required":["kind","state"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"maintenance","type":"string"},"state":{"$ref":"#/$defs/WorkerMaintenanceState"}},"required":["kind","state"],"type":"object"}]},"WorkerCommandAcknowledgement":{"additionalProperties":false,"properties":{"command":{"$ref":"#/$defs/WorkerCommandKind"},"command_id":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"disposition":{"$ref":"#/$defs/WorkerCommandDisposition"},"state":{"$ref":"#/$defs/WorkerStateSnapshot","description":"The complete authoritative state observed after command admission."}},"required":["command_id","command","disposition","state"],"type":"object"},"WorkerCommandDisposition":{"enum":["accepted","stale_command_id","conflict","invalid_state"],"type":"string"},"WorkerCommandEnvelope":{"additionalProperties":false,"description":"Caller-owned identity for one state-changing Worker command.\n\nA controller accepts command ids in strictly increasing order. Exact retries\nof an accepted id must retain the same command kind.","properties":{"command_id":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["command_id"],"type":"object"},"WorkerCommandKind":{"enum":["resume","cancel","pause","compact","shutdown"],"type":"string"},"WorkerEvent":{"description":"Typed lifecycle events sent from a child Worker to its parent.\n\nDelivered as `Method::WorkerEvent` over the parent's Unix socket. The\nparent Controller always applies variant-specific side effects\n(registry / worker-allocation updates). Agent-visible variants are also\nqueued into the notification buffer; control-plane-only variants are\nnot injected into the parent's LLM context.\n\nTransport is fire-and-forget; receivers must tolerate out-of-order\ndelivery (e.g. `TurnEnded` arriving after `ShutDown` for the same\nchild Worker).","oneOf":[{"additionalProperties":false,"description":"Child finished one turn and is back to IDLE.","properties":{"kind":{"const":"turn_ended","type":"string"},"worker_name":{"type":"string"}},"required":["kind","worker_name"],"type":"object"},{"additionalProperties":false,"description":"Engine execution error occurred inside the child's turn.\n\nLimited to worker runtime failures (provider / tool errors) —\ndoes not include transient method-rejection responses such as\n`AlreadyRunning`.","properties":{"kind":{"const":"errored","type":"string"},"message":{"type":"string"},"worker_name":{"type":"string"}},"required":["kind","worker_name","message"],"type":"object"},{"additionalProperties":false,"description":"Child has stopped (controller loop is exiting).","properties":{"kind":{"const":"shut_down","type":"string"},"worker_name":{"type":"string"}},"required":["kind","worker_name"],"type":"object"},{"additionalProperties":false,"description":"Child SubWorker sub-delegated scope to a grandchild SubWorker via `SubWorkerSpawn`.\n\nControl-plane only: receivers apply registry side effects and\npropagate upward, but do not expose this as an agent notification.\n\nThe parent uses this to add the grandchild to its own\n`spawned_workers.json` so it can manage the grandchild directly\neven if the intermediate child dies. The parent then re-fires\nthis event upward (if it has a parent of its own) to maintain\nthe chain to root.","properties":{"kind":{"const":"scope_sub_delegated","type":"string"},"parent_worker":{"description":"Sub-delegating Worker (= the sender itself).","type":"string"},"scope":{"description":"Scope delegated to the grandchild.","items":{"$ref":"#/$defs/ScopeRule"},"type":"array"},"sub_socket":{"description":"Unix-socket path where the grandchild is reachable.","type":"string"},"sub_worker":{"description":"Name of the grandchild Worker.","type":"string"}},"required":["kind","parent_worker","sub_worker","sub_socket","scope"],"type":"object"}]},"WorkerMaintenanceState":{"enum":["compacting"],"type":"string"},"WorkerRunState":{"enum":["running","pausing","paused","cancelling"],"type":"string"},"WorkerState":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"idle","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"busy","type":"string"},"state":{"$ref":"#/$defs/WorkerBusyState"}},"required":["kind","state"],"type":"object"}]},"WorkerStateSnapshot":{"additionalProperties":false,"properties":{"last_command_id":{"description":"Highest lifecycle command id observed by this controller instance.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"last_finished_submission_request_id":{"description":"Most recent Submit request whose controller Run has returned and whose\npost-Run state transition has been published. This is a teardown fence,\nnot a successful-result claim; cancelled and failed Runs may also finish.","type":["string","null"]},"state":{"$ref":"#/$defs/WorkerState"}},"required":["last_command_id","state"],"type":"object"},"WorkerStatus":{"enum":["idle","running","paused","stopped"],"type":"string"},"WorkspaceSubscriptionWorkdir":{"additionalProperties":false,"description":"Workspace-facing Workdir summary. Backend-generated Repository UUIDs never\nenter this DTO; Workspace Server must resolve the required Repository key.","properties":{"repository_key":{"type":"string"},"state":{"type":"string"},"working_directory_id":{"$ref":"#/$defs/SubscriptionWorkdirId"}},"required":["working_directory_id","repository_key","state"],"type":"object"}},"$schema":"https://json-schema.org/draft/2020-12/schema","description":"Direction is explicit at the outer envelope, while concrete message kinds\nremain typed within each lane.","oneOf":[{"additionalProperties":false,"properties":{"frame":{"const":"request","type":"string"},"message":{"$ref":"#/$defs/SubscriptionRequest"},"protocol_version":{"format":"uint16","maximum":65535.0,"minimum":0.0,"type":"integer"}},"required":["frame","message","protocol_version"],"type":"object"},{"additionalProperties":false,"properties":{"frame":{"const":"response","type":"string"},"message":{"$ref":"#/$defs/SubscriptionResponse"},"protocol_version":{"format":"uint16","maximum":65535.0,"minimum":0.0,"type":"integer"}},"required":["frame","message","protocol_version"],"type":"object"},{"additionalProperties":false,"properties":{"frame":{"const":"event","type":"string"},"message":{"$ref":"#/$defs/SubscriptionEvent"},"protocol_version":{"format":"uint16","maximum":65535.0,"minimum":0.0,"type":"integer"}},"required":["frame","message","protocol_version"],"type":"object"},{"additionalProperties":false,"properties":{"frame":{"const":"worker_protocol","type":"string"},"message":{"$ref":"#/$defs/SubscriptionWorkerProtocolMethod"},"protocol_version":{"format":"uint16","maximum":65535.0,"minimum":0.0,"type":"integer"}},"required":["frame","message","protocol_version"],"type":"object"}],"title":"SubscriptionFrame"}; +const subscriptionFrameSchema: unknown = {"$defs":{"Alert":{"additionalProperties":false,"description":"User-facing alert emitted from the Worker layer.\n\nThis is a separate channel from `tracing` (developer logs): entries\nhere are assembled explicitly by the Worker when a condition should be\nsurfaced to the person driving the client. Keep messages short and\nhuman-readable.","properties":{"level":{"$ref":"#/$defs/AlertLevel"},"message":{"type":"string"},"source":{"$ref":"#/$defs/AlertSource"},"timestamp_ms":{"description":"Milliseconds since the Unix epoch.","format":"int64","maximum":9007199254740991.0,"minimum":-9007199254740991.0,"type":"integer"}},"required":["level","source","message","timestamp_ms"],"type":"object"},"AlertLevel":{"enum":["warn","error"],"type":"string"},"AlertSource":{"enum":["worker","engine","compactor","agents_md"],"type":"string"},"CommandEvent":{"oneOf":[{"additionalProperties":false,"properties":{"command_id":{"type":"string"},"kind":{"const":"started","type":"string"},"observed_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"tool_call_id":{"type":["string","null"]}},"required":["kind","command_id","observed_at_ms"],"type":"object"},{"additionalProperties":false,"properties":{"command_id":{"type":"string"},"content":{"type":"string"},"end_offset":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"kind":{"const":"output","type":"string"},"observed_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"start_offset":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"stream":{"$ref":"#/$defs/CommandStream"}},"required":["kind","command_id","stream","start_offset","end_offset","content","observed_at_ms"],"type":"object"},{"additionalProperties":false,"properties":{"command_id":{"type":"string"},"exit_code":{"format":"int32","maximum":2147483647.0,"minimum":-2147483648.0,"type":["integer","null"]},"kind":{"const":"terminal","type":"string"},"observed_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"status":{"$ref":"#/$defs/CommandStatus"},"stderr_end_offset":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"stdout_end_offset":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","command_id","status","stdout_end_offset","stderr_end_offset","observed_at_ms"],"type":"object"}]},"CommandSnapshot":{"additionalProperties":false,"properties":{"command_id":{"type":"string"},"exit_code":{"format":"int32","maximum":2147483647.0,"minimum":-2147483648.0,"type":["integer","null"]},"last_output_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":["integer","null"]},"observed_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"started_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"status":{"$ref":"#/$defs/CommandStatus"},"stderr":{"$ref":"#/$defs/CommandStreamSlice"},"stdout":{"$ref":"#/$defs/CommandStreamSlice"},"tool_call_id":{"type":["string","null"]}},"required":["command_id","status","started_at_ms","observed_at_ms","stdout","stderr"],"type":"object"},"CommandStatus":{"description":"Live provider-owned command status. These values are operational Console\nstate only and are never appended to Worker history.","enum":["running","completed","failed","timed_out","cancelled"],"type":"string"},"CommandStream":{"enum":["stdout","stderr"],"type":"string"},"CommandStreamSlice":{"additionalProperties":false,"properties":{"content":{"type":"string"},"end_offset":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"start_offset":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"truncated":{"type":"boolean"}},"required":["start_offset","end_offset","content","truncated"],"type":"object"},"CompactionLifecycle":{"additionalProperties":false,"description":"Stable parent-owned lifecycle for one compaction run.","properties":{"compaction_id":{"type":"string"},"ended_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":["integer","null"]},"error":{"type":["string","null"]},"internal_worker":{"anyOf":[{"$ref":"#/$defs/InternalWorkerRef"},{"type":"null"}]},"new_segment_id":{"type":["string","null"]},"revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"schema_version":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"started_at_ms":{"description":"Milliseconds since the Unix epoch.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"state":{"$ref":"#/$defs/CompactionLifecycleState"},"summary":{"type":["string","null"]}},"required":["schema_version","compaction_id","revision","state","started_at_ms"],"type":"object"},"CompactionLifecycleState":{"enum":["running","done","failed","interrupted"],"type":"string"},"CompactionPhase":{"enum":["preparing","summarizing","committing"],"type":"string"},"CompactionTrigger":{"enum":["manual","pre_run","request_threshold"],"type":"string"},"CompletionContext":{"additionalProperties":false,"properties":{"argument":{"type":["string","null"]},"invocation":{"$ref":"#/$defs/FeatureInvocationIdentity"}},"required":["invocation"],"type":"object"},"CompletionEntry":{"additionalProperties":false,"description":"One completion candidate for a prefix query.","properties":{"description":{"type":["string","null"]},"invocation":{"anyOf":[{"$ref":"#/$defs/FeatureInvocationDescriptor"},{"type":"null"}]},"is_dir":{"default":false,"type":"boolean"},"usage":{"type":["string","null"]},"value":{"type":"string"}},"required":["value","is_dir"],"type":"object"},"CompletionKind":{"description":"Kind of completion requested by `Method::ListCompletions`.\n\n`File` is the existing `@` path lane. `Feature` lists installed chat\ninvocations. `FeatureArgument` resolves a declared argument using the\nsource-qualified [`CompletionContext`].","enum":["file","feature","feature_argument"],"type":"string"},"ContextTokenSource":{"description":"Provenance of the current-context token count in a [`Greeting`].","oneOf":[{"const":"measured","description":"Provider-reported input occupancy from the most recent LLM request.","type":"string"},{"const":"estimated","description":"Current history projected from the Worker's token accounting records.","type":"string"}]},"ContextUsage":{"additionalProperties":false,"description":"Current prompt-context occupancy, distinct from cumulative run traffic.","properties":{"source":{"$ref":"#/$defs/ContextTokenSource"},"tokens":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["tokens","source"],"type":"object"},"ErrorCode":{"enum":["already_running","not_running","not_paused","provider_error","tool_error","invalid_request","internal"],"type":"string"},"Event":{"oneOf":[{"additionalProperties":false,"description":"Durable Submit acceptance. A `Queued` receipt follows the durable FIFO\ncheckpoint. A `Started` receipt follows the pending activation checkpoint;\nthe corresponding [`Event::UserMessage`] marks the later history commit.\nRepeating the same request id and exact payload returns the same receipt\nwithout appending or activating twice.","properties":{"data":{"additionalProperties":false,"properties":{"disposition":{"$ref":"#/$defs/SubmissionDisposition"},"submission_id":{"type":"string"},"submission_request_id":{"type":"string"}},"required":["submission_request_id","submission_id","disposition"],"type":"object"},"event":{"const":"submission_accepted","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Correlated rejection before durable acceptance.","properties":{"data":{"additionalProperties":false,"properties":{"message":{"type":"string"},"submission_request_id":{"type":"string"}},"required":["submission_request_id","message"],"type":"object"},"event":{"const":"submission_rejected","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Durable notification acceptance. The request identity is persisted with\nthe notification receipt before this event is emitted. Repeating the\nsame request id and exact payload returns the same acknowledgement\nwithout appending the notification twice.","properties":{"data":{"additionalProperties":false,"properties":{"notification_request_id":{"type":"string"}},"required":["notification_request_id"],"type":"object"},"event":{"const":"notification_accepted","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Correlated notification rejection before durable acceptance.","properties":{"data":{"additionalProperties":false,"properties":{"message":{"type":"string"},"notification_request_id":{"type":"string"}},"required":["notification_request_id","message"],"type":"object"},"event":{"const":"notification_rejected","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Revisioned FIFO replacement following enqueue, activation, cancel, or clear.","properties":{"data":{"additionalProperties":false,"properties":{"pending":{"$ref":"#/$defs/PendingSubmissionsSnapshot"}},"required":["pending"],"type":"object"},"event":{"const":"pending_submissions_changed","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A user input message persisted as `LogEntry::AnnotatedUserInput` and\nactivated for a turn. Broadcast to every subscribed client so TUI / GUI\ninstances show the same user line that reconnect snapshots replay.","properties":{"data":{"additionalProperties":false,"properties":{"entry_id":{"description":"Stable durable history identity. Older mixed-version producers omit it.","type":["string","null"]},"segments":{"items":{"$ref":"#/$defs/Segment"},"type":"array"}},"required":["segments"],"type":"object"},"event":{"const":"user_message","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A canonical assistant/tool entry has committed to durable history.\nStreaming events remain the in-flight authority; this event supplies the\nstable identity used to reconcile that live block with paged history.","properties":{"data":{"additionalProperties":false,"properties":{"entry":{"$ref":"#/$defs/SessionSnapshotEntry"}},"required":["entry"],"type":"object"},"event":{"const":"session_entry_committed","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"One agent-injected system item committed to history.\n\nCarries the JSON form of `session_store::SystemItem`. Covers\n`Method::Notify` echoes, child-Worker lifecycle events from\n`Method::WorkerEvent`, `@` resolution payloads, and any future\nClients dispatch on the `kind` tag for typed rendering instead\nof parsing free-text prefixes like `[Notification] …` or\n`[File: …]`.\n\nOne event per `LogEntry::AnnotatedSystemItem` commit. Disk-side and\nwire-side are 1:1.","properties":{"data":{"additionalProperties":false,"properties":{"entry_id":{"description":"Stable durable history identity. Synthetic compatibility events omit it.","type":["string","null"]},"item":true},"required":["item"],"type":"object"},"event":{"const":"system_item","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A new self-driving cycle has begun (IDLE → active transition).\n\nMarker event for the start of an Invoke range; the range extends\nimplicitly until the next `InvokeStart`. Fires for every accepted\n`Method::Submit` (kind=`UserSend`), `Method::Notify` (kind=`Notify`),\n`Method::WorkerEvent` re-injection (kind=`WorkerEvent`), and any other\nIDLE-breaking trigger. Mid-run interrupts (e.g. hook output,\ntyped system reminder insertion that doesn't break IDLE) do not\nemit `InvokeStart` — they appear as `SystemItem` only.\n\nCarries `kind` only; the payload (user text / notify message /\nworker event body) is delivered separately via the immediately\nfollowing `UserMessage` / `SystemItem` event.","properties":{"data":{"additionalProperties":false,"properties":{"kind":{"$ref":"#/$defs/InvokeKind"}},"required":["kind"],"type":"object"},"event":{"const":"invoke_start","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"One AgentTurn boundary opened. An AgentTurn is a maximal run of\nLLM generation calls whose input messages are identical (i.e.\nretries from network errors / 5xx / stream disconnects collapse\ninto the same AgentTurn). When the input changes (a new tool\nresult lands, a user interrupts, etc.), the next LLM call belongs\nto a new AgentTurn.\n\n`turn` is the AgentTurn index from `Engine::turn_count`.\n\nCurrently retry is not yet implemented (`agen-stream-continuation`)\nso AgentTurn and `LlmCall` fire 1:1, but the contract here is\nthe AgentTurn boundary; consumers that want per-LLM-call signals\nmust subscribe to `LlmCallStart` / `LlmCallEnd` instead.","properties":{"data":{"additionalProperties":false,"properties":{"turn":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["turn"],"type":"object"},"event":{"const":"turn_start","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"AgentTurn closed.","properties":{"data":{"additionalProperties":false,"properties":{"result":{"$ref":"#/$defs/TurnResult"},"turn":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["turn","result"],"type":"object"},"event":{"const":"turn_end","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"One LLM generation call started (1 request → 1 generation, retry\nincluded). Multiple `LlmCall*` pairs may fire inside a single\n`TurnStart` / `TurnEnd` pair when a request is retried; today\nthey fire 1:1 because retry is not implemented.\n\n`llm_call` is the worker-wide running counter from\n`Engine::llm_call_count`.","properties":{"data":{"additionalProperties":false,"properties":{"llm_call":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["llm_call"],"type":"object"},"event":{"const":"llm_call_start","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"LLM generation call ended.","properties":{"data":{"additionalProperties":false,"properties":{"llm_call":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["llm_call"],"type":"object"},"event":{"const":"llm_call_end","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A transport-level LLM request retry has been scheduled.\n\nThis is operational state for clients to render while the worker is\nwaiting in backoff. It is not part of conversation history.","properties":{"data":{"additionalProperties":false,"properties":{"elapsed_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"error":{"type":"string"},"failed_attempt":{"description":"The attempt that just failed. 1 origin.","format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"llm_call":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"max_attempts":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"status":{"format":"uint16","maximum":65535.0,"minimum":0.0,"type":["integer","null"]},"wait_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["llm_call","failed_attempt","max_attempts","wait_ms","elapsed_ms","error"],"type":"object"},"event":{"const":"llm_retry","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Stream generation was interrupted after events had begun and the worker\nis continuing with a follow-up LLM request.","properties":{"data":{"additionalProperties":false,"properties":{"attempt":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"llm_call":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"max_attempts":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"reason":{"type":"string"}},"required":["llm_call","attempt","max_attempts","reason"],"type":"object"},"event":{"const":"llm_continuation","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"text":{"type":"string"}},"required":["text"],"type":"object"},"event":{"const":"text_delta","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"text":{"type":"string"}},"required":["text"],"type":"object"},"event":{"const":"text_done","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A reasoning / thinking block has started.\n\nAlways paired with a `ThinkingDone`. `ThinkingDelta` is optional:\nsome providers (or some configurations) emit thinking metadata\nwithout plaintext, in which case Start → Done arrive with no\ndeltas in between. Multiple thinking blocks per turn are allowed.","properties":{"event":{"const":"thinking_start","type":"string"}},"required":["event"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"text":{"type":"string"}},"required":["text"],"type":"object"},"event":{"const":"thinking_delta","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Thinking block completed. `text` is the full accumulated body\n(empty string when the provider didn't emit plaintext).","properties":{"data":{"additionalProperties":false,"properties":{"text":{"type":"string"}},"required":["text"],"type":"object"},"event":{"const":"thinking_done","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"id":{"type":"string"},"name":{"type":"string"}},"required":["id","name"],"type":"object"},"event":{"const":"tool_call_start","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"id":{"type":"string"},"json":{"type":"string"}},"required":["id","json"],"type":"object"},"event":{"const":"tool_call_args_delta","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"arguments":{"type":"string"},"id":{"type":"string"},"name":{"type":"string"}},"required":["id","name","arguments"],"type":"object"},"event":{"const":"tool_call_done","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"disposition":{"anyOf":[{"$ref":"#/$defs/ToolResultDisposition"},{"type":"null"}]},"id":{"type":"string"},"is_error":{"default":false,"type":"boolean"},"output":{"description":"Full tool output. Absent when the tool chose to return\nsummary-only, or when the result was pruned.","type":["string","null"]},"summary":{"description":"Short human-readable summary. Always present; used by clients\nthat only want a 1-line rendering (e.g. collapsed views).","type":"string"}},"required":["id","summary","is_error"],"type":"object"},"event":{"const":"tool_result","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Token accounting for one LLM request.\n\n`input_tokens` is the prompt prefix occupancy (cache reads /\ncache writes included), as normalised by the worker layer.\n`cache_read_input_tokens` is the cache-hit subset of that\noccupancy; subtracting it yields the \"net upload\" the client\nactually paid full price to send on this request, which is what\nthe TUI status line accumulates per turn.","properties":{"data":{"additionalProperties":false,"properties":{"cache_read_input_tokens":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":["integer","null"]},"input_tokens":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":["integer","null"]},"output_tokens":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":["integer","null"]}},"type":"object"},"event":{"const":"usage","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Authoritative current-context occupancy after a non-request boundary.\n\nUnlike [`Event::Usage`], this is not cumulative request traffic. It lets\nconnected clients converge with reconnect snapshots after run completion,\ncompaction, rewind, and other history rewrites. `None` explicitly means\nthe current value is unavailable and stale occupancy must be cleared.","properties":{"data":{"additionalProperties":false,"properties":{"usage":{"anyOf":[{"$ref":"#/$defs/ContextUsage"},{"type":"null"}]}},"type":"object"},"event":{"const":"context_usage","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"result":{"$ref":"#/$defs/RunResult"}},"required":["result"],"type":"object"},"event":{"const":"run_end","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"code":{"$ref":"#/$defs/ErrorCode"},"message":{"type":"string"}},"required":["code","message"],"type":"object"},"event":{"const":"error","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Sent exactly once at the start of every client connection.\n\n`entries` is the session-log mirror at subscribe time, serialised\nas the JSON form of `session_store::LogEntry`. This is the\nbulk-reconstruction lane: clients walk the entries to seed their\nderived view.\n\n`greeting` and `status` accompany the snapshot so clients render\nworker identity and current controller state without an extra round\ntrip.\n\nLive updates after the snapshot arrive through the streaming\nevents (`TextDelta` / `ToolCall*` / `ToolResult` / etc.) plus\nrole-specific entry events (`SegmentRotated` / `SystemItem`) —\nthere is no generic \"every committed entry\" broadcast.","properties":{"data":{"additionalProperties":false,"properties":{"greeting":{"$ref":"#/$defs/Greeting"},"in_flight":{"$ref":"#/$defs/InFlightSnapshot","description":"Unfinished model output that has already streamed in the current\nrun but is not yet represented by committed snapshot entries."},"internal_workers":{"description":"Parent-owned Internal Worker sessions visible to this client.\nService-private Internal Workers are deliberately excluded.","items":{"$ref":"#/$defs/InternalWorkerSnapshot"},"type":"array"},"session":{"$ref":"#/$defs/SessionSnapshot"},"state":{"$ref":"#/$defs/WorkerStateSnapshot","description":"Full revisioned live execution state. `Stopped` remains Runtime\ncatalog authority and is deliberately not represented here."}},"required":["session","greeting","state"],"type":"object"},"event":{"const":"snapshot","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A live event from a parent-owned Internal Worker. The payload reuses the\nnormal Worker event vocabulary while the wrapper carries stable origin\nidentity and a per-child revision fence.","properties":{"data":{"additionalProperties":false,"properties":{"event":{"$ref":"#/$defs/Event"},"revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"worker":{"$ref":"#/$defs/InternalWorkerRef"}},"required":["worker","revision","event"],"type":"object"},"event":{"const":"internal_worker","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Terminal removal fence for one parent-owned Internal Worker session.\n\nClients discard the matching child and descendants, then ignore later\nnested events for this identity until an authoritative snapshot replaces\nthe projection.","properties":{"data":{"additionalProperties":false,"properties":{"revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"worker":{"$ref":"#/$defs/InternalWorkerRef"}},"required":["worker","revision"],"type":"object"},"event":{"const":"internal_worker_removed","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Server-side segment log rotated to a fresh `SegmentStart`.\n\nFires on compaction and on auto-fork when the store head drifts\nA compaction/fork has replaced the authoritative segment. Clients drop\ntheir derived view and reseed from the canonical committed snapshot.","properties":{"data":{"additionalProperties":false,"properties":{"session":{"$ref":"#/$defs/SessionSnapshot"}},"required":["session"],"type":"object"},"event":{"const":"segment_rotated","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Current Worker controller status. Broadcast on every controller-level\ntransition and included in `History` snapshots for late attach.","properties":{"data":{"additionalProperties":false,"properties":{"snapshot":{"$ref":"#/$defs/WorkerStateSnapshot"}},"required":["snapshot"],"type":"object"},"event":{"const":"worker_state","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"acknowledgement":{"$ref":"#/$defs/WorkerCommandAcknowledgement"}},"required":["acknowledgement"],"type":"object"},"event":{"const":"command_acknowledged","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Bounded, provider-owned command telemetry for the live Console. This is\nintentionally not a history entry and is reconstructed from\n`Snapshot.in_flight.commands` after reconnect.","properties":{"data":{"additionalProperties":false,"properties":{"event":{"$ref":"#/$defs/CommandEvent"}},"required":["event"],"type":"object"},"event":{"const":"command","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Reply to `Method::ListCompletions`. Delivered only to the\nrequesting socket (not broadcast). `entries` is empty when no\ncandidates match or when the requested kind has no resolver\nwired up.","properties":{"data":{"additionalProperties":false,"properties":{"context":{"anyOf":[{"$ref":"#/$defs/CompletionContext"},{"type":"null"}]},"entries":{"items":{"$ref":"#/$defs/CompletionEntry"},"type":"array"},"kind":{"$ref":"#/$defs/CompletionKind"},"prefix":{"default":"","type":"string"},"request_id":{"type":["string","null"]}},"required":["kind","entries","prefix"],"type":"object"},"event":{"const":"completions","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Reply to `Method::ListRewindTargets`. Clients should only open a picker\nin response to their own pending request; the event may be broadcast.","properties":{"data":{"additionalProperties":false,"properties":{"head_entries":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"targets":{"items":{"$ref":"#/$defs/RewindTarget"},"type":"array"}},"required":["head_entries","targets"],"type":"object"},"event":{"const":"rewind_targets","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"A rewind has truncated the authoritative session. `session` is the\nretained canonical snapshot clients should use to reseed display state.","properties":{"data":{"additionalProperties":false,"properties":{"input":{"items":{"$ref":"#/$defs/Segment"},"type":"array"},"session":{"$ref":"#/$defs/SessionSnapshot"},"summary":{"$ref":"#/$defs/RewindSummary"}},"required":["session","input","summary"],"type":"object"},"event":{"const":"rewind_applied","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Reply to `Method::ListWorkers`. Payload is a stable JSON value so the Worker\ncrate can evolve discovery fields without introducing a protocol\ndependency on session-store.","properties":{"data":{"additionalProperties":false,"properties":{"workers":true},"required":["workers"],"type":"object"},"event":{"const":"workers_listed","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Reply to `Method::RestoreWorker`.","properties":{"data":{"additionalProperties":false,"properties":{"result":true},"required":["result"],"type":"object"},"event":{"const":"worker_restored","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Reply to `Method::RegisterPeer`.","properties":{"data":{"additionalProperties":false,"properties":{"result":true},"required":["result"],"type":"object"},"event":{"const":"peer_registered","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"$ref":"#/$defs/Alert"},"event":{"const":"alert","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Latest memory extract/consolidation lifecycle event for UI observability.\n\nThis is not part of LLM history or prompt context; clients may display it\nbriefly as operational status.","properties":{"data":{"$ref":"#/$defs/MemoryWorkerEvent"},"event":{"const":"memory_worker","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Runtime-only compaction progress. `None` clears the current status.\nThis never enters Session history and carries no operation or Segment identity.","properties":{"data":{"additionalProperties":false,"properties":{"compaction":{"anyOf":[{"$ref":"#/$defs/InFlightCompaction"},{"type":"null"}]}},"type":"object"},"event":{"const":"compaction_progress","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Legacy compaction lifecycle event retained for wire read compatibility.","properties":{"data":{"additionalProperties":false,"properties":{"lifecycle":{"$ref":"#/$defs/CompactionLifecycle"}},"required":["lifecycle"],"type":"object"},"event":{"const":"compact_start","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Compaction completed and the session was rotated.","properties":{"data":{"additionalProperties":false,"properties":{"lifecycle":{"$ref":"#/$defs/CompactionLifecycle"}},"required":["lifecycle"],"type":"object"},"event":{"const":"compact_done","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"description":"Compaction failed or was cancelled. The session is unchanged.","properties":{"data":{"additionalProperties":false,"properties":{"lifecycle":{"$ref":"#/$defs/CompactionLifecycle"}},"required":["lifecycle"],"type":"object"},"event":{"const":"compact_failed","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"event":{"const":"shutdown","type":"string"}},"required":["event"],"type":"object"}]},"EventSubscriptionSelector":{"oneOf":[{"additionalProperties":false,"properties":{"topic":{"const":"runtime_workers","type":"string"}},"required":["topic"],"type":"object"},{"additionalProperties":false,"properties":{"topic":{"const":"worker_lifecycle","type":"string"},"worker_ids":{"$ref":"#/$defs/SubscriptionWorkerIds"}},"required":["topic","worker_ids"],"type":"object"},{"additionalProperties":false,"properties":{"runtime_id":{"type":["string","null"]},"topic":{"const":"worker_protocol","type":"string"},"worker_id":{"$ref":"#/$defs/SubscriptionWorkerId"}},"required":["topic","worker_id"],"type":"object"},{"additionalProperties":false,"description":"Server-derived Workspace projection. Workspace identity comes from the\nauthenticated connection and is deliberately absent from this selector.","properties":{"topic":{"const":"workspace_workers","type":"string"}},"required":["topic"],"type":"object"},{"additionalProperties":false,"description":"Typed extension point for the later Workdir subscription slice.","properties":{"topic":{"const":"workspace_workdirs","type":"string"}},"required":["topic"],"type":"object"}]},"FeatureInvocation":{"additionalProperties":false,"properties":{"arguments":{"items":{"$ref":"#/$defs/InvocationArgumentValue"},"type":"array"},"identity":{"$ref":"#/$defs/FeatureInvocationIdentity"},"invocation_id":{"description":"Stable client-generated id retained across queueing and transport retry.","type":"string"},"name":{"description":"Selected public name, retained for exact draft/history restoration. Host\nresolution is always by `identity`.","type":"string"}},"required":["invocation_id","identity","name","arguments"],"type":"object"},"FeatureInvocationDescriptor":{"additionalProperties":false,"properties":{"aliases":{"default":[],"items":{"type":"string"},"type":"array"},"arguments":{"default":[],"items":{"$ref":"#/$defs/InvocationArgumentDescriptor"},"type":"array"},"client_adapter":{"anyOf":[{"$ref":"#/$defs/InvocationClientAdapter"},{"type":"null"}]},"description":{"type":"string"},"display_name":{"type":"string"},"identity":{"$ref":"#/$defs/FeatureInvocationIdentity"},"name":{"type":"string"},"syntax":{"$ref":"#/$defs/FeatureInvocationSyntax"}},"required":["identity","name","display_name","description","syntax","aliases","arguments"],"type":"object"},"FeatureInvocationIdentity":{"description":"Source-qualified identity of one invocation contribution, for example\n`builtin:attachments/attach`. Display names and aliases are never authority.","type":"string"},"FeatureInvocationSyntax":{"oneOf":[{"const":"parenthesized","description":"`/name(value, key=value)` with JSON-style quoted strings and `)` as the\nexplicit boundary before following natural language.","type":"string"}]},"Greeting":{"additionalProperties":false,"description":"Worker self-description rendered by clients when a session starts empty.\n\nSnapshot metadata contains only resolved public execution information so\nclients do not need their own copy of the manifest.","properties":{"context_tokens":{"default":0,"description":"Legacy numeric mirror retained for mixed-version clients.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"context_usage":{"anyOf":[{"$ref":"#/$defs/ContextUsage"},{"type":"null"}],"description":"Typed current-context value. Older snapshots omit it; clients may fall\nback to `context_tokens` only when `context_window` is known."},"context_window":{"default":0,"description":"Resolved effective model context window in tokens. Legacy producers use\nzero when this is unavailable; clients must treat that as unknown.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"cwd":{"type":"string"},"model":{"type":"string"},"provider":{"type":"string"},"reasoning":{"anyOf":[{"$ref":"#/$defs/ReasoningConfig"},{"type":"null"}],"description":"Effective reasoning control. Absent means unspecified or unavailable."},"scope_summary":{"type":"string"},"tools":{"items":{"type":"string"},"type":"array"},"worker_name":{"type":"string"}},"required":["worker_name","cwd","provider","model","scope_summary","tools","context_tokens","context_window"],"type":"object"},"InFlightBlock":{"oneOf":[{"additionalProperties":false,"properties":{"finished":{"type":"boolean"},"kind":{"const":"text","type":"string"},"text":{"type":"string"}},"required":["kind","text"],"type":"object"},{"additionalProperties":false,"properties":{"finished":{"type":"boolean"},"kind":{"const":"thinking","type":"string"},"text":{"type":"string"}},"required":["kind","text"],"type":"object"},{"additionalProperties":false,"properties":{"args":{"type":"string"},"id":{"type":"string"},"kind":{"const":"tool_call","type":"string"},"name":{"type":"string"},"state":{"$ref":"#/$defs/InFlightToolCallState"}},"required":["kind","id","name","args"],"type":"object"}]},"InFlightCompaction":{"additionalProperties":false,"properties":{"phase":{"$ref":"#/$defs/CompactionPhase"},"started_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"trigger":{"$ref":"#/$defs/CompactionTrigger"}},"required":["phase","started_at_ms","trigger"],"type":"object"},"InFlightSnapshot":{"additionalProperties":false,"description":"Unfinished model output and active command state included in\n`Event::Snapshot` for clients that attach while work is still streaming.\n\nThese blocks are presentation state only: they are reconstructed from the\nactive Worker controller and must not be treated as committed assistant\nhistory. Finalized assistant items continue to come from ordinary snapshot\nentries.","properties":{"blocks":{"items":{"$ref":"#/$defs/InFlightBlock"},"type":"array"},"commands":{"items":{"$ref":"#/$defs/CommandSnapshot"},"type":"array"},"compaction":{"anyOf":[{"$ref":"#/$defs/InFlightCompaction"},{"type":"null"}],"description":"The currently running compaction, if any.\n\nThis is lifecycle progress only. Candidate history and the staged\nSegment remain private until the Segment is activated atomically."}},"type":"object"},"InFlightToolCallState":{"enum":["pending","streaming_args","done"],"type":"string"},"InternalWorkerKind":{"description":"Presentation category for an Internal Worker exposed through its parent's\nprotocol stream. Internal Workers never become independently addressable\nprotocol subjects.","oneOf":[{"enum":["sub_worker"],"type":"string"},{"additionalProperties":false,"properties":{"service":{"additionalProperties":false,"properties":{"kind":{"type":"string"}},"required":["kind"],"type":"object"}},"required":["service"],"type":"object"}]},"InternalWorkerRef":{"additionalProperties":false,"description":"Stable presentation identity for one parent-owned Internal Worker session.\n`name` is display-only; `session_id` is the identity used by clients.","properties":{"kind":{"$ref":"#/$defs/InternalWorkerKind"},"name":{"type":"string"},"parent_session_id":{"type":["string","null"]},"session_id":{"type":"string"}},"required":["session_id","name","kind"],"type":"object"},"InternalWorkerSnapshot":{"additionalProperties":false,"description":"Reconnect state for one visible Internal Worker. The revision fences live\n`Event::InternalWorker` updates that raced with parent snapshot assembly.","properties":{"error":{"type":["string","null"]},"greeting":{"anyOf":[{"$ref":"#/$defs/Greeting"},{"type":"null"}],"description":"Public execution metadata for this child. Older snapshots and service\nworkers may omit it; clients must never substitute the parent's values."},"in_flight":{"$ref":"#/$defs/InFlightSnapshot"},"internal_workers":{"items":{"$ref":"#/$defs/InternalWorkerSnapshot"},"type":"array"},"revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"session":{"$ref":"#/$defs/SessionSnapshot"},"status":{"$ref":"#/$defs/WorkerStatus","default":"idle"},"worker":{"$ref":"#/$defs/InternalWorkerRef"}},"required":["worker","revision","session","status"],"type":"object"},"InvocationArgumentDescriptor":{"additionalProperties":false,"properties":{"completion":{"$ref":"#/$defs/InvocationCompletion","default":{"kind":"none"}},"description":{"type":["string","null"]},"name":{"type":"string"},"position":{"description":"Zero-based positional index. `None` means named-only. A positional\nargument may also be supplied by name; this makes chip editing stable.","format":"uint16","maximum":65535.0,"minimum":0.0,"type":["integer","null"]},"required":{"default":false,"type":"boolean"},"value_type":{"$ref":"#/$defs/InvocationArgumentType"}},"required":["name","value_type","completion","required"],"type":"object"},"InvocationArgumentType":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"string","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"integer","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"boolean","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"enum","type":"string"},"values":{"items":{"type":"string"},"type":"array"}},"required":["kind","values"],"type":"object"},{"additionalProperties":false,"description":"A Worker-readable path. This does not grant filesystem authority.","properties":{"kind":{"const":"worker_file","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"description":"A resource selected and staged by the client. It must never be resolved\nas a path on the Worker host.","properties":{"kind":{"const":"client_file","type":"string"}},"required":["kind"],"type":"object"}]},"InvocationArgumentValue":{"additionalProperties":false,"properties":{"name":{"type":"string"},"value":{"$ref":"#/$defs/InvocationValue"}},"required":["name","value"],"type":"object"},"InvocationClientAdapter":{"oneOf":[{"const":"attachment","description":"The client stages a local file and replaces the draft invocation with an\n`UploadedFile` chip. Local paths are not sent to or persisted by Worker.","type":"string"}]},"InvocationCompletion":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"none","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"static","type":"string"},"values":{"items":{"type":"string"},"type":"array"}},"required":["kind","values"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"worker_file","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"client_file","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"description":"Host-owned provider selected by source-qualified id. Metadata lookup is\nside-effect free; implementations never cross this wire boundary.","properties":{"kind":{"const":"provider","type":"string"},"provider":{"type":"string"}},"required":["kind","provider"],"type":"object"}]},"InvocationValue":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"string","type":"string"},"value":{"type":"string"}},"required":["kind","value"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"integer","type":"string"},"value":{"format":"int32","maximum":2147483647.0,"minimum":-2147483648.0,"type":"integer"}},"required":["kind","value"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"boolean","type":"string"},"value":{"type":"boolean"}},"required":["kind","value"],"type":"object"}]},"InvokeKind":{"description":"Kind of trigger that opened a new Invoke (IDLE → active) range.\n\nOne Invoke groups all entries from this trigger up to the next\n`Invoke` marker. The kind is the only payload — content (user text,\nnotify message, worker event body) is delivered by the immediately\nfollowing Turn entry, not by the marker itself.","oneOf":[{"const":"user_send","description":"`Method::Submit` — a user submission.","type":"string"},{"const":"notify","description":"`Method::Notify` — free-text notification injected into history.","type":"string"},{"const":"worker_event","description":"`Method::WorkerEvent` — typed lifecycle report from a child Worker.","type":"string"},{"const":"system_reminder","description":"A typed system reminder that crosses an IDLE boundary (mid-run\nreminders that don't break IDLE are SystemItem-only and do not\nopen a new Invoke).","type":"string"},{"const":"wakeup","description":"Cron / RemoteTrigger style scheduled wake-up. Reserved; no\nproducer is wired today but the variant is part of the initial\nset so future schedulers don't have to amend the wire enum.","type":"string"}]},"MemoryWorkerEvent":{"additionalProperties":false,"properties":{"message":{"description":"Human-readable compact form for actionbar rendering.","type":"string"},"reason":{"type":"string"},"run_id":{"type":"string"},"status":{"type":"string"},"timestamp_ms":{"description":"Milliseconds since the Unix epoch.","format":"int64","maximum":9007199254740991.0,"minimum":-9007199254740991.0,"type":"integer"},"trigger":{"type":"string"},"worker":{"type":"string"}},"required":["worker","status","run_id","trigger","reason","message","timestamp_ms"],"type":"object"},"Method":{"oneOf":[{"additionalProperties":false,"description":"Durably accept typed input for immediate activation or the session FIFO.\nRunning or Paused Workers queue Submit for a later turn; use Notify for\nadvisory information to incorporate into work already in progress.\n\n`submission_request_id` is generated by the authenticated caller and is\nused only for idempotent retry. Worker allocates the durable\n`submission_id` returned by [`Event::SubmissionAccepted`].","properties":{"method":{"const":"submit","type":"string"},"params":{"additionalProperties":false,"properties":{"input":{"items":{"$ref":"#/$defs/Segment"},"type":"array"},"submission_request_id":{"type":"string"}},"required":["submission_request_id","input"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Human-readable text injected into the target Worker's LLM context as a\nnon-blocking system message. Accepted notifications are delivered at the\nnext between-turn boundary, or start a coalesced notification run while\nthe Worker is idle. No side effects beyond LLM context; use\n`WorkerEvent` for typed lifecycle reports.","properties":{"method":{"const":"notify","type":"string"},"params":{"additionalProperties":false,"properties":{"message":{"type":"string"},"notification_request_id":{"type":"string"}},"required":["notification_request_id","message"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Typed lifecycle report from a child Worker to its direct parent.","properties":{"method":{"const":"worker_event","type":"string"},"params":{"$ref":"#/$defs/WorkerEvent"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Return the authoritative FIFO summary without exposing queued payloads.","properties":{"method":{"const":"list_pending_submissions","type":"string"}},"required":["method"],"type":"object"},{"additionalProperties":false,"description":"Remove one queued submission. Running or already activated submissions\nare immutable and therefore cannot be cancelled here.","properties":{"method":{"const":"cancel_pending_submission","type":"string"},"params":{"additionalProperties":false,"properties":{"expected_revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"submission_id":{"type":"string"}},"required":["submission_id","expected_revision"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Remove every queued submission while preserving the active run.","properties":{"method":{"const":"clear_pending_submissions","type":"string"},"params":{"additionalProperties":false,"properties":{"expected_revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["expected_revision"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Activate the next queued submission while the Worker is idle. This is an\nexplicit recovery operation and never resumes a paused run implicitly.","properties":{"method":{"const":"continue_pending","type":"string"},"params":{"additionalProperties":false,"properties":{"expected_head_id":{"type":"string"},"expected_revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["expected_revision","expected_head_id"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"properties":{"method":{"const":"resume","type":"string"},"params":{"additionalProperties":false,"properties":{"command":{"$ref":"#/$defs/WorkerCommandEnvelope"}},"required":["command"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"properties":{"method":{"const":"cancel","type":"string"},"params":{"additionalProperties":false,"properties":{"command":{"$ref":"#/$defs/WorkerCommandEnvelope"}},"required":["command"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Stop the in-flight turn and transition to `Paused`.\n\nUnlike `Cancel` (which discards and returns to `Idle`), a paused\nWorker can resume the interrupted work via `Resume`, or accept a\nfresh `Submit` (orphan `tool_use` items are closed with a\nsynthetic tool result before the new user message is appended).","properties":{"method":{"const":"pause","type":"string"},"params":{"additionalProperties":false,"properties":{"command":{"$ref":"#/$defs/WorkerCommandEnvelope"}},"required":["command"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Request an explicit compaction while the Worker is otherwise idle.\n\nThis is a typed control method: clients must not send `compact` as a\n`Method::Submit` user message.","properties":{"method":{"const":"compact","type":"string"},"params":{"additionalProperties":false,"properties":{"command":{"$ref":"#/$defs/WorkerCommandEnvelope"}},"required":["command"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Ask the Worker to list valid rewind targets from its authoritative session log.","properties":{"method":{"const":"list_rewind_targets","type":"string"}},"required":["method"],"type":"object"},{"additionalProperties":false,"description":"Truncate the current session back to the selected rewind target and\nreturn the selected user input to the client composer.","properties":{"method":{"const":"rewind_to","type":"string"},"params":{"additionalProperties":false,"properties":{"expected_head_entries":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"target":{"$ref":"#/$defs/RewindTargetId"}},"required":["target","expected_head_entries"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"properties":{"method":{"const":"shutdown","type":"string"},"params":{"additionalProperties":false,"properties":{"command":{"$ref":"#/$defs/WorkerCommandEnvelope"}},"required":["command"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Request a list of completion candidates from the Worker.\n\nReply is sent on the same socket as `Event::Completions` (not\nbroadcast). The IPC server handles this directly and writes\nthe response straight back to the requesting socket.","properties":{"method":{"const":"list_completions","type":"string"},"params":{"additionalProperties":false,"properties":{"context":{"anyOf":[{"$ref":"#/$defs/CompletionContext"},{"type":"null"}]},"kind":{"$ref":"#/$defs/CompletionKind"},"prefix":{"type":"string"},"request_id":{"description":"Client query generation. Echoed verbatim so identical-prefix ABA\nresponses cannot cross edits, target switches, or permission changes.","type":["string","null"]}},"required":["kind","prefix"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"List Workers visible to this Worker from durable Worker state and the spawned-child\nregistry. This is not a host-wide Worker universe query.","properties":{"method":{"const":"list_workers","type":"string"}},"required":["method"],"type":"object"},{"additionalProperties":false,"description":"Restore a visible stopped/restorable Worker, or report that it is already\nlive. Missing state and not-visible state are distinct errors.","properties":{"method":{"const":"restore_worker","type":"string"},"params":{"additionalProperties":false,"properties":{"name":{"type":"string"}},"required":["name"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"description":"Register another existing Worker as a reciprocal peer of this Worker.\n\nThis is metadata/control state only: it does not ask the target's live\ncontroller for consent, and it must not grant delegated scope,\nspawned-child ownership, output cursors, or child lifecycle authority.","properties":{"method":{"const":"register_peer","type":"string"},"params":{"additionalProperties":false,"properties":{"name":{"type":"string"}},"required":["name"],"type":"object"}},"required":["method","params"],"type":"object"}]},"PasteArtifactAvailability":{"enum":["available","unavailable","integrity_failed"],"type":"string"},"PasteArtifactMediaType":{"description":"One typed piece of a user submission.\n\n`Method::Submit` and `Event::UserMessage` carry `Vec`. Dumb\nclients (CLI piping, scripts) only need to produce a single\n`Segment::Text`; richer clients (TUI / GUI) construct typed atoms\n(paste chips, file refs) and\nsend them through directly so the Worker side never has to re-parse a\nflattened string.\n\nForward compat: payloads with unknown `kind` deserialize to\n`Segment::Unknown`. Worker treats this the same as known-but-unresolved\nvariants — emits an alert and inserts a `[unknown input segment]`\nplaceholder into the LLM context so neither user nor LLM is blind to\nthe dropped intent.","enum":["text_plain_utf8"],"type":"string"},"PasteArtifactRef":{"additionalProperties":false,"description":"Session-owned reference to a large pasted-input artifact.\n\nThe reference contains only bounded integrity and provenance metadata. The\nartifact body remains in session storage and is available to the model only\nthrough the scoped paste-artifact tools installed by Worker.","properties":{"artifact_id":{"type":"string"},"availability":{"$ref":"#/$defs/PasteArtifactAvailability","description":"Availability observed when this immutable reference was committed.\nReads revalidate storage and integrity rather than trusting this field."},"byte_len":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"char_count":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"created_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"line_count":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"media_type":{"$ref":"#/$defs/PasteArtifactMediaType"},"sha256":{"type":"string"},"source_entry_id":{"type":"string"}},"required":["artifact_id","created_at_ms","media_type","availability","byte_len","char_count","line_count","sha256","source_entry_id"],"type":"object"},"PendingSubmissionSummary":{"additionalProperties":false,"description":"Bounded public projection of one pending submission. Only a short display\npreview is exposed; full payload segments and provenance remain in the log.","properties":{"accepted_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"byte_len":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"preview":{"description":"Single-line display text, at most 240 Unicode scalars plus an ellipsis.\nAbsent on older runtimes. Never used to reconstruct the submitted input.","type":["string","null"]},"segment_count":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"submission_id":{"type":"string"}},"required":["submission_id","accepted_at_ms","segment_count","byte_len"],"type":"object"},"PendingSubmissionsSnapshot":{"additionalProperties":false,"description":"Revisioned session-owned FIFO projection used by snapshots and live events.","properties":{"head_id":{"default":null,"type":["string","null"]},"notification_count":{"default":0,"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"notification_previews":{"description":"Ordered display-only previews of waiting notifications, each bounded like\nPendingSubmissionSummary::preview. Older runtimes omit this field.","items":{"type":"string"},"type":"array"},"revision":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"submissions":{"default":[],"items":{"$ref":"#/$defs/PendingSubmissionSummary"},"type":"array"}},"required":["revision","head_id","notification_count","submissions"],"type":"object"},"Permission":{"description":"Permission lattice used by [`ScopeRule`].\n\nThe derived `Ord` instance follows declaration order, so\n`Read < Write`. Allow rules grant the stated level (and by extension\neverything below); deny rules cap the effective level **strictly\nbelow** the stated level.","enum":["read","write"],"type":"string"},"ReasoningConfig":{"description":"Public reasoning configuration resolved for the model client.\n\nThis deliberately exposes only the effective provider control, not the\nWorker manifest or provider credentials. The tagged shape prevents token\nbudgets from being presented as effort labels.","oneOf":[{"additionalProperties":false,"properties":{"effort":{"type":"string"},"kind":{"const":"effort","type":"string"}},"required":["kind","effort"],"type":"object"},{"additionalProperties":false,"properties":{"budget_tokens":{"format":"int32","maximum":2147483647.0,"minimum":-2147483648.0,"type":"integer"},"kind":{"const":"budget_tokens","type":"string"}},"required":["kind","budget_tokens"],"type":"object"}]},"RewindSummary":{"additionalProperties":false,"properties":{"discarded_entries":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"tool_side_effect_warning":{"type":"boolean"},"truncated_to_entries":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["truncated_to_entries","discarded_entries","tool_side_effect_warning"],"type":"object"},"RewindTarget":{"additionalProperties":false,"properties":{"disabled_reason":{"type":["string","null"]},"eligible":{"type":"boolean"},"expected_head_entries":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"id":{"$ref":"#/$defs/RewindTargetId"},"preview":{"type":"string"},"timestamp_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":["integer","null"]},"truncate_entries":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"turn_index":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"warning":{"type":["string","null"]}},"required":["id","expected_head_entries","truncate_entries","turn_index","preview","eligible"],"type":"object"},"RewindTargetId":{"additionalProperties":false,"properties":{"segment_id":{"type":"string"},"user_input_entry_index":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["segment_id","user_input_entry_index"],"type":"object"},"RunFailureKind":{"enum":["engine","compaction"],"type":"string"},"RunResult":{"oneOf":[{"enum":["finished","paused","limit_reached"],"type":"string"},{"const":"cancelled","description":"The logical Run ended because the user intentionally cancelled it after\nobservable work. Unlike `RolledBack`, committed history is retained.","type":"string"},{"const":"rolled_back","description":"The accepted Method::Submit produced no assistant/tool output before\nuser interruption, so the Worker rolled the submit-time turn state back\nto its pre-submit snapshot. Clients should treat the Worker as Idle and\nrestore the just-submitted input into the editable composer if desired.","type":"string"}]},"RunResumeSource":{"enum":["compaction","pause"],"type":"string"},"RunYieldReason":{"enum":["compaction"],"type":"string"},"ScopeRule":{"additionalProperties":false,"description":"A single allow or deny rule inside a scope configuration.","properties":{"permission":{"$ref":"#/$defs/Permission","description":"Permission level this rule grants (allow) or caps strictly below\n(deny)."},"recursive":{"default":true,"description":"When `false`, the rule only matches the target itself and its\ndirect children. Defaults to `true`.","type":"boolean"},"symlink_policy":{"$ref":"#/$defs/SymlinkPolicy","default":"resolved","description":"Which path identity an allow rule uses when symbolic links are\nencountered. Deny rules always inspect both identities."},"target":{"description":"Target path. Must be absolute by the time a `Scope` is built from\nthis rule — relative paths are resolved per-layer against the\nmanifest file's directory (cwd for overlay layers) before cascade\nmerge.","type":"string"}},"required":["target","permission","recursive","symlink_policy"],"type":"object"},"Segment":{"oneOf":[{"additionalProperties":false,"description":"Free-form text. The fallback every client can produce.","properties":{"content":{"type":"string"},"kind":{"const":"text","type":"string"}},"required":["kind","content"],"type":"object"},{"additionalProperties":false,"description":"Bracketed-paste capture from a TUI-style client. `id`, `chars`\nand `lines` carry the metadata needed to re-render a\n`[Clipboard #N | X chars, Y lines]` chip in `Event::UserMessage`\nre-broadcast.","properties":{"chars":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"content":{"type":"string"},"id":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"},"kind":{"const":"paste","type":"string"},"lines":{"format":"uint32","maximum":4294967295.0,"minimum":0.0,"type":"integer"}},"required":["kind","id","chars","lines","content"],"type":"object"},{"additionalProperties":false,"description":"Internal reference produced when Worker stores a large `Paste` before\ncommitting input. Clients may receive this in history/event projections;\nthe body is intentionally absent.","properties":{"artifact":{"$ref":"#/$defs/PasteArtifactRef"},"kind":{"const":"paste_artifact","type":"string"}},"required":["kind","artifact"],"type":"object"},{"additionalProperties":false,"description":"Client-local file uploaded into the owning Worker session before submit.\nThe Session Log stores only this immutable reference, never file bytes or\nthe client's local path.","properties":{"file":{"$ref":"#/$defs/UploadedFileRef"},"kind":{"const":"uploaded_file","type":"string"}},"required":["kind","file"],"type":"object"},{"additionalProperties":false,"description":"`@` file-system reference. Worker resolves readable files to\n`[File: ]` attachments and readable normal directories to shallow\n`[Dir: ]` listings; the flattened user text keeps the literal\n`@` placeholder either way.","properties":{"kind":{"const":"file_ref","type":"string"},"path":{"type":"string"}},"required":["kind","path"],"type":"object"},{"additionalProperties":false,"description":"Explicitly selected, source-qualified Feature invocation. The host never\npromotes free text to this variant and validates it against the installed\ndeclaration before execution.","properties":{"invocation":{"$ref":"#/$defs/FeatureInvocation"},"kind":{"const":"feature_invoke","type":"string"}},"required":["kind","invocation"],"type":"object"},{"additionalProperties":false,"description":"Source-qualified request for the host to start a Flow before committing\nthis user input. Runtime resolves it through Workspace authority and\nreplaces it with the entered state's instructions before Worker-side\ninput resolution.","properties":{"kind":{"const":"flow","type":"string"},"selector":{"type":"string"}},"required":["kind","selector"],"type":"object"},{"additionalProperties":false,"description":"Unknown variant from a newer client. Worker treats this as an\nunresolved input — surfaces an alert and inserts a placeholder.\nRound-trip is lossy: re-serializing yields `{\"kind\":\"unknown\"}`.","properties":{"kind":{"const":"unknown","type":"string"}},"required":["kind"],"type":"object"}]},"SessionContentPart":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"text","type":"string"},"text":{"type":"string"}},"required":["kind","text"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"refusal","type":"string"},"refusal":{"type":"string"}},"required":["kind","refusal"],"type":"object"}]},"SessionEntryProvenance":{"enum":["human_input","worker_input","flow_instruction","backend_instruction","model_output","tool_output","derived_summary","legacy_unknown"],"type":"string"},"SessionMessageRole":{"enum":["user","assistant"],"type":"string"},"SessionSnapshot":{"additionalProperties":false,"description":"Canonical, storage-independent projection of committed session history.\n\nWorker protocols expose this DTO instead of append-log records. New\nstorage variants can therefore be added without teaching every client how\nto replay the durable log format.","properties":{"entries":{"items":{"$ref":"#/$defs/SessionSnapshotEntry"},"type":"array"},"pending_submissions":{"$ref":"#/$defs/PendingSubmissionsSnapshot","default":{"head_id":null,"notification_count":0,"revision":0,"submissions":[]}}},"required":["entries","pending_submissions"],"type":"object"},"SessionSnapshotEntry":{"oneOf":[{"additionalProperties":false,"properties":{"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"kind":{"const":"user_input","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"segments":{"items":{"$ref":"#/$defs/Segment"},"type":"array"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","segments","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"properties":{"content":{"items":{"$ref":"#/$defs/SessionContentPart"},"type":"array"},"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"kind":{"const":"message","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"role":{"$ref":"#/$defs/SessionMessageRole"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","role","content","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"properties":{"arguments":{"type":"string"},"call_id":{"type":"string"},"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"kind":{"const":"tool_call","type":"string"},"name":{"type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","call_id","name","arguments","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"properties":{"attachments":{"items":{"$ref":"#/$defs/SessionToolAttachment"},"type":"array"},"call_id":{"type":"string"},"content":{"type":["string","null"]},"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"is_error":{"type":"boolean"},"kind":{"const":"tool_result","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"summary":{"type":"string"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","call_id","summary","is_error","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"properties":{"content":{"type":"string"},"data":true,"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"item_kind":{"type":"string"},"kind":{"const":"system_item","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","item_kind","content","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"description":"Durable, non-terminal control return from one logical Run. This is not a\ncompleted run and must not be used to infer the current Worker state.","properties":{"active_run_turn_count":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"kind":{"const":"run_yielded","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"reason":{"$ref":"#/$defs/RunYieldReason"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","reason","active_run_turn_count","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"description":"Durable boundary immediately before Engine execution resumes. The source\ndistinguishes automatic compaction continuation from an intentional\nuser-pause resume; it does not prove that later model output occurred.","properties":{"active_run_turn_count":{"format":"uint","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"kind":{"const":"run_resumed","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"source":{"$ref":"#/$defs/RunResumeSource"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","source","active_run_turn_count","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"description":"Durable terminal for an intentional cancellation. Consumers must not\npresent this lifecycle result as an execution failure.","properties":{"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"kind":{"const":"run_cancelled","type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","entry_id","timestamp","provenance"],"type":"object"},{"additionalProperties":false,"properties":{"derived_from":{"items":{"type":"string"},"type":"array"},"entry_id":{"description":"Stable identity from durable history metadata, or a deterministic\nidentity derived from the legacy segment and log position.","type":"string"},"failure":{"anyOf":[{"$ref":"#/$defs/RunFailureKind"},{"type":"null"}]},"kind":{"const":"run_error","type":"string"},"message":{"type":"string"},"provenance":{"$ref":"#/$defs/SessionEntryProvenance"},"timestamp":{"description":"Timestamp copied from the durable log record that commits this entry.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["kind","message","entry_id","timestamp","provenance"],"type":"object"}]},"SessionToolAttachment":{"additionalProperties":false,"properties":{"attachment_id":{"description":"Stable, bounded identity derived from the owning Session entry, attachment\nposition, and immutable image bytes. It is a locator, not a bearer secret.","type":"string"},"byte_len":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"media_type":{"type":"string"}},"required":["attachment_id","media_type","byte_len"],"type":"object"},"SubmissionDisposition":{"description":"Durable acceptance result for one idempotent Submit request.","oneOf":[{"enum":["started"],"type":"string"},{"const":"queued","description":"Input is durably held in the session FIFO until eligible for activation.","type":"string"}]},"SubscriptionEvent":{"oneOf":[{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"payload":{"$ref":"#/$defs/SubscriptionEventPayload"},"subscription_id":{"$ref":"#/$defs/SubscriptionId"}},"required":["subscription_id","payload"],"type":"object"},"event":{"const":"event","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"code":{"$ref":"#/$defs/SubscriptionTerminationCode"},"message":{"type":"string"},"subscription_id":{"$ref":"#/$defs/SubscriptionId"}},"required":["subscription_id","code","message"],"type":"object"},"event":{"const":"subscription_closed","type":"string"}},"required":["event","data"],"type":"object"}]},"SubscriptionEventPayload":{"oneOf":[{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"worker":{"$ref":"#/$defs/SubscriptionWorker"}},"required":["worker"],"type":"object"},"event":{"const":"worker_upserted","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"runtime_id":{"type":["string","null"]},"worker_id":{"$ref":"#/$defs/SubscriptionWorkerId"}},"required":["worker_id"],"type":"object"},"event":{"const":"worker_removed","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"event":{"$ref":"#/$defs/Event"},"worker_id":{"$ref":"#/$defs/SubscriptionWorkerId"}},"required":["worker_id","event"],"type":"object"},"event":{"const":"worker_protocol","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"workdir":{"$ref":"#/$defs/WorkspaceSubscriptionWorkdir"}},"required":["workdir"],"type":"object"},"event":{"const":"workdir_upserted","type":"string"}},"required":["event","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"working_directory_id":{"$ref":"#/$defs/SubscriptionWorkdirId"}},"required":["working_directory_id"],"type":"object"},"event":{"const":"workdir_removed","type":"string"}},"required":["event","data"],"type":"object"}]},"SubscriptionId":{"type":"string"},"SubscriptionRejectionCode":{"enum":["invalid_request","unsupported_protocol_version","unsupported_selector","unauthorized","resource_not_found","capacity_exceeded","internal"],"type":"string"},"SubscriptionRequest":{"oneOf":[{"additionalProperties":false,"properties":{"method":{"const":"subscribe_events","type":"string"},"params":{"additionalProperties":false,"properties":{"request_id":{"$ref":"#/$defs/SubscriptionRequestId"},"selector":{"$ref":"#/$defs/EventSubscriptionSelector"}},"required":["request_id","selector"],"type":"object"}},"required":["method","params"],"type":"object"},{"additionalProperties":false,"properties":{"method":{"const":"unsubscribe_events","type":"string"},"params":{"additionalProperties":false,"properties":{"request_id":{"$ref":"#/$defs/SubscriptionRequestId"},"subscription_id":{"$ref":"#/$defs/SubscriptionId"}},"required":["request_id","subscription_id"],"type":"object"}},"required":["method","params"],"type":"object"}]},"SubscriptionRequestId":{"type":"string"},"SubscriptionResponse":{"oneOf":[{"additionalProperties":false,"properties":{"payload":{"additionalProperties":false,"properties":{"request_id":{"$ref":"#/$defs/SubscriptionRequestId"},"selector":{"$ref":"#/$defs/EventSubscriptionSelector"},"snapshot":{"$ref":"#/$defs/SubscriptionSnapshot"},"subscription_id":{"$ref":"#/$defs/SubscriptionId"}},"required":["request_id","subscription_id","selector","snapshot"],"type":"object"},"result":{"const":"subscribed","type":"string"}},"required":["result","payload"],"type":"object"},{"additionalProperties":false,"properties":{"payload":{"additionalProperties":false,"properties":{"request_id":{"$ref":"#/$defs/SubscriptionRequestId"},"subscription_id":{"$ref":"#/$defs/SubscriptionId"}},"required":["request_id","subscription_id"],"type":"object"},"result":{"const":"unsubscribed","type":"string"}},"required":["result","payload"],"type":"object"},{"additionalProperties":false,"properties":{"payload":{"additionalProperties":false,"properties":{"code":{"$ref":"#/$defs/SubscriptionRejectionCode"},"message":{"type":"string"},"request_id":{"$ref":"#/$defs/SubscriptionRequestId"},"subscription_id":{"anyOf":[{"$ref":"#/$defs/SubscriptionId"},{"type":"null"}]}},"required":["request_id","code","message"],"type":"object"},"result":{"const":"subscription_rejected","type":"string"}},"required":["result","payload"],"type":"object"}]},"SubscriptionSnapshot":{"oneOf":[{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"workers":{"items":{"$ref":"#/$defs/SubscriptionWorker"},"type":"array"}},"required":["workers"],"type":"object"},"topic":{"const":"workers","type":"string"}},"required":["topic","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"events":{"items":{"$ref":"#/$defs/Event"},"type":"array"},"worker_id":{"$ref":"#/$defs/SubscriptionWorkerId"}},"required":["worker_id","events"],"type":"object"},"topic":{"const":"worker_protocol","type":"string"}},"required":["topic","data"],"type":"object"},{"additionalProperties":false,"properties":{"data":{"additionalProperties":false,"properties":{"workdirs":{"items":{"$ref":"#/$defs/WorkspaceSubscriptionWorkdir"},"type":"array"}},"required":["workdirs"],"type":"object"},"topic":{"const":"workspace_workdirs","type":"string"}},"required":["topic","data"],"type":"object"}]},"SubscriptionTerminationCode":{"enum":["lagged","resource_gone","unauthorized","server_shutdown"],"type":"string"},"SubscriptionWorkdirId":{"type":"string"},"SubscriptionWorker":{"additionalProperties":false,"properties":{"availability":{"$ref":"#/$defs/SubscriptionWorkerAvailability","default":"observed","description":"Freshness of the Runtime-backed observation carried by this projection.\n`Unavailable` preserves catalog membership without claiming that execution stopped."},"display_name":{"type":["string","null"]},"has_running_internal_workers":{"default":false,"type":"boolean"},"job":{"anyOf":[{"$ref":"#/$defs/SubscriptionWorkerJob"},{"type":"null"}],"description":"Backend-owned Job binding. Runtime producers leave this unset; the\nWorkspace projection enriches both snapshots and live updates."},"profile":{"type":["string","null"]},"resource_key":{"description":"Workspace-scoped canonical resource key. Runtime producers leave this unset;\nWorkspace-facing projections must populate it before publishing the Worker.","type":["string","null"]},"restore_observation_token":{"description":"Opaque Runtime-owned Restore observation identity, never a display-state hash.","type":["string","null"]},"runtime_id":{"description":"Set by the Workspace Server when projecting a Runtime-owned Worker to clients.\nRuntime producers leave this unset because the connection identifies the Runtime.","type":["string","null"]},"state":{"$ref":"#/$defs/SubscriptionWorkerState","description":"Runtime catalog lifecycle compatibility projection; not foreground-state authority."},"workdir_attachments":{"items":{"$ref":"#/$defs/SubscriptionWorkerWorkdirAttachment"},"type":"array"},"worker_id":{"$ref":"#/$defs/SubscriptionWorkerId"},"worker_state":{"anyOf":[{"$ref":"#/$defs/WorkerStateSnapshot"},{"type":"null"}],"description":"Latest foreground state observed from the Worker. This remains\nabsent until an authoritative Worker snapshot/event has been applied."},"workspace_id":{"type":["string","null"]}},"required":["worker_id","state","availability","has_running_internal_workers"],"type":"object"},"SubscriptionWorkerAvailability":{"description":"Whether the Workspace Server has a current Runtime observation for a Worker.\n\n`Unavailable` deliberately does not imply `Stopped`: it means the last observed\nlifecycle state is retained while the Runtime observation source is unavailable.","enum":["observed","unavailable"],"type":"string"},"SubscriptionWorkerId":{"type":"string"},"SubscriptionWorkerIds":{"description":"A bounded, canonical worker-id set. Ordering on the wire and in equality/hash\nsemantics is lexical, so equivalent selectors aggregate to one upstream key.","items":{"$ref":"#/$defs/SubscriptionWorkerId"},"type":"array"},"SubscriptionWorkerJob":{"additionalProperties":false,"properties":{"attempt_id":{"type":"string"},"job_id":{"type":"string"},"purpose":{"type":"string"}},"required":["job_id","attempt_id","purpose"],"type":"object"},"SubscriptionWorkerProtocolMethod":{"additionalProperties":false,"properties":{"method":{"$ref":"#/$defs/Method"},"subscription_id":{"$ref":"#/$defs/SubscriptionId"}},"required":["subscription_id","method"],"type":"object"},"SubscriptionWorkerState":{"enum":["idle","running","paused","stopped"],"type":"string"},"SubscriptionWorkerWorkdirAttachment":{"additionalProperties":false,"properties":{"alias":{"description":"Stable Worker-local routing alias.","type":"string"},"repository_key":{"type":["string","null"]},"working_directory_id":{"$ref":"#/$defs/SubscriptionWorkdirId"}},"required":["alias","working_directory_id"],"type":"object"},"SymlinkPolicy":{"description":"Symbolic-link identity used by one filesystem allow rule.\n\n`Resolved` is the least authority and the default: access is matched\nagainst the provider-resolved target. `Logical` intentionally grants the\npath as presented through the Workdir, even when it aliases another target.","enum":["resolved","logical"],"type":"string"},"ToolResultDisposition":{"enum":["success","error","interrupted","cancelled","outcome_unknown"],"type":"string"},"TurnResult":{"enum":["finished","paused"],"type":"string"},"UploadedFileAvailability":{"description":"Availability recorded for an uploaded client-local file.","enum":["available","unavailable","integrity_failed"],"type":"string"},"UploadedFileRef":{"additionalProperties":false,"description":"Session-owned immutable reference to a client-local uploaded file.\n\nUpload transports return an unbound reference. Worker fills\n`source_entry_id` immediately before the containing user input is committed;\ncommitted Session Log and public snapshot records therefore always retain\nthe durable source-entry identity without storing the file body.","properties":{"artifact_id":{"type":"string"},"availability":{"$ref":"#/$defs/UploadedFileAvailability"},"byte_len":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"created_at_ms":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"file_name":{"type":"string"},"media_type":{"type":"string"},"sha256":{"type":"string"},"source_entry_id":{"type":["string","null"]}},"required":["artifact_id","file_name","media_type","created_at_ms","availability","byte_len","sha256"],"type":"object"},"WorkerBusyState":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"run","type":"string"},"state":{"$ref":"#/$defs/WorkerRunState"}},"required":["kind","state"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"maintenance","type":"string"},"state":{"$ref":"#/$defs/WorkerMaintenanceState"}},"required":["kind","state"],"type":"object"}]},"WorkerCommandAcknowledgement":{"additionalProperties":false,"properties":{"command":{"$ref":"#/$defs/WorkerCommandKind"},"command_id":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"disposition":{"$ref":"#/$defs/WorkerCommandDisposition"},"state":{"$ref":"#/$defs/WorkerStateSnapshot","description":"The complete authoritative state observed after command admission."}},"required":["command_id","command","disposition","state"],"type":"object"},"WorkerCommandDisposition":{"enum":["accepted","stale_command_id","conflict","invalid_state"],"type":"string"},"WorkerCommandEnvelope":{"additionalProperties":false,"description":"Caller-owned identity for one state-changing Worker command.\n\nA controller accepts command ids in strictly increasing order. Exact retries\nof an accepted id must retain the same command kind.","properties":{"command_id":{"format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"}},"required":["command_id"],"type":"object"},"WorkerCommandKind":{"enum":["resume","cancel","pause","compact","shutdown"],"type":"string"},"WorkerEvent":{"description":"Typed lifecycle events sent from a child Worker to its parent.\n\nDelivered as `Method::WorkerEvent` over the parent's Unix socket. The\nparent Controller always applies variant-specific side effects\n(registry / worker-allocation updates). Agent-visible variants are also\nqueued into the notification buffer; control-plane-only variants are\nnot injected into the parent's LLM context.\n\nTransport is fire-and-forget; receivers must tolerate out-of-order\ndelivery (e.g. `TurnEnded` arriving after `ShutDown` for the same\nchild Worker).","oneOf":[{"additionalProperties":false,"description":"Child finished one turn and is back to IDLE.","properties":{"kind":{"const":"turn_ended","type":"string"},"worker_name":{"type":"string"}},"required":["kind","worker_name"],"type":"object"},{"additionalProperties":false,"description":"Engine execution error occurred inside the child's turn.\n\nLimited to worker runtime failures (provider / tool errors) —\ndoes not include transient method-rejection responses such as\n`AlreadyRunning`.","properties":{"kind":{"const":"errored","type":"string"},"message":{"type":"string"},"worker_name":{"type":"string"}},"required":["kind","worker_name","message"],"type":"object"},{"additionalProperties":false,"description":"Child has stopped (controller loop is exiting).","properties":{"kind":{"const":"shut_down","type":"string"},"worker_name":{"type":"string"}},"required":["kind","worker_name"],"type":"object"},{"additionalProperties":false,"description":"Child SubWorker sub-delegated scope to a grandchild SubWorker via `SubWorkerSpawn`.\n\nControl-plane only: receivers apply registry side effects and\npropagate upward, but do not expose this as an agent notification.\n\nThe parent uses this to add the grandchild to its own\n`spawned_workers.json` so it can manage the grandchild directly\neven if the intermediate child dies. The parent then re-fires\nthis event upward (if it has a parent of its own) to maintain\nthe chain to root.","properties":{"kind":{"const":"scope_sub_delegated","type":"string"},"parent_worker":{"description":"Sub-delegating Worker (= the sender itself).","type":"string"},"scope":{"description":"Scope delegated to the grandchild.","items":{"$ref":"#/$defs/ScopeRule"},"type":"array"},"sub_socket":{"description":"Unix-socket path where the grandchild is reachable.","type":"string"},"sub_worker":{"description":"Name of the grandchild Worker.","type":"string"}},"required":["kind","parent_worker","sub_worker","sub_socket","scope"],"type":"object"}]},"WorkerMaintenanceState":{"enum":["compacting"],"type":"string"},"WorkerRunState":{"enum":["running","pausing","paused","cancelling"],"type":"string"},"WorkerState":{"oneOf":[{"additionalProperties":false,"properties":{"kind":{"const":"idle","type":"string"}},"required":["kind"],"type":"object"},{"additionalProperties":false,"properties":{"kind":{"const":"busy","type":"string"},"state":{"$ref":"#/$defs/WorkerBusyState"}},"required":["kind","state"],"type":"object"}]},"WorkerStateSnapshot":{"additionalProperties":false,"properties":{"last_command_id":{"description":"Highest lifecycle command id observed by this controller instance.","format":"uint64","maximum":9007199254740991.0,"minimum":0.0,"type":"integer"},"last_finished_submission_request_id":{"description":"Most recent Submit request whose controller Run has returned and whose\npost-Run state transition has been published. This is a teardown fence,\nnot a successful-result claim; cancelled and failed Runs may also finish.","type":["string","null"]},"state":{"$ref":"#/$defs/WorkerState"}},"required":["last_command_id","state"],"type":"object"},"WorkerStatus":{"enum":["idle","running","paused","stopped"],"type":"string"},"WorkspaceSubscriptionWorkdir":{"additionalProperties":false,"description":"Workspace-facing Workdir summary. Backend-generated Repository UUIDs never\nenter this DTO; Workspace Server must resolve the required Repository key.","properties":{"repository_key":{"type":"string"},"state":{"type":"string"},"working_directory_id":{"$ref":"#/$defs/SubscriptionWorkdirId"}},"required":["working_directory_id","repository_key","state"],"type":"object"}},"$schema":"https://json-schema.org/draft/2020-12/schema","description":"Direction is explicit at the outer envelope, while concrete message kinds\nremain typed within each lane.","oneOf":[{"additionalProperties":false,"properties":{"frame":{"const":"request","type":"string"},"message":{"$ref":"#/$defs/SubscriptionRequest"},"protocol_version":{"format":"uint16","maximum":65535.0,"minimum":0.0,"type":"integer"}},"required":["frame","message","protocol_version"],"type":"object"},{"additionalProperties":false,"properties":{"frame":{"const":"response","type":"string"},"message":{"$ref":"#/$defs/SubscriptionResponse"},"protocol_version":{"format":"uint16","maximum":65535.0,"minimum":0.0,"type":"integer"}},"required":["frame","message","protocol_version"],"type":"object"},{"additionalProperties":false,"properties":{"frame":{"const":"event","type":"string"},"message":{"$ref":"#/$defs/SubscriptionEvent"},"protocol_version":{"format":"uint16","maximum":65535.0,"minimum":0.0,"type":"integer"}},"required":["frame","message","protocol_version"],"type":"object"},{"additionalProperties":false,"properties":{"frame":{"const":"worker_protocol","type":"string"},"message":{"$ref":"#/$defs/SubscriptionWorkerProtocolMethod"},"protocol_version":{"format":"uint16","maximum":65535.0,"minimum":0.0,"type":"integer"}},"required":["frame","message","protocol_version"],"type":"object"}],"title":"SubscriptionFrame"}; export const RUST_SERIALIZED_SUBSCRIPTION_FRAME_FIXTURES: readonly unknown[] = - [{"protocol_version":2,"frame":"response","message":{"result":"subscribed","payload":{"request_id":"fixture-request","subscription_id":"fixture-subscription","selector":{"topic":"workspace_workers"},"snapshot":{"topic":"workers","data":{"workers":[{"worker_id":"fixture-worker","runtime_id":"fixture-runtime","resource_key":"fixture-worker-resource","availability":"observed","state":"idle","has_running_internal_workers":false,"workspace_id":"fixture-workspace","workdir_attachments":[{"alias":"external","working_directory_id":"fixture-external-workdir"}]}]}}}}},{"protocol_version":2,"frame":"event","message":{"event":"event","data":{"subscription_id":"fixture-subscription","payload":{"event":"worker_protocol","data":{"worker_id":"fixture-worker","event":{"event":"pending_submissions_changed","data":{"pending":{"revision":0,"notification_count":0,"head_id":null,"submissions":[]}}}}}}}},{"protocol_version":2,"frame":"event","message":{"event":"event","data":{"subscription_id":"fixture-subscription","payload":{"event":"worker_protocol","data":{"worker_id":"fixture-worker","event":{"event":"snapshot","data":{"session":{"pending_submissions":{"revision":0,"notification_count":0,"head_id":null,"submissions":[]},"entries":[]},"greeting":{"worker_name":"fixture-worker","cwd":"/fixture","provider":"openai_responses","model":"gpt-6-astra","reasoning":{"kind":"effort","effort":"high"},"scope_summary":"fixture scope","tools":["Read"],"context_window":272000,"context_tokens":142000,"context_usage":{"tokens":142000,"source":"measured"}},"state":{"last_command_id":0,"state":{"kind":"idle"}},"internal_workers":[{"worker":{"session_id":"fixture-child-session","name":"fixture-child","parent_session_id":"fixture-parent-session","kind":"sub_worker"},"revision":1,"session":{"pending_submissions":{"revision":0,"notification_count":0,"head_id":null,"submissions":[]},"entries":[]},"greeting":{"worker_name":"fixture-worker","cwd":"/fixture","provider":"openai_responses","model":"gpt-6-astra","reasoning":{"kind":"effort","effort":"high"},"scope_summary":"fixture scope","tools":["Read"],"context_window":272000,"context_tokens":142000,"context_usage":{"tokens":142000,"source":"measured"}},"status":"idle"}]}}}}}}}]; + [{"protocol_version":2,"frame":"response","message":{"result":"subscribed","payload":{"request_id":"fixture-request","subscription_id":"fixture-subscription","selector":{"topic":"workspace_workers"},"snapshot":{"topic":"workers","data":{"workers":[{"restore_observation_token":"fixture-restore-observation","worker_id":"fixture-worker","runtime_id":"fixture-runtime","resource_key":"fixture-worker-resource","availability":"observed","state":"idle","has_running_internal_workers":false,"workspace_id":"fixture-workspace","workdir_attachments":[{"alias":"external","working_directory_id":"fixture-external-workdir"}]}]}}}}},{"protocol_version":2,"frame":"event","message":{"event":"event","data":{"subscription_id":"fixture-subscription","payload":{"event":"worker_protocol","data":{"worker_id":"fixture-worker","event":{"event":"pending_submissions_changed","data":{"pending":{"revision":0,"notification_count":0,"head_id":null,"submissions":[]}}}}}}}},{"protocol_version":2,"frame":"event","message":{"event":"event","data":{"subscription_id":"fixture-subscription","payload":{"event":"worker_protocol","data":{"worker_id":"fixture-worker","event":{"event":"snapshot","data":{"session":{"pending_submissions":{"revision":0,"notification_count":0,"head_id":null,"submissions":[]},"entries":[]},"greeting":{"worker_name":"fixture-worker","cwd":"/fixture","provider":"openai_responses","model":"gpt-6-astra","reasoning":{"kind":"effort","effort":"high"},"scope_summary":"fixture scope","tools":["Read"],"context_window":272000,"context_tokens":142000,"context_usage":{"tokens":142000,"source":"measured"}},"state":{"last_command_id":0,"state":{"kind":"idle"}},"internal_workers":[{"worker":{"session_id":"fixture-child-session","name":"fixture-child","parent_session_id":"fixture-parent-session","kind":"sub_worker"},"revision":1,"session":{"pending_submissions":{"revision":0,"notification_count":0,"head_id":null,"submissions":[]},"entries":[]},"greeting":{"worker_name":"fixture-worker","cwd":"/fixture","provider":"openai_responses","model":"gpt-6-astra","reasoning":{"kind":"effort","effort":"high"},"scope_summary":"fixture scope","tools":["Read"],"context_window":272000,"context_tokens":142000,"context_usage":{"tokens":142000,"source":"measured"}},"status":"idle"}]}}}}}}}]; const utf8 = new TextEncoder(); type JsonObject = Record; @@ -379,6 +379,7 @@ function validateWorkers(workers: SubscriptionWorker[]): boolean { } function validateWorker(worker: SubscriptionWorker): boolean { + if (worker.restore_observation_token != null && !validateIdentifier(worker.restore_observation_token, MAX_RESOURCE_ID_BYTES)) return false; if (!validateIdentifier(worker.worker_id, MAX_RESOURCE_ID_BYTES)) return false; if (worker.runtime_id != null && !validateIdentifier(worker.runtime_id, MAX_RESOURCE_ID_BYTES)) { return false; diff --git a/web/workspace/src/lib/generated/protocol.ts b/web/workspace/src/lib/generated/protocol.ts index 926075657..637f7c16b 100644 --- a/web/workspace/src/lib/generated/protocol.ts +++ b/web/workspace/src/lib/generated/protocol.ts @@ -359,7 +359,11 @@ alias: string, repository_key?: string | null, working_directory_id: Subscriptio export type SubscriptionWorkerJob = { job_id: string, attempt_id: string, purpose: string, }; -export type SubscriptionWorker = { worker_id: SubscriptionWorkerId, +export type SubscriptionWorker = { +/** + * Opaque Runtime-owned Restore observation identity, never a display-state hash. + */ +restore_observation_token?: string | null, worker_id: SubscriptionWorkerId, /** * Set by the Workspace Server when projecting a Runtime-owned Worker to clients. * Runtime producers leave this unset because the connection identifies the Runtime. diff --git a/web/workspace/src/lib/generated/repository-api.ts b/web/workspace/src/lib/generated/repository-api.ts index a259cd8ae..5b72d2098 100644 --- a/web/workspace/src/lib/generated/repository-api.ts +++ b/web/workspace/src/lib/generated/repository-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-repository-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:6001e9f7273288195fb5a9859e4a5932a521d2f2e7c3a9ecd8ec660fd6a37779) +// Input contract: 0.1.0 (sha256:fd06f382c6d33a5c3cc98ef9e264545923c53fa95a9be555a85680f98c6065da) // Options: repository-operation-schema-closure,openapi-3.1,format:uint64=safe-number,strict-safe-integers,sorted-output // Output: web/workspace/src/lib/generated/repository-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_repository_openapi_types -- --check @@ -71,7 +71,7 @@ export type WorkerState = { kind: "idle"; } | { kind: "busy"; state: WorkerBusyS export type WorkerStateSnapshot = { last_command_id: number; last_finished_submission_request_id?: string | null; state: WorkerState; }; -export type WorkerSummary = { availability: SubscriptionWorkerAvailability; diagnostics?: Array; display_name?: string; host_id: string; implementation: WorkerImplementationSummary; label: string; last_seen_at?: string | null; pinned?: boolean; profile?: string | null; resource_key: string; retention_state?: string; runtime_id: string; singleton_key?: string | null; state: string; tags?: Array; workdir_attachments?: Array; worker_id: string; worker_state?: WorkerStateSnapshot | null; workspace: WorkerWorkspaceSummary; }; +export type WorkerSummary = { availability: SubscriptionWorkerAvailability; diagnostics?: Array; display_name?: string; host_id: string; implementation: WorkerImplementationSummary; label: string; last_seen_at?: string | null; pinned?: boolean; profile?: string | null; resource_key: string; restore_observation_token?: string | null; retention_state?: string; runtime_id: string; singleton_key?: string | null; state: string; tags?: Array; workdir_attachments?: Array; worker_id: string; worker_state?: WorkerStateSnapshot | null; workspace: WorkerWorkspaceSummary; }; export type WorkerWorkdirAttachmentSummary = { alias: string; effective_permissions: ExternalWorkdirPermissions; working_directory: WorkingDirectorySummary; }; diff --git a/web/workspace/src/lib/generated/runtime-api.ts b/web/workspace/src/lib/generated/runtime-api.ts index 901b4716e..53eeae412 100644 --- a/web/workspace/src/lib/generated/runtime-api.ts +++ b/web/workspace/src/lib/generated/runtime-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-runtime-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:6001e9f7273288195fb5a9859e4a5932a521d2f2e7c3a9ecd8ec660fd6a37779) +// Input contract: 0.1.0 (sha256:fd06f382c6d33a5c3cc98ef9e264545923c53fa95a9be555a85680f98c6065da) // Options: runtime-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Output: web/workspace/src/lib/generated/runtime-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_runtime_api_types -- --check @@ -139,7 +139,7 @@ export type RuntimeWorkerLifecycleResult = { diagnostics?: Array; ru export type RuntimeWorkerListResponse = { diagnostics?: Array; items: Array; limit: number; source: string; workspace_id: string; }; -export type RuntimeWorkerShowResponse = { availability: SubscriptionWorkerAvailability; diagnostics?: Array; display_name?: string; host_id: string; implementation: WorkerImplementationSummary; label: string; last_seen_at?: string | null; pinned?: boolean; profile?: string | null; resource_key: string; retention_state?: string; runtime_id: string; singleton_key?: string | null; state: string; tags?: Array; updated_at: string; workdir_attachments?: Array; worker_id: string; worker_state?: WorkerStateSnapshot | null; workspace: WorkerWorkspaceSummary; }; +export type RuntimeWorkerShowResponse = { availability: SubscriptionWorkerAvailability; diagnostics?: Array; display_name?: string; host_id: string; implementation: WorkerImplementationSummary; label: string; last_seen_at?: string | null; pinned?: boolean; profile?: string | null; resource_key: string; restore_observation_token?: string | null; retention_state?: string; runtime_id: string; singleton_key?: string | null; state: string; tags?: Array; updated_at: string; workdir_attachments?: Array; worker_id: string; worker_state?: WorkerStateSnapshot | null; workspace: WorkerWorkspaceSummary; }; export type RuntimeWorkerSpawnAcceptanceEvidence = { detail: string; kind: string; }; @@ -181,6 +181,8 @@ export type WorkerMaintenanceState = "compacting"; export type WorkerOperationState = "accepted" | "unsupported" | "rejected"; +export type WorkerRestoreRequest = { expected_observation_token: string; request_id: string; }; + export type WorkerRestoreResponse = { result: WorkerRestoreResult; runtime_id: string; worker_id: string; workspace_id: string; }; export type WorkerRestoreResult = { diagnostics?: Array; state: WorkerRestoreState; worker?: WorkerSummary | null; }; @@ -195,7 +197,7 @@ export type WorkerState = { kind: "idle"; } | { kind: "busy"; state: WorkerBusyS export type WorkerStateSnapshot = { last_command_id: number; last_finished_submission_request_id?: string | null; state: WorkerState; }; -export type WorkerSummary = { availability: SubscriptionWorkerAvailability; diagnostics?: Array; display_name?: string; host_id: string; implementation: WorkerImplementationSummary; label: string; last_seen_at?: string | null; pinned?: boolean; profile?: string | null; resource_key: string; retention_state?: string; runtime_id: string; singleton_key?: string | null; state: string; tags?: Array; workdir_attachments?: Array; worker_id: string; worker_state?: WorkerStateSnapshot | null; workspace: WorkerWorkspaceSummary; }; +export type WorkerSummary = { availability: SubscriptionWorkerAvailability; diagnostics?: Array; display_name?: string; host_id: string; implementation: WorkerImplementationSummary; label: string; last_seen_at?: string | null; pinned?: boolean; profile?: string | null; resource_key: string; restore_observation_token?: string | null; retention_state?: string; runtime_id: string; singleton_key?: string | null; state: string; tags?: Array; workdir_attachments?: Array; worker_id: string; worker_state?: WorkerStateSnapshot | null; workspace: WorkerWorkspaceSummary; }; export type WorkerWorkdirAttachmentSummary = { alias: string; effective_permissions: ExternalWorkdirPermissions; working_directory: WorkingDirectorySummary; }; diff --git a/web/workspace/src/lib/generated/ticket-api.ts b/web/workspace/src/lib/generated/ticket-api.ts index e6fb0a177..657aa7d1b 100644 --- a/web/workspace/src/lib/generated/ticket-api.ts +++ b/web/workspace/src/lib/generated/ticket-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-ticket-browser-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:6001e9f7273288195fb5a9859e4a5932a521d2f2e7c3a9ecd8ec660fd6a37779) +// Input contract: 0.1.0 (sha256:fd06f382c6d33a5c3cc98ef9e264545923c53fa95a9be555a85680f98c6065da) // Options: ticket-objective-merge-request-browser-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Output: web/workspace/src/lib/generated/ticket-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_ticket_api_types -- --check diff --git a/web/workspace/src/lib/generated/workdir-api.ts b/web/workspace/src/lib/generated/workdir-api.ts index b4a8b9d21..cf273be40 100644 --- a/web/workspace/src/lib/generated/workdir-api.ts +++ b/web/workspace/src/lib/generated/workdir-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-workdir-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:6001e9f7273288195fb5a9859e4a5932a521d2f2e7c3a9ecd8ec660fd6a37779) +// Input contract: 0.1.0 (sha256:fd06f382c6d33a5c3cc98ef9e264545923c53fa95a9be555a85680f98c6065da) // Options: workdir-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Output: web/workspace/src/lib/generated/workdir-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_workdir_api_types -- --check diff --git a/web/workspace/src/lib/generated/worker-launch-api.ts b/web/workspace/src/lib/generated/worker-launch-api.ts index f3519c205..f56aba174 100644 --- a/web/workspace/src/lib/generated/worker-launch-api.ts +++ b/web/workspace/src/lib/generated/worker-launch-api.ts @@ -1,6 +1,6 @@ // @generated by yoi-worker-openapi-typescript@1. Do not edit. // Input: openapi/server-api.json -// Input contract: 0.1.0 (sha256:6001e9f7273288195fb5a9859e4a5932a521d2f2e7c3a9ecd8ec660fd6a37779) +// Input contract: 0.1.0 (sha256:fd06f382c6d33a5c3cc98ef9e264545923c53fa95a9be555a85680f98c6065da) // Options: worker-operation-schema-closure,openapi-3.1,strict-integer-ranges,sorted-output // Output: web/workspace/src/lib/generated/worker-launch-api.ts // Regenerate/check: cargo run -q -p server-api --example generate_worker_launch_api_types -- --check @@ -77,7 +77,7 @@ export type WorkerState = { kind: "idle"; } | { kind: "busy"; state: WorkerBusyS export type WorkerStateSnapshot = { last_command_id: number; last_finished_submission_request_id?: string | null; state: WorkerState; }; -export type WorkerSummary = { availability: SubscriptionWorkerAvailability; diagnostics?: Array; display_name?: string; host_id: string; implementation: WorkerImplementationSummary; label: string; last_seen_at?: string | null; pinned?: boolean; profile?: string | null; resource_key: string; retention_state?: string; runtime_id: string; singleton_key?: string | null; state: string; tags?: Array; workdir_attachments?: Array; worker_id: string; worker_state?: WorkerStateSnapshot | null; workspace: WorkerWorkspaceSummary; }; +export type WorkerSummary = { availability: SubscriptionWorkerAvailability; diagnostics?: Array; display_name?: string; host_id: string; implementation: WorkerImplementationSummary; label: string; last_seen_at?: string | null; pinned?: boolean; profile?: string | null; resource_key: string; restore_observation_token?: string | null; retention_state?: string; runtime_id: string; singleton_key?: string | null; state: string; tags?: Array; workdir_attachments?: Array; worker_id: string; worker_state?: WorkerStateSnapshot | null; workspace: WorkerWorkspaceSummary; }; export type WorkerWorkdirAttachmentSummary = { alias: string; effective_permissions: ExternalWorkdirPermissions; working_directory: WorkingDirectorySummary; }; diff --git a/web/workspace/src/lib/workspace/api/runtime-workers.ts b/web/workspace/src/lib/workspace/api/runtime-workers.ts index 5247b8366..646fb0d32 100644 --- a/web/workspace/src/lib/workspace/api/runtime-workers.ts +++ b/web/workspace/src/lib/workspace/api/runtime-workers.ts @@ -10,9 +10,13 @@ import type { RuntimeCleanupPlanResponse, RuntimeWorkerLifecycleResult, WorkerOperationState, + WorkerRestoreResponse, + WorkerRestoreState, WorkerRetentionResponse, } from "#lib/generated/runtime-api.ts"; +import { parseWorkerSummary } from "#lib/workspace/api/workers.ts"; + const MAX_STRING_BYTES = 4_096; const MAX_CANDIDATES = 1_000; const MAX_RESULTS = 2_000; @@ -68,14 +72,20 @@ function exactKeys( ): void { const allowed = new Set([...required, ...optional]); for (const key of Object.keys(value)) { - if (!allowed.has(key)) fail(`${path}.${key}`, "is not part of the wire contract"); + if (!allowed.has(key)) { + fail(`${path}.${key}`, "is not part of the wire contract"); + } } for (const key of required) { if (!Object.hasOwn(value, key)) fail(`${path}.${key}`, "is required"); } } -function boundedString(value: unknown, path: string, allowEmpty = false): string { +function boundedString( + value: unknown, + path: string, + allowEmpty = false, +): string { if (typeof value !== "string") return fail(path, "must be a string"); if (!allowEmpty && value.length === 0) return fail(path, "must not be empty"); if (encoder.encode(value).byteLength > MAX_STRING_BYTES) { @@ -91,7 +101,9 @@ function boolean(value: unknown, path: string): boolean { function array(value: unknown, path: string, max: number): unknown[] { if (!Array.isArray(value)) return fail(path, "must be an array"); - if (value.length > max) return fail(path, `must contain at most ${max} items`); + if (value.length > max) { + return fail(path, `must contain at most ${max} items`); + } return value; } @@ -101,12 +113,18 @@ function strings(value: unknown, path: string): string[] { ); } -function optionalNullableString(value: unknown, path: string): string | null | undefined { +function optionalNullableString( + value: unknown, + path: string, +): string | null | undefined { if (value === undefined || value === null) return value; return boundedString(value, path); } -function optionalNullableSafeInteger(value: unknown, path: string): number | null | undefined { +function optionalNullableSafeInteger( + value: unknown, + path: string, +): number | null | undefined { if (value === undefined || value === null) return value; if (!Number.isSafeInteger(value) || (value as number) < 0) { return fail(path, "must be a non-negative safe integer"); @@ -115,7 +133,9 @@ function optionalNullableSafeInteger(value: unknown, path: string): number | nul } function targetKind(value: unknown, path: string): CleanupTargetKind { - if (typeof value !== "string" || !TARGET_KINDS.has(value as CleanupTargetKind)) { + if ( + typeof value !== "string" || !TARGET_KINDS.has(value as CleanupTargetKind) + ) { return fail(path, "has an unknown cleanup action"); } return value as CleanupTargetKind; @@ -164,13 +184,25 @@ function workerCandidate(value: unknown, path: string): CleanupWorkerCandidate { target_id: boundedString(item.target_id, `${path}.target_id`), action: targetKind(item.action, `${path}.action`), worker_id: boundedString(item.worker_id, `${path}.worker_id`), - runtime_worker_id: boundedString(item.runtime_worker_id, `${path}.runtime_worker_id`), + runtime_worker_id: boundedString( + item.runtime_worker_id, + `${path}.runtime_worker_id`, + ), runtime_id: boundedString(item.runtime_id, `${path}.runtime_id`), reason: boundedString(item.reason, `${path}.reason`, true), - blocking_reason: optionalNullableString(item.blocking_reason, `${path}.blocking_reason`), + blocking_reason: optionalNullableString( + item.blocking_reason, + `${path}.blocking_reason`, + ), pinned: boolean(item.pinned, `${path}.pinned`), - retention_state: boundedString(item.retention_state, `${path}.retention_state`), - linked_workdir_ids: strings(item.linked_workdir_ids, `${path}.linked_workdir_ids`), + retention_state: boundedString( + item.retention_state, + `${path}.retention_state`, + ), + linked_workdir_ids: strings( + item.linked_workdir_ids, + `${path}.linked_workdir_ids`, + ), running_linked: boolean(item.running_linked, `${path}.running_linked`), estimated_reclaim_bytes: optionalNullableSafeInteger( item.estimated_reclaim_bytes, @@ -179,7 +211,10 @@ function workerCandidate(value: unknown, path: string): CleanupWorkerCandidate { }; } -function workdirCandidate(value: unknown, path: string): CleanupWorkdirCandidate { +function workdirCandidate( + value: unknown, + path: string, +): CleanupWorkdirCandidate { const item = object(value, path); exactKeys( item, @@ -213,10 +248,19 @@ function workdirCandidate(value: unknown, path: string): CleanupWorkdirCandidate action: targetKind(item.action, `${path}.action`), workdir_id: boundedString(item.workdir_id, `${path}.workdir_id`), runtime_id: boundedString(item.runtime_id, `${path}.runtime_id`), - repository_key: boundedString(item.repository_key, `${path}.repository_key`), + repository_key: boundedString( + item.repository_key, + `${path}.repository_key`, + ), reason: boundedString(item.reason, `${path}.reason`, true), - blocking_reason: optionalNullableString(item.blocking_reason, `${path}.blocking_reason`), - linked_worker_ids: strings(item.linked_worker_ids, `${path}.linked_worker_ids`), + blocking_reason: optionalNullableString( + item.blocking_reason, + `${path}.blocking_reason`, + ), + linked_worker_ids: strings( + item.linked_worker_ids, + `${path}.linked_worker_ids`, + ), linked_running_worker_ids: strings( item.linked_running_worker_ids, `${path}.linked_running_worker_ids`, @@ -232,7 +276,9 @@ function workdirCandidate(value: unknown, path: string): CleanupWorkdirCandidate }; } -export function parseWorkerRetentionResponse(value: unknown): WorkerRetentionResponse { +export function parseWorkerRetentionResponse( + value: unknown, +): WorkerRetentionResponse { const item = object(value, "Worker retention response"); exactKeys( item, @@ -241,9 +287,18 @@ export function parseWorkerRetentionResponse(value: unknown): WorkerRetentionRes "Worker retention response", ); return { - workspace_id: boundedString(item.workspace_id, "Worker retention response.workspace_id"), - runtime_id: boundedString(item.runtime_id, "Worker retention response.runtime_id"), - worker_id: boundedString(item.worker_id, "Worker retention response.worker_id"), + workspace_id: boundedString( + item.workspace_id, + "Worker retention response.workspace_id", + ), + runtime_id: boundedString( + item.runtime_id, + "Worker retention response.runtime_id", + ), + worker_id: boundedString( + item.worker_id, + "Worker retention response.worker_id", + ), pinned: boolean(item.pinned, "Worker retention response.pinned"), retention_state: boundedString( item.retention_state, @@ -268,7 +323,10 @@ export function parseRuntimeWorkerLifecycleResult( [], "Runtime Worker lifecycle result", ); - const state = boundedString(item.state, "Runtime Worker lifecycle result.state"); + const state = boundedString( + item.state, + "Runtime Worker lifecycle result.state", + ); if (!WORKER_OPERATION_STATES.has(state as WorkerOperationState)) { fail("Runtime Worker lifecycle result.state", "has an unknown value"); } @@ -289,7 +347,58 @@ export function parseRuntimeWorkerLifecycleResult( }; } -export function parseRuntimeCleanupPlan(value: unknown): RuntimeCleanupPlanResponse { +/** Restore is a wrapped Workspace response, not the flat Stop lifecycle result. */ +export function parseWorkerRestoreResponse( + value: unknown, +): WorkerRestoreResponse { + const path = "Worker Restore response"; + const item = object(value, path); + exactKeys( + item, + ["workspace_id", "runtime_id", "worker_id", "result"], + [], + path, + ); + const result = object(item.result, `${path}.result`); + exactKeys(result, ["state"], ["diagnostics", "worker"], `${path}.result`); + const state = boundedString(result.state, `${path}.result.state`); + if ( + !["accepted", "rejected", "rolled_back", "reconciliation_required"] + .includes(state) + ) { + fail(`${path}.result.state`, "has an unknown value"); + } + const workspace_id = boundedString(item.workspace_id, `${path}.workspace_id`); + const runtime_id = boundedString(item.runtime_id, `${path}.runtime_id`); + const worker_id = boundedString(item.worker_id, `${path}.worker_id`); + const worker = result.worker == null + ? result.worker as null | undefined + : parseWorkerSummary(result.worker); + if ( + worker && + (worker.runtime_id !== runtime_id || worker.worker_id !== worker_id || + (worker.workspace.workspace_id != null && + worker.workspace.workspace_id !== workspace_id)) + ) { + fail(`${path}.result.worker`, "does not match the response identity"); + } + return { + workspace_id, + runtime_id, + worker_id, + result: { + state: state as WorkerRestoreState, + diagnostics: result.diagnostics === undefined + ? [] + : diagnostics(result.diagnostics, `${path}.result.diagnostics`), + worker, + }, + }; +} + +export function parseRuntimeCleanupPlan( + value: unknown, +): RuntimeCleanupPlanResponse { const item = object(value, "Runtime cleanup plan"); exactKeys( item, @@ -307,22 +416,44 @@ export function parseRuntimeCleanupPlan(value: unknown): RuntimeCleanupPlanRespo "Runtime cleanup plan", ); return { - workspace_id: boundedString(item.workspace_id, "Runtime cleanup plan.workspace_id"), - runtime_id: boundedString(item.runtime_id, "Runtime cleanup plan.runtime_id"), - generated_at: boundedString(item.generated_at, "Runtime cleanup plan.generated_at"), + workspace_id: boundedString( + item.workspace_id, + "Runtime cleanup plan.workspace_id", + ), + runtime_id: boundedString( + item.runtime_id, + "Runtime cleanup plan.runtime_id", + ), + generated_at: boundedString( + item.generated_at, + "Runtime cleanup plan.generated_at", + ), revision: boundedString(item.revision, "Runtime cleanup plan.revision"), digest: boundedString(item.digest, "Runtime cleanup plan.digest"), - workers: array(item.workers, "Runtime cleanup plan.workers", MAX_CANDIDATES).map( - (entry, index) => workerCandidate(entry, `Runtime cleanup plan.workers[${index}]`), + workers: array(item.workers, "Runtime cleanup plan.workers", MAX_CANDIDATES) + .map( + (entry, index) => + workerCandidate(entry, `Runtime cleanup plan.workers[${index}]`), + ), + workdirs: array( + item.workdirs, + "Runtime cleanup plan.workdirs", + MAX_CANDIDATES, + ).map( + (entry, index) => + workdirCandidate(entry, `Runtime cleanup plan.workdirs[${index}]`), ), - workdirs: array(item.workdirs, "Runtime cleanup plan.workdirs", MAX_CANDIDATES).map( - (entry, index) => workdirCandidate(entry, `Runtime cleanup plan.workdirs[${index}]`), + diagnostics: diagnostics( + item.diagnostics, + "Runtime cleanup plan.diagnostics", ), - diagnostics: diagnostics(item.diagnostics, "Runtime cleanup plan.diagnostics"), }; } -function executionResult(value: unknown, path: string): RuntimeCleanupExecutionResult { +function executionResult( + value: unknown, + path: string, +): RuntimeCleanupExecutionResult { const item = object(value, path); exactKeys(item, ["target_id", "action", "status", "message"], [], path); return { @@ -339,18 +470,42 @@ export function parseRuntimeCleanupExecution( const item = object(value, "Runtime cleanup execution"); exactKeys( item, - ["workspace_id", "runtime_id", "executed_at", "results", "plan_after", "diagnostics"], + [ + "workspace_id", + "runtime_id", + "executed_at", + "results", + "plan_after", + "diagnostics", + ], [], "Runtime cleanup execution", ); return { - workspace_id: boundedString(item.workspace_id, "Runtime cleanup execution.workspace_id"), - runtime_id: boundedString(item.runtime_id, "Runtime cleanup execution.runtime_id"), - executed_at: boundedString(item.executed_at, "Runtime cleanup execution.executed_at"), - results: array(item.results, "Runtime cleanup execution.results", MAX_RESULTS).map( - (entry, index) => executionResult(entry, `Runtime cleanup execution.results[${index}]`), + workspace_id: boundedString( + item.workspace_id, + "Runtime cleanup execution.workspace_id", + ), + runtime_id: boundedString( + item.runtime_id, + "Runtime cleanup execution.runtime_id", + ), + executed_at: boundedString( + item.executed_at, + "Runtime cleanup execution.executed_at", + ), + results: array( + item.results, + "Runtime cleanup execution.results", + MAX_RESULTS, + ).map( + (entry, index) => + executionResult(entry, `Runtime cleanup execution.results[${index}]`), ), plan_after: parseRuntimeCleanupPlan(item.plan_after), - diagnostics: diagnostics(item.diagnostics, "Runtime cleanup execution.diagnostics"), + diagnostics: diagnostics( + item.diagnostics, + "Runtime cleanup execution.diagnostics", + ), }; } diff --git a/web/workspace/src/lib/workspace/api/workers.ts b/web/workspace/src/lib/workspace/api/workers.ts index e1f765a7c..856cadd7c 100644 --- a/web/workspace/src/lib/workspace/api/workers.ts +++ b/web/workspace/src/lib/workspace/api/workers.ts @@ -97,7 +97,11 @@ function optional( label: string, parse: (item: unknown, label: string) => T, ): T | null | undefined { - return value === undefined ? undefined : value === null ? null : parse(value, label); + return value === undefined + ? undefined + : value === null + ? null + : parse(value, label); } function diagnostic(value: unknown, label: string): Diagnostic { @@ -372,6 +376,7 @@ export function parseWorkerSummary( "availability", "state", "worker_state", + "restore_observation_token", "last_seen_at", "pinned", "retention_state", @@ -406,13 +411,28 @@ export function parseWorkerSummary( `${label}.singleton_key`, string, ), - tags: item.tags === undefined ? [] : array(item.tags, `${label}.tags`, string), + tags: item.tags === undefined + ? [] + : array(item.tags, `${label}.tags`, string), workspace: workspaceSummary(item.workspace, `${label}.workspace`), availability, state: string(item.state, `${label}.state`), worker_state: workerState, + restore_observation_token: optional( + item.restore_observation_token, + `${label}.restore_observation_token`, + (value, path) => { + const token = string(value, path); + if (new TextEncoder().encode(token).byteLength > 256) { + throw new Error(`${path} exceeds its 256 UTF-8 byte limit`); + } + return token; + }, + ), last_seen_at: optional(item.last_seen_at, `${label}.last_seen_at`, string), - pinned: item.pinned === undefined ? false : boolean(item.pinned, `${label}.pinned`), + pinned: item.pinned === undefined + ? false + : boolean(item.pinned, `${label}.pinned`), retention_state: item.retention_state === undefined ? "" : string(item.retention_state, `${label}.retention_state`), @@ -420,11 +440,13 @@ export function parseWorkerSummary( item.implementation, `${label}.implementation`, ), - workdir_attachments: item.workdir_attachments === undefined ? undefined : array( - item.workdir_attachments, - `${label}.workdir_attachments`, - workerWorkdirAttachment, - ), + workdir_attachments: item.workdir_attachments === undefined + ? undefined + : array( + item.workdir_attachments, + `${label}.workdir_attachments`, + workerWorkdirAttachment, + ), diagnostics: item.diagnostics === undefined ? [] : array(item.diagnostics, `${label}.diagnostics`, diagnostic), @@ -634,11 +656,13 @@ function workerSummary( item.implementation, `${label}.implementation`, ), - workdir_attachments: item.workdir_attachments === undefined ? undefined : array( - item.workdir_attachments, - `${label}.workdir_attachments`, - runtimeWorkdirAttachment, - ), + workdir_attachments: item.workdir_attachments === undefined + ? undefined + : array( + item.workdir_attachments, + `${label}.workdir_attachments`, + runtimeWorkdirAttachment, + ), diagnostics: array(item.diagnostics, `${label}.diagnostics`, diagnostic), }; } @@ -728,11 +752,17 @@ function featureConnectionsRequest( ): WorkspaceWorkerFeatureConnectionsRequest { const item = record(value, label); exact(item, ["subjektiv"], label); - const subjektiv = optional(item.subjektiv, `${label}.subjektiv`, (value, itemLabel) => { - const connection = record(value, itemLabel); - exact(connection, ["subject_id"], itemLabel); - return { subject_id: string(connection.subject_id, `${itemLabel}.subject_id`) }; - }); + const subjektiv = optional( + item.subjektiv, + `${label}.subjektiv`, + (value, itemLabel) => { + const connection = record(value, itemLabel); + exact(connection, ["subject_id"], itemLabel); + return { + subject_id: string(connection.subject_id, `${itemLabel}.subject_id`), + }; + }, + ); return subjektiv === undefined ? {} : { subjektiv }; } @@ -841,18 +871,32 @@ function uploadedFile( function invocationValue( value: unknown, label: string, -): Extract["invocation"]["arguments"][number], { name: string }>["value"] { +): Extract< + Extract< + Segment, + { kind: "feature_invoke" } + >["invocation"]["arguments"][number], + { name: string } +>["value"] { const item = record(value, label); const kind = string(item.kind, `${label}.kind`); exact(item, ["kind", "value"], label); - if (kind === "string") return { kind, value: string(item.value, `${label}.value`) }; + if (kind === "string") { + return { kind, value: string(item.value, `${label}.value`) }; + } if (kind === "integer") { const parsed = number(item.value, `${label}.value`); - if (!Number.isSafeInteger(parsed)) throw new Error(`${label}.value must be a safe integer`); - if (parsed < -2147483648 || parsed > 2147483647) throw new Error(`${label}.value must be a signed 32-bit integer`); + if (!Number.isSafeInteger(parsed)) { + throw new Error(`${label}.value must be a safe integer`); + } + if (parsed < -2147483648 || parsed > 2147483647) { + throw new Error(`${label}.value must be a signed 32-bit integer`); + } return { kind, value: parsed }; } - if (kind === "boolean") return { kind, value: boolean(item.value, `${label}.value`) }; + if (kind === "boolean") { + return { kind, value: boolean(item.value, `${label}.value`) }; + } throw new Error(`${label}.kind is invalid`); } @@ -866,14 +910,18 @@ function featureInvocation( invocation_id: string(item.invocation_id, `${label}.invocation_id`), identity: string(item.identity, `${label}.identity`), name: string(item.name, `${label}.name`), - arguments: array(item.arguments, `${label}.arguments`, (argument, argumentLabel) => { - const entry = record(argument, argumentLabel); - exact(entry, ["name", "value"], argumentLabel); - return { - name: string(entry.name, `${argumentLabel}.name`), - value: invocationValue(entry.value, `${argumentLabel}.value`), - }; - }), + arguments: array( + item.arguments, + `${label}.arguments`, + (argument, argumentLabel) => { + const entry = record(argument, argumentLabel); + exact(entry, ["name", "value"], argumentLabel); + return { + name: string(entry.name, `${argumentLabel}.name`), + value: invocationValue(entry.value, `${argumentLabel}.value`), + }; + }, + ), }; } diff --git a/web/workspace/src/lib/workspace/sidebar/WorkersNavSection.svelte b/web/workspace/src/lib/workspace/sidebar/WorkersNavSection.svelte index a2180f3a8..3b90e2fa7 100644 --- a/web/workspace/src/lib/workspace/sidebar/WorkersNavSection.svelte +++ b/web/workspace/src/lib/workspace/sidebar/WorkersNavSection.svelte @@ -1,8 +1,15 @@ @@ -221,6 +276,23 @@ {#if openWorkerKey === key}