From 2f74e00e07bbf5023c4780e64bfe30b91287b759 Mon Sep 17 00:00:00 2001 From: Hare Date: Sat, 31 Oct 2026 13:08:47 +0900 Subject: [PATCH] feat: preserve Subject body lifecycle candidate for integration Checkpoint all source-only T807 work, including Runtime stop fencing, Subject adapters, shared UI and unexecuted regression sources before merging published T806. Not a validated or reviewed release. --- crates/runtime-api/src/lib.rs | 97 +- crates/server-api/Cargo.toml | 4 + .../generate_subject_body_api_types.rs | 6 + crates/server-api/src/lib.rs | 68 +- crates/server-api/src/subject_body.rs | 195 +++ crates/worker-runtime/src/catalog.rs | 41 + crates/worker-runtime/src/error.rs | 18 + crates/worker-runtime/src/fs_store.rs | 79 + crates/worker-runtime/src/http_server.rs | 131 +- .../src/http_server/runtime_management_api.rs | 57 +- crates/worker-runtime/src/runtime.rs | 1297 ++++++++++++++++- .../src/feature/builtin/manage_worker.rs | 2 + crates/workspace-server/src/hosts.rs | 194 ++- crates/workspace-server/src/server.rs | 457 +++++- .../src/server/subject_authorization.rs | 1 + .../src/server/subject_body.rs | 522 +++++++ .../src/server/subjektiv_deletion.rs | 1 + .../src/server/subjektiv_job.rs | 1 + .../src/server/tests/api_logging_tests.rs | 1 + .../tests/protocol_backpressure_tests.rs | 2 + .../src/server/tests/restore_workdir_tests.rs | 2 + .../tests/restored_parent_control_tests.rs | 1 + .../tests/subject_authorization_tests.rs | 2 + .../src/server/tests/subject_body_tests.rs | 278 ++++ .../server/tests/subject_deletion_tests.rs | 2 + .../src/server/tests/subject_spawn_tests.rs | 54 + .../server/tests/subjektiv_journal_tests.rs | 4 + .../tests/ticket_worker_launch_tests.rs | 1 + .../tests/workdir_session_close_tests.rs | 135 ++ .../src/server/tests/worker_drive_tests.rs | 2 + .../server/tests/worker_operations_tests.rs | 2 + .../workspace_protocol_backpressure_tests.rs | 1 + .../src/server/worker_operations.rs | 256 +++- .../src/server_assignment_attention_tests.rs | 5 + .../src/server_assignment_delivery_tests.rs | 1 + .../src/server_plugin_controller_tests.rs | 1 + .../server_plugin_remote_controller_tests.rs | 1 + .../src/server_residual_execution_tests.rs | 1 + crates/workspace-server/src/store.rs | 14 + web/workspace/deno.json | 2 +- .../src/lib/workspace/api/runtime-workers.ts | 7 +- .../memory/SubjectBody.browser.spec.ts | 1073 ++++++++++++++ .../lib/workspace/memory/SubjectBody.svelte | 288 ++++ web/workspace/src/lib/workspace/memory/api.ts | 31 +- .../lib/workspace/memory/subject-body.test.ts | 721 +++++++++ .../src/lib/workspace/memory/subject-body.ts | 551 +++++++ .../sidebar/WorkersNavSection.svelte | 4 +- .../lib/workspace/sidebar/worker-actions.ts | 15 +- .../w/[workspaceId]/memory/PageContent.svelte | 2 +- .../memory/[subjectId]/PageContent.svelte | 19 +- 50 files changed, 6531 insertions(+), 119 deletions(-) create mode 100644 crates/server-api/examples/generate_subject_body_api_types.rs create mode 100644 crates/server-api/src/subject_body.rs create mode 100644 crates/workspace-server/src/server/subject_body.rs create mode 100644 crates/workspace-server/src/server/tests/subject_body_tests.rs create mode 100644 web/workspace/src/lib/workspace/memory/SubjectBody.browser.spec.ts create mode 100644 web/workspace/src/lib/workspace/memory/SubjectBody.svelte create mode 100644 web/workspace/src/lib/workspace/memory/subject-body.test.ts create mode 100644 web/workspace/src/lib/workspace/memory/subject-body.ts diff --git a/crates/runtime-api/src/lib.rs b/crates/runtime-api/src/lib.rs index 7457ec08b..a205e7893 100644 --- a/crates/runtime-api/src/lib.rs +++ b/crates/runtime-api/src/lib.rs @@ -623,9 +623,29 @@ pub struct WorkerDeleteResponse { } #[derive(Clone, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] pub struct WorkerLifecycleRequest { #[serde(default, skip_serializing_if = "Option::is_none")] pub reason: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stop_intent: Option, +} + +/// Fences a normal Worker stop to the observed lifecycle, not a Subject ledger. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(deny_unknown_fields)] +pub struct WorkerStopIntent { + pub expected_observation_token: String, + pub request_id: String, +} + +/// Historical stop evidence. Stopped does not describe the current execution. +#[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] +#[serde(tag = "state", rename_all = "snake_case", deny_unknown_fields)] +pub enum WorkerStopOutcome { + NotFound, + Pending, + Stopped { ack: WorkerLifecycleAck }, } #[derive(Clone, Copy, Debug, Default, Deserialize, Eq, PartialEq, Serialize)] @@ -633,6 +653,10 @@ pub struct EmptyObjectRequest {} #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] pub struct WorkerLifecycleAck { + /// Historical typed stop receipt, not a fresh stop of the current execution. + /// Consumers must not repeat cleanup of current resources when this is true. + #[serde(default)] + pub replayed: bool, pub worker_ref: WorkerRef, pub status: WorkerStatus, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -1410,13 +1434,29 @@ pub trait RuntimeApi { #[body] input: WorkerInput, ) -> Result; - #[post("/v1/workers/{worker_id}/stop", status = 200, error_status = 400)] + #[post("/v1/workers/{worker_id}/stop", status = 200, error_status = 400, additional_error_statuses = [409])] async fn stop_worker( &self, #[path] worker_id: String, #[body] request: WorkerLifecycleRequest, ) -> Result; + /// Admit the same fenced stop without backend cleanup; requires stop_intent. + #[post("/v1/workers/{worker_id}/stop/admit", status = 200, error_status = 400, additional_error_statuses = [409])] + async fn coordinate_worker_stop( + &self, + #[path] worker_id: String, + #[body] request: WorkerLifecycleRequest, + ) -> Result; + + /// Read-only exact-identity lookup; never initiates backend stop cleanup. + #[post("/v1/workers/{worker_id}/stop/outcome", status = 200, error_status = 400, additional_error_statuses = [409])] + async fn worker_stop_outcome( + &self, + #[path] worker_id: String, + #[body] request: WorkerLifecycleRequest, + ) -> Result; + #[post("/v1/workers/{worker_id}/cancel", status = 200, error_status = 400)] async fn cancel_worker( &self, @@ -1667,6 +1707,47 @@ mod tests { assert_eq!(roundtrip, binding); } + #[test] + fn lifecycle_ack_legacy_default_and_replay_marker_round_trip() { + let mut value = serde_json::json!({ + "worker_ref": {"worker_id": WorkerId::now_v7()}, "status": "stopped" + }); + let legacy: WorkerLifecycleAck = serde_json::from_value(value.clone()).unwrap(); + assert!(!legacy.replayed); + value["replayed"] = serde_json::json!(false); + assert_eq!(serde_json::to_value(legacy).unwrap(), value); + value["replayed"] = serde_json::json!(true); + let replay: WorkerLifecycleAck = serde_json::from_value(value.clone()).unwrap(); + assert!(replay.replayed); + assert_eq!(serde_json::to_value(replay).unwrap(), value); + } + + #[test] + fn lifecycle_stop_intent_defaults_and_strict_external_fields() { + for value in [ + serde_json::json!({}), + serde_json::json!({"reason": "cancel"}), + ] { + let request: WorkerLifecycleRequest = serde_json::from_value(value.clone()).unwrap(); + assert!(request.stop_intent.is_none()); + assert_eq!(serde_json::to_value(request).unwrap(), value); + } + let value = serde_json::json!({"reason":"stop", "stop_intent": { + "expected_observation_token":"observed", "request_id":"same-request" + }}); + let request: WorkerLifecycleRequest = serde_json::from_value(value.clone()).unwrap(); + assert_eq!(serde_json::to_value(request).unwrap(), value); + for invalid in [ + serde_json::json!({"unknown":true}), + serde_json::json!({"stop_intent":{}}), + serde_json::json!({"stop_intent":{"request_id":"r"}}), + serde_json::json!({"stop_intent":{"expected_observation_token":"t"}}), + serde_json::json!({"stop_intent":{"expected_observation_token":"t", "request_id":"r", "unknown":true}}), + ] { + assert!(serde_json::from_value::(invalid).is_err()); + } + } + #[test] fn restore_request_is_mandatory_and_conflict_is_not_an_operation_state() { for invalid in [ @@ -1974,6 +2055,20 @@ mod tests { ) -> Result { Err(test_error(501)) } + async fn coordinate_worker_stop( + &self, + _worker_id: String, + _request: WorkerLifecycleRequest, + ) -> Result { + Err(test_error(501)) + } + async fn worker_stop_outcome( + &self, + _worker_id: String, + _request: WorkerLifecycleRequest, + ) -> Result { + Err(test_error(501)) + } async fn cancel_worker( &self, _worker_id: String, diff --git a/crates/server-api/Cargo.toml b/crates/server-api/Cargo.toml index 75daa75fc..40b920bb3 100644 --- a/crates/server-api/Cargo.toml +++ b/crates/server-api/Cargo.toml @@ -43,6 +43,10 @@ name = "generate_worker_launch_api_types" [[example]] name = "generate_companion_api_types" +[[example]] +name = "generate_subject_body_api_types" +required-features = ["typescript"] + [[example]] name = "generate_memory_api_types" required-features = ["typescript"] diff --git a/crates/server-api/examples/generate_subject_body_api_types.rs b/crates/server-api/examples/generate_subject_body_api_types.rs new file mode 100644 index 000000000..9d82a3849 --- /dev/null +++ b/crates/server-api/examples/generate_subject_body_api_types.rs @@ -0,0 +1,6 @@ +fn main() { + print!( + "{}", + server_api::normalized_typescript(server_api::subject_body_api_typescript()) + ); +} diff --git a/crates/server-api/src/lib.rs b/crates/server-api/src/lib.rs index aa151c016..f8ac54ea1 100644 --- a/crates/server-api/src/lib.rs +++ b/crates/server-api/src/lib.rs @@ -9,6 +9,8 @@ pub use api_macros::{ApiContract, BinaryBody, HttpMethod, TransportMetadata, Web pub mod authorization; mod chat; pub use chat::*; +mod subject_body; +pub use subject_body::*; mod chat_transport; pub use chat_transport::*; mod drive; @@ -250,6 +252,11 @@ impl_openapi_schema!( PluginArtifactDeliveryRequest, PluginArtifactDeliveryMetadata, PluginWorkerDiagnostics, + SubjectBodyResponse, + SubjectBodyStartRequest, + SubjectBodyStopRequest, + SubjectBodyMutationResponse, + SubjectBodyStopOutcomeResponse, SubjectPermissionsResponse, SubjectPermissionsMutationResponse, SubjectPermissionInterruption, @@ -1540,6 +1547,45 @@ pub trait ServerApi { #[path] workspace_id: String, #[path] subject_id: String, ) -> Result; + #[get("/api/w/{workspace_id}/subjektiv/subjects/{subject_id}/body", status = 200, error_status = 404, + additional_error_statuses = [400, 401, 403, 409, 500], bearer_auth = true, browser_auth = true)] + async fn subjektiv_subject_body_get( + &self, + #[extension] context: ServerRequestContext, + #[path] workspace_id: String, + #[path] subject_id: String, + ) -> Result; + + #[post("/api/w/{workspace_id}/subjektiv/subjects/{subject_id}/body/start", status = 200, error_status = 400, + additional_error_statuses = [401, 403, 404, 409, 500], bearer_auth = true, browser_auth = true)] + async fn subjektiv_subject_body_start( + &self, + #[extension] context: ServerRequestContext, + #[path] workspace_id: String, + #[path] subject_id: String, + #[body] request: SubjectBodyStartRequest, + ) -> Result; + + #[post("/api/w/{workspace_id}/subjektiv/subjects/{subject_id}/body/stop", status = 200, error_status = 400, + additional_error_statuses = [401, 403, 404, 409, 500], bearer_auth = true, browser_auth = true)] + async fn subjektiv_subject_body_stop( + &self, + #[extension] context: ServerRequestContext, + #[path] workspace_id: String, + #[path] subject_id: String, + #[body] request: SubjectBodyStopRequest, + ) -> Result; + + #[post("/api/w/{workspace_id}/subjektiv/subjects/{subject_id}/body/stop/outcome", status = 200, error_status = 400, + additional_error_statuses = [401, 403, 404, 409, 500, 502], bearer_auth = true, browser_auth = true)] + async fn subjektiv_subject_body_stop_outcome( + &self, + #[extension] context: ServerRequestContext, + #[path] workspace_id: String, + #[path] subject_id: String, + #[body] request: SubjectBodyStopRequest, + ) -> Result; + #[patch( "/api/w/{workspace_id}/subjektiv/subjects/{subject_id}/behavior", status = 200, @@ -5687,8 +5733,12 @@ pub struct SubjektivSubjectResponse { pub memory_fingerprint: String, pub created_at: String, pub updated_at: String, - /// Current keyed-singleton owner, when one exists. This is a live Yoi - /// projection, not state duplicated in the subjektiv store. + /// Current owner remains present even when its Runtime cannot be observed. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub current_body: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub current_worker_diagnostics: Vec, + /// Available launch metadata, not evidence of absence when observation failed. #[serde(default, skip_serializing_if = "Option::is_none")] pub current_worker: Option, } @@ -9676,9 +9726,18 @@ pub struct RuntimeWorkerInputResult { pub diagnostics: Vec, } +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] +#[serde(deny_unknown_fields)] +pub struct WorkerStopIntent { + pub expected_observation_token: String, + pub request_id: String, +} + #[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] #[serde(deny_unknown_fields)] pub struct RuntimeWorkerLifecycleRequest { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stop_intent: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub reason: Option, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -9688,6 +9747,9 @@ pub struct RuntimeWorkerLifecycleRequest { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, JsonSchema)] #[serde(deny_unknown_fields)] pub struct RuntimeWorkerLifecycleResult { + /// Historical receipt replay, not a new stop or evidence of current state. + #[serde(default)] + pub replayed: bool, pub state: WorkerOperationState, pub runtime_id: String, pub worker_id: String, @@ -11261,7 +11323,7 @@ export type SubjektivSubjectDeleteResponse = { subject_id: string, operation_id: export type SubjektivSubjectBehaviorUpdateRequest = { expected_behavior_md: string, behavior_md: string, }; -export type SubjektivSubjectResponse = { id: string, role: string, behavior_md: string, state: SubjektivSubjectState, memory_fingerprint: string, created_at: string, updated_at: string, current_worker?: | import("./worker-launch-api").WorkerLaunchWorkerSummary | null, }; +export type SubjektivSubjectResponse = { id: string, role: string, behavior_md: string, state: SubjektivSubjectState, memory_fingerprint: string, created_at: string, updated_at: string, current_body?: import("./subject-body-api").SubjectBodyRef | null, current_worker_diagnostics?: Array, current_worker?: | import("./worker-launch-api").WorkerLaunchWorkerSummary | null, }; export type SubjektivSubjectListResponse = { limit: number, items: Array, next_cursor?: string | null, has_more: boolean, }; diff --git a/crates/server-api/src/subject_body.rs b/crates/server-api/src/subject_body.rs new file mode 100644 index 000000000..fac571613 --- /dev/null +++ b/crates/server-api/src/subject_body.rs @@ -0,0 +1,195 @@ +//! Subject-facing projections and intents over ordinary Worker lifecycle authority. +//! No current-body mapping or execution state is owned by these DTOs. +use crate::{Diagnostic, WorkerSummary}; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] +pub struct SubjectBodyRef { + pub runtime_id: String, + pub worker_id: String, + /// Decimal text preserves the full singleton generation through browsers. + pub generation: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum SubjectBodyObservation { + Absent, + Available { + worker: WorkerSummary, + }, + /// The owner is still occupied; this must not invite a replacement launch. + Unavailable { + diagnostics: Vec, + }, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] +pub struct SubjectBodyResponse { + pub workspace_id: String, + pub subject_id: String, + pub current: Option, + pub observation: SubjectBodyObservation, + /// Account authority, not a claim that Profile/model/Runtime admission will succeed. + pub can_start: bool, + pub can_stop: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] +pub struct SubjectBodyStartRequest { + pub request_id: String, + /// Null explicitly means the caller observed no owner, not an unspecified target. + #[serde(deserialize_with = "required_nullable_current")] + #[schemars(required)] + pub expected_current: Option, + pub expected_observation_token: Option, +} + +fn required_nullable_current<'de, D: serde::Deserializer<'de>>( + deserializer: D, +) -> Result, D::Error> { + Option::::deserialize(deserializer) +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] +pub struct SubjectBodyStopRequest { + pub request_id: String, + pub expected_current: SubjectBodyRef, + pub expected_observation_token: String, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(rename_all = "snake_case")] +pub enum SubjectBodyDisposition { + Accepted, + Rejected, + Unknown, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] +pub struct SubjectBodyMutationResponse { + pub request_id: String, + pub disposition: SubjectBodyDisposition, + pub diagnostics: Vec, + /// Fresh observation is independent of the historical operation receipt. + pub body: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(rename_all = "snake_case")] +pub enum SubjectBodyStopOutcome { + NotFound, + Pending, + Stopped, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, JsonSchema)] +#[cfg_attr(feature = "typescript", derive(ts_rs::TS))] +#[serde(deny_unknown_fields)] +pub struct SubjectBodyStopOutcomeResponse { + pub request_id: String, + /// Historical operation evidence, never the current body's running state. + pub outcome: SubjectBodyStopOutcome, + pub body: Option, +} + +#[cfg(feature = "typescript")] +pub fn subject_body_api_typescript() -> String { + use ts_rs::TS; + let config = ts_rs::Config::default(); + let declarations = [ + SubjectBodyRef::decl(&config), + SubjectBodyObservation::decl(&config), + SubjectBodyResponse::decl(&config), + SubjectBodyStartRequest::decl(&config), + SubjectBodyStopRequest::decl(&config), + SubjectBodyDisposition::decl(&config), + SubjectBodyMutationResponse::decl(&config), + SubjectBodyStopOutcome::decl(&config), + SubjectBodyStopOutcomeResponse::decl(&config), + ]; + format!( + "// Generated by `cargo run -p server-api --features typescript --example generate_subject_body_api_types`.\n// Do not edit manually.\n\nimport type {{ Diagnostic, WorkerSummary }} from './worker-launch-api';\n\n{}\n", + declarations + .into_iter() + .map(|declaration| format!("export {declaration}")) + .collect::>() + .join("\n\n") + ) +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn start_requires_an_explicit_nullable_current_observation() { + assert!( + serde_json::from_value::(json!({"request_id":"start"})) + .is_err() + ); + let request: SubjectBodyStartRequest = serde_json::from_value(json!({"request_id":"start", "expected_current":null, "expected_observation_token":null})).unwrap(); + assert!(request.expected_current.is_none()); + } + + #[cfg(feature = "typescript")] + #[test] + fn generated_subject_body_contract_is_current() { + let path = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("../../web/workspace/src/lib/generated/subject-body-api.ts"); + let actual = std::fs::read_to_string(&path) + .unwrap_or_else(|error| panic!("failed to read {}: {error}", path.display())); + let normalize = |value: &str| { + value + .chars() + .filter_map(|character| match character { + character if character.is_whitespace() => None, + ',' => Some(';'), + character => Some(character), + }) + .collect::() + .replace("=|", "=") + .replace(";}", "}") + }; + assert_eq!( + normalize(&actual), + normalize(&subject_body_api_typescript()), + "regenerate subject-body-api.ts using generate_subject_body_api_types and format the result" + ); + } + + #[test] + fn stop_intent_requires_target_and_observation() { + let valid = json!({ + "request_id": "stop-1", + "expected_current": {"runtime_id":"runtime", "worker_id":"worker", "generation":"18446744073709551615"}, + "expected_observation_token":"observed" + }); + let request: SubjectBodyStopRequest = serde_json::from_value(valid.clone()).unwrap(); + assert_eq!(request.expected_current.generation, "18446744073709551615"); + for field in [ + "expected_current", + "expected_observation_token", + "request_id", + ] { + let mut missing = valid.clone(); + missing.as_object_mut().unwrap().remove(field); + assert!(serde_json::from_value::(missing).is_err()); + } + } +} diff --git a/crates/worker-runtime/src/catalog.rs b/crates/worker-runtime/src/catalog.rs index 913a918c4..128c71eec 100644 --- a/crates/worker-runtime/src/catalog.rs +++ b/crates/worker-runtime/src/catalog.rs @@ -397,16 +397,57 @@ pub struct WorkerDetail { /// Acknowledgement returned by stop/cancel lifecycle operations. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct WorkerLifecycleAck { + /// Historical typed stop receipt; never authorizes cleanup of current resources. + #[serde(default)] + pub replayed: bool, pub worker_ref: WorkerRef, pub status: WorkerStatus, #[serde(default, skip_serializing_if = "Option::is_none")] pub worker_state: Option, } +impl WorkerLifecycleAck { + pub(crate) fn historical_stop_receipt(&self) -> Self { + Self { + replayed: true, + ..self.clone() + } + } +} + +/// Retained stop evidence, independent of the Worker's current execution. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +#[serde(tag = "state", rename_all = "snake_case", deny_unknown_fields)] +pub enum WorkerStopOutcome { + NotFound, + Pending, + Stopped { ack: WorkerLifecycleAck }, +} + #[cfg(test)] mod tests { use super::WorkspaceApiRef; + #[test] + fn legacy_stop_receipt_without_marker_is_marked_only_when_returned_as_history() { + let value = serde_json::json!({ + "worker_ref": {"worker_id": crate::identity::WorkerId::now_v7()}, "status":"stopped" + }); + let receipt: super::WorkerLifecycleAck = serde_json::from_value(value).unwrap(); + assert!(!receipt.replayed); + let returned = receipt.historical_stop_receipt(); + assert!(returned.replayed); + assert!( + !receipt.replayed, + "the retained receipt must not be rewritten" + ); + let wire = serde_json::to_value(&returned).unwrap(); + let external: runtime_api::WorkerLifecycleAck = + serde_json::from_value(wire.clone()).unwrap(); + assert!(external.replayed); + assert_eq!(serde_json::to_value(external).unwrap(), wire); + } + #[test] fn workspace_api_ref_public_schema_contains_no_source_credentials_or_claim_choices() { let value = serde_json::to_value(WorkspaceApiRef { diff --git a/crates/worker-runtime/src/error.rs b/crates/worker-runtime/src/error.rs index ef9609cc0..f299d3eb8 100644 --- a/crates/worker-runtime/src/error.rs +++ b/crates/worker-runtime/src/error.rs @@ -15,6 +15,24 @@ pub enum RuntimeError { #[error("worker {worker_id} Restore observation conflicts with the current operation")] RestoreObservationConflict { worker_id: WorkerId }, + #[error( + "worker {worker_id} lifecycle lock is busy; release caller gates and retry the same Restore intent" + )] + RestoreAdmissionBusy { worker_id: WorkerId }, + + #[error("worker {worker_id} Stop observation conflicts with the current lifecycle")] + StopObservationConflict { worker_id: WorkerId }, + + #[error( + "worker {worker_id} Stop request ID is already bound to different input or another pending stop" + )] + StopRequestConflict { worker_id: WorkerId }, + + #[error( + "worker {worker_id} lifecycle lock is busy; release caller gates and retry the same Stop intent" + )] + StopAdmissionBusy { worker_id: WorkerId }, + #[error("worker {worker_id} was not found")] WorkerNotFound { worker_id: WorkerId }, diff --git a/crates/worker-runtime/src/fs_store.rs b/crates/worker-runtime/src/fs_store.rs index fe750e2ff..28f005b10 100644 --- a/crates/worker-runtime/src/fs_store.rs +++ b/crates/worker-runtime/src/fs_store.rs @@ -278,11 +278,27 @@ impl FsRuntimeStore { atomic_write_json(&self.runtime_path(), &snapshot, "write runtime snapshot") } + /// Use the canonical pretty-JSON representation, including its final newline, + /// to refuse writes that our bounded canonical reader could not reopen. + pub(crate) fn validate_worker_record_size( + &self, + worker: &PersistedWorkerRecord, + ) -> Result<(), RuntimeError> { + let bytes = worker_record_serialized_size(worker)?; + if bytes > MAX_WORKER_RECORD_BYTES { + return Err(RuntimeError::InvalidRequest(format!( + "Worker aggregate requires {bytes} bytes, exceeding canonical reader limit {MAX_WORKER_RECORD_BYTES}; retained lifecycle owners cannot be evicted" + ))); + } + Ok(()) + } + pub(crate) fn write_worker_record( &self, worker: &PersistedWorkerRecord, ) -> Result<(), RuntimeError> { self.ensure_worker_ref(&worker.worker_ref)?; + self.validate_worker_record_size(worker)?; let worker_dir = self.worker_dir(&worker.worker_id); fs::create_dir_all(&worker_dir).map_err(|source| RuntimeError::StoreIo { operation: "create Worker store", @@ -3033,6 +3049,33 @@ where }) } +// Count rather than allocate a second aggregate-sized buffer during admission. +// This is the same serializer and trailing newline as atomic_write_json_with_fault. +fn worker_record_serialized_size(worker: &PersistedWorkerRecord) -> Result { + struct ByteCount(u64); + impl Write for ByteCount { + fn write(&mut self, bytes: &[u8]) -> std::io::Result { + self.0 = self + .0 + .checked_add(bytes.len() as u64) + .ok_or_else(|| std::io::Error::other("Worker aggregate size overflow"))?; + Ok(bytes.len()) + } + fn flush(&mut self) -> std::io::Result<()> { + Ok(()) + } + } + let mut count = ByteCount(0); + serde_json::to_writer_pretty(&mut count, &WorkerAggregateRecord::from_persisted(worker)) + .map_err(|error| { + RuntimeError::InvalidRequest(format!("serialize Worker aggregate: {error}")) + })?; + count + .write_all(b"\n") + .map_err(|error| RuntimeError::InvalidRequest(error.to_string()))?; + Ok(count.0) +} + fn atomic_write_json(path: &Path, value: &T, operation: &'static str) -> Result<(), RuntimeError> where T: Serialize, @@ -4024,6 +4067,42 @@ mod tests { assert!(error.to_string().contains("exceeds limit")); } + #[test] + fn canonical_worker_writer_enforces_reader_byte_limit_including_final_newline() { + let (_temp, store, mut worker, path) = canonical_worker_store(); + worker.execution_state = PersistedWorkerExecutionState::Unavailable; + worker.status = WorkerStatus::Stopped; + worker.display_name = Some(String::new()); + let base = worker_record_serialized_size(&worker).unwrap(); + worker.display_name = Some("x".repeat((MAX_WORKER_RECORD_BYTES - base) as usize)); + store.write_worker_record(&worker).unwrap(); + assert_eq!(fs::metadata(&path).unwrap().len(), MAX_WORKER_RECORD_BYTES); + assert_eq!( + worker_record_serialized_size(&worker).unwrap(), + MAX_WORKER_RECORD_BYTES + ); + store.read_worker_record(&worker.worker_ref).unwrap(); + let before = fs::read(&path).unwrap(); + worker.display_name.as_mut().unwrap().push('x'); + assert_eq!( + worker_record_serialized_size(&worker).unwrap(), + MAX_WORKER_RECORD_BYTES + 1 + ); + assert!( + matches!(store.write_worker_record(&worker), Err(RuntimeError::InvalidRequest(message)) if message.contains("canonical reader limit")) + ); + assert_eq!(fs::read(&path).unwrap(), before); + assert_eq!( + store + .read_worker_record(&worker.worker_ref) + .unwrap() + .display_name + .unwrap() + .len(), + (MAX_WORKER_RECORD_BYTES - base) as usize + ); + } + #[test] fn worker_aggregate_write_distinguishes_pre_and_post_rename_failures() { let temp = tempfile::tempdir().unwrap(); diff --git a/crates/worker-runtime/src/http_server.rs b/crates/worker-runtime/src/http_server.rs index fd87a6a04..90a54f04c 100644 --- a/crates/worker-runtime/src/http_server.rs +++ b/crates/worker-runtime/src/http_server.rs @@ -523,11 +523,7 @@ pub type RuntimeHttpWorkerCompletionsRequest = runtime_api::CompletionRequest; pub type RuntimeHttpWorkerCompletionsResponse = runtime_api::CompletionResponse; /// Worker lifecycle request body used by stop/cancel endpoints. -#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] -pub struct RuntimeHttpWorkerLifecycleRequest { - #[serde(default, skip_serializing_if = "Option::is_none")] - pub reason: Option, -} +pub type RuntimeHttpWorkerLifecycleRequest = runtime_api::WorkerLifecycleRequest; /// Worker lifecycle acknowledgement response. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] @@ -1902,8 +1898,8 @@ async fn stop_worker( let ack = match auth_workspace_scope(&state, auth.as_ref())? { Some(scope) => state .runtime - .stop_worker_scoped(&scope, &worker_ref, request.reason), - None => state.runtime.stop_worker(&worker_ref, request.reason), + .stop_worker_request_scoped(&scope, &worker_ref, request), + None => state.runtime.stop_worker_request(&worker_ref, request), } .map_err(RuntimeHttpRestError::runtime)?; Ok(Json(RuntimeHttpWorkerLifecycleResponse { ack })) @@ -1918,6 +1914,7 @@ async fn cancel_worker( ) -> RestResult { let worker_ref = worker_ref_for(&state.runtime, worker_id)?; let request = parse_optional_lifecycle_request(body)?; + validate_cancel_request(&request)?; let ack = match auth_workspace_scope(&state, auth.as_ref())? { Some(scope) => state .runtime @@ -1942,6 +1939,19 @@ fn worker_ref_for( Ok(WorkerRef::new(worker_id)) } +fn validate_cancel_request( + request: &runtime_api::WorkerLifecycleRequest, +) -> Result<(), RuntimeHttpRestError> { + if request.stop_intent.is_some() { + return Err(RuntimeHttpRestError::new( + StatusCode::BAD_REQUEST, + "invalid_request", + "stop_intent is only valid for stop, not cancel", + )); + } + Ok(()) +} + fn parse_optional_lifecycle_request( body: Bytes, ) -> Result { @@ -2160,6 +2170,8 @@ fn required_runtime_permission(method: &Method, path: &str) -> Option<&'static s return Some("workers:delete"); } if path.ends_with("/stop") + || path.ends_with("/stop/outcome") + || path.ends_with("/stop/admit") || path.ends_with("/cancel") || (path == "/v1/subject-release" && *method == Method::POST) { @@ -2432,6 +2444,10 @@ fn status_for_runtime_error(error: &RuntimeError) -> StatusCode { } RuntimeError::RuntimeStopped | RuntimeError::RestoreObservationConflict { .. } + | RuntimeError::RestoreAdmissionBusy { .. } + | RuntimeError::StopObservationConflict { .. } + | RuntimeError::StopRequestConflict { .. } + | RuntimeError::StopAdmissionBusy { .. } | RuntimeError::RuntimeStoreAlreadyOpen { .. } | RuntimeError::WorkerExecutionUnavailable { .. } | RuntimeError::ExecutionBackendUnavailable { .. } @@ -2459,6 +2475,10 @@ fn code_for_runtime_error(error: &RuntimeError) -> String { RuntimeError::RestoreObservationConflict { .. } => { "restore_observation_conflict".to_string() } + RuntimeError::RestoreAdmissionBusy { .. } => "restore_admission_busy".to_string(), + RuntimeError::StopObservationConflict { .. } => "stop_observation_conflict".to_string(), + RuntimeError::StopRequestConflict { .. } => "stop_request_conflict".to_string(), + RuntimeError::StopAdmissionBusy { .. } => "stop_admission_busy".to_string(), RuntimeError::RuntimeStoreAlreadyOpen { .. } => "runtime_store_already_open".to_string(), RuntimeError::WorkerNotFound { .. } => "worker_not_found".to_string(), RuntimeError::WorkerExecutionUnavailable { .. } => { @@ -3728,13 +3748,80 @@ mod tests { input.submission_request_id ); } - client - .stop_worker( + let stop_request = runtime_api::WorkerLifecycleRequest { + reason: Some("generated fenced stop".into()), + stop_intent: Some(runtime_api::WorkerStopIntent { + expected_observation_token: client + .get_worker(generated_worker_id.clone()) + .await + .unwrap() + .worker + .restore_observation_token + .unwrap(), + request_id: "generated-stop".into(), + }), + }; + assert_eq!( + client + .worker_stop_outcome(generated_worker_id.clone(), stop_request.clone()) + .await + .unwrap(), + runtime_api::WorkerStopOutcome::NotFound + ); + let invalid_cancel = client + .cancel_worker(generated_worker_id.clone(), stop_request.clone()) + .await + .unwrap_err(); + assert!( + matches!(invalid_cancel, runtime_api::client_support::ClientError::Public { status, .. } if status == StatusCode::BAD_REQUEST) + ); + assert_eq!( + client + .coordinate_worker_stop(generated_worker_id.clone(), stop_request.clone()) + .await + .unwrap(), + runtime_api::WorkerStopOutcome::Pending + ); + assert_eq!( + client + .coordinate_worker_stop(generated_worker_id.clone(), stop_request.clone()) + .await + .unwrap(), + runtime_api::WorkerStopOutcome::Pending + ); + assert_eq!( + client + .get_worker(generated_worker_id.clone()) + .await + .unwrap() + .worker + .status, + runtime_api::WorkerStatus::Idle + ); + let mut stop_receipt = client + .stop_worker(generated_worker_id.clone(), stop_request.clone()) + .await + .unwrap() + .ack; + let cancelled = client + .cancel_worker( generated_worker_id.clone(), runtime_api::WorkerLifecycleRequest::default(), ) .await .unwrap(); + assert!(!cancelled.ack.replayed); + assert!(!stop_receipt.replayed); + stop_receipt.replayed = true; // Outcome and retry return historical evidence. + assert_eq!( + client + .worker_stop_outcome(generated_worker_id.clone(), stop_request.clone()) + .await + .unwrap(), + runtime_api::WorkerStopOutcome::Stopped { + ack: stop_receipt.clone() + } + ); client .restore_worker( generated_worker_id.clone(), @@ -3752,6 +3839,32 @@ mod tests { ) .await .unwrap(); + assert_eq!( + client + .stop_worker(generated_worker_id.clone(), stop_request.clone()) + .await + .unwrap() + .ack, + stop_receipt + ); + assert_eq!( + client + .get_worker(generated_worker_id.clone()) + .await + .unwrap() + .worker + .status, + runtime_api::WorkerStatus::Idle + ); + let mut changed_stop = stop_request.clone(); + changed_stop.reason = Some("different request input".into()); + let conflict = client + .stop_worker(generated_worker_id.clone(), changed_stop) + .await + .unwrap_err(); + assert!( + matches!(conflict, runtime_api::client_support::ClientError::Public { status, .. } if status == StatusCode::CONFLICT) + ); let conflict = client.create_worker(conflicting_request).await.unwrap_err(); assert!( matches!( diff --git a/crates/worker-runtime/src/http_server/runtime_management_api.rs b/crates/worker-runtime/src/http_server/runtime_management_api.rs index 8d3a1157e..860a8f266 100644 --- a/crates/worker-runtime/src/http_server/runtime_management_api.rs +++ b/crates/worker-runtime/src/http_server/runtime_management_api.rs @@ -678,11 +678,12 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi { let ack = match auth_workspace_scope(&self.state, auth_extension().as_ref()) .map_err(api_error)? { - Some(scope) => self - .state - .runtime - .stop_worker_scoped(&scope, &worker_ref, value.reason), - None => self.state.runtime.stop_worker(&worker_ref, value.reason), + Some(scope) => { + self.state + .runtime + .stop_worker_request_scoped(&scope, &worker_ref, value) + } + None => self.state.runtime.stop_worker_request(&worker_ref, value), } .map_err(RuntimeHttpRestError::runtime) .map_err(api_error)?; @@ -690,11 +691,57 @@ impl runtime_api::RuntimeApi for RuntimeManagementApi { Ok(runtime_api::WorkerLifecycleResponse { ack }) } + async fn coordinate_worker_stop( + &self, + worker_id: String, + value: runtime_api::WorkerLifecycleRequest, + ) -> Result { + let worker_ref = worker_ref_for(&self.state.runtime, worker_id).map_err(api_error)?; + let outcome = match auth_workspace_scope(&self.state, auth_extension().as_ref()) + .map_err(api_error)? + { + Some(scope) => { + self.state + .runtime + .coordinate_worker_stop_scoped(&scope, &worker_ref, value) + } + None => self + .state + .runtime + .coordinate_worker_stop(&worker_ref, value), + } + .map_err(RuntimeHttpRestError::runtime) + .map_err(api_error)?; + response(outcome) + } + + async fn worker_stop_outcome( + &self, + worker_id: String, + value: runtime_api::WorkerLifecycleRequest, + ) -> Result { + let worker_ref = worker_ref_for(&self.state.runtime, worker_id).map_err(api_error)?; + let outcome = match auth_workspace_scope(&self.state, auth_extension().as_ref()) + .map_err(api_error)? + { + Some(scope) => { + self.state + .runtime + .worker_stop_outcome_scoped(&scope, &worker_ref, value) + } + None => self.state.runtime.worker_stop_outcome(&worker_ref, value), + } + .map_err(RuntimeHttpRestError::runtime) + .map_err(api_error)?; + response(outcome) + } + async fn cancel_worker( &self, worker_id: String, value: runtime_api::WorkerLifecycleRequest, ) -> Result { + validate_cancel_request(&value).map_err(api_error)?; let worker_ref = worker_ref_for(&self.state.runtime, worker_id).map_err(api_error)?; let ack = match auth_workspace_scope(&self.state, auth_extension().as_ref()) .map_err(api_error)? diff --git a/crates/worker-runtime/src/runtime.rs b/crates/worker-runtime/src/runtime.rs index 72d78e499..c27c76e27 100644 --- a/crates/worker-runtime/src/runtime.rs +++ b/crates/worker-runtime/src/runtime.rs @@ -2140,9 +2140,20 @@ impl Runtime { RuntimeError, > { let operation_lock = self.worker_operation_lock(worker_ref.worker_id)?; - let _operation_guard = operation_lock - .lock() - .map_err(|_| RuntimeError::StatePoisoned)?; + let _operation_guard = if coordinate { + // Admission may run under a Backend gate needed by in-flight + // lifecycle callbacks; never wait for their Worker lock here. + operation_lock.try_lock().map_err(|error| match error { + std::sync::TryLockError::WouldBlock => RuntimeError::RestoreAdmissionBusy { + worker_id: worker_ref.worker_id, + }, + std::sync::TryLockError::Poisoned(_) => RuntimeError::StatePoisoned, + })? + } else { + operation_lock + .lock() + .map_err(|_| RuntimeError::StatePoisoned)? + }; self.lock()? .ensure_subject_not_reserved(worker_ref, false)?; if let Some(scope) = scope { @@ -3881,7 +3892,14 @@ impl Runtime { worker_ref: &WorkerRef, reason: Option, ) -> Result { - self.stop_worker_with_scope(worker_ref, reason, Some(scope)) + self.stop_worker_request_scoped( + scope, + worker_ref, + runtime_api::WorkerLifecycleRequest { + reason, + ..Default::default() + }, + ) } /// Stop a Worker. Repeated stops are idempotent. The per-Worker lifecycle @@ -3892,24 +3910,246 @@ impl Runtime { worker_ref: &WorkerRef, reason: Option, ) -> Result { - self.stop_worker_with_scope(worker_ref, reason, None) + self.stop_worker_request( + worker_ref, + runtime_api::WorkerLifecycleRequest { + reason, + ..Default::default() + }, + ) + } + + /// Stop with durable caller identity and an observation fence. Exact replay + /// returns only the historical receipt, even after a subsequent Restore. + pub fn stop_worker_request_scoped( + &self, + scope: &RuntimeWorkspaceScope, + worker_ref: &WorkerRef, + request: runtime_api::WorkerLifecycleRequest, + ) -> Result { + self.stop_worker_with_scope(worker_ref, request, Some(scope)) + } + + pub fn stop_worker_request( + &self, + worker_ref: &WorkerRef, + request: runtime_api::WorkerLifecycleRequest, + ) -> Result { + self.stop_worker_with_scope(worker_ref, request, None) + } + + /// Look up an exact stop intent without admitting it, reconciling backend + /// cleanup, rotating observations, or changing lifecycle state. Canonical + /// reads may refresh an earlier uncertain write; errors remain unknown. + pub fn worker_stop_outcome( + &self, + worker_ref: &WorkerRef, + request: runtime_api::WorkerLifecycleRequest, + ) -> Result { + self.worker_stop_outcome_with_scope(worker_ref, request, None) + } + + pub fn worker_stop_outcome_scoped( + &self, + scope: &RuntimeWorkspaceScope, + worker_ref: &WorkerRef, + request: runtime_api::WorkerLifecycleRequest, + ) -> Result { + self.worker_stop_outcome_with_scope(worker_ref, request, Some(scope)) + } + + fn worker_stop_outcome_with_scope( + &self, + worker_ref: &WorkerRef, + request: runtime_api::WorkerLifecycleRequest, + scope: Option<&RuntimeWorkspaceScope>, + ) -> Result { + let operation_lock = self.worker_operation_lock(worker_ref.worker_id)?; + let _operation_guard = operation_lock + .lock() + .map_err(|_| RuntimeError::StatePoisoned)?; + if let Some(scope) = scope { + let mut state = self.lock()?; + if !state + .worker(worker_ref)? + .belongs_to_workspace(&scope.workspace_id) + { + return Err(RuntimeError::WorkerNotFound { + worker_id: worker_ref.worker_id, + }); + } + state.ensure_workspace_owner(scope, false)?; + } + let intent = validate_stop_intent(&request)?.ok_or_else(|| { + RuntimeError::InvalidRequest("stop outcome requires stop_intent".into()) + })?; + let mut state = self.lock()?; + state.refresh_uncertain_worker_write(worker_ref)?; + let worker = state.worker(worker_ref)?; + let Some(owner) = worker.restore_guard.stop_owners.get(&intent.request_id) else { + return Ok(crate::catalog::WorkerStopOutcome::NotFound); + }; + owner.ensure_request(worker_ref, &request)?; + if let Some(ack) = &owner.receipt { + return Ok(crate::catalog::WorkerStopOutcome::Stopped { + ack: ack.historical_stop_receipt(), + }); + } + owner.ensure_pending(worker_ref, worker.pending_stop)?; + Ok(crate::catalog::WorkerStopOutcome::Pending) + } + + /// Admit the exact stop under the Worker lifecycle lock without invoking + /// backend cleanup. Callers must release their own lifecycle gates before + /// dispatching the same request through stop_worker_request. + pub fn coordinate_worker_stop( + &self, + worker_ref: &WorkerRef, + request: runtime_api::WorkerLifecycleRequest, + ) -> Result { + self.coordinate_worker_stop_with_scope(worker_ref, request, None) + } + + pub fn coordinate_worker_stop_scoped( + &self, + scope: &RuntimeWorkspaceScope, + worker_ref: &WorkerRef, + request: runtime_api::WorkerLifecycleRequest, + ) -> Result { + self.coordinate_worker_stop_with_scope(worker_ref, request, Some(scope)) + } + + fn coordinate_worker_stop_with_scope( + &self, + worker_ref: &WorkerRef, + request: runtime_api::WorkerLifecycleRequest, + scope: Option<&RuntimeWorkspaceScope>, + ) -> Result { + if validate_stop_intent(&request)?.is_none() { + return Err(RuntimeError::InvalidRequest( + "stop admission requires stop_intent".into(), + )); + } + if let Some(scope) = scope { + let mut state = self.lock()?; + if !state + .worker(worker_ref)? + .belongs_to_workspace(&scope.workspace_id) + { + return Err(RuntimeError::WorkerNotFound { + worker_id: worker_ref.worker_id, + }); + } + state.ensure_workspace_owner(scope, false)?; + } + let operation_lock = self.worker_operation_lock(worker_ref.worker_id)?; + // A caller may hold a Backend gate needed by an in-flight cleanup's + // callbacks. Never wait for that cleanup while retaining the gate. + let _operation_guard = operation_lock.try_lock().map_err(|error| match error { + std::sync::TryLockError::WouldBlock => RuntimeError::StopAdmissionBusy { + worker_id: worker_ref.worker_id, + }, + std::sync::TryLockError::Poisoned(_) => RuntimeError::StatePoisoned, + })?; + match self.admit_worker_stop_under_lock(worker_ref, request, scope)? { + StopAdmission::Pending(_) => Ok(crate::catalog::WorkerStopOutcome::Pending), + StopAdmission::Stopped(ack) => Ok(crate::catalog::WorkerStopOutcome::Stopped { + ack: ack.historical_stop_receipt(), + }), + } } fn stop_worker_with_scope( &self, worker_ref: &WorkerRef, - reason: Option, + request: runtime_api::WorkerLifecycleRequest, scope: Option<&RuntimeWorkspaceScope>, ) -> Result { let operation_lock = self.worker_operation_lock(worker_ref.worker_id)?; let _operation_guard = operation_lock .lock() .map_err(|_| RuntimeError::StatePoisoned)?; + let operation_id = match self.admit_worker_stop_under_lock(worker_ref, request, scope)? { + StopAdmission::Stopped(ack) => return Ok(ack), + StopAdmission::Pending(operation_id) => operation_id, + }; + let backend = self.lock()?.execution_backend.clone().ok_or_else(|| { + RuntimeError::WorkerExecutionUnavailable { + worker_id: worker_ref.worker_id, + message: "runtime has no execution backend".into(), + } + })?; + let result = backend.stop_worker_operation(WorkerExecutionStopRequest { + operation_id, + worker_ref: worker_ref.clone(), + }); + if !result.is_accepted() { + return Err(RuntimeError::WorkerExecutionRejected { + worker_id: worker_ref.worker_id, + operation: result.operation, + outcome: result.outcome, + message: result.message_or_default(), + result, + }); + } + self.commit_stopped_worker(worker_ref, operation_id) + } + + fn admit_worker_stop_under_lock( + &self, + worker_ref: &WorkerRef, + request: runtime_api::WorkerLifecycleRequest, + scope: Option<&RuntimeWorkspaceScope>, + ) -> Result { if let Some(scope) = scope { - self.ensure_worker_in_workspace(scope, worker_ref)?; + let mut state = self.lock()?; + if !state + .worker(worker_ref)? + .belongs_to_workspace(&scope.workspace_id) + { + return Err(RuntimeError::WorkerNotFound { + worker_id: worker_ref.worker_id, + }); + } + // Replays and rejected fences do not bind ownership or write snapshots. + state.ensure_workspace_owner(scope, false)?; } self.lock()?.refresh_uncertain_worker_write(worker_ref)?; + let intent = validate_stop_intent(&request)?; + if let Some(intent) = intent { + let state = self.lock()?; + let worker = state.worker(worker_ref)?; + if let Some(owner) = worker.restore_guard.stop_owners.get(&intent.request_id) { + owner.ensure_request(worker_ref, &request)?; + if let Some(ack) = &owner.receipt { + return Ok(StopAdmission::Stopped(ack.historical_stop_receipt())); + } + owner.ensure_pending(worker_ref, worker.pending_stop)?; + } else { + if intent.expected_observation_token != worker.restore_guard.observation_token { + return Err(RuntimeError::StopObservationConflict { + worker_id: worker_ref.worker_id, + }); + } + // A public caller may recover a legacy pending stop, but cannot + // steal another public intent's pending operation. + if worker.pending_stop.is_some_and(|pending| { + worker + .restore_guard + .stop_owners + .values() + .any(|owner| owner.operation_id == pending.operation_id) + }) { + return Err(RuntimeError::StopRequestConflict { + worker_id: worker_ref.worker_id, + }); + } + } + } + if let Some(scope) = scope { + self.ensure_worker_in_workspace(scope, worker_ref)?; + } self.lock()?.ensure_subject_not_reserved(worker_ref, true)?; let (backend, pending_stop, pending_restore, status) = { let state = self.lock()?; @@ -3929,44 +4169,67 @@ impl Runtime { worker .restore_guard .finish(RuntimeWorkerRestoreResult::superseded_by_stop(None)); - state.persist_worker_record(&worker)?; let ack = WorkerLifecycleAck { + replayed: false, worker_ref: worker_ref.clone(), status: worker.status, worker_state: worker.worker_state.clone(), }; + if let Some(intent) = intent { + worker.restore_guard.stop_owners.insert( + intent.request_id.clone(), + StopRequestOwner { + request: request.clone(), + operation_id: WorkerLifecycleOperationId::new(), + receipt: Some(ack.clone()), + }, + ); + } + state.persist_stop_worker_record(&worker)?; state.workers.insert(worker_ref.worker_id, worker); state.publish_worker_upsert(worker_ref.worker_id)?; - return Ok(ack); + return Ok(StopAdmission::Stopped(ack)); } - let backend = backend.ok_or_else(|| RuntimeError::WorkerExecutionUnavailable { + let _backend = backend.ok_or_else(|| RuntimeError::WorkerExecutionUnavailable { worker_id: worker_ref.worker_id, message: "runtime has no execution backend".to_string(), })?; let operation_id = match pending_stop { - Some(pending) => pending.operation_id, + Some(pending) => { + if let Some(intent) = intent { + let mut state = self.lock()?; + let mut candidate = state.worker(worker_ref)?.clone(); + if !candidate + .restore_guard + .stop_owners + .contains_key(&intent.request_id) + { + candidate.restore_guard.stop_owners.insert( + intent.request_id.clone(), + StopRequestOwner { + request: request.clone(), + operation_id: pending.operation_id, + receipt: None, + }, + ); + state.preflight_stop_owner_growth(&candidate)?; + state.persist_stop_worker_record(&candidate)?; + state.workers.insert(worker_ref.worker_id, candidate); + } + } + pending.operation_id + } None => { let operation_id = WorkerLifecycleOperationId::new(); - self.begin_stop_operation(worker_ref, operation_id)?; + self.begin_stop_operation_request( + worker_ref, + operation_id, + intent.map(|_| request.clone()), + )?; operation_id } }; - let result = backend.stop_worker_operation(WorkerExecutionStopRequest { - operation_id, - worker_ref: worker_ref.clone(), - }); - if !result.is_accepted() { - return Err(RuntimeError::WorkerExecutionRejected { - worker_id: worker_ref.worker_id, - operation: result.operation, - outcome: result.outcome, - message: result.message_or_default(), - result, - }); - } - let ack = self.commit_stopped_worker(worker_ref, operation_id)?; - let _ = reason; - Ok(ack) + Ok(StopAdmission::Pending(operation_id)) } /// Cancel a Worker through a workspace-scoped Runtime authorization context. @@ -4013,6 +4276,7 @@ impl Runtime { } if worker.status == WorkerStatus::Stopped { return Ok(WorkerLifecycleAck { + replayed: false, worker_ref: worker_ref.clone(), status: WorkerStatus::Stopped, worker_state: None, @@ -4024,6 +4288,7 @@ impl Runtime { let state = self.lock()?; let worker = state.worker(worker_ref)?; Ok(WorkerLifecycleAck { + replayed: false, worker_ref: worker_ref.clone(), status: worker.status, worker_state: worker.worker_state.clone(), @@ -4583,6 +4848,15 @@ impl Runtime { &self, worker_ref: &WorkerRef, operation_id: WorkerLifecycleOperationId, + ) -> Result<(), RuntimeError> { + self.begin_stop_operation_request(worker_ref, operation_id, None) + } + + fn begin_stop_operation_request( + &self, + worker_ref: &WorkerRef, + operation_id: WorkerLifecycleOperationId, + request: Option, ) -> Result<(), RuntimeError> { let mut state = self.lock()?; let mut candidate = state.worker(worker_ref)?.clone(); @@ -4603,7 +4877,19 @@ impl Runtime { operation_id, last_settled_status: candidate.status, }); - match state.persist_worker_record(&candidate) { + if let Some(request) = request { + let intent = request.stop_intent.as_ref().expect("validated stop intent"); + candidate.restore_guard.stop_owners.insert( + intent.request_id.clone(), + StopRequestOwner { + request, + operation_id, + receipt: None, + }, + ); + } + state.preflight_stop_owner_growth(&candidate)?; + match state.persist_stop_worker_record(&candidate) { Ok(()) => { state.workers.insert(worker_ref.worker_id, candidate); Ok(()) @@ -4632,26 +4918,11 @@ impl Runtime { worker_ref.worker_id ))); } - let restore_receipt = - RuntimeWorkerRestoreResult::superseded_by_stop(candidate.pending_restore); - candidate.pending_stop = None; - candidate.pending_restore = None; if let Some(context) = candidate.restore_candidate_context.take() { context.discard_candidate(); } - candidate.restore_guard.rotate(); - candidate.status = WorkerStatus::Stopped; - candidate.worker_state = None; - candidate.restore_intent = WorkerRestoreIntent::Explicit; - - candidate.restore_guard.finish(restore_receipt); - candidate.internal_workers.clear(); - state.persist_worker_record(&candidate)?; - let ack = WorkerLifecycleAck { - worker_ref: worker_ref.clone(), - status: candidate.status, - worker_state: None, - }; + let (candidate, ack) = candidate.stopped_candidate(operation_id); + state.persist_stop_worker_record(&candidate)?; state.workers.insert(worker_ref.worker_id, candidate); state.publish_worker_upsert(worker_ref.worker_id)?; if let Err(error) = state.persist_runtime_snapshot() { @@ -5859,6 +6130,30 @@ impl RuntimeState { Ok(()) } + fn preflight_stop_owner_growth(&self, worker: &WorkerRecord) -> Result<(), RuntimeError> { + #[cfg(feature = "fs-store")] + if let Some(store) = self.fs_store() { + store.validate_worker_record_size(&worker.persisted_record())?; + if let Some(pending) = worker.pending_stop { + // Reserve the receipt and any superseded Restore receipt too, + // not merely the smaller pending admission representation. + let (settled, _) = worker.stopped_candidate(pending.operation_id); + store.validate_worker_record_size(&settled.persisted_record())?; + } + } + let _ = worker; + Ok(()) + } + + /// An unknown Stop write fences later mutations until a canonical reread. + fn persist_stop_worker_record(&mut self, worker: &WorkerRecord) -> Result<(), RuntimeError> { + let result = self.persist_worker_record(worker); + if matches!(&result, Err(RuntimeError::StoreCommitOutcomeUnknown { .. })) { + self.unreconciled_worker_writes.insert(worker.worker_id); + } + result + } + /// An uncertain Restore write is resolved only from the locked canonical /// aggregate and a successful durability barrier, never a second blind write. fn persist_restore_worker_record(&mut self, worker: &WorkerRecord) -> Result<(), RuntimeError> { @@ -6788,12 +7083,71 @@ struct PendingWorkerRestore { last_settled_status: WorkerStatus, } +enum StopAdmission { + Pending(WorkerLifecycleOperationId), + Stopped(WorkerLifecycleAck), +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] struct PendingWorkerStop { operation_id: WorkerLifecycleOperationId, last_settled_status: WorkerStatus, } +fn validate_stop_intent( + request: &runtime_api::WorkerLifecycleRequest, +) -> Result, RuntimeError> { + if let Some(intent) = &request.stop_intent + && (intent.request_id.trim().is_empty() + || intent.request_id.len() > 512 + || intent.expected_observation_token.is_empty() + || intent.expected_observation_token.len() > 256) + { + return Err(RuntimeError::InvalidRequest( + "Stop requires a nonempty request_id (max 512 bytes) and expected_observation_token (max 256 bytes)".into(), + )); + } + Ok(request.stop_intent.as_ref()) +} + +/// Stop ownership is part of the same canonical lifecycle aggregate as Restore. +/// Full typed input equality binds reason as well as the observation and ID. +#[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] +#[serde(deny_unknown_fields)] +struct StopRequestOwner { + request: runtime_api::WorkerLifecycleRequest, + operation_id: WorkerLifecycleOperationId, + receipt: Option, +} + +impl StopRequestOwner { + fn ensure_request( + &self, + worker_ref: &WorkerRef, + request: &runtime_api::WorkerLifecycleRequest, + ) -> Result<(), RuntimeError> { + if self.request != *request { + return Err(RuntimeError::StopRequestConflict { + worker_id: worker_ref.worker_id, + }); + } + Ok(()) + } + + fn ensure_pending( + &self, + worker_ref: &WorkerRef, + pending: Option, + ) -> Result<(), RuntimeError> { + if pending.is_none_or(|pending| pending.operation_id != self.operation_id) { + return Err(RuntimeError::StopRequestConflict { + worker_id: worker_ref.worker_id, + }); + } + Ok(()) + } +} + /// Durable Restore admission/replay metadata. Never discard receipts on stop. #[derive(Clone, Debug, serde::Serialize, serde::Deserialize)] pub(crate) struct RestoreRequestOwner { @@ -6848,6 +7202,12 @@ pub(crate) struct RestoreGuard { observation_token: String, active_request_id: Option, owners: BTreeMap, + // Older aggregates have no public stop owner. Existing pending_stop remains + // their recovery authority; do not synthesize public request receipts. + // Like Restore owners, retain these for the Worker aggregate's lifetime: + // eviction would forget request-ID conflicts and historical receipts. + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + stop_owners: BTreeMap, } impl Default for RestoreGuard { @@ -6856,6 +7216,7 @@ impl Default for RestoreGuard { observation_token: uuid::Uuid::now_v7().to_string(), active_request_id: None, owners: BTreeMap::new(), + stop_owners: BTreeMap::new(), } } } @@ -6917,6 +7278,38 @@ struct WorkerRecord { } impl WorkerRecord { + // Pure projection shared by terminal commit and stop-owner capacity preflight. + // Resource cleanup is deliberately outside this method. + fn stopped_candidate( + &self, + operation_id: WorkerLifecycleOperationId, + ) -> (Self, WorkerLifecycleAck) { + let mut candidate = self.clone(); + let restore_receipt = + RuntimeWorkerRestoreResult::superseded_by_stop(candidate.pending_restore); + candidate.pending_stop = None; + candidate.pending_restore = None; + candidate.restore_candidate_context = None; + candidate.restore_guard.rotate(); + candidate.status = WorkerStatus::Stopped; + candidate.worker_state = None; + candidate.restore_intent = WorkerRestoreIntent::Explicit; + candidate.restore_guard.finish(restore_receipt); + candidate.internal_workers.clear(); + let ack = WorkerLifecycleAck { + replayed: false, + worker_ref: candidate.worker_ref.clone(), + status: candidate.status, + worker_state: None, + }; + for owner in candidate.restore_guard.stop_owners.values_mut() { + if owner.operation_id == operation_id && owner.receipt.is_none() { + owner.receipt = Some(ack.clone()); + } + } + (candidate, ack) + } + fn ensure_restore_rollback_allowed(&self) -> Result<(), RuntimeError> { if self.pending_restore.is_some_and(|pending| { pending.commit != RestoreCommitFact::NotCommitted || pending.source_repair_possible @@ -7567,6 +7960,10 @@ fn runtime_worker_create_failure_fields( RuntimeError::RestoreObservationConflict { .. } => { ("restore_observation_conflict", None, None) } + RuntimeError::RestoreAdmissionBusy { .. } => ("restore_admission_busy", None, None), + RuntimeError::StopObservationConflict { .. } => ("stop_observation_conflict", None, None), + RuntimeError::StopRequestConflict { .. } => ("stop_request_conflict", None, None), + RuntimeError::StopAdmissionBusy { .. } => ("stop_admission_busy", None, None), RuntimeError::RuntimeStopped => ("runtime_stopped", None, None), RuntimeError::RuntimeStoreAlreadyOpen { .. } => ("runtime_store_already_open", None, None), RuntimeError::InvalidInitialInputKind { .. } => ("invalid_initial_input_kind", None, None), @@ -10904,6 +11301,735 @@ mod tests { } } + fn test_stop_request( + runtime: &Runtime, + worker_ref: &WorkerRef, + id: &str, + ) -> runtime_api::WorkerLifecycleRequest { + runtime_api::WorkerLifecycleRequest { + reason: Some("normal stop".into()), + stop_intent: Some(runtime_api::WorkerStopIntent { + expected_observation_token: runtime + .worker_detail(worker_ref) + .unwrap() + .restore_observation_token + .unwrap(), + request_id: id.into(), + }), + } + } + + #[test] + fn coordinate_stop_admits_without_cleanup_and_dispatches_the_same_operation() { + let (runtime, backend) = runtime_and_backend(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("two-phase stop")) + .unwrap(); + let request = test_stop_request(&runtime, &worker.worker_ref, "admit-then-dispatch"); + assert!(matches!( + runtime.coordinate_worker_stop( + &worker.worker_ref, + runtime_api::WorkerLifecycleRequest::default() + ), + Err(RuntimeError::InvalidRequest(_)) + )); + assert_eq!( + runtime + .coordinate_worker_stop(&worker.worker_ref, request.clone()) + .unwrap(), + crate::catalog::WorkerStopOutcome::Pending + ); + let operation_id = runtime + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .pending_stop + .unwrap() + .operation_id; + let admitted_token = runtime + .worker_detail(&worker.worker_ref) + .unwrap() + .restore_observation_token; + assert_eq!( + runtime + .coordinate_worker_stop(&worker.worker_ref, request.clone()) + .unwrap(), + crate::catalog::WorkerStopOutcome::Pending + ); + assert_eq!( + runtime + .worker_detail(&worker.worker_ref) + .unwrap() + .restore_observation_token, + admitted_token + ); + assert_eq!(*backend.stop_count.lock().unwrap(), 0); + assert_eq!( + runtime.worker_detail(&worker.worker_ref).unwrap().status, + WorkerStatus::Idle + ); + let mut changed = request.clone(); + changed.reason = Some("changed admission input".into()); + assert!(matches!( + runtime.coordinate_worker_stop(&worker.worker_ref, changed), + Err(RuntimeError::StopRequestConflict { .. }) + )); + let ack = runtime + .stop_worker_request(&worker.worker_ref, request.clone()) + .unwrap(); + assert!(!ack.replayed); + assert_eq!( + backend.stop_operation_ids.lock().unwrap().as_slice(), + &[operation_id] + ); + runtime + .restore_worker( + &worker.worker_ref, + runtime.test_restore_request(&worker.worker_ref), + ) + .unwrap(); + let restored_token = runtime + .worker_detail(&worker.worker_ref) + .unwrap() + .restore_observation_token; + assert!( + matches!(runtime.coordinate_worker_stop(&worker.worker_ref, request.clone()).unwrap(), crate::catalog::WorkerStopOutcome::Stopped { ack } if ack.replayed) + ); + let mut unseen_old = request; + unseen_old.stop_intent.as_mut().unwrap().request_id = "unseen-old-generation".into(); + assert!(matches!( + runtime.coordinate_worker_stop(&worker.worker_ref, unseen_old), + Err(RuntimeError::StopObservationConflict { .. }) + )); + assert_eq!(*backend.stop_count.lock().unwrap(), 1); + assert_eq!( + runtime + .worker_detail(&worker.worker_ref) + .unwrap() + .restore_observation_token, + restored_token + ); + } + + #[test] + fn coordinate_stop_never_waits_for_cleanup_while_a_caller_gate_may_be_held() { + let (runtime, backend) = runtime_and_backend(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("stop admission busy")) + .unwrap(); + let request = test_stop_request(&runtime, &worker.worker_ref, "busy-stop"); + let operation_lock = runtime.worker_operation_lock(worker.worker_id).unwrap(); + let guard = operation_lock.lock().unwrap(); + assert!(matches!( + runtime.coordinate_worker_stop(&worker.worker_ref, request.clone()), + Err(RuntimeError::StopAdmissionBusy { .. }) + )); + assert!( + runtime + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .restore_guard + .stop_owners + .is_empty() + ); + assert_eq!(*backend.stop_count.lock().unwrap(), 0); + drop(guard); + assert_eq!( + runtime + .coordinate_worker_stop(&worker.worker_ref, request) + .unwrap(), + crate::catalog::WorkerStopOutcome::Pending + ); + } + + #[cfg(feature = "fs-store")] + #[test] + fn coordinate_stop_unknown_admission_is_reobserved_without_backend_cleanup() { + let dir = tempfile::tempdir().unwrap(); + let options = crate::fs_store::FsRuntimeStoreOptions { + root: dir.path().join("runtime"), + runtime_id: "stop-admit-unknown".into(), + display_name: None, + }; + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = + Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()).unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("unknown stop admission")) + .unwrap(); + let request = test_stop_request(&runtime, &worker.worker_ref, "unknown-admission"); + runtime_store(&runtime) + .fail_next_worker_write(crate::fs_store::AtomicWriteFault::AfterRename); + assert!(matches!( + runtime.coordinate_worker_stop(&worker.worker_ref, request.clone()), + Err(RuntimeError::StoreCommitOutcomeUnknown { .. }) + )); + assert_eq!(*backend.stop_count.lock().unwrap(), 0); + runtime_store(&runtime).fail_next_worker_reconcile_read(); + assert!(matches!( + runtime.coordinate_worker_stop(&worker.worker_ref, request.clone()), + Err(RuntimeError::StoreCommitOutcomeUnknown { .. }) + )); + assert_eq!(*backend.stop_count.lock().unwrap(), 0); + assert_eq!( + runtime + .coordinate_worker_stop(&worker.worker_ref, request.clone()) + .unwrap(), + crate::catalog::WorkerStopOutcome::Pending + ); + let operation_id = runtime + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .pending_stop + .unwrap() + .operation_id; + drop(runtime); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = + Runtime::with_fs_store_and_execution_backend(options, backend.clone()).unwrap(); + assert_eq!( + backend.stop_operation_ids.lock().unwrap().as_slice(), + &[operation_id] + ); + assert!( + matches!(runtime.coordinate_worker_stop(&worker.worker_ref, request).unwrap(), crate::catalog::WorkerStopOutcome::Stopped { ack } if ack.replayed) + ); + assert_eq!(*backend.stop_count.lock().unwrap(), 1); + } + + #[cfg(feature = "fs-store")] + #[test] + fn stop_owner_capacity_rejects_admission_before_mutation_or_cleanup() { + let dir = tempfile::tempdir().unwrap(); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = Runtime::with_fs_store_and_execution_backend( + crate::fs_store::FsRuntimeStoreOptions { + root: dir.path().join("runtime"), + runtime_id: "stop-admit-capacity".into(), + display_name: None, + }, + backend.clone(), + ) + .unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("stop admission capacity")) + .unwrap(); + let mut request = test_stop_request(&runtime, &worker.worker_ref, "oversized-stop-owner"); + request.reason = Some("x".repeat(8 * 1024 * 1024)); + for result in [ + runtime + .coordinate_worker_stop(&worker.worker_ref, request.clone()) + .map(|_| ()), + runtime + .stop_worker_request(&worker.worker_ref, request.clone()) + .map(|_| ()), + ] { + assert!( + matches!(result, Err(RuntimeError::InvalidRequest(message)) if message.contains("canonical reader limit")) + ); + } + let observed = runtime_store(&runtime) + .read_worker_record(&worker.worker_ref) + .unwrap(); + assert_eq!(observed.status, WorkerStatus::Idle); + let state = runtime.lock().unwrap(); + let record = state.worker(&worker.worker_ref).unwrap(); + assert!(record.pending_stop.is_none()); + assert!(record.restore_guard.stop_owners.is_empty()); + assert_eq!( + Some(record.restore_guard.observation_token.clone()), + worker.restore_observation_token + ); + assert_eq!(*backend.stop_count.lock().unwrap(), 0); + } + + #[test] + fn stop_intent_rejects_unseen_old_observation_after_restore_and_replays_historical_receipt() { + let (runtime, backend) = runtime_and_backend(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime.create_worker(task_request("fenced stop")).unwrap(); + let old = test_stop_request(&runtime, &worker.worker_ref, "late-unseen"); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let restore = runtime.test_restore_request(&worker.worker_ref); + runtime + .restore_worker(&worker.worker_ref, restore.clone()) + .unwrap(); + assert!(matches!( + runtime.stop_worker_request(&worker.worker_ref, old.clone()), + Err(RuntimeError::StopObservationConflict { .. }) + )); + assert_eq!( + runtime + .worker_stop_outcome(&worker.worker_ref, old) + .unwrap(), + crate::catalog::WorkerStopOutcome::NotFound + ); + assert_eq!(*backend.stop_count.lock().unwrap(), 1); + + let stop = test_stop_request(&runtime, &worker.worker_ref, "owned-stop"); + let mut receipt = runtime + .stop_worker_request(&worker.worker_ref, stop.clone()) + .unwrap(); + assert!(!receipt.replayed, "first stop is not a historical receipt"); + receipt.replayed = true; // Expected transport evidence on replay/query only. + // A stop must preserve the earlier Restore receipt as historical only. + runtime.restore_worker(&worker.worker_ref, restore).unwrap(); + assert_eq!( + runtime.worker_detail(&worker.worker_ref).unwrap().status, + WorkerStatus::Stopped + ); + runtime + .restore_worker( + &worker.worker_ref, + runtime.test_restore_request(&worker.worker_ref), + ) + .unwrap(); + let before = runtime.worker_detail(&worker.worker_ref).unwrap(); + assert_eq!( + runtime + .stop_worker_request(&worker.worker_ref, stop.clone()) + .unwrap(), + receipt + ); + assert_eq!( + runtime + .worker_stop_outcome(&worker.worker_ref, stop) + .unwrap(), + crate::catalog::WorkerStopOutcome::Stopped { ack: receipt } + ); + let after = runtime.worker_detail(&worker.worker_ref).unwrap(); + assert_eq!(after.status, WorkerStatus::Idle); + assert_eq!( + after.restore_observation_token, + before.restore_observation_token + ); + assert_eq!(*backend.stop_count.lock().unwrap(), 2); + assert_eq!(*backend.restore_count.lock().unwrap(), 2); + } + + #[test] + fn current_stopped_status_and_cancel_are_not_historical_stop_receipts() { + let (runtime, backend) = runtime_and_backend(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("current lifecycle ack")) + .unwrap(); + assert!( + !runtime + .cancel_worker(&worker.worker_ref, None) + .unwrap() + .replayed + ); + assert!( + !runtime + .stop_worker(&worker.worker_ref, None) + .unwrap() + .replayed + ); + assert!( + !runtime + .cancel_worker(&worker.worker_ref, None) + .unwrap() + .replayed + ); + let request = test_stop_request(&runtime, &worker.worker_ref, "already-stopped"); + let current = runtime + .stop_worker_request(&worker.worker_ref, request.clone()) + .unwrap(); + assert!( + !current.replayed, + "new stop intent on current stopped status is fresh" + ); + assert!( + runtime + .stop_worker_request(&worker.worker_ref, request.clone()) + .unwrap() + .replayed + ); + assert!( + matches!(runtime.worker_stop_outcome(&worker.worker_ref, request).unwrap(), crate::catalog::WorkerStopOutcome::Stopped { ack } if ack.replayed) + ); + assert!( + !runtime + .stop_worker(&worker.worker_ref, None) + .unwrap() + .replayed + ); + assert_eq!(*backend.stop_count.lock().unwrap(), 1); + } + + #[test] + fn stop_intent_same_id_binds_reason_and_observation_for_stop_and_lookup() { + let (runtime, backend) = runtime_and_backend(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("stop input binding")) + .unwrap(); + let request = test_stop_request(&runtime, &worker.worker_ref, "bound-stop"); + runtime + .stop_worker_request(&worker.worker_ref, request.clone()) + .unwrap(); + let mut changed_reason = request.clone(); + changed_reason.reason = None; + let mut changed_token = request.clone(); + changed_token + .stop_intent + .as_mut() + .unwrap() + .expected_observation_token + .push_str("-changed"); + for changed in [changed_reason, changed_token] { + assert!(matches!( + runtime.stop_worker_request(&worker.worker_ref, changed.clone()), + Err(RuntimeError::StopRequestConflict { .. }) + )); + assert!(matches!( + runtime.worker_stop_outcome(&worker.worker_ref, changed), + Err(RuntimeError::StopRequestConflict { .. }) + )); + } + assert_eq!(*backend.stop_count.lock().unwrap(), 1); + } + + #[test] + fn stop_intent_pending_replay_and_legacy_recovery_keep_one_operation_and_owner() { + let (runtime, backend) = runtime_and_backend(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("pending typed stop")) + .unwrap(); + let request = test_stop_request(&runtime, &worker.worker_ref, "pending-stop"); + backend.set_stop_result(WorkerExecutionResult::errored( + WorkerExecutionOperation::Stop, + "cleanup unknown", + )); + assert!(matches!( + runtime.stop_worker_request(&worker.worker_ref, request.clone()), + Err(RuntimeError::WorkerExecutionRejected { .. }) + )); + assert_eq!( + runtime + .worker_stop_outcome(&worker.worker_ref, request.clone()) + .unwrap(), + crate::catalog::WorkerStopOutcome::Pending + ); + assert_eq!( + *backend.stop_count.lock().unwrap(), + 1, + "lookup must not perform cleanup" + ); + let thief = test_stop_request(&runtime, &worker.worker_ref, "another-request"); + assert!(matches!( + runtime.stop_worker_request(&worker.worker_ref, thief), + Err(RuntimeError::StopRequestConflict { .. }) + )); + let mut altered = request.clone(); + altered.reason = Some("changed pending input".into()); + assert!(matches!( + runtime.stop_worker_request(&worker.worker_ref, altered), + Err(RuntimeError::StopRequestConflict { .. }) + )); + let mut receipt = runtime.stop_worker(&worker.worker_ref, None).unwrap(); + assert!(!receipt.replayed, "internal cleanup is a fresh stop"); + receipt.replayed = true; + assert_eq!( + runtime + .stop_worker_request(&worker.worker_ref, request) + .unwrap(), + receipt + ); + let ids = backend.stop_operation_ids.lock().unwrap(); + assert_eq!(ids.len(), 2); + assert_eq!(ids[0], ids[1]); + } + + #[test] + fn stop_intent_can_bind_existing_legacy_pending_stop_without_new_operation() { + let (runtime, backend) = runtime_and_backend(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("legacy pending stop")) + .unwrap(); + backend.set_stop_result(WorkerExecutionResult::errored( + WorkerExecutionOperation::Stop, + "cleanup unknown", + )); + runtime.stop_worker(&worker.worker_ref, None).unwrap_err(); + let request = test_stop_request(&runtime, &worker.worker_ref, "legacy-recovery"); + let mut receipt = runtime + .stop_worker_request(&worker.worker_ref, request.clone()) + .unwrap(); + assert!( + !receipt.replayed, + "settling legacy pending cleanup is fresh" + ); + receipt.replayed = true; + assert_eq!( + runtime + .worker_stop_outcome(&worker.worker_ref, request) + .unwrap(), + crate::catalog::WorkerStopOutcome::Stopped { ack: receipt } + ); + let ids = backend.stop_operation_ids.lock().unwrap(); + assert_eq!(ids.len(), 2); + assert_eq!(ids[0], ids[1]); + } + + #[cfg(feature = "fs-store")] + #[test] + fn legacy_restore_guard_without_stop_owners_reopens_without_losing_observation() { + let dir = tempfile::tempdir().unwrap(); + let options = crate::fs_store::FsRuntimeStoreOptions { + root: dir.path().join("runtime"), + runtime_id: "legacy-stop-guard".into(), + display_name: None, + }; + let runtime = Runtime::with_fs_store_and_execution_backend( + options.clone(), + Arc::new(TestExecutionBackend::default()), + ) + .unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("legacy stop guard")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + let observed = runtime + .worker_detail(&worker.worker_ref) + .unwrap() + .restore_observation_token; + let document: serde_json::Value = serde_json::from_slice( + &std::fs::read( + options + .root + .join("workers") + .join(worker.worker_id.to_string()) + .join("worker.json"), + ) + .unwrap(), + ) + .unwrap(); + assert!(document["restore_guard"].get("stop_owners").is_none()); + drop(runtime); + let runtime = Runtime::with_fs_store_and_execution_backend( + options, + Arc::new(TestExecutionBackend::default()), + ) + .unwrap(); + assert_eq!( + runtime + .worker_detail(&worker.worker_ref) + .unwrap() + .restore_observation_token, + observed + ); + let request = test_stop_request(&runtime, &worker.worker_ref, "never-admitted"); + assert_eq!( + runtime + .worker_stop_outcome(&worker.worker_ref, request) + .unwrap(), + crate::catalog::WorkerStopOutcome::NotFound + ); + } + + #[cfg(feature = "fs-store")] + #[test] + fn stop_intent_unknown_admission_reopens_with_same_owner_and_historical_receipt() { + let dir = tempfile::tempdir().unwrap(); + let options = crate::fs_store::FsRuntimeStoreOptions { + root: dir.path().join("runtime"), + runtime_id: "stop-intent-reopen".into(), + display_name: None, + }; + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = + Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()).unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("persist stop owner")) + .unwrap(); + let request = test_stop_request(&runtime, &worker.worker_ref, "persisted-stop"); + runtime_store(&runtime) + .fail_next_worker_write(crate::fs_store::AtomicWriteFault::AfterRename); + assert!(matches!( + runtime.stop_worker_request(&worker.worker_ref, request.clone()), + Err(RuntimeError::StoreCommitOutcomeUnknown { .. }) + )); + assert_eq!(*backend.stop_count.lock().unwrap(), 0); + assert_eq!( + runtime + .worker_stop_outcome(&worker.worker_ref, request.clone()) + .unwrap(), + crate::catalog::WorkerStopOutcome::Pending + ); + assert_eq!(*backend.stop_count.lock().unwrap(), 0); + let operation_id = runtime + .lock() + .unwrap() + .worker(&worker.worker_ref) + .unwrap() + .pending_stop + .unwrap() + .operation_id; + drop(runtime); + + let backend = Arc::new(TestExecutionBackend::default()); + let reopened = + Runtime::with_fs_store_and_execution_backend(options.clone(), backend.clone()).unwrap(); + assert_eq!( + backend.stop_operation_ids.lock().unwrap().as_slice(), + &[operation_id] + ); + let receipt = reopened + .stop_worker_request(&worker.worker_ref, request.clone()) + .unwrap(); + assert!( + receipt.replayed, + "startup already settled the retained stop" + ); + assert_eq!(*backend.stop_count.lock().unwrap(), 1); + reopened + .restore_worker( + &worker.worker_ref, + reopened.test_restore_request(&worker.worker_ref), + ) + .unwrap(); + drop(reopened); + let backend = Arc::new(TestExecutionBackend::default()); + let reopened = + Runtime::with_fs_store_and_execution_backend(options, backend.clone()).unwrap(); + let before = reopened.worker_detail(&worker.worker_ref).unwrap(); + assert_eq!( + reopened + .stop_worker_request(&worker.worker_ref, request.clone()) + .unwrap(), + receipt + ); + assert_eq!( + reopened + .worker_stop_outcome(&worker.worker_ref, request) + .unwrap(), + crate::catalog::WorkerStopOutcome::Stopped { ack: receipt } + ); + assert_eq!(*backend.stop_count.lock().unwrap(), 0); + assert_eq!( + reopened + .worker_detail(&worker.worker_ref) + .unwrap() + .restore_observation_token, + before.restore_observation_token + ); + } + + #[cfg(feature = "fs-store")] + #[test] + fn stop_intent_failed_admission_does_not_bind_id_or_rotate_observation() { + let dir = tempfile::tempdir().unwrap(); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = Runtime::with_fs_store_and_execution_backend( + crate::fs_store::FsRuntimeStoreOptions { + root: dir.path().join("runtime"), + runtime_id: "stop-intent-failed-write".into(), + display_name: None, + }, + backend.clone(), + ) + .unwrap(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("failed stop admission")) + .unwrap(); + let request = test_stop_request(&runtime, &worker.worker_ref, "retry-admission"); + runtime_store(&runtime) + .fail_next_worker_write(crate::fs_store::AtomicWriteFault::BeforeRename); + assert!( + runtime + .stop_worker_request(&worker.worker_ref, request.clone()) + .is_err() + ); + assert_eq!( + runtime + .worker_stop_outcome(&worker.worker_ref, request.clone()) + .unwrap(), + crate::catalog::WorkerStopOutcome::NotFound + ); + assert_eq!(*backend.stop_count.lock().unwrap(), 0); + assert_eq!( + runtime + .worker_detail(&worker.worker_ref) + .unwrap() + .restore_observation_token, + worker.restore_observation_token + ); + runtime + .stop_worker_request(&worker.worker_ref, request) + .unwrap(); + assert_eq!(*backend.stop_count.lock().unwrap(), 1); + } + + #[cfg(feature = "fs-store")] + #[test] + fn stop_intent_terminal_unknown_lookup_recovers_receipt_without_restop() { + let dir = tempfile::tempdir().unwrap(); + let backend = Arc::new(TestExecutionBackend::default()); + let runtime = Arc::new( + Runtime::with_fs_store_and_execution_backend( + crate::fs_store::FsRuntimeStoreOptions { + root: dir.path().join("runtime"), + runtime_id: "stop-intent-terminal-unknown".into(), + display_name: None, + }, + backend.clone(), + ) + .unwrap(), + ); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("terminal stop unknown")) + .unwrap(); + let request = test_stop_request(&runtime, &worker.worker_ref, "terminal-unknown"); + let gate = Arc::new(RestoreGate::default()); + *backend.stop_gate.lock().unwrap() = Some(gate.clone()); + let stopping = { + let runtime = runtime.clone(); + let worker_ref = worker.worker_ref.clone(); + let request = request.clone(); + std::thread::spawn(move || runtime.stop_worker_request(&worker_ref, request)) + }; + let entered = gate.wait_for_entered(1, std::time::Duration::from_secs(2)); + if entered { + runtime_store(&runtime) + .fail_next_worker_write(crate::fs_store::AtomicWriteFault::AfterRename); + } + gate.release(); + let result = stopping.join().unwrap(); + assert!(entered, "backend stop did not reach the test gate"); + assert!(matches!( + result, + Err(RuntimeError::StoreCommitOutcomeUnknown { .. }) + )); + assert!(matches!( + runtime + .worker_stop_outcome(&worker.worker_ref, request.clone()) + .unwrap(), + crate::catalog::WorkerStopOutcome::Stopped { ack } if ack.replayed + )); + let replay = runtime + .stop_worker_request(&worker.worker_ref, request) + .unwrap(); + assert!(replay.replayed); + assert_eq!(*backend.stop_count.lock().unwrap(), 1); + } + #[test] fn failed_stop_cleanup_retains_execution_for_retry() { let (runtime, backend) = runtime_and_backend(); @@ -11304,6 +12430,85 @@ mod tests { assert!(runtime.worker_operations.lock().unwrap().is_empty()); } + #[test] + fn restore_coordination_busy_preserves_pending_stop_and_retained_receipts() { + let (runtime, backend) = runtime_and_backend(); + runtime.store_config_bundle(test_bundle()).unwrap(); + let worker = runtime + .create_worker(task_request("restore admission busy")) + .unwrap(); + runtime.stop_worker(&worker.worker_ref, None).unwrap(); + runtime + .restore_worker( + &worker.worker_ref, + runtime.test_restore_request(&worker.worker_ref), + ) + .unwrap(); + let stop = test_stop_request(&runtime, &worker.worker_ref, "admitted-stop"); + runtime + .coordinate_worker_stop(&worker.worker_ref, stop) + .unwrap(); + let snapshot = || { + let state = runtime.lock().unwrap(); + let record = state.worker(&worker.worker_ref).unwrap(); + assert!( + record + .restore_guard + .owners + .values() + .any(|owner| owner.receipt.is_some()) + ); + ( + serde_json::to_value(&record.restore_guard).unwrap(), + record.pending_stop, + record.pending_restore, + record.status, + ) + }; + let before = snapshot(); + for preparation in [Some(runtime_api::WorkerRestorePreparation::default()), None] { + let operation_lock = runtime.worker_operation_lock(worker.worker_id).unwrap(); + let guard = operation_lock.lock().unwrap(); + let request = runtime_api::WorkerRestoreCoordinationRequest { + expected_observation_token: runtime + .worker_detail(&worker.worker_ref) + .unwrap() + .restore_observation_token + .unwrap(), + request_id: "busy-restore".into(), + preparation, + }; + let (tx, rx) = std::sync::mpsc::channel(); + let coordinating = { + let runtime = runtime.clone(); + let worker_ref = worker.worker_ref.clone(); + std::thread::spawn(move || { + let _ = tx.send(runtime.coordinate_worker_restore_operation( + &worker_ref, + request, + None, + )); + }) + }; + let result = rx.recv_timeout(std::time::Duration::from_secs(2)); + // Release before joining even on regression, so a blocking lock + // produces a test failure rather than leaving the test hung. + drop(guard); + coordinating.join().unwrap(); + assert!(matches!( + result, + Ok(Err(RuntimeError::RestoreAdmissionBusy { .. })) + )); + assert_eq!( + snapshot(), + before, + "busy coordination changed lifecycle ownership or receipts" + ); + } + assert_eq!(*backend.stop_count.lock().unwrap(), 1); + assert_eq!(*backend.restore_count.lock().unwrap(), 1); + } + #[test] fn restore_guard_coordination_is_authorized_and_none_cannot_admit_new_intent() { let (runtime, backend) = runtime_and_backend(); diff --git a/crates/worker/src/feature/builtin/manage_worker.rs b/crates/worker/src/feature/builtin/manage_worker.rs index c59cf4560..fa0919499 100644 --- a/crates/worker/src/feature/builtin/manage_worker.rs +++ b/crates/worker/src/feature/builtin/manage_worker.rs @@ -275,6 +275,7 @@ impl WorkerControlService for WorkspaceWorkerControlService { runtime_id, worker_id, request: server_api::RuntimeWorkerLifecycleRequest { + stop_intent: None, reason, ticket_assignment: None, }, @@ -285,6 +286,7 @@ impl WorkerControlService for WorkspaceWorkerControlService { runtime_id, worker_id, request: server_api::RuntimeWorkerLifecycleRequest { + stop_intent: None, reason, ticket_assignment: None, }, diff --git a/crates/workspace-server/src/hosts.rs b/crates/workspace-server/src/hosts.rs index 0d0ca3fa8..e95042a65 100644 --- a/crates/workspace-server/src/hosts.rs +++ b/crates/workspace-server/src/hosts.rs @@ -711,6 +711,8 @@ pub struct WorkerStopResult { #[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] pub struct WorkerLifecycleRequest { + #[serde(default, skip_serializing_if = "Option::is_none")] + pub stop_intent: Option, #[serde(default, skip_serializing_if = "Option::is_none")] pub reason: Option, #[serde(default, skip_serializing_if = "Option::is_none")] @@ -719,6 +721,8 @@ pub struct WorkerLifecycleRequest { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct WorkerLifecycleResult { + #[serde(default)] + pub replayed: bool, pub state: InternalWorkerOperationState, #[serde(flatten)] pub worker: RuntimeWorkerRef, @@ -878,6 +882,14 @@ impl RuntimeRegistryError { } => { if code == "restore_observation_conflict" { Error::RestoreObservationConflict + } else if matches!( + code.as_str(), + "stop_observation_conflict" + | "stop_request_conflict" + | "stop_admission_busy" + | "restore_admission_busy" + ) { + Error::RepositoryConflict(message) } else { Error::RuntimeOperationFailed { runtime_id, @@ -1188,12 +1200,37 @@ pub trait WorkspaceWorkerRuntime: Send + Sync { }) } + fn coordinate_worker_stop( + &self, + _worker_id: &str, + _request: WorkerLifecycleRequest, + ) -> Result { + Err(diagnostic( + "worker_stop_admission_unsupported", + HostDiagnosticSeverity::Error, + "Runtime does not support fenced stop admission".into(), + )) + } + + fn worker_stop_outcome( + &self, + _worker_id: &str, + _request: WorkerLifecycleRequest, + ) -> Result { + Err(diagnostic( + "worker_stop_outcome_unsupported", + HostDiagnosticSeverity::Error, + "Runtime does not support stop outcome queries".into(), + )) + } + fn stop_worker( &self, worker_id: &str, _request: WorkerLifecycleRequest, ) -> WorkerLifecycleResult { WorkerLifecycleResult { + replayed: false, state: InternalWorkerOperationState::Unsupported, worker: RuntimeWorkerRef::new(self.runtime_id().to_string(), worker_id.to_string()), diagnostics: vec![diagnostic( @@ -1212,6 +1249,7 @@ pub trait WorkspaceWorkerRuntime: Send + Sync { _request: WorkerLifecycleRequest, ) -> WorkerLifecycleResult { WorkerLifecycleResult { + replayed: false, state: InternalWorkerOperationState::Unsupported, worker: RuntimeWorkerRef::new(self.runtime_id().to_string(), worker_id.to_string()), diagnostics: vec![diagnostic( @@ -2119,6 +2157,40 @@ impl RuntimeRegistry { Ok(runtime.worker_completions(worker_id, request)) } + pub fn coordinate_worker_stop( + &self, + worker: &RuntimeWorkerRef, + request: WorkerLifecycleRequest, + ) -> Result { + validate_backend_identifier("runtime_id", &worker.runtime_id)?; + validate_backend_identifier("worker_id", &worker.worker_id)?; + let runtime = self.runtime(&worker.runtime_id)?; + runtime + .coordinate_worker_stop(&worker.worker_id, request) + .map_err(|diagnostic| RuntimeRegistryError::RuntimeOperationFailed { + runtime_id: worker.runtime_id.clone(), + code: diagnostic.code, + message: diagnostic.message, + }) + } + + pub fn worker_stop_outcome( + &self, + worker: &RuntimeWorkerRef, + request: WorkerLifecycleRequest, + ) -> Result { + validate_backend_identifier("runtime_id", &worker.runtime_id)?; + validate_backend_identifier("worker_id", &worker.worker_id)?; + let runtime = self.runtime(&worker.runtime_id)?; + runtime + .worker_stop_outcome(&worker.worker_id, request) + .map_err(|diagnostic| RuntimeRegistryError::RuntimeOperationFailed { + runtime_id: worker.runtime_id.clone(), + code: diagnostic.code, + message: diagnostic.message, + }) + } + pub fn stop_worker( &self, worker: &RuntimeWorkerRef, @@ -3223,6 +3295,56 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime { } } + fn coordinate_worker_stop( + &self, + worker_id: &str, + request: WorkerLifecycleRequest, + ) -> Result { + let worker_ref = self.worker_ref(worker_id).ok_or_else(|| { + diagnostic( + "embedded_worker_id_invalid", + HostDiagnosticSeverity::Error, + "Worker ID could not be resolved".into(), + ) + })?; + let outcome = self + .runtime + .coordinate_worker_stop( + &worker_ref, + runtime_api::WorkerLifecycleRequest { + reason: request.reason, + stop_intent: request.stop_intent, + }, + ) + .map_err(|error| embedded_runtime_diagnostic(&error))?; + runtime_contract_convert(outcome) + } + + fn worker_stop_outcome( + &self, + worker_id: &str, + request: WorkerLifecycleRequest, + ) -> Result { + let worker_ref = self.worker_ref(worker_id).ok_or_else(|| { + diagnostic( + "embedded_worker_id_invalid", + HostDiagnosticSeverity::Error, + "Worker ID could not be resolved".into(), + ) + })?; + let outcome = self + .runtime + .worker_stop_outcome( + &worker_ref, + runtime_api::WorkerLifecycleRequest { + reason: request.reason, + stop_intent: request.stop_intent, + }, + ) + .map_err(|error| embedded_runtime_diagnostic(&error))?; + runtime_contract_convert(outcome) + } + fn stop_worker( &self, worker_id: &str, @@ -3250,8 +3372,15 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime { ), ); }; - match self.runtime.stop_worker(&worker_ref, request.reason) { - Ok(_) => WorkerLifecycleResult { + match self.runtime.stop_worker_request( + &worker_ref, + runtime_api::WorkerLifecycleRequest { + reason: request.reason, + stop_intent: request.stop_intent, + }, + ) { + Ok(response) => WorkerLifecycleResult { + replayed: response.replayed, state: InternalWorkerOperationState::Accepted, worker: RuntimeWorkerRef::new(self.runtime_id.clone(), worker_id.to_string()), diagnostics: Vec::new(), @@ -3295,6 +3424,7 @@ impl WorkspaceWorkerRuntime for EmbeddedWorkerRuntime { }; match self.runtime.cancel_worker(&worker_ref, request.reason) { Ok(_) => WorkerLifecycleResult { + replayed: false, state: InternalWorkerOperationState::Accepted, worker: RuntimeWorkerRef::new(self.runtime_id.clone(), worker_id.to_string()), diagnostics: Vec::new(), @@ -4899,6 +5029,7 @@ impl RemoteWorkerRuntime { response: worker_runtime::catalog::WorkerLifecycleAck, ) -> WorkerLifecycleResult { WorkerLifecycleResult { + replayed: response.replayed, state: InternalWorkerOperationState::Accepted, worker: RuntimeWorkerRef::new(self.runtime_id.clone(), worker_id.to_string()), diagnostics: vec![diagnostic( @@ -5574,6 +5705,40 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime { } } + fn coordinate_worker_stop( + &self, + worker_id: &str, + request: WorkerLifecycleRequest, + ) -> Result { + let worker_id = worker_id.to_string(); + let request = runtime_api::WorkerLifecycleRequest { + reason: request.reason, + stop_intent: request.stop_intent, + }; + self.run_runtime_api( + self.request_timeout, + MAX_REMOTE_RUNTIME_RESPONSE_BYTES, + move |client| async move { client.coordinate_worker_stop(worker_id, request).await }, + ) + } + + fn worker_stop_outcome( + &self, + worker_id: &str, + request: WorkerLifecycleRequest, + ) -> Result { + let worker_id = worker_id.to_string(); + let request = runtime_api::WorkerLifecycleRequest { + reason: request.reason, + stop_intent: request.stop_intent, + }; + self.run_runtime_api( + self.request_timeout, + MAX_REMOTE_RUNTIME_RESPONSE_BYTES, + move |client| async move { client.worker_stop_outcome(worker_id, request).await }, + ) + } + fn stop_worker( &self, worker_id: &str, @@ -5581,6 +5746,7 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime { ) -> WorkerLifecycleResult { let body = runtime_api::WorkerLifecycleRequest { reason: request.reason, + stop_intent: request.stop_intent, }; let worker_id_owned = worker_id.to_string(); match self @@ -5606,6 +5772,7 @@ impl WorkspaceWorkerRuntime for RemoteWorkerRuntime { ) -> WorkerLifecycleResult { let body = runtime_api::WorkerLifecycleRequest { reason: request.reason, + stop_intent: request.stop_intent, }; let worker_id_owned = worker_id.to_string(); match self @@ -6399,6 +6566,7 @@ fn embedded_lifecycle_rejected( diagnostic: RuntimeDiagnostic, ) -> WorkerLifecycleResult { WorkerLifecycleResult { + replayed: false, state: InternalWorkerOperationState::Rejected, worker: RuntimeWorkerRef::new(runtime_id.to_string(), worker_id.to_string()), diagnostics: vec![diagnostic], @@ -6411,6 +6579,7 @@ fn remote_lifecycle_rejected( diagnostic: RuntimeDiagnostic, ) -> WorkerLifecycleResult { WorkerLifecycleResult { + replayed: false, state: InternalWorkerOperationState::Rejected, worker: RuntimeWorkerRef::new(runtime_id.to_string(), worker_id.to_string()), diagnostics: vec![diagnostic], @@ -6484,6 +6653,26 @@ fn sanitize_embedded_execution_detail(summary: &str, message: &str) -> String { fn embedded_runtime_diagnostic(error: &EmbeddedRuntimeError) -> RuntimeDiagnostic { match error { + EmbeddedRuntimeError::RestoreAdmissionBusy { .. } => diagnostic( + "restore_admission_busy", + HostDiagnosticSeverity::Warning, + "Worker lifecycle admission is busy; retry the same Restore intent".into(), + ), + EmbeddedRuntimeError::StopAdmissionBusy { .. } => diagnostic( + "stop_admission_busy", + HostDiagnosticSeverity::Warning, + "Worker lifecycle admission is busy; retry the same stop intent".into(), + ), + EmbeddedRuntimeError::StopObservationConflict { .. } => diagnostic( + "stop_observation_conflict", + HostDiagnosticSeverity::Warning, + "Stop observation changed; reload before a new intent".into(), + ), + EmbeddedRuntimeError::StopRequestConflict { .. } => diagnostic( + "stop_request_conflict", + HostDiagnosticSeverity::Warning, + "Stop request ID was already used with different input".into(), + ), EmbeddedRuntimeError::RestoreObservationConflict { .. } => diagnostic( "restore_observation_conflict", HostDiagnosticSeverity::Warning, @@ -7288,6 +7477,7 @@ mod tests { assert!(value.get("worker").is_none()); let lifecycle = WorkerLifecycleResult { + replayed: false, state: InternalWorkerOperationState::Accepted, worker: RuntimeWorkerRef::new("arcadia", "30"), diagnostics: Vec::new(), diff --git a/crates/workspace-server/src/server.rs b/crates/workspace-server/src/server.rs index 98a32276d..c211f442b 100644 --- a/crates/workspace-server/src/server.rs +++ b/crates/workspace-server/src/server.rs @@ -29,6 +29,7 @@ mod backend_job_profile; mod chat; mod external_operation_audit; mod subject_authorization; +mod subject_body; mod subjektiv_deletion; mod subjektiv_job; mod subjektiv_journal; @@ -659,6 +660,77 @@ impl WorkdirSessionRegistry { .collect() } + // Stop owns the concrete provider sessions observed before dispatch, not + // every session later registered under the same restored Worker identity. + fn observe_worker(&self, worker: &RuntimeWorkerRef) -> Vec { + let mut observed = Vec::new(); + if let Some(session) = self.attachments.get(worker) { + observed.push(RegisteredWorkdirSession::Attachment { + worker: worker.clone(), + session: session.clone(), + }); + } + observed.extend( + self.commands + .iter() + .filter_map(|((owner, handle), session)| { + (owner == worker).then(|| RegisteredWorkdirSession::Command { + worker: worker.clone(), + external_handle: handle.clone(), + session: session.clone(), + }) + }), + ); + observed + } + + fn take_observed( + &mut self, + observed: &[RegisteredWorkdirSession], + ) -> Vec { + observed + .iter() + .filter_map(|item| match item { + RegisteredWorkdirSession::Attachment { worker, session } => self + .attachments + .get(worker) + .is_some_and(|current| Arc::ptr_eq(current, session)) + .then(|| { + self.attachments.remove(worker).map(|session| { + RegisteredWorkdirSession::Attachment { + worker: worker.clone(), + session, + } + }) + }) + .flatten(), + RegisteredWorkdirSession::Command { + worker, + external_handle, + session, + } => { + let key = (worker.clone(), external_handle.clone()); + self.commands + .get(&key) + .is_some_and(|current| { + Arc::ptr_eq(¤t.source, &session.source) + && current.provider_handle == session.provider_handle + }) + .then(|| { + self.commands.remove(&key).map(|session| { + RegisteredWorkdirSession::Command { + worker: worker.clone(), + external_handle: external_handle.clone(), + session, + } + }) + }) + .flatten() + } + }) + .collect() + } + fn take_worker(&mut self, worker: &RuntimeWorkerRef) -> Vec { let mut sessions = Vec::new(); if let Some(session) = self.attachments.remove(worker) { @@ -759,6 +831,17 @@ async fn close_registered_workdir_sessions( } } +async fn close_observed_worker_workdir_sessions( + registry: &Arc>, + observed: &[RegisteredWorkdirSession], +) -> std::result::Result<(), String> { + let registered = registry + .lock() + .map_err(|_| "Workdir session registry was poisoned".to_string())? + .take_observed(observed); + close_registered_workdir_sessions(registry, registered).await +} + async fn close_worker_workdir_sessions( registry: &Arc>, worker: &RuntimeWorkerRef, @@ -2220,6 +2303,7 @@ impl WorkerRemovalService { return (false, diagnostics); }; let lifecycle_request = WorkerLifecycleRequest { + stop_intent: None, reason: Some("Backend spawn finalize failed; compensating Runtime Worker".to_string()), ticket_assignment: None, }; @@ -5513,6 +5597,7 @@ impl WorkspaceApi { match self.runtime.stop_worker( &worker, WorkerLifecycleRequest { + stop_intent: None, reason: Some(format!( "Backend Job attempt {attempt_id} reached terminal state" )), @@ -6580,6 +6665,15 @@ impl WorkspaceApi { &self, worker: &RuntimeWorkerRef, request: server_api::WorkerRestoreRequest, + ) -> ApiResult { + self.restore_workspace_worker_with_subject(worker, request, None) + } + + fn restore_workspace_worker_with_subject( + &self, + worker: &RuntimeWorkerRef, + request: server_api::WorkerRestoreRequest, + expected_subject: Option<(&str, &server_api::SubjectBodyRef)>, ) -> ApiResult { if request.request_id.trim().is_empty() || request.request_id.len() > 512 @@ -6591,6 +6685,19 @@ impl WorkspaceApi { ) .into()); } + let subject_gate = expected_subject + .map(|(subject, _)| subjektiv_journal::lifecycle_lock(self, subject)) + .transpose()?; + let subject_guard = subject_gate + .as_ref() + .map(|gate| { + gate.lock() + .map_err(|_| Error::Store("Subject lifecycle lock poisoned".into())) + }) + .transpose()?; + if let Some((subject, expected)) = expected_subject { + subject_body::require_expected(self, subject, Some(expected))?; + } // Auth is checked by WorkspaceWorker before this read-only owner dispatch. // Existing owners reach receipts/reconciliation before NEW launch eligibility. let (recovered, owned_preparation) = self @@ -6608,6 +6715,24 @@ impl WorkspaceApi { result.worker = self.current_restore_summary(worker, result.worker.is_some()); return Ok(result); } + if expected_subject.is_some() && owned_preparation.is_none() { + let live = self + .runtime + .worker(worker) + .map_err(RuntimeRegistryError::into_error)?; + if live.restore_observation_token.as_deref() + == Some(request.expected_observation_token.as_str()) + && live.availability + == protocol::subscription::SubscriptionWorkerAvailability::Observed + && live.worker_state.is_some() + { + return Ok(InternalWorkerRestoreResult { + state: server_api::WorkerRestoreState::Accepted, + worker: Some(live), + diagnostics: Vec::new(), + }); + } + } let preparation = if let Some(preparation) = owned_preparation { preparation } else { @@ -6642,6 +6767,10 @@ impl WorkspaceApi { Error::Config("Runtime did not retain admitted Restore preparation".into()) })? }; + // Runtime has retained the exact Restore owner/preparation. Release the + // Subject gate before execution can call back into Session/Journal APIs. + drop(subject_guard); + drop(subject_gate); let mut restored = self .runtime .restore_worker( @@ -7703,6 +7832,16 @@ fn generated_workspace_contract_router(service: ServerApiContractService) -> Rou .merge(server_api::server_api_axum::subjektiv_subject_get( service.clone(), )) + .merge(server_api::server_api_axum::subjektiv_subject_body_get( + service.clone(), + )) + .merge(server_api::server_api_axum::subjektiv_subject_body_start( + service.clone(), + )) + .merge(server_api::server_api_axum::subjektiv_subject_body_stop( + service.clone(), + )) + .merge(server_api::server_api_axum::subjektiv_subject_body_stop_outcome(service.clone())) .merge(server_api::server_api_axum::subjektiv_subject_behavior_update(service.clone())) .merge(server_api::server_api_axum::subjektiv_resident_surface( service.clone(), @@ -8335,6 +8474,12 @@ fn worker_lifecycle_request_from_api( request: server_api::RuntimeWorkerLifecycleRequest, ) -> WorkerLifecycleRequest { WorkerLifecycleRequest { + stop_intent: request + .stop_intent + .map(|intent| runtime_api::WorkerStopIntent { + expected_observation_token: intent.expected_observation_token, + request_id: intent.request_id, + }), reason: request.reason, ticket_assignment: request .ticket_assignment @@ -8443,6 +8588,7 @@ fn worker_lifecycle_result_to_api( result: WorkerLifecycleResult, ) -> server_api::RuntimeWorkerLifecycleResult { server_api::RuntimeWorkerLifecycleResult { + replayed: result.replayed, state: result.state.into(), runtime_id: result.worker.runtime_id, worker_id: result.worker.worker_id, @@ -10074,6 +10220,81 @@ impl server_api::ServerApi for ServerApiContractService { .map_err(ApiError::into_repository_api_error) } + async fn subjektiv_subject_body_get( + &self, + context: server_api::ServerRequestContext, + workspace_id: String, + subject_id: String, + ) -> std::result::Result { + subject_body::get( + self.workspace_api()?.clone(), + context, + workspace_id, + subject_id, + ) + .await + .map_err(ApiError::into_repository_api_error) + } + + async fn subjektiv_subject_body_start( + &self, + context: server_api::ServerRequestContext, + workspace_id: String, + subject_id: String, + request: server_api::SubjectBodyStartRequest, + ) -> std::result::Result + { + subject_body::start( + self.workspace_api()?.clone(), + context, + workspace_id, + subject_id, + request, + ) + .await + .map_err(ApiError::into_repository_api_error) + } + + async fn subjektiv_subject_body_stop( + &self, + context: server_api::ServerRequestContext, + workspace_id: String, + subject_id: String, + request: server_api::SubjectBodyStopRequest, + ) -> std::result::Result + { + subject_body::stop( + self.workspace_api()?.clone(), + context, + workspace_id, + subject_id, + request, + ) + .await + .map_err(ApiError::into_repository_api_error) + } + + async fn subjektiv_subject_body_stop_outcome( + &self, + context: server_api::ServerRequestContext, + workspace_id: String, + subject_id: String, + request: server_api::SubjectBodyStopRequest, + ) -> std::result::Result< + server_api::SubjectBodyStopOutcomeResponse, + server_api::RepositoryApiError, + > { + subject_body::stop_outcome( + self.workspace_api()?.clone(), + context, + workspace_id, + subject_id, + request, + ) + .await + .map_err(ApiError::into_repository_api_error) + } + async fn subjektiv_subject_behavior_update( &self, workspace_id: String, @@ -16266,6 +16487,7 @@ async fn compensate_manual_ticket_worker_binding( match api.runtime.stop_worker( worker, WorkerLifecycleRequest { + stop_intent: None, reason: Some( "roll back failed manual Worker assignment capability binding".to_string(), ), @@ -16595,6 +16817,7 @@ async fn start_manual_ticket_worker_assignment_on_host( .stop_worker( &worker, WorkerLifecycleRequest { + stop_intent: None, reason: Some(format!( "bind authoritative Workdir capabilities for Ticket {} manual Worker assignment", record.ticket_id @@ -17329,6 +17552,7 @@ async fn cancel_ticket_worker( match api.runtime.cancel_worker( worker, WorkerLifecycleRequest { + stop_intent: None, reason: Some(format!("Ticket implementation cancelled: {reason}")), ticket_assignment: None, }, @@ -23815,16 +24039,28 @@ fn subjektiv_subject_response( let memory_fingerprint = open_subjektiv_store(api)? .memory_fingerprint(&subject.id) .map_err(subjektiv_store_error)?; - let current_worker = crate::subjektiv::subject_worker_singleton_key(&subject.id) - .ok() - .and_then(|key| { - api.store - .current_worker_singleton_owner(&api.config.workspace_id, &key) - .ok() - .flatten() - }) - .and_then(|lease| api.runtime.worker(&lease.worker).ok()) - .map(worker_launch_worker_summary); + let singleton_key = crate::subjektiv::subject_worker_singleton_key(&subject.id) + .map_err(|error| Error::InvalidInput(error.to_string()))?; + // Only an authoritative absent owner means no body. A failed lookup must + // not invite a second launch by presenting an occupied singleton as empty. + let owner = api + .store + .current_worker_singleton_owner(&api.config.workspace_id, &singleton_key)?; + let current_body = owner.as_ref().map(subject_body::reference); + let (current_worker, current_worker_diagnostics) = match owner { + None => (None, vec![]), + Some(lease) => match api.runtime.worker(&lease.worker) { + Ok(worker) => (Some(worker_launch_worker_summary(worker)), vec![]), + Err(error) => ( + None, + vec![server_api::Diagnostic { + code: "subject_body_observation_unavailable".into(), + severity: server_api::DiagnosticSeverity::Warning, + message: sanitize_backend_error(&error.message()), + }], + ), + }, + }; Ok(server_api::SubjektivSubjectResponse { id: subject.id, role: subject.role.as_str().to_string(), @@ -23838,6 +24074,8 @@ fn subjektiv_subject_response( }, created_at: subject.created_at, updated_at: subject.updated_at, + current_body, + current_worker_diagnostics, current_worker, }) } @@ -37515,13 +37753,24 @@ fn sync_all_runtime_workdir_observations(api: &WorkspaceApi) -> Vec ApiResult<()> { - let links = api + // The caller holds the Worker session fence. Rebinding may have won before + // that fence was reacquired; never project its new connection for an old stop. + let current = api .store - .list_worker_workdir_links(&api.config.workspace_id, &worker_record.worker)?; - for link in links { + .list_worker_workdir_links(api.workspace_id(), worker)?; + for link in observed_links.iter().filter(|observed| { + observed.unlinked_at.is_none() + && current.iter().any(|current| { + current.unlinked_at.is_none() + && current.connection_id == observed.connection_id + && current.alias == observed.alias + && current.workdir_id == observed.workdir_id + }) + }) { + let runtime_id = &worker.runtime_id; let result = api .runtime .working_directory(runtime_id, link.workdir_id.as_str()) @@ -42274,6 +42523,7 @@ mod tests { callback(); } WorkerLifecycleResult { + replayed: false, state: InternalWorkerOperationState::Accepted, worker: RuntimeWorkerRef::new(Self::RUNTIME_ID, worker_id), diagnostics: Vec::new(), @@ -42286,6 +42536,7 @@ mod tests { _request: WorkerLifecycleRequest, ) -> WorkerLifecycleResult { WorkerLifecycleResult { + replayed: false, state: InternalWorkerOperationState::Accepted, worker: RuntimeWorkerRef::new(Self::RUNTIME_ID, worker_id), diagnostics: Vec::new(), @@ -42955,15 +43206,23 @@ mod tests { register_test_runtime(&api, WorkdirlessFixtureRuntime::RUNTIME_ID).await; let stopped = Arc::new(tokio::sync::Notify::new()); let notification = stopped.clone(); - let runtime = WorkdirlessFixtureRuntime { - stop_callback: Some(Arc::new(move || notification.notify_one())), - ..Default::default() - }; - api.runtime.register_or_replace(runtime.clone()); let identity = RuntimeWorkerRef::new( WorkdirlessFixtureRuntime::RUNTIME_ID, Uuid::now_v7().to_string(), ); + let cleanup_lock = current_worker_session_lock(&api, &identity); + let session_guard = Arc::new(Mutex::new(None)); + let held_guard = session_guard.clone(); + let runtime = WorkdirlessFixtureRuntime { + stop_callback: Some(Arc::new(move || { + // Capture finished before dispatch. Hold the same fence only + // after admission, so cancellation still precedes cleanup. + *held_guard.lock().unwrap() = Some(cleanup_lock.clone().try_lock_owned().unwrap()); + notification.notify_one(); + })), + ..Default::default() + }; + api.runtime.register_or_replace(runtime.clone()); api.store .upsert_worker_registry(&WorkerRegistryRecord { workspace_id: TEST_WORKSPACE_ID.into(), @@ -42997,11 +43256,8 @@ mod tests { .lock() .unwrap() .insert_attachment(identity.clone(), session); - // An actual session fence prevents cleanup until after cancellation; + // The callback takes the session fence after capture and before ACK; // the Runtime receipt is the synchronization event, not an elapsed delay. - let session_guard = current_worker_session_lock(&api, &identity) - .lock_owned() - .await; let worker = WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap(); let waiter = tokio::spawn(async move { @@ -43009,6 +43265,7 @@ mod tests { .stop( &WorkerOperationContext::Backend, WorkerLifecycleRequest { + stop_intent: None, reason: None, ticket_assignment: None, }, @@ -43033,7 +43290,7 @@ mod tests { .attachments .contains_key(&identity) ); - drop(session_guard); + drop(session_guard.lock().unwrap().take()); tokio::time::timeout( std::time::Duration::from_secs(2), api.host_executors.host().unwrap().drain(), @@ -43052,6 +43309,145 @@ mod tests { ); } + #[tokio::test(flavor = "current_thread")] + async fn fresh_stop_does_not_close_sessions_rebound_before_cleanup() { + let directory = tempfile::tempdir().unwrap(); + std::fs::write(directory.path().join("alive"), "retained").unwrap(); + let api = test_api(directory.path()).await; + api.worker_projection.shutdown_async().await; + api.store.db_executor().drain().await.unwrap(); + register_test_runtime(&api, WorkdirlessFixtureRuntime::RUNTIME_ID).await; + let identity = RuntimeWorkerRef::new( + WorkdirlessFixtureRuntime::RUNTIME_ID, + Uuid::now_v7().to_string(), + ); + api.store + .upsert_worker_registry(&WorkerRegistryRecord { + workspace_id: TEST_WORKSPACE_ID.into(), + worker: identity.clone(), + display_name: "fresh stop resource lifetime".into(), + profile: None, + retention_state: "normal".into(), + transcript_ref: None, + session_ref: None, + summary_ref: None, + diagnostics_ref: None, + created_at: "1".into(), + updated_at: "1".into(), + }) + .unwrap(); + let session = || -> WorkdirSessionHandle { + Arc::new(workdir::LocalWorkdirSession::new( + manifest::Scope::writable(directory.path()).unwrap(), + directory.path().to_path_buf(), + )) + }; + let old_attachment = session(); + let old_command = session(); + let new_attachment = session(); + let new_command = session(); + let old_handle = { + let mut registry = api.workdir_sessions.lock().unwrap(); + registry.insert_attachment(identity.clone(), old_attachment.clone()); + registry.register_command( + identity.clone(), + "checkout".into(), + old_command.clone(), + CommandHandle("old-provider".into()), + ) + }; + let dispatched = Arc::new(tokio::sync::Notify::new()); + let notification = dispatched.clone(); + let cleanup_lock = current_worker_session_lock(&api, &identity); + let held = Arc::new(Mutex::new(None)); + let held_by_callback = held.clone(); + api.runtime.register_or_replace(WorkdirlessFixtureRuntime { + stop_callback: Some(Arc::new(move || { + // A genuine fresh-ACK path, not historical receipt replay. Capture + // must release this fence before Runtime dispatch/callbacks. + *held_by_callback.lock().unwrap() = + Some(cleanup_lock.clone().try_lock_owned().unwrap()); + notification.notify_one(); + })), + ..Default::default() + }); + let worker = + WorkspaceWorker::resolve(&api, &identity.runtime_id, &identity.worker_id).unwrap(); + let waiter = tokio::spawn(async move { + worker + .stop( + &WorkerOperationContext::Backend, + WorkerLifecycleRequest::default(), + ) + .await + }); + tokio::time::timeout(std::time::Duration::from_secs(2), dispatched.notified()) + .await + .unwrap(); + let guard = held.lock().unwrap().take().unwrap(); + // The same fence and registry mutations used by restore's first Workdir + // operation/rebind win before the old stop's delayed cleanup. + let new_handle = { + let mut registry = api.workdir_sessions.lock().unwrap(); + let replaced = registry + .insert_attachment(identity.clone(), new_attachment.clone()) + .unwrap(); + assert!(Arc::ptr_eq(&replaced, &old_attachment)); + registry.register_command( + identity.clone(), + "checkout".into(), + new_command.clone(), + CommandHandle("new-provider".into()), + ) + }; + old_attachment.close().await.unwrap(); + drop(guard); + let result = tokio::time::timeout(std::time::Duration::from_secs(2), waiter) + .await + .unwrap() + .unwrap() + .unwrap(); + assert_eq!(result.state, InternalWorkerOperationState::Accepted); + assert!(!result.replayed); + { + let registry = api.workdir_sessions.lock().unwrap(); + assert!(Arc::ptr_eq( + registry.attachments.get(&identity).unwrap(), + &new_attachment + )); + assert!( + registry + .command(&identity, "checkout", &old_handle) + .is_none() + ); + assert!( + registry + .command(&identity, "checkout", &new_handle) + .is_some() + ); + } + for session in [&new_attachment, &new_command] { + session + .stat(workdir::StatRequest { + path: workdir::WorkdirPath::new("alive").unwrap(), + }) + .await + .unwrap(); + } + assert!( + old_command + .stat(workdir::StatRequest { + path: workdir::WorkdirPath::new("alive").unwrap() + }) + .await + .is_err() + ); + // Ordinary revocation still releases the surviving current resources. + close_worker_workdir_sessions(&api.workdir_sessions, &identity) + .await + .unwrap(); + } + #[tokio::test] async fn command_session_survives_attachment_refresh_until_worker_revocation() { let directory = tempfile::tempdir().unwrap(); @@ -45270,6 +45666,7 @@ mod tests { .stop_worker( &worker, WorkerLifecycleRequest { + stop_intent: None, reason: Some("exercise logical attachment reconciliation".to_string()), ticket_assignment: None, }, @@ -45407,6 +45804,7 @@ mod tests { .stop_worker( &worker, WorkerLifecycleRequest { + stop_intent: None, reason: Some("production Orchestrator restore regression test".to_string()), ticket_assignment: None, }, @@ -46511,6 +46909,7 @@ mod tests { .stop_worker( &worker, WorkerLifecycleRequest { + stop_intent: None, reason: Some("singleton restore test".to_string()), ticket_assignment: None, }, @@ -46968,6 +47367,7 @@ mod tests { .stop_worker( &target, WorkerLifecycleRequest { + stop_intent: None, reason: Some("prepare archive revocation regression".to_string()), ticket_assignment: None, }, @@ -48063,6 +48463,7 @@ mod tests { .stop_worker( &worker, WorkerLifecycleRequest { + stop_intent: None, reason: Some("test subject Worker restart".to_string()), ticket_assignment: None, }, @@ -49820,6 +50221,7 @@ mod tests { .stop_worker( &forged_internal.worker, WorkerLifecycleRequest { + stop_intent: None, reason: Some("remove forged Orchestrator claimant".to_string()), ticket_assignment: None, }, @@ -57300,6 +57702,7 @@ mod tests { .stop_worker( &singleton_worker, WorkerLifecycleRequest { + stop_intent: None, reason: Some("replace completed Memory singleton".to_string()), ticket_assignment: None, }, @@ -61626,6 +62029,7 @@ mod tests { .stop_worker( &target, WorkerLifecycleRequest { + stop_intent: None, reason: Some("prepare WorkerRemove regression".to_string()), ticket_assignment: None, }, @@ -65258,6 +65662,7 @@ mod tests { .stop_worker( &worker, WorkerLifecycleRequest { + stop_intent: None, reason: Some("prepare singleton cleanup".to_string()), ticket_assignment: None, }, @@ -65383,6 +65788,7 @@ mod tests { .stop_worker( &worker, WorkerLifecycleRequest { + stop_intent: None, reason: Some("prepare cleanup projection regression".to_string()), ticket_assignment: None, }, @@ -69006,6 +69412,7 @@ mod tests { .stop_worker( &worker_ref, WorkerLifecycleRequest { + stop_intent: None, reason: Some("retire Worker after rejected automatic restore".to_string()), ticket_assignment: None, }, diff --git a/crates/workspace-server/src/server/subject_authorization.rs b/crates/workspace-server/src/server/subject_authorization.rs index 3d533ab9b..141d48098 100644 --- a/crates/workspace-server/src/server/subject_authorization.rs +++ b/crates/workspace-server/src/server/subject_authorization.rs @@ -885,6 +885,7 @@ pub(super) async fn interrupt_subject_workers( let cancellation = match api.host_executors.host() { Ok(host) => host.run(move || -> ApiResult<_> { Ok(cancellation_api.runtime.cancel_worker(&cancellation_worker, WorkerLifecycleRequest { + stop_intent: None, reason: Some("Subject permission assignment changed; new operations are fenced".into()), ticket_assignment: None, }).map_err(RuntimeRegistryError::into_error)?) }).await, diff --git a/crates/workspace-server/src/server/subject_body.rs b/crates/workspace-server/src/server/subject_body.rs new file mode 100644 index 000000000..fe28e3b8d --- /dev/null +++ b/crates/workspace-server/src/server/subject_body.rs @@ -0,0 +1,522 @@ +//! Subject adapters derive targets from the ordinary singleton and dispatch ordinary +//! Worker lifecycle operations. Receipts are historical; observations are read separately. +use super::*; +use server_api::{ + SubjectBodyDisposition, SubjectBodyMutationResponse, SubjectBodyObservation, SubjectBodyRef, + SubjectBodyResponse, SubjectBodyStartRequest, SubjectBodyStopRequest, +}; + +fn diagnostic(code: &str, message: String) -> server_api::Diagnostic { + server_api::Diagnostic { + code: code.into(), + severity: server_api::DiagnosticSeverity::Warning, + message: sanitize_backend_error(&message), + } +} + +fn require_subject( + api: &WorkspaceApi, + subject: &str, +) -> ApiResult { + let record = open_subjektiv_store(api)? + .subject(subject) + .map_err(subjektiv_store_error)? + .ok_or_else(|| Error::SubjektivSubjectNotFound(subject.into()))?; + if record.state != crate::subjektiv::SubjectState::Active + || subjektiv_deletion::pending(api, subject)? + { + return Err(Error::WorkspacePermissionDenied( + "Subject is deleted, being deleted or no longer active".into(), + ) + .into()); + } + Ok(record) +} + +fn current( + api: &WorkspaceApi, + subject: &str, +) -> ApiResult> { + require_subject(api, subject)?; + let key = crate::subjektiv::subject_worker_singleton_key(subject) + .map_err(|error| Error::InvalidInput(error.to_string()))?; + Ok(api + .store + .current_worker_singleton_owner(api.workspace_id(), &key)?) +} + +pub(super) fn reference(lease: &crate::store::WorkerSingletonLease) -> SubjectBodyRef { + SubjectBodyRef { + runtime_id: lease.worker.runtime_id.clone(), + worker_id: lease.worker.worker_id.clone(), + generation: lease.generation.to_string(), + } +} + +/// Call while holding the existing Subject lifecycle gate through Runtime admission. +pub(super) fn require_expected( + api: &WorkspaceApi, + subject: &str, + expected: Option<&SubjectBodyRef>, +) -> ApiResult> { + let lease = current(api, subject)?; + if lease.as_ref().map(reference).as_ref() != expected { + return Err(Error::RepositoryConflict( + "Subject body changed; reload before a new lifecycle intent".into(), + ) + .into()); + } + Ok(lease) +} + +pub(super) fn snapshot(api: &WorkspaceApi, subject: &str) -> ApiResult { + let lease = current(api, subject)?; + let observation = match lease.as_ref() { + None => SubjectBodyObservation::Absent, + Some(lease) => { + let observe = || -> ApiResult { + let live = api + .runtime + .worker(&lease.worker) + .map_err(RuntimeRegistryError::into_error)?; + let record = api + .store + .get_worker_registry(api.workspace_id(), &lease.worker)? + .ok_or_else(|| Error::UnknownWorker { + worker: lease.worker.clone(), + })?; + let links = api + .store + .list_worker_workdir_links(api.workspace_id(), &lease.worker)?; + let workdirs = api.store.list_workdir_registry(api.workspace_id(), 500)?; + project_worker_registry_projection(api, Some(&live), &record, links, &workdirs) + }; + match observe() { + Ok(worker) => SubjectBodyObservation::Available { worker }, + Err(error) => SubjectBodyObservation::Unavailable { + diagnostics: vec![diagnostic( + "subject_body_observation_unavailable", + error.error.to_string(), + )], + }, + } + } + }; + if current(api, subject)? != lease { + return Err(Error::RepositoryConflict( + "Subject body changed during observation; reload its current state".into(), + ) + .into()); + } + Ok(SubjectBodyResponse { + workspace_id: api.workspace_id().into(), + subject_id: subject.into(), + current: lease.as_ref().map(reference), + observation, + can_start: false, + can_stop: false, + }) +} + +async fn account_context( + api: &WorkspaceApi, + context: &server_api::ServerRequestContext, + workspace: &str, +) -> ApiResult<(String, WorkerOperationContext)> { + validate_workspace_scope(api, workspace)?; + let actor = subject_authorization::administration_actor(context)?; + require_workspace_owner(api, workspace, actor, "Subject body lifecycle").await?; + let authority = WorkerOperationContext::from_request_async(api, context) + .await + .map_err(|error| Error::WorkspacePermissionDenied(error.message))?; + authority.authorize_workspace(api).await?; + Ok((actor.account_id.clone(), authority)) +} + +pub(super) async fn get( + api: WorkspaceApi, + context: server_api::ServerRequestContext, + workspace: String, + subject: String, +) -> ApiResult { + account_context(&api, &context, &workspace).await?; + api.host_executors + .host()? + .run(move || { + let mut body = snapshot(&api, &subject)?; + body.can_start = true; + body.can_stop = true; + Ok(body) + }) + .await +} + +fn validate_request_id(id: &str) -> ApiResult<()> { + if id.trim().is_empty() || id.len() > 128 || id.chars().any(char::is_control) { + return Err(Error::InvalidInput( + "request_id must contain 1..128 bytes without control characters".into(), + ) + .into()); + } + Ok(()) +} + +fn operation_id(account: &str, workspace: &str, subject: &str, request: &str) -> String { + let bytes = serde_json::to_vec(&(account, workspace, subject, request)) + .expect("lifecycle identity is text"); + let digest = Sha256::digest(bytes) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::(); + format!("subject-body-start:{digest}") +} + +async fn finish( + api: WorkspaceApi, + context: server_api::ServerRequestContext, + workspace: String, + subject: String, + request_id: String, + disposition: SubjectBodyDisposition, + diagnostics: Vec, +) -> SubjectBodyMutationResponse { + let body = get(api, context, workspace, subject).await.ok(); + SubjectBodyMutationResponse { + request_id, + disposition, + diagnostics, + body, + } +} + +fn stop_request(request: &SubjectBodyStopRequest) -> WorkerLifecycleRequest { + WorkerLifecycleRequest { + reason: Some("Stopped from the Subject screen; Subject data is retained".into()), + ticket_assignment: None, + stop_intent: Some(runtime_api::WorkerStopIntent { + expected_observation_token: request.expected_observation_token.clone(), + request_id: request.request_id.clone(), + }), + } +} + +pub(super) async fn stop_outcome( + api: WorkspaceApi, + context: server_api::ServerRequestContext, + workspace: String, + subject: String, + request: SubjectBodyStopRequest, +) -> ApiResult { + validate_request_id(&request.request_id)?; + let (_, authority) = account_context(&api, &context, &workspace).await?; + let lookup_api = api.clone(); + let lookup_subject = subject.clone(); + let expected = request.expected_current.clone(); + let lease = api + .store + .db_executor() + .run(move || { + require_expected(&lookup_api, &lookup_subject, Some(&expected))?.ok_or_else(|| { + ApiError::from(Error::RepositoryConflict( + "Subject has no current body".into(), + )) + }) + }) + .await?; + let worker = + WorkspaceWorker::resolve_async(&api, &lease.worker.runtime_id, &lease.worker.worker_id) + .await?; + let result = worker + .stop_outcome_current_subject( + &authority, + stop_request(&request), + subject.clone(), + request.expected_current, + ) + .await?; + let outcome = match result { + runtime_api::WorkerStopOutcome::NotFound => server_api::SubjectBodyStopOutcome::NotFound, + runtime_api::WorkerStopOutcome::Pending => server_api::SubjectBodyStopOutcome::Pending, + runtime_api::WorkerStopOutcome::Stopped { .. } => { + server_api::SubjectBodyStopOutcome::Stopped + } + }; + let body = get(api, context, workspace, subject).await.ok(); + Ok(server_api::SubjectBodyStopOutcomeResponse { + request_id: request.request_id, + outcome, + body, + }) +} + +pub(super) async fn stop( + api: WorkspaceApi, + context: server_api::ServerRequestContext, + workspace: String, + subject: String, + request: SubjectBodyStopRequest, +) -> ApiResult { + validate_request_id(&request.request_id)?; + let (_, authority) = account_context(&api, &context, &workspace).await?; + let lookup_api = api.clone(); + let lookup_subject = subject.clone(); + let expected = request.expected_current.clone(); + let lease = api + .store + .db_executor() + .run(move || { + require_expected(&lookup_api, &lookup_subject, Some(&expected))?.ok_or_else(|| { + ApiError::from(Error::RepositoryConflict( + "Subject has no current body".into(), + )) + }) + }) + .await?; + let worker = + WorkspaceWorker::resolve_async(&api, &lease.worker.runtime_id, &lease.worker.worker_id) + .await?; + let result = worker + .stop_current_subject( + &authority, + stop_request(&request), + subject.clone(), + request.expected_current, + ) + .await; + let (disposition, diagnostics) = match result { + Ok(result) => ( + if result.state == InternalWorkerOperationState::Accepted { + SubjectBodyDisposition::Accepted + } else { + SubjectBodyDisposition::Unknown + }, + result.diagnostics.into_iter().map(Into::into).collect(), + ), + Err(error) + if matches!( + error.error, + Error::RepositoryConflict(_) + | Error::WorkspacePermissionDenied(_) + | Error::RestoreObservationConflict + ) => + { + return Err(error); + } + // A post-dispatch failure may have followed a real stop. Keep the exact intent. + Err(error) => ( + SubjectBodyDisposition::Unknown, + vec![diagnostic( + "subject_body_stop_outcome_unknown", + error.error.to_string(), + )], + ), + }; + Ok(finish( + api, + context, + workspace, + subject, + request.request_id, + disposition, + diagnostics, + ) + .await) +} + +pub(super) async fn start( + api: WorkspaceApi, + context: server_api::ServerRequestContext, + workspace: String, + subject: String, + request: SubjectBodyStartRequest, +) -> ApiResult { + validate_request_id(&request.request_id)?; + let (account, authority) = account_context(&api, &context, &workspace).await?; + if let Some(expected) = request.expected_current.as_ref() { + let observed = get( + api.clone(), + context.clone(), + workspace.clone(), + subject.clone(), + ) + .await?; + if observed.current.as_ref() != Some(expected) { + return Err( + Error::RepositoryConflict("Subject body changed before start".into()).into(), + ); + } + let SubjectBodyObservation::Available { .. } = observed.observation else { + return Err(Error::RepositoryConflict( + "Current body cannot be observed; do not create a replacement".into(), + ) + .into()); + }; + let token = request.expected_observation_token.clone().ok_or_else(|| { + Error::InvalidInput( + "Starting an existing body requires its observed Runtime token".into(), + ) + })?; + // Even a live-body reuse checks Runtime's existing request owner first; + // changed input cannot bypass a retained Restore receipt via a fresh GET. + let worker = + WorkspaceWorker::resolve_async(&api, &expected.runtime_id, &expected.worker_id).await?; + let result = worker + .restore_current_subject( + &authority, + server_api::WorkerRestoreRequest { + expected_observation_token: token, + request_id: operation_id(&account, &workspace, &subject, &request.request_id), + }, + subject.clone(), + expected.clone(), + ) + .await; + let (disposition, diagnostics) = match result { + Ok(result) => ( + match result.state { + server_api::WorkerRestoreState::Accepted => SubjectBodyDisposition::Accepted, + server_api::WorkerRestoreState::Rejected + | server_api::WorkerRestoreState::RolledBack + | server_api::WorkerRestoreState::Superseded => { + SubjectBodyDisposition::Rejected + } + server_api::WorkerRestoreState::ReconciliationRequired => { + SubjectBodyDisposition::Unknown + } + }, + result.diagnostics.into_iter().map(Into::into).collect(), + ), + Err(error) + if matches!( + error.error, + Error::RepositoryConflict(_) + | Error::WorkspacePermissionDenied(_) + | Error::RestoreObservationConflict + ) => + { + return Err(error); + } + Err(error) => ( + SubjectBodyDisposition::Unknown, + vec![diagnostic( + "subject_body_restore_outcome_unknown", + error.error.to_string(), + )], + ), + }; + return Ok(finish( + api, + context, + workspace, + subject, + request.request_id, + disposition, + diagnostics, + ) + .await); + } + if request.expected_observation_token.is_some() { + return Err( + Error::InvalidInput("An absent body has no Runtime observation token".into()).into(), + ); + } + let create_api = api.clone(); + let create_context = context.clone(); + let create_subject = subject.clone(); + let create_workspace = workspace.clone(); + let request_id = request.request_id.clone(); + let operation = operation_id(&account, &workspace, &subject, &request_id); + let result = run_host_workflow(api.host_executors.host()?, async move { + let subject_record = require_subject(&create_api, &create_subject)?; + // An exact creation retry uses the same ordinary reservation, including its + // original Runtime. It never changes an unknown request into a new launch. + let reserved_runtime = create_api + .config_store + .reserved_worker_create_runtime(&create_workspace, &operation)?; + if reserved_runtime.is_none() { + require_expected(&create_api, &create_subject, None)?; + } + let runtime_id = match reserved_runtime { + Some(runtime) => runtime, + None => { + let runtimes = create_api.runtime.list_runtimes(100); + runtimes + .items + .iter() + .filter(|runtime| runtime.worker_creation_available) + .min_by_key(|runtime| { + ( + runtime.runtime_id != EMBEDDED_WORKER_RUNTIME_ID, + runtime.runtime_id.clone(), + ) + }) + .map(|runtime| runtime.runtime_id.clone()) + .ok_or_else(|| { + Error::Config("No Runtime is available to start this Subject".into()) + })? + } + }; + let input = serde_json::to_vec(&(&account, &create_workspace, &create_subject, &request)) + .map_err(|error| Error::InvalidInput(error.to_string()))?; + let fingerprint = Sha256::digest(input) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::(); + create_workspace_worker_authenticated_on_host( + create_api, + contract_request_headers(&create_context) + .map_err(|error| Error::WorkspacePermissionDenied(error.message))?, + server_api::CreateWorkspaceWorkerRequest { + runtime_id, + display_name: subject_record.role.as_str().into(), + singleton_key: None, + profile: Some("builtin:async-chat".into()), + ticket_assignment: None, + initial_submit: vec![], + workdir_attachments: vec![], + feature_connections: server_api::WorkspaceWorkerFeatureConnectionsRequest { + subjektiv: Some(server_api::SubjektivWorkerConnectionRequest { + subject_id: create_subject, + }), + }, + control_operation_id: None, + execution_grant: None, + }, + Some(WorkerControlOperation { + operation_id: operation, + input_fingerprint: format!("sha256:{fingerprint}"), + }), + Some(&create_context), + ) + .await + }) + .await; + let (disposition, diagnostics) = match result { + Ok(Json(result)) => (SubjectBodyDisposition::Accepted, result.diagnostics), + Err(error) + if matches!( + error.error, + Error::RepositoryConflict(_) | Error::WorkspacePermissionDenied(_) + ) => + { + return Err(error); + } + Err(error) => ( + SubjectBodyDisposition::Unknown, + vec![diagnostic( + "subject_body_start_outcome_unknown", + error.error.to_string(), + )], + ), + }; + Ok(finish( + api, + context, + workspace, + subject, + request_id, + disposition, + diagnostics, + ) + .await) +} diff --git a/crates/workspace-server/src/server/subjektiv_deletion.rs b/crates/workspace-server/src/server/subjektiv_deletion.rs index 9063f3b90..f925b8e47 100644 --- a/crates/workspace-server/src/server/subjektiv_deletion.rs +++ b/crates/workspace-server/src/server/subjektiv_deletion.rs @@ -240,6 +240,7 @@ async fn remove_worker( .stop_worker( worker, WorkerLifecycleRequest { + stop_intent: None, reason: Some(format!("Explicit Subject deletion {operation}")), ticket_assignment: None, }, diff --git a/crates/workspace-server/src/server/subjektiv_job.rs b/crates/workspace-server/src/server/subjektiv_job.rs index d376a2781..e816010c9 100644 --- a/crates/workspace-server/src/server/subjektiv_job.rs +++ b/crates/workspace-server/src/server/subjektiv_job.rs @@ -113,6 +113,7 @@ pub(super) async fn retire_legacy_worker(api: &WorkspaceApi, subject_id: &str) - .stop_worker( &owner.worker, WorkerLifecycleRequest { + stop_intent: None, reason: Some( "Retire legacy subjektiv consolidator after foreground completion" .into(), diff --git a/crates/workspace-server/src/server/tests/api_logging_tests.rs b/crates/workspace-server/src/server/tests/api_logging_tests.rs index 604296ac5..4c045e51d 100644 --- a/crates/workspace-server/src/server/tests/api_logging_tests.rs +++ b/crates/workspace-server/src/server/tests/api_logging_tests.rs @@ -114,6 +114,7 @@ async fn cleanup_retention_failures_preserve_cause_on_execution_and_recovery() { .stop_worker( &worker, WorkerLifecycleRequest { + stop_intent: None, reason: Some("retention diagnostic fixture".into()), ticket_assignment: None, }, diff --git a/crates/workspace-server/src/server/tests/protocol_backpressure_tests.rs b/crates/workspace-server/src/server/tests/protocol_backpressure_tests.rs index e6e3ea742..9e80d1cc3 100644 --- a/crates/workspace-server/src/server/tests/protocol_backpressure_tests.rs +++ b/crates/workspace-server/src/server/tests/protocol_backpressure_tests.rs @@ -308,6 +308,7 @@ async fn browser_method_queue_overflow_fails_fast_and_does_not_pin_session_clean worker.stop( &WorkerOperationContext::Backend, WorkerLifecycleRequest { + stop_intent: None, reason: Some("cleanup-after-method-overflow".into()), ticket_assignment: None, }, @@ -674,6 +675,7 @@ async fn assert_same_worker_reconnect_and_stop( worker.stop( &WorkerOperationContext::Backend, WorkerLifecycleRequest { + stop_intent: None, reason: Some("cleanup-after-stalled-protocol".into()), ticket_assignment: None, }, diff --git a/crates/workspace-server/src/server/tests/restore_workdir_tests.rs b/crates/workspace-server/src/server/tests/restore_workdir_tests.rs index f11276a78..cfd69913b 100644 --- a/crates/workspace-server/src/server/tests/restore_workdir_tests.rs +++ b/crates/workspace-server/src/server/tests/restore_workdir_tests.rs @@ -116,6 +116,7 @@ async fn restore_keeps_offline_external_and_unmaterialized_repository_attachment .stop_worker( &worker, WorkerLifecycleRequest { + stop_intent: None, reason: None, ticket_assignment: None, }, @@ -186,6 +187,7 @@ async fn restore_keeps_offline_external_and_unmaterialized_repository_attachment .stop_worker( &worker, WorkerLifecycleRequest { + stop_intent: None, reason: None, ticket_assignment: None, }, diff --git a/crates/workspace-server/src/server/tests/restored_parent_control_tests.rs b/crates/workspace-server/src/server/tests/restored_parent_control_tests.rs index 1c1b9ffd9..fc8a4cf28 100644 --- a/crates/workspace-server/src/server/tests/restored_parent_control_tests.rs +++ b/crates/workspace-server/src/server/tests/restored_parent_control_tests.rs @@ -442,6 +442,7 @@ async fn restored_runtime_parent_lists_and_inputs_existing_child() { .stop_worker( &parent, WorkerLifecycleRequest { + stop_intent: None, reason: Some("restart parent".into()), ticket_assignment: None, }, diff --git a/crates/workspace-server/src/server/tests/subject_authorization_tests.rs b/crates/workspace-server/src/server/tests/subject_authorization_tests.rs index f1abbda75..b67100676 100644 --- a/crates/workspace-server/src/server/tests/subject_authorization_tests.rs +++ b/crates/workspace-server/src/server/tests/subject_authorization_tests.rs @@ -9,6 +9,8 @@ use super::*; mod chat_delegation_connected; #[path = "chat_tests.rs"] mod chat_tests; +#[path = "subject_body_tests.rs"] +mod subject_body_tests; #[path = "subject_job_model_supply_tests.rs"] mod subject_job_model_supply; #[path = "subject_permission_management_tests.rs"] diff --git a/crates/workspace-server/src/server/tests/subject_body_tests.rs b/crates/workspace-server/src/server/tests/subject_body_tests.rs new file mode 100644 index 000000000..e086bdb06 --- /dev/null +++ b/crates/workspace-server/src/server/tests/subject_body_tests.rs @@ -0,0 +1,278 @@ +//! Source for the authenticated Subject lifecycle adapter contract. Router/Account, +//! Subject SQLite and ordinary create reservations are real; no model or product +//! process is started. Runtime execution and restore/stop callbacks are exercised +//! by the connected suites, not inferred from this projection/admission fixture. +use super::*; + +struct BodyFixture { + api: WorkspaceApi, + owner: String, + subject: String, + _dir: tempfile::TempDir, +} + +impl BodyFixture { + async fn new() -> Self { + let dir = tempfile::tempdir().unwrap(); + let api = test_api(dir.path()).await; + let owner = seed_test_workspace_owner_token(api.store.as_ref(), "body-owner").await; + let subject = open_subjektiv_store(&api) + .unwrap() + .create_subject(crate::subjektiv::SubjectRole::new("companion").unwrap()) + .unwrap() + .id; + Self { + api, + owner, + subject, + _dir: dir, + } + } + fn path(&self) -> String { + format!( + "/api/w/{TEST_WORKSPACE_ID}/subjektiv/subjects/{}/body", + self.subject + ) + } + async fn read(&self) -> Value { + let (status, body) = response( + &self.api, + human_request(&self.owner, "GET", &self.path(), None), + ) + .await; + assert_eq!(status, StatusCode::OK, "{body}"); + body + } + fn reserve(&self) -> crate::store::WorkerSingletonLease { + let key = crate::subjektiv::subject_worker_singleton_key(&self.subject).unwrap(); + self.api + .config_store + .reserve_worker_create( + TEST_WORKSPACE_ID, + "runtime-test", + "body-start-reservation", + &"f".repeat(64), + Some(&key), + &self + .api + .config_store + .get_workspace_memory_settings(TEST_WORKSPACE_ID) + .unwrap(), + ) + .unwrap() + .singleton + .unwrap() + } +} + +#[tokio::test] +async fn body_read_retains_an_occupied_owner_when_runtime_observation_is_unavailable() { + let f = BodyFixture::new().await; + let absent = f.read().await; + assert!(absent["current"].is_null()); + assert_eq!(absent["observation"]["kind"], "absent"); + assert_eq!(absent["can_start"], true); + let lease = f.reserve(); + let occupied = f.read().await; + assert_eq!(occupied["current"]["worker_id"], lease.worker.worker_id); + assert_eq!( + occupied["current"]["generation"], + lease.generation.to_string() + ); + assert_eq!(occupied["observation"]["kind"], "unavailable"); + let (status, subject) = response( + &f.api, + human_request( + &f.owner, + "GET", + f.path().strip_suffix("/body").unwrap(), + None, + ), + ) + .await; + assert_eq!(status, StatusCode::OK, "{subject}"); + assert_eq!(subject["current_body"], occupied["current"]); + assert!( + !subject["current_worker_diagnostics"] + .as_array() + .unwrap() + .is_empty() + ); +} + +#[tokio::test] +async fn body_operations_reject_nonowners_and_cross_workspace_targets() { + let f = BodyFixture::new().await; + let reader = seed_test_api_token(f.api.store.as_ref(), "body-reader"); + for (method, suffix, payload) in [ + ("GET", "", None), + ( + "POST", + "/start", + Some( + json!({"request_id":"start", "expected_current":null, "expected_observation_token":null}), + ), + ), + ] { + let (status, body) = response( + &f.api, + human_request(&reader, method, &format!("{}{suffix}", f.path()), payload), + ) + .await; + assert_eq!(status, StatusCode::FORBIDDEN, "{body}"); + } + let path = f.path().replace(TEST_WORKSPACE_ID, "another-workspace"); + let (status, body) = response(&f.api, human_request(&f.owner, "GET", &path, None)).await; + assert_eq!(status, StatusCode::NOT_FOUND, "{body}"); + assert!(f.read().await["current"].is_null()); +} + +#[tokio::test] +async fn body_stop_and_outcome_reject_a_target_from_another_subject_before_runtime_dispatch() { + let f = BodyFixture::new().await; + let lease = f.reserve(); + let other = open_subjektiv_store(&f.api) + .unwrap() + .create_subject(crate::subjektiv::SubjectRole::new("other-companion").unwrap()) + .unwrap(); + let path = format!( + "/api/w/{TEST_WORKSPACE_ID}/subjektiv/subjects/{}/body", + other.id + ); + let request = json!({"request_id":"old-screen", "expected_current":subject_body::reference(&lease), "expected_observation_token":"old-token"}); + for suffix in ["/stop", "/stop/outcome"] { + let (status, body) = response( + &f.api, + human_request( + &f.owner, + "POST", + &format!("{path}{suffix}"), + Some(request.clone()), + ), + ) + .await; + assert_eq!(status, StatusCode::CONFLICT, "{body}"); + } + assert_eq!( + f.read().await["current"]["worker_id"], + lease.worker.worker_id + ); + assert!( + open_subjektiv_store(&f.api) + .unwrap() + .subject_permissions(&f.subject) + .unwrap() + .is_none() + ); +} + +#[tokio::test] +async fn body_mutations_reject_an_old_generation_without_releasing_its_owner() { + let f = BodyFixture::new().await; + let lease = f.reserve(); + let mut old = subject_body::reference(&lease); + old.generation = (lease.generation + 1).to_string(); + for (suffix, payload) in [ + ( + "/start", + json!({"request_id":"old-start", "expected_current":old, "expected_observation_token":"old-token"}), + ), + ( + "/stop", + json!({"request_id":"old-stop", "expected_current":old, "expected_observation_token":"old-token"}), + ), + ( + "/stop/outcome", + json!({"request_id":"old-stop", "expected_current":old, "expected_observation_token":"old-token"}), + ), + ] { + let (status, body) = response( + &f.api, + human_request( + &f.owner, + "POST", + &format!("{}{suffix}", f.path()), + Some(payload), + ), + ) + .await; + assert_eq!(status, StatusCode::CONFLICT, "{body}"); + } + assert_eq!( + f.read().await["current"], + serde_json::to_value(subject_body::reference(&lease)).unwrap() + ); +} + +#[tokio::test] +async fn body_operations_reject_a_subject_with_deletion_already_admitted() { + let f = BodyFixture::new().await; + let lease = f.reserve(); + open_subjektiv_store(&f.api) + .unwrap() + .begin_subject_deletion(&f.subject, "delete-body-fixture") + .unwrap(); + let stop = json!({"request_id":"stop", "expected_current":subject_body::reference(&lease), "expected_observation_token":"observed"}); + for (method, suffix, payload) in [ + ("GET", "", None), + ( + "POST", + "/start", + Some( + json!({"request_id":"start", "expected_current":null, "expected_observation_token":null}), + ), + ), + ("POST", "/stop", Some(stop.clone())), + ("POST", "/stop/outcome", Some(stop)), + ] { + let (status, body) = response( + &f.api, + human_request(&f.owner, method, &format!("{}{suffix}", f.path()), payload), + ) + .await; + assert!( + matches!(status, StatusCode::FORBIDDEN | StatusCode::NOT_FOUND), + "{status}: {body}" + ); + } + assert!( + open_subjektiv_store(&f.api) + .unwrap() + .subject(&f.subject) + .unwrap() + .is_some() + ); + assert_eq!( + f.api + .store + .current_worker_singleton_owner(TEST_WORKSPACE_ID, &lease.key) + .unwrap(), + Some(lease) + ); +} + +#[tokio::test] +async fn body_mutation_json_requires_explicit_identity_and_fences() { + let f = BodyFixture::new().await; + for (suffix, payload) in [ + ("/start", json!({"request_id":"start"})), + ( + "/stop", + json!({"request_id":"stop", "expected_observation_token":"observed"}), + ), + ("/stop/outcome", json!({"expected_current":null})), + ] { + let (status, body) = response( + &f.api, + human_request( + &f.owner, + "POST", + &format!("{}{suffix}", f.path()), + Some(payload), + ), + ) + .await; + assert!(status.is_client_error(), "{status}: {body}"); + } + assert!(f.read().await["current"].is_null()); +} diff --git a/crates/workspace-server/src/server/tests/subject_deletion_tests.rs b/crates/workspace-server/src/server/tests/subject_deletion_tests.rs index 5cc349ee1..bbec96f14 100644 --- a/crates/workspace-server/src/server/tests/subject_deletion_tests.rs +++ b/crates/workspace-server/src/server/tests/subject_deletion_tests.rs @@ -817,6 +817,7 @@ async fn old_job_removal_reconciles_exact_input_then_erases_copies_across_http_r .stop_worker( &worker, WorkerLifecycleRequest { + stop_intent: None, reason: Some("Prior independent removal".into()), ticket_assignment: None, }, @@ -1131,6 +1132,7 @@ async fn old_body_archive_reconciles_unchanged_and_remains_retained_after_subjec .stop_worker( &worker, WorkerLifecycleRequest { + stop_intent: None, reason: Some("Earlier removal".into()), ticket_assignment: None, }, diff --git a/crates/workspace-server/src/server/tests/subject_spawn_tests.rs b/crates/workspace-server/src/server/tests/subject_spawn_tests.rs index 95e7ba25f..91dd7358e 100644 --- a/crates/workspace-server/src/server/tests/subject_spawn_tests.rs +++ b/crates/workspace-server/src/server/tests/subject_spawn_tests.rs @@ -1,5 +1,59 @@ use super::*; +#[tokio::test] +async fn subject_current_worker_lookup_failure_is_not_reported_as_no_body() { + let workspace = tempfile::tempdir().unwrap(); + let api = test_api(workspace.path()).await; + let subject = open_subjektiv_store(&api) + .unwrap() + .create_subject(crate::subjektiv::SubjectRole::new("companion").unwrap()) + .unwrap(); + assert!( + subjektiv_subject_response(&api, subject.clone()) + .unwrap() + .current_worker + .is_none() + ); + let singleton_key = crate::subjektiv::subject_worker_singleton_key(&subject.id).unwrap(); + let memory = api + .config_store + .get_workspace_memory_settings(TEST_WORKSPACE_ID) + .unwrap(); + // Real reservation SQL owns the body, but no Runtime Worker exists yet. + // This is an unavailable observation, not evidence that another launch is safe. + api.config_store + .reserve_worker_create( + TEST_WORKSPACE_ID, + "runtime-test", + "subject-start", + &"f".repeat(64), + Some(&singleton_key), + &memory, + ) + .unwrap(); + assert!( + api.store + .current_worker_singleton_owner(TEST_WORKSPACE_ID, &singleton_key) + .unwrap() + .is_some() + ); + let projection = subjektiv_subject_response(&api, subject.clone()).unwrap(); + assert!(projection.current_body.is_some()); + assert!(projection.current_worker.is_none()); + assert!( + projection + .current_worker_diagnostics + .iter() + .any(|diagnostic| diagnostic.code == "subject_body_observation_unavailable") + ); + let body = subject_body::snapshot(&api, &subject.id).unwrap(); + assert_eq!(body.current, projection.current_body); + assert!(matches!( + body.observation, + server_api::SubjectBodyObservation::Unavailable { .. } + )); +} + #[tokio::test(flavor = "current_thread")] async fn subject_callbacks_finish_before_waiting_host_parents_release_capacity() { let workspace = tempfile::tempdir().unwrap(); diff --git a/crates/workspace-server/src/server/tests/subjektiv_journal_tests.rs b/crates/workspace-server/src/server/tests/subjektiv_journal_tests.rs index a746f39a7..56327c331 100644 --- a/crates/workspace-server/src/server/tests/subjektiv_journal_tests.rs +++ b/crates/workspace-server/src/server/tests/subjektiv_journal_tests.rs @@ -858,6 +858,7 @@ impl RuntimeFixture { .stop_worker( &self.source, WorkerLifecycleRequest { + stop_intent: None, reason: Some("settle original source before permanent erase".into()), ticket_assignment: None, }, @@ -1488,6 +1489,7 @@ async fn stopped_source_pin_releases_after_saved_completion_and_removal() { .stop_worker( &f.source, WorkerLifecycleRequest { + stop_intent: None, reason: Some("archive original".into()), ticket_assignment: None, }, @@ -1763,6 +1765,7 @@ async fn stopped_partial_journal_recovery_fences_old_attempt_and_resumes_same_im .stop_worker( worker, WorkerLifecycleRequest { + stop_intent: None, reason: Some("definitive stop before checkpoint".into()), ticket_assignment: None, }, @@ -2130,6 +2133,7 @@ impl SubjectRemovalFixture { .stop_worker( &target, WorkerLifecycleRequest { + stop_intent: None, reason: Some("test stopped Subject removal".into()), ticket_assignment: None, }, diff --git a/crates/workspace-server/src/server/tests/ticket_worker_launch_tests.rs b/crates/workspace-server/src/server/tests/ticket_worker_launch_tests.rs index 802589f2a..21275c2f8 100644 --- a/crates/workspace-server/src/server/tests/ticket_worker_launch_tests.rs +++ b/crates/workspace-server/src/server/tests/ticket_worker_launch_tests.rs @@ -1822,6 +1822,7 @@ async fn mr_open_retains_source_before_first_review_and_observed_result() { .stop_worker( &coder, WorkerLifecycleRequest { + stop_intent: None, reason: None, ticket_assignment: None, }, diff --git a/crates/workspace-server/src/server/tests/workdir_session_close_tests.rs b/crates/workspace-server/src/server/tests/workdir_session_close_tests.rs index 5bb808785..76d1a0fac 100644 --- a/crates/workspace-server/src/server/tests/workdir_session_close_tests.rs +++ b/crates/workspace-server/src/server/tests/workdir_session_close_tests.rs @@ -652,6 +652,140 @@ async fn attachment_close_cancellation_restores_remote_session_for_retry() { cancellation_restoration(ClosePath::Attachment).await; } +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn observed_stop_close_failure_retains_old_command_without_replacing_new_attachment() { + let fixture = Fixture::new().await; + let worker = fixture.registry_owner(); + let old = fixture.open().await; + let new = fixture.open().await; + let handle = fixture + .api() + .workdir_sessions + .lock() + .unwrap() + .register_command( + worker.clone(), + "checkout".into(), + old.clone(), + CommandHandle("observed-provider-command".into()), + ); + let observed = fixture + .api() + .workdir_sessions + .lock() + .unwrap() + .observe_worker(&worker); + fixture + .api() + .workdir_sessions + .lock() + .unwrap() + .insert_attachment(worker.clone(), new.clone()); + *fixture.transport.fault.lock().unwrap() = Some(CloseFault::Reject); + let _guard = current_worker_session_lock(fixture.api(), &worker) + .lock_owned() + .await; + assert!( + close_observed_worker_workdir_sessions(&fixture.api().workdir_sessions, &observed) + .await + .is_err() + ); + { + let registry = fixture.api().workdir_sessions.lock().unwrap(); + assert!(Arc::ptr_eq( + registry.attachments.get(&worker).unwrap(), + &new + )); + assert!(Arc::ptr_eq( + ®istry + .command(&worker, "checkout", &handle) + .unwrap() + .source, + &old + )); + } + fixture.assert_live(&old).await; + fixture.assert_live(&new).await; + close_observed_worker_workdir_sessions(&fixture.api().workdir_sessions, &observed) + .await + .unwrap(); + assert_eq!(fixture.transport.attempts().len(), 2); + assert_eq!( + fixture.transport.attempts()[0], + fixture.transport.attempts()[1] + ); + fixture.assert_live(&new).await; + assert!( + fixture + .api() + .workdir_sessions + .lock() + .unwrap() + .command(&worker, "checkout", &handle) + .is_none() + ); + close_worker_workdir_sessions(&fixture.api().workdir_sessions, &worker) + .await + .unwrap(); +} + +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn stop_projection_does_not_follow_a_rebound_connection_with_identical_worker_alias_and_workdir() + { + let fixture = Fixture::new().await; + let api = fixture.api(); + let worker = fixture.registry_owner(); + let link = WorkerWorkdirLinkRecord { + connection_id: String::new(), + workspace_id: TEST_WORKSPACE_ID.into(), + worker: worker.clone(), + workdir_id: fixture.workdir_id.clone(), + alias: "checkout".into(), + capabilities: workdir::WorkdirSessionCapabilities::READ_ONLY, + linked_at: TEST_CREATED_AT.into(), + unlinked_at: None, + }; + let _guard = current_worker_session_lock(api, &worker).lock_owned().await; + let old = api.store.attach_worker_workdir(&link).unwrap(); + api.store + .detach_worker_workdir_connection( + TEST_WORKSPACE_ID, + &worker, + &old.alias, + &old.connection_id, + TEST_CREATED_AT, + ) + .unwrap(); + let new = api.store.attach_worker_workdir(&link).unwrap(); + assert_ne!(old.connection_id, new.connection_id); + let mut record = api + .store + .get_workdir_registry(TEST_WORKSPACE_ID, &fixture.workdir_id) + .unwrap() + .unwrap(); + record.display_name = "new binding not owned by old stop".into(); + api.store.upsert_workdir_registry(&record).unwrap(); + sync_linked_workdir_after_worker_stop(api, &worker, &[old]).unwrap(); + assert_eq!( + api.store + .get_workdir_registry(TEST_WORKSPACE_ID, &fixture.workdir_id) + .unwrap() + .unwrap() + .display_name, + record.display_name + ); + sync_linked_workdir_after_worker_stop(api, &worker, &[new]).unwrap(); + assert_ne!( + api.store + .get_workdir_registry(TEST_WORKSPACE_ID, &fixture.workdir_id) + .unwrap() + .unwrap() + .display_name, + record.display_name, + "the same current connection still projects normally" + ); +} + #[tokio::test(flavor = "multi_thread", worker_threads = 2)] async fn worker_removal_close_failure_retains_attachment_and_retry_releases_runtime_lease() { let fixture = Fixture::new().await; @@ -680,6 +814,7 @@ async fn worker_removal_close_failure_retains_attachment_and_retry_releases_runt .stop_worker( &worker, WorkerLifecycleRequest { + stop_intent: None, reason: Some("prepare removal".into()), ticket_assignment: None, }, diff --git a/crates/workspace-server/src/server/tests/worker_drive_tests.rs b/crates/workspace-server/src/server/tests/worker_drive_tests.rs index c0001b321..05c9c8f93 100644 --- a/crates/workspace-server/src/server/tests/worker_drive_tests.rs +++ b/crates/workspace-server/src/server/tests/worker_drive_tests.rs @@ -714,6 +714,7 @@ async fn worker_removal_preserves_drive_reference_for_explicitly_granted_replace .stop( &WorkerOperationContext::Backend, WorkerLifecycleRequest { + stop_intent: None, reason: Some("Drive lifetime test".into()), ticket_assignment: None, }, @@ -816,6 +817,7 @@ async fn worker_removal_preserves_drive_reference_for_explicitly_granted_replace .stop_worker( target, WorkerLifecycleRequest { + stop_intent: None, reason: None, ticket_assignment: None, }, diff --git a/crates/workspace-server/src/server/tests/worker_operations_tests.rs b/crates/workspace-server/src/server/tests/worker_operations_tests.rs index 901c22dc3..9b665ac95 100644 --- a/crates/workspace-server/src/server/tests/worker_operations_tests.rs +++ b/crates/workspace-server/src/server/tests/worker_operations_tests.rs @@ -12,6 +12,7 @@ fn operation_input(kind: WorkerInputKind) -> WorkerInputRequest { fn operation_lifecycle() -> WorkerLifecycleRequest { WorkerLifecycleRequest { + stop_intent: None, reason: Some("operation-boundary-test".into()), ticket_assignment: None, } @@ -718,6 +719,7 @@ async fn generic_management_aliases_cannot_bypass_worker_grants_or_target_bound_ .is_err() ); let lifecycle = || server_api::RuntimeWorkerLifecycleRequest { + stop_intent: None, reason: None, ticket_assignment: None, }; diff --git a/crates/workspace-server/src/server/tests/workspace_protocol_backpressure_tests.rs b/crates/workspace-server/src/server/tests/workspace_protocol_backpressure_tests.rs index add77016e..637ba73d1 100644 --- a/crates/workspace-server/src/server/tests/workspace_protocol_backpressure_tests.rs +++ b/crates/workspace-server/src/server/tests/workspace_protocol_backpressure_tests.rs @@ -124,6 +124,7 @@ async fn actual_workspace_multiplexer_stalled_writer_timeout_or_disconnect_recla .stop( &WorkerOperationContext::Backend, WorkerLifecycleRequest { + stop_intent: None, reason: Some("cleanup-after-multiplexer-exit".into()), ticket_assignment: None, }, diff --git a/crates/workspace-server/src/server/worker_operations.rs b/crates/workspace-server/src/server/worker_operations.rs index f6735e689..c838dd70f 100644 --- a/crates/workspace-server/src/server/worker_operations.rs +++ b/crates/workspace-server/src/server/worker_operations.rs @@ -48,6 +48,42 @@ pub(crate) enum WorkerOperationContext { } impl WorkerOperationContext { + fn bind_stop_intent( + &self, + workspace: &str, + request: &mut WorkerLifecycleRequest, + ) -> ApiResult<()> { + let Some(intent) = request.stop_intent.as_mut() else { + return Ok(()); + }; + if intent.request_id.trim().is_empty() + || intent.request_id.len() > 512 + || intent.request_id.chars().any(char::is_control) + { + return Err(Error::InvalidInput( + "Stop request_id must be nonempty bounded text".into(), + ) + .into()); + } + let source = match self { + Self::Browser { source, .. } => source.clone(), + Self::WorkerControl { controller } => protocol::AuthenticatedInputSource::Worker { + runtime_id: controller.runtime_id.clone(), + worker_id: controller.worker_id.clone(), + }, + #[cfg(test)] + Self::Backend => return Ok(()), + }; + let canonical = serde_json::to_vec(&(workspace, source.namespace(), &intent.request_id)) + .expect("stop identity contains only strings"); + let digest = Sha256::digest(canonical) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect::(); + intent.request_id = format!("runtime-stop:{digest}"); + Ok(()) + } + fn bind_input(&self, request: &mut WorkerInputRequest) { // Public adapters cannot choose provenance or consume another input path's // receipt IDs. Bind these only after the current caller is authorized. @@ -443,42 +479,179 @@ impl WorkspaceWorker { self.api.restore_workspace_worker(&self.identity, request) } + pub(super) async fn restore_current_subject( + &self, + context: &WorkerOperationContext, + request: server_api::WorkerRestoreRequest, + subject: String, + expected: server_api::SubjectBodyRef, + ) -> ApiResult { + let guard = self.authorize_operation(context, "restore").await?; + let worker = self.clone(); + self.api + .host_executors + .host()? + .run(move || { + let _control = guard; + worker.current_record()?; + worker.api.restore_workspace_worker_with_subject( + &worker.identity, + request, + Some((&subject, &expected)), + ) + }) + .await + } + + pub(super) async fn stop_outcome_current_subject( + &self, + context: &WorkerOperationContext, + mut request: WorkerLifecycleRequest, + subject: String, + expected: server_api::SubjectBodyRef, + ) -> ApiResult { + let control = self.authorize_operation(context, "stop").await?; + context.bind_stop_intent(self.api.workspace_id(), &mut request)?; + let worker = self.clone(); + self.api + .host_executors + .host()? + .run(move || { + let _control = control; + { + let gate = subjektiv_journal::lifecycle_lock(&worker.api, &subject)?; + let _subject = gate + .lock() + .map_err(|_| Error::Store("Subject lifecycle lock poisoned".into()))?; + subject_body::require_expected(&worker.api, &subject, Some(&expected))?; + } + // Read-only receipt lookup may wait for an ongoing cleanup callback; + // never retain the Subject gate while waiting for the Worker lock. + worker + .api + .runtime + .worker_stop_outcome(&worker.identity, request) + .map_err(|error| error.into_error().into()) + }) + .await + } + + pub(super) async fn stop_current_subject( + &self, + context: &WorkerOperationContext, + request: WorkerLifecycleRequest, + subject: String, + expected: server_api::SubjectBodyRef, + ) -> ApiResult { + self.stop_impl(context, request, Some((subject, expected))) + .await + } + pub(super) async fn stop( &self, context: &WorkerOperationContext, request: WorkerLifecycleRequest, + ) -> ApiResult { + self.stop_impl(context, request, None).await + } + + async fn stop_impl( + &self, + context: &WorkerOperationContext, + mut request: WorkerLifecycleRequest, + expected_subject: Option<(String, server_api::SubjectBodyRef)>, ) -> ApiResult { let guard = self.authorize_operation(context, "stop").await?; + context.bind_stop_intent(self.api.workspace_id(), &mut request)?; let worker = self.clone(); // Once admitted, Runtime stop, session close and linked Workdir // publication have one owner. Never hand an accepted stop's guards // back to its cancellable waiter or acquire this Host lane again. run_host_workflow(self.api.host_executors.host()?, async move { let _control = guard; + let mut expected_subject = expected_subject; + if expected_subject.is_none() { + if let Some(lease) = worker + .api + .store + .retained_worker_singleton_lease(worker.api.workspace_id(), &worker.identity)? + { + if let Some(subject) = lease.key.strip_prefix(SUBJEKTIV_SINGLETON_PREFIX) { + expected_subject = + Some((subject.to_owned(), subject_body::reference(&lease))); + } + } + } + if expected_subject.is_some() && request.stop_intent.is_none() { + return Err(Error::InvalidInput( + "Subject body stop requires request identity and its observed Runtime token" + .into(), + ) + .into()); + } + // Observe concrete resource lifetimes under the existing session + // fence, then release it before Runtime/cleanup can call back here. + let (observed_sessions, observed_links) = { + let _session = current_worker_session_lock(&worker.api, &worker.identity) + .lock_owned() + .await; + let links = worker + .api + .store + .list_worker_workdir_links(worker.api.workspace_id(), &worker.identity)?; + let sessions = worker + .api + .workdir_sessions + .lock() + .map_err(|_| Error::Store("Workdir session registry was poisoned".into()))? + .observe_worker(&worker.identity); + (sessions, links) + }; + if let Some((subject, expected)) = expected_subject.as_ref() { + let gate = subjektiv_journal::lifecycle_lock(&worker.api, subject)?; + let _subject = gate + .lock() + .map_err(|_| Error::Store("Subject lifecycle lock poisoned".into()))?; + subject_body::require_expected(&worker.api, subject, Some(expected))?; + worker + .api + .runtime + .coordinate_worker_stop(&worker.identity, request.clone()) + .map_err(RuntimeRegistryError::into_error)?; + } + // Admission owns the exact pending stop before releasing the Subject + // gate. Execution may call back into Session/Journal/Subject services. let result = worker .api .runtime .stop_worker(&worker.identity, request) - .map_err(|error| error.into_error())?; - if result.state != InternalWorkerOperationState::Accepted { + .map_err(RuntimeRegistryError::into_error)?; + if let Some(conflict) = result.diagnostics.iter().find(|diagnostic| { + matches!( + diagnostic.code.as_str(), + "stop_observation_conflict" | "stop_request_conflict" + ) + }) { + return Err(Error::RepositoryConflict(conflict.message.clone()).into()); + } + if result.state != InternalWorkerOperationState::Accepted || result.replayed { return Ok(result); } parse_runtime_worker_id_for_registry(&worker.identity.worker_id)?; let _session = current_worker_session_lock(&worker.api, &worker.identity) .lock_owned() .await; - close_current_worker_session_locked(&worker.api, &worker.identity).await?; - if let Some(record) = worker - .api - .store - .get_worker_registry(worker.api.workspace_id(), &worker.identity)? - { - sync_linked_workdir_after_worker_stop( - &worker.api, - &worker.identity.runtime_id, - &record, - )?; - } + close_observed_worker_workdir_sessions( + &worker.api.workdir_sessions, + &observed_sessions, + ) + .await + .map_err(|message| Error::RuntimeOperationFailed { + runtime_id: worker.identity.runtime_id.clone(), + code: "workdir_session_close_failed".into(), + message, + })?; + sync_linked_workdir_after_worker_stop(&worker.api, &worker.identity, &observed_links)?; Ok(result) }) .await @@ -489,6 +662,9 @@ impl WorkspaceWorker { context: &WorkerOperationContext, request: WorkerLifecycleRequest, ) -> ApiResult { + if request.stop_intent.is_some() { + return Err(Error::InvalidInput("stop_intent is not valid for Cancel".into()).into()); + } let guard = self.authorize_operation(context, "cancel").await?; let worker = self.clone(); self.api @@ -890,6 +1066,58 @@ mod input_binding_tests { } } + #[test] + fn stop_identity_is_bound_to_authenticated_account_and_workspace_without_replacing_the_observation() + { + let context = |account: &str| WorkerOperationContext::Browser { + headers: HeaderMap::new(), + source: protocol::AuthenticatedInputSource::Account { + account_id: account.into(), + }, + }; + let request = || WorkerLifecycleRequest { + stop_intent: Some(runtime_api::WorkerStopIntent { + request_id: "same-request".into(), + expected_observation_token: "observed-lifecycle".into(), + }), + ..Default::default() + }; + let mut first = request(); + context("account-a") + .bind_stop_intent("workspace-a", &mut first) + .unwrap(); + let mut retry = request(); + context("account-a") + .bind_stop_intent("workspace-a", &mut retry) + .unwrap(); + assert_eq!(first, retry); + assert_eq!( + first + .stop_intent + .as_ref() + .unwrap() + .expected_observation_token, + "observed-lifecycle" + ); + for (account, workspace) in [("account-b", "workspace-a"), ("account-a", "workspace-b")] { + let mut other = request(); + context(account) + .bind_stop_intent(workspace, &mut other) + .unwrap(); + assert_ne!( + first.stop_intent.as_ref().unwrap().request_id, + other.stop_intent.as_ref().unwrap().request_id + ); + } + let mut empty = request(); + empty.stop_intent.as_mut().unwrap().request_id.clear(); + assert!( + context("account-a") + .bind_stop_intent("workspace-a", &mut empty) + .is_err() + ); + } + #[test] fn generic_input_binds_caller_source_and_isolates_client_ids_from_chat_and_other_paths() { let browser = |id: &str| WorkerOperationContext::Browser { diff --git a/crates/workspace-server/src/server_assignment_attention_tests.rs b/crates/workspace-server/src/server_assignment_attention_tests.rs index da5b97abc..2947a92d4 100644 --- a/crates/workspace-server/src/server_assignment_attention_tests.rs +++ b/crates/workspace-server/src/server_assignment_attention_tests.rs @@ -376,6 +376,7 @@ async fn orchestrator_attention_input_rechecks_reserved_noop_before_and_after_co .stop_worker( &worker, WorkerLifecycleRequest { + stop_intent: None, reason: None, ticket_assignment: None, }, @@ -479,6 +480,7 @@ async fn orchestrator_attention_restore_keeps_pending_unknown_delivery_and_id() .stop_worker( &worker, WorkerLifecycleRequest { + stop_intent: None, reason: None, ticket_assignment: None, }, @@ -1023,6 +1025,7 @@ async fn real_attention_worker_start_restore(lose_response: bool) { .stop_worker( &worker, WorkerLifecycleRequest { + stop_intent: None, reason: None, ticket_assignment: None, }, @@ -1131,6 +1134,7 @@ async fn real_attention_worker_start_restore(lose_response: bool) { .stop_worker( &worker, WorkerLifecycleRequest { + stop_intent: None, reason: None, ticket_assignment: None, }, @@ -1151,6 +1155,7 @@ async fn orchestrator_attention_restore_and_replacement_do_not_resend_acknowledg .stop_worker( &worker, WorkerLifecycleRequest { + stop_intent: None, reason: None, ticket_assignment: None, }, diff --git a/crates/workspace-server/src/server_assignment_delivery_tests.rs b/crates/workspace-server/src/server_assignment_delivery_tests.rs index b8d7e2777..32d457350 100644 --- a/crates/workspace-server/src/server_assignment_delivery_tests.rs +++ b/crates/workspace-server/src/server_assignment_delivery_tests.rs @@ -138,6 +138,7 @@ fn delivery_stop(api: &WorkspaceApi, worker: &RuntimeWorkerRef) { .stop_worker( worker, WorkerLifecycleRequest { + stop_intent: None, reason: None, ticket_assignment: None, }, diff --git a/crates/workspace-server/src/server_plugin_controller_tests.rs b/crates/workspace-server/src/server_plugin_controller_tests.rs index 3812ed97c..834ec5994 100644 --- a/crates/workspace-server/src/server_plugin_controller_tests.rs +++ b/crates/workspace-server/src/server_plugin_controller_tests.rs @@ -443,6 +443,7 @@ mod plugin_controller_tests { api.runtime.stop_worker( &worker, WorkerLifecycleRequest { + stop_intent: None, reason: None, ticket_assignment: None, }, diff --git a/crates/workspace-server/src/server_plugin_remote_controller_tests.rs b/crates/workspace-server/src/server_plugin_remote_controller_tests.rs index 09768a865..6729c67cf 100644 --- a/crates/workspace-server/src/server_plugin_remote_controller_tests.rs +++ b/crates/workspace-server/src/server_plugin_remote_controller_tests.rs @@ -336,6 +336,7 @@ mod plugin_remote_controller_tests { api.runtime.stop_worker( &stopped, WorkerLifecycleRequest { + stop_intent: None, reason: None, ticket_assignment: None, }, diff --git a/crates/workspace-server/src/server_residual_execution_tests.rs b/crates/workspace-server/src/server_residual_execution_tests.rs index 759dbf040..ff81318b2 100644 --- a/crates/workspace-server/src/server_residual_execution_tests.rs +++ b/crates/workspace-server/src/server_residual_execution_tests.rs @@ -658,6 +658,7 @@ async fn worker_control_source_lookup_waits_off_tokio_and_retains_finite_host_ow let leases = Arc::new(std::sync::atomic::AtomicUsize::new(0)); let request = || { Json(server_api::RuntimeWorkerLifecycleRequest { + stop_intent: None, reason: None, ticket_assignment: None, }) diff --git a/crates/workspace-server/src/store.rs b/crates/workspace-server/src/store.rs index ae6df14a0..227fffbfa 100644 --- a/crates/workspace-server/src/store.rs +++ b/crates/workspace-server/src/store.rs @@ -3668,6 +3668,20 @@ impl SqliteWorkspaceStore { }) } + pub(crate) fn reserved_worker_create_runtime( + &self, + workspace_id: &str, + allocation_key: &str, + ) -> Result> { + self.with_conn(|conn| { + Ok(conn.query_row( + "SELECT runtime_id FROM worker_create_reservations WHERE workspace_id = ?1 AND allocation_key = ?2", + params![workspace_id, allocation_key], + |row| row.get(0), + ).optional()?) + }) + } + pub(crate) fn current_worker_singleton_owner( &self, workspace_id: &str, diff --git a/web/workspace/deno.json b/web/workspace/deno.json index 26070d9ba..bdbe832f5 100644 --- a/web/workspace/deno.json +++ b/web/workspace/deno.json @@ -6,7 +6,7 @@ "dev": "deno run -A npm:vite@8.3.2 dev", "dev:backend": "cd ../.. && cargo run -p yoi-workspace-server --bin yoi-server -- serve --listen 127.0.0.1:8787", "check": "deno run -A npm:@sveltejs/kit@3.0.0 sync && deno run -A npm:svelte-check@4.7.6 --tsconfig ./tsconfig.json", - "test": "deno test --allow-read=src,test,tests,../../docs,../../resources/config-schema --allow-env=LOG,VSCODE_TEXTMATE_DEBUG,NODE_ENV tests/workspace-model.test.ts tests/repository-api.test.ts tests/workspace-catalog.test.ts tests/profile-api.test.ts tests/skill-api.test.ts tests/tooltip-contract.test.ts tests/bevel-contract.test.ts tests/bevel-line-contract.test.ts tests/typography-contract.test.ts src/lib/workspace/navigation/deferred.test.ts src/lib/workspace/auth/model.test.ts src/lib/workspace/auth/login.test.ts src/lib/workspace/auth/routes.test.ts src/lib/workspace/auth/hosting.test.ts src/lib/workspace/auth/hosting-enrollment.test.ts src/lib/workspace/auth/activation.test.ts src/lib/workspace/auth/hosting-accounts.test.ts src/lib/workspace/protocol-validator.test.ts tests/auth-api.test.ts src/lib/workspace/api/http.test.ts src/lib/workspace/api/workers.test.ts src/lib/workspace/api/ticket-browser.test.ts src/lib/workspace/api/ticket-case.test.ts src/lib/workspace/api/ticket-operations.test.ts src/lib/workspace/header/breadcrumb-model.test.ts src/lib/workspace/console/chat-submit.test.ts src/lib/workspace/console/composer-delivery.test.ts test/composer-history.test.ts tests/composer-paste.test.ts src/lib/workspace/console/composer-command.test.ts src/lib/workspace/console/composer-draft.test.ts src/lib/workspace/console/composer-completion.test.ts src/lib/workspace/console/composer-invocation.test.ts src/lib/workspace/console/file-completions.test.ts test/console/ansi.test.ts src/lib/workspace/console/history.test.ts src/lib/workspace/console/model.test.ts src/lib/workspace/console/alert-order.test.ts src/lib/workspace/console/turn-navigation.test.ts src/lib/workspace/console/run-status.test.ts src/lib/workspace/companion/api.test.ts tests/workdir-api.test.ts src/lib/workspace/settings/workdir-checkout.test.ts tests/workdir-removal.test.ts tests/workdir-permissions.test.ts src/lib/workspace/console/tasks.test.ts test/ticket-detail-route-reuse.test.ts test/spa-shell.test.ts test/objective-pages.test.ts test/repositories/ui.test.ts src/lib/workspace/console/worker-console.ui.test.ts src/lib/workspace/settings/model.test.ts tests/runtime-connection.test.ts tests/runtime-management.test.ts tests/runtime-management-source.test.ts src/lib/workspace/sidebar/override-stack.test.ts src/lib/workspace/sidebar/route-ownership.test.ts src/lib/workspace/sidebar/workers.test.ts src/lib/workspace/sidebar/workspace-switcher.test.ts src/lib/workspace/sidebar/worker-subscription.test.ts src/lib/workspace/sidebar/worker-workdir-meta.test.ts src/lib/workspace/sidebar/worker-launch.test.ts src/lib/workspace/workers/worker-launch.test.ts tests/runtime-workers.test.ts test/sidebar/worker-actions.test.ts src/lib/workspace/tickets/merge-request-resources.test.ts src/lib/workspace/tickets/ticket-panel.test.ts src/lib/workspace/tickets/ticket-search.test.ts test/merge-request-status.test.ts test/config-source/decodal-grammar.test.ts test/config-source/editor-state.test.ts test/config-source/fixed-schema-wrapper.test.ts test/config-source/toolchain.test.ts test/config-source/wasm-parity.test.ts test/config-source/api.test.ts tests/memory-api.test.ts src/lib/workspace/memory/permissions.test.ts tests/subject-delete.test.ts tests/journal-api.test.ts test/repository-access/api.test.ts test/repository-access/loader.test.ts test/repository-access/model.test.ts test/repository-access/structure.test.ts test/repository-ssh-connection-ui.test.ts src/lib/workspace/drive/api.test.ts src/lib/workspace/drive/controller.test.ts src/lib/workspace/drive-ui/policy.test.ts", + "test": "deno test --allow-read=src,test,tests,../../docs,../../resources/config-schema --allow-env=LOG,VSCODE_TEXTMATE_DEBUG,NODE_ENV tests/workspace-model.test.ts tests/repository-api.test.ts tests/workspace-catalog.test.ts tests/profile-api.test.ts tests/skill-api.test.ts tests/tooltip-contract.test.ts tests/bevel-contract.test.ts tests/bevel-line-contract.test.ts tests/typography-contract.test.ts src/lib/workspace/navigation/deferred.test.ts src/lib/workspace/auth/model.test.ts src/lib/workspace/auth/login.test.ts src/lib/workspace/auth/routes.test.ts src/lib/workspace/auth/hosting.test.ts src/lib/workspace/auth/hosting-enrollment.test.ts src/lib/workspace/auth/activation.test.ts src/lib/workspace/auth/hosting-accounts.test.ts src/lib/workspace/protocol-validator.test.ts tests/auth-api.test.ts src/lib/workspace/api/http.test.ts src/lib/workspace/api/workers.test.ts src/lib/workspace/api/ticket-browser.test.ts src/lib/workspace/api/ticket-case.test.ts src/lib/workspace/api/ticket-operations.test.ts src/lib/workspace/header/breadcrumb-model.test.ts src/lib/workspace/console/chat-submit.test.ts src/lib/workspace/console/composer-delivery.test.ts test/composer-history.test.ts tests/composer-paste.test.ts src/lib/workspace/console/composer-command.test.ts src/lib/workspace/console/composer-draft.test.ts src/lib/workspace/console/composer-completion.test.ts src/lib/workspace/console/composer-invocation.test.ts src/lib/workspace/console/file-completions.test.ts test/console/ansi.test.ts src/lib/workspace/console/history.test.ts src/lib/workspace/console/model.test.ts src/lib/workspace/console/alert-order.test.ts src/lib/workspace/console/turn-navigation.test.ts src/lib/workspace/console/run-status.test.ts src/lib/workspace/companion/api.test.ts tests/workdir-api.test.ts src/lib/workspace/settings/workdir-checkout.test.ts tests/workdir-removal.test.ts tests/workdir-permissions.test.ts src/lib/workspace/console/tasks.test.ts test/ticket-detail-route-reuse.test.ts test/spa-shell.test.ts test/objective-pages.test.ts test/repositories/ui.test.ts src/lib/workspace/console/worker-console.ui.test.ts src/lib/workspace/settings/model.test.ts tests/runtime-connection.test.ts tests/runtime-management.test.ts tests/runtime-management-source.test.ts src/lib/workspace/sidebar/override-stack.test.ts src/lib/workspace/sidebar/route-ownership.test.ts src/lib/workspace/sidebar/workers.test.ts src/lib/workspace/sidebar/workspace-switcher.test.ts src/lib/workspace/sidebar/worker-subscription.test.ts src/lib/workspace/sidebar/worker-workdir-meta.test.ts src/lib/workspace/sidebar/worker-launch.test.ts src/lib/workspace/workers/worker-launch.test.ts tests/runtime-workers.test.ts test/sidebar/worker-actions.test.ts src/lib/workspace/tickets/merge-request-resources.test.ts src/lib/workspace/tickets/ticket-panel.test.ts src/lib/workspace/tickets/ticket-search.test.ts test/merge-request-status.test.ts test/config-source/decodal-grammar.test.ts test/config-source/editor-state.test.ts test/config-source/fixed-schema-wrapper.test.ts test/config-source/toolchain.test.ts test/config-source/wasm-parity.test.ts test/config-source/api.test.ts tests/memory-api.test.ts src/lib/workspace/memory/permissions.test.ts src/lib/workspace/memory/subject-body.test.ts tests/subject-delete.test.ts tests/journal-api.test.ts test/repository-access/api.test.ts test/repository-access/loader.test.ts test/repository-access/model.test.ts test/repository-access/structure.test.ts test/repository-ssh-connection-ui.test.ts src/lib/workspace/drive/api.test.ts src/lib/workspace/drive/controller.test.ts src/lib/workspace/drive-ui/policy.test.ts", "test:journal:backend-contract": "cd ../.. && mkdir -p target && cargo test -p yoi-workspace-server --lib journal_browser_web_contract -- --nocapture > target/journal-browser-contract.log && cd web/workspace && deno run --allow-read=../../target/journal-browser-contract.log test/journal-backend-contract.ts ../../target/journal-browser-contract.log", "test:component": "deno run -A npm:vitest@5.0.3 run --config vitest.config.ts", "test:browser:activation": "mkdir -p ../../target/web-ux/tmp && TMPDIR=../../target/web-ux/tmp deno test --config ../../tools/web-ux/deno.json --allow-env --allow-net --allow-read --allow-write --allow-run --allow-sys test/activation-browser/activation-shell.browser.test.ts", diff --git a/web/workspace/src/lib/workspace/api/runtime-workers.ts b/web/workspace/src/lib/workspace/api/runtime-workers.ts index fcd88470c..49c6861a1 100644 --- a/web/workspace/src/lib/workspace/api/runtime-workers.ts +++ b/web/workspace/src/lib/workspace/api/runtime-workers.ts @@ -319,8 +319,8 @@ export function parseRuntimeWorkerLifecycleResult( const item = object(value, "Runtime Worker lifecycle result"); exactKeys( item, - ["state", "runtime_id", "worker_id", "diagnostics"], - [], + ["state", "runtime_id", "worker_id", "diagnostics", "replayed"], + ["replayed"], "Runtime Worker lifecycle result", ); const state = boundedString( @@ -331,6 +331,9 @@ export function parseRuntimeWorkerLifecycleResult( fail("Runtime Worker lifecycle result.state", "has an unknown value"); } return { + replayed: item.replayed === undefined + ? false + : boolean(item.replayed, "Runtime Worker lifecycle result.replayed"), state: state as WorkerOperationState, runtime_id: boundedString( item.runtime_id, diff --git a/web/workspace/src/lib/workspace/memory/SubjectBody.browser.spec.ts b/web/workspace/src/lib/workspace/memory/SubjectBody.browser.spec.ts new file mode 100644 index 000000000..7bcb427d5 --- /dev/null +++ b/web/workspace/src/lib/workspace/memory/SubjectBody.browser.spec.ts @@ -0,0 +1,1073 @@ +// @vitest-environment happy-dom +import { + cleanup, + fireEvent, + render, + screen, + waitFor, +} from "@testing-library/svelte"; +import { afterEach, expect, test, vi } from "vitest"; +import { tick } from "svelte"; +import SubjectBody from "./SubjectBody.svelte"; +import { + readSubjectBodyAttempt, + subjectBodyStorageKey, +} from "./subject-body.ts"; + +// component・client/parser・sessionStorageは本番実装を使い、HTTP境界だけを置き換える。 +const props = { + accountId: "account-a", + workspaceId: "workspace", + subjectId: "subject", +}; +const owner = { runtime_id: "runtime", worker_id: "worker", generation: "17" }; +function absent(subjectId = props.subjectId) { + return { + workspace_id: props.workspaceId, + subject_id: subjectId, + current: null, + observation: { kind: "absent" }, + can_start: true, + can_stop: true, + }; +} +function available(state = "stopped", observationToken = "observation-1") { + return { + ...absent(), + current: owner, + observation: { + kind: "available", + worker: { + runtime_id: owner.runtime_id, + worker_id: owner.worker_id, + resource_key: "W-1", + host_id: "host", + label: "Body", + workspace: { + visibility: "workspace", + identity: "workspace", + workspace_id: "workspace", + }, + implementation: { kind: "embedded", display_hint: "" }, + availability: "observed", + state, + worker_state: state === "stopped" + ? null + : { last_command_id: 1, state: { kind: "idle" } }, + restore_observation_token: observationToken, + }, + }, + }; +} +function savedStop(accountId = props.accountId) { + return { + accountId, + workspaceId: props.workspaceId, + subjectId: props.subjectId, + action: "stop", + request: { + request_id: `saved-stop-${accountId}`, + expected_current: owner, + expected_observation_token: "saved-observation", + }, + }; +} + +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise((accept) => { + resolve = accept; + }); + return { promise, resolve }; +} +afterEach(() => { + cleanup(); + vi.restoreAllMocks(); + vi.unstubAllGlobals(); + sessionStorage.clear(); +}); + +test("unavailable owner is not absence and cannot invite a replacement start", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => + Response.json({ + ...absent(), + current: owner, + observation: { + kind: "unavailable", + diagnostics: [{ + code: "runtime_unavailable", + severity: "warning", + message: "Runtime cannot be observed", + }], + }, + }) + ), + ); + render(SubjectBody, props); + await screen.findByText("Body owner present; observation unavailable."); + expect(screen.getByText("Runtime cannot be observed")).toBeTruthy(); + expect( + screen.queryByRole("button", { + name: /^(Start|Restart|Stop) conversation body$/, + }), + ).toBeNull(); + expect(screen.queryByText("No current conversation body.")) + .toBeNull(); + expect(screen.queryByText(owner.worker_id)).toBeNull(); +}); + +test("unauthorized controls are hidden and permission diagnosis stays visible", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async () => + Response.json({ ...absent(), can_start: false, can_stop: false }) + ), + ); + render(SubjectBody, props); + await screen.findByText(/You do not have permission to start or stop/); + expect(screen.queryByRole("button", { name: "Start conversation body" })) + .toBeNull(); + expect(screen.queryByRole("button", { name: "Stop conversation body" })) + .toBeNull(); + expect(screen.getByRole("button", { name: "Refresh body state" })) + .toBeTruthy(); +}); + +test("unknown survives refresh and remount and retries the complete original request", async () => { + const posts: string[] = []; + let observation: unknown = absent(); + vi.stubGlobal( + "fetch", + vi.fn(async (_path: string, init?: RequestInit) => { + if (init?.method !== "POST") return Response.json(observation); + posts.push(init.body as string); + const request = JSON.parse(init.body as string); + return Response.json({ + request_id: request.request_id, + disposition: "unknown", + diagnostics: [], + body: observation, + }); + }), + ); + const first = render(SubjectBody, props); + await fireEvent.click( + await screen.findByRole("button", { name: "Start conversation body" }), + ); + await screen.findByText("Request outcome unknown."); + await waitFor(() => + expect( + (screen.getByRole("button", { + name: "Retry same start request", + }) as HTMLButtonElement).disabled, + ).toBe(false) + ); + const saved = sessionStorage.getItem( + subjectBodyStorageKey(props.accountId, props.workspaceId, props.subjectId), + ); + expect(saved).not.toBeNull(); + expect(JSON.parse(posts[0])).toMatchObject({ + expected_current: null, + expected_observation_token: null, + }); + observation = available("stopped", "new-observation"); + await fireEvent.click( + screen.getByRole("button", { name: "Refresh body state" }), + ); + await screen.findByText("Body stopped."); + expect( + sessionStorage.getItem( + subjectBodyStorageKey( + props.accountId, + props.workspaceId, + props.subjectId, + ), + ), + ).toBe(saved); + expect(posts).toHaveLength(1); + first.unmount(); + render(SubjectBody, props); + await screen.findByText(/An unresolved start request is retained/); + await waitFor(() => + expect( + (screen.getByRole("button", { + name: "Retry same start request", + }) as HTMLButtonElement).disabled, + ).toBe(false) + ); + expect(screen.queryByRole("button", { name: "Restart conversation body" })) + .toBeNull(); + await fireEvent.click( + screen.getByRole("button", { name: "Retry same start request" }), + ); + await waitFor(() => expect(posts).toHaveLength(2)); + expect(posts[1]).toBe(posts[0]); + expect( + readSubjectBodyAttempt( + sessionStorage, + props.accountId, + props.workspaceId, + props.subjectId, + ) + ?.request.expected_current, + ).toBeNull(); +}); + +test("transport loss retains the sent input without automatic resubmission", async () => { + const posts: string[] = []; + vi.stubGlobal( + "fetch", + vi.fn(async (_path: string, init?: RequestInit) => { + if (init?.method !== "POST") return Response.json(available()); + posts.push(init.body as string); + throw new TypeError("connection lost"); + }), + ); + render(SubjectBody, props); + await fireEvent.click( + await screen.findByRole("button", { name: "Restart conversation body" }), + ); + await screen.findByText(/Request outcome unknown\. The request may/); + await waitFor(() => + expect( + (screen.getByRole("button", { + name: "Retry same start request", + }) as HTMLButtonElement).disabled, + ).toBe(false) + ); + expect(posts).toHaveLength(1); + expect( + JSON.stringify( + readSubjectBodyAttempt( + sessionStorage, + props.accountId, + props.workspaceId, + props.subjectId, + ) + ?.request, + ), + ).toBe(posts[0]); + const request = JSON.parse(posts[0]); + expect(request.expected_current).toEqual(owner); + expect(request.expected_observation_token).toBe("observation-1"); +}); + +test("accepted stop receipt does not display a stopped body before observation", async () => { + let stopped = false; + const posts: string[] = []; + vi.stubGlobal( + "fetch", + vi.fn(async (_path: string, init?: RequestInit) => { + if (init?.method !== "POST") { + return Response.json(available(stopped ? "stopped" : "idle")); + } + posts.push(init.body as string); + return Response.json({ + request_id: JSON.parse(init.body as string).request_id, + disposition: "accepted", + diagnostics: [], + body: null, + }); + }), + ); + render(SubjectBody, props); + await fireEvent.click( + await screen.findByRole("button", { name: "Stop conversation body" }), + ); + await screen.findByText("Request accepted."); + await waitFor(() => + expect( + (screen.getByRole("button", { + name: "Refresh body state", + }) as HTMLButtonElement).disabled, + ).toBe(false) + ); + expect(screen.queryByText("Body stopped.")).toBeNull(); + expect(screen.getByText("Live execution")).toBeTruthy(); + expect(JSON.parse(posts[0])).toMatchObject({ + expected_current: owner, + expected_observation_token: "observation-1", + }); + stopped = true; + await fireEvent.click( + screen.getByRole("button", { name: "Refresh body state" }), + ); + await screen.findByText("Body stopped."); + expect( + screen.getByText( + /Start restores this body; retained conversation and Memory/, + ), + ).toBeTruthy(); +}); + +test("structured configuration rejection is visible without inventing running state", async () => { + vi.stubGlobal( + "fetch", + vi.fn(async (_path: string, init?: RequestInit) => + Response.json( + init?.method !== "POST" ? absent() : { + request_id: JSON.parse(init.body as string).request_id, + disposition: "rejected", + diagnostics: [{ + code: "permission_missing", + severity: "error", + message: "Required permission session.record is missing", + }], + body: absent(), + }, + ) + ), + ); + render(SubjectBody, props); + await fireEvent.click( + await screen.findByRole("button", { name: "Start conversation body" }), + ); + await screen.findByText("Required permission session.record is missing"); + expect(screen.getByText("Request rejected.")).toBeTruthy(); + expect(screen.getByText("No current conversation body.")) + .toBeTruthy(); +}); + +test("storage write failure sends nothing and exposes a recovery diagnostic", async () => { + const fetch = vi.fn(async () => Response.json(absent())); + vi.stubGlobal("fetch", fetch); + render(SubjectBody, props); + const start = await screen.findByRole("button", { + name: "Start conversation body", + }); + vi.spyOn(Storage.prototype, "setItem").mockImplementation(() => { + throw new Error("storage denied"); + }); + await fireEvent.click(start); + await screen.findByText(/The exact request could not be retained/); + expect(fetch).toHaveBeenCalledTimes(1); +}); + +test("late GET for an old route cannot overwrite the selected Subject", async () => { + const old = deferred(); + vi.stubGlobal( + "fetch", + vi.fn((path: string) => + path.includes("/subjects/subject/body") ? old.promise : Promise.resolve( + Response.json({ + ...absent("other"), + can_start: false, + can_stop: false, + }), + ) + ), + ); + const view = render(SubjectBody, props); + await waitFor(() => + expect(screen.getByText("Loading body state…")).toBeTruthy() + ); + await view.rerender({ ...props, subjectId: "other" }); + await screen.findByText(/You do not have permission to start or stop/); + const response = Response.json(available()); + old.resolve(response); + await waitFor(() => expect(response.bodyUsed).toBe(true)); + await tick(); + await waitFor(() => + expect(screen.getByText("No current conversation body.")) + .toBeTruthy() + ); + expect(screen.queryByText("Body stopped.")).toBeNull(); + expect(screen.queryByRole("button", { name: "Restart conversation body" })) + .toBeNull(); +}); + +test("late old-route mutation receipt cannot update new route or discard old recovery input", async () => { + const pending = deferred(); + let requestId = ""; + vi.stubGlobal( + "fetch", + vi.fn(async (path: string, init?: RequestInit) => { + if (init?.method === "POST") { + requestId = JSON.parse(init.body as string).request_id; + return pending.promise; + } + return Response.json( + path.includes("/subjects/other/body") + ? { ...absent("other"), can_start: false, can_stop: false } + : absent(), + ); + }), + ); + const view = render(SubjectBody, props); + await fireEvent.click( + await screen.findByRole("button", { name: "Start conversation body" }), + ); + await screen.findByText("Sending request; body state is not yet confirmed."); + const saved = sessionStorage.getItem( + subjectBodyStorageKey(props.accountId, props.workspaceId, props.subjectId), + ); + await view.rerender({ ...props, subjectId: "other" }); + await screen.findByText(/You do not have permission to start or stop/); + // 同じrouteへ戻っても、以前のmount世代の応答を採用しない。 + await view.rerender(props); + await screen.findByText(/An unresolved start request is retained/); + const response = Response.json({ + request_id: requestId, + disposition: "accepted", + diagnostics: [], + body: available(), + }); + pending.resolve(response); + await waitFor(() => expect(response.bodyUsed).toBe(true)); + await tick(); + await waitFor(() => + expect(screen.getByText("No current conversation body.")) + .toBeTruthy() + ); + expect(screen.queryByText("Request accepted.")).toBeNull(); + expect(screen.queryByText("Body stopped.")).toBeNull(); + expect( + sessionStorage.getItem( + subjectBodyStorageKey( + props.accountId, + props.workspaceId, + props.subjectId, + ), + ), + ).toBe(saved); +}); + +test("a newer refresh observation is not replaced by an older mutation body", async () => { + const mutation = deferred(); + const finalRefresh = deferred(); + let gets = 0; + let requestId = ""; + vi.stubGlobal( + "fetch", + vi.fn(async (_path: string, init?: RequestInit) => { + if (init?.method === "POST") { + requestId = JSON.parse(init.body as string).request_id; + return mutation.promise; + } + gets += 1; + if (gets === 1) return Response.json(absent()); + if (gets === 2) return Response.json(available("idle")); + return finalRefresh.promise; + }), + ); + render(SubjectBody, props); + await fireEvent.click( + await screen.findByRole("button", { name: "Start conversation body" }), + ); + await screen.findByText("Sending request; body state is not yet confirmed."); + await fireEvent.click( + screen.getByRole("button", { name: "Refresh body state" }), + ); + await screen.findByText("Live execution"); + mutation.resolve( + Response.json({ + request_id: requestId, + disposition: "accepted", + diagnostics: [], + body: available(), + }), + ); + await screen.findByText("Request accepted."); + await waitFor(() => expect(gets).toBe(3)); + expect(screen.queryByText("Body stopped.")).toBeNull(); + expect(screen.getByText("Live execution")).toBeTruthy(); + finalRefresh.resolve(Response.json(available("idle"))); + await waitFor(() => + expect( + (screen.getByRole("button", { + name: "Refresh body state", + }) as HTMLButtonElement).disabled, + ).toBe(false) + ); +}); + +test.each(["not_found", "pending"])( + "read-only %s stop lookup and body GET retain the complete unknown request", + async (outcome) => { + const posts: { path: string; body: string }[] = []; + let observation = available("idle"); + vi.stubGlobal( + "fetch", + vi.fn(async (path: string, init?: RequestInit) => { + if (init?.method !== "POST") return Response.json(observation); + posts.push({ path, body: init.body as string }); + const request = JSON.parse(init.body as string); + if (path.endsWith("/stop/outcome")) { + return Response.json({ + request_id: request.request_id, + outcome, + body: observation, + }); + } + // 停止した本体のGETを得ても、その停止依頼のunknownは未解決のまま。 + observation = available("stopped", "new-observation"); + return Response.json({ + request_id: request.request_id, + disposition: "unknown", + diagnostics: [], + body: null, + }); + }), + ); + const view = render(SubjectBody, props); + await fireEvent.click( + await screen.findByRole("button", { name: "Stop conversation body" }), + ); + await screen.findByText("Request outcome unknown."); + await screen.findByText("Body stopped."); + const key = subjectBodyStorageKey( + props.accountId, + props.workspaceId, + props.subjectId, + ); + const saved = sessionStorage.getItem(key); + expect(saved).not.toBeNull(); + await waitFor(() => + expect( + (screen.getByRole("button", { + name: "Refresh body state", + }) as HTMLButtonElement).disabled, + ).toBe(false) + ); + await fireEvent.click( + screen.getByRole("button", { name: "Refresh body state" }), + ); + await waitFor(() => + expect( + (screen.getByRole("button", { + name: "Refresh body state", + }) as HTMLButtonElement).disabled, + ).toBe(false) + ); + expect(sessionStorage.getItem(key)).toBe(saved); + expect(screen.queryByRole("button", { name: "Restart conversation body" })) + .toBeNull(); + // 再読込み相当のremount後も、初回停止時の全入力で照会する。 + view.unmount(); + render(SubjectBody, props); + await fireEvent.click( + await screen.findByRole("button", { name: "Check stop outcome" }), + ); + await screen.findByText( + outcome === "pending" + ? "Stop outcome pending." + : "No stop receipt found.", + ); + await waitFor(() => + expect( + (screen.getByRole("button", { + name: "Refresh body state", + }) as HTMLButtonElement).disabled, + ).toBe(false) + ); + expect(posts.filter((request) => request.path.endsWith("/stop"))) + .toHaveLength(1); + const checks = posts.filter((request) => + request.path.endsWith("/stop/outcome") + ); + expect(checks).toHaveLength(1); + expect(checks[0].body).toBe(posts[0].body); + expect(JSON.parse(checks[0].body)).toMatchObject({ + expected_current: owner, + expected_observation_token: "observation-1", + }); + expect(sessionStorage.getItem(key)).toBe(saved); + expect(screen.queryByRole("button", { name: "Restart conversation body" })) + .toBeNull(); + expect(screen.getByRole("button", { name: "Retry same stop request" })) + .toBeTruthy(); + }, +); + +test("historical stopped outcome settles only the old request while restored body remains running", async () => { + const posts: { path: string; body: string }[] = []; + let observation = available("idle"); + vi.stubGlobal( + "fetch", + vi.fn(async (path: string, init?: RequestInit) => { + if (init?.method !== "POST") return Response.json(observation); + posts.push({ path, body: init.body as string }); + const request = JSON.parse(init.body as string); + if (path.endsWith("/stop/outcome")) { + observation = available("running", "restored-observation"); + return Response.json({ + request_id: request.request_id, + outcome: "stopped", + body: observation, + }); + } + return Response.json({ + request_id: request.request_id, + disposition: "unknown", + diagnostics: [], + body: null, + }); + }), + ); + render(SubjectBody, props); + await fireEvent.click( + await screen.findByRole("button", { name: "Stop conversation body" }), + ); + await screen.findByText("Request outcome unknown."); + await fireEvent.click( + screen.getByRole("button", { name: "Check stop outcome" }), + ); + await screen.findByText("Historical stop confirmed."); + await screen.findByText("running"); + expect(screen.queryByText("Body stopped.")).toBeNull(); + expect(screen.queryByText("Request outcome unknown.")).toBeNull(); + expect(screen.getByText(/The body may have been restored since then/)) + .toBeTruthy(); + expect( + sessionStorage.getItem( + subjectBodyStorageKey( + props.accountId, + props.workspaceId, + props.subjectId, + ), + ), + ).toBeNull(); + expect(posts.filter((request) => request.path.endsWith("/stop"))) + .toHaveLength(1); + expect(posts[1].path.endsWith("/stop/outcome")).toBe(true); + expect(posts[1].body).toBe(posts[0].body); +}); + +test.each([ + "forbidden", + "lookup", + "http-not-found", + "transport", + "wrong-receipt", + "invalid-json", +])( + "%s stop lookup failure is not a missing receipt or settled stop", + async (failure) => { + const posts: string[] = []; + vi.stubGlobal( + "fetch", + vi.fn(async (path: string, init?: RequestInit) => { + if (init?.method !== "POST") return Response.json(available("idle")); + posts.push(path); + if (path.endsWith("/stop/outcome")) { + if (failure === "transport") throw new Error("connection lost"); + if ( + failure === "forbidden" || failure === "lookup" || + failure === "http-not-found" + ) { + return Response.json({ error: "unavailable" }, { + status: failure === "forbidden" + ? 403 + : failure === "http-not-found" + ? 404 + : 502, + }); + } + if (failure === "invalid-json") return new Response("invalid-json"); + return Response.json({ + request_id: "other-request", + outcome: "not_found", + body: null, + }); + } + return Response.json({ + request_id: JSON.parse(init.body as string).request_id, + disposition: "unknown", + diagnostics: [], + body: null, + }); + }), + ); + render(SubjectBody, props); + await fireEvent.click( + await screen.findByRole("button", { name: "Stop conversation body" }), + ); + await screen.findByText("Request outcome unknown."); + const key = subjectBodyStorageKey( + props.accountId, + props.workspaceId, + props.subjectId, + ); + const saved = sessionStorage.getItem(key); + await fireEvent.click( + screen.getByRole("button", { name: "Check stop outcome" }), + ); + await screen.findByText(/Stop outcome could not be checked/); + expect(screen.queryByText("No stop receipt found.")).toBeNull(); + expect(screen.queryByText("Stop outcome pending.")).toBeNull(); + expect(screen.queryByText("Historical stop confirmed.")).toBeNull(); + expect(sessionStorage.getItem(key)).toBe(saved); + expect(posts.filter((path) => path.endsWith("/stop"))).toHaveLength(1); + expect(posts.filter((path) => path.endsWith("/stop/outcome"))).toHaveLength( + 1, + ); + }, +); + +test("late stop outcome cannot settle an old route request after navigation", async () => { + const pending = deferred(); + let requestId = ""; + vi.stubGlobal( + "fetch", + vi.fn(async (path: string, init?: RequestInit) => { + if (path.endsWith("/stop/outcome")) return pending.promise; + if (init?.method === "POST") { + requestId = JSON.parse(init.body as string).request_id; + return Response.json({ + request_id: requestId, + disposition: "unknown", + diagnostics: [], + body: null, + }); + } + return Response.json( + path.includes("/subjects/other/body") + ? { ...absent("other"), can_start: false, can_stop: false } + : available("idle"), + ); + }), + ); + const view = render(SubjectBody, props); + await fireEvent.click( + await screen.findByRole("button", { name: "Stop conversation body" }), + ); + await screen.findByText("Request outcome unknown."); + await fireEvent.click( + screen.getByRole("button", { name: "Check stop outcome" }), + ); + await screen.findByText( + "Checking saved stop outcome… No stop request is sent.", + ); + const key = subjectBodyStorageKey( + props.accountId, + props.workspaceId, + props.subjectId, + ); + const saved = sessionStorage.getItem(key); + await view.rerender({ ...props, subjectId: "other" }); + await screen.findByText(/You do not have permission to start or stop/); + const response = Response.json({ + request_id: requestId, + outcome: "stopped", + body: available(), + }); + pending.resolve(response); + await waitFor(() => expect(response.bodyUsed).toBe(true)); + await tick(); + expect(screen.queryByText("Historical stop confirmed.")).toBeNull(); + expect(screen.queryByRole("button", { name: "Check stop outcome" })) + .toBeNull(); + expect(sessionStorage.getItem(key)).toBe(saved); +}); + +test("account switch hides old body and retry input while same-account reload preserves exact request", async () => { + const accountB = { ...props, accountId: "account-b" }; + const first = savedStop(); + const keyA = subjectBodyStorageKey( + props.accountId, + props.workspaceId, + props.subjectId, + ); + const keyB = subjectBodyStorageKey( + accountB.accountId, + props.workspaceId, + props.subjectId, + ); + const rawA = JSON.stringify(first); + sessionStorage.setItem(keyA, rawA); + const pendingB = deferred(); + let account = props.accountId; + let firstB = true; + const posts: { path: string; body: string }[] = []; + vi.stubGlobal( + "fetch", + vi.fn(async (path: string, init?: RequestInit) => { + if (init?.method === "POST") { + posts.push({ path, body: init.body as string }); + return Response.json({ + request_id: JSON.parse(init.body as string).request_id, + disposition: "unknown", + diagnostics: [], + body: null, + }); + } + if (account === accountB.accountId && firstB) { + firstB = false; + return pendingB.promise; + } + return Response.json( + account === props.accountId ? available() : absent(), + ); + }), + ); + const view = render(SubjectBody, props); + await screen.findByText("Body stopped."); + await screen.findByText(/An unresolved stop request is retained/); + const reads = vi.spyOn(Storage.prototype, "getItem"); + reads.mockClear(); + account = accountB.accountId; + await view.rerender(accountB); + await screen.findByText("Loading body state…"); + expect(screen.queryByText("Body stopped.")).toBeNull(); + expect(screen.queryByText(/An unresolved stop request is retained/)) + .toBeNull(); + expect(screen.queryByText(first.request.request_id)).toBeNull(); + expect(screen.queryByRole("button", { name: "Retry same stop request" })) + .toBeNull(); + expect(reads).toHaveBeenCalledWith(keyB); + expect(reads).not.toHaveBeenCalledWith(keyA); + pendingB.resolve(Response.json(absent())); + await fireEvent.click( + await screen.findByRole("button", { name: "Start conversation body" }), + ); + await screen.findByText("Request outcome unknown."); + const rawB = sessionStorage.getItem(keyB); + expect(rawB).not.toBeNull(); + expect(JSON.parse(rawB!)).toMatchObject({ accountId: accountB.accountId }); + expect(sessionStorage.getItem(keyA)).toBe(rawA); + expect(posts).toHaveLength(1); + expect(JSON.parse(posts[0].body).request_id).not.toBe( + first.request.request_id, + ); + expect(JSON.parse(posts[0].body)).not.toHaveProperty("accountId"); + view.unmount(); + const reloadedB = render(SubjectBody, accountB); + await screen.findByText(/An unresolved start request is retained/); + expect(screen.queryByText(first.request.request_id)).toBeNull(); + expect(screen.queryByRole("button", { name: "Check stop outcome" })) + .toBeNull(); + expect(posts).toHaveLength(1); + reloadedB.unmount(); + account = props.accountId; + render(SubjectBody, props); + await waitFor(() => + expect( + (screen.getByRole("button", { + name: "Retry same stop request", + }) as HTMLButtonElement).disabled, + ).toBe(false) + ); + await fireEvent.click( + screen.getByRole("button", { name: "Retry same stop request" }), + ); + await waitFor(() => expect(posts).toHaveLength(2)); + expect(posts[1].path.endsWith("/stop")).toBe(true); + expect(posts[1].body).toBe(JSON.stringify(first.request)); + expect(JSON.parse(posts[1].body)).not.toHaveProperty("accountId"); + expect(sessionStorage.getItem(keyA)).toBe(rawA); + expect(sessionStorage.getItem(keyB)).toBe(rawB); +}); + +test("late GET from another account cannot replace the same route's new observation", async () => { + const pendingA = deferred(); + let account = props.accountId; + const fetch = vi.fn(() => + account === props.accountId ? pendingA.promise : Promise.resolve( + Response.json({ ...absent(), can_start: false, can_stop: false }), + ) + ); + vi.stubGlobal("fetch", fetch); + const view = render(SubjectBody, props); + await waitFor(() => expect(fetch).toHaveBeenCalledTimes(1)); + account = "account-b"; + await view.rerender({ ...props, accountId: account }); + await screen.findByText(/You do not have permission to start or stop/); + const oldResponse = Response.json(available()); + pendingA.resolve(oldResponse); + await waitFor(() => expect(oldResponse.bodyUsed).toBe(true)); + await tick(); + expect(screen.getByText("No current conversation body.")).toBeTruthy(); + expect(screen.queryByText("Body stopped.")).toBeNull(); + expect(screen.queryByRole("button", { name: "Restart conversation body" })) + .toBeNull(); + expect(fetch).toHaveBeenCalledTimes(2); +}); + +test.each([false, true])( + "late mutation cannot update or clear another account's requests, round trip=%s", + async (roundTrip) => { + const pendingA = deferred(); + const accountB = { ...props, accountId: "account-b" }; + let account = props.accountId; + let requestA = ""; + let gets = 0; + const posts: string[] = []; + vi.stubGlobal( + "fetch", + vi.fn(async (_path: string, init?: RequestInit) => { + if (init?.method !== "POST") { + gets += 1; + return Response.json(absent()); + } + posts.push(init.body as string); + if (account === props.accountId) { + requestA = JSON.parse(init.body as string).request_id; + return pendingA.promise; + } + return Response.json({ + request_id: JSON.parse(init.body as string).request_id, + disposition: "unknown", + diagnostics: [], + body: null, + }); + }), + ); + const view = render(SubjectBody, props); + await fireEvent.click( + await screen.findByRole("button", { name: "Start conversation body" }), + ); + await screen.findByText( + "Sending request; body state is not yet confirmed.", + ); + const keyA = subjectBodyStorageKey( + props.accountId, + props.workspaceId, + props.subjectId, + ); + const rawA = sessionStorage.getItem(keyA); + account = accountB.accountId; + await view.rerender(accountB); + expect( + screen.queryByText("Sending request; body state is not yet confirmed."), + ).toBeNull(); + await fireEvent.click( + await screen.findByRole("button", { name: "Start conversation body" }), + ); + await screen.findByText("Request outcome unknown."); + await waitFor(() => + expect( + (screen.getByRole("button", { + name: "Refresh body state", + }) as HTMLButtonElement).disabled, + ).toBe(false) + ); + const keyB = subjectBodyStorageKey( + accountB.accountId, + props.workspaceId, + props.subjectId, + ); + const rawB = sessionStorage.getItem(keyB); + if (roundTrip) { + account = props.accountId; + await view.rerender(props); + await screen.findByText(/An unresolved start request is retained/); + expect(screen.queryByText("Request outcome unknown.")).toBeNull(); + await waitFor(() => + expect( + (screen.getByRole("button", { + name: "Refresh body state", + }) as HTMLButtonElement).disabled, + ).toBe(false) + ); + } + const removals = vi.spyOn(Storage.prototype, "removeItem"); + const getsBeforeResponse = gets; + const oldResponse = Response.json({ + request_id: requestA, + disposition: "accepted", + diagnostics: [], + body: available(), + }); + pendingA.resolve(oldResponse); + await waitFor(() => expect(oldResponse.bodyUsed).toBe(true)); + await tick(); + expect(screen.queryByText("Request accepted.")).toBeNull(); + expect(screen.queryByText("Body stopped.")).toBeNull(); + expect(removals).not.toHaveBeenCalled(); + expect(sessionStorage.getItem(keyA)).toBe(rawA); + expect(sessionStorage.getItem(keyB)).toBe(rawB); + expect(posts).toHaveLength(2); + expect(gets).toBe(getsBeforeResponse); + }, +); + +test("late stop outcome from another account cannot settle or clear either account's saved request", async () => { + const first = savedStop(); + const second = savedStop("account-b"); + const keyA = subjectBodyStorageKey( + first.accountId, + props.workspaceId, + props.subjectId, + ); + const keyB = subjectBodyStorageKey( + second.accountId, + props.workspaceId, + props.subjectId, + ); + const rawA = JSON.stringify(first); + const rawB = JSON.stringify(second); + sessionStorage.setItem(keyA, rawA); + sessionStorage.setItem(keyB, rawB); + const pending = deferred(); + const posts: string[] = []; + vi.stubGlobal( + "fetch", + vi.fn(async (path: string, init?: RequestInit) => { + if (init?.method === "POST") { + posts.push(init.body as string); + return pending.promise; + } + return Response.json(available("idle")); + }), + ); + const view = render(SubjectBody, props); + await fireEvent.click( + await screen.findByRole("button", { name: "Check stop outcome" }), + ); + await screen.findByText( + "Checking saved stop outcome… No stop request is sent.", + ); + await view.rerender({ ...props, accountId: second.accountId }); + await screen.findByText(/An unresolved stop request is retained/); + expect( + screen.queryByText("Checking saved stop outcome… No stop request is sent."), + ).toBeNull(); + const removals = vi.spyOn(Storage.prototype, "removeItem"); + const oldResponse = Response.json({ + request_id: first.request.request_id, + outcome: "stopped", + body: available(), + }); + pending.resolve(oldResponse); + await waitFor(() => expect(oldResponse.bodyUsed).toBe(true)); + await tick(); + expect(screen.queryByText("Historical stop confirmed.")).toBeNull(); + expect(screen.queryByText("Body stopped.")).toBeNull(); + expect(screen.getByRole("button", { name: "Check stop outcome" })) + .toBeTruthy(); + expect(removals).not.toHaveBeenCalled(); + expect(posts).toEqual([JSON.stringify(first.request)]); + expect(sessionStorage.getItem(keyA)).toBe(rawA); + expect(sessionStorage.getItem(keyB)).toBe(rawB); +}); + +test("signout hides body and request details and never falls back to a dummy identity", async () => { + const first = savedStop(); + const key = subjectBodyStorageKey( + first.accountId, + props.workspaceId, + props.subjectId, + ); + const raw = JSON.stringify(first); + sessionStorage.setItem(key, raw); + const fetch = vi.fn(async () => Response.json(available())); + vi.stubGlobal("fetch", fetch); + const view = render(SubjectBody, props); + await screen.findByText("Body stopped."); + expect(screen.getByText(first.request.request_id)).toBeTruthy(); + const reads = vi.spyOn(Storage.prototype, "getItem"); + reads.mockClear(); + await view.rerender({ ...props, accountId: "" }); + await screen.findByText("Sign in to view this conversation body."); + expect(screen.queryByText("Body stopped.")).toBeNull(); + expect(screen.queryByText(first.request.request_id)).toBeNull(); + expect(screen.queryByRole("button", { name: "Check stop outcome" })) + .toBeNull(); + expect(screen.queryByRole("button", { name: "Refresh body state" })) + .toBeNull(); + expect(reads).not.toHaveBeenCalled(); + expect(fetch).toHaveBeenCalledTimes(1); + expect(sessionStorage.getItem(key)).toBe(raw); +}); diff --git a/web/workspace/src/lib/workspace/memory/SubjectBody.svelte b/web/workspace/src/lib/workspace/memory/SubjectBody.svelte new file mode 100644 index 000000000..22edb6330 --- /dev/null +++ b/web/workspace/src/lib/workspace/memory/SubjectBody.svelte @@ -0,0 +1,288 @@ + + +
+
+

Conversation body

+ {#if hasAccount}{/if} +
+ {#if !hasAccount} +

Sign in to view this conversation body.

+ {:else if !visible || (!snapshot && loading)} +

Loading body state…

+ {:else if snapshot} + {#if !fresh}

{loading ? 'Updating body state…' : 'Last confirmed observation; refresh before acting.'}

{/if} + {#if snapshot.observation.kind === 'absent'} +

No current conversation body.

+ {:else if snapshot.observation.kind === 'unavailable'} +

Body owner present; observation unavailable. This is not an absent or stopped body. A replacement will not be started.

+ {#each snapshot.observation.diagnostics as diagnostic} +

{diagnostic.message}

+ {/each} + {:else if worker} + {#if worker.availability !== 'observed'} +

Body owner present; Worker observation unavailable. Running or stopped state is not confirmed.

+ {:else if liveWorkerState(worker) === 'stopped'} +

Body stopped. Start restores this body; retained conversation and Memory remain available.

+ {:else if worker.worker_state} +
Worker lifecycle
{worker.state}
Live execution
{liveWorkerState(worker)}
+ {:else} +

Body owner present; live execution not confirmed. Refresh before acting.

+ {/if} + {#if worker.availability === 'observed' && !worker.restore_observation_token} +

Lifecycle controls are unavailable until the current body’s observation token can be confirmed. Refresh body state before acting.

+ {/if} + {#each workerDiagnostics as diagnostic} +

{diagnostic.message}

+ {/each} + {/if} + {#if !snapshot.can_start || !snapshot.can_stop} +

{!snapshot.can_start && !snapshot.can_stop ? 'You do not have permission to start or stop this body.' : !snapshot.can_start ? 'You do not have permission to start this body.' : 'You do not have permission to stop this body.'} Ask a Workspace administrator to review access; this control does not add permissions.

+ {/if} + {#if !attempt} +
+ {#if controls.start} + + {/if} + {#if controls.stop} + + {/if} +
+ {#if controls.stop}

Stopping ends the body Worker, not the Subject. Conversation and Memory are retained.

{/if} + {/if} + {/if} + {#if visible} + {#if loadError}{/if} + {#if storageError}{/if} + {#if sending}

Sending request; body state is not yet confirmed.

{/if} + {#if receipt} +
+ {#if receipt.disposition === 'accepted'} +

Request accepted. Acceptance is not confirmation of running or stopped state. Check the body observation above.

+ {:else if receipt.disposition === 'rejected'} +

Request rejected. Review the diagnostics and refresh before a new action. Check Subject permissions, the default profile and model configuration; no permissions were added.

+ {:else} +

Request outcome unknown. {attempt?.action === 'stop' ? 'Check the saved stop outcome without sending another stop, or explicitly retry the same saved request.' : 'Refresh observations, then retry the same saved request to reconcile.'} Refresh alone does not settle this receipt.

+ {/if} + {#each receipt.diagnostics as diagnostic}

{diagnostic.message}

{/each} +
+ {/if} + {#if checkingOutcome}

Checking saved stop outcome… No stop request is sent.

{/if} + {#if stopOutcome} +
+ {#if stopOutcome.outcome === 'stopped'} +

Historical stop confirmed. This settles the saved stop operation, not the current body’s state. The body may have been restored since then; see the current observation above.

+ {:else if stopOutcome.outcome === 'pending'} +

Stop outcome pending. Stopping is not confirmed. The complete saved request is retained; check again or explicitly retry the same request.

+ {:else} +

No stop receipt found. This does not prove that stopping failed or that the current body is stopped. The complete saved request is retained; check again or explicitly retry the same request.

+ {/if} +
+ {/if} + {#if outcomeError}{/if} + {#if operationError}{/if} + {#if attempt && !sending} + {#if !receipt && !operationError && stopOutcome?.outcome !== 'stopped'}

An unresolved {attempt.action} request is retained in this tab. Refresh and retry the same request; do not create a replacement.

{/if} + {#if attempt.action === 'stop' && snapshot?.can_stop} + + {/if} + {#if retryAuthorized} + + {/if} +
Request diagnostics

Request ID: {attempt.request.request_id}

The complete frozen request is retained across reloads in this tab. Retrying does not use a new request ID or a newer observation token.

+ {/if} + {/if} +
+ + diff --git a/web/workspace/src/lib/workspace/memory/api.ts b/web/workspace/src/lib/workspace/memory/api.ts index 8bb095fd1..beb955aa7 100644 --- a/web/workspace/src/lib/workspace/memory/api.ts +++ b/web/workspace/src/lib/workspace/memory/api.ts @@ -607,9 +607,11 @@ export function parseSubjektivSubjectResponse( "created_at", "updated_at", "current_worker", + "current_body", + "current_worker_diagnostics", ], "Subject response", - ["current_worker"], + ["current_worker", "current_body", "current_worker_diagnostics"], ); const id = requiredIdentifier(record, "id"); if (expectedSubjectId !== undefined && id !== expectedSubjectId) { @@ -638,6 +640,33 @@ export function parseSubjektivSubjectResponse( "Subject current worker", ); } + if ("current_body" in record) { + if (record.current_body === null) result.current_body = null; + else { + const body = strictRecord(record.current_body, [ + "runtime_id", + "worker_id", + "generation", + ], "Subject body owner"); + const generation = requiredString(body, "generation", 20); + if ( + !/^[1-9][0-9]{0,19}$/.test(generation) || + BigInt(generation) > 18446744073709551615n + ) invalid("invalid Subject body generation"); + result.current_body = { + runtime_id: requiredIdentifier(body, "runtime_id"), + worker_id: requiredIdentifier(body, "worker_id"), + generation, + }; + } + } + if ("current_worker_diagnostics" in record) { + result.current_worker_diagnostics = boundedArray( + record.current_worker_diagnostics, + 100, + "current worker diagnostics", + ).map(parseDiagnostic); + } return result; } diff --git a/web/workspace/src/lib/workspace/memory/subject-body.test.ts b/web/workspace/src/lib/workspace/memory/subject-body.test.ts new file mode 100644 index 000000000..6294a6795 --- /dev/null +++ b/web/workspace/src/lib/workspace/memory/subject-body.test.ts @@ -0,0 +1,721 @@ +import { + clearSubjectBodyAttempt, + createSubjectBodyAttempt, + getSubjectBody, + getSubjectBodyStopOutcome, + parseSubjectBodyMutationResponse, + parseSubjectBodyResponse, + parseSubjectBodyStopOutcomeResponse, + readSubjectBodyAttempt, + saveSubjectBodyAttempt, + startSubjectBody, + stopSubjectBody, + subjectBodyControls, + subjectBodyStorageKey, +} from "./subject-body.ts"; + +declare const Deno: { + test(name: string, fn: () => void | Promise): void; +}; +function assert(value: unknown): asserts value { + if (!value) throw new Error("Assertion failed"); +} +function rejects(fn: () => unknown): void { + let rejected = false; + try { + fn(); + } catch { + rejected = true; + } + assert(rejected); +} +const owner = { + runtime_id: "runtime", + worker_id: "worker", + generation: "18446744073709551615", +}; +function absent() { + return { + workspace_id: "workspace", + subject_id: "subject", + current: null, + observation: { kind: "absent" }, + can_start: true, + can_stop: true, + }; +} +function available(state = "stopped") { + return { + ...absent(), + current: owner, + observation: { + kind: "available", + worker: { + runtime_id: owner.runtime_id, + worker_id: owner.worker_id, + resource_key: "W-1", + host_id: "host", + label: "Body", + workspace: { + visibility: "workspace", + identity: "workspace", + workspace_id: "workspace", + }, + implementation: { kind: "embedded", display_hint: "" }, + availability: "observed", + state, + worker_state: state === "stopped" + ? null + : { last_command_id: 1, state: { kind: "idle" } }, + restore_observation_token: "observation-1", + }, + }, + }; +} +function parse(value: unknown) { + return parseSubjectBodyResponse(value, "workspace", "subject"); +} +function storage() { + const values = new Map(); + return { + getItem: (key: string) => values.get(key) ?? null, + setItem: (key: string, value: string) => { + values.set(key, value); + }, + removeItem: (key: string) => { + values.delete(key); + }, + }; +} + +Deno.test("absent permits initial start while unavailable retains owner and forbids replacement", () => { + assert(subjectBodyControls(parse(absent())).start); + const unavailable = parse({ + ...absent(), + current: owner, + observation: { + kind: "unavailable", + diagnostics: [{ + code: "runtime_unavailable", + severity: "warning", + message: "Cannot observe Runtime", + }], + }, + }); + assert(unavailable.current?.generation === "18446744073709551615"); + assert( + !subjectBodyControls(unavailable).start && + !subjectBodyControls(unavailable).stop, + ); + rejects(() => parse({ ...absent(), current: owner })); + rejects(() => + parse({ + ...absent(), + observation: { kind: "unavailable", diagnostics: [] }, + }) + ); +}); + +Deno.test("ordinary Worker parsing keeps availability and live execution distinct from lifecycle", () => { + const stopped = parse(available()); + assert( + subjectBodyControls(stopped).start && !subjectBodyControls(stopped).stop, + ); + const idle = parse(available("idle")); + assert(subjectBodyControls(idle).stop && !subjectBodyControls(idle).start); + const unobserved = available("idle"); + unobserved.observation.worker.availability = "unavailable"; + unobserved.observation.worker.worker_state = null; + assert( + !subjectBodyControls(parse(unobserved)).start && + !subjectBodyControls(parse(unobserved)).stop, + ); + const invalid = available(); + invalid.observation.worker.availability = "invented"; + rejects(() => parse(invalid)); + const noPermission = parse({ + ...available("idle"), + can_start: false, + can_stop: false, + }); + assert( + !subjectBodyControls(noPermission).start && + !subjectBodyControls(noPermission).stop, + ); +}); + +Deno.test("body responses reject cross-route identities and mismatched owner Worker", () => { + rejects(() => parse({ ...absent(), workspace_id: "other" })); + rejects(() => parse({ ...absent(), subject_id: "other" })); + const mismatch = available(); + mismatch.observation.worker.worker_id = "old-worker"; + rejects(() => parse(mismatch)); + const foreignWorkspace = available(); + foreignWorkspace.observation.worker.workspace.workspace_id = "other"; + rejects(() => parse(foreignWorkspace)); +}); + +Deno.test("external body JSON requires nullable current and bounds generations diagnostics and unknown fields", () => { + const missing: Record = absent(); + delete missing.current; + rejects(() => parse(missing)); + for ( + const generation of [1, "1e3", "-1", "18446744073709551616", "1".repeat(30)] + ) { + rejects(() => parse({ ...available(), current: { ...owner, generation } })); + } + rejects(() => parse({ ...absent(), can_start: "true" })); + rejects(() => parse({ ...absent(), extra: true })); + rejects(() => + parse({ + ...available(), + observation: { + kind: "unavailable", + diagnostics: Array(129).fill({ + code: "x", + severity: "warning", + message: "x", + }), + }, + }) + ); +}); + +Deno.test("mutation receipts validate request identity and nested route without deriving execution state", () => { + const receipt = { + request_id: "request", + disposition: "accepted", + diagnostics: [], + body: absent(), + }; + const parsed = parseSubjectBodyMutationResponse( + receipt, + "workspace", + "subject", + "request", + ); + assert( + parsed.disposition === "accepted" && + parsed.body?.observation.kind === "absent", + ); + rejects(() => + parseSubjectBodyMutationResponse( + receipt, + "workspace", + "subject", + "different", + ) + ); + rejects(() => + parseSubjectBodyMutationResponse(receipt, "other", "subject", "request") + ); + rejects(() => + parseSubjectBodyMutationResponse( + { ...receipt, disposition: "running" }, + "workspace", + "subject", + "request", + ) + ); +}); + +Deno.test("durable retry input is frozen complete account-and-route-scoped and never silently replaced", () => { + const saved = storage(); + const attempt = createSubjectBodyAttempt( + "account", + parse(available()), + "start", + "stable-request", + ); + saveSubjectBodyAttempt(saved, "account", attempt); + const restored = readSubjectBodyAttempt( + saved, + "account", + "workspace", + "subject", + ); + assert(restored && JSON.stringify(restored) === JSON.stringify(attempt)); + assert( + Object.isFrozen(restored) && Object.isFrozen(restored.request) && + Object.isFrozen(restored.request.expected_current), + ); + assert(restored.request.expected_observation_token === "observation-1"); + assert( + readSubjectBodyAttempt(saved, "account", "workspace", "other") === null, + ); + const newer = createSubjectBodyAttempt( + "account", + parse(available()), + "start", + "new-request", + ); + rejects(() => saveSubjectBodyAttempt(saved, "account", newer)); + rejects(() => clearSubjectBodyAttempt(saved, "account", newer)); + saveSubjectBodyAttempt(saved, "account", restored); + clearSubjectBodyAttempt(saved, "account", restored); + assert( + readSubjectBodyAttempt(saved, "account", "workspace", "subject") === null, + ); +}); + +Deno.test("unreadable or incomplete saved request blocks replacement rather than generating identity", () => { + const saved = storage(); + const key = subjectBodyStorageKey("account", "workspace", "subject"); + for ( + const raw of [ + "not-json", + "x".repeat(17000), + JSON.stringify({ + accountId: "account", + workspaceId: "workspace", + subjectId: "subject", + action: "start", + request: { request_id: "only-id" }, + }), + JSON.stringify({ + accountId: "account", + workspaceId: "workspace", + subjectId: "other", + action: "start", + request: { + request_id: "request", + expected_current: null, + expected_observation_token: null, + }, + }), + ] + ) { + saved.setItem(key, raw); + rejects(() => + readSubjectBodyAttempt(saved, "account", "workspace", "subject") + ); + rejects(() => + saveSubjectBodyAttempt( + saved, + "account", + createSubjectBodyAttempt( + "account", + parse(absent()), + "start", + "replacement", + ), + ) + ); + assert(saved.getItem(key) === raw); + } +}); + +Deno.test("shared HTTP functions encode route segments and preserve exact start stop inputs", async () => { + const workspace = "space /"; + const subject = "subject /"; + const requests: { path: string; init?: RequestInit }[] = []; + const body = { ...absent(), workspace_id: workspace, subject_id: subject }; + const fetchFn = ((path: string, init?: RequestInit) => { + requests.push({ path, init }); + return Promise.resolve(Response.json( + init?.method === "POST" + ? { + request_id: JSON.parse(init.body as string).request_id, + disposition: "unknown", + diagnostics: [], + body, + } + : body, + )); + }) as typeof fetch; + const start = { + request_id: "same-start", + expected_current: null, + expected_observation_token: null, + }; + const stop = { + request_id: "same-stop", + expected_current: owner, + expected_observation_token: "frozen-token", + }; + await getSubjectBody(fetchFn, workspace, subject); + const result = await startSubjectBody(fetchFn, workspace, subject, start); + assert(result.disposition === "unknown"); + await startSubjectBody(fetchFn, workspace, subject, start); + await stopSubjectBody(fetchFn, workspace, subject, stop); + assert( + requests[0].path === + "/api/w/space%20%2F/subjektiv/subjects/subject%20%2F/body", + ); + assert(requests[1].path === `${requests[0].path}/start`); + assert(requests[3].path === `${requests[0].path}/stop`); + assert(requests[1].init?.body === JSON.stringify(start)); + assert(requests[2].init?.body === requests[1].init?.body); + assert(requests[3].init?.body === JSON.stringify(stop)); +}); + +Deno.test("stop outcome validates historical receipt identity independently of current body", () => { + for (const outcome of ["not_found", "pending", "stopped"]) { + const result = parseSubjectBodyStopOutcomeResponse( + { + request_id: "saved-stop", + outcome, + body: available("running"), + }, + "workspace", + "subject", + "saved-stop", + ); + assert(result.outcome === outcome); + assert( + result.body?.observation.kind === "available" && + result.body.observation.worker.state === "running", + ); + } + const receipt = { + request_id: "saved-stop", + outcome: "stopped", + body: absent(), + }; + rejects(() => + parseSubjectBodyStopOutcomeResponse( + receipt, + "workspace", + "subject", + "other-request", + ) + ); + rejects(() => + parseSubjectBodyStopOutcomeResponse( + receipt, + "other-workspace", + "subject", + "saved-stop", + ) + ); + rejects(() => + parseSubjectBodyStopOutcomeResponse( + receipt, + "workspace", + "other-subject", + "saved-stop", + ) + ); + rejects(() => + parseSubjectBodyStopOutcomeResponse( + { ...receipt, outcome: "unknown" }, + "workspace", + "subject", + "saved-stop", + ) + ); + rejects(() => + parseSubjectBodyStopOutcomeResponse( + { ...receipt, body: undefined }, + "workspace", + "subject", + "saved-stop", + ) + ); + rejects(() => + parseSubjectBodyStopOutcomeResponse( + { ...receipt, unexpected: true }, + "workspace", + "subject", + "saved-stop", + ) + ); +}); + +Deno.test("stop outcome query sends only the read-only endpoint with full frozen input unchanged", async () => { + const saved = storage(); + const attempt = createSubjectBodyAttempt( + "account", + parse(available("idle")), + "stop", + "saved-stop", + ); + assert(attempt.action === "stop"); + saveSubjectBodyAttempt(saved, "account", attempt); + const requests: { path: string; init?: RequestInit }[] = []; + const fetchFn = ((path: string, init?: RequestInit) => { + requests.push({ path, init }); + return Promise.resolve( + Response.json({ + request_id: "saved-stop", + outcome: "pending", + body: null, + }), + ); + }) as typeof fetch; + const result = await getSubjectBodyStopOutcome( + fetchFn, + "workspace", + "subject", + attempt.request, + ); + assert(result.outcome === "pending"); + assert(requests.length === 1); + assert( + requests[0].path === + "/api/w/workspace/subjektiv/subjects/subject/body/stop/outcome", + ); + assert(requests[0].init?.method === "POST"); + assert(requests[0].init?.body === JSON.stringify(attempt.request)); + assert( + JSON.stringify( + readSubjectBodyAttempt(saved, "account", "workspace", "subject"), + ) === JSON.stringify(attempt), + ); +}); + +Deno.test("stop outcome authorization transport lookup and malformed receipt failures never become not_found", async () => { + const request = { + request_id: "saved-stop", + expected_current: owner, + expected_observation_token: "saved-token", + }; + for ( + const failure of [ + "forbidden", + "lookup", + "http-not-found", + "transport", + "wrong-receipt", + "invalid-json", + ] + ) { + const fetchFn = (async () => { + if (failure === "transport") throw new Error("connection lost"); + if ( + failure === "forbidden" || failure === "lookup" || + failure === "http-not-found" + ) { + return Response.json({ error: "unavailable" }, { + status: failure === "forbidden" + ? 403 + : failure === "http-not-found" + ? 404 + : 502, + }); + } + if (failure === "invalid-json") return new Response("invalid-json"); + return Response.json({ + request_id: "other-request", + outcome: "not_found", + body: null, + }); + }) as typeof fetch; + let rejected = false; + try { + await getSubjectBodyStopOutcome(fetchFn, "workspace", "subject", request); + } catch { + rejected = true; + } + assert(rejected); + } +}); + +Deno.test("request IDs enforce 128 UTF-8 bytes and control-free input before sending or retaining", async () => { + for (const requestId of ["x".repeat(128), "é".repeat(64)]) { + const attempt = createSubjectBodyAttempt( + "account", + parse(absent()), + "start", + requestId, + ); + assert(attempt.request.request_id === requestId); + } + for ( + const requestId of [ + "", + "x".repeat(129), + "é".repeat(65), + "request\n", + "request\u007f", + "request\u0085", + ] + ) { + rejects(() => + createSubjectBodyAttempt("account", parse(absent()), "start", requestId) + ); + const saved = storage(); + saved.setItem( + subjectBodyStorageKey("account", "workspace", "subject"), + JSON.stringify({ + accountId: "account", + workspaceId: "workspace", + subjectId: "subject", + action: "start", + request: { + request_id: requestId, + expected_current: null, + expected_observation_token: null, + }, + }), + ); + rejects(() => + readSubjectBodyAttempt(saved, "account", "workspace", "subject") + ); + let calls = 0; + const fetchFn = (() => { + calls += 1; + return Promise.resolve(Response.json({})); + }) as typeof fetch; + const stop = { + request_id: requestId, + expected_current: owner, + expected_observation_token: "saved-token", + }; + const start = { + request_id: requestId, + expected_current: null, + expected_observation_token: null, + }; + for ( + const invoke of [ + () => startSubjectBody(fetchFn, "workspace", "subject", start), + () => stopSubjectBody(fetchFn, "workspace", "subject", stop), + () => getSubjectBodyStopOutcome(fetchFn, "workspace", "subject", stop), + ] + ) { + let rejected = false; + try { + await invoke(); + } catch { + rejected = true; + } + assert(rejected); + } + assert(calls === 0); + } +}); + +Deno.test("durable requests are account-scoped and another account cannot read save or clear them", () => { + const saved = storage(); + const first = createSubjectBodyAttempt( + "account-a", + parse(available()), + "start", + "first-request", + ); + saveSubjectBodyAttempt(saved, "account-a", first); + const firstKey = subjectBodyStorageKey("account-a", "workspace", "subject"); + const secondKey = subjectBodyStorageKey("account-b", "workspace", "subject"); + assert( + firstKey === + `yoi.subject-body:${ + JSON.stringify(["account-a", "workspace", "subject"]) + }`, + ); + assert(firstKey !== secondKey); + const firstRaw = saved.getItem(firstKey); + const accesses: string[] = []; + const observed = { + getItem(key: string) { + accesses.push(`get:${key}`); + return saved.getItem(key); + }, + setItem(key: string, value: string) { + accesses.push(`set:${key}`); + saved.setItem(key, value); + }, + removeItem(key: string) { + accesses.push(`remove:${key}`); + saved.removeItem(key); + }, + }; + assert( + readSubjectBodyAttempt(observed, "account-b", "workspace", "subject") === + null, + ); + assert(accesses.length === 1 && accesses[0] === `get:${secondKey}`); + accesses.length = 0; + rejects(() => saveSubjectBodyAttempt(observed, "account-b", first)); + rejects(() => clearSubjectBodyAttempt(observed, "account-b", first)); + assert(accesses.length === 0); + const second = createSubjectBodyAttempt( + "account-b", + parse(absent()), + "start", + "second-request", + ); + saveSubjectBodyAttempt(saved, "account-b", second); + clearSubjectBodyAttempt(saved, "account-b", second); + assert(saved.getItem(firstKey) === firstRaw); + const restored = readSubjectBodyAttempt( + saved, + "account-a", + "workspace", + "subject", + ); + assert(restored?.accountId === "account-a"); + assert(JSON.stringify(restored) === JSON.stringify(first)); + assert( + Object.isFrozen(restored) && Object.isFrozen(restored.request) && + Object.isFrozen(restored.request.expected_current), + ); +}); + +Deno.test("a foreign or accountless attempt copied into the current account key is rejected unchanged", () => { + const saved = storage(); + const first = createSubjectBodyAttempt( + "account-a", + parse(available("idle")), + "stop", + "first-stop", + ); + const secondKey = subjectBodyStorageKey("account-b", "workspace", "subject"); + const accountless = { + workspaceId: first.workspaceId, + subjectId: first.subjectId, + action: first.action, + request: first.request, + }; + for (const raw of [JSON.stringify(first), JSON.stringify(accountless)]) { + saved.setItem(secondKey, raw); + rejects(() => + readSubjectBodyAttempt(saved, "account-b", "workspace", "subject") + ); + const replacement = createSubjectBodyAttempt( + "account-b", + parse(absent()), + "start", + "new-request", + ); + rejects(() => saveSubjectBodyAttempt(saved, "account-b", replacement)); + assert(saved.getItem(secondKey) === raw); + } +}); + +Deno.test("accountless legacy keys are neither discovered migrated nor cleared for a signed-in account", () => { + const saved = storage(); + const legacyKey = `yoi.subject-body:${ + JSON.stringify(["workspace", "subject"]) + }`; + saved.setItem(legacyKey, "unattributed-request"); + const accessed: string[] = []; + const observed = { + getItem(key: string) { + accessed.push(key); + return saved.getItem(key); + }, + }; + assert( + readSubjectBodyAttempt(observed, "account-a", "workspace", "subject") === + null, + ); + assert( + accessed.length === 1 && + accessed[0] === + subjectBodyStorageKey("account-a", "workspace", "subject"), + ); + const own = createSubjectBodyAttempt( + "account-a", + parse(absent()), + "start", + "own-request", + ); + saveSubjectBodyAttempt(saved, "account-a", own); + clearSubjectBodyAttempt(saved, "account-a", own); + assert(saved.getItem(legacyKey) === "unattributed-request"); + rejects(() => subjectBodyStorageKey("", "workspace", "subject")); + rejects(() => + createSubjectBodyAttempt("", parse(absent()), "start", "no-account") + ); +}); diff --git a/web/workspace/src/lib/workspace/memory/subject-body.ts b/web/workspace/src/lib/workspace/memory/subject-body.ts new file mode 100644 index 000000000..5aa3794fb --- /dev/null +++ b/web/workspace/src/lib/workspace/memory/subject-body.ts @@ -0,0 +1,551 @@ +import type { + SubjectBodyMutationResponse, + SubjectBodyObservation, + SubjectBodyRef, + SubjectBodyResponse, + SubjectBodyStartRequest, + SubjectBodyStopOutcomeResponse, + SubjectBodyStopRequest, +} from "#lib/generated/subject-body-api.ts"; +import { parseWorkerSummary } from "#lib/workspace/api/workers.ts"; +import { readBoundedJson, workspaceApiPath } from "#lib/workspace/api/http.ts"; + +type Diagnostic = SubjectBodyMutationResponse["diagnostics"][number]; +const MAX_RESPONSE_BYTES = 2 * 1024 * 1024; +const MAX_SAVED_LENGTH = 16 * 1024; + +function record(value: unknown, keys: string[]): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) { + throw new Error("Subject body value must be an object"); + } + if (Object.keys(value).some((key) => !keys.includes(key))) { + throw new Error("Subject body value contains unknown fields"); + } + return value as Record; +} + +function text(value: unknown, max = 4096): string { + if (typeof value !== "string" || value.length === 0 || value.length > max) { + throw new Error("Subject body string is invalid"); + } + return value; +} + +function parseRequestId(value: unknown): string { + const result = text(value, 128); + if ( + new TextEncoder().encode(result).byteLength > 128 || /\p{Cc}/u.test(result) + ) { + throw new Error( + "Subject body request ID must be at most 128 UTF-8 bytes and contain no control characters", + ); + } + return result; +} + +function token(value: unknown): string { + const result = text(value, 256); + if (new TextEncoder().encode(result).byteLength > 256) { + throw new Error("Subject body observation token is too long"); + } + return result; +} + +function flag(value: unknown): boolean { + if (typeof value !== "boolean") { + throw new Error("Subject body flag is invalid"); + } + return value; +} + +function diagnostics(value: unknown): Diagnostic[] { + if (!Array.isArray(value) || value.length > 128) { + throw new Error("Subject body diagnostics exceed their limit"); + } + return value.map((entry) => { + const item = record(entry, ["code", "severity", "message"]); + const severity = item.severity; + if (severity !== "info" && severity !== "warning" && severity !== "error") { + throw new Error("Subject body diagnostic severity is invalid"); + } + return { + code: text(item.code), + severity, + message: text(item.message, 65536), + }; + }); +} + +function bodyRef(value: unknown): SubjectBodyRef { + const item = record(value, ["runtime_id", "worker_id", "generation"]); + const generation = text(item.generation, 20); + if ( + !/^(0|[1-9][0-9]*)$/.test(generation) || + (generation.length === 20 && generation > "18446744073709551615") + ) { + throw new Error("Subject body generation must be unsigned decimal text"); + } + return { + runtime_id: text(item.runtime_id), + worker_id: text(item.worker_id), + generation, + }; +} + +export function parseSubjectBodyResponse( + value: unknown, + workspaceId: string, + subjectId: string, +): SubjectBodyResponse { + const item = record(value, [ + "workspace_id", + "subject_id", + "current", + "observation", + "can_start", + "can_stop", + ]); + const workspace = text(item.workspace_id); + const subject = text(item.subject_id); + if (workspace !== workspaceId || subject !== subjectId) { + throw new Error( + "Subject body response does not match the requested target", + ); + } + const current = item.current === null ? null : bodyRef(item.current); + const observed = record(item.observation, ["kind", "worker", "diagnostics"]); + let observation: SubjectBodyObservation; + switch (observed.kind) { + case "absent": + record(observed, ["kind"]); + if (current !== null) throw new Error("Absent body cannot have an owner"); + observation = { kind: "absent" }; + break; + case "unavailable": + record(observed, ["kind", "diagnostics"]); + if (current === null) { + throw new Error("Unavailable body must retain its owner"); + } + observation = { + kind: "unavailable", + diagnostics: diagnostics(observed.diagnostics), + }; + break; + case "available": { + record(observed, ["kind", "worker"]); + const worker = parseWorkerSummary(observed.worker); + if ( + current === null || worker.runtime_id !== current.runtime_id || + worker.worker_id !== current.worker_id || + (worker.workspace.workspace_id != null && + worker.workspace.workspace_id !== workspaceId) + ) { + throw new Error("Subject body Worker does not match the current owner"); + } + observation = { kind: "available", worker }; + break; + } + default: + throw new Error("Subject body observation kind is invalid"); + } + return { + workspace_id: workspace, + subject_id: subject, + current, + observation, + can_start: flag(item.can_start), + can_stop: flag(item.can_stop), + }; +} + +export function parseSubjectBodyMutationResponse( + value: unknown, + workspaceId: string, + subjectId: string, + requestId: string, +): SubjectBodyMutationResponse { + const item = record(value, [ + "request_id", + "disposition", + "diagnostics", + "body", + ]); + const request_id = parseRequestId(item.request_id); + if (request_id !== requestId) { + throw new Error("Subject body receipt does not match the request"); + } + const disposition = item.disposition; + if ( + disposition !== "accepted" && disposition !== "rejected" && + disposition !== "unknown" + ) { + throw new Error("Subject body disposition is invalid"); + } + return { + request_id, + disposition, + diagnostics: diagnostics(item.diagnostics), + body: item.body === null + ? null + : parseSubjectBodyResponse(item.body, workspaceId, subjectId), + }; +} + +export function parseSubjectBodyStopOutcomeResponse( + value: unknown, + workspaceId: string, + subjectId: string, + requestId: string, +): SubjectBodyStopOutcomeResponse { + const item = record(value, ["request_id", "outcome", "body"]); + const request_id = parseRequestId(item.request_id); + if (request_id !== requestId) { + throw new Error("Subject body stop outcome does not match the request"); + } + const outcome = item.outcome; + if ( + outcome !== "not_found" && outcome !== "pending" && outcome !== "stopped" + ) { + throw new Error("Subject body stop outcome is invalid"); + } + return { + request_id, + outcome, + body: item.body === null + ? null + : parseSubjectBodyResponse(item.body, workspaceId, subjectId), + }; +} + +function bodyPath(workspaceId: string, subjectId: string): string { + return workspaceApiPath( + workspaceId, + `/subjektiv/subjects/${encodeURIComponent(subjectId)}/body`, + ); +} + +export class SubjectBodyApiError extends Error {} + +async function checkedJson(response: Response): Promise { + if (!response.ok) { + // 生のerror bodyを表示せず、公開Diagnosticだけを境界で検査する。 + let detail = ""; + try { + const error = record( + await readBoundedJson(response, MAX_RESPONSE_BYTES), + ["error", "message", "diagnostics"], + ); + if (error.diagnostics !== undefined) { + detail = diagnostics(error.diagnostics).map((item) => item.message) + .join("; ").slice(0, 1024); + } + } catch { /* HTTP分類はbodyのdecode失敗でも保持する。 */ } + throw new SubjectBodyApiError( + `Subject body request failed with HTTP ${response.status}${ + detail ? `: ${detail}` : "" + }`, + ); + } + return readBoundedJson(response, MAX_RESPONSE_BYTES); +} + +export async function getSubjectBody( + fetchFn: typeof fetch, + workspaceId: string, + subjectId: string, +): Promise { + return parseSubjectBodyResponse( + await checkedJson( + await fetchFn(bodyPath(workspaceId, subjectId), { cache: "no-store" }), + ), + workspaceId, + subjectId, + ); +} + +async function mutateSubjectBody( + fetchFn: typeof fetch, + workspaceId: string, + subjectId: string, + action: "start" | "stop", + request: SubjectBodyStartRequest | SubjectBodyStopRequest, +): Promise { + const requestId = parseRequestId(request.request_id); + const response = await fetchFn( + `${bodyPath(workspaceId, subjectId)}/${action}`, + { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(request), + }, + ); + return parseSubjectBodyMutationResponse( + await checkedJson(response), + workspaceId, + subjectId, + requestId, + ); +} + +export function startSubjectBody( + fetchFn: typeof fetch, + workspaceId: string, + subjectId: string, + request: SubjectBodyStartRequest, +): Promise { + return mutateSubjectBody(fetchFn, workspaceId, subjectId, "start", request); +} + +export function stopSubjectBody( + fetchFn: typeof fetch, + workspaceId: string, + subjectId: string, + request: SubjectBodyStopRequest, +): Promise { + return mutateSubjectBody(fetchFn, workspaceId, subjectId, "stop", request); +} + +// POSTは凍結入力を照合するためであり、stopの再実行ではない。 +export async function getSubjectBodyStopOutcome( + fetchFn: typeof fetch, + workspaceId: string, + subjectId: string, + request: SubjectBodyStopRequest, +): Promise { + const requestId = parseRequestId(request.request_id); + const response = await fetchFn( + `${bodyPath(workspaceId, subjectId)}/stop/outcome`, + { + method: "POST", + headers: { "content-type": "application/json" }, + cache: "no-store", + body: JSON.stringify(request), + }, + ); + // 認可拒否・通信失敗・不正な応答をnot_foundに読み替えない。 + return parseSubjectBodyStopOutcomeResponse( + await checkedJson(response), + workspaceId, + subjectId, + requestId, + ); +} + +export function subjectBodyControls( + body: SubjectBodyResponse, +): { start: boolean; stop: boolean } { + if (body.observation.kind === "absent") { + return { start: body.can_start, stop: false }; + } + if (body.observation.kind !== "available") { + return { start: false, stop: false }; + } + const worker = body.observation.worker; + const observed = worker.availability === "observed" && + Boolean(worker.restore_observation_token); + return { + start: body.can_start && observed && worker.state === "stopped" && + worker.worker_state == null, + stop: body.can_stop && observed && worker.worker_state != null && + ["idle", "running", "paused"].includes(worker.state), + }; +} + +// accountIdはwhoami由来のローカルな隔離境界であり、Serverへ送るactor指定ではない。 +export type SubjectBodyAttempt = + & { + accountId: string; + workspaceId: string; + subjectId: string; + } + & ( + | { action: "start"; request: SubjectBodyStartRequest } + | { action: "stop"; request: SubjectBodyStopRequest } + ); + +export function subjectBodyStorageKey( + accountId: string, + workspaceId: string, + subjectId: string, +): string { + // accountを特定できない旧2要素keyは読取り・移行・削除しない。 + return `yoi.subject-body:${ + JSON.stringify([text(accountId), workspaceId, subjectId]) + }`; +} + +function parseAttempt( + value: unknown, + accountId: string, + workspaceId: string, + subjectId: string, +): SubjectBodyAttempt { + text(accountId); + const item = record(value, [ + "accountId", + "workspaceId", + "subjectId", + "action", + "request", + ]); + if ( + item.accountId !== accountId || item.workspaceId !== workspaceId || + item.subjectId !== subjectId + ) { + throw new Error( + "Saved Subject body request belongs to a different account or target", + ); + } + const input = record(item.request, [ + "request_id", + "expected_current", + "expected_observation_token", + ]); + const request_id = parseRequestId(input.request_id); + const expected_current = input.expected_current === null + ? null + : Object.freeze(bodyRef(input.expected_current)); + const expected_observation_token = input.expected_observation_token === null + ? null + : token(input.expected_observation_token); + if ( + item.action === "stop" && expected_current !== null && + expected_observation_token !== null + ) { + return Object.freeze({ + accountId, + workspaceId, + subjectId, + action: "stop", + request: Object.freeze({ + request_id, + expected_current, + expected_observation_token, + }), + }); + } + if ( + item.action === "start" && + ((expected_current === null) === (expected_observation_token === null)) + ) { + return Object.freeze({ + accountId, + workspaceId, + subjectId, + action: "start", + request: Object.freeze({ + request_id, + expected_current, + expected_observation_token, + }), + }); + } + throw new Error( + "Saved Subject body request is invalid; do not create a replacement request", + ); +} + +export function createSubjectBodyAttempt( + accountId: string, + body: SubjectBodyResponse, + action: "start" | "stop", + requestId: string, +): SubjectBodyAttempt { + if (!subjectBodyControls(body)[action]) { + throw new Error("Refresh Subject body before this action"); + } + return parseAttempt( + { + accountId, + workspaceId: body.workspace_id, + subjectId: body.subject_id, + action, + request: { + request_id: requestId, + expected_current: body.current, + expected_observation_token: body.observation.kind === "available" + ? body.observation.worker.restore_observation_token + : null, + }, + }, + accountId, + body.workspace_id, + body.subject_id, + ); +} + +export function readSubjectBodyAttempt( + storage: Pick, + accountId: string, + workspaceId: string, + subjectId: string, +): SubjectBodyAttempt | null { + const raw = storage.getItem( + subjectBodyStorageKey(accountId, workspaceId, subjectId), + ); + if (raw === null) return null; + if (raw.length > MAX_SAVED_LENGTH) { + throw new Error("Saved Subject body request is too large"); + } + return parseAttempt(JSON.parse(raw), accountId, workspaceId, subjectId); +} + +export function saveSubjectBodyAttempt( + storage: Pick, + accountId: string, + attempt: SubjectBodyAttempt, +): void { + const safe = parseAttempt( + attempt, + accountId, + attempt.workspaceId, + attempt.subjectId, + ); + const raw = JSON.stringify(safe); + if (raw.length > MAX_SAVED_LENGTH) { + throw new Error("Subject body request is too large to retain"); + } + const previous = readSubjectBodyAttempt( + storage, + accountId, + safe.workspaceId, + safe.subjectId, + ); + if (previous !== null && JSON.stringify(previous) !== raw) { + throw new Error( + "An unresolved Subject body request already exists; retry that request", + ); + } + storage.setItem( + subjectBodyStorageKey(accountId, safe.workspaceId, safe.subjectId), + raw, + ); +} + +export function clearSubjectBodyAttempt( + storage: Pick, + accountId: string, + attempt: SubjectBodyAttempt, +): void { + // 呼出し時のaccountと凍結入力を、storageアクセスより前に照合する。 + const safe = parseAttempt( + attempt, + accountId, + attempt.workspaceId, + attempt.subjectId, + ); + const saved = readSubjectBodyAttempt( + storage, + accountId, + safe.workspaceId, + safe.subjectId, + ); + if (saved && JSON.stringify(saved) !== JSON.stringify(safe)) { + throw new Error( + "The saved Subject body request changed; do not replace it", + ); + } + storage.removeItem( + subjectBodyStorageKey(accountId, safe.workspaceId, safe.subjectId), + ); +} diff --git a/web/workspace/src/lib/workspace/sidebar/WorkersNavSection.svelte b/web/workspace/src/lib/workspace/sidebar/WorkersNavSection.svelte index 3b90e2fa7..425270cbd 100644 --- a/web/workspace/src/lib/workspace/sidebar/WorkersNavSection.svelte +++ b/web/workspace/src/lib/workspace/sidebar/WorkersNavSection.svelte @@ -143,8 +143,8 @@ if (!scope.isCurrent()) return; await refreshWorkspaceWorkers(scope.workspaceId); if (!scope.isCurrent()) return; - pushWorkspaceAlert('info', `${worker.display_name || worker.label} stopped`, { - title: 'Worker stopped', + pushWorkspaceAlert('info', `Stop accepted for ${worker.display_name || worker.label}. Check the refreshed Worker state for confirmation.`, { + title: 'Stop accepted', }); } catch (cause) { if (!scope.isCurrent()) return; diff --git a/web/workspace/src/lib/workspace/sidebar/worker-actions.ts b/web/workspace/src/lib/workspace/sidebar/worker-actions.ts index 26bb28bc9..5ef46d042 100644 --- a/web/workspace/src/lib/workspace/sidebar/worker-actions.ts +++ b/web/workspace/src/lib/workspace/sidebar/worker-actions.ts @@ -196,13 +196,23 @@ export function canDeleteSidebarWorker(worker: WorkerActionTarget): boolean { export async function stopSidebarWorker( workspaceId: string, - worker: WorkerActionTarget, + worker: WorkerActionTarget & { restore_observation_token?: string | null }, fetchFn: FetchFn = fetch, ): Promise { const response = await fetchFn(`${workerPath(workspaceId, worker)}/stop`, { method: "POST", headers: { "content-type": "application/json" }, - body: JSON.stringify({ reason: "stopped from Workspace sidebar" }), + body: JSON.stringify({ + reason: "stopped from Workspace sidebar", + ...(worker.restore_observation_token + ? { + stop_intent: { + request_id: crypto.randomUUID(), + expected_observation_token: worker.restore_observation_token, + }, + } + : {}), + }), }); if (!response.ok) throw new Error(await responseError(response)); @@ -239,7 +249,6 @@ export async function deleteSidebarWorker( method: "POST", headers: { "content-type": "application/json" }, body: JSON.stringify({ - expected_plan_digest: plan.digest, worker_target_ids: [candidate.target_id], workdir_target_ids: [], diff --git a/web/workspace/src/routes/w/[workspaceId]/memory/PageContent.svelte b/web/workspace/src/routes/w/[workspaceId]/memory/PageContent.svelte index dd672b917..6a8264d69 100644 --- a/web/workspace/src/routes/w/[workspaceId]/memory/PageContent.svelte +++ b/web/workspace/src/routes/w/[workspaceId]/memory/PageContent.svelte @@ -224,7 +224,7 @@
Worker connection
- {subject.current_worker ? 'Connected' : 'Not connected'} + {subject.current_worker_diagnostics?.length ? 'Body observation unavailable' : subject.current_body ? 'Current body assigned' : 'No current body'} {#if subject.current_worker}{subject.current_worker.display_name}{/if}
diff --git a/web/workspace/src/routes/w/[workspaceId]/memory/[subjectId]/PageContent.svelte b/web/workspace/src/routes/w/[workspaceId]/memory/[subjectId]/PageContent.svelte index 1a9c0f8a1..dfc207562 100644 --- a/web/workspace/src/routes/w/[workspaceId]/memory/[subjectId]/PageContent.svelte +++ b/web/workspace/src/routes/w/[workspaceId]/memory/[subjectId]/PageContent.svelte @@ -4,9 +4,11 @@ // behaviorは利用者が管理する本文、surfaceはMemoryからの生成物として分ける。 // ready-empty／ungenerated／stale／failedとrequest failureを区別し、surfaceの現在性を // 実行中Workerの履歴へ反映済みであることの証明には使わない。診断値は詳細へ分離する。 - import { untrack } from 'svelte'; + import { onMount, untrack } from 'svelte'; + import { browserSession } from '#lib/workspace/auth/session-state.ts'; import DocumentMarkdown from '#lib/workspace/markdown/DocumentMarkdown.svelte'; import SubjectDelete from '#lib/workspace/memory/SubjectDelete.svelte'; + import SubjectBody from '#lib/workspace/memory/SubjectBody.svelte'; import { formatDate, workspaceRoute } from '#lib/workspace/api/http.ts'; import { SubjektivSubjectCreateError, @@ -19,6 +21,11 @@ type PageProps = { data: ResolvedPageData }; let { data }: PageProps = $props(); + let accountId = $state(null); + onMount(() => browserSession.subscribe((snapshot) => { + accountId = snapshot.kind === 'ready' && snapshot.value.session.kind === 'authenticated' + ? snapshot.value.actor?.account_id ?? null : null; + })); let subject = $state(untrack(() => data.subject.data)); let editingBehavior = $state(false); @@ -190,9 +197,13 @@

{subject.state === 'active' ? 'Available for new Worker connections.' : 'No longer available for new Worker connections.'}

- Worker connection - {subject.current_worker ? 'Connected' : 'Not connected'} -

{subject.current_worker?.display_name ?? 'No Worker currently owns this Subject connection.'}

+ {#if accountId} + {#key JSON.stringify([accountId, data.workspaceId, data.subjectId])} + + {/key} + {:else} +

Confirming account identity before loading body controls…

+ {/if}
Committed Memories